<commit_before>//! Implement thread-local storage.

use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use std::collections::HashSet;

use log::trace;

use rustc_middle::ty;
use rustc_target::abi::{Size, HasDataLayout};

use crate::{
    HelpersEvalContextExt, InterpResult, MPlaceTy, Scalar, StackPopCleanup, Tag, ThreadId,
    ThreadsEvalContextExt,
};

pub type TlsKey = u128;

#[derive(Clone, Debug)]
pub struct TlsEntry<'tcx> {
    /// The data for this key. None is used to represent NULL.
    /// (We normalize this early to avoid having to do a NULL-ptr-test each time we access the data.)
    data: BTreeMap<ThreadId, Scalar<Tag>>,
    dtor: Option<ty::Instance<'tcx>>,
}

#[derive(Debug)]
pub struct TlsData<'tcx> {
    /// The Key to use for the next thread-local allocation.
    next_key: TlsKey,

    /// pthreads-style thread-local storage.
    keys: BTreeMap<TlsKey, TlsEntry<'tcx>>,

    /// A single per thread destructor of the thread local storage (that's how
    /// things work on macOS) with a data argument.
    thread_dtors: BTreeMap<ThreadId, (ty::Instance<'tcx>, Scalar<Tag>)>,

    /// Whether we are in the "destruct" phase, during which some operations are UB.
    dtors_running: HashSet<ThreadId>,

    /// The last TlsKey used to retrieve a TLS destructor.
    last_dtor_key: BTreeMap<ThreadId, TlsKey>,
}

impl<'tcx> Default for TlsData<'tcx> {
    fn default() -> Self {
        TlsData {
            next_key: 1, // start with 1 as we must not use 0 on Windows
            keys: Default::default(),
            thread_dtors: Default::default(),
            dtors_running: Default::default(),
            last_dtor_key: Default::default(),
        }
    }
}

impl<'tcx> TlsData<'tcx> {
    /// Generate a new TLS key with the given destructor.
    /// `max_size` determines the integer size the key has to fit in.
    pub fn create_tls_key(&mut self, dtor: Option<ty::Instance<'tcx>>, max_size: Size) -> InterpResult<'tcx, TlsKey> {
        let new_key = self.next_key;
        self.next_key += 1;
        self.keys.insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap_none();
        trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor);

        if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits() as u128) {
            throw_unsup_format!("we ran out of TLS key space");
        }
        Ok(new_key)
    }

    pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> {
        match self.keys.remove(&key) {
            Some(_) => {
                trace!("TLS key {} removed", key);
                Ok(())
            }
            None => throw_ub_format!("removing a non-existig TLS key: {}", key),
        }
    }

    pub fn load_tls(
        &self,
        key: TlsKey,
        thread_id: ThreadId,
        cx: &impl HasDataLayout,
    ) -> InterpResult<'tcx, Scalar<Tag>> {
        match self.keys.get(&key) {
            Some(TlsEntry { data, .. }) => {
                let value = data.get(&thread_id).copied();
                trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value);
                Ok(value.unwrap_or_else(|| Scalar::null_ptr(cx).into()))
            }
            None => throw_ub_format!("loading from a non-existing TLS key: {}", key),
        }
    }

    pub fn store_tls(
        &mut self,
        key: TlsKey,
        thread_id: ThreadId,
        new_data: Option<Scalar<Tag>>
    ) -> InterpResult<'tcx> {
        match self.keys.get_mut(&key) {
            Some(TlsEntry { data, .. }) => {
                match new_data {
                    Some(scalar) => {
                        trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, scalar);
                        data.insert(thread_id, scalar);
                    }
                    None => {
                        trace!("TLS key {} for thread {:?} removed", key, thread_id);
                        data.remove(&thread_id);
                    }
                }
                Ok(())
            }
            None => throw_ub_format!("storing to a non-existing TLS key: {}", key),
        }
    }

    /// Set the thread wide destructor of the thread local storage for the given
    /// thread. This function is used to implement `_tlv_atexit` shim on MacOS.
    ///
    /// Thread wide dtors are available only on MacOS. There is one destructor
    /// per thread as can be guessed from the following comment in the
    /// [`_tlv_atexit`
    /// implementation](https://github.com/opensource-apple/dyld/blob/195030646877261f0c8c7ad8b001f52d6a26f514/src/threadLocalVariables.c#L389):
    ///
    ///     // NOTE: this does not need locks because it only operates on current thread data
    pub fn set_thread_dtor(
        &mut self,
        thread: ThreadId,
        dtor: ty::Instance<'tcx>,
        data: Scalar<Tag>
    ) -> InterpResult<'tcx> {
        if self.dtors_running.contains(&thread) {
            // UB, according to libstd docs.
            throw_ub_format!("setting thread's local storage destructor while destructors are already running");
        }
        if self.thread_dtors.insert(thread, (dtor, data)).is_some() {
            throw_unsup_format!("setting more than one thread local storage destructor for the same thread is not supported");
        }
        Ok(())
    }

    /// Returns a dtor, its argument and its index, if one is supposed to run.
    /// `key` is the last dtors that was run; we return the *next* one after that.
    ///
    /// An optional destructor function may be associated with each key value.
    /// At thread exit, if a key value has a non-NULL destructor pointer,
    /// and the thread has a non-NULL value associated with that key,
    /// the value of the key is set to NULL, and then the function pointed
    /// to is called with the previously associated value as its sole argument.
    /// The order of destructor calls is unspecified if more than one destructor
    /// exists for a thread when it exits.
    ///
    /// If, after all the destructors have been called for all non-NULL values
    /// with associated destructors, there are still some non-NULL values with
    /// associated destructors, then the process is repeated.
    /// If, after at least {PTHREAD_DESTRUCTOR_ITERATIONS} iterations of destructor
    /// calls for outstanding non-NULL values, there are still some non-NULL values
    /// with associated destructors, implementations may stop calling destructors,
    /// or they may continue calling destructors until no non-NULL values with
    /// associated destructors exist, even though this might result in an infinite loop.
    fn fetch_tls_dtor(
        &mut self,
        key: Option<TlsKey>,
        thread_id: ThreadId,
    ) -> Option<(ty::Instance<'tcx>, Scalar<Tag>, TlsKey)> {
        use std::collections::Bound::*;

        let thread_local = &mut self.keys;
        let start = match key {
            Some(key) => Excluded(key),
            None => Unbounded,
        };
        for (&key, TlsEntry { data, dtor }) in
            thread_local.range_mut((start, Unbounded))
        {
            match data.entry(thread_id) {
                Entry::Occupied(entry) => {
                    let data_scalar = entry.remove();
                    if let Some(dtor) = dtor {
                        let ret = Some((*dtor, data_scalar, key));
                        return ret;
                    }
                }
                Entry::Vacant(_) => {}
            }
        }
        None
    }
}

impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
trait EvalContextPrivExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
    /// Schedule TLS destructors for the main thread on Windows. The
    /// implementation assumes that we do not support concurrency on Windows
    /// yet.
    fn schedule_windows_tls_dtors(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let active_thread = this.get_active_thread()?;
        assert_eq!(this.get_total_thread_count()?, 1, "concurrency on Windows not supported");
        this.machine.tls.dtors_running.insert(active_thread);
        // Windows has a special magic linker section that is run on certain events.
        // Instead of searching for that section and supporting arbitrary hooks in there
        // (that would be basically https://github.com/rust-lang/miri/issues/450),
        // we specifically look up the static in libstd that we know is placed
        // in that section.
        let thread_callback = this.eval_path_scalar(&["std", "sys", "windows", "thread_local", "p_thread_callback"])?;
        let thread_callback = this.memory.get_fn(thread_callback.not_undef()?)?.as_instance()?;

        // The signature of this function is `unsafe extern "system" fn(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID)`.
        let reason = this.eval_path_scalar(&["std", "sys", "windows", "c", "DLL_PROCESS_DETACH"])?;
        let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
        this.call_function(
            thread_callback,
            &[Scalar::null_ptr(this).into(), reason.into(), Scalar::null_ptr(this).into()],
            Some(ret_place),
            StackPopCleanup::None { cleanup: true },
        )?;

        this.enable_thread(active_thread)?;
        Ok(())
    }

    /// Schedule the MacOS thread destructor of the thread local storage to be
    /// executed.
    ///
    /// Note: It is safe to call this function also on other Unixes.
    fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let thread_id = this.get_active_thread()?;
        if let Some((instance, data)) = this.machine.tls.thread_dtors.remove(&thread_id) {
            trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id);

            let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
            this.call_function(
                instance,
                &[data.into()],
                Some(ret_place),
                StackPopCleanup::None { cleanup: true },
            )?;

            // Enable the thread so that it steps through the destructor which
            // we just scheduled. Since we deleted the destructor, it is
            // guaranteed that we will schedule it again. The `dtors_running`
            // flag will prevent the code from adding the destructor again.
            this.enable_thread(thread_id)?;
        }
        Ok(())
    }

    /// Schedule a pthread TLS destructor.
    fn schedule_pthread_tls_dtors(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let active_thread = this.get_active_thread()?;

        assert!(this.has_terminated(active_thread)?, "running TLS dtors for non-terminated thread");
        // Fetch next dtor after `key`.
        let last_key = this.machine.tls.last_dtor_key.get(&active_thread).cloned();
        let dtor = match this.machine.tls.fetch_tls_dtor(last_key, active_thread) {
            dtor @ Some(_) => dtor,
            // We ran each dtor once, start over from the beginning.
            None => {
                this.machine.tls.fetch_tls_dtor(None, active_thread)
            }
        };
        if let Some((instance, ptr, key)) = dtor {
            this.machine.tls.last_dtor_key.insert(active_thread, key);
            trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread);
            assert!(!this.is_null(ptr).unwrap(), "data can't be NULL when dtor is called!");

            let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
            this.call_function(
                instance,
                &[ptr.into()],
                Some(ret_place),
                StackPopCleanup::None { cleanup: true },
            )?;

            this.enable_thread(active_thread)?;
            return Ok(());
        }
        this.machine.tls.last_dtor_key.remove(&active_thread);

        Ok(())
    }
}

impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {

    /// Schedule an active thread's TLS destructor to run on the active thread.
    /// Note that this function does not run the destructors itself, it just
    /// schedules them one by one each time it is called and reenables the
    /// thread so that it can be executed normally by the main execution loop.
    ///
    /// FIXME: we do not support yet deallocation of thread local statics.
    /// Issue: https://github.com/rust-lang/miri/issues/1369
    fn schedule_next_tls_dtor_for_active_thread(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let active_thread = this.get_active_thread()?;

        if this.tcx.sess.target.target.target_os == "windows" {
            if !this.machine.tls.dtors_running.contains(&active_thread) {
                this.machine.tls.dtors_running.insert(active_thread);
                this.schedule_windows_tls_dtors()?;
            }
        } else {
            this.machine.tls.dtors_running.insert(active_thread);
            // The macOS thread wide destructor runs "before any TLS slots get
            // freed", so do that first.
            this.schedule_macos_tls_dtor()?;
            this.schedule_pthread_tls_dtors()?;
        }

        Ok(())
    }
}
<commit_msg>Make sure to remove thread local data only if we have destructor.<commit_after>//! Implement thread-local storage.

use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use std::collections::HashSet;

use log::trace;

use rustc_middle::ty;
use rustc_target::abi::{Size, HasDataLayout};

use crate::{
    HelpersEvalContextExt, InterpResult, MPlaceTy, Scalar, StackPopCleanup, Tag, ThreadId,
    ThreadsEvalContextExt,
};

pub type TlsKey = u128;

#[derive(Clone, Debug)]
pub struct TlsEntry<'tcx> {
    /// The data for this key. None is used to represent NULL.
    /// (We normalize this early to avoid having to do a NULL-ptr-test each time we access the data.)
    data: BTreeMap<ThreadId, Scalar<Tag>>,
    dtor: Option<ty::Instance<'tcx>>,
}

#[derive(Debug)]
pub struct TlsData<'tcx> {
    /// The Key to use for the next thread-local allocation.
    next_key: TlsKey,

    /// pthreads-style thread-local storage.
    keys: BTreeMap<TlsKey, TlsEntry<'tcx>>,

    /// A single per thread destructor of the thread local storage (that's how
    /// things work on macOS) with a data argument.
    thread_dtors: BTreeMap<ThreadId, (ty::Instance<'tcx>, Scalar<Tag>)>,

    /// Whether we are in the "destruct" phase, during which some operations are UB.
    dtors_running: HashSet<ThreadId>,

    /// The last TlsKey used to retrieve a TLS destructor.
    last_dtor_key: BTreeMap<ThreadId, TlsKey>,
}

impl<'tcx> Default for TlsData<'tcx> {
    fn default() -> Self {
        TlsData {
            next_key: 1, // start with 1 as we must not use 0 on Windows
            keys: Default::default(),
            thread_dtors: Default::default(),
            dtors_running: Default::default(),
            last_dtor_key: Default::default(),
        }
    }
}

impl<'tcx> TlsData<'tcx> {
    /// Generate a new TLS key with the given destructor.
    /// `max_size` determines the integer size the key has to fit in.
    pub fn create_tls_key(&mut self, dtor: Option<ty::Instance<'tcx>>, max_size: Size) -> InterpResult<'tcx, TlsKey> {
        let new_key = self.next_key;
        self.next_key += 1;
        self.keys.insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap_none();
        trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor);

        if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits() as u128) {
            throw_unsup_format!("we ran out of TLS key space");
        }
        Ok(new_key)
    }

    pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> {
        match self.keys.remove(&key) {
            Some(_) => {
                trace!("TLS key {} removed", key);
                Ok(())
            }
            None => throw_ub_format!("removing a non-existig TLS key: {}", key),
        }
    }

    pub fn load_tls(
        &self,
        key: TlsKey,
        thread_id: ThreadId,
        cx: &impl HasDataLayout,
    ) -> InterpResult<'tcx, Scalar<Tag>> {
        match self.keys.get(&key) {
            Some(TlsEntry { data, .. }) => {
                let value = data.get(&thread_id).copied();
                trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value);
                Ok(value.unwrap_or_else(|| Scalar::null_ptr(cx).into()))
            }
            None => throw_ub_format!("loading from a non-existing TLS key: {}", key),
        }
    }

    pub fn store_tls(
        &mut self,
        key: TlsKey,
        thread_id: ThreadId,
        new_data: Option<Scalar<Tag>>
    ) -> InterpResult<'tcx> {
        match self.keys.get_mut(&key) {
            Some(TlsEntry { data, .. }) => {
                match new_data {
                    Some(scalar) => {
                        trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, scalar);
                        data.insert(thread_id, scalar);
                    }
                    None => {
                        trace!("TLS key {} for thread {:?} removed", key, thread_id);
                        data.remove(&thread_id);
                    }
                }
                Ok(())
            }
            None => throw_ub_format!("storing to a non-existing TLS key: {}", key),
        }
    }

    /// Set the thread wide destructor of the thread local storage for the given
    /// thread. This function is used to implement `_tlv_atexit` shim on MacOS.
    ///
    /// Thread wide dtors are available only on MacOS. There is one destructor
    /// per thread as can be guessed from the following comment in the
    /// [`_tlv_atexit`
    /// implementation](https://github.com/opensource-apple/dyld/blob/195030646877261f0c8c7ad8b001f52d6a26f514/src/threadLocalVariables.c#L389):
    ///
    ///     // NOTE: this does not need locks because it only operates on current thread data
    pub fn set_thread_dtor(
        &mut self,
        thread: ThreadId,
        dtor: ty::Instance<'tcx>,
        data: Scalar<Tag>
    ) -> InterpResult<'tcx> {
        if self.dtors_running.contains(&thread) {
            // UB, according to libstd docs.
            throw_ub_format!("setting thread's local storage destructor while destructors are already running");
        }
        if self.thread_dtors.insert(thread, (dtor, data)).is_some() {
            throw_unsup_format!("setting more than one thread local storage destructor for the same thread is not supported");
        }
        Ok(())
    }

    /// Returns a dtor, its argument and its index, if one is supposed to run.
    /// `key` is the last dtors that was run; we return the *next* one after that.
    ///
    /// An optional destructor function may be associated with each key value.
    /// At thread exit, if a key value has a non-NULL destructor pointer,
    /// and the thread has a non-NULL value associated with that key,
    /// the value of the key is set to NULL, and then the function pointed
    /// to is called with the previously associated value as its sole argument.
    /// The order of destructor calls is unspecified if more than one destructor
    /// exists for a thread when it exits.
    ///
    /// If, after all the destructors have been called for all non-NULL values
    /// with associated destructors, there are still some non-NULL values with
    /// associated destructors, then the process is repeated.
    /// If, after at least {PTHREAD_DESTRUCTOR_ITERATIONS} iterations of destructor
    /// calls for outstanding non-NULL values, there are still some non-NULL values
    /// with associated destructors, implementations may stop calling destructors,
    /// or they may continue calling destructors until no non-NULL values with
    /// associated destructors exist, even though this might result in an infinite loop.
    fn fetch_tls_dtor(
        &mut self,
        key: Option<TlsKey>,
        thread_id: ThreadId,
    ) -> Option<(ty::Instance<'tcx>, Scalar<Tag>, TlsKey)> {
        use std::collections::Bound::*;

        let thread_local = &mut self.keys;
        let start = match key {
            Some(key) => Excluded(key),
            None => Unbounded,
        };
        for (&key, TlsEntry { data, dtor }) in
            thread_local.range_mut((start, Unbounded))
        {
            match data.entry(thread_id) {
                Entry::Occupied(entry) => {
                    if let Some(dtor) = dtor {
                        let data_scalar = entry.remove();
                        let ret = Some((*dtor, data_scalar, key));
                        return ret;
                    }
                }
                Entry::Vacant(_) => {}
            }
        }
        None
    }
}

impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
trait EvalContextPrivExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
    /// Schedule TLS destructors for the main thread on Windows. The
    /// implementation assumes that we do not support concurrency on Windows
    /// yet.
    fn schedule_windows_tls_dtors(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let active_thread = this.get_active_thread()?;
        assert_eq!(this.get_total_thread_count()?, 1, "concurrency on Windows not supported");
        this.machine.tls.dtors_running.insert(active_thread);
        // Windows has a special magic linker section that is run on certain events.
        // Instead of searching for that section and supporting arbitrary hooks in there
        // (that would be basically https://github.com/rust-lang/miri/issues/450),
        // we specifically look up the static in libstd that we know is placed
        // in that section.
        let thread_callback = this.eval_path_scalar(&["std", "sys", "windows", "thread_local", "p_thread_callback"])?;
        let thread_callback = this.memory.get_fn(thread_callback.not_undef()?)?.as_instance()?;

        // The signature of this function is `unsafe extern "system" fn(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID)`.
        let reason = this.eval_path_scalar(&["std", "sys", "windows", "c", "DLL_PROCESS_DETACH"])?;
        let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
        this.call_function(
            thread_callback,
            &[Scalar::null_ptr(this).into(), reason.into(), Scalar::null_ptr(this).into()],
            Some(ret_place),
            StackPopCleanup::None { cleanup: true },
        )?;

        this.enable_thread(active_thread)?;
        Ok(())
    }

    /// Schedule the MacOS thread destructor of the thread local storage to be
    /// executed.
    ///
    /// Note: It is safe to call this function also on other Unixes.
    fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let thread_id = this.get_active_thread()?;
        if let Some((instance, data)) = this.machine.tls.thread_dtors.remove(&thread_id) {
            trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id);

            let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
            this.call_function(
                instance,
                &[data.into()],
                Some(ret_place),
                StackPopCleanup::None { cleanup: true },
            )?;

            // Enable the thread so that it steps through the destructor which
            // we just scheduled. Since we deleted the destructor, it is
            // guaranteed that we will schedule it again. The `dtors_running`
            // flag will prevent the code from adding the destructor again.
            this.enable_thread(thread_id)?;
        }
        Ok(())
    }

    /// Schedule a pthread TLS destructor.
    fn schedule_pthread_tls_dtors(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let active_thread = this.get_active_thread()?;

        assert!(this.has_terminated(active_thread)?, "running TLS dtors for non-terminated thread");
        // Fetch next dtor after `key`.
        let last_key = this.machine.tls.last_dtor_key.get(&active_thread).cloned();
        let dtor = match this.machine.tls.fetch_tls_dtor(last_key, active_thread) {
            dtor @ Some(_) => dtor,
            // We ran each dtor once, start over from the beginning.
            None => {
                this.machine.tls.fetch_tls_dtor(None, active_thread)
            }
        };
        if let Some((instance, ptr, key)) = dtor {
            this.machine.tls.last_dtor_key.insert(active_thread, key);
            trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread);
            assert!(!this.is_null(ptr).unwrap(), "data can't be NULL when dtor is called!");

            let ret_place = MPlaceTy::dangling(this.machine.layouts.unit, this).into();
            this.call_function(
                instance,
                &[ptr.into()],
                Some(ret_place),
                StackPopCleanup::None { cleanup: true },
            )?;

            this.enable_thread(active_thread)?;
            return Ok(());
        }
        this.machine.tls.last_dtor_key.remove(&active_thread);

        Ok(())
    }
}

impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {

    /// Schedule an active thread's TLS destructor to run on the active thread.
    /// Note that this function does not run the destructors itself, it just
    /// schedules them one by one each time it is called and reenables the
    /// thread so that it can be executed normally by the main execution loop.
    ///
    /// FIXME: we do not support yet deallocation of thread local statics.
    /// Issue: https://github.com/rust-lang/miri/issues/1369
    fn schedule_next_tls_dtor_for_active_thread(&mut self) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let active_thread = this.get_active_thread()?;

        if this.tcx.sess.target.target.target_os == "windows" {
            if !this.machine.tls.dtors_running.contains(&active_thread) {
                this.machine.tls.dtors_running.insert(active_thread);
                this.schedule_windows_tls_dtors()?;
            }
        } else {
            this.machine.tls.dtors_running.insert(active_thread);
            // The macOS thread wide destructor runs "before any TLS slots get
            // freed", so do that first.
            this.schedule_macos_tls_dtor()?;
            this.schedule_pthread_tls_dtors()?;
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>// Copyright 2021 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

pub mod device;
pub mod vmm;

use std::fmt::Debug;

pub use self::device::*;

cfg_if::cfg_if! {
    if #[cfg(unix)] {
        pub mod proxy;
        pub use self::proxy::*;
    } else if #[cfg(windows)] {}
}

use argh::FromArgValue;
use serde::Deserialize;

/// Extends any device configuration with a mandatory extra "vhost" parameter to specify the socket
/// or PCI device to use in order to communicate with a vhost client.
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VhostUserParams<T: Debug> {
    pub vhost: String,
    #[serde(flatten)]
    pub device: T,
}

impl<T> FromArgValue for VhostUserParams<T>
where
    T: Debug + for<'de> Deserialize<'de>,
{
    fn from_arg_value(value: &str) -> std::result::Result<Self, String> {
        serde_keyvalue::from_key_values(value).map_err(|e| e.to_string())
    }
}
<commit_msg>virtio: vhost: user: introduce tests for VhostUserParams parsing<commit_after>// Copyright 2021 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

pub mod device;
pub mod vmm;

use std::fmt::Debug;

pub use self::device::*;

cfg_if::cfg_if! {
    if #[cfg(unix)] {
        pub mod proxy;
        pub use self::proxy::*;
    } else if #[cfg(windows)] {}
}

use argh::FromArgValue;
use serde::Deserialize;

/// Extends any device configuration with a mandatory extra "vhost" parameter to specify the socket
/// or PCI device to use in order to communicate with a vhost client.
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VhostUserParams<T: Debug> {
    pub vhost: String,
    #[serde(flatten)]
    pub device: T,
}

impl<T> FromArgValue for VhostUserParams<T>
where
    T: Debug + for<'de> Deserialize<'de>,
{
    fn from_arg_value(value: &str) -> std::result::Result<Self, String> {
        serde_keyvalue::from_key_values(value).map_err(|e| e.to_string())
    }
}

#[cfg(test)]
mod tests {
    use std::path::PathBuf;

    use argh::FromArgValue;
    use serde::Deserialize;
    use serde_keyvalue::*;

    use super::VhostUserParams;

    #[derive(Debug, Deserialize, PartialEq, Eq)]
    #[serde(deny_unknown_fields, rename_all = "kebab-case")]
    struct DummyDevice {
        path: PathBuf,
        #[serde(default)]
        boom_range: u32,
    }

    fn from_arg_value(s: &str) -> Result<VhostUserParams<DummyDevice>, String> {
        VhostUserParams::<DummyDevice>::from_arg_value(s)
    }

    #[test]
    fn vhost_user_params() {
        let device = from_arg_value("vhost=vhost_sock,path=/path/to/dummy,boom-range=42").unwrap();
        assert_eq!(device.vhost.as_str(), "vhost_sock");
        assert_eq!(
            device.device,
            DummyDevice {
                path: "/path/to/dummy".into(),
                boom_range: 42,
            }
        );

        // Default parameter of device not specified.
        let device = from_arg_value("vhost=vhost_sock,path=/path/to/dummy").unwrap();
        assert_eq!(device.vhost.as_str(), "vhost_sock");
        assert_eq!(
            device.device,
            DummyDevice {
                path: "/path/to/dummy".into(),
                boom_range: Default::default(),
            }
        );

        // Invalid parameter is rejected.
        assert_eq!(
            from_arg_value("vhost=vhost_sock,path=/path/to/dummy,boom-range=42,invalid-param=10")
                .unwrap_err(),
            "unknown field `invalid-param`".to_string(),
        );
    }
}
<|endoftext|>


<commit_before>//! An example that determines the time stamp counter frequency (RDTSC, RDTSCP) .
extern crate raw_cpuid;

use std::time;

const MHZ_TO_HZ: u64 = 1000000;
const KHZ_TO_HZ: u64 = 1000;

#[rustversion::nightly]
use core::arch::x86_64::_rdtsc as rdtsc;

#[rustversion::stable]
unsafe fn rdtsc() -> u64 {
    0
}

fn main() {
    let cpuid = raw_cpuid::CpuId::new();
    let has_tsc = cpuid
        .get_feature_info()
        .map_or(false, |finfo| finfo.has_tsc());

    let has_invariant_tsc = cpuid
        .get_extended_function_info()
        .map_or(false, |efinfo| efinfo.has_invariant_tsc());

    let tsc_frequency_hz = cpuid.get_tsc_info().map(|tinfo| {
        if tinfo.nominal_frequency() != 0 {
            Some(tinfo.tsc_frequency())
        } else if tinfo.numerator() != 0 && tinfo.denominator() != 0 {
            // Skylake and Kabylake don't report the crystal clock, approximate with base frequency:
            cpuid
                .get_processor_frequency_info()
                .map(|pinfo| pinfo.processor_base_frequency() as u64 * MHZ_TO_HZ)
                .map(|cpu_base_freq_hz| {
                    let crystal_hz =
                        cpu_base_freq_hz * tinfo.denominator() as u64 / tinfo.numerator() as u64;
                    crystal_hz * tinfo.numerator() as u64 / tinfo.denominator() as u64
                })
        } else {
            None
        }
    });

    if has_tsc {
        // Try to figure out TSC frequency with CPUID
        println!(
            "TSC Frequency is: {} ({})",
            match tsc_frequency_hz {
                Some(x) => format!("{} Hz", x.unwrap_or(0)),
                None => String::from("unknown"),
            },
            if has_invariant_tsc {
                "invariant"
            } else {
                "TSC frequency varies with speed-stepping"
            }
        );

        // Check if we run in a VM and the hypervisor can give us the TSC frequency
        cpuid.get_hypervisor_info().map(|hv| {
            hv.tsc_frequency().map(|tsc_khz| {
                let virtual_tsc_frequency_hz = tsc_khz as u64 * KHZ_TO_HZ;
                println!(
                    "Hypervisor reports TSC Frequency at: {} Hz",
                    virtual_tsc_frequency_hz
                );
            })
        });

        // Determine TSC frequency by measuring it (loop for a second, record ticks)
        let one_second = time::Duration::from_secs(1);
        let now = time::Instant::now();
        let start = unsafe { rdtsc() };
        if start > 0 {
            loop {
                if now.elapsed() >= one_second {
                    break;
                }
            }
            let end = unsafe { rdtsc() };
            println!(
                "Empirical measurement of TSC frequency was: {} Hz",
                (end - start)
            );
        } else {
            // Don't have rdtsc on stable!
        }
    } else {
        println!("System does not have a TSC.")
    }
}
<commit_msg>Update so it builds on stable 32bit rust too<commit_after>//! An example that determines the time stamp counter frequency (RDTSC, RDTSCP) .
extern crate raw_cpuid;

use std::time;

const MHZ_TO_HZ: u64 = 1000000;
const KHZ_TO_HZ: u64 = 1000;

#[cfg(target_arch = "x86_64")]
#[rustversion::nightly]
use core::arch::x86_64::_rdtsc as rdtsc;

#[cfg(target_arch = "x86")]
#[rustversion::nightly]
use core::arch::x86::_rdtsc as rdtsc;

#[rustversion::stable]
unsafe fn rdtsc() -> u64 {
    0
}

fn main() {
    let cpuid = raw_cpuid::CpuId::new();
    let has_tsc = cpuid
        .get_feature_info()
        .map_or(false, |finfo| finfo.has_tsc());

    let has_invariant_tsc = cpuid
        .get_extended_function_info()
        .map_or(false, |efinfo| efinfo.has_invariant_tsc());

    let tsc_frequency_hz = cpuid.get_tsc_info().map(|tinfo| {
        if tinfo.nominal_frequency() != 0 {
            Some(tinfo.tsc_frequency())
        } else if tinfo.numerator() != 0 && tinfo.denominator() != 0 {
            // Skylake and Kabylake don't report the crystal clock, approximate with base frequency:
            cpuid
                .get_processor_frequency_info()
                .map(|pinfo| pinfo.processor_base_frequency() as u64 * MHZ_TO_HZ)
                .map(|cpu_base_freq_hz| {
                    let crystal_hz =
                        cpu_base_freq_hz * tinfo.denominator() as u64 / tinfo.numerator() as u64;
                    crystal_hz * tinfo.numerator() as u64 / tinfo.denominator() as u64
                })
        } else {
            None
        }
    });

    if has_tsc {
        // Try to figure out TSC frequency with CPUID
        println!(
            "TSC Frequency is: {} ({})",
            match tsc_frequency_hz {
                Some(x) => format!("{} Hz", x.unwrap_or(0)),
                None => String::from("unknown"),
            },
            if has_invariant_tsc {
                "invariant"
            } else {
                "TSC frequency varies with speed-stepping"
            }
        );

        // Check if we run in a VM and the hypervisor can give us the TSC frequency
        cpuid.get_hypervisor_info().map(|hv| {
            hv.tsc_frequency().map(|tsc_khz| {
                let virtual_tsc_frequency_hz = tsc_khz as u64 * KHZ_TO_HZ;
                println!(
                    "Hypervisor reports TSC Frequency at: {} Hz",
                    virtual_tsc_frequency_hz
                );
            })
        });

        // Determine TSC frequency by measuring it (loop for a second, record ticks)
        let one_second = time::Duration::from_secs(1);
        let now = time::Instant::now();
        let start = unsafe { rdtsc() };
        if start > 0 {
            loop {
                if now.elapsed() >= one_second {
                    break;
                }
            }
            let end = unsafe { rdtsc() };
            println!(
                "Empirical measurement of TSC frequency was: {} Hz",
                (end - start)
            );
        } else {
            // Don't have rdtsc on stable!
        }
    } else {
        println!("System does not have a TSC.")
    }
}
<|endoftext|>


<commit_before>use std::iter::{Iterator, Peekable};
use std::collections::btree_map::{BTreeMap, Range};
use std::cmp::Ordering::*;

use super::Result;

use self::NextIterValue::*;

pub type Patch = BTreeMap<Vec<u8>, Change>;
pub type Iter<'a> = Box<Iterator<Item=(&'a [u8], &'a [u8])> + 'a>;

pub enum Change {
    Put(Vec<u8>),
    Delete,
}

pub struct Fork {
    snapshot: Box<Snapshot>,
    changes: Patch
}

pub struct ForkIter<'a> {
    snapshot: Peekable<Iter<'a>>,
    changes: Peekable<Range<'a, Vec<u8>, Change>>
}

enum NextIterValue<'a> {
    Stored(&'a [u8], &'a [u8]),
    Replaced(&'a [u8], &'a [u8]),
    Inserted(&'a [u8], &'a [u8]),
    Deleted,
    MissDeleted
}

pub trait Database: Sized + Clone + Send + Sync + 'static {
    fn snapshot(&self) -> Box<Snapshot>;
    fn fork(&self) -> Fork {
        Fork {
            snapshot: self.snapshot(),
            changes: Patch::new(),
        }
    }
    fn merge(&mut self, patch: Patch) -> Result<()>;
}

pub trait Snapshot {
    fn get(&self, key: &[u8]) -> Option<Vec<u8>>;
    fn contains(&self, key: &[u8]) -> bool {
        self.get(key).is_some()
    }
    fn iter<'a>(&'a self, from: &[u8]) -> Iter<'a>;
}

impl Snapshot for Fork {
    fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
        match self.changes.get(key) {
            Some(change) => match *change {
                Change::Put(ref v) => Some(v.clone()),
                Change::Delete => None,
            },
            None => self.snapshot.get(key)
        }
    }

    fn contains(&self, key: &[u8]) -> bool {
        match self.changes.get(key) {
            Some(change) => match *change {
                Change::Put(..) => true,
                Change::Delete => false,
            },
            None => self.snapshot.get(key).is_some()
        }
    }

    fn iter<'a>(&'a self, from: &[u8]) -> Iter<'a> {
        use std::collections::Bound::*;
        let range = (Included(from), Unbounded);
        Box::new(ForkIter {
            snapshot: self.snapshot.iter(from).peekable(),
            changes: self.changes.range::<[u8], _>(range).peekable()
        })
    }
}

impl Fork {
    pub fn put(&mut self, key: Vec<u8>, value: Vec<u8>) {
        self.changes.insert(key, Change::Put(value));
    }

    pub fn delete(&mut self, key: Vec<u8>) {
        self.changes.insert(key, Change::Delete);
    }

    pub fn delete_by_prefix(&mut self, prefix: &[u8]) {
        for (k, _) in self.snapshot.iter(prefix) {
            if !k.starts_with(prefix) {
                return
            }
            self.changes.insert(k.to_vec(), Change::Delete);
        }
    }

    pub fn into_patch(self) -> Patch {
        self.changes
    }
}

// TODO: Does we needed this AsRef / AsMut impls?

impl AsRef<Snapshot> for Snapshot {
    fn as_ref<'a>(&'a self) -> &'a (Snapshot + 'static) {
        self
    }
}

impl AsRef<Snapshot + 'static> for Fork {
    fn as_ref<'a>(&'a self) -> &'a (Snapshot + 'static) {
        &*self
    }
}

impl AsMut<Fork> for Fork {
    fn as_mut<'a>(&'a mut self) -> &'a mut Fork {
        self
    }
}

impl<'a> NextIterValue<'a> {
    fn skip_changes(&self) -> bool {
        match *self {
            Replaced(..) | Inserted(..) | Deleted | MissDeleted => true,
            Stored(..) => false,
        }
    }

    fn skip_snapshot(&self) -> bool {
        match *self {
            Stored(..) | Replaced(..) | Deleted => true,
            Inserted(..) | MissDeleted => false
        }
    }

    fn value(&self) -> Option<(&'a [u8], &'a [u8])> {
        match *self {
            Stored(k, v) => Some((k, v)),
            Replaced(k, v) => Some((k, v)),
            Inserted(k, v) => Some((k, v)),
            Deleted | MissDeleted => None
        }
    }
}

impl<'a> Iterator for ForkIter<'a> {
    type Item = (&'a [u8], &'a [u8]);

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            let next = match self.changes.peek() {
                Some(&(k, ref change)) => match self.snapshot.peek() {
                    Some(&(key, ref value)) => match **change {
                        Change::Put(ref v) => match k[..].cmp(key) {
                            Equal => Replaced(k, v),
                            Less => Inserted(k, v),
                            Greater => Stored(key, value)
                        },
                        Change::Delete => match k[..].cmp(key) {
                            Equal => Deleted,
                            Less => MissDeleted,
                            Greater => Stored(key, value)
                        }
                    },
                    None => match **change {
                        Change::Put(ref v) => Inserted(k, v),
                        Change::Delete => MissDeleted,
                    }
                },
                None => match self.snapshot.peek() {
                    Some(&(key, ref value)) => Stored(key, value),
                    None => return None,
                }
            };
            if next.skip_changes() {
                self.changes.next();
            }
            if next.skip_snapshot() {
                self.snapshot.next();
            }
            if let Some(value) = next.value() {
                return Some(value)
            }
        }
    }
}
<commit_msg>Implement changelog mechanism<commit_after>use std::iter::{Iterator, Peekable};
use std::collections::btree_map::{BTreeMap, Range};
use std::cmp::Ordering::*;

use super::Result;

use self::NextIterValue::*;

pub type Patch = BTreeMap<Vec<u8>, Change>;
pub type Iter<'a> = Box<Iterator<Item=(&'a [u8], &'a [u8])> + 'a>;

pub enum Change {
    Put(Vec<u8>),
    Delete,
}

pub struct Fork {
    snapshot: Box<Snapshot>,
    changes: Patch,
    changelog: Vec<(Vec<u8>, Option<Change>)>
}

pub struct ForkIter<'a> {
    snapshot: Peekable<Iter<'a>>,
    changes: Peekable<Range<'a, Vec<u8>, Change>>
}

enum NextIterValue<'a> {
    Stored(&'a [u8], &'a [u8]),
    Replaced(&'a [u8], &'a [u8]),
    Inserted(&'a [u8], &'a [u8]),
    Deleted,
    MissDeleted
}

pub trait Database: Sized + Clone + Send + Sync + 'static {
    fn snapshot(&self) -> Box<Snapshot>;
    fn fork(&self) -> Fork {
        Fork {
            snapshot: self.snapshot(),
            changes: Patch::new(),
            changelog: Vec::new()
        }
    }
    fn merge(&mut self, patch: Patch) -> Result<()>;
}

pub trait Snapshot {
    fn get(&self, key: &[u8]) -> Option<Vec<u8>>;
    fn contains(&self, key: &[u8]) -> bool {
        self.get(key).is_some()
    }
    fn iter<'a>(&'a self, from: &[u8]) -> Iter<'a>;
}

impl Snapshot for Fork {
    fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
        match self.changes.get(key) {
            Some(change) => match *change {
                Change::Put(ref v) => Some(v.clone()),
                Change::Delete => None,
            },
            None => self.snapshot.get(key)
        }
    }

    fn contains(&self, key: &[u8]) -> bool {
        match self.changes.get(key) {
            Some(change) => match *change {
                Change::Put(..) => true,
                Change::Delete => false,
            },
            None => self.snapshot.get(key).is_some()
        }
    }

    fn iter<'a>(&'a self, from: &[u8]) -> Iter<'a> {
        use std::collections::Bound::*;
        let range = (Included(from), Unbounded);
        Box::new(ForkIter {
            snapshot: self.snapshot.iter(from).peekable(),
            changes: self.changes.range::<[u8], _>(range).peekable()
        })
    }
}

impl Fork {
    pub fn checkpoint(&mut self) {
        self.changelog.clear()
    }

    pub fn rollback(&mut self) {
        for (k, c) in self.changelog.drain(..).rev() {
            match c {
                Some(change) => self.changes.insert(k, change),
                None => self.changes.remove(&k)
            };
        }
    }

    pub fn put(&mut self, key: Vec<u8>, value: Vec<u8>) {
        self.changelog.push((key.clone(),
                             self.changes.insert(key, Change::Put(value))))
    }

    pub fn delete(&mut self, key: Vec<u8>) {
        self.changelog.push((key.clone(),
                             self.changes.insert(key, Change::Delete)));
    }

    pub fn delete_by_prefix(&mut self, prefix: &[u8]) {
        for (k, _) in self.snapshot.iter(prefix) {
            if !k.starts_with(prefix) {
                return
            }
            self.changes.insert(k.to_vec(), Change::Delete);
        }
    }

    pub fn into_patch(self) -> Patch {
        self.changes
    }
}

// TODO: Does we needed this AsRef / AsMut impls?

impl AsRef<Snapshot> for Snapshot {
    fn as_ref<'a>(&'a self) -> &'a (Snapshot + 'static) {
        self
    }
}

impl AsRef<Snapshot + 'static> for Fork {
    fn as_ref<'a>(&'a self) -> &'a (Snapshot + 'static) {
        &*self
    }
}

impl AsMut<Fork> for Fork {
    fn as_mut<'a>(&'a mut self) -> &'a mut Fork {
        self
    }
}

impl<'a> NextIterValue<'a> {
    fn skip_changes(&self) -> bool {
        match *self {
            Replaced(..) | Inserted(..) | Deleted | MissDeleted => true,
            Stored(..) => false,
        }
    }

    fn skip_snapshot(&self) -> bool {
        match *self {
            Stored(..) | Replaced(..) | Deleted => true,
            Inserted(..) | MissDeleted => false
        }
    }

    fn value(&self) -> Option<(&'a [u8], &'a [u8])> {
        match *self {
            Stored(k, v) => Some((k, v)),
            Replaced(k, v) => Some((k, v)),
            Inserted(k, v) => Some((k, v)),
            Deleted | MissDeleted => None
        }
    }
}

impl<'a> Iterator for ForkIter<'a> {
    type Item = (&'a [u8], &'a [u8]);

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            let next = match self.changes.peek() {
                Some(&(k, ref change)) => match self.snapshot.peek() {
                    Some(&(key, ref value)) => match **change {
                        Change::Put(ref v) => match k[..].cmp(key) {
                            Equal => Replaced(k, v),
                            Less => Inserted(k, v),
                            Greater => Stored(key, value)
                        },
                        Change::Delete => match k[..].cmp(key) {
                            Equal => Deleted,
                            Less => MissDeleted,
                            Greater => Stored(key, value)
                        }
                    },
                    None => match **change {
                        Change::Put(ref v) => Inserted(k, v),
                        Change::Delete => MissDeleted,
                    }
                },
                None => match self.snapshot.peek() {
                    Some(&(key, ref value)) => Stored(key, value),
                    None => return None,
                }
            };
            if next.skip_changes() {
                self.changes.next();
            }
            if next.skip_snapshot() {
                self.snapshot.next();
            }
            if let Some(value) = next.value() {
                return Some(value)
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::slice;
use libc::c_void;

use objc::Id;
use INSObject;

pub trait INSData : INSObject {
    fn len(&self) -> usize {
        unsafe {
            msg_send![self, length]
        }
    }

    fn bytes(&self) -> &[u8] {
        let len = self.len();
        unsafe {
            let ptr: *const c_void = msg_send![self, bytes];
            slice::from_raw_parts(ptr as *const u8, len)
        }
    }

    fn with_bytes(bytes: &[u8]) -> Id<Self> {
        let cls = <Self as INSObject>::class();
        unsafe {
            let obj: *mut Self = msg_send![cls, alloc];
            let obj: *mut Self = msg_send![obj, initWithBytes:bytes.as_ptr()
                                                       length:bytes.len()];
            Id::from_retained_ptr(obj)
        }
    }
}
<commit_msg>Added NSData and tests.<commit_after>use std::slice;
use libc::c_void;

use objc::Id;
use {INSObject, INSCopying};

pub trait INSData : INSObject {
    fn len(&self) -> usize {
        unsafe {
            msg_send![self, length]
        }
    }

    fn bytes(&self) -> &[u8] {
        let len = self.len();
        unsafe {
            let ptr: *const c_void = msg_send![self, bytes];
            slice::from_raw_parts(ptr as *const u8, len)
        }
    }

    fn with_bytes(bytes: &[u8]) -> Id<Self> {
        let cls = <Self as INSObject>::class();
        unsafe {
            let obj: *mut Self = msg_send![cls, alloc];
            let obj: *mut Self = msg_send![obj, initWithBytes:bytes.as_ptr()
                                                       length:bytes.len()];
            Id::from_retained_ptr(obj)
        }
    }
}

object_struct!(NSData);

impl INSData for NSData { }

impl INSCopying for NSData {
    type Output = NSData;
}

#[cfg(test)]
mod tests {
    use objc::Id;
    use super::{INSData, NSData};

    #[test]
    fn test_bytes() {
        let bytes = [3u8, 7, 16, 52, 112, 19];
        let data: Id<NSData> = INSData::with_bytes(&bytes);
        assert!(data.len() == bytes.len());
        assert!(data.bytes() == bytes.as_slice());
    }
}
<|endoftext|>


<commit_before>use serde;

#[derive(Clone, Debug)]
pub struct Timestamp(f64);
impl From<f64> for Timestamp {
    fn from(t: f64) -> Self {
        Timestamp(t)
    }
}

impl std::ops::Deref for Timestamp {
    type Target = f64;
    fn deref(&self) -> &f64 {
        &self.0
    }
}

impl<'de> ::serde::Deserialize<'de> for Timestamp {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: ::serde::Deserializer<'de>,
    {
        use serde::de::Error as SerdeError;

        let value = ::serde_json::Value::deserialize(deserializer)?;
        if let Some(s) = value.as_str() {
            s.parse::<f64>()
            .map_err(|e| D::Error::custom(e))
            .map(Into::into)
        } else if let Some(f) = value.as_f64() {
            Ok(f.into())
        } else if let Some(u) = value.as_u64() {
            Ok((u as f64).into())
        } else {
            Err(D::Error::custom(
                format!("expected a timestamp but got: {}", value.to_string()),
            ))
        }
    }
}

impl Default for Timestamp {
    fn default() -> Self {
        0_f64.into()
    }
}

impl Timestamp {
    pub fn to_param_value(&self) -> String {
        let t: &f64 = self;
        serde_json::to_string(t).unwrap()
    }
}
<commit_msg>derive Copy and Default for Timestamp<commit_after>use serde;

#[derive(Debug, Clone, Copy, Default)]
pub struct Timestamp(f64);
impl From<f64> for Timestamp {
    fn from(t: f64) -> Self {
        Timestamp(t)
    }
}

impl std::ops::Deref for Timestamp {
    type Target = f64;
    fn deref(&self) -> &f64 {
        &self.0
    }
}

impl<'de> ::serde::Deserialize<'de> for Timestamp {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: ::serde::Deserializer<'de>,
    {
        use serde::de::Error as SerdeError;

        let value = ::serde_json::Value::deserialize(deserializer)?;
        if let Some(s) = value.as_str() {
            s.parse::<f64>()
            .map_err(|e| D::Error::custom(e))
            .map(Into::into)
        } else if let Some(f) = value.as_f64() {
            Ok(f.into())
        } else if let Some(u) = value.as_u64() {
            Ok((u as f64).into())
        } else {
            Err(D::Error::custom(
                format!("expected a timestamp but got: {}", value.to_string()),
            ))
        }
    }
}

impl Timestamp {
    pub fn to_param_value(&self) -> String {
        let t: &f64 = self;
        serde_json::to_string(t).unwrap()
    }
}
<|endoftext|>


<commit_before>
use vertex::{RawVertex, Vertex};
use edge::{RawEdge, Edge};
use graph::TraversableToVertex;
use path::{Path, Element};
use std::iter::Extend;

// use std::iter::Iterator;

// define a traversal as a series of stages

pub struct GraphQuery {
    // many paths
    pub paths: Vec<Path>
}

impl GraphQuery {
    pub fn new(vertices: Vec<Vertex>) -> GraphQuery {
        // create a new path for each proxy, add the proxy
        let mut paths = Vec::new();
        for p in vertices {
            let mut path = Path::new(p);
            paths.push(path)
        }
        GraphQuery{paths:paths}
    }

    pub fn empty() -> GraphQuery {
        // return an empty graph query
        GraphQuery{paths:Vec::new()}
    }

    pub fn get_paths(&self) -> &Vec<Path> {
        &self.paths
    }
    pub fn push(&mut self, path:Path) {
        self.paths.push(path)
    }

    /*
    generic mapper for our queries
    future this may use worker threads to perform
    traversals
     */
    fn map<F: Fn(&Path) -> Vec<Element>>(&self, closure: F) -> GraphQuery  {
        let mut result = GraphQuery::empty(); // result
        for path in self.paths.iter() {
            let mut tmp = closure(&path);
            // let mut tmp2 = tmp.as_slice();
            // currently gets back a Vec<Path> but what if it gets Elements?
            let new_elements = path.permute(&tmp);
            result.paths.extend(new_elements);
        }
        result
    }

    pub fn outV(&self, labels: &[String]) -> GraphQuery {
        // map can figure out the rest
        let f = |path: &Path| {
            //take the final element in the path
            let element = path.last().unwrap();

            let result : Vec<Element> = match element {
                &Element::Vertex(v) =>
                    v.outV().iter().map(|v| Element::Vertex(*v)).collect(),
                    // Vec::new(),
                &Element::Edge(ref e) =>
                    Vec::new()
            };
            // apply outV
            result
        };
        self.map(f)
    }

//    pub fn inV(&self) -> GraphQuery {
//        let g = GraphQuery::empty();
//        let f = |element: &Element| {
//            element.inV()
//        };
//        self.map(f)
//    }
}
<commit_msg>back to vec<commit_after>
use vertex::{RawVertex, Vertex};
use edge::{RawEdge, Edge};
use graph::TraversableToVertex;
use path::{Path, Element};
use std::iter::Extend;

// use std::iter::Iterator;

// define a traversal as a series of stages

pub struct GraphQuery {
    // many paths
    pub paths: Vec<Path>
}

impl GraphQuery {
    pub fn new(vertices: Vec<Vertex>) -> GraphQuery {
        // create a new path for each proxy, add the proxy
        let mut paths = Vec::new();
        for p in vertices {
            let mut path = Path::new(p);
            paths.push(path)
        }
        GraphQuery{paths:paths}
    }

    pub fn empty() -> GraphQuery {
        // return an empty graph query
        GraphQuery{paths:Vec::new()}
    }

    pub fn get_paths(&self) -> &Vec<Path> {
        &self.paths
    }
    pub fn push(&mut self, path:Path) {
        self.paths.push(path)
    }

    /*
    generic mapper for our queries
    future this may use worker threads to perform
    traversals
     */
    fn map<F: Fn(&Path) -> Vec<Element>>(&self, closure: F) -> GraphQuery  {
        let mut result = GraphQuery::empty(); // result
        for path in self.paths.iter() {
            let mut tmp = closure(&path);
            // let mut tmp2 = tmp.as_slice();
            // currently gets back a Vec<Path> but what if it gets Elements?
            let new_elements = path.permute(&tmp);
            result.paths.extend(new_elements);
        }
        result
    }

    pub fn outV(&self, labels: Vec<String>) -> GraphQuery {
        // map can figure out the rest
        let f = |path: &Path| {
            //take the final element in the path
            let element = path.last().unwrap();

            let result : Vec<Element> = match element {
                &Element::Vertex(v) =>
                    v.outV().iter().map(|v| Element::Vertex(*v)).collect(),
                    // Vec::new(),
                &Element::Edge(ref e) =>
                    Vec::new()
            };
            // apply outV
            result
        };
        self.map(f)
    }

//    pub fn inV(&self) -> GraphQuery {
//        let g = GraphQuery::empty();
//        let f = |element: &Element| {
//            element.inV()
//        };
//        self.map(f)
//    }
}
<|endoftext|>


<commit_before>//! Serde Serialization Framework
//!
//! Serde is a powerful framework that enables serialization libraries to generically serialize
//! Rust data structures without the overhead of runtime type information. In many situations, the
//! handshake protocol between serializers and serializees can be completely optimized away,
//! leaving serde to perform roughly the same speed as a hand written serializer for a specific
//! type.
//!
//! For a detailed tutorial on the different ways to use serde please check out the
//! [github repository](https://github.com/serde-rs/serde)

#![doc(html_root_url="https://serde-rs.github.io/serde/serde")]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(feature = "unstable", feature(reflect_marker, unicode, nonzero, plugin, step_trait, zero_one))]
#![cfg_attr(feature = "alloc", feature(alloc))]
#![cfg_attr(feature = "collections", feature(collections, enumset))]
#![cfg_attr(feature = "clippy", plugin(clippy))]
#![cfg_attr(feature = "clippy", allow(linkedlist))]

#![cfg_attr(any(not(feature = "std"), feature = "unstable"), allow(unused_variables, unused_imports, unused_features, dead_code))]

#![deny(missing_docs)]

#[cfg(all(feature = "unstable", feature = "collections"))]
extern crate collections;

#[cfg(all(feature = "unstable", feature = "alloc"))]
extern crate alloc;

#[cfg(feature = "std")]
mod core {
    pub use std::{ops, hash, fmt, cmp, marker, mem, i8, i16, i32, i64, u8, u16, u32, u64, isize,
            usize, f32, f64, char, str, num, slice, iter};
    #[cfg(feature = "unstable")]
    extern crate core;
    #[cfg(feature = "unstable")]
    pub use self::core::nonzero;
}

pub use ser::{Serialize, Serializer};
pub use de::{Deserialize, Deserializer, Error};

#[cfg(not(feature = "std"))]
macro_rules! format {
    ($s:expr, $($rest:tt)*) => ($s)
}

pub mod bytes;
pub mod de;
#[cfg(feature = "std")]
pub mod iter;
pub mod ser;
#[cfg(not(feature = "std"))]
pub mod error;
mod utils;
<commit_msg>Adjust doc(html_root_url)<commit_after>//! Serde Serialization Framework
//!
//! Serde is a powerful framework that enables serialization libraries to generically serialize
//! Rust data structures without the overhead of runtime type information. In many situations, the
//! handshake protocol between serializers and serializees can be completely optimized away,
//! leaving serde to perform roughly the same speed as a hand written serializer for a specific
//! type.
//!
//! For a detailed tutorial on the different ways to use serde please check out the
//! [github repository](https://github.com/serde-rs/serde)

#![doc(html_root_url="https://serde-rs.github.io/serde")]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(feature = "unstable", feature(reflect_marker, unicode, nonzero, plugin, step_trait, zero_one))]
#![cfg_attr(feature = "alloc", feature(alloc))]
#![cfg_attr(feature = "collections", feature(collections, enumset))]
#![cfg_attr(feature = "clippy", plugin(clippy))]
#![cfg_attr(feature = "clippy", allow(linkedlist))]

#![cfg_attr(any(not(feature = "std"), feature = "unstable"), allow(unused_variables, unused_imports, unused_features, dead_code))]

#![deny(missing_docs)]

#[cfg(all(feature = "unstable", feature = "collections"))]
extern crate collections;

#[cfg(all(feature = "unstable", feature = "alloc"))]
extern crate alloc;

#[cfg(feature = "std")]
mod core {
    pub use std::{ops, hash, fmt, cmp, marker, mem, i8, i16, i32, i64, u8, u16, u32, u64, isize,
            usize, f32, f64, char, str, num, slice, iter};
    #[cfg(feature = "unstable")]
    extern crate core;
    #[cfg(feature = "unstable")]
    pub use self::core::nonzero;
}

pub use ser::{Serialize, Serializer};
pub use de::{Deserialize, Deserializer, Error};

#[cfg(not(feature = "std"))]
macro_rules! format {
    ($s:expr, $($rest:tt)*) => ($s)
}

pub mod bytes;
pub mod de;
#[cfg(feature = "std")]
pub mod iter;
pub mod ser;
#[cfg(not(feature = "std"))]
pub mod error;
mod utils;
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Traits for physical query plan, supporting parallel execution for partitioned relations.

use std::fmt::{Debug, Display};
use std::sync::Arc;
use std::{any::Any, pin::Pin};

use crate::execution::context::ExecutionContextState;
use crate::logical_plan::LogicalPlan;
use crate::{error::Result, scalar::ScalarValue};
use arrow::datatypes::{DataType, Schema, SchemaRef};
use arrow::error::Result as ArrowResult;
use arrow::record_batch::RecordBatch;
use arrow::{array::ArrayRef, datatypes::Field};

use async_trait::async_trait;
use futures::stream::Stream;

use self::merge::MergeExec;

/// Trait for types that stream [arrow::record_batch::RecordBatch]
pub trait RecordBatchStream: Stream<Item = ArrowResult<RecordBatch>> {
    /// Returns the schema of this `RecordBatchStream`.
    ///
    /// Implementation of this trait should guarantee that all `RecordBatch`'s returned by this
    /// stream should have the same schema as returned from this method.
    fn schema(&self) -> SchemaRef;
}

/// Trait for a stream of record batches.
pub type SendableRecordBatchStream = Pin<Box<dyn RecordBatchStream + Send>>;

/// Physical query planner that converts a `LogicalPlan` to an
/// `ExecutionPlan` suitable for execution.
pub trait PhysicalPlanner {
    /// Create a physical plan from a logical plan
    fn create_physical_plan(
        &self,
        logical_plan: &LogicalPlan,
        ctx_state: &ExecutionContextState,
    ) -> Result<Arc<dyn ExecutionPlan>>;
}

/// Partition-aware execution plan for a relation
#[async_trait]
pub trait ExecutionPlan: Debug + Send + Sync {
    /// Returns the execution plan as [`Any`](std::any::Any) so that it can be
    /// downcast to a specific implementation.
    fn as_any(&self) -> &dyn Any;
    /// Get the schema for this execution plan
    fn schema(&self) -> SchemaRef;
    /// Specifies the output partitioning scheme of this plan
    fn output_partitioning(&self) -> Partitioning;
    /// Specifies the data distribution requirements of all the children for this operator
    fn required_child_distribution(&self) -> Distribution {
        Distribution::UnspecifiedDistribution
    }
    /// Get a list of child execution plans that provide the input for this plan. The returned list
    /// will be empty for leaf nodes, will contain a single value for unary nodes, or two
    /// values for binary nodes (such as joins).
    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>>;
    /// Returns a new plan where all children were replaced by new plans.
    /// The size of `children` must be equal to the size of `ExecutionPlan::children()`.
    fn with_new_children(
        &self,
        children: Vec<Arc<dyn ExecutionPlan>>,
    ) -> Result<Arc<dyn ExecutionPlan>>;

    /// creates an iterator
    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream>;
}

/// Execute the [ExecutionPlan] and collect the results in memory
pub async fn collect(plan: Arc<dyn ExecutionPlan>) -> Result<Vec<RecordBatch>> {
    match plan.output_partitioning().partition_count() {
        0 => Ok(vec![]),
        1 => {
            let it = plan.execute(0).await?;
            common::collect(it).await
        }
        _ => {
            // merge into a single partition
            let plan = MergeExec::new(plan.clone());
            // MergeExec must produce a single partition
            assert_eq!(1, plan.output_partitioning().partition_count());
            common::collect(plan.execute(0).await?).await
        }
    }
}

/// Execute the [ExecutionPlan] and collect the results in memory
pub async fn collect_partitioned(
    plan: Arc<dyn ExecutionPlan>,
) -> Result<Vec<Vec<RecordBatch>>> {
    match plan.output_partitioning().partition_count() {
        0 => Ok(vec![]),
        1 => {
            let it = plan.execute(0).await?;
            Ok(vec![common::collect(it).await?])
        }
        _ => {
            let mut partitions = vec![];
            for i in 0..plan.output_partitioning().partition_count() {
                partitions.push(common::collect(plan.execute(i).await?).await?)
            }
            Ok(partitions)
        }
    }
}

/// Partitioning schemes supported by operators.
#[derive(Debug, Clone)]
pub enum Partitioning {
    /// Allocate batches using a round-robin algorithm and the specified number of partitions
    RoundRobinBatch(usize),
    /// Allocate rows based on a hash of one of more expressions and the specified
    /// number of partitions
    /// This partitioning scheme is not yet fully supported. See [ARROW-11011](https://issues.apache.org/jira/browse/ARROW-11011)
    Hash(Vec<Arc<dyn PhysicalExpr>>, usize),
    /// Unknown partitioning scheme with a known number of partitions
    UnknownPartitioning(usize),
}

impl Partitioning {
    /// Returns the number of partitions in this partitioning scheme
    pub fn partition_count(&self) -> usize {
        use Partitioning::*;
        match self {
            RoundRobinBatch(n) => *n,
            Hash(_, n) => *n,
            UnknownPartitioning(n) => *n,
        }
    }
}

/// Distribution schemes
#[derive(Debug, Clone)]
pub enum Distribution {
    /// Unspecified distribution
    UnspecifiedDistribution,
    /// A single partition is required
    SinglePartition,
}

/// Represents the result from an expression
pub enum ColumnarValue {
    /// Array of values
    Array(ArrayRef),
    /// A single value
    Scalar(ScalarValue),
}

impl ColumnarValue {
    fn data_type(&self) -> DataType {
        match self {
            ColumnarValue::Array(array_value) => array_value.data_type().clone(),
            ColumnarValue::Scalar(scalar_value) => scalar_value.get_datatype(),
        }
    }

    fn into_array(self, num_rows: usize) -> ArrayRef {
        match self {
            ColumnarValue::Array(array) => array,
            ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(num_rows),
        }
    }
}

/// Expression that can be evaluated against a RecordBatch
/// A Physical expression knows its type, nullability and how to evaluate itself.
pub trait PhysicalExpr: Send + Sync + Display + Debug {
    /// Returns the physical expression as [`Any`](std::any::Any) so that it can be
    /// downcast to a specific implementation.
    fn as_any(&self) -> &dyn Any;
    /// Get the data type of this expression, given the schema of the input
    fn data_type(&self, input_schema: &Schema) -> Result<DataType>;
    /// Determine whether this expression is nullable, given the schema of the input
    fn nullable(&self, input_schema: &Schema) -> Result<bool>;
    /// Evaluate an expression against a RecordBatch
    fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue>;
}

/// An aggregate expression that:
/// * knows its resulting field
/// * knows how to create its accumulator
/// * knows its accumulator's state's field
/// * knows the expressions from whose its accumulator will receive values
pub trait AggregateExpr: Send + Sync + Debug {
    /// Returns the aggregate expression as [`Any`](std::any::Any) so that it can be
    /// downcast to a specific implementation.
    fn as_any(&self) -> &dyn Any;
    /// the field of the final result of this aggregation.
    fn field(&self) -> Result<Field>;

    /// the accumulator used to accumulate values from the expressions.
    /// the accumulator expects the same number of arguments as `expressions` and must
    /// return states with the same description as `state_fields`
    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>>;

    /// the fields that encapsulate the Accumulator's state
    /// the number of fields here equals the number of states that the accumulator contains
    fn state_fields(&self) -> Result<Vec<Field>>;

    /// expressions that are passed to the Accumulator.
    /// Single-column aggregations such as `sum` return a single value, others (e.g. `cov`) return many.
    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>>;
}

/// An accumulator represents a stateful object that lives throughout the evaluation of multiple rows and
/// generically accumulates values. An accumulator knows how to:
/// * update its state from inputs via `update`
/// * convert its internal state to a vector of scalar values
/// * update its state from multiple accumulators' states via `merge`
/// * compute the final value from its internal state via `evaluate`
pub trait Accumulator: Send + Sync + Debug {
    /// Returns the state of the accumulator at the end of the accumulation.
    // in the case of an average on which we track `sum` and `n`, this function should return a vector
    // of two values, sum and n.
    fn state(&self) -> Result<Vec<ScalarValue>>;

    /// updates the accumulator's state from a vector of scalars.
    fn update(&mut self, values: &[ScalarValue]) -> Result<()>;

    /// updates the accumulator's state from a vector of arrays.
    fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
        if values.is_empty() {
            return Ok(());
        };
        (0..values[0].len()).try_for_each(|index| {
            let v = values
                .iter()
                .map(|array| ScalarValue::try_from_array(array, index))
                .collect::<Result<Vec<_>>>()?;
            self.update(&v)
        })
    }

    /// updates the accumulator's state from a vector of scalars.
    fn merge(&mut self, states: &[ScalarValue]) -> Result<()>;

    /// updates the accumulator's state from a vector of states.
    fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
        if states.is_empty() {
            return Ok(());
        };
        (0..states[0].len()).try_for_each(|index| {
            let v = states
                .iter()
                .map(|array| ScalarValue::try_from_array(array, index))
                .collect::<Result<Vec<_>>>()?;
            self.merge(&v)
        })
    }

    /// returns its value based on its current state.
    fn evaluate(&self) -> Result<ScalarValue>;
}

pub mod aggregates;
pub mod array_expressions;
pub mod coalesce_batches;
pub mod common;
pub mod crypto_expressions;
pub mod csv;
pub mod datetime_expressions;
pub mod distinct_expressions;
pub mod empty;
pub mod explain;
pub mod expressions;
pub mod filter;
pub mod functions;
pub mod group_scalar;
pub mod hash_aggregate;
pub mod hash_join;
pub mod hash_utils;
pub mod limit;
pub mod math_expressions;
pub mod memory;
pub mod merge;
pub mod parquet;
pub mod planner;
pub mod projection;
pub mod repartition;
pub mod sort;
pub mod string_expressions;
pub mod type_coercion;
pub mod udaf;
pub mod udf;
<commit_msg>ARROW-11545: [Rust] [DataFusion] SendableRecordBatchStream should implement Sync<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Traits for physical query plan, supporting parallel execution for partitioned relations.

use std::fmt::{Debug, Display};
use std::sync::Arc;
use std::{any::Any, pin::Pin};

use crate::execution::context::ExecutionContextState;
use crate::logical_plan::LogicalPlan;
use crate::{error::Result, scalar::ScalarValue};
use arrow::datatypes::{DataType, Schema, SchemaRef};
use arrow::error::Result as ArrowResult;
use arrow::record_batch::RecordBatch;
use arrow::{array::ArrayRef, datatypes::Field};

use async_trait::async_trait;
use futures::stream::Stream;

use self::merge::MergeExec;

/// Trait for types that stream [arrow::record_batch::RecordBatch]
pub trait RecordBatchStream: Stream<Item = ArrowResult<RecordBatch>> {
    /// Returns the schema of this `RecordBatchStream`.
    ///
    /// Implementation of this trait should guarantee that all `RecordBatch`'s returned by this
    /// stream should have the same schema as returned from this method.
    fn schema(&self) -> SchemaRef;
}

/// Trait for a stream of record batches.
pub type SendableRecordBatchStream = Pin<Box<dyn RecordBatchStream + Send + Sync>>;

/// Physical query planner that converts a `LogicalPlan` to an
/// `ExecutionPlan` suitable for execution.
pub trait PhysicalPlanner {
    /// Create a physical plan from a logical plan
    fn create_physical_plan(
        &self,
        logical_plan: &LogicalPlan,
        ctx_state: &ExecutionContextState,
    ) -> Result<Arc<dyn ExecutionPlan>>;
}

/// Partition-aware execution plan for a relation
#[async_trait]
pub trait ExecutionPlan: Debug + Send + Sync {
    /// Returns the execution plan as [`Any`](std::any::Any) so that it can be
    /// downcast to a specific implementation.
    fn as_any(&self) -> &dyn Any;
    /// Get the schema for this execution plan
    fn schema(&self) -> SchemaRef;
    /// Specifies the output partitioning scheme of this plan
    fn output_partitioning(&self) -> Partitioning;
    /// Specifies the data distribution requirements of all the children for this operator
    fn required_child_distribution(&self) -> Distribution {
        Distribution::UnspecifiedDistribution
    }
    /// Get a list of child execution plans that provide the input for this plan. The returned list
    /// will be empty for leaf nodes, will contain a single value for unary nodes, or two
    /// values for binary nodes (such as joins).
    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>>;
    /// Returns a new plan where all children were replaced by new plans.
    /// The size of `children` must be equal to the size of `ExecutionPlan::children()`.
    fn with_new_children(
        &self,
        children: Vec<Arc<dyn ExecutionPlan>>,
    ) -> Result<Arc<dyn ExecutionPlan>>;

    /// creates an iterator
    async fn execute(&self, partition: usize) -> Result<SendableRecordBatchStream>;
}

/// Execute the [ExecutionPlan] and collect the results in memory
pub async fn collect(plan: Arc<dyn ExecutionPlan>) -> Result<Vec<RecordBatch>> {
    match plan.output_partitioning().partition_count() {
        0 => Ok(vec![]),
        1 => {
            let it = plan.execute(0).await?;
            common::collect(it).await
        }
        _ => {
            // merge into a single partition
            let plan = MergeExec::new(plan.clone());
            // MergeExec must produce a single partition
            assert_eq!(1, plan.output_partitioning().partition_count());
            common::collect(plan.execute(0).await?).await
        }
    }
}

/// Execute the [ExecutionPlan] and collect the results in memory
pub async fn collect_partitioned(
    plan: Arc<dyn ExecutionPlan>,
) -> Result<Vec<Vec<RecordBatch>>> {
    match plan.output_partitioning().partition_count() {
        0 => Ok(vec![]),
        1 => {
            let it = plan.execute(0).await?;
            Ok(vec![common::collect(it).await?])
        }
        _ => {
            let mut partitions = vec![];
            for i in 0..plan.output_partitioning().partition_count() {
                partitions.push(common::collect(plan.execute(i).await?).await?)
            }
            Ok(partitions)
        }
    }
}

/// Partitioning schemes supported by operators.
#[derive(Debug, Clone)]
pub enum Partitioning {
    /// Allocate batches using a round-robin algorithm and the specified number of partitions
    RoundRobinBatch(usize),
    /// Allocate rows based on a hash of one of more expressions and the specified
    /// number of partitions
    /// This partitioning scheme is not yet fully supported. See [ARROW-11011](https://issues.apache.org/jira/browse/ARROW-11011)
    Hash(Vec<Arc<dyn PhysicalExpr>>, usize),
    /// Unknown partitioning scheme with a known number of partitions
    UnknownPartitioning(usize),
}

impl Partitioning {
    /// Returns the number of partitions in this partitioning scheme
    pub fn partition_count(&self) -> usize {
        use Partitioning::*;
        match self {
            RoundRobinBatch(n) => *n,
            Hash(_, n) => *n,
            UnknownPartitioning(n) => *n,
        }
    }
}

/// Distribution schemes
#[derive(Debug, Clone)]
pub enum Distribution {
    /// Unspecified distribution
    UnspecifiedDistribution,
    /// A single partition is required
    SinglePartition,
}

/// Represents the result from an expression
pub enum ColumnarValue {
    /// Array of values
    Array(ArrayRef),
    /// A single value
    Scalar(ScalarValue),
}

impl ColumnarValue {
    fn data_type(&self) -> DataType {
        match self {
            ColumnarValue::Array(array_value) => array_value.data_type().clone(),
            ColumnarValue::Scalar(scalar_value) => scalar_value.get_datatype(),
        }
    }

    fn into_array(self, num_rows: usize) -> ArrayRef {
        match self {
            ColumnarValue::Array(array) => array,
            ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(num_rows),
        }
    }
}

/// Expression that can be evaluated against a RecordBatch
/// A Physical expression knows its type, nullability and how to evaluate itself.
pub trait PhysicalExpr: Send + Sync + Display + Debug {
    /// Returns the physical expression as [`Any`](std::any::Any) so that it can be
    /// downcast to a specific implementation.
    fn as_any(&self) -> &dyn Any;
    /// Get the data type of this expression, given the schema of the input
    fn data_type(&self, input_schema: &Schema) -> Result<DataType>;
    /// Determine whether this expression is nullable, given the schema of the input
    fn nullable(&self, input_schema: &Schema) -> Result<bool>;
    /// Evaluate an expression against a RecordBatch
    fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue>;
}

/// An aggregate expression that:
/// * knows its resulting field
/// * knows how to create its accumulator
/// * knows its accumulator's state's field
/// * knows the expressions from whose its accumulator will receive values
pub trait AggregateExpr: Send + Sync + Debug {
    /// Returns the aggregate expression as [`Any`](std::any::Any) so that it can be
    /// downcast to a specific implementation.
    fn as_any(&self) -> &dyn Any;
    /// the field of the final result of this aggregation.
    fn field(&self) -> Result<Field>;

    /// the accumulator used to accumulate values from the expressions.
    /// the accumulator expects the same number of arguments as `expressions` and must
    /// return states with the same description as `state_fields`
    fn create_accumulator(&self) -> Result<Box<dyn Accumulator>>;

    /// the fields that encapsulate the Accumulator's state
    /// the number of fields here equals the number of states that the accumulator contains
    fn state_fields(&self) -> Result<Vec<Field>>;

    /// expressions that are passed to the Accumulator.
    /// Single-column aggregations such as `sum` return a single value, others (e.g. `cov`) return many.
    fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>>;
}

/// An accumulator represents a stateful object that lives throughout the evaluation of multiple rows and
/// generically accumulates values. An accumulator knows how to:
/// * update its state from inputs via `update`
/// * convert its internal state to a vector of scalar values
/// * update its state from multiple accumulators' states via `merge`
/// * compute the final value from its internal state via `evaluate`
pub trait Accumulator: Send + Sync + Debug {
    /// Returns the state of the accumulator at the end of the accumulation.
    // in the case of an average on which we track `sum` and `n`, this function should return a vector
    // of two values, sum and n.
    fn state(&self) -> Result<Vec<ScalarValue>>;

    /// updates the accumulator's state from a vector of scalars.
    fn update(&mut self, values: &[ScalarValue]) -> Result<()>;

    /// updates the accumulator's state from a vector of arrays.
    fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
        if values.is_empty() {
            return Ok(());
        };
        (0..values[0].len()).try_for_each(|index| {
            let v = values
                .iter()
                .map(|array| ScalarValue::try_from_array(array, index))
                .collect::<Result<Vec<_>>>()?;
            self.update(&v)
        })
    }

    /// updates the accumulator's state from a vector of scalars.
    fn merge(&mut self, states: &[ScalarValue]) -> Result<()>;

    /// updates the accumulator's state from a vector of states.
    fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
        if states.is_empty() {
            return Ok(());
        };
        (0..states[0].len()).try_for_each(|index| {
            let v = states
                .iter()
                .map(|array| ScalarValue::try_from_array(array, index))
                .collect::<Result<Vec<_>>>()?;
            self.merge(&v)
        })
    }

    /// returns its value based on its current state.
    fn evaluate(&self) -> Result<ScalarValue>;
}

pub mod aggregates;
pub mod array_expressions;
pub mod coalesce_batches;
pub mod common;
pub mod crypto_expressions;
pub mod csv;
pub mod datetime_expressions;
pub mod distinct_expressions;
pub mod empty;
pub mod explain;
pub mod expressions;
pub mod filter;
pub mod functions;
pub mod group_scalar;
pub mod hash_aggregate;
pub mod hash_join;
pub mod hash_utils;
pub mod limit;
pub mod math_expressions;
pub mod memory;
pub mod merge;
pub mod parquet;
pub mod planner;
pub mod projection;
pub mod repartition;
pub mod sort;
pub mod string_expressions;
pub mod type_coercion;
pub mod udaf;
pub mod udf;
<|endoftext|>


<commit_before>use rustc::lint::*;
use syntax::ast::*;
use syntax::codemap::Spanned;
use utils::{span_lint_and_sugg, snippet};

/// **What it does:** Checks for
///  - () being assigned to a variable
///  - () being passed to a function
///
/// **Why is this bad?** It is extremely unlikely that a user intended to assign '()' to valiable. Instead,
///   Unit is what a block evaluates to when it returns nothing. This is typically caused by a trailing
///   unintended semicolon.
///
/// **Known problems:** None.
///
/// **Example:**
/// * `let x = {"foo" ;}` when the user almost certainly intended `let x ={"foo"}`

declare_lint! {
    pub UNIT_EXPR,
    Warn,
    "unintended assignment or use of a unit typed value"
}

#[derive(Copy, Clone)]
pub struct UnitExpr;

impl LintPass for UnitExpr {
    fn get_lints(&self) -> LintArray {
        lint_array!(UNIT_EXPR)
    }
}

impl EarlyLintPass for UnitExpr {
    fn check_expr(&mut self, cx: &EarlyContext, expr: &Expr) {
        if let ExprKind::Assign(ref left, ref right) = expr.node {
            if is_unit_expr(right){
                span_lint_and_sugg(
                    cx,
                    UNIT_EXPR,
                    right.span,
                    "trailing semicolons can be tricky",
                    "remove the last semicolon",
                    "TODO".to_owned()
                )
            }
        }
        if let ExprKind::MethodCall(_, ref args) = expr.node {
            for ref arg in args{
                if is_unit_expr(arg){
                    span_lint_and_sugg(
                        cx,
                        UNIT_EXPR,
                        arg.span,
                        "trailing semicolons can be tricky",
                        "remove the last semicolon",
                        "TODO".to_owned()
                    )
                }            }
        }
        if let ExprKind::Call( _, ref args) = expr.node{
            for ref arg in args{
                if is_unit_expr(arg){
                    span_lint_and_sugg(
                        cx,
                        UNIT_EXPR,
                        arg.span,
                        "trailing semicolons can be tricky",
                        "remove the last semicolon",
                        "TODO".to_owned()
                    )
                }            }        }
    }

    fn check_stmt(&mut self, cx: &EarlyContext, stmt: &Stmt) {
        if let StmtKind::Local(ref local) = stmt.node{
            if local.pat.node == PatKind::Wild {return;}
            if let Some(ref expr) = local.init{
                if is_unit_expr(expr){
                    span_lint_and_sugg(
                        cx,
                        UNIT_EXPR,
                        local.span,
                        "trailing semicolons can be tricky",
                        "remove the last semicolon",
                        "TODO".to_owned()
                    )
                }
            }
            }
    }
}

fn is_unit_expr(expr: &Expr)->bool{
    match expr.node{
         ExprKind::Block(ref next) => {
            let ref final_stmt = &next.stmts[next.stmts.len()-1];
            if let StmtKind::Expr(_) = final_stmt.node{
                return false;
            }
            else{
                return true;
            }
        },
        _ => return false,
    }
}<commit_msg>Fix brace indentation<commit_after>use rustc::lint::*;
use syntax::ast::*;
use syntax::codemap::Spanned;
use utils::{span_lint_and_sugg, snippet};

/// **What it does:** Checks for
///  - () being assigned to a variable
///  - () being passed to a function
///
/// **Why is this bad?** It is extremely unlikely that a user intended to assign '()' to valiable. Instead,
///   Unit is what a block evaluates to when it returns nothing. This is typically caused by a trailing
///   unintended semicolon.
///
/// **Known problems:** None.
///
/// **Example:**
/// * `let x = {"foo" ;}` when the user almost certainly intended `let x ={"foo"}`

declare_lint! {
    pub UNIT_EXPR,
    Warn,
    "unintended assignment or use of a unit typed value"
}

#[derive(Copy, Clone)]
pub struct UnitExpr;

impl LintPass for UnitExpr {
    fn get_lints(&self) -> LintArray {
        lint_array!(UNIT_EXPR)
    }
}

impl EarlyLintPass for UnitExpr {
    fn check_expr(&mut self, cx: &EarlyContext, expr: &Expr) {
        if let ExprKind::Assign(ref left, ref right) = expr.node {
            if is_unit_expr(right){
                span_lint_and_sugg(
                    cx,
                    UNIT_EXPR,
                    right.span,
                    "trailing semicolons can be tricky",
                    "remove the last semicolon",
                    "TODO".to_owned()
                )
            }
        }
        if let ExprKind::MethodCall(_, ref args) = expr.node {
            for ref arg in args{
                if is_unit_expr(arg){
                    span_lint_and_sugg(
                        cx,
                        UNIT_EXPR,
                        arg.span,
                        "trailing semicolons can be tricky",
                        "remove the last semicolon",
                        "TODO".to_owned()
                    )
                }
            }
        }
        if let ExprKind::Call( _, ref args) = expr.node{
            for ref arg in args{
                if is_unit_expr(arg){
                    span_lint_and_sugg(
                        cx,
                        UNIT_EXPR,
                        arg.span,
                        "trailing semicolons can be tricky",
                        "remove the last semicolon",
                        "TODO".to_owned()
                    )
                }
            }
        }
    }

    fn check_stmt(&mut self, cx: &EarlyContext, stmt: &Stmt) {
        if let StmtKind::Local(ref local) = stmt.node{
            if local.pat.node == PatKind::Wild {return;}
            if let Some(ref expr) = local.init{
                if is_unit_expr(expr){
                    span_lint_and_sugg(
                        cx,
                        UNIT_EXPR,
                        local.span,
                        "trailing semicolons can be tricky",
                        "remove the last semicolon",
                        "TODO".to_owned()
                    )
                }
            }
            }
    }
}

fn is_unit_expr(expr: &Expr)->bool{
    match expr.node{
         ExprKind::Block(ref next) => {
            let ref final_stmt = &next.stmts[next.stmts.len()-1];
            if let StmtKind::Expr(_) = final_stmt.node{
                return false;
            }
            else{
                return true;
            }
        },
        _ => return false,
    }
}<|endoftext|>


<commit_before>// 当且仅当测试套件没有运行时，才条件编译 `main` 函数。
#[cfg(not(test))]
fn main() {
    println!("If you see this, the tests were not compiled nor ran!");
}

// 当且仅当测试套件没有启动时，才条件编译 `test` 模块。
#[cfg(test)]
mod test {
    // 需要一个辅助函数 `distance_test`。
    fn distance(a: (f32, f32), b: (f32, f32)) -> f32 {
        (
            (b.0 - a.0).powi(2) +
            (b.1 - a.1).powi(2)
        ).sqrt()
    }

    #[test]
    fn distance_test() {
        assert!(distance((0f32, 0f32), (1f32, 1f32)) == (2f32).sqrt());
    }

    #[test]
    #[should_panic]
    fn failing_test() {
        assert!(1i32 == 2i32);
    }
}
<commit_msg>修正一处翻译错误<commit_after>// 当且仅当测试套件没有运行时，才条件编译 `main` 函数。
#[cfg(not(test))]
fn main() {
    println!("If you see this, the tests were not compiled nor ran!");
}

// 当且仅当测试套件运行时，才条件编译 `test` 模块。
#[cfg(test)]
mod test {
    // 需要一个辅助函数 `distance_test`。
    fn distance(a: (f32, f32), b: (f32, f32)) -> f32 {
        (
            (b.0 - a.0).powi(2) +
            (b.1 - a.1).powi(2)
        ).sqrt()
    }

    #[test]
    fn distance_test() {
        assert!(distance((0f32, 0f32), (1f32, 1f32)) == (2f32).sqrt());
    }

    #[test]
    #[should_panic]
    fn failing_test() {
        assert!(1i32 == 2i32);
    }
}
<|endoftext|>


<commit_before>use std::fmt;
use std::marker::PhantomData;

use crate::reflect::runtime_types::RuntimeTypeEnumOrUnknown;
use crate::reflect::EnumDescriptor;
use crate::reflect::ProtobufValue;
use crate::Enum;
use crate::EnumFull;

/// Protobuf enums with possibly unknown values are preserved in this struct.
#[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone)]
#[repr(transparent)]
// TODO: specify <E: ProtobufEnum> when it no longer prevents using const fns
pub struct EnumOrUnknown<E> {
    value: i32,
    _marker: PhantomData<E>,
}

// TODO: move into <E: ProtobufEnum> when no longer:
//  trait bounds other than `Sized` on const fn parameters are unstable
impl<E> EnumOrUnknown<E> {
    /// Construct from any `i32` value.
    ///
    /// Note passed value is not required to be a valid enum value.
    pub const fn from_i32(value: i32) -> EnumOrUnknown<E> {
        EnumOrUnknown {
            value,
            _marker: PhantomData,
        }
    }
}

impl<E: Enum> EnumOrUnknown<E> {
    /// Construct from typed enum
    pub fn new(e: E) -> EnumOrUnknown<E> {
        EnumOrUnknown::from_i32(e.value())
    }

    /// Get contained `i32` value of enum
    pub fn value(&self) -> i32 {
        self.value
    }

    /// Get `i32` value as typed enum. Return `None` is value is unknown.
    pub fn enum_value(&self) -> Result<E, i32> {
        E::from_i32(self.value).ok_or(self.value)
    }

    /// Get contained enum, panic if value is unknown.
    pub fn unwrap(&self) -> E {
        self.enum_value().unwrap()
    }

    /// Get `i32` value as typed enum.
    /// Return default enum value (first value) if value is unknown.
    pub fn enum_value_or_default(&self) -> E {
        self.enum_value().unwrap_or_default()
    }

    /// Get `i32` value as typed enum.
    /// Return given enum value if value is unknown.
    pub fn enum_value_or(&self, map_unknown: E) -> E {
        self.enum_value().unwrap_or(map_unknown)
    }
}

impl<E: EnumFull> EnumOrUnknown<E> {
    /// Get enum descriptor by type.
    pub fn enum_descriptor_static() -> EnumDescriptor {
        E::enum_descriptor_static()
    }
}

impl<E: Enum> From<E> for EnumOrUnknown<E> {
    fn from(e: E) -> Self {
        EnumOrUnknown::new(e)
    }
}

impl<E: EnumFull> Default for EnumOrUnknown<E> {
    fn default() -> EnumOrUnknown<E> {
        EnumOrUnknown::new(E::default())
    }
}

impl<E: EnumFull> fmt::Debug for EnumOrUnknown<E> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self.enum_value() {
            Ok(e) => fmt::Debug::fmt(&e, f),
            Err(e) => fmt::Debug::fmt(&e, f),
        }
    }
}

impl<E: EnumFull> ProtobufValue for EnumOrUnknown<E> {
    type RuntimeType = RuntimeTypeEnumOrUnknown<E>;
}
<commit_msg>Do not require EnumFull for Default for EnumOrUnknown<commit_after>use std::fmt;
use std::marker::PhantomData;

use crate::reflect::runtime_types::RuntimeTypeEnumOrUnknown;
use crate::reflect::EnumDescriptor;
use crate::reflect::ProtobufValue;
use crate::Enum;
use crate::EnumFull;

/// Protobuf enums with possibly unknown values are preserved in this struct.
#[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone)]
#[repr(transparent)]
// TODO: specify <E: ProtobufEnum> when it no longer prevents using const fns
pub struct EnumOrUnknown<E> {
    value: i32,
    _marker: PhantomData<E>,
}

// TODO: move into <E: ProtobufEnum> when no longer:
//  trait bounds other than `Sized` on const fn parameters are unstable
impl<E> EnumOrUnknown<E> {
    /// Construct from any `i32` value.
    ///
    /// Note passed value is not required to be a valid enum value.
    pub const fn from_i32(value: i32) -> EnumOrUnknown<E> {
        EnumOrUnknown {
            value,
            _marker: PhantomData,
        }
    }
}

impl<E: Enum> EnumOrUnknown<E> {
    /// Construct from typed enum
    pub fn new(e: E) -> EnumOrUnknown<E> {
        EnumOrUnknown::from_i32(e.value())
    }

    /// Get contained `i32` value of enum
    pub fn value(&self) -> i32 {
        self.value
    }

    /// Get `i32` value as typed enum. Return `None` is value is unknown.
    pub fn enum_value(&self) -> Result<E, i32> {
        E::from_i32(self.value).ok_or(self.value)
    }

    /// Get contained enum, panic if value is unknown.
    pub fn unwrap(&self) -> E {
        self.enum_value().unwrap()
    }

    /// Get `i32` value as typed enum.
    /// Return default enum value (first value) if value is unknown.
    pub fn enum_value_or_default(&self) -> E {
        self.enum_value().unwrap_or_default()
    }

    /// Get `i32` value as typed enum.
    /// Return given enum value if value is unknown.
    pub fn enum_value_or(&self, map_unknown: E) -> E {
        self.enum_value().unwrap_or(map_unknown)
    }
}

impl<E: EnumFull> EnumOrUnknown<E> {
    /// Get enum descriptor by type.
    pub fn enum_descriptor_static() -> EnumDescriptor {
        E::enum_descriptor_static()
    }
}

impl<E: Enum> From<E> for EnumOrUnknown<E> {
    fn from(e: E) -> Self {
        EnumOrUnknown::new(e)
    }
}

impl<E: Enum> Default for EnumOrUnknown<E> {
    fn default() -> EnumOrUnknown<E> {
        EnumOrUnknown::new(E::default())
    }
}

impl<E: EnumFull> fmt::Debug for EnumOrUnknown<E> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self.enum_value() {
            Ok(e) => fmt::Debug::fmt(&e, f),
            Err(e) => fmt::Debug::fmt(&e, f),
        }
    }
}

impl<E: EnumFull> ProtobufValue for EnumOrUnknown<E> {
    type RuntimeType = RuntimeTypeEnumOrUnknown<E>;
}
<|endoftext|>