<commit_before>//! Endpoints for client configuration.

/// [PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}](https://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-user-userid-rooms-roomid-account-data-type)
pub mod set_room_account_data {
    use ruma_api_macros::ruma_api;
    use ruma_identifiers::{RoomId, UserId};
    use serde_json::Value;

    ruma_api! {
        metadata {
            description: "Associate account data with a room.",
            method: Method::Put,
            name: "set_room_account_data",
            path: "/_matrix/client/r0/user/:user_id/rooms/:room_id/account_data/:event_type",
            rate_limited: false,
            requires_authentication: true,
        }

        request {
            /// Arbitrary JSON to store as config data.
            #[ruma_api(body)]
            pub data: Value,
            /// The event type of the account_data to set.
            ///
            /// Custom types should be namespaced to avoid clashes.
            #[ruma_api(path)]
            #[serde(rename = "type")]
            pub event_type: String,
            /// The ID of the room to set account_data on.
            #[ruma_api(path)]
            pub room_id: RoomId,
            /// The ID of the user to set account_data for.
            ///
            /// The access token must be authorized to make requests for this user ID.
            #[ruma_api(path)]
            pub user_id: UserId,
        }

        response {}
    }
}

/// [PUT /_matrix/client/r0/user/{userId}/account_data/{type}](https://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-user-userid-account-data-type)
pub mod set_global_account_data  {
    use ruma_api_macros::ruma_api;
    use ruma_identifiers::UserId;
    use serde_json::Value;

    ruma_api! {
        metadata {
            description: "Sets global account data.",
            method: Method::Put,
            name: "set_global_account_data",
            path: "/_matrix/client/r0/user/:user_id/account_data/:event_type",
            rate_limited: false,
            requires_authentication: true,
        }

        request {
            /// Arbitrary JSON to store as config data.
            #[ruma_api(body)]
            pub data: Value,
            /// The event type of the account_data to set.
            ///
            /// Custom types should be namespaced to avoid clashes.
            #[ruma_api(path)]
            #[serde(rename = "type")]
            pub event_type: String,
            /// The ID of the user to set account_data for.
            ///
            /// The access token must be authorized to make requests for this user ID.
            #[ruma_api(path)]
            pub user_id: UserId,
        }

        response {}
    }
}
<commit_msg>Remove unused serde attributes<commit_after>//! Endpoints for client configuration.

/// [PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}](https://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-user-userid-rooms-roomid-account-data-type)
pub mod set_room_account_data {
    use ruma_api_macros::ruma_api;
    use ruma_identifiers::{RoomId, UserId};
    use serde_json::Value;

    ruma_api! {
        metadata {
            description: "Associate account data with a room.",
            method: Method::Put,
            name: "set_room_account_data",
            path: "/_matrix/client/r0/user/:user_id/rooms/:room_id/account_data/:event_type",
            rate_limited: false,
            requires_authentication: true,
        }

        request {
            /// Arbitrary JSON to store as config data.
            #[ruma_api(body)]
            pub data: Value,
            /// The event type of the account_data to set.
            ///
            /// Custom types should be namespaced to avoid clashes.
            #[ruma_api(path)]
            pub event_type: String,
            /// The ID of the room to set account_data on.
            #[ruma_api(path)]
            pub room_id: RoomId,
            /// The ID of the user to set account_data for.
            ///
            /// The access token must be authorized to make requests for this user ID.
            #[ruma_api(path)]
            pub user_id: UserId,
        }

        response {}
    }
}

/// [PUT /_matrix/client/r0/user/{userId}/account_data/{type}](https://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-user-userid-account-data-type)
pub mod set_global_account_data  {
    use ruma_api_macros::ruma_api;
    use ruma_identifiers::UserId;
    use serde_json::Value;

    ruma_api! {
        metadata {
            description: "Sets global account data.",
            method: Method::Put,
            name: "set_global_account_data",
            path: "/_matrix/client/r0/user/:user_id/account_data/:event_type",
            rate_limited: false,
            requires_authentication: true,
        }

        request {
            /// Arbitrary JSON to store as config data.
            #[ruma_api(body)]
            pub data: Value,
            /// The event type of the account_data to set.
            ///
            /// Custom types should be namespaced to avoid clashes.
            #[ruma_api(path)]
            pub event_type: String,
            /// The ID of the user to set account_data for.
            ///
            /// The access token must be authorized to make requests for this user ID.
            #[ruma_api(path)]
            pub user_id: UserId,
        }

        response {}
    }
}
<|endoftext|>


<commit_before>//! This module contains shareable mutable containers designed for enabling
//! and detecting race conditions in thread synchronization testing code.
//!
//! # Motivation
//!
//! The main purpose of a thread synchronization protocol is to ensure that
//! some operations which are not atomic in hardware, such as writing to two
//! unrelated memory locations, appear to occur as atomic transactions from the
//! point of view of other threads: at any point of time, either a given
//! operation appears to be done, or it appears not to have started.
//!
//! Testing a thread synchronization primitive involves showing that
//! inconsistent states (where an operation appears to be half-performed) have a
//! negligible probability of being exposed to the outside world in all expected
//! usage scenarios. It is typically done by showing that a given non-atomic
//! operation is properly encapsulated by the transactional semantics of the
//! synchronization protocol, and will never appear as half-done to observers.
//!
//! Non-atomic operations are easier said than done, however, when the set of
//! operations which can be atomic in hardware is larger than most people think
//! (at the time of writing, current Intel CPUs can access memory in blocks of
//! 128 bits, and current NVidia GPUs can do so in blocks of 1024 bits), not
//! well-defined by the architecture (and thus subjected to increase in future
//! hardware), and highly dependent on the compiler's optimization choices in a
//! high-level programming language such as Rust.
//!
//! Which is why I think we need some types whose operations are guaranteed to
//! be non-atomic under a set of reasonable assumptions, and which can easily be
//! observed to be in an inconsistent state.
//!
//! # Functionality
//!
//! This module provides the RaceCell type, which can hold a value of a
//! certain type T like a `Cell<T>` would, but is guaranteed **not** to be read
//! or written to in a single atomic operation, even if the corresponding type T
//! can be atomically read or written to in hardware.
//!
//! Furthermore, a RaceCell can detect scenarios where it is read at the same
//! time as it is being written (which constitutes a read-after-write race
//! condition) and report them to the reader thread.
//!
//! Such "controlled data races" can, in turn, be used to detect failures in
//! thread synchronization protocols, manifesting as inconsistent shared states
//! being exposed to the outside world.
//!
//! # Requirements on T
//!
//! In principle, any Clone + Eq type T whose equality operator and clone()
//! implementation behave well even if the inner data is in a inconsistent state
//! (e.g. filled with random bits) could be used. This is true of all primitive
//! integral types and aggregates thereof, for example.
//!
//! However, in practice, unsynchronized concurrent read/write access to
//! arbitrary data from multiple threads constitutes a data race, which is
//! undefined behaviour in Rust even when it occurs inside an UnsafeCell. This
//! is problematic because rustc's optimizer allows itself to transform code
//! which relies on undefined behaviour however it likes, leading to breakages
//! in release builds such as infinite loops appearing out of nowhere.
//!
//! For this reason, we currently only support some specific types T which have
//! atomic load and store operations, implemented as part of an atomic wrapper.
//! Note that although individual loads and stores to T are atomic, loads and
//! stores to RaceCell<T> are still guaranteed not to be atomic.

#![deny(missing_docs)]

use std::sync::atomic::{
    AtomicBool, AtomicI16, AtomicI32, AtomicI64, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16,
    AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering,
};

/// Shareable mutable container for triggering and detecting write-after-read
/// data races in a well-controlled fashion.
#[derive(Debug)]
pub struct RaceCell<T: AtomicData> {
    /// Two copies of a value of type T are made. One is stored on the stack...
    local_contents: T::AtomicWrapper,

    /// ...and one is stored on the heap, which in all popular OSs is too far
    /// away from the stack to allow any significant probability of the hardware
    /// writing both copies in a single atomic transactions.
    ///
    /// Of course, a malicious optimizer could still use hardware transactional
    /// memory or a software emulation thereof to achieve this effect, but there
    /// are no performance benefits in doing so, and in fact it will rather have
    /// an averse effect on performance, so a realistic optimizer won't do it.
    ///
    remote_version: Box<T::AtomicWrapper>,
}
//
impl<T: AtomicData> RaceCell<T> {
    /// Create a new RaceCell with a certain initial content
    pub fn new(value: T) -> Self {
        RaceCell {
            local_contents: T::AtomicWrapper::new(value.clone()),
            remote_version: Box::new(T::AtomicWrapper::new(value)),
        }
    }

    /// Update the internal contents of the RaceCell in a non-atomic fashion
    pub fn set(&self, value: T) {
        self.local_contents.relaxed_store(value.clone());
        self.remote_version.relaxed_store(value);
    }

    /// Read the current contents of the RaceCell, detecting any data race
    /// caused by a concurrently occurring write along the way.
    pub fn get(&self) -> Racey<T> {
        let local_data = self.local_contents.relaxed_load();
        let remote_data = self.remote_version.relaxed_load();
        if local_data == remote_data {
            Racey::Consistent(local_data)
        } else {
            Racey::Inconsistent
        }
    }
}
//
impl<T: AtomicData> Clone for RaceCell<T> {
    /// Making RaceCells cloneable allows putting them in concurrent containers
    fn clone(&self) -> Self {
        let local_copy = self.local_contents.relaxed_load();
        let remote_copy = self.remote_version.relaxed_load();
        RaceCell {
            local_contents: T::AtomicWrapper::new(local_copy),
            remote_version: Box::new(T::AtomicWrapper::new(remote_copy)),
        }
    }
}
//
impl<T: AtomicData + Default> Default for RaceCell<T> {
    /// A RaceCell has a default value if the inner type has
    fn default() -> Self {
        Self::new(T::default())
    }
}

/// This is the result of a RaceCell read
#[derive(Debug, Eq, PartialEq)]
pub enum Racey<U: AtomicData> {
    /// The RaceCell was internally consistent, and its content was copied
    Consistent(U),

    /// The RaceCell was internally inconsistent: a data race has occurred
    Inconsistent,
}

/// Requirements on the data held by a RaceCell
pub trait AtomicData: Clone + Eq + Sized {
    /// Atomic wrapper type for this data implementing relaxed atomic load/store
    type AtomicWrapper: AtomicLoadStore<Content = Self>;
}
///
/// Atomic wrapper type for a certain kind of value
///
/// For the implementation of RaceCell not to be considered as undefined
/// behaviour and trashed by the Rust compiler in release mode, the underlying
/// type must be loaded and stored atomically. This requires synchronizing
/// accesses to it using things like the std::sync::atomic::AtomicXyz wrappers.
///
/// The only guarantee that we need is that loads and stores are atomic. We do
/// not need any other memory ordering guarantee. This is why we allow for
/// more wrapper type implementation freedom by not specifying memory orderings,
/// and internally relying only on relaxed ordering.
///
pub trait AtomicLoadStore: Sized {
    /// Type of data that is being wrapped
    type Content: AtomicData<AtomicWrapper = Self>;

    /// Create an atomic wrapper for a value of type U
    fn new(v: Self::Content) -> Self;

    /// Atomically load a value from the wrapper
    fn relaxed_load(&self) -> Self::Content;

    /// Atomically store a new value into the wrapper
    fn relaxed_store(&self, val: Self::Content);
}
///
/// This macro implements support for non-generic standard atomic types
///
macro_rules! impl_atomic_data {
    ($($data:ty => $wrapper:ty),*) => ($(
        impl AtomicData for $data {
            type AtomicWrapper = $wrapper;
        }

        impl AtomicLoadStore for $wrapper {
            type Content = $data;

            fn new(v: $data) -> $wrapper {
                <$wrapper>::new(v)
            }

            fn relaxed_load(&self) -> $data {
                <$wrapper>::load(self, Ordering::Relaxed)
            }

            fn relaxed_store(&self, val: $data) {
                <$wrapper>::store(self, val, Ordering::Relaxed)
            }
        }
    )*)
}
///
impl_atomic_data! {
    bool  => AtomicBool,
    i8    => AtomicI8,
    i16   => AtomicI16,
    i32   => AtomicI32,
    i64   => AtomicI64,
    isize => AtomicIsize,
    u8    => AtomicU8,
    u16   => AtomicU16,
    u32   => AtomicU32,
    u64   => AtomicU64,
    usize => AtomicUsize
}
//
// Atomic pointers are a bit special as they are generic, for now we will just
// treat them as a special case.
//
impl<V> AtomicData for *mut V {
    type AtomicWrapper = AtomicPtr<V>;
}
///
impl<V> AtomicLoadStore for AtomicPtr<V> {
    type Content = *mut V;

    fn new(v: *mut V) -> AtomicPtr<V> {
        <AtomicPtr<V>>::new(v)
    }

    fn relaxed_load(&self) -> *mut V {
        <AtomicPtr<V>>::load(self, Ordering::Relaxed)
    }

    fn relaxed_store(&self, val: *mut V) {
        <AtomicPtr<V>>::store(self, val, Ordering::Relaxed)
    }
}

// FIXME: The astute reader will have noted that any data could be theoretically
//        put in a RaceCell by using a Mutex as the AtomicWrapper. However, this
//        will only be implemented once Rust has specialization, to avoid
//        pessimizing the common case where a primitive type is enough.

/// Here are some RaceCell tests
#[cfg(test)]
mod tests {
    use super::{AtomicLoadStore, RaceCell, Racey};
    use std::sync::{Arc, Mutex};

    /// A RaceCell should be created in a consistent and correct state
    #[test]
    fn initial_state() {
        let cell = RaceCell::new(true);
        assert_eq!(cell.local_contents.relaxed_load(), true);
        assert_eq!(cell.remote_version.relaxed_load(), true);
    }

    /// Reading a consistent RaceCell should work as expected
    #[test]
    fn consistent_read() {
        let cell = RaceCell::new(-42_isize);
        assert_eq!(cell.get(), Racey::Consistent(-42));
    }

    /// Reading an inconsistent RaceCell should work as expected
    #[test]
    fn inconsistent_read() {
        let cell = RaceCell::new(0xbad_usize);
        cell.local_contents.relaxed_store(0xdead);
        assert_eq!(cell.get(), Racey::Inconsistent);
    }

    /// RaceCells should be cloned as-is, even if in an inconsistent state
    #[test]
    fn clone() {
        let cell = RaceCell::new(0xbeef_usize);
        cell.local_contents.relaxed_store(0xdeaf);
        let clone = cell.clone();
        assert_eq!(clone.local_contents.relaxed_load(), 0xdeaf);
        assert_eq!(clone.remote_version.relaxed_load(), 0xbeef);
    }

    /// Unprotected concurrent reads and writes to a RaceCell should trigger
    /// detectable race conditions, illustrating its non-atomic nature.
    ///
    /// To maximize the odds of race conditions, this kind of test should be run
    /// in single-threaded mode.
    ///
    #[test]
    #[ignore]
    fn unprotected_race() {
        // Amount of writes to carry out
        const WRITES_COUNT: usize = 100_000_000;

        // RaceCell in which the writes will be carried out
        let initial_value = 0usize;
        let cell1 = Arc::new(RaceCell::new(initial_value));
        let cell2 = cell1.clone();

        // Make sure that RaceCell does expose existing data races, with a
        // detection probability better than 1% for very obvious ones :)
        crate::concurrent_test_2(
            move || {
                for i in 1..=WRITES_COUNT {
                    cell1.set(i);
                }
            },
            move || {
                let mut last_value = 0;
                let mut data_race_count = 0usize;
                while last_value != WRITES_COUNT {
                    match cell2.get() {
                        Racey::Consistent(value) => last_value = value,
                        Racey::Inconsistent => data_race_count += 1,
                    }
                }
                print!("{} races detected: ", data_race_count);
                assert!(data_race_count > WRITES_COUNT / 100);
            },
        );
    }

    /// Appropriately protected concurrent reads and writes to a RaceCell should
    /// not yield any detectable race conditions.
    ///
    /// To maximize the odds of race conditions, this kind of test should be run
    /// in single-threaded mode.
    ///
    #[test]
    #[ignore]
    fn protected_transaction() {
        // Amount of writes to carry out
        const WRITES_COUNT: usize = 10_000_000;

        // Mutex-protected RaceCell in which the writes will be carried out
        let initial_value = 0usize;
        let cell1 = Arc::new(Mutex::new(RaceCell::new(initial_value)));
        let cell2 = cell1.clone();

        // Make sure that RaceCell does not incorrectly detect race conditions
        crate::concurrent_test_2(
            move || {
                for i in 1..=WRITES_COUNT {
                    cell1.lock().unwrap().set(i);
                }
            },
            move || {
                let mut last_value = 0;
                let mut data_race_count = 0usize;
                while last_value != WRITES_COUNT {
                    match cell2.lock().unwrap().get() {
                        Racey::Consistent(value) => last_value = value,
                        Racey::Inconsistent => data_race_count += 1,
                    }
                }
                assert_eq!(data_race_count, 0);
            },
        );
    }
}
<commit_msg>Fix lint<commit_after>//! This module contains shareable mutable containers designed for enabling
//! and detecting race conditions in thread synchronization testing code.
//!
//! # Motivation
//!
//! The main purpose of a thread synchronization protocol is to ensure that
//! some operations which are not atomic in hardware, such as writing to two
//! unrelated memory locations, appear to occur as atomic transactions from the
//! point of view of other threads: at any point of time, either a given
//! operation appears to be done, or it appears not to have started.
//!
//! Testing a thread synchronization primitive involves showing that
//! inconsistent states (where an operation appears to be half-performed) have a
//! negligible probability of being exposed to the outside world in all expected
//! usage scenarios. It is typically done by showing that a given non-atomic
//! operation is properly encapsulated by the transactional semantics of the
//! synchronization protocol, and will never appear as half-done to observers.
//!
//! Non-atomic operations are easier said than done, however, when the set of
//! operations which can be atomic in hardware is larger than most people think
//! (at the time of writing, current Intel CPUs can access memory in blocks of
//! 128 bits, and current NVidia GPUs can do so in blocks of 1024 bits), not
//! well-defined by the architecture (and thus subjected to increase in future
//! hardware), and highly dependent on the compiler's optimization choices in a
//! high-level programming language such as Rust.
//!
//! Which is why I think we need some types whose operations are guaranteed to
//! be non-atomic under a set of reasonable assumptions, and which can easily be
//! observed to be in an inconsistent state.
//!
//! # Functionality
//!
//! This module provides the RaceCell type, which can hold a value of a
//! certain type T like a `Cell<T>` would, but is guaranteed **not** to be read
//! or written to in a single atomic operation, even if the corresponding type T
//! can be atomically read or written to in hardware.
//!
//! Furthermore, a RaceCell can detect scenarios where it is read at the same
//! time as it is being written (which constitutes a read-after-write race
//! condition) and report them to the reader thread.
//!
//! Such "controlled data races" can, in turn, be used to detect failures in
//! thread synchronization protocols, manifesting as inconsistent shared states
//! being exposed to the outside world.
//!
//! # Requirements on T
//!
//! In principle, any Clone + Eq type T whose equality operator and clone()
//! implementation behave well even if the inner data is in a inconsistent state
//! (e.g. filled with random bits) could be used. This is true of all primitive
//! integral types and aggregates thereof, for example.
//!
//! However, in practice, unsynchronized concurrent read/write access to
//! arbitrary data from multiple threads constitutes a data race, which is
//! undefined behaviour in Rust even when it occurs inside an UnsafeCell. This
//! is problematic because rustc's optimizer allows itself to transform code
//! which relies on undefined behaviour however it likes, leading to breakages
//! in release builds such as infinite loops appearing out of nowhere.
//!
//! For this reason, we currently only support some specific types T which have
//! atomic load and store operations, implemented as part of an atomic wrapper.
//! Note that although individual loads and stores to T are atomic, loads and
//! stores to RaceCell<T> are still guaranteed not to be atomic.

#![deny(missing_docs)]

use std::sync::atomic::{
    AtomicBool, AtomicI16, AtomicI32, AtomicI64, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16,
    AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering,
};

/// Shareable mutable container for triggering and detecting write-after-read
/// data races in a well-controlled fashion.
#[derive(Debug)]
pub struct RaceCell<T: AtomicData> {
    /// Two copies of a value of type T are made. One is stored on the stack...
    local_contents: T::AtomicWrapper,

    /// ...and one is stored on the heap, which in all popular OSs is too far
    /// away from the stack to allow any significant probability of the hardware
    /// writing both copies in a single atomic transactions.
    ///
    /// Of course, a malicious optimizer could still use hardware transactional
    /// memory or a software emulation thereof to achieve this effect, but there
    /// are no performance benefits in doing so, and in fact it will rather have
    /// an averse effect on performance, so a realistic optimizer won't do it.
    ///
    remote_version: Box<T::AtomicWrapper>,
}
//
impl<T: AtomicData> RaceCell<T> {
    /// Create a new RaceCell with a certain initial content
    pub fn new(value: T) -> Self {
        RaceCell {
            local_contents: T::AtomicWrapper::new(value.clone()),
            remote_version: Box::new(T::AtomicWrapper::new(value)),
        }
    }

    /// Update the internal contents of the RaceCell in a non-atomic fashion
    pub fn set(&self, value: T) {
        self.local_contents.relaxed_store(value.clone());
        self.remote_version.relaxed_store(value);
    }

    /// Read the current contents of the RaceCell, detecting any data race
    /// caused by a concurrently occurring write along the way.
    pub fn get(&self) -> Racey<T> {
        let local_data = self.local_contents.relaxed_load();
        let remote_data = self.remote_version.relaxed_load();
        if local_data == remote_data {
            Racey::Consistent(local_data)
        } else {
            Racey::Inconsistent
        }
    }
}
//
impl<T: AtomicData> Clone for RaceCell<T> {
    /// Making RaceCells cloneable allows putting them in concurrent containers
    fn clone(&self) -> Self {
        let local_copy = self.local_contents.relaxed_load();
        let remote_copy = self.remote_version.relaxed_load();
        RaceCell {
            local_contents: T::AtomicWrapper::new(local_copy),
            remote_version: Box::new(T::AtomicWrapper::new(remote_copy)),
        }
    }
}
//
impl<T: AtomicData + Default> Default for RaceCell<T> {
    /// A RaceCell has a default value if the inner type has
    fn default() -> Self {
        Self::new(T::default())
    }
}

/// This is the result of a RaceCell read
#[derive(Debug, Eq, PartialEq)]
pub enum Racey<U: AtomicData> {
    /// The RaceCell was internally consistent, and its content was copied
    Consistent(U),

    /// The RaceCell was internally inconsistent: a data race has occurred
    Inconsistent,
}

/// Requirements on the data held by a RaceCell
pub trait AtomicData: Clone + Eq + Sized {
    /// Atomic wrapper type for this data implementing relaxed atomic load/store
    type AtomicWrapper: AtomicLoadStore<Content = Self>;
}
///
/// Atomic wrapper type for a certain kind of value
///
/// For the implementation of RaceCell not to be considered as undefined
/// behaviour and trashed by the Rust compiler in release mode, the underlying
/// type must be loaded and stored atomically. This requires synchronizing
/// accesses to it using things like the std::sync::atomic::AtomicXyz wrappers.
///
/// The only guarantee that we need is that loads and stores are atomic. We do
/// not need any other memory ordering guarantee. This is why we allow for
/// more wrapper type implementation freedom by not specifying memory orderings,
/// and internally relying only on relaxed ordering.
///
pub trait AtomicLoadStore: Sized {
    /// Type of data that is being wrapped
    type Content: AtomicData<AtomicWrapper = Self>;

    /// Create an atomic wrapper for a value of type U
    fn new(v: Self::Content) -> Self;

    /// Atomically load a value from the wrapper
    fn relaxed_load(&self) -> Self::Content;

    /// Atomically store a new value into the wrapper
    fn relaxed_store(&self, val: Self::Content);
}
///
/// This macro implements support for non-generic standard atomic types
///
macro_rules! impl_atomic_data {
    ($($data:ty => $wrapper:ty),*) => ($(
        impl AtomicData for $data {
            type AtomicWrapper = $wrapper;
        }

        impl AtomicLoadStore for $wrapper {
            type Content = $data;

            fn new(v: $data) -> $wrapper {
                <$wrapper>::new(v)
            }

            fn relaxed_load(&self) -> $data {
                <$wrapper>::load(self, Ordering::Relaxed)
            }

            fn relaxed_store(&self, val: $data) {
                <$wrapper>::store(self, val, Ordering::Relaxed)
            }
        }
    )*)
}
//
impl_atomic_data! {
    bool  => AtomicBool,
    i8    => AtomicI8,
    i16   => AtomicI16,
    i32   => AtomicI32,
    i64   => AtomicI64,
    isize => AtomicIsize,
    u8    => AtomicU8,
    u16   => AtomicU16,
    u32   => AtomicU32,
    u64   => AtomicU64,
    usize => AtomicUsize
}
//
// Atomic pointers are a bit special as they are generic, for now we will just
// treat them as a special case.
//
impl<V> AtomicData for *mut V {
    type AtomicWrapper = AtomicPtr<V>;
}
///
impl<V> AtomicLoadStore for AtomicPtr<V> {
    type Content = *mut V;

    fn new(v: *mut V) -> AtomicPtr<V> {
        <AtomicPtr<V>>::new(v)
    }

    fn relaxed_load(&self) -> *mut V {
        <AtomicPtr<V>>::load(self, Ordering::Relaxed)
    }

    fn relaxed_store(&self, val: *mut V) {
        <AtomicPtr<V>>::store(self, val, Ordering::Relaxed)
    }
}

// FIXME: The astute reader will have noted that any data could be theoretically
//        put in a RaceCell by using a Mutex as the AtomicWrapper. However, this
//        will only be implemented once Rust has specialization, to avoid
//        pessimizing the common case where a primitive type is enough.

/// Here are some RaceCell tests
#[cfg(test)]
mod tests {
    use super::{AtomicLoadStore, RaceCell, Racey};
    use std::sync::{Arc, Mutex};

    /// A RaceCell should be created in a consistent and correct state
    #[test]
    fn initial_state() {
        let cell = RaceCell::new(true);
        assert_eq!(cell.local_contents.relaxed_load(), true);
        assert_eq!(cell.remote_version.relaxed_load(), true);
    }

    /// Reading a consistent RaceCell should work as expected
    #[test]
    fn consistent_read() {
        let cell = RaceCell::new(-42_isize);
        assert_eq!(cell.get(), Racey::Consistent(-42));
    }

    /// Reading an inconsistent RaceCell should work as expected
    #[test]
    fn inconsistent_read() {
        let cell = RaceCell::new(0xbad_usize);
        cell.local_contents.relaxed_store(0xdead);
        assert_eq!(cell.get(), Racey::Inconsistent);
    }

    /// RaceCells should be cloned as-is, even if in an inconsistent state
    #[test]
    fn clone() {
        let cell = RaceCell::new(0xbeef_usize);
        cell.local_contents.relaxed_store(0xdeaf);
        let clone = cell.clone();
        assert_eq!(clone.local_contents.relaxed_load(), 0xdeaf);
        assert_eq!(clone.remote_version.relaxed_load(), 0xbeef);
    }

    /// Unprotected concurrent reads and writes to a RaceCell should trigger
    /// detectable race conditions, illustrating its non-atomic nature.
    ///
    /// To maximize the odds of race conditions, this kind of test should be run
    /// in single-threaded mode.
    ///
    #[test]
    #[ignore]
    fn unprotected_race() {
        // Amount of writes to carry out
        const WRITES_COUNT: usize = 100_000_000;

        // RaceCell in which the writes will be carried out
        let initial_value = 0usize;
        let cell1 = Arc::new(RaceCell::new(initial_value));
        let cell2 = cell1.clone();

        // Make sure that RaceCell does expose existing data races, with a
        // detection probability better than 1% for very obvious ones :)
        crate::concurrent_test_2(
            move || {
                for i in 1..=WRITES_COUNT {
                    cell1.set(i);
                }
            },
            move || {
                let mut last_value = 0;
                let mut data_race_count = 0usize;
                while last_value != WRITES_COUNT {
                    match cell2.get() {
                        Racey::Consistent(value) => last_value = value,
                        Racey::Inconsistent => data_race_count += 1,
                    }
                }
                print!("{} races detected: ", data_race_count);
                assert!(data_race_count > WRITES_COUNT / 100);
            },
        );
    }

    /// Appropriately protected concurrent reads and writes to a RaceCell should
    /// not yield any detectable race conditions.
    ///
    /// To maximize the odds of race conditions, this kind of test should be run
    /// in single-threaded mode.
    ///
    #[test]
    #[ignore]
    fn protected_transaction() {
        // Amount of writes to carry out
        const WRITES_COUNT: usize = 10_000_000;

        // Mutex-protected RaceCell in which the writes will be carried out
        let initial_value = 0usize;
        let cell1 = Arc::new(Mutex::new(RaceCell::new(initial_value)));
        let cell2 = cell1.clone();

        // Make sure that RaceCell does not incorrectly detect race conditions
        crate::concurrent_test_2(
            move || {
                for i in 1..=WRITES_COUNT {
                    cell1.lock().unwrap().set(i);
                }
            },
            move || {
                let mut last_value = 0;
                let mut data_race_count = 0usize;
                while last_value != WRITES_COUNT {
                    match cell2.lock().unwrap().get() {
                        Racey::Consistent(value) => last_value = value,
                        Racey::Inconsistent => data_race_count += 1,
                    }
                }
                assert_eq!(data_race_count, 0);
            },
        );
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::env;

use crate::stacked_borrows::Tag;
use crate::*;
use rustc::ty::layout::Size;
use rustc_mir::interpret::{Memory, Pointer};

#[derive(Default)]
pub struct EnvVars {
    /// Stores pointers to the environment variables. These variables must be stored as
    /// null-terminated C strings with the `"{name}={value}"` format.
    map: HashMap<Vec<u8>, Pointer<Tag>>,
}

impl EnvVars {
    pub(crate) fn init<'mir, 'tcx>(
        ecx: &mut InterpCx<'mir, 'tcx, Evaluator<'tcx>>,
        mut excluded_env_vars: Vec<String>,
    ) {
        // Exclude `TERM` var to avoid terminfo trying to open the termcap file.
        excluded_env_vars.push("TERM".to_owned());

        if ecx.machine.communicate {
            for (name, value) in env::vars() {
                if !excluded_env_vars.contains(&name) {
                    let var_ptr =
                        alloc_env_var(name.as_bytes(), value.as_bytes(), ecx.memory_mut());
                    ecx.machine.env_vars.map.insert(name.into_bytes(), var_ptr);
                }
            }
        }
    }
}

fn alloc_env_var<'mir, 'tcx>(
    name: &[u8],
    value: &[u8],
    memory: &mut Memory<'mir, 'tcx, Evaluator<'tcx>>,
) -> Pointer<Tag> {
    let mut bytes = name.to_vec();
    bytes.push(b'=');
    bytes.extend_from_slice(value);
    bytes.push(0);
    memory.allocate_static_bytes(bytes.as_slice(), MiriMemoryKind::Env.into())
}

impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
    fn getenv(&mut self, name_op: OpTy<'tcx, Tag>) -> InterpResult<'tcx, Scalar<Tag>> {
        let this = self.eval_context_mut();

        let name_ptr = this.read_scalar(name_op)?.not_undef()?;
        let name = this.memory().read_c_str(name_ptr)?;
        Ok(match this.machine.env_vars.map.get(name) {
            // The offset is used to strip the "{name}=" part of the string.
            Some(var_ptr) => {
                Scalar::Ptr(var_ptr.offset(Size::from_bytes(name.len() as u64 + 1), this)?)
            }
            None => Scalar::ptr_null(&*this.tcx),
        })
    }

    fn setenv(
        &mut self,
        name_op: OpTy<'tcx, Tag>,
        value_op: OpTy<'tcx, Tag>,
    ) -> InterpResult<'tcx, i32> {
        let this = self.eval_context_mut();

        let name_ptr = this.read_scalar(name_op)?.not_undef()?;
        let value_ptr = this.read_scalar(value_op)?.not_undef()?;
        let value = this.memory().read_c_str(value_ptr)?;
        let mut new = None;
        if !this.is_null(name_ptr)? {
            let name = this.memory().read_c_str(name_ptr)?;
            if !name.is_empty() && !name.contains(&b'=') {
                new = Some((name.to_owned(), value.to_owned()));
            }
        }
        if let Some((name, value)) = new {
            let var_ptr = alloc_env_var(&name, &value, this.memory_mut());
            if let Some(var) = this.machine.env_vars.map.insert(name.to_owned(), var_ptr) {
                this.memory_mut()
                    .deallocate(var, None, MiriMemoryKind::Env.into())?;
            }
            Ok(0)
        } else {
            Ok(-1)
        }
    }

    fn unsetenv(&mut self, name_op: OpTy<'tcx, Tag>) -> InterpResult<'tcx, i32> {
        let this = self.eval_context_mut();

        let name_ptr = this.read_scalar(name_op)?.not_undef()?;
        let mut success = None;
        if !this.is_null(name_ptr)? {
            let name = this.memory().read_c_str(name_ptr)?.to_owned();
            if !name.is_empty() && !name.contains(&b'=') {
                success = Some(this.machine.env_vars.map.remove(&name));
            }
        }
        if let Some(old) = success {
            if let Some(var) = old {
                this.memory_mut()
                    .deallocate(var, None, MiriMemoryKind::Env.into())?;
            }
            Ok(0)
        } else {
            Ok(-1)
        }
    }

    fn getcwd(
        &mut self,
        buf_op: OpTy<'tcx, Tag>,
        size_op: OpTy<'tcx, Tag>,
    ) -> InterpResult<'tcx, Scalar<Tag>> {
        let this = self.eval_context_mut();

        if !this.machine.communicate {
            throw_unsup_format!("Function not available when isolation is enabled")
        }

        let tcx = &{ this.tcx.tcx };

        let buf = this.force_ptr(this.read_scalar(buf_op)?.not_undef()?)?;
        let size = this.read_scalar(size_op)?.to_usize(&*this.tcx)?;
        // If we cannot get the current directory, we return null
        // FIXME: Technically we have to set the `errno` global too
        match env::current_dir() {
            Ok(cwd) => {
                // It is not clear what happens with non-utf8 paths here
                let mut bytes = cwd.display().to_string().into_bytes();
                // If the buffer is smaller or equal than the path, we return null.
                if (bytes.len() as u64) < size {
                    // We add a `/0` terminator
                    bytes.push(0);
                    // This is ok because the buffer is larger than the path with the null terminator.
                    this.memory_mut()
                        .get_mut(buf.alloc_id)?
                        .write_bytes(tcx, buf, &bytes)?;
                    return Ok(Scalar::Ptr(buf));
                }
                this.machine.last_error = this
                    .eval_path_scalar(&["libc", "ERANGE"])?
                    .unwrap()
                    .to_u32()?;
            }
            Err(e) => this.machine.last_error = e.raw_os_error().unwrap() as u32,
        }
        Ok(Scalar::ptr_null(&*this.tcx))
    }
}
<commit_msg>Fix comments<commit_after>use std::collections::HashMap;
use std::env;

use crate::stacked_borrows::Tag;
use crate::*;
use rustc::ty::layout::Size;
use rustc_mir::interpret::{Memory, Pointer};

#[derive(Default)]
pub struct EnvVars {
    /// Stores pointers to the environment variables. These variables must be stored as
    /// null-terminated C strings with the `"{name}={value}"` format.
    map: HashMap<Vec<u8>, Pointer<Tag>>,
}

impl EnvVars {
    pub(crate) fn init<'mir, 'tcx>(
        ecx: &mut InterpCx<'mir, 'tcx, Evaluator<'tcx>>,
        mut excluded_env_vars: Vec<String>,
    ) {
        // Exclude `TERM` var to avoid terminfo trying to open the termcap file.
        excluded_env_vars.push("TERM".to_owned());

        if ecx.machine.communicate {
            for (name, value) in env::vars() {
                if !excluded_env_vars.contains(&name) {
                    let var_ptr =
                        alloc_env_var(name.as_bytes(), value.as_bytes(), ecx.memory_mut());
                    ecx.machine.env_vars.map.insert(name.into_bytes(), var_ptr);
                }
            }
        }
    }
}

fn alloc_env_var<'mir, 'tcx>(
    name: &[u8],
    value: &[u8],
    memory: &mut Memory<'mir, 'tcx, Evaluator<'tcx>>,
) -> Pointer<Tag> {
    let mut bytes = name.to_vec();
    bytes.push(b'=');
    bytes.extend_from_slice(value);
    bytes.push(0);
    memory.allocate_static_bytes(bytes.as_slice(), MiriMemoryKind::Env.into())
}

impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
    fn getenv(&mut self, name_op: OpTy<'tcx, Tag>) -> InterpResult<'tcx, Scalar<Tag>> {
        let this = self.eval_context_mut();

        let name_ptr = this.read_scalar(name_op)?.not_undef()?;
        let name = this.memory().read_c_str(name_ptr)?;
        Ok(match this.machine.env_vars.map.get(name) {
            // The offset is used to strip the "{name}=" part of the string.
            Some(var_ptr) => {
                Scalar::Ptr(var_ptr.offset(Size::from_bytes(name.len() as u64 + 1), this)?)
            }
            None => Scalar::ptr_null(&*this.tcx),
        })
    }

    fn setenv(
        &mut self,
        name_op: OpTy<'tcx, Tag>,
        value_op: OpTy<'tcx, Tag>,
    ) -> InterpResult<'tcx, i32> {
        let this = self.eval_context_mut();

        let name_ptr = this.read_scalar(name_op)?.not_undef()?;
        let value_ptr = this.read_scalar(value_op)?.not_undef()?;
        let value = this.memory().read_c_str(value_ptr)?;
        let mut new = None;
        if !this.is_null(name_ptr)? {
            let name = this.memory().read_c_str(name_ptr)?;
            if !name.is_empty() && !name.contains(&b'=') {
                new = Some((name.to_owned(), value.to_owned()));
            }
        }
        if let Some((name, value)) = new {
            let var_ptr = alloc_env_var(&name, &value, this.memory_mut());
            if let Some(var) = this.machine.env_vars.map.insert(name.to_owned(), var_ptr) {
                this.memory_mut()
                    .deallocate(var, None, MiriMemoryKind::Env.into())?;
            }
            Ok(0)
        } else {
            Ok(-1)
        }
    }

    fn unsetenv(&mut self, name_op: OpTy<'tcx, Tag>) -> InterpResult<'tcx, i32> {
        let this = self.eval_context_mut();

        let name_ptr = this.read_scalar(name_op)?.not_undef()?;
        let mut success = None;
        if !this.is_null(name_ptr)? {
            let name = this.memory().read_c_str(name_ptr)?.to_owned();
            if !name.is_empty() && !name.contains(&b'=') {
                success = Some(this.machine.env_vars.map.remove(&name));
            }
        }
        if let Some(old) = success {
            if let Some(var) = old {
                this.memory_mut()
                    .deallocate(var, None, MiriMemoryKind::Env.into())?;
            }
            Ok(0)
        } else {
            Ok(-1)
        }
    }

    fn getcwd(
        &mut self,
        buf_op: OpTy<'tcx, Tag>,
        size_op: OpTy<'tcx, Tag>,
    ) -> InterpResult<'tcx, Scalar<Tag>> {
        let this = self.eval_context_mut();

        if !this.machine.communicate {
            throw_unsup_format!("`getcwd` not available when isolation is enabled")
        }

        let tcx = &{ this.tcx.tcx };

        let buf = this.force_ptr(this.read_scalar(buf_op)?.not_undef()?)?;
        let size = this.read_scalar(size_op)?.to_usize(&*this.tcx)?;
        // If we cannot get the current directory, we return null
        match env::current_dir() {
            Ok(cwd) => {
                // It is not clear what happens with non-utf8 paths here
                let mut bytes = cwd.display().to_string().into_bytes();
                // If the buffer is smaller or equal than the path, we return null.
                if (bytes.len() as u64) < size {
                    // We add a `/0` terminator
                    bytes.push(0);
                    // This is ok because the buffer is larger than the path with the null terminator.
                    this.memory_mut()
                        .get_mut(buf.alloc_id)?
                        .write_bytes(tcx, buf, &bytes)?;
                    return Ok(Scalar::Ptr(buf));
                }
                this.machine.last_error = this
                    .eval_path_scalar(&["libc", "ERANGE"])?
                    .unwrap()
                    .to_u32()?;
            }
            Err(e) => this.machine.last_error = e.raw_os_error().unwrap() as u32,
        }
        Ok(Scalar::ptr_null(&*this.tcx))
    }
}
<|endoftext|>


<commit_before>use position::Position;
use cppn::{Cppn, CppnNodeType};
use acyclic_network::NodeType;
use std::fmt::Debug;

/// Represents a node in the substrate. `T` stores  additional information about that node.

pub struct Node<P: Position, T: NodeType> {
    pub position: P,
    pub node_type: T,
}

pub struct Substrate<P: Position, T: NodeType> {
    nodes: Vec<Node<P, T>>,
}

pub struct LinkIterator<'a,
                        N: CppnNodeType + 'a,
                        L: Copy + Debug + Send + Sized + Into<f64> + 'a,
                        EXTID: Copy + Debug + Send + Sized + Ord + 'a,
                        P: Position + 'a,
                        T: NodeType + 'a>
{
    nodes: &'a [Node<P, T>],
    cppn: &'a mut Cppn<'a, N, L, EXTID>,
    inner: usize,
    outer: usize,
    max_distance: Option<f64>,
}

#[derive(Copy, Clone)]
pub struct Link<'a, P: Position + 'a, T: NodeType + 'a> {
    pub source: &'a Node<P, T>,
    pub target: &'a Node<P, T>,
    pub source_idx: usize,
    pub target_idx: usize,
    pub weight: f64,
    pub distance: f64,
}

impl<'a, N: CppnNodeType + 'a,
   L: Copy + Debug + Send + Sized + Into<f64> + 'a,
    EXTID: Copy + Debug + Send + Sized + Ord + 'a,
P: Position + 'a, T: NodeType + 'a> Iterator for LinkIterator<'a, N, L, EXTID, P, T> {
    type Item = Link<'a, P, T>;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if self.inner >= self.nodes.len() {
                self.inner = 0;
                self.outer += 1;
            }
            if self.outer >= self.nodes.len() {
                return None;
            }
            if self.inner == self.outer {
                self.inner += 1;
                continue;
            }

            assert!(self.inner < self.nodes.len() && self.outer < self.nodes.len() &&
                    self.inner != self.outer);
            let source = &self.nodes[self.inner];
            let target = &self.nodes[self.outer];

// Reject invalid connections.

            if !source.node_type.accept_outgoing_links() || !target.node_type.accept_incoming_links() {
                    self.inner += 1;
                    continue;
            }

            let distance = source.position.distance(&target.position);

// reject a pair of nodes based on `max_distance`.

            if let Some(max_d) = self.max_distance {
                if distance > max_d {
                    self.inner += 1;
                    continue;
                }
            }

// Calculate the weight between source and target using the CPPN.

            let distance_between: &[_] = &[distance];
            let inputs_to_cppn = [source.position.coords(), target.position.coords(), distance_between];
            let outputs_from_cppn = self.cppn.calculate(&inputs_to_cppn);
            assert!(outputs_from_cppn.len() == 1);
            let weight = outputs_from_cppn[0];

            let link = Link {
                source: source,
                target: target,
                source_idx: self.inner,
                target_idx: self.outer,
                weight: weight,
                distance: distance,
            };
            self.inner += 1;
            return Some(link);
        }
    }
}

impl<P: Position, T: NodeType> Substrate<P, T> {
    pub fn new() -> Substrate<P, T> {
        Substrate { nodes: Vec::new() }
    }

    pub fn nodes(&self) -> &[Node<P, T>] {
        &self.nodes
    }

    pub fn add_node(&mut self, position: P, node_type: T) {
        self.nodes.push(Node {
            position: position,
            node_type: node_type,
        });
    }

    /// Iterate over all produced links of Cppn.
    pub fn iter_links<'a, N, L, EXTID>(&'a self,
                                       cppn: &'a mut Cppn<'a, N, L, EXTID>,
                                       max_distance: Option<f64>)
                                       -> LinkIterator<'a, N, L, EXTID, P, T>
        where N: CppnNodeType,
              L: Copy + Debug + Send + Sized + Into<f64> + 'a,
              EXTID: Copy + Debug + Send + Sized + Ord + 'a
    {
        LinkIterator {
            nodes: &self.nodes,
            cppn: cppn,
            inner: 0,
            outer: 0,
            max_distance: max_distance,
        }
    }
}
<commit_msg>No functional changes<commit_after>use position::Position;
use cppn::{Cppn, CppnNodeType};
use acyclic_network::NodeType;
use std::fmt::Debug;

/// Represents a node in the substrate. `T` stores additional information about that node.

pub struct Node<P, T>
    where P: Position,
          T: NodeType
{
    pub position: P,
    pub node_type: T,
}

pub struct Substrate<P, T>
    where P: Position,
          T: NodeType
{
    nodes: Vec<Node<P, T>>,
}

pub struct LinkIterator<'a,
                        N: CppnNodeType + 'a,
                        L: Copy + Debug + Send + Sized + Into<f64> + 'a,
                        EXTID: Copy + Debug + Send + Sized + Ord + 'a,
                        P: Position + 'a,
                        T: NodeType + 'a>
{
    nodes: &'a [Node<P, T>],
    cppn: &'a mut Cppn<'a, N, L, EXTID>,
    inner: usize,
    outer: usize,
    max_distance: Option<f64>,
}

#[derive(Copy, Clone)]
pub struct Link<'a, P, T>
    where P: Position + 'a,
          T: NodeType + 'a
{
    pub source: &'a Node<P, T>,
    pub target: &'a Node<P, T>,
    pub source_idx: usize,
    pub target_idx: usize,
    pub weight: f64,
    pub distance: f64,
}

impl<'a, N: CppnNodeType + 'a,
   L: Copy + Debug + Send + Sized + Into<f64> + 'a,
    EXTID: Copy + Debug + Send + Sized + Ord + 'a,
P: Position + 'a, T: NodeType + 'a> Iterator for LinkIterator<'a, N, L, EXTID, P, T> {
    type Item = Link<'a, P, T>;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if self.inner >= self.nodes.len() {
                self.inner = 0;
                self.outer += 1;
            }
            if self.outer >= self.nodes.len() {
                return None;
            }
            if self.inner == self.outer {
                self.inner += 1;
                continue;
            }

            assert!(self.inner < self.nodes.len() && self.outer < self.nodes.len() &&
                    self.inner != self.outer);
            let source = &self.nodes[self.inner];
            let target = &self.nodes[self.outer];

// Reject invalid connections.

            if !source.node_type.accept_outgoing_links() || !target.node_type.accept_incoming_links() {
                    self.inner += 1;
                    continue;
            }

            let distance = source.position.distance(&target.position);

// reject a pair of nodes based on `max_distance`.

            if let Some(max_d) = self.max_distance {
                if distance > max_d {
                    self.inner += 1;
                    continue;
                }
            }

// Calculate the weight between source and target using the CPPN.

            let distance_between: &[_] = &[distance];
            let inputs_to_cppn = [source.position.coords(), target.position.coords(), distance_between];
            let outputs_from_cppn = self.cppn.calculate(&inputs_to_cppn);
            assert!(outputs_from_cppn.len() == 1);
            let weight = outputs_from_cppn[0];

            let link = Link {
                source: source,
                target: target,
                source_idx: self.inner,
                target_idx: self.outer,
                weight: weight,
                distance: distance,
            };
            self.inner += 1;
            return Some(link);
        }
    }
}

impl<P, T> Substrate<P, T>
    where P: Position,
          T: NodeType
{
    pub fn new() -> Self {
        Substrate { nodes: Vec::new() }
    }

    pub fn nodes(&self) -> &[Node<P, T>] {
        &self.nodes
    }

    pub fn add_node(&mut self, position: P, node_type: T) {
        self.nodes.push(Node {
            position: position,
            node_type: node_type,
        });
    }

    /// Iterate over all produced links of Cppn.
    pub fn iter_links<'a, N, L, EXTID>(&'a self,
                                       cppn: &'a mut Cppn<'a, N, L, EXTID>,
                                       max_distance: Option<f64>)
                                       -> LinkIterator<'a, N, L, EXTID, P, T>
        where N: CppnNodeType,
              L: Copy + Debug + Send + Sized + Into<f64> + 'a,
              EXTID: Copy + Debug + Send + Sized + Ord + 'a
    {
        LinkIterator {
            nodes: &self.nodes,
            cppn: cppn,
            inner: 0,
            outer: 0,
            max_distance: max_distance,
        }
    }
}
<|endoftext|>


<commit_before>use super::Tokens;

use std::borrow::Cow;

use proc_macro2::{Literal, Span, Term, TokenStream, TokenTree};

/// Types that can be interpolated inside a [`quote!`] invocation.
///
/// [`quote!`]: macro.quote.html
pub trait ToTokens {
    /// Write `self` to the given `Tokens`.
    ///
    /// Example implementation for a struct representing Rust paths like
    /// `std::cmp::PartialEq`:
    ///
    /// ```
    /// extern crate quote;
    /// use quote::{Tokens, ToTokens};
    ///
    /// extern crate proc_macro2;
    /// use proc_macro2::{TokenTree, Spacing, Span, Op};
    ///
    /// pub struct Path {
    ///     pub global: bool,
    ///     pub segments: Vec<PathSegment>,
    /// }
    ///
    /// impl ToTokens for Path {
    ///     fn to_tokens(&self, tokens: &mut Tokens) {
    ///         for (i, segment) in self.segments.iter().enumerate() {
    ///             if i > 0 || self.global {
    ///                 // Double colon `::`
    ///                 tokens.append(Op::new(':', Spacing::Joint));
    ///                 tokens.append(Op::new(':', Spacing::Alone));
    ///             }
    ///             segment.to_tokens(tokens);
    ///         }
    ///     }
    /// }
    /// #
    /// # pub struct PathSegment;
    /// #
    /// # impl ToTokens for PathSegment {
    /// #     fn to_tokens(&self, tokens: &mut Tokens) {
    /// #         unimplemented!()
    /// #     }
    /// # }
    /// #
    /// # fn main() {}
    /// ```
    fn to_tokens(&self, tokens: &mut Tokens);

    /// Convert `self` directly into a `Tokens` object.
    ///
    /// This method is implicitly implemented using `to_tokens`, and acts as a
    /// convenience method for consumers of the `ToTokens` trait.
    fn into_tokens(self) -> Tokens
    where
        Self: Sized,
    {
        let mut tokens = Tokens::new();
        self.to_tokens(&mut tokens);
        tokens
    }
}

impl<'a, T: ?Sized + ToTokens> ToTokens for &'a T {
    fn to_tokens(&self, tokens: &mut Tokens) {
        (**self).to_tokens(tokens);
    }
}

impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ?Sized + ToTokens> ToTokens for Box<T> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ToTokens> ToTokens for Option<T> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        if let Some(ref t) = *self {
            t.to_tokens(tokens);
        }
    }
}

impl ToTokens for str {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(Literal::string(self));
    }
}

impl ToTokens for String {
    fn to_tokens(&self, tokens: &mut Tokens) {
        self.as_str().to_tokens(tokens);
    }
}

macro_rules! primitive {
    ($($t:ident => $name:ident)*) => ($(
        impl ToTokens for $t {
            fn to_tokens(&self, tokens: &mut Tokens) {
                tokens.append(Literal::$name(*self));
            }
        }
    )*)
}

primitive! {
    i8 => i8_suffixed
    i16 => i16_suffixed
    i32 => i32_suffixed
    i64 => i64_suffixed
    isize => isize_suffixed

    u8 => u8_suffixed
    u16 => u16_suffixed
    u32 => u32_suffixed
    u64 => u64_suffixed
    usize => usize_suffixed

    f32 => f32_suffixed
    f64 => f64_suffixed
}

impl ToTokens for char {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(Literal::character(*self));
    }
}

impl ToTokens for bool {
    fn to_tokens(&self, tokens: &mut Tokens) {
        let word = if *self { "true" } else { "false" };
        tokens.append(Term::new(word, Span::call_site()));
    }
}

impl ToTokens for Term {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Literal {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.clone());
    }
}

impl ToTokens for TokenTree {
    fn to_tokens(&self, dst: &mut Tokens) {
        dst.append(self.clone());
    }
}

impl ToTokens for TokenStream {
    fn to_tokens(&self, dst: &mut Tokens) {
        dst.append_all(self.clone().into_iter());
    }
}
<commit_msg>Implement ToTokens for proc_macro2::Op, Group<commit_after>use super::Tokens;

use std::borrow::Cow;

use proc_macro2::{Group, Literal, Op, Span, Term, TokenStream, TokenTree};

/// Types that can be interpolated inside a [`quote!`] invocation.
///
/// [`quote!`]: macro.quote.html
pub trait ToTokens {
    /// Write `self` to the given `Tokens`.
    ///
    /// Example implementation for a struct representing Rust paths like
    /// `std::cmp::PartialEq`:
    ///
    /// ```
    /// extern crate quote;
    /// use quote::{Tokens, ToTokens};
    ///
    /// extern crate proc_macro2;
    /// use proc_macro2::{TokenTree, Spacing, Span, Op};
    ///
    /// pub struct Path {
    ///     pub global: bool,
    ///     pub segments: Vec<PathSegment>,
    /// }
    ///
    /// impl ToTokens for Path {
    ///     fn to_tokens(&self, tokens: &mut Tokens) {
    ///         for (i, segment) in self.segments.iter().enumerate() {
    ///             if i > 0 || self.global {
    ///                 // Double colon `::`
    ///                 tokens.append(Op::new(':', Spacing::Joint));
    ///                 tokens.append(Op::new(':', Spacing::Alone));
    ///             }
    ///             segment.to_tokens(tokens);
    ///         }
    ///     }
    /// }
    /// #
    /// # pub struct PathSegment;
    /// #
    /// # impl ToTokens for PathSegment {
    /// #     fn to_tokens(&self, tokens: &mut Tokens) {
    /// #         unimplemented!()
    /// #     }
    /// # }
    /// #
    /// # fn main() {}
    /// ```
    fn to_tokens(&self, tokens: &mut Tokens);

    /// Convert `self` directly into a `Tokens` object.
    ///
    /// This method is implicitly implemented using `to_tokens`, and acts as a
    /// convenience method for consumers of the `ToTokens` trait.
    fn into_tokens(self) -> Tokens
    where
        Self: Sized,
    {
        let mut tokens = Tokens::new();
        self.to_tokens(&mut tokens);
        tokens
    }
}

impl<'a, T: ?Sized + ToTokens> ToTokens for &'a T {
    fn to_tokens(&self, tokens: &mut Tokens) {
        (**self).to_tokens(tokens);
    }
}

impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ?Sized + ToTokens> ToTokens for Box<T> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ToTokens> ToTokens for Option<T> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        if let Some(ref t) = *self {
            t.to_tokens(tokens);
        }
    }
}

impl ToTokens for str {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(Literal::string(self));
    }
}

impl ToTokens for String {
    fn to_tokens(&self, tokens: &mut Tokens) {
        self.as_str().to_tokens(tokens);
    }
}

macro_rules! primitive {
    ($($t:ident => $name:ident)*) => ($(
        impl ToTokens for $t {
            fn to_tokens(&self, tokens: &mut Tokens) {
                tokens.append(Literal::$name(*self));
            }
        }
    )*)
}

primitive! {
    i8 => i8_suffixed
    i16 => i16_suffixed
    i32 => i32_suffixed
    i64 => i64_suffixed
    isize => isize_suffixed

    u8 => u8_suffixed
    u16 => u16_suffixed
    u32 => u32_suffixed
    u64 => u64_suffixed
    usize => usize_suffixed

    f32 => f32_suffixed
    f64 => f64_suffixed
}

impl ToTokens for char {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(Literal::character(*self));
    }
}

impl ToTokens for bool {
    fn to_tokens(&self, tokens: &mut Tokens) {
        let word = if *self { "true" } else { "false" };
        tokens.append(Term::new(word, Span::call_site()));
    }
}

impl ToTokens for Group {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Term {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Op {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Literal {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.clone());
    }
}

impl ToTokens for TokenTree {
    fn to_tokens(&self, dst: &mut Tokens) {
        dst.append(self.clone());
    }
}

impl ToTokens for TokenStream {
    fn to_tokens(&self, dst: &mut Tokens) {
        dst.append_all(self.clone().into_iter());
    }
}
<|endoftext|>


<commit_before>//! Utility module for lexer implementations,
//! providing types to help manage states and tokens.

use std::str::Chars;
use super::token::Token;
use super::token::Category;

/// A recursive function type used by lexers to manage their state.
/// Based on Rob Pike's "Lexical Scanning in Go" talk, these functions are
/// invoked in a call/return loop (letting the current function determine
/// the next) until a `None` value is returned, after which lexing is complete.
///
/// See the `lexers` module for examples.
pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store tokens for lexers.
pub struct Tokenizer<'a> {
    data: Chars<'a>,
    current_token: String,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let tokenizer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.chars(),
      current_token: String::new(),
      tokens: vec![],
      states: vec![]
    }
}

impl<'a> Tokenizer<'a> {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// tokenizer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Ensure that we're at the first character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the first character.
    /// tokenizer.advance();
    ///
    /// // Ensure that we're at the next character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        match self.data.next() {
            Some(c) => self.current_token.push(c),
            None => ()
        }
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("l");
    ///
    /// // Ensure that the current character is correct.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the last bit of data.
    /// tokenizer.advance();
    ///
    /// // Ensure that there is no current character.
    /// assert_eq!(tokenizer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        match self.data.clone().peekable().peek() {
            Some(c) => Some(c.clone()),
            None => None
        }
    }

    /// Whether or not the remaining data starts with the specified prefix.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("lex");
    ///
    /// assert!(tokenizer.has_prefix("le"));
    /// ```
    pub fn has_prefix(&self, prefix: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let mut data_iter = self.data.clone();

        // Check that the subject is a prefix, character by character.
        // This is much faster than building a string of equal length from
        // self.data and deferring to a straight string comparison using ==.
        prefix.chars().all(|c| {
            match data_iter.next() {
                Some(d) => c == d,
                None => false
            }
        })
    }

    /// Whether or not the remaining data starts with the specified lexeme.
    /// Ensures that the specified lexeme is not just a prefix by checking
    /// that the data that follows it is a newline, space, or nothing at all.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("lex\nluthor lib");
    ///
    /// // Prefixes don't count.
    /// assert!(!tokenizer.starts_with_lexeme("le"));
    ///
    /// // Newlines delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lex"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // Spaces delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("luthor"));
    ///
    /// // Consume 7 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(7, Category::Text);
    ///
    /// // End of string delineates lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lib"));
    /// ```
    pub fn starts_with_lexeme(&self, lexeme: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let data_iter = self.data.clone();

        self.has_prefix(lexeme) && match data_iter.skip(lexeme.len()).next() {
            Some(' ') | Some('\n') => true,
            None => true,
            _ => false
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume two characters and then tokenize them.
    /// tokenizer.advance();
    /// tokenizer.advance();
    /// tokenizer.tokenize(Category::Text);
    ///
    /// // Ensure that we have a correctly-categorized token.
    /// assert_eq!(tokenizer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if !self.current_token.is_empty() {
            let token = Token{
                lexeme: self.current_token.clone(),
                category: category,
            };
            self.tokens.push(token);
            self.current_token = String::new();
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// `Category::Text` category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume one character, and then tokenize the next 5.
    /// tokenizer.advance();
    /// tokenizer.tokenize_next(5, Category::Keyword);
    ///
    /// // Ensure that we have two properly-categorized tokens.
    /// assert_eq!(
    ///     tokenizer.tokens()[0],
    ///     Token{ lexeme: "l".to_string(), category: Category::Text }
    /// );
    /// assert_eq!(
    ///     tokenizer.tokens()[1],
    ///     Token{ lexeme: "uthor".to_string(), category: Category::Keyword }
    /// );
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        // If there's any data that has yet
        // to be tokenized, take care of that.
        self.tokenize(Category::Text);

        // Mark the next series of characters.
        for _ in 0..amount { self.advance(); }

        // Tokenize the marked characters.
        self.tokenize(category);
    }
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    #[test]
    fn current_char_returns_the_char_at_head() {
        let data = "él";
        let tokenizer = new(data);

        assert_eq!(tokenizer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let data = "él";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();

        assert_eq!(tokenizer.current_char(), None);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize(Category::Text);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.tokenize(Category::Text);

        assert_eq!(tokenizer.tokens.len(), 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(1, Category::Keyword);

        let token = tokenizer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(5, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(15, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }
}
<commit_msg>Allow commas to delineate lexemes.<commit_after>//! Utility module for lexer implementations,
//! providing types to help manage states and tokens.

use std::str::Chars;
use super::token::Token;
use super::token::Category;

/// A recursive function type used by lexers to manage their state.
/// Based on Rob Pike's "Lexical Scanning in Go" talk, these functions are
/// invoked in a call/return loop (letting the current function determine
/// the next) until a `None` value is returned, after which lexing is complete.
///
/// See the `lexers` module for examples.
pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store tokens for lexers.
pub struct Tokenizer<'a> {
    data: Chars<'a>,
    current_token: String,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let tokenizer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.chars(),
      current_token: String::new(),
      tokens: vec![],
      states: vec![]
    }
}

impl<'a> Tokenizer<'a> {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// tokenizer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Ensure that we're at the first character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the first character.
    /// tokenizer.advance();
    ///
    /// // Ensure that we're at the next character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        match self.data.next() {
            Some(c) => self.current_token.push(c),
            None => ()
        }
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("l");
    ///
    /// // Ensure that the current character is correct.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the last bit of data.
    /// tokenizer.advance();
    ///
    /// // Ensure that there is no current character.
    /// assert_eq!(tokenizer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        match self.data.clone().peekable().peek() {
            Some(c) => Some(c.clone()),
            None => None
        }
    }

    /// Whether or not the remaining data starts with the specified prefix.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("lex");
    ///
    /// assert!(tokenizer.has_prefix("le"));
    /// ```
    pub fn has_prefix(&self, prefix: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let mut data_iter = self.data.clone();

        // Check that the subject is a prefix, character by character.
        // This is much faster than building a string of equal length from
        // self.data and deferring to a straight string comparison using ==.
        prefix.chars().all(|c| {
            match data_iter.next() {
                Some(d) => c == d,
                None => false
            }
        })
    }

    /// Whether or not the remaining data starts with the specified lexeme.
    /// Ensures that the specified lexeme is not just a prefix by checking that
    /// the data that follows it is a newline, space, comma, or nothing at all.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("lex\nluthor lib,rary");
    ///
    /// // Prefixes don't count.
    /// assert!(!tokenizer.starts_with_lexeme("le"));
    ///
    /// // Newlines delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lex"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // Spaces delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("luthor"));
    ///
    /// // Consume 7 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(7, Category::Text);
    ///
    /// // Commas delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lib"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // End of string delineates lexemes.
    /// assert!(tokenizer.starts_with_lexeme("rary"));
    /// ```
    pub fn starts_with_lexeme(&self, lexeme: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let data_iter = self.data.clone();

        self.has_prefix(lexeme) && match data_iter.skip(lexeme.len()).next() {
            Some(' ') | Some('\n') | Some(',') => true,
            None => true,
            _ => false
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume two characters and then tokenize them.
    /// tokenizer.advance();
    /// tokenizer.advance();
    /// tokenizer.tokenize(Category::Text);
    ///
    /// // Ensure that we have a correctly-categorized token.
    /// assert_eq!(tokenizer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if !self.current_token.is_empty() {
            let token = Token{
                lexeme: self.current_token.clone(),
                category: category,
            };
            self.tokens.push(token);
            self.current_token = String::new();
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// `Category::Text` category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume one character, and then tokenize the next 5.
    /// tokenizer.advance();
    /// tokenizer.tokenize_next(5, Category::Keyword);
    ///
    /// // Ensure that we have two properly-categorized tokens.
    /// assert_eq!(
    ///     tokenizer.tokens()[0],
    ///     Token{ lexeme: "l".to_string(), category: Category::Text }
    /// );
    /// assert_eq!(
    ///     tokenizer.tokens()[1],
    ///     Token{ lexeme: "uthor".to_string(), category: Category::Keyword }
    /// );
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        // If there's any data that has yet
        // to be tokenized, take care of that.
        self.tokenize(Category::Text);

        // Mark the next series of characters.
        for _ in 0..amount { self.advance(); }

        // Tokenize the marked characters.
        self.tokenize(category);
    }
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    #[test]
    fn current_char_returns_the_char_at_head() {
        let data = "él";
        let tokenizer = new(data);

        assert_eq!(tokenizer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let data = "él";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();

        assert_eq!(tokenizer.current_char(), None);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize(Category::Text);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.tokenize(Category::Text);

        assert_eq!(tokenizer.tokens.len(), 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(1, Category::Keyword);

        let token = tokenizer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(5, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(15, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! write docs here

use std::comm::channel;

/// write docs here
pub type WorkUnit<ARG, RV> = (ARG, Sender<RV>);

/// write docs here
pub enum MessageToWorker<ARG, RV> {
    /// A work unit.
    Work(WorkUnit<ARG, RV>),
    /// Shut down the worker task.
    Halt,
}

/// write docs here
pub enum MessageToDispatcher<ARG, RV> {
    /// A work unit that needs to be dispatched to a worker task.
    Dispatch(WorkUnit<ARG, RV>),
    /// Halt all worker tasks.
    HaltAll,
    /// Register a new worker task.
    RegisterWorker(Sender<Sender<Sender<MessageToWorker<ARG, RV>>>>),
}

/// A queue that distributes work items to worker tasks.
pub struct WorkQueue<ARG, RV> {
    /// write docs here
    dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

/// A proxy to a `WorkQueue`. It can be freely cloned to use from multiple tasks.
pub struct Dispatcher<ARG, RV> {
    /// write docs here
    inbox: Receiver<MessageToDispatcher<ARG, RV>>,
}

/// A proxy to a `WorkQueue`. It can be freely cloned to use from multiple tasks.
pub struct WorkQueueProxy<ARG, RV> {
    /// write docs here
    dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

/// A worker that executes tasks from its parent queue.
pub struct Worker<ARG, RV> {
    priv ask_for_work: Sender<Sender<MessageToWorker<ARG, RV>>>,
}

/// Create a new work queue.
pub fn WorkQueue<ARG:Send, RV:Send>() -> (WorkQueue<ARG, RV>, Dispatcher<ARG, RV>) {
    let (dispatcher, dispatcher_inbox) = channel::<MessageToDispatcher<ARG, RV>>();
    return (
        WorkQueue{dispatcher: dispatcher},
        Dispatcher{inbox: dispatcher_inbox},
    );
}

impl<ARG:Send, RV:Send> WorkQueue<ARG, RV> {
    /// Create a copyable proxy that can be used to push work units.
    pub fn proxy(&self) -> WorkQueueProxy<ARG, RV> {
        return WorkQueueProxy{dispatcher: self.dispatcher.clone()};
    }

    /// Register a new worker. It will receive a sender where it can ask for tasks.
    pub fn register_worker(&self) -> Sender<Sender<MessageToWorker<ARG, RV>>> {
        let (reg_s, reg_r) = channel::<Sender<Sender<MessageToWorker<ARG, RV>>>>();
        self.dispatcher.send(RegisterWorker(reg_s));
        return reg_r.recv();
    }

    /// Create a new worker.
    pub fn worker(&self) -> Worker<ARG, RV> {
        return Worker{ask_for_work: self.register_worker()};
    }

    /// Push a work item to this queue.
    pub fn push(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch((arg, rv)));
        return wait_for_rv;
    }
}

// rustc complais "cannot implement a destructor on a structure with
// type parameters", but our destruction is safe, we only send
// a simple message on a channel.
#[unsafe_destructor]
impl<ARG:Send, RV:Send> Drop for WorkQueue<ARG, RV> {
    fn drop(&mut self) {
        self.dispatcher.send(HaltAll);
    }
}

impl<ARG:Send, RV:Send> Dispatcher<ARG, RV> {
    /// Run the dispatcher loop. It will stop when the parent WorkQueue
    /// object is dropped.
    pub fn run(&self) {
        let (want_work, idle_worker) = channel::<Sender<MessageToWorker<ARG, RV>>>();
        let mut worker_count = 0;
        let idle_worker = idle_worker;
        loop {
            match self.inbox.recv() {
                Dispatch(work_item) => {
                    idle_worker.recv().send(Work(work_item));
                },
                RegisterWorker(want_idle_sender) => {
                    worker_count += 1;
                    want_idle_sender.send(want_work.clone());
                }
                HaltAll => {
                    while worker_count > 0 {
                        idle_worker.recv().send(Halt);
                        worker_count -= 1;
                    }
                    return;
                },
            };
        }
    }
}

impl<ARG:Send, RV:Send> WorkQueueProxy<ARG, RV> {
    /// Push a work item to this queue.
    pub fn push(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch((arg, rv)));
        return wait_for_rv;
    }
}

impl<ARG:Send, RV:Send> Clone for WorkQueueProxy<ARG, RV> {
    fn clone(&self) -> WorkQueueProxy<ARG, RV> {
        return WorkQueueProxy{dispatcher: self.dispatcher.clone()};
    }
}

impl<ARG:Send, RV:Send> Worker<ARG, RV> {
    pub fn run(&self, fun: |arg: ARG| -> RV) {
        loop {
            let (idle, work_unit) = channel::<MessageToWorker<ARG, RV>>();
            self.ask_for_work.send(idle);
            match work_unit.recv() {
                Work((arg, rv)) => rv.send((fun)(arg)),
                Halt            => return
            };
        }
    }
}

#[cfg(test)]
mod test {
    use std::task::spawn;
    use super::WorkQueue;

    #[test]
    fn test_queue() {
        let (queue, dispatcher) = WorkQueue::<int, int>();
        spawn(proc() { dispatcher.run() });
        for _ in range(0, 3) {
            let worker = queue.worker();
            spawn(proc() { worker.run(|arg| arg * 2); });
        }

        let return_list: ~[int] =
            range(0, 10)
            .map(|c| queue.push(c))
            .map(|rv| rv.recv())
            .collect();

        assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
    }

    #[test]
    fn test_enqueue_from_tasks() {
        let (queue, dispatcher) = WorkQueue::<int, int>();
        spawn(proc() { dispatcher.run() });
        for _ in range(0, 3) {
            let worker = queue.worker();
            spawn(proc() { worker.run(|arg| arg * 2); });
        }
        let mut promise_list: ~[Receiver<int>] = ~[];
        let queue_proxy = queue.proxy();
        for c in range(0, 10) {
            let queue_proxy_clone = queue_proxy.clone();
            let (done, promise) = channel::<int>();
            promise_list.push(promise);
            spawn(proc() {
                let done = done;
                let queue = queue_proxy_clone;
                let rv = queue.push(c);
                done.send(rv.recv());
            });
        }

        let return_list: ~[int] =
            promise_list
            .iter()
            .map(|promise| promise.recv())
            .collect();
        assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
    }
}

#[cfg(test)]
mod bench {
    extern crate test;

    use self::test::BenchHarness;
    use super::WorkQueue;

    #[bench]
    fn bench_50_tasks_4_threads(b: &mut BenchHarness) {
        let (queue, dispatcher) = WorkQueue::<int, int>();
        spawn(proc() { dispatcher.run() });
        for _ in range(0, 4) {
            let worker = queue.worker();
            spawn(proc() { worker.run(|arg| arg * 2); });
        }
        b.iter(|| {
            let _: ~[int] =
                range(0, 50)
                .map(|_| queue.push(1))
                .map(|rv| rv.recv())
                .collect();
        });
    }

    #[bench]
    fn bench_spawn_5_workers(b: &mut BenchHarness) {
        b.iter(|| {
            let (queue, dispatcher) = WorkQueue::<int, int>();
            spawn(proc() { dispatcher.run() });
            for _ in range(0, 5) {
                let worker = queue.worker();
                spawn(proc() { worker.run(|arg| arg * 2); });
            }
        });
    }
}
<commit_msg>fewer things need to be public<commit_after>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! write docs here

use std::comm::channel;

type WorkUnit<ARG, RV> = (ARG, Sender<RV>);

enum MessageToWorker<ARG, RV> {
    Work(WorkUnit<ARG, RV>),
    Halt,
}

enum MessageToDispatcher<ARG, RV> {
    Dispatch(WorkUnit<ARG, RV>),
    HaltAll,
    RegisterWorker(Sender<Sender<Sender<MessageToWorker<ARG, RV>>>>),
}

/// A queue that distributes work items to worker tasks.
pub struct WorkQueue<ARG, RV> {
    priv dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

/// A proxy to a `WorkQueue`. It can be freely cloned to use from multiple tasks.
pub struct Dispatcher<ARG, RV> {
    priv inbox: Receiver<MessageToDispatcher<ARG, RV>>,
}

/// A proxy to a `WorkQueue`. It can be freely cloned to use from multiple tasks.
pub struct WorkQueueProxy<ARG, RV> {
    priv dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

/// A worker that executes tasks from its parent queue.
pub struct Worker<ARG, RV> {
    priv ask_for_work: Sender<Sender<MessageToWorker<ARG, RV>>>,
}

/// Create a new work queue.
pub fn WorkQueue<ARG:Send, RV:Send>() -> (WorkQueue<ARG, RV>, Dispatcher<ARG, RV>) {
    let (dispatcher, dispatcher_inbox) = channel::<MessageToDispatcher<ARG, RV>>();
    return (
        WorkQueue{dispatcher: dispatcher},
        Dispatcher{inbox: dispatcher_inbox},
    );
}

impl<ARG:Send, RV:Send> WorkQueue<ARG, RV> {
    /// Create a copyable proxy that can be used to push work units.
    pub fn proxy(&self) -> WorkQueueProxy<ARG, RV> {
        return WorkQueueProxy{dispatcher: self.dispatcher.clone()};
    }

    /// Register a new worker. It will receive a sender where it can ask for tasks.
    fn register_worker(&self) -> Sender<Sender<MessageToWorker<ARG, RV>>> {
        let (reg_s, reg_r) = channel::<Sender<Sender<MessageToWorker<ARG, RV>>>>();
        self.dispatcher.send(RegisterWorker(reg_s));
        return reg_r.recv();
    }

    /// Create a new worker.
    pub fn worker(&self) -> Worker<ARG, RV> {
        return Worker{ask_for_work: self.register_worker()};
    }

    /// Push a work item to this queue.
    pub fn push(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch((arg, rv)));
        return wait_for_rv;
    }
}

// rustc complais "cannot implement a destructor on a structure with
// type parameters", but our destruction is safe, we only send
// a simple message on a channel.
#[unsafe_destructor]
impl<ARG:Send, RV:Send> Drop for WorkQueue<ARG, RV> {
    fn drop(&mut self) {
        self.dispatcher.send(HaltAll);
    }
}

impl<ARG:Send, RV:Send> Dispatcher<ARG, RV> {
    /// Run the dispatcher loop. It will stop when the parent WorkQueue
    /// object is dropped.
    pub fn run(&self) {
        let (want_work, idle_worker) = channel::<Sender<MessageToWorker<ARG, RV>>>();
        let mut worker_count = 0;
        let idle_worker = idle_worker;
        loop {
            match self.inbox.recv() {
                Dispatch(work_item) => {
                    idle_worker.recv().send(Work(work_item));
                },
                RegisterWorker(want_idle_sender) => {
                    worker_count += 1;
                    want_idle_sender.send(want_work.clone());
                }
                HaltAll => {
                    while worker_count > 0 {
                        idle_worker.recv().send(Halt);
                        worker_count -= 1;
                    }
                    return;
                },
            };
        }
    }
}

impl<ARG:Send, RV:Send> WorkQueueProxy<ARG, RV> {
    /// Push a work item to this queue.
    pub fn push(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch((arg, rv)));
        return wait_for_rv;
    }
}

impl<ARG:Send, RV:Send> Clone for WorkQueueProxy<ARG, RV> {
    fn clone(&self) -> WorkQueueProxy<ARG, RV> {
        return WorkQueueProxy{dispatcher: self.dispatcher.clone()};
    }
}

impl<ARG:Send, RV:Send> Worker<ARG, RV> {
    pub fn run(&self, fun: |arg: ARG| -> RV) {
        loop {
            let (idle, work_unit) = channel::<MessageToWorker<ARG, RV>>();
            self.ask_for_work.send(idle);
            match work_unit.recv() {
                Work((arg, rv)) => rv.send((fun)(arg)),
                Halt            => return
            };
        }
    }
}

#[cfg(test)]
mod test {
    use std::task::spawn;
    use super::WorkQueue;

    #[test]
    fn test_queue() {
        let (queue, dispatcher) = WorkQueue::<int, int>();
        spawn(proc() { dispatcher.run() });
        for _ in range(0, 3) {
            let worker = queue.worker();
            spawn(proc() { worker.run(|arg| arg * 2); });
        }

        let return_list: ~[int] =
            range(0, 10)
            .map(|c| queue.push(c))
            .map(|rv| rv.recv())
            .collect();

        assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
    }

    #[test]
    fn test_enqueue_from_tasks() {
        let (queue, dispatcher) = WorkQueue::<int, int>();
        spawn(proc() { dispatcher.run() });
        for _ in range(0, 3) {
            let worker = queue.worker();
            spawn(proc() { worker.run(|arg| arg * 2); });
        }
        let mut promise_list: ~[Receiver<int>] = ~[];
        let queue_proxy = queue.proxy();
        for c in range(0, 10) {
            let queue_proxy_clone = queue_proxy.clone();
            let (done, promise) = channel::<int>();
            promise_list.push(promise);
            spawn(proc() {
                let done = done;
                let queue = queue_proxy_clone;
                let rv = queue.push(c);
                done.send(rv.recv());
            });
        }

        let return_list: ~[int] =
            promise_list
            .iter()
            .map(|promise| promise.recv())
            .collect();
        assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
    }
}

#[cfg(test)]
mod bench {
    extern crate test;

    use self::test::BenchHarness;
    use super::WorkQueue;

    #[bench]
    fn bench_50_tasks_4_threads(b: &mut BenchHarness) {
        let (queue, dispatcher) = WorkQueue::<int, int>();
        spawn(proc() { dispatcher.run() });
        for _ in range(0, 4) {
            let worker = queue.worker();
            spawn(proc() { worker.run(|arg| arg * 2); });
        }
        b.iter(|| {
            let _: ~[int] =
                range(0, 50)
                .map(|_| queue.push(1))
                .map(|rv| rv.recv())
                .collect();
        });
    }

    #[bench]
    fn bench_spawn_5_workers(b: &mut BenchHarness) {
        b.iter(|| {
            let (queue, dispatcher) = WorkQueue::<int, int>();
            spawn(proc() { dispatcher.run() });
            for _ in range(0, 5) {
                let worker = queue.worker();
                spawn(proc() { worker.run(|arg| arg * 2); });
            }
        });
    }
}
<|endoftext|>