<commit_before>//! Broadcasts progress information among workers.

use progress::Timestamp;
use progress::ChangeBatch;
use timely_communication::Allocate;
use {Push, Pull};

// TODO : Extend ProgressVec<T> to contain a source worker id and a sequence number.

/// A list of progress updates corresponding to `((child_scope, [in/out]_port, timestamp), delta)`
pub type ProgressVec<T> = Vec<((usize, usize, T), i64)>;

/// Manages broadcasting of progress updates to and receiving updates from workers.
pub struct Progcaster<T:Timestamp> {
    pushers: Vec<Box<Push<(ProgressVec<T>, ProgressVec<T>)>>>,
    puller: Box<Pull<(ProgressVec<T>, ProgressVec<T>)>>,
}

impl<T:Timestamp+Send> Progcaster<T> {
    /// Creates a new `Progcaster` using a channel from the supplied allocator.
    pub fn new<A: Allocate>(allocator: &mut A) -> Progcaster<T> {
        let (pushers, puller) = allocator.allocate();
        Progcaster { pushers: pushers, puller: puller }
    }

    // TODO : puller.pull() forcibly decodes, whereas we would be just as happy to read data from
    // TODO : binary. Investigate fabric::Wrapper for this purpose.
    /// Sends and receives progress updates, broadcasting the contents of `messages` and `internal`,
    /// and updating each with updates from other workers.
    pub fn send_and_recv(
        &mut self,
        messages: &mut ChangeBatch<(usize, usize, T)>,
        internal: &mut ChangeBatch<(usize, usize, T)>)
    {
        if self.pushers.len() > 1 {  // if the length is one, just return the updates...
            if !messages.is_empty() || !internal.is_empty() {
                for pusher in self.pushers.iter_mut() {
                    pusher.push(&mut Some((messages.clone().into_inner(), internal.clone().into_inner())));
                }

                messages.clear();
                internal.clear();
            }

            // TODO : Could take ownership, and recycle / reuse for next broadcast ...
            while let Some((ref mut recv_messages, ref mut recv_internal)) = *self.puller.pull() {

                for (update, delta) in recv_messages.drain(..) {
                    messages.update(update, delta);
                }

                for (update, delta) in recv_internal.drain(..) {
                    internal.update(update, delta);
                }
            }
        }
    }
}
<commit_msg>progress broadcast optimizations<commit_after>//! Broadcasts progress information among workers.

use progress::Timestamp;
use progress::ChangeBatch;
use timely_communication::Allocate;
use {Push, Pull};

// TODO : Extend ProgressVec<T> to contain a source worker id and a sequence number.

/// A list of progress updates corresponding to `((child_scope, [in/out]_port, timestamp), delta)`
pub type ProgressVec<T> = Vec<((usize, usize, T), i64)>;

/// Manages broadcasting of progress updates to and receiving updates from workers.
pub struct Progcaster<T:Timestamp> {
    pushers: Vec<Box<Push<(ProgressVec<T>, ProgressVec<T>)>>>,
    puller: Box<Pull<(ProgressVec<T>, ProgressVec<T>)>>,
}

impl<T:Timestamp+Send> Progcaster<T> {
    /// Creates a new `Progcaster` using a channel from the supplied allocator.
    pub fn new<A: Allocate>(allocator: &mut A) -> Progcaster<T> {
        let (pushers, puller) = allocator.allocate();
        Progcaster { pushers: pushers, puller: puller }
    }

    /// Sends and receives progress updates, broadcasting the contents of `messages` and `internal`,
    /// and updating each with updates from other workers.
    pub fn send_and_recv(
        &mut self,
        messages: &mut ChangeBatch<(usize, usize, T)>,
        internal: &mut ChangeBatch<(usize, usize, T)>)
    {
        if self.pushers.len() > 1 {  // if the length is one, just return the updates...
            if !messages.is_empty() || !internal.is_empty() {
                for pusher in self.pushers.iter_mut() {
                    // TODO: This should probably use a broadcast channel, or somehow serialize only once.
                    //       It really shouldn't be doing all of this cloning, that's for sure.
                    pusher.push(&mut Some((messages.clone().into_inner(), internal.clone().into_inner())));
                }

                messages.clear();
                internal.clear();
            }

            while let Some((ref mut recv_messages, ref mut recv_internal)) = *self.puller.pull() {

                // We clone rather than drain to avoid deserialization.
                for &(ref update, delta) in recv_messages.iter() {
                    messages.update(update.clone(), delta);
                }

                // We clone rather than drain to avoid deserialization.
                for &(ref update, delta) in recv_internal.iter() {
                    internal.update(update.clone(), delta);
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use ruma_events_macros::event_content_enum;
use serde::{
    de::{self, Error as _},
    Serialize,
};
use serde_json::{from_value as from_json_value, Value as JsonValue};

use crate::{
    event_kinds::{
        BasicEvent, EphemeralRoomEvent, MessageEvent, MessageEventStub, StateEvent, StateEventStub,
        StrippedStateEventStub, ToDeviceEvent,
    },
    presence::PresenceEvent,
    room::redaction::{RedactionEvent, RedactionEventStub},
    util,
};

event_content_enum! {
    /// Any basic event.
    name: AnyBasicEventContent,
    events: [
        "m.direct",
        "m.dummy",
        "m.ignored_user_list",
        "m.push_rules",
        "m.room_key",
        "m.tag",
    ]
}

event_content_enum! {
    /// Any ephemeral room event.
    name: AnyEphemeralRoomEventContent,
    events: [
        "m.fully_read",
        "m.receipt",
        "m.typing",
    ]
}

event_content_enum! {
    /// Any message event.
    name: AnyMessageEventContent,
    events: [
        "m.call.answer",
        "m.call.invite",
        "m.call.hangup",
        "m.call.candidates",
        "m.room.encrypted",
        "m.room.message",
        "m.room.message.feedback",
        "m.sticker",
    ]
}

event_content_enum! {
    /// Any state event.
    name: AnyStateEventContent,
    events: [
        "m.room.aliases",
        "m.room.avatar",
        "m.room.canonical_alias",
        "m.room.create",
        "m.room.encryption",
        "m.room.guest_access",
        "m.room.history_visibility",
        "m.room.join_rules",
        "m.room.member",
        "m.room.name",
        "m.room.pinned_events",
        "m.room.power_levels",
        "m.room.redaction",
        "m.room.server_acl",
        "m.room.third_party_invite",
        "m.room.tombstone",
        "m.room.topic",
    ]
}

event_content_enum! {
    /// Any to-device event.
    name: AnyToDeviceEventContent,
    events: [
        "m.dummy",
        "m.room_key",
        "m.room_key_request",
        "m.forwarded_room_key",
        "m.key.verification.request",
        "m.key.verification.start",
        "m.key.verification.cancel",
        "m.key.verification.accept",
        "m.key.verification.key",
        "m.key.verification.mac",
        "m.room.encrypted",
    ]
}

/// Any basic event, one that has no (well-known) fields outside of `content`.
pub type AnyBasicEvent = BasicEvent<AnyBasicEventContent>;

/// Any ephemeral room event.
pub type AnyEphemeralRoomEvent = EphemeralRoomEvent<AnyEphemeralRoomEventContent>;

/// Any message event.
pub type AnyMessageEvent = MessageEvent<AnyMessageEventContent>;

/// Any message event stub (message event without a `room_id`, as returned in `/sync` responses)
pub type AnyMessageEventStub = MessageEventStub<AnyMessageEventContent>;

/// Any state event.
pub type AnyStateEvent = StateEvent<AnyStateEventContent>;

/// Any state event stub (state event without a `room_id`, as returned in `/sync` responses)
pub type AnyStateEventStub = StateEventStub<AnyStateEventContent>;

/// Any stripped state event stub (stripped-down state event, as returned for rooms the user has
/// been invited to in `/sync` responses)
pub type AnyStrippedStateEventStub = StrippedStateEventStub<AnyStateEventContent>;

/// Any to-device event.
pub type AnyToDeviceEvent = ToDeviceEvent<AnyToDeviceEventContent>;

/// Any event.
#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum AnyEvent {
    /// Any basic event.
    Basic(AnyBasicEvent),
    /// `"m.presence"`, the only non-room event with a `sender` field.
    Presence(PresenceEvent),
    /// Any ephemeral room event.
    Ephemeral(AnyEphemeralRoomEvent),
    /// Any message event.
    Message(AnyMessageEvent),
    /// `"m.room.redaction"`, the only room event with a `redacts` field.
    Redaction(RedactionEvent),
    /// Any state event.
    State(AnyStateEvent),
}

/// Any room event.
#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum AnyRoomEvent {
    /// Any message event.
    Message(AnyMessageEvent),
    /// `"m.room.redaction"`, the only room event with a `redacts` field.
    Redaction(RedactionEvent),
    /// Any state event.
    State(AnyStateEvent),
}

/// Any room event stub (room event without a `room_id`, as returned in `/sync` responses)
#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum AnyRoomEventStub {
    /// Any message event stub
    Message(AnyMessageEventStub),
    /// `"m.room.redaction"` stub
    Redaction(RedactionEventStub),
    /// Any state event stub
    State(AnyStateEventStub),
}

// FIXME `#[serde(untagged)]` deserialization fails for these enums which
// is odd as we are doing basically the same thing here, investigate?
impl<'de> de::Deserialize<'de> for AnyEvent {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = JsonValue::deserialize(deserializer)?;
        let ev_type: String = util::get_field(&json, "type")?;

        match ev_type.as_str() {
            "m.room.redaction" => {
                Ok(AnyEvent::Redaction(from_json_value(json).map_err(D::Error::custom)?))
            }
            "m.presence" => {
                Ok(AnyEvent::Presence(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyBasicEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::Basic(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyEphemeralRoomEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::Ephemeral(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyMessageEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::Message(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyStateEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::State(from_json_value(json).map_err(D::Error::custom)?))
            }
            _ => Err(D::Error::custom(format!("event type `{}` is not a valid event", ev_type))),
        }
    }
}

impl<'de> de::Deserialize<'de> for AnyRoomEvent {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = JsonValue::deserialize(deserializer)?;
        let ev_type: String = util::get_field(&json, "type")?;

        match ev_type.as_str() {
            "m.room.redaction" => {
                Ok(AnyRoomEvent::Redaction(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyMessageEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEvent::Message(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyStateEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEvent::State(from_json_value(json).map_err(D::Error::custom)?))
            }
            _ => Err(D::Error::custom(format!("event type `{}` is not a valid event", ev_type))),
        }
    }
}

impl<'de> de::Deserialize<'de> for AnyRoomEventStub {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = JsonValue::deserialize(deserializer)?;
        let ev_type: String = util::get_field(&json, "type")?;

        match ev_type.as_str() {
            "m.room.redaction" => {
                Ok(AnyRoomEventStub::Redaction(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyMessageEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEventStub::Message(from_json_value(json).map_err(D::Error::custom)?))
            }
            ev_type if AnyStateEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEventStub::State(from_json_value(json).map_err(D::Error::custom)?))
            }
            _ => Err(D::Error::custom(format!("event type `{}` is not a valid event", ev_type))),
        }
    }
}
<commit_msg>Optimize event enum deserialization<commit_after>use ruma_events_macros::event_content_enum;
use serde::{
    de::{self, DeserializeOwned, Error as _},
    Deserialize, Serialize,
};
use serde_json::value::RawValue as RawJsonValue;

use crate::{
    event_kinds::{
        BasicEvent, EphemeralRoomEvent, MessageEvent, MessageEventStub, StateEvent, StateEventStub,
        StrippedStateEventStub, ToDeviceEvent,
    },
    presence::PresenceEvent,
    room::redaction::{RedactionEvent, RedactionEventStub},
};

event_content_enum! {
    /// Any basic event.
    name: AnyBasicEventContent,
    events: [
        "m.direct",
        "m.dummy",
        "m.ignored_user_list",
        "m.push_rules",
        "m.room_key",
        "m.tag",
    ]
}

event_content_enum! {
    /// Any ephemeral room event.
    name: AnyEphemeralRoomEventContent,
    events: [
        "m.fully_read",
        "m.receipt",
        "m.typing",
    ]
}

event_content_enum! {
    /// Any message event.
    name: AnyMessageEventContent,
    events: [
        "m.call.answer",
        "m.call.invite",
        "m.call.hangup",
        "m.call.candidates",
        "m.room.encrypted",
        "m.room.message",
        "m.room.message.feedback",
        "m.sticker",
    ]
}

event_content_enum! {
    /// Any state event.
    name: AnyStateEventContent,
    events: [
        "m.room.aliases",
        "m.room.avatar",
        "m.room.canonical_alias",
        "m.room.create",
        "m.room.encryption",
        "m.room.guest_access",
        "m.room.history_visibility",
        "m.room.join_rules",
        "m.room.member",
        "m.room.name",
        "m.room.pinned_events",
        "m.room.power_levels",
        "m.room.redaction",
        "m.room.server_acl",
        "m.room.third_party_invite",
        "m.room.tombstone",
        "m.room.topic",
    ]
}

event_content_enum! {
    /// Any to-device event.
    name: AnyToDeviceEventContent,
    events: [
        "m.dummy",
        "m.room_key",
        "m.room_key_request",
        "m.forwarded_room_key",
        "m.key.verification.request",
        "m.key.verification.start",
        "m.key.verification.cancel",
        "m.key.verification.accept",
        "m.key.verification.key",
        "m.key.verification.mac",
        "m.room.encrypted",
    ]
}

/// Any basic event, one that has no (well-known) fields outside of `content`.
pub type AnyBasicEvent = BasicEvent<AnyBasicEventContent>;

/// Any ephemeral room event.
pub type AnyEphemeralRoomEvent = EphemeralRoomEvent<AnyEphemeralRoomEventContent>;

/// Any message event.
pub type AnyMessageEvent = MessageEvent<AnyMessageEventContent>;

/// Any message event stub (message event without a `room_id`, as returned in `/sync` responses)
pub type AnyMessageEventStub = MessageEventStub<AnyMessageEventContent>;

/// Any state event.
pub type AnyStateEvent = StateEvent<AnyStateEventContent>;

/// Any state event stub (state event without a `room_id`, as returned in `/sync` responses)
pub type AnyStateEventStub = StateEventStub<AnyStateEventContent>;

/// Any stripped state event stub (stripped-down state event, as returned for rooms the user has
/// been invited to in `/sync` responses)
pub type AnyStrippedStateEventStub = StrippedStateEventStub<AnyStateEventContent>;

/// Any to-device event.
pub type AnyToDeviceEvent = ToDeviceEvent<AnyToDeviceEventContent>;

/// Any event.
#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum AnyEvent {
    /// Any basic event.
    Basic(AnyBasicEvent),
    /// `"m.presence"`, the only non-room event with a `sender` field.
    Presence(PresenceEvent),
    /// Any ephemeral room event.
    Ephemeral(AnyEphemeralRoomEvent),
    /// Any message event.
    Message(AnyMessageEvent),
    /// `"m.room.redaction"`, the only room event with a `redacts` field.
    Redaction(RedactionEvent),
    /// Any state event.
    State(AnyStateEvent),
}

/// Any room event.
#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum AnyRoomEvent {
    /// Any message event.
    Message(AnyMessageEvent),
    /// `"m.room.redaction"`, the only room event with a `redacts` field.
    Redaction(RedactionEvent),
    /// Any state event.
    State(AnyStateEvent),
}

/// Any room event stub (room event without a `room_id`, as returned in `/sync` responses)
#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum AnyRoomEventStub {
    /// Any message event stub
    Message(AnyMessageEventStub),
    /// `"m.room.redaction"` stub
    Redaction(RedactionEventStub),
    /// Any state event stub
    State(AnyStateEventStub),
}

#[derive(Deserialize)]
struct EventDeHelper {
    #[serde(rename = "type")]
    ev_type: String,
}

fn from_raw_json_value<T, E>(val: &RawJsonValue) -> Result<T, E>
where
    T: DeserializeOwned,
    E: de::Error,
{
    serde_json::from_str(val.get()).map_err(E::custom)
}

// FIXME `#[serde(untagged)]` deserialization fails for these enums which
// is odd as we are doing basically the same thing here, investigate?
impl<'de> de::Deserialize<'de> for AnyEvent {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = <&RawJsonValue>::deserialize(deserializer)?;
        let EventDeHelper { ev_type } = from_raw_json_value(json)?;

        match ev_type.as_str() {
            "m.room.redaction" => Ok(AnyEvent::Redaction(from_raw_json_value(json)?)),
            "m.presence" => Ok(AnyEvent::Presence(from_raw_json_value(json)?)),
            ev_type if AnyBasicEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::Basic(from_raw_json_value(json)?))
            }
            ev_type if AnyEphemeralRoomEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::Ephemeral(from_raw_json_value(json)?))
            }
            ev_type if AnyMessageEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::Message(from_raw_json_value(json)?))
            }
            ev_type if AnyStateEventContent::is_compatible(ev_type) => {
                Ok(AnyEvent::State(from_raw_json_value(json)?))
            }
            _ => Err(D::Error::custom(format!("event type `{}` is not a valid event", ev_type))),
        }
    }
}

impl<'de> de::Deserialize<'de> for AnyRoomEvent {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = <&RawJsonValue>::deserialize(deserializer)?;
        let EventDeHelper { ev_type } = from_raw_json_value(json)?;

        match ev_type.as_str() {
            "m.room.redaction" => Ok(AnyRoomEvent::Redaction(from_raw_json_value(json)?)),
            ev_type if AnyMessageEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEvent::Message(from_raw_json_value(json)?))
            }
            ev_type if AnyStateEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEvent::State(from_raw_json_value(json)?))
            }
            _ => Err(D::Error::custom(format!("event type `{}` is not a valid event", ev_type))),
        }
    }
}

impl<'de> de::Deserialize<'de> for AnyRoomEventStub {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = <&RawJsonValue>::deserialize(deserializer)?;
        let EventDeHelper { ev_type } = from_raw_json_value(json)?;

        match ev_type.as_str() {
            "m.room.redaction" => Ok(AnyRoomEventStub::Redaction(from_raw_json_value(json)?)),
            ev_type if AnyMessageEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEventStub::Message(from_raw_json_value(json)?))
            }
            ev_type if AnyStateEventContent::is_compatible(ev_type) => {
                Ok(AnyRoomEventStub::State(from_raw_json_value(json)?))
            }
            _ => Err(D::Error::custom(format!("event type `{}` is not a valid event", ev_type))),
        }
    }
}
<|endoftext|>


<commit_before>///! File system utilities, particularly functionality that requires
///! alternative implementations for alternative operating systems.
use std::{
    fs::{self, File},
    io, os,
    path::Path,
};

use common::eyre::{eyre, Result};

/// Set permissions on a file
#[allow(unused_variables)]
pub fn set_perms<File: AsRef<Path>>(path: File, mode: u32) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    {
        use os::unix::fs::PermissionsExt;
        fs::set_permissions(path, fs::Permissions::from_mode(mode))?;
    }

    Ok(())
}

/// Create a symbolic (soft) link to a file
pub fn symlink_file<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_file(original, link)?;

    Ok(())
}

/// Create a symbolic (soft) link to a directory
pub fn symlink_dir<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_dir(original, link)?;

    Ok(())
}

/// Remove a file or directory if it exists
pub fn remove_if_exists(path: impl AsRef<Path>) -> Result<()> {
    let path = path.as_ref();
    if path.exists() {
        if path.is_dir() {
            fs::remove_dir_all(path)?;
        } else {
            fs::remove_file(path)?;
        }
    }
    Ok(())
}

/// Copy a file or directory if it exists
pub fn copy_if_exists(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    let src = src.as_ref();
    if src.exists() {
        if src.is_dir() {
            copy_dir_all(src, dest)?;
        } else {
            fs::copy(src, dest)?;
        }
    }
    Ok(())
}

/// Clear a directory
pub fn clear_dir_all(dir: impl AsRef<Path>) -> Result<()> {
    fs::remove_dir_all(&dir)?;
    fs::create_dir_all(&dir)?;
    Ok(())
}

/// Recursively copy a directory to another
pub fn copy_dir_all(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    fs::create_dir_all(&dest)?;
    for entry in fs::read_dir(src)? {
        let entry = entry?;
        if entry.file_type()?.is_dir() {
            copy_dir_all(entry.path(), dest.as_ref().join(entry.file_name()))?;
        } else if let Err(error) = fs::copy(entry.path(), dest.as_ref().join(entry.file_name())) {
            // Ignore "the source path is neither a regular file nor a symlink to a regular file" errors
            if !matches!(error.kind(), io::ErrorKind::InvalidInput) {
                return Err(eyre!(error));
            }
        }
    }
    Ok(())
}

/// Move a directory
///
/// This is a lot less efficient than `std::fs::rename` but will work across mounts
pub fn move_dir_all(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    copy_dir_all(&src, &dest)?;
    fs::remove_dir_all(&src)?;
    Ok(())
}

/// Open a file in 600 mode (only read and writeable by current user)
pub fn open_file_600(path: impl AsRef<Path>) -> Result<File> {
    let mut options = fs::OpenOptions::new();
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    {
        use std::os::unix::fs::OpenOptionsExt;
        options.mode(0o600);
    }
    #[cfg(any(target_os = "windows"))]
    {
        use std::os::windows::fs::OpenOptionsExt;
        options.share_mode(0);
    }

    let file = options
        .read(true)
        .write(true)
        .create_new(true)
        .open(&path)?;

    Ok(file)
}
<commit_msg>fix(Filesystem utils): Use `create` and `truncate` options for usual behaviour<commit_after>///! File system utilities, particularly functionality that requires
///! alternative implementations for alternative operating systems.
use std::{
    fs::{self, File},
    io, os,
    path::Path,
};

use common::eyre::{eyre, Result};

/// Set permissions on a file
#[allow(unused_variables)]
pub fn set_perms<File: AsRef<Path>>(path: File, mode: u32) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    {
        use os::unix::fs::PermissionsExt;
        fs::set_permissions(path, fs::Permissions::from_mode(mode))?;
    }

    Ok(())
}

/// Create a symbolic (soft) link to a file
pub fn symlink_file<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_file(original, link)?;

    Ok(())
}

/// Create a symbolic (soft) link to a directory
pub fn symlink_dir<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_dir(original, link)?;

    Ok(())
}

/// Remove a file or directory if it exists
pub fn remove_if_exists(path: impl AsRef<Path>) -> Result<()> {
    let path = path.as_ref();
    if path.exists() {
        if path.is_dir() {
            fs::remove_dir_all(path)?;
        } else {
            fs::remove_file(path)?;
        }
    }
    Ok(())
}

/// Copy a file or directory if it exists
pub fn copy_if_exists(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    let src = src.as_ref();
    if src.exists() {
        if src.is_dir() {
            copy_dir_all(src, dest)?;
        } else {
            fs::copy(src, dest)?;
        }
    }
    Ok(())
}

/// Clear a directory
pub fn clear_dir_all(dir: impl AsRef<Path>) -> Result<()> {
    fs::remove_dir_all(&dir)?;
    fs::create_dir_all(&dir)?;
    Ok(())
}

/// Recursively copy a directory to another
pub fn copy_dir_all(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    fs::create_dir_all(&dest)?;
    for entry in fs::read_dir(src)? {
        let entry = entry?;
        if entry.file_type()?.is_dir() {
            copy_dir_all(entry.path(), dest.as_ref().join(entry.file_name()))?;
        } else if let Err(error) = fs::copy(entry.path(), dest.as_ref().join(entry.file_name())) {
            // Ignore "the source path is neither a regular file nor a symlink to a regular file" errors
            if !matches!(error.kind(), io::ErrorKind::InvalidInput) {
                return Err(eyre!(error));
            }
        }
    }
    Ok(())
}

/// Move a directory
///
/// This is a lot less efficient than `std::fs::rename` but will work across mounts
pub fn move_dir_all(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    copy_dir_all(&src, &dest)?;
    fs::remove_dir_all(&src)?;
    Ok(())
}

/// Open a file in 600 mode (only read and writeable by current user)
pub fn open_file_600(path: impl AsRef<Path>) -> Result<File> {
    let mut options = fs::OpenOptions::new();
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    {
        use std::os::unix::fs::OpenOptionsExt;
        options.mode(0o600);
    }
    #[cfg(any(target_os = "windows"))]
    {
        use std::os::windows::fs::OpenOptionsExt;
        options.share_mode(0);
    }

    let file = options
        .read(true)
        .write(true)
        .create(true)
        .truncate(true)
        .open(&path)?;

    Ok(file)
}
<|endoftext|>


<commit_before>use crate::{future::FutureExt, try_future::TryFutureExt};
use std::{
    future::FutureObj,
    task::{Executor, SpawnErrorKind, SpawnObjError},
};
use tokio_executor::{DefaultExecutor, Executor as TokioExecutor};

/// An executor that delegates to `tokio`'s
/// [`DefaultExecutor`][tokio_executor::DefaultExecutor], will panic if used in
/// the context of a task that is not running on `tokio`'s executor.
///
/// # Examples
///
/// ```ignore
/// #![feature(async_await, await_macro, futures_api, pin, use_extern_macros)]
///
/// use std::thread;
/// use std::boxed::PinBox;
///
/// use futures::channel::oneshot;
/// use futures::compat::TokioDefaultExecutor;
/// use futures::future::{FutureExt, TryFutureExt};
/// use futures::spawn;
///
/// let (sender, receiver) = oneshot::channel::<i32>();
///
/// let handle = thread::spawn(move || {
///     futures::executor::block_on(async move {
///         await!(receiver).expect("the sender will send us something")
///     })
/// });
///
/// thread::spawn(move || {
///     let future = async move {
///         spawn!(async move {
///             sender.send(5).expect("the receiver is still waiting");
///         }).expect("can spawn on the executor")
///     };
///
///     let compat_future = PinBox::new(future)
///         .unit_error()
///         .compat(TokioDefaultExecutor);
///
///     tokio::run(compat_future);
/// }).join().unwrap();
///
/// assert_eq!(handle.join().unwrap(), 5);
/// ```
#[derive(Debug, Copy, Clone)]
pub struct TokioDefaultExecutor;

impl Executor for TokioDefaultExecutor {
    fn spawn_obj(
        &mut self,
        task: FutureObj<'static, ()>,
    ) -> Result<(), SpawnObjError> {
        let fut = Box::new(task.unit_error().compat(*self));
        DefaultExecutor::current().spawn(fut).map_err(|err| {
            panic!(
                "tokio failed to spawn and doesn't return the future: {:?}",
                err
            )
        })
    }

    fn status(&self) -> Result<(), SpawnErrorKind> {
        DefaultExecutor::current().status().map_err(|e| {
            if e.is_shutdown() {
                SpawnErrorKind::shutdown()
            } else {
                panic!("tokio executor failed for non-shutdown reason: {:?}", e)
            }
        })
    }
}
<commit_msg>Add forward compatibility notice to tokio compat.<commit_after>use crate::{future::FutureExt, try_future::TryFutureExt};
use std::{
    future::FutureObj,
    task::{Executor, SpawnErrorKind, SpawnObjError},
};
use tokio_executor::{DefaultExecutor, Executor as TokioExecutor};

/// An executor that delegates to `tokio`'s
/// [`DefaultExecutor`][tokio_executor::DefaultExecutor], will panic if used in
/// the context of a task that is not running on `tokio`'s executor.
///
/// *NOTE* The future of this struct in `futures` is uncertain. It may be
/// deprecated before or soon after the initial 0.3 release and moved to a
/// feature in `tokio` instead.
///
/// # Examples
///
/// ```ignore
/// #![feature(async_await, await_macro, futures_api, pin, use_extern_macros)]
///
/// use std::thread;
/// use std::boxed::PinBox;
///
/// use futures::channel::oneshot;
/// use futures::compat::TokioDefaultExecutor;
/// use futures::future::{FutureExt, TryFutureExt};
/// use futures::spawn;
///
/// let (sender, receiver) = oneshot::channel::<i32>();
///
/// let handle = thread::spawn(move || {
///     futures::executor::block_on(async move {
///         await!(receiver).expect("the sender will send us something")
///     })
/// });
///
/// thread::spawn(move || {
///     let future = async move {
///         spawn!(async move {
///             sender.send(5).expect("the receiver is still waiting");
///         }).expect("can spawn on the executor")
///     };
///
///     let compat_future = PinBox::new(future)
///         .unit_error()
///         .compat(TokioDefaultExecutor);
///
///     tokio::run(compat_future);
/// }).join().unwrap();
///
/// assert_eq!(handle.join().unwrap(), 5);
/// ```
#[derive(Debug, Copy, Clone)]
pub struct TokioDefaultExecutor;

impl Executor for TokioDefaultExecutor {
    fn spawn_obj(
        &mut self,
        task: FutureObj<'static, ()>,
    ) -> Result<(), SpawnObjError> {
        let fut = Box::new(task.unit_error().compat(*self));
        DefaultExecutor::current().spawn(fut).map_err(|err| {
            panic!(
                "tokio failed to spawn and doesn't return the future: {:?}",
                err
            )
        })
    }

    fn status(&self) -> Result<(), SpawnErrorKind> {
        DefaultExecutor::current().status().map_err(|e| {
            if e.is_shutdown() {
                SpawnErrorKind::shutdown()
            } else {
                panic!("tokio executor failed for non-shutdown reason: {:?}", e)
            }
        })
    }
}
<|endoftext|>


<commit_before>use hashing::hash;
use int_to_bytes::{int_to_bytes1, int_to_bytes4};
use std::cmp::max;

/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
///
/// Utilizes 'swap or not' shuffling found in
/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
/// See the 'generalized domain' algorithm on page 3.
///
/// Note: this function is significantly slower than the `shuffle_list` function in this crate.
/// Using `get_permutated_list` to shuffle an entire list, index by index, has been observed to be
/// 250x slower than `shuffle_list`. Therefore, this function is only useful when shuffling a small
/// portion of a much larger list.
///
/// Returns `None` under any of the following conditions:
///  - `list_size == 0`
///  - `index >= list_size`
///  - `list_size > 2**24`
///  - `list_size > usize::max_value() / 2`
pub fn get_permutated_index(
    index: usize,
    list_size: usize,
    seed: &[u8],
    shuffle_round_count: u8,
) -> Option<usize> {
    if list_size == 0
        || index >= list_size
        || list_size > usize::max_value() / 2
        || list_size > 2_usize.pow(24)
    {
        return None;
    }

    let mut index = index;
    for round in 0..shuffle_round_count {
        let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size;
        index = do_round(seed, index, pivot, round, list_size)?;
    }
    Some(index)
}

fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option<usize> {
    let flip = (pivot + list_size - index) % list_size;
    let position = max(index, flip);
    let source = hash_with_round_and_position(seed, round, position)?;
    let byte = source[(position % 256) / 8];
    let bit = (byte >> (position % 8)) % 2;
    Some(if bit == 1 { flip } else { index })
}

fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option<Vec<u8>> {
    let mut seed = seed.to_vec();
    seed.append(&mut int_to_bytes1(round));
    /*
     * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 <
     * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`.
     */
    seed.append(&mut int_to_bytes4((position / 256) as u32));
    Some(hash(&seed[..]))
}

fn hash_with_round(seed: &[u8], round: u8) -> Vec<u8> {
    let mut seed = seed.to_vec();
    seed.append(&mut int_to_bytes1(round));
    hash(&seed[..])
}

fn bytes_to_int64(slice: &[u8]) -> u64 {
    let mut bytes = [0; 8];
    bytes.copy_from_slice(&slice[0..8]);
    u64::from_le_bytes(bytes)
}

#[cfg(test)]
mod tests {
    use super::*;
    use ethereum_types::H256 as Hash256;
    use hex;
    use std::{fs::File, io::prelude::*, path::PathBuf};
    use yaml_rust::yaml;

    #[test]
    #[ignore]
    fn fuzz_test() {
        let max_list_size = 2_usize.pow(24);
        let test_runs = 1000;

        // Test at max list_size with the end index.
        for _ in 0..test_runs {
            let index = max_list_size - 1;
            let list_size = max_list_size;
            let seed = Hash256::random();
            let shuffle_rounds = 90;

            assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
        }

        // Test at max list_size low indices.
        for i in 0..test_runs {
            let index = i;
            let list_size = max_list_size;
            let seed = Hash256::random();
            let shuffle_rounds = 90;

            assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
        }

        // Test at max list_size high indices.
        for i in 0..test_runs {
            let index = max_list_size - 1 - i;
            let list_size = max_list_size;
            let seed = Hash256::random();
            let shuffle_rounds = 90;

            assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
        }
    }

    #[test]
    fn returns_none_for_zero_length_list() {
        assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90));
    }

    #[test]
    fn returns_none_for_out_of_bounds_index() {
        assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90));
    }

    #[test]
    fn returns_none_for_too_large_list() {
        assert_eq!(
            None,
            get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90)
        );
    }

    #[test]
    fn test_vectors() {
        /*
         * Test vectors are generated here:
         *
         * https://github.com/ethereum/eth2.0-test-generators
         */
        let mut file = {
            let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
            file_path_buf.push("src/specs/test_vector_permutated_index.yml");

            File::open(file_path_buf).unwrap()
        };

        let mut yaml_str = String::new();

        file.read_to_string(&mut yaml_str).unwrap();

        let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap();
        let doc = &docs[0];
        let test_cases = doc["test_cases"].as_vec().unwrap();

        for (i, test_case) in test_cases.iter().enumerate() {
            let index = test_case["index"].as_i64().unwrap() as usize;
            let list_size = test_case["list_size"].as_i64().unwrap() as usize;
            let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize;
            let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap();
            let seed_string = test_case["seed"].clone().into_string().unwrap();
            let seed = hex::decode(seed_string.replace("0x", "")).unwrap();

            let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) {
                shuffle_round_count as u8
            } else {
                panic!("shuffle_round_count must be a u8")
            };

            assert_eq!(
                Some(permutated_index),
                get_permutated_index(index, list_size, &seed[..], shuffle_round_count),
                "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}",
                i,
                index,
                list_size,
                shuffle_round_count,
                seed_string,
            );
        }
    }
}
<commit_msg>Clarify order-of-operations in shuffling<commit_after>use hashing::hash;
use int_to_bytes::{int_to_bytes1, int_to_bytes4};
use std::cmp::max;

/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
///
/// Utilizes 'swap or not' shuffling found in
/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
/// See the 'generalized domain' algorithm on page 3.
///
/// Note: this function is significantly slower than the `shuffle_list` function in this crate.
/// Using `get_permutated_list` to shuffle an entire list, index by index, has been observed to be
/// 250x slower than `shuffle_list`. Therefore, this function is only useful when shuffling a small
/// portion of a much larger list.
///
/// Returns `None` under any of the following conditions:
///  - `list_size == 0`
///  - `index >= list_size`
///  - `list_size > 2**24`
///  - `list_size > usize::max_value() / 2`
pub fn get_permutated_index(
    index: usize,
    list_size: usize,
    seed: &[u8],
    shuffle_round_count: u8,
) -> Option<usize> {
    if list_size == 0
        || index >= list_size
        || list_size > usize::max_value() / 2
        || list_size > 2_usize.pow(24)
    {
        return None;
    }

    let mut index = index;
    for round in 0..shuffle_round_count {
        let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size;
        index = do_round(seed, index, pivot, round, list_size)?;
    }
    Some(index)
}

fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option<usize> {
    let flip = (pivot + (list_size - index)) % list_size;
    let position = max(index, flip);
    let source = hash_with_round_and_position(seed, round, position)?;
    let byte = source[(position % 256) / 8];
    let bit = (byte >> (position % 8)) % 2;
    Some(if bit == 1 { flip } else { index })
}

fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option<Vec<u8>> {
    let mut seed = seed.to_vec();
    seed.append(&mut int_to_bytes1(round));
    /*
     * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 <
     * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`.
     */
    seed.append(&mut int_to_bytes4((position / 256) as u32));
    Some(hash(&seed[..]))
}

fn hash_with_round(seed: &[u8], round: u8) -> Vec<u8> {
    let mut seed = seed.to_vec();
    seed.append(&mut int_to_bytes1(round));
    hash(&seed[..])
}

fn bytes_to_int64(slice: &[u8]) -> u64 {
    let mut bytes = [0; 8];
    bytes.copy_from_slice(&slice[0..8]);
    u64::from_le_bytes(bytes)
}

#[cfg(test)]
mod tests {
    use super::*;
    use ethereum_types::H256 as Hash256;
    use hex;
    use std::{fs::File, io::prelude::*, path::PathBuf};
    use yaml_rust::yaml;

    #[test]
    #[ignore]
    fn fuzz_test() {
        let max_list_size = 2_usize.pow(24);
        let test_runs = 1000;

        // Test at max list_size with the end index.
        for _ in 0..test_runs {
            let index = max_list_size - 1;
            let list_size = max_list_size;
            let seed = Hash256::random();
            let shuffle_rounds = 90;

            assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
        }

        // Test at max list_size low indices.
        for i in 0..test_runs {
            let index = i;
            let list_size = max_list_size;
            let seed = Hash256::random();
            let shuffle_rounds = 90;

            assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
        }

        // Test at max list_size high indices.
        for i in 0..test_runs {
            let index = max_list_size - 1 - i;
            let list_size = max_list_size;
            let seed = Hash256::random();
            let shuffle_rounds = 90;

            assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
        }
    }

    #[test]
    fn returns_none_for_zero_length_list() {
        assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90));
    }

    #[test]
    fn returns_none_for_out_of_bounds_index() {
        assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90));
    }

    #[test]
    fn returns_none_for_too_large_list() {
        assert_eq!(
            None,
            get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90)
        );
    }

    #[test]
    fn test_vectors() {
        /*
         * Test vectors are generated here:
         *
         * https://github.com/ethereum/eth2.0-test-generators
         */
        let mut file = {
            let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
            file_path_buf.push("src/specs/test_vector_permutated_index.yml");

            File::open(file_path_buf).unwrap()
        };

        let mut yaml_str = String::new();

        file.read_to_string(&mut yaml_str).unwrap();

        let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap();
        let doc = &docs[0];
        let test_cases = doc["test_cases"].as_vec().unwrap();

        for (i, test_case) in test_cases.iter().enumerate() {
            let index = test_case["index"].as_i64().unwrap() as usize;
            let list_size = test_case["list_size"].as_i64().unwrap() as usize;
            let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize;
            let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap();
            let seed_string = test_case["seed"].clone().into_string().unwrap();
            let seed = hex::decode(seed_string.replace("0x", "")).unwrap();

            let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) {
                shuffle_round_count as u8
            } else {
                panic!("shuffle_round_count must be a u8")
            };

            assert_eq!(
                Some(permutated_index),
                get_permutated_index(index, list_size, &seed[..], shuffle_round_count),
                "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}",
                i,
                index,
                list_size,
                shuffle_round_count,
                seed_string,
            );
        }
    }
}
<|endoftext|>


<commit_before>//
//  SOS: the Stupid Operating System
//  by Eliza Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2016 Eliza Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Initial 32-bit bootloader for x86_64
#![crate_name = "boot"]
#![crate_type = "staticlib"]
#![feature(asm)]
#![feature(lang_items)]
#![no_std]

const TABLE_LENGTH: usize = 512;

type Table = [u64; TABLE_LENGTH];

use core::fmt::Arguments;

#[lang = "panic_fmt"]
extern fn panic_fmt(fmt: Arguments, file: &'static str, line: u32) -> ! {
	loop {}
}

extern {
	static mut pml4_table: Table;
	static mut pdp_table: Table;
	static mut pd_table: Table;
	static mut page_table: Table;
    //
	// static low_end: u8;
	// static kernel_size: u8; // &kernel_size == kernel size in pages
}

#[no_mangle]
pub unsafe extern fn create_page_tables() {
    // recursive map last PML4 entry
    pml4_table[511] = (&pml4_table as *const Table as u64) | 3;
    // map first PML4 entry to PDP table
    pml4_table[0] = (&pdp_table as *const Table as u64) | 3;
    // map first PDPT entry to PD table
    pdp_table[0] = (&pd_table as *const Table as u64) | 3;

    // map the PD table
    // TODO: implement this in Rust
    asm!("
        // map each PD table entry to its own 2mB page
        mov         ecx, 0
    .pd_table_map: // maps the PD table ----------------------------------------
        mov     eax, 0x200000   // 2 mB
        mul     ecx             // times the start address of the page
        or      eax, 0b10000011 // check if present + writable + huge
        mov     [pd_table + ecx * 8], eax // map nth entry from pd -> own page
        // increment counter and check if done
        inc     ecx
        cmp     ecx, 512
        jne     .pd_table_map
        " :::: "volatile", "intel");
}
<commit_msg>[boot] implement page-table mapping in pure Rust<commit_after>//
//  SOS: the Stupid Operating System
//  by Eliza Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2016-2017 Eliza Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Initial 32-bit bootloader for x86_64
#![crate_name = "boot"]
#![crate_type = "staticlib"]
#![feature(asm)]
#![feature(lang_items)]
#![feature(naked_functions)]
#![no_std]

const TABLE_LENGTH: usize = 512;

type Table = [u64; TABLE_LENGTH];

use core::fmt::Arguments;

#[lang = "panic_fmt"]
extern fn panic_fmt(fmt: Arguments, file: &'static str, line: u32) -> ! {
	loop {}
}

extern {
    static mut pml4_table: Table;
    static mut pdp_table: Table;
    static mut pd_table: Table;
    static mut page_table: Table;
    //
    // static low_end: u8;
    // static kernel_size: u8; // &kernel_size == kernel size in pages
}

#[no_mangle]
#[naked]
pub unsafe extern fn create_page_tables() {
    const HUGE_PAGE_SIZE: u64 = 2 * 1024 * 1024; // 2 MiB

    //-- map the PML4 and PDP tables -----------------------------------------
    // recursive map last PML4 entry
    pml4_table[511] = (&pml4_table as *const Table as u64) | 3;
    // map first PML4 entry to PDP table
    pml4_table[0] = (&pdp_table as *const Table as u64) | 3;
    // map first PDPT entry to PD table
    pdp_table[0] = (&pd_table as *const Table as u64) | 3;

    //-- map the PD table ----------------------------------------------------
    for (number, entry) in pd_table.iter_mut().enumerate() {
        // set each PD entry equal to the start address of the page (the page
        // number times the page's size)
        let addr = number as u64 * HUGE_PAGE_SIZE;
        // with the appropriate flags (present + writable + huge)
        // TODO: do we want to do this using bitflags, or is that too
        //       heavyweight for the boot module?
        //          - eliza, 1/23/2017
        *entry = addr | 0b10000011;
    }
}
<|endoftext|>


<commit_before>//! Source file support for diagnostic reporting.

use std::ops::Range;

/// Files that can be used for pretty diagnostic rendering.
///
/// A lifetime parameter `'a` is provided to allow any of the returned values to returned by reference.
/// This is to workaround the lack of higher kinded lifetime parameters.
/// This can be ignored if this is not needed, however.
pub trait Files<'a> {
    type FileId: 'a + Copy + PartialEq;
    type Origin: 'a + std::fmt::Display;
    type Source: 'a + AsRef<str>;

    /// The origin of a file.
    fn origin(&'a self, id: Self::FileId) -> Option<Self::Origin>;

    /// The source of a file.
    fn source(&'a self, id: Self::FileId) -> Option<Self::Source>;

    /// The index of the line at the given byte index.
    fn line_index(&'a self, id: Self::FileId, byte_index: usize) -> Option<usize>;

    /// The user-facing line number at the given line index.
    ///
    /// This can be useful for implementing something like the
    /// [C preprocessor's `#line` macro][line-macro],
    /// but is usually 1-indexed from the beginning of the file.
    ///
    /// [line-macro]: https://en.cppreference.com/w/c/preprocessor/line
    #[allow(unused_variables)]
    fn line_number(&'a self, id: Self::FileId, line_index: usize) -> Option<usize> {
        Some(line_index + 1)
    }

    /// The index of the line at the given byte index.
    fn line_range(&'a self, id: Self::FileId, line_index: usize) -> Option<Range<usize>>;
}

/// The column index at the given byte index in the source file.
/// This is the number of characters to the given byte index.
///
/// If the byte index is smaller than the start of the line, then `0` is returned.
/// If the byte index is past the end of the line, the column index of the last
/// character `+ 1` is returned.
///
/// # Example
///
/// ```rust
/// use codespan_reporting::files;
///
/// let source = "\n\n🗻∈🌏";
///
/// assert_eq!(files::column_index(source, 2..13, 0), 0);
/// assert_eq!(files::column_index(source, 2..13, 2 + 0), 0);
/// assert_eq!(files::column_index(source, 2..13, 2 + 1), 0);
/// assert_eq!(files::column_index(source, 2..13, 2 + 4), 1);
/// assert_eq!(files::column_index(source, 2..13, 2 + 8), 2);
/// assert_eq!(files::column_index(source, 2..13, source.len()), 3);
/// ```
pub fn column_index(source: &str, line_range: Range<usize>, byte_index: usize) -> usize {
    let relative_index = match byte_index.checked_sub(line_range.start) {
        None => return 0,
        Some(relative_index) => relative_index,
    };
    let line_source = &source[line_range];
    let column_index = line_source
        .char_indices()
        .map(|(i, _)| i)
        .take_while(|i| *i < relative_index)
        .count();

    match () {
        () if relative_index >= line_source.len() => column_index,
        () if line_source.is_char_boundary(relative_index) => column_index,
        () => column_index - 1,
    }
}

/// The 1-indexed column number at the given byte index.
///
/// # Example
///
/// ```rust
/// use codespan_reporting::files;
///
/// let source = "\n\n🗻∈🌏";
/// let line_range = 2..13;
///
/// assert_eq!(files::column_number(source, 2..13, 0), 1);
/// assert_eq!(files::column_number(source, 2..13, 2 + 0), 1);
/// assert_eq!(files::column_number(source, 2..13, 2 + 1), 1);
/// assert_eq!(files::column_number(source, 2..13, 2 + 4), 2);
/// assert_eq!(files::column_number(source, 2..13, 2 + 8), 3);
/// assert_eq!(files::column_number(source, 2..13, source.len()), 4);
/// ```
pub fn column_number(source: &str, line_range: Range<usize>, byte_index: usize) -> usize {
    column_index(source, line_range, byte_index) + 1
}

/// A single source file.
///
/// This is useful for simple language tests, but it might be worth creating a
/// custom implementation when a language scales beyond a certain size.
#[derive(Debug, Clone)]
pub struct SimpleFile<Origin, Source> {
    /// The origin of the file.
    origin: Origin,
    /// The source code of the file.
    source: Source,
    /// The starting byte indices in the source code.
    line_starts: Vec<usize>,
}

/// Return the starting byte index of each line in the source string.
///
/// This can make it easier to implement new `Files` implementations.
///
/// # Example
///
/// ```rust
/// use codespan_reporting::files;
///
/// let source = "foo\nbar\r\n\nbaz";
/// let line_starts: Vec<_> = files::line_starts(source).collect();
///
/// assert_eq!(
///     line_starts,
///     [
///         0,  // "foo\n"
///         4,  // "bar\r\n"
///         9,  // ""
///         10, // "baz"
///     ],
/// );
///
/// fn line_index(line_starts: &[usize], byte_index: usize) -> Option<usize> {
///     match line_starts.binary_search(&byte_index) {
///         Ok(line) => Some(line),
///         Err(next_line) => Some(next_line - 1),
///     }
/// }
///
/// assert_eq!(line_index(&line_starts, 5), Some(1));
/// ```
pub fn line_starts<'source>(source: &'source str) -> impl 'source + Iterator<Item = usize> {
    std::iter::once(0).chain(source.match_indices('\n').map(|(i, _)| i + 1))
}

impl<Origin, Source> SimpleFile<Origin, Source>
where
    Origin: std::fmt::Display,
    Source: AsRef<str>,
{
    /// Create a new source file.
    pub fn new(origin: Origin, source: Source) -> SimpleFile<Origin, Source> {
        SimpleFile {
            origin,
            line_starts: line_starts(source.as_ref()).collect(),
            source,
        }
    }

    /// Return the origin of the file.
    pub fn origin(&self) -> &Origin {
        &self.origin
    }

    /// Return the source of the file.
    pub fn source(&self) -> &Source {
        &self.source
    }

    fn line_start(&self, line_index: usize) -> Option<usize> {
        use std::cmp::Ordering;

        match line_index.cmp(&self.line_starts.len()) {
            Ordering::Less => self.line_starts.get(line_index).cloned(),
            Ordering::Equal => Some(self.source.as_ref().len()),
            Ordering::Greater => None,
        }
    }
}

impl<'a, Origin, Source> Files<'a> for SimpleFile<Origin, Source>
where
    Origin: 'a + std::fmt::Display + Clone,
    Source: 'a + AsRef<str>,
{
    type FileId = ();
    type Origin = Origin;
    type Source = &'a str;

    fn origin(&self, (): ()) -> Option<Origin> {
        Some(self.origin.clone())
    }

    fn source(&self, (): ()) -> Option<&str> {
        Some(self.source.as_ref())
    }

    fn line_index(&self, (): (), byte_index: usize) -> Option<usize> {
        match self.line_starts.binary_search(&byte_index) {
            Ok(line) => Some(line),
            Err(next_line) => Some(next_line - 1),
        }
    }

    fn line_range(&self, (): (), line_index: usize) -> Option<Range<usize>> {
        let line_start = self.line_start(line_index)?;
        let next_line_start = self.line_start(line_index + 1)?;

        Some(line_start..next_line_start)
    }
}

/// A file database that can store multiple source files.
///
/// This is useful for simple language tests, but it might be worth creating a
/// custom implementation when a language scales beyond a certain size.
#[derive(Debug, Clone)]
pub struct SimpleFiles<Origin, Source> {
    files: Vec<SimpleFile<Origin, Source>>,
}

impl<Origin, Source> SimpleFiles<Origin, Source>
where
    Origin: std::fmt::Display,
    Source: AsRef<str>,
{
    /// Create a new files database.
    pub fn new() -> SimpleFiles<Origin, Source> {
        SimpleFiles { files: Vec::new() }
    }

    /// Add a file to the database, returning the handle that can be used to
    /// refer to it again.
    pub fn add(&mut self, origin: Origin, source: Source) -> usize {
        let file_id = self.files.len();
        self.files.push(SimpleFile::new(origin, source));
        file_id
    }

    /// Get the file corresponding to the given id.
    pub fn get(&self, file_id: usize) -> Option<&SimpleFile<Origin, Source>> {
        self.files.get(file_id)
    }
}

impl<'a, Origin, Source> Files<'a> for SimpleFiles<Origin, Source>
where
    Origin: 'a + std::fmt::Display + Clone,
    Source: 'a + AsRef<str>,
{
    type FileId = usize;
    type Origin = Origin;
    type Source = &'a str;

    fn origin(&self, file_id: usize) -> Option<Origin> {
        Some(self.get(file_id)?.origin().clone())
    }

    fn source(&self, file_id: usize) -> Option<&str> {
        Some(self.get(file_id)?.source().as_ref())
    }

    fn line_index(&self, file_id: usize, byte_index: usize) -> Option<usize> {
        self.get(file_id)?.line_index((), byte_index)
    }

    fn line_range(&self, file_id: usize, line_index: usize) -> Option<Range<usize>> {
        self.get(file_id)?.line_range((), line_index)
    }
}

#[cfg(test)]
mod test {
    use super::*;

    const TEST_SOURCE: &str = "foo\nbar\r\n\nbaz";

    #[test]
    fn line_starts() {
        let file = SimpleFile::new("test", TEST_SOURCE);

        assert_eq!(
            file.line_starts,
            [
                0,  // "foo\n"
                4,  // "bar\r\n"
                9,  // ""
                10, // "baz"
            ],
        );
    }

    #[test]
    fn line_span_sources() {
        let file = SimpleFile::new("test", TEST_SOURCE);

        let line_sources = (0..4)
            .map(|line| {
                let line_range = file.line_range((), line).unwrap();
                &file.source[line_range]
            })
            .collect::<Vec<_>>();

        assert_eq!(line_sources, ["foo\n", "bar\r\n", "\n", "baz"]);
    }
}
<commit_msg>Simplify column index implementation<commit_after>//! Source file support for diagnostic reporting.

use std::ops::Range;

/// Files that can be used for pretty diagnostic rendering.
///
/// A lifetime parameter `'a` is provided to allow any of the returned values to returned by reference.
/// This is to workaround the lack of higher kinded lifetime parameters.
/// This can be ignored if this is not needed, however.
pub trait Files<'a> {
    type FileId: 'a + Copy + PartialEq;
    type Origin: 'a + std::fmt::Display;
    type Source: 'a + AsRef<str>;

    /// The origin of a file.
    fn origin(&'a self, id: Self::FileId) -> Option<Self::Origin>;

    /// The source of a file.
    fn source(&'a self, id: Self::FileId) -> Option<Self::Source>;

    /// The index of the line at the given byte index.
    fn line_index(&'a self, id: Self::FileId, byte_index: usize) -> Option<usize>;

    /// The user-facing line number at the given line index.
    ///
    /// This can be useful for implementing something like the
    /// [C preprocessor's `#line` macro][line-macro],
    /// but is usually 1-indexed from the beginning of the file.
    ///
    /// [line-macro]: https://en.cppreference.com/w/c/preprocessor/line
    #[allow(unused_variables)]
    fn line_number(&'a self, id: Self::FileId, line_index: usize) -> Option<usize> {
        Some(line_index + 1)
    }

    /// The index of the line at the given byte index.
    fn line_range(&'a self, id: Self::FileId, line_index: usize) -> Option<Range<usize>>;
}

/// The column index at the given byte index in the source file.
/// This is the number of characters to the given byte index.
///
/// If the byte index is smaller than the start of the line, then `0` is returned.
/// If the byte index is past the end of the line, the column index of the last
/// character `+ 1` is returned.
///
/// # Example
///
/// ```rust
/// use codespan_reporting::files;
///
/// let source = "\n\n🗻∈🌏\n\n";
///
/// assert_eq!(files::column_index(source, 0..1, 0), 0);
/// assert_eq!(files::column_index(source, 2..13, 0), 0);
/// assert_eq!(files::column_index(source, 2..13, 2 + 0), 0);
/// assert_eq!(files::column_index(source, 2..13, 2 + 1), 0);
/// assert_eq!(files::column_index(source, 2..13, 2 + 4), 1);
/// assert_eq!(files::column_index(source, 2..13, 2 + 8), 2);
/// assert_eq!(files::column_index(source, 2..13, 2 + 10), 2);
/// assert_eq!(files::column_index(source, 2..13, 2 + 11), 3);
/// assert_eq!(files::column_index(source, 2..13, 2 + 12), 3);
/// ```
pub fn column_index(source: &str, line_range: Range<usize>, byte_index: usize) -> usize {
    let end_index = std::cmp::min(byte_index, std::cmp::min(line_range.end, source.len()));

    (line_range.start..end_index)
        .filter(|byte_index| source.is_char_boundary(byte_index + 1))
        .count()
}

/// The 1-indexed column number at the given byte index.
///
/// # Example
///
/// ```rust
/// use codespan_reporting::files;
///
/// let source = "\n\n🗻∈🌏";
/// let line_range = 2..13;
///
/// assert_eq!(files::column_number(source, 0..1, 0), 1);
/// assert_eq!(files::column_number(source, 2..13, 0), 1);
/// assert_eq!(files::column_number(source, 2..13, 2 + 0), 1);
/// assert_eq!(files::column_number(source, 2..13, 2 + 1), 1);
/// assert_eq!(files::column_number(source, 2..13, 2 + 4), 2);
/// assert_eq!(files::column_number(source, 2..13, 2 + 8), 3);
/// assert_eq!(files::column_number(source, 2..13, 2 + 10), 3);
/// assert_eq!(files::column_number(source, 2..13, 2 + 11), 4);
/// assert_eq!(files::column_number(source, 2..13, 2 + 12), 4);
/// ```
pub fn column_number(source: &str, line_range: Range<usize>, byte_index: usize) -> usize {
    column_index(source, line_range, byte_index) + 1
}

/// A single source file.
///
/// This is useful for simple language tests, but it might be worth creating a
/// custom implementation when a language scales beyond a certain size.
#[derive(Debug, Clone)]
pub struct SimpleFile<Origin, Source> {
    /// The origin of the file.
    origin: Origin,
    /// The source code of the file.
    source: Source,
    /// The starting byte indices in the source code.
    line_starts: Vec<usize>,
}

/// Return the starting byte index of each line in the source string.
///
/// This can make it easier to implement new `Files` implementations.
///
/// # Example
///
/// ```rust
/// use codespan_reporting::files;
///
/// let source = "foo\nbar\r\n\nbaz";
/// let line_starts: Vec<_> = files::line_starts(source).collect();
///
/// assert_eq!(
///     line_starts,
///     [
///         0,  // "foo\n"
///         4,  // "bar\r\n"
///         9,  // ""
///         10, // "baz"
///     ],
/// );
///
/// fn line_index(line_starts: &[usize], byte_index: usize) -> Option<usize> {
///     match line_starts.binary_search(&byte_index) {
///         Ok(line) => Some(line),
///         Err(next_line) => Some(next_line - 1),
///     }
/// }
///
/// assert_eq!(line_index(&line_starts, 5), Some(1));
/// ```
pub fn line_starts<'source>(source: &'source str) -> impl 'source + Iterator<Item = usize> {
    std::iter::once(0).chain(source.match_indices('\n').map(|(i, _)| i + 1))
}

impl<Origin, Source> SimpleFile<Origin, Source>
where
    Origin: std::fmt::Display,
    Source: AsRef<str>,
{
    /// Create a new source file.
    pub fn new(origin: Origin, source: Source) -> SimpleFile<Origin, Source> {
        SimpleFile {
            origin,
            line_starts: line_starts(source.as_ref()).collect(),
            source,
        }
    }

    /// Return the origin of the file.
    pub fn origin(&self) -> &Origin {
        &self.origin
    }

    /// Return the source of the file.
    pub fn source(&self) -> &Source {
        &self.source
    }

    fn line_start(&self, line_index: usize) -> Option<usize> {
        use std::cmp::Ordering;

        match line_index.cmp(&self.line_starts.len()) {
            Ordering::Less => self.line_starts.get(line_index).cloned(),
            Ordering::Equal => Some(self.source.as_ref().len()),
            Ordering::Greater => None,
        }
    }
}

impl<'a, Origin, Source> Files<'a> for SimpleFile<Origin, Source>
where
    Origin: 'a + std::fmt::Display + Clone,
    Source: 'a + AsRef<str>,
{
    type FileId = ();
    type Origin = Origin;
    type Source = &'a str;

    fn origin(&self, (): ()) -> Option<Origin> {
        Some(self.origin.clone())
    }

    fn source(&self, (): ()) -> Option<&str> {
        Some(self.source.as_ref())
    }

    fn line_index(&self, (): (), byte_index: usize) -> Option<usize> {
        match self.line_starts.binary_search(&byte_index) {
            Ok(line) => Some(line),
            Err(next_line) => Some(next_line - 1),
        }
    }

    fn line_range(&self, (): (), line_index: usize) -> Option<Range<usize>> {
        let line_start = self.line_start(line_index)?;
        let next_line_start = self.line_start(line_index + 1)?;

        Some(line_start..next_line_start)
    }
}

/// A file database that can store multiple source files.
///
/// This is useful for simple language tests, but it might be worth creating a
/// custom implementation when a language scales beyond a certain size.
#[derive(Debug, Clone)]
pub struct SimpleFiles<Origin, Source> {
    files: Vec<SimpleFile<Origin, Source>>,
}

impl<Origin, Source> SimpleFiles<Origin, Source>
where
    Origin: std::fmt::Display,
    Source: AsRef<str>,
{
    /// Create a new files database.
    pub fn new() -> SimpleFiles<Origin, Source> {
        SimpleFiles { files: Vec::new() }
    }

    /// Add a file to the database, returning the handle that can be used to
    /// refer to it again.
    pub fn add(&mut self, origin: Origin, source: Source) -> usize {
        let file_id = self.files.len();
        self.files.push(SimpleFile::new(origin, source));
        file_id
    }

    /// Get the file corresponding to the given id.
    pub fn get(&self, file_id: usize) -> Option<&SimpleFile<Origin, Source>> {
        self.files.get(file_id)
    }
}

impl<'a, Origin, Source> Files<'a> for SimpleFiles<Origin, Source>
where
    Origin: 'a + std::fmt::Display + Clone,
    Source: 'a + AsRef<str>,
{
    type FileId = usize;
    type Origin = Origin;
    type Source = &'a str;

    fn origin(&self, file_id: usize) -> Option<Origin> {
        Some(self.get(file_id)?.origin().clone())
    }

    fn source(&self, file_id: usize) -> Option<&str> {
        Some(self.get(file_id)?.source().as_ref())
    }

    fn line_index(&self, file_id: usize, byte_index: usize) -> Option<usize> {
        self.get(file_id)?.line_index((), byte_index)
    }

    fn line_range(&self, file_id: usize, line_index: usize) -> Option<Range<usize>> {
        self.get(file_id)?.line_range((), line_index)
    }
}

#[cfg(test)]
mod test {
    use super::*;

    const TEST_SOURCE: &str = "foo\nbar\r\n\nbaz";

    #[test]
    fn line_starts() {
        let file = SimpleFile::new("test", TEST_SOURCE);

        assert_eq!(
            file.line_starts,
            [
                0,  // "foo\n"
                4,  // "bar\r\n"
                9,  // ""
                10, // "baz"
            ],
        );
    }

    #[test]
    fn line_span_sources() {
        let file = SimpleFile::new("test", TEST_SOURCE);

        let line_sources = (0..4)
            .map(|line| {
                let line_range = file.line_range((), line).unwrap();
                &file.source[line_range]
            })
            .collect::<Vec<_>>();

        assert_eq!(line_sources, ["foo\n", "bar\r\n", "\n", "baz"]);
    }
}
<|endoftext|>


<commit_before>//! Initialization logic for a generic instance of the `Allocate` channel allocation trait.

use std::thread;
use std::io::BufRead;
use getopts;
use std::sync::Arc;

use allocator::{Thread, Process, Generic};
use networking::initialize_networking;

/// Possible configurations for the communication infrastructure.
pub enum Configuration {
    /// Use one thread.
    Thread,
    /// Use one process with an indicated number of threads.
    Process(usize),
    /// Expect multiple processes indicated by `(threads, process, host_list, report)`.
    Cluster(usize, usize, Vec<String>, bool)
}

impl Configuration {

    /// Constructs a new configuration by parsing supplied text arguments.
    ///
    /// Most commonly, this uses `std::env::Args()` as the supplied iterator.
    pub fn from_args<I: Iterator<Item=String>>(args: I) -> Result<Configuration,String> {

        let mut opts = getopts::Options::new();
        opts.optopt("w", "threads", "number of per-process worker threads", "NUM");
        opts.optopt("p", "process", "identity of this process", "IDX");
        opts.optopt("n", "processes", "number of processes", "NUM");
        opts.optopt("h", "hostfile", "text file whose lines are process addresses", "FILE");
        opts.optflag("r", "report", "reports connection progress");

        opts.parse(args)
            .map_err(|e| format!("{:?}", e))
            .map(|matches| {

            // let mut config = Configuration::new(1, 0, Vec::new());
            let threads = matches.opt_str("w").map(|x| x.parse().unwrap_or(1)).unwrap_or(1);
            let process = matches.opt_str("p").map(|x| x.parse().unwrap_or(0)).unwrap_or(0);
            let processes = matches.opt_str("n").map(|x| x.parse().unwrap_or(1)).unwrap_or(1);
            let report = matches.opt_present("report");

            assert!(process < processes);

            if processes > 1 {
                let mut addresses = Vec::new();
                if let Some(hosts) = matches.opt_str("h") {
                    let reader = ::std::io::BufReader::new(::std::fs::File::open(hosts.clone()).unwrap());
                    for x in reader.lines().take(processes) {
                        addresses.push(x.unwrap());
                    }
                    if addresses.len() < processes {
                        panic!("could only read {} addresses from {}, but -n: {}", addresses.len(), hosts, processes);
                    }
                }
                else {
                    for index in 0..processes {
                        addresses.push(format!("localhost:{}", 2101 + index));
                    }
                }

                assert!(processes == addresses.len());
                Configuration::Cluster(threads, process, addresses, report)
            }
            else {
                if threads > 1 { Configuration::Process(threads) }
                else { Configuration::Thread }
            }
        })
    }
}

fn create_allocators(config: Configuration, logger: Arc<Fn(::logging::CommsSetup)->::logging::CommsLogger+Send+Sync>) -> Result<Vec<Generic>,String> {
    match config {
        Configuration::Thread => Ok(vec![Generic::Thread(Thread)]),
        Configuration::Process(threads) => Ok(Process::new_vector(threads).into_iter().map(|x| Generic::Process(x)).collect()),
        Configuration::Cluster(threads, process, addresses, report) => {
            if let Ok(stuff) = initialize_networking(addresses, process, threads, report, logger) {
                Ok(stuff.into_iter().map(|x| Generic::Binary(x)).collect())
            }
            else {
                Err("failed to initialize networking".to_owned())
            }
        },
    }
}

/// Initializes communication and executes a distributed computation.
///
/// This method allocates an `allocator::Generic` for each thread, spawns local worker threads,
/// and invokes the supplied function with the allocator.
/// The method returns a `WorkerGuards<T>` which can be `join`ed to retrieve the return values
/// (or errors) of the workers.
///
///
/// #Examples
/// ```
/// // configure for two threads, just one process.
/// let config = timely_communication::Configuration::Process(2);
///
/// // create a source of inactive loggers.
/// let logger = ::std::sync::Arc::new(|_| timely_communication::logging::BufferingLogger::new_inactive());
///
/// // initializes communication, spawns workers
/// let guards = timely_communication::initialize(config, logger, |mut allocator| {
///     println!("worker {} started", allocator.index());
///
///     // allocates pair of senders list and one receiver.
///     let (mut senders, mut receiver, _) = allocator.allocate();
///
///     // send typed data along each channel
///     senders[0].send(format!("hello, {}", 0));
///     senders[1].send(format!("hello, {}", 1));
///
///     // no support for termination notification,
///     // we have to count down ourselves.
///     let mut expecting = 2;
///     while expecting > 0 {
///         if let Some(message) = receiver.recv() {
///             println!("worker {}: received: <{}>", allocator.index(), message);
///             expecting -= 1;
///         }
///     }
///
///     // optionally, return something
///     allocator.index()
/// });
///
/// // computation runs until guards are joined or dropped.
/// if let Ok(guards) = guards {
///     for guard in guards.join() {
///         println!("result: {:?}", guard);
///     }
/// }
/// else { println!("error in computation"); }
/// ```
///
/// The should produce output like:
///
/// ```ignore
/// worker 0 started
/// worker 1 started
/// worker 0: received: <hello, 0>
/// worker 1: received: <hello, 1>
/// worker 0: received: <hello, 0>
/// worker 1: received: <hello, 1>
/// result: Ok(0)
/// result: Ok(1)
/// ```
pub fn initialize<T:Send+'static, F: Fn(Generic)->T+Send+Sync+'static>(
    config: Configuration,
    log_sender: Arc<Fn(::logging::CommsSetup)->::logging::CommsLogger+Send+Sync>,
    func: F,
) -> Result<WorkerGuards<T>,String> {

    let allocators = try!(create_allocators(config, log_sender));
    let logic = Arc::new(func);

    let mut guards = Vec::new();
    for allocator in allocators.into_iter() {
        let clone = logic.clone();
        guards.push(try!(thread::Builder::new()
                            .name(format!("worker thread {}", allocator.index()))
                            .spawn(move || {
                                (*clone)(allocator)
                            })
                            .map_err(|e| format!("{:?}", e))));
    }

    Ok(WorkerGuards { guards: guards })
}

/// Maintains `JoinHandle`s for worker threads.
pub struct WorkerGuards<T:Send+'static> {
    guards: Vec<::std::thread::JoinHandle<T>>
}

impl<T:Send+'static> WorkerGuards<T> {
    /// Waits on the worker threads and returns the results they produce.
    pub fn join(mut self) -> Vec<Result<T,String>> {
        self.guards.drain(..)
                   .map(|guard| guard.join().map_err(|e| format!("{:?}", e)))
                   .collect()
    }
}

impl<T:Send+'static> Drop for WorkerGuards<T> {
    fn drop(&mut self) {
        for guard in self.guards.drain(..) {
            guard.join().unwrap();
        }
    }
}
<commit_msg>timely-communication: Remove redundant closures.<commit_after>//! Initialization logic for a generic instance of the `Allocate` channel allocation trait.

use std::thread;
use std::io::BufRead;
use getopts;
use std::sync::Arc;

use allocator::{Thread, Process, Generic};
use networking::initialize_networking;

/// Possible configurations for the communication infrastructure.
pub enum Configuration {
    /// Use one thread.
    Thread,
    /// Use one process with an indicated number of threads.
    Process(usize),
    /// Expect multiple processes indicated by `(threads, process, host_list, report)`.
    Cluster(usize, usize, Vec<String>, bool)
}

impl Configuration {

    /// Constructs a new configuration by parsing supplied text arguments.
    ///
    /// Most commonly, this uses `std::env::Args()` as the supplied iterator.
    pub fn from_args<I: Iterator<Item=String>>(args: I) -> Result<Configuration,String> {

        let mut opts = getopts::Options::new();
        opts.optopt("w", "threads", "number of per-process worker threads", "NUM");
        opts.optopt("p", "process", "identity of this process", "IDX");
        opts.optopt("n", "processes", "number of processes", "NUM");
        opts.optopt("h", "hostfile", "text file whose lines are process addresses", "FILE");
        opts.optflag("r", "report", "reports connection progress");

        opts.parse(args)
            .map_err(|e| format!("{:?}", e))
            .map(|matches| {

            // let mut config = Configuration::new(1, 0, Vec::new());
            let threads = matches.opt_str("w").map(|x| x.parse().unwrap_or(1)).unwrap_or(1);
            let process = matches.opt_str("p").map(|x| x.parse().unwrap_or(0)).unwrap_or(0);
            let processes = matches.opt_str("n").map(|x| x.parse().unwrap_or(1)).unwrap_or(1);
            let report = matches.opt_present("report");

            assert!(process < processes);

            if processes > 1 {
                let mut addresses = Vec::new();
                if let Some(hosts) = matches.opt_str("h") {
                    let reader = ::std::io::BufReader::new(::std::fs::File::open(hosts.clone()).unwrap());
                    for x in reader.lines().take(processes) {
                        addresses.push(x.unwrap());
                    }
                    if addresses.len() < processes {
                        panic!("could only read {} addresses from {}, but -n: {}", addresses.len(), hosts, processes);
                    }
                }
                else {
                    for index in 0..processes {
                        addresses.push(format!("localhost:{}", 2101 + index));
                    }
                }

                assert!(processes == addresses.len());
                Configuration::Cluster(threads, process, addresses, report)
            }
            else {
                if threads > 1 { Configuration::Process(threads) }
                else { Configuration::Thread }
            }
        })
    }
}

fn create_allocators(config: Configuration, logger: Arc<Fn(::logging::CommsSetup)->::logging::CommsLogger+Send+Sync>) -> Result<Vec<Generic>,String> {
    match config {
        Configuration::Thread => Ok(vec![Generic::Thread(Thread)]),
        Configuration::Process(threads) => Ok(Process::new_vector(threads).into_iter().map(Generic::Process).collect()),
        Configuration::Cluster(threads, process, addresses, report) => {
            if let Ok(stuff) = initialize_networking(addresses, process, threads, report, logger) {
                Ok(stuff.into_iter().map(Generic::Binary).collect())
            }
            else {
                Err("failed to initialize networking".to_owned())
            }
        },
    }
}

/// Initializes communication and executes a distributed computation.
///
/// This method allocates an `allocator::Generic` for each thread, spawns local worker threads,
/// and invokes the supplied function with the allocator.
/// The method returns a `WorkerGuards<T>` which can be `join`ed to retrieve the return values
/// (or errors) of the workers.
///
///
/// #Examples
/// ```
/// // configure for two threads, just one process.
/// let config = timely_communication::Configuration::Process(2);
///
/// // create a source of inactive loggers.
/// let logger = ::std::sync::Arc::new(|_| timely_communication::logging::BufferingLogger::new_inactive());
///
/// // initializes communication, spawns workers
/// let guards = timely_communication::initialize(config, logger, |mut allocator| {
///     println!("worker {} started", allocator.index());
///
///     // allocates pair of senders list and one receiver.
///     let (mut senders, mut receiver, _) = allocator.allocate();
///
///     // send typed data along each channel
///     senders[0].send(format!("hello, {}", 0));
///     senders[1].send(format!("hello, {}", 1));
///
///     // no support for termination notification,
///     // we have to count down ourselves.
///     let mut expecting = 2;
///     while expecting > 0 {
///         if let Some(message) = receiver.recv() {
///             println!("worker {}: received: <{}>", allocator.index(), message);
///             expecting -= 1;
///         }
///     }
///
///     // optionally, return something
///     allocator.index()
/// });
///
/// // computation runs until guards are joined or dropped.
/// if let Ok(guards) = guards {
///     for guard in guards.join() {
///         println!("result: {:?}", guard);
///     }
/// }
/// else { println!("error in computation"); }
/// ```
///
/// The should produce output like:
///
/// ```ignore
/// worker 0 started
/// worker 1 started
/// worker 0: received: <hello, 0>
/// worker 1: received: <hello, 1>
/// worker 0: received: <hello, 0>
/// worker 1: received: <hello, 1>
/// result: Ok(0)
/// result: Ok(1)
/// ```
pub fn initialize<T:Send+'static, F: Fn(Generic)->T+Send+Sync+'static>(
    config: Configuration,
    log_sender: Arc<Fn(::logging::CommsSetup)->::logging::CommsLogger+Send+Sync>,
    func: F,
) -> Result<WorkerGuards<T>,String> {

    let allocators = try!(create_allocators(config, log_sender));
    let logic = Arc::new(func);

    let mut guards = Vec::new();
    for allocator in allocators.into_iter() {
        let clone = logic.clone();
        guards.push(try!(thread::Builder::new()
                            .name(format!("worker thread {}", allocator.index()))
                            .spawn(move || {
                                (*clone)(allocator)
                            })
                            .map_err(|e| format!("{:?}", e))));
    }

    Ok(WorkerGuards { guards: guards })
}

/// Maintains `JoinHandle`s for worker threads.
pub struct WorkerGuards<T:Send+'static> {
    guards: Vec<::std::thread::JoinHandle<T>>
}

impl<T:Send+'static> WorkerGuards<T> {
    /// Waits on the worker threads and returns the results they produce.
    pub fn join(mut self) -> Vec<Result<T,String>> {
        self.guards.drain(..)
                   .map(|guard| guard.join().map_err(|e| format!("{:?}", e)))
                   .collect()
    }
}

impl<T:Send+'static> Drop for WorkerGuards<T> {
    fn drop(&mut self) {
        for guard in self.guards.drain(..) {
            guard.join().unwrap();
        }
    }
}
<|endoftext|>


<commit_before>extern mod xlib;

use std::ptr;
use std::option::Option;
use xlib::xlib;
use xlib::{Display, XSelectInput, XOpenDisplay, Window, XDefaultRootWindow};
use xlib::{XNextEvent, Union__XEvent};

mod x;

fn getDisplay() -> Option<*mut Display> {
    unsafe {
        let null = ptr::null();
        let display = XOpenDisplay(null);

        if ptr::is_null(display) {
            return None
        } else {
            return Some(display);
        }

    }
}

fn getRootWindow(display: *mut Display) -> Window {
    unsafe {
        return XDefaultRootWindow(display);
    }
}

fn setSelectInput(display: *mut Display, window: Window) {
    unsafe {
        let XSelectInputMask = x::SubstructureNotifyMask|x::SubstructureRedirectMask;
        XSelectInput(display, window, XSelectInputMask);
    }
}

fn main() {
    let maybeDisplay = getDisplay();

    match maybeDisplay {
        Some(display) => {
            println!("Got display {}", display);
            let root = getRootWindow(display);
            println!("Got root window {}", root);

            setSelectInput(display, root);
            println!("select input set!");

            unsafe {
                let mut wa = xlib::XSetWindowAttributes {
                    background_pixmap: 0,
                    background_pixel: 0,
                    border_pixmap: 0,
                    border_pixel: 0,
                    bit_gravity: 0,
                    win_gravity: 0,
                    backing_store: 0,
                    backing_planes: 0,
                    backing_pixel: 0,
                    save_under: 0,
                    event_mask: 0,
                    do_not_propagate_mask: 0,
                    override_redirect: 0,
                    colormap: 0,
                    cursor: 0,
                };
                wa.cursor = xlib::XCreateFontCursor(display,
                                                x::cursorfont::XC_left_ptr);
                xlib::XChangeWindowAttributes(display, root,
                    x::CWCursor, &mut wa);
                xlib::XGrabKey(display, 53, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);
                xlib::XGrabKey(display, 45, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);
                xlib::XGrabKey(display, 36, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);
                xlib::XGrabKey(display, 56, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);

                let mut xevent = Union__XEvent{ data: [0, ..24] };
                loop {
                    XNextEvent(display, ptr::to_mut_unsafe_ptr(&mut xevent));
                    let eventType = *xevent._type();

                    match eventType {
                        x::MapRequest => {
                            let screen = xlib::XDefaultScreen(display);
                            let width = xlib::XDisplayWidth(display, screen);
                            let height = xlib::XDisplayHeight(display, screen);

                            println!("Map request screen {} width {} height {}",
                                screen, width, height);

                            let event = *xevent.xmaprequest();
                            xlib::XMapWindow(display, event.window);
                            xlib::XSetInputFocus(display, event.window,
                                x::RevertToPointerRoot, x::CurrentTime);
                            xlib::XMoveResizeWindow(display, event.window, 0, 0,
                                width as u32, height as u32);
                        }
                        x::KeyRelease => {
                            let event = *xevent.xkey();
                            println!("Key {}", event.keycode);
                            if event.keycode == 45 { // k
                                    let mut focus: Window = 0;
                                    let mut revert: i32 = 0;
                                    xlib::XGetInputFocus(display, &mut focus,
                                        &mut revert);
                                    xlib::XDestroyWindow(display, focus);
                            } else if event.keycode == 36 { // enter
                                println!("running terminal")
                                "gnome-terminal".with_c_str(|command| {
                                    std::libc::execvp(command,
                                        ptr::null());
                                });
                            } else if event.keycode == 56 { // b
                                println!("running browser")
                                "firefox".with_c_str(|command| {
                                    std::libc::execvp(command,
                                        ptr::null());
                                });
                            }
                        }
                        _ => {
                            println!("Unknown Event Type {}", eventType);
                        }
                    }
                }
            }
        }
        None => println!("Error getting display")
    }
}
<commit_msg>refactor run command<commit_after>extern mod xlib;

use std::ptr;
use std::option::Option;
use xlib::xlib;
use xlib::{Display, XSelectInput, XOpenDisplay, Window, XDefaultRootWindow};
use xlib::{XNextEvent, Union__XEvent};

mod x;

fn getDisplay() -> Option<*mut Display> {
    unsafe {
        let null = ptr::null();
        let display = XOpenDisplay(null);

        if ptr::is_null(display) {
            return None
        } else {
            return Some(display);
        }

    }
}

fn getRootWindow(display: *mut Display) -> Window {
    unsafe {
        return XDefaultRootWindow(display);
    }
}

fn setSelectInput(display: *mut Display, window: Window) {
    unsafe {
        let XSelectInputMask = x::SubstructureNotifyMask|x::SubstructureRedirectMask;
        XSelectInput(display, window, XSelectInputMask);
    }
}

fn runCommand(command: ~str) {
    println!("running {}", command);
    command.with_c_str(|c_command| unsafe {
        std::libc::execvp(c_command,
        ptr::null());
    });
}

fn main() {
    let maybeDisplay = getDisplay();

    match maybeDisplay {
        Some(display) => {
            println!("Got display {}", display);
            let root = getRootWindow(display);
            println!("Got root window {}", root);

            setSelectInput(display, root);
            println!("select input set!");

            unsafe {
                let mut wa = xlib::XSetWindowAttributes {
                    background_pixmap: 0,
                    background_pixel: 0,
                    border_pixmap: 0,
                    border_pixel: 0,
                    bit_gravity: 0,
                    win_gravity: 0,
                    backing_store: 0,
                    backing_planes: 0,
                    backing_pixel: 0,
                    save_under: 0,
                    event_mask: 0,
                    do_not_propagate_mask: 0,
                    override_redirect: 0,
                    colormap: 0,
                    cursor: 0,
                };
                wa.cursor = xlib::XCreateFontCursor(display,
                                                x::cursorfont::XC_left_ptr);
                xlib::XChangeWindowAttributes(display, root,
                    x::CWCursor, &mut wa);
                xlib::XGrabKey(display, 53, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);
                xlib::XGrabKey(display, 45, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);
                xlib::XGrabKey(display, 36, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);
                xlib::XGrabKey(display, 56, x::Mod4Mask, root, 1, x::GrabModeAsync,
                    x::GrabModeAsync);

                let mut xevent = Union__XEvent{ data: [0, ..24] };
                loop {
                    XNextEvent(display, ptr::to_mut_unsafe_ptr(&mut xevent));
                    let eventType = *xevent._type();

                    match eventType {
                        x::MapRequest => {
                            let screen = xlib::XDefaultScreen(display);
                            let width = xlib::XDisplayWidth(display, screen);
                            let height = xlib::XDisplayHeight(display, screen);

                            println!("Map request screen {} width {} height {}",
                                screen, width, height);

                            let event = *xevent.xmaprequest();
                            xlib::XMapWindow(display, event.window);
                            xlib::XSetInputFocus(display, event.window,
                                x::RevertToPointerRoot, x::CurrentTime);
                            xlib::XMoveResizeWindow(display, event.window, 0, 0,
                                width as u32, height as u32);
                        }
                        x::KeyRelease => {
                            let event = *xevent.xkey();
                            println!("Key {}", event.keycode);
                            if event.keycode == 45 { // k
                                    let mut focus: Window = 0;
                                    let mut revert: i32 = 0;
                                    xlib::XGetInputFocus(display, &mut focus,
                                        &mut revert);
                                    xlib::XDestroyWindow(display, focus);
                            } else if event.keycode == 36 { // enter
                                runCommand(~"gnome-terminal");
                            } else if event.keycode == 56 { // b
                                runCommand(~"firefox");
                            }
                        }
                        _ => {
                            println!("Unknown Event Type {}", eventType);
                        }
                    }
                }
            }
        }
        None => println!("Error getting display")
    }
}
<|endoftext|>