<commit_before>#![feature(phase)]

#[macro_use]
extern crate objc;

use objc::{encode, Id, ToMessage, WeakId};
use objc::runtime::{Class, Object, Sel};

fn main() {
    // Get a class
    let cls = Class::get("NSObject").unwrap();
    println!("NSObject size: {}", cls.instance_size());

    // Inspect its ivars
    println!("NSObject ivars:");
    for ivar in cls.instance_variables().as_slice().iter() {
        println!("{}", ivar.name());
    }

    // Allocate an instance
    let obj: Id<Object> = unsafe {
        let obj = msg_send![cls, alloc];
        let obj = msg_send![obj, init];
        Id::from_retained_ptr(obj)
    };
    println!("NSObject address: {}", obj.as_ptr());

    // Access an ivar of the object
    let isa: *const Class = unsafe {
        *obj.get_ivar("isa")
    };
    println!("NSObject isa: {}", isa);

    // Inspect a method of the class
    let hash_sel = Sel::register("hash");
    let hash_method = cls.instance_method(hash_sel).unwrap();
    let hash_return = hash_method.return_type();
    println!("-[NSObject hash] return type: {}", hash_return.as_slice());
    assert!(encode::<uint>() == hash_return.as_slice());

    // Invoke a method on the object
    let hash = unsafe {
        (msg_send![obj, hash]) as uint
    };
    println!("NSObject hash: {}", hash);

    // Take a weak reference to the object
    let obj = obj.share();
    let weak = WeakId::new(&obj);
    println!("Weak reference is nil? {}", weak.load().is_none());

    println!("Releasing object");
    drop(obj);
    println!("Weak reference is nil? {}", weak.load().is_none());
}
<commit_msg>Updated core example for the latest rust.<commit_after>#[macro_use]
extern crate objc;

use objc::{encode, Id, ToMessage, WeakId};
use objc::runtime::{Class, Object, Sel};

fn main() {
    // Get a class
    let cls = Class::get("NSObject").unwrap();
    println!("NSObject size: {}", cls.instance_size());

    // Inspect its ivars
    println!("NSObject ivars:");
    for ivar in cls.instance_variables().as_slice().iter() {
        println!("{}", ivar.name());
    }

    // Allocate an instance
    let obj: Id<Object> = unsafe {
        let obj = msg_send![cls, alloc];
        let obj = msg_send![obj, init];
        Id::from_retained_ptr(obj)
    };
    println!("NSObject address: {:?}", obj.as_ptr());

    // Access an ivar of the object
    let isa: *const Class = unsafe {
        *obj.get_ivar("isa")
    };
    println!("NSObject isa: {:?}", isa);

    // Inspect a method of the class
    let hash_sel = Sel::register("hash");
    let hash_method = cls.instance_method(hash_sel).unwrap();
    let hash_return = hash_method.return_type();
    println!("-[NSObject hash] return type: {}", hash_return.as_slice());
    assert!(encode::<usize>() == hash_return.as_slice());

    // Invoke a method on the object
    let hash = unsafe {
        (msg_send![obj, hash]) as usize
    };
    println!("NSObject hash: {}", hash);

    // Take a weak reference to the object
    let obj = obj.share();
    let weak = WeakId::new(&obj);
    println!("Weak reference is nil? {}", weak.load().is_none());

    println!("Releasing object");
    drop(obj);
    println!("Weak reference is nil? {}", weak.load().is_none());
}
<|endoftext|>


<commit_before>//! [GET /_matrix/client/r0/sync](https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-sync)

use std::collections::HashMap;

use js_int::UInt;
use ruma_api::{ruma_api, Outgoing};
use ruma_events::{
    collections::{
        all::{RoomEvent, StateEvent},
        only::Event as NonRoomEvent,
    },
    presence::PresenceEvent,
    stripped::AnyStrippedStateEvent,
    EventResult,
};
use ruma_identifiers::RoomId;
use serde::{Deserialize, Serialize};

use crate::r0::filter::FilterDefinition;

ruma_api! {
    metadata {
        description: "Get all new events from all rooms since the last sync or a given point of time.",
        method: GET,
        name: "sync",
        path: "/_matrix/client/r0/sync",
        rate_limited: false,
        requires_authentication: true,
    }

    request {
        /// A filter represented either as its full JSON definition or the ID of a saved filter.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub filter: Option<Filter>,
        /// A point in time to continue a sync from.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub since: Option<String>,
        /// Controls whether to include the full state for all rooms the user is a member of.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub full_state: Option<bool>,
        /// Controls whether the client is automatically marked as online by polling this API.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub set_presence: Option<SetPresence>,
        /// The maximum time to poll in milliseconds before returning this request.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub timeout: Option<UInt>,
    }

    response {
        /// The batch token to supply in the `since` param of the next `/sync` request.
        pub next_batch: String,
        /// Updates to rooms.
        #[wrap_incoming]
        pub rooms: Rooms,
        /// Updates to the presence status of other users.
        #[wrap_incoming]
        pub presence: Presence,
    }
}

/// Whether to set presence or not during sync.
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
pub enum SetPresence {
    /// Do not set the presence of the user calling this API.
    #[serde(rename = "offline")]
    Offline,
}

/// A filter represented either as its full JSON definition or the ID of a saved filter.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[allow(clippy::large_enum_variant)]
#[serde(untagged)]
pub enum Filter {
    // The filter definition needs to be (de)serialized twice because it is a URL-encoded JSON
    // string. Since #[ruma_api(query)] only does the latter and this is a very uncommon
    // setup, we implement it through custom serde logic for this specific enum variant rather than
    // adding another ruma_api attribute.
    //
    // On the deserialization side, because this is an enum with #[serde(untagged)], serde will
    // try the variants in order (https://serde.rs/enum-representations.html). That means because
    // FilterDefinition is the first variant, JSON decoding is attempted first which is almost
    // functionally equivalent to looking at whether the first symbol is a '{' as the spec says.
    // (there are probably some corner cases like leading whitespace)
    #[serde(with = "filter_def_serde")]
    /// A complete filter definition serialized to JSON.
    FilterDefinition(FilterDefinition),
    /// The ID of a filter saved on the server.
    FilterId(String),
}

/// Serialization and deserialization logic for filter definitions.
mod filter_def_serde {
    use serde::{de::Error as _, ser::Error as _, Deserialize, Deserializer, Serializer};

    use crate::r0::filter::FilterDefinition;

    /// Serialization logic for filter definitions.
    pub fn serialize<S>(filter_def: &FilterDefinition, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let string = serde_json::to_string(filter_def).map_err(S::Error::custom)?;
        serializer.serialize_str(&string)
    }

    /// Deserialization logic for filter definitions.
    pub fn deserialize<'de, D>(deserializer: D) -> Result<FilterDefinition, D::Error>
    where
        D: Deserializer<'de>,
    {
        let filter_str = <&str>::deserialize(deserializer)?;

        serde_json::from_str(filter_str).map_err(D::Error::custom)
    }
}

/// Updates to rooms.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Rooms {
    /// The rooms that the user has left or been banned from.
    #[wrap_incoming(LeftRoom)]
    pub leave: HashMap<RoomId, LeftRoom>,
    /// The rooms that the user has joined.
    #[wrap_incoming(JoinedRoom)]
    pub join: HashMap<RoomId, JoinedRoom>,
    /// The rooms that the user has been invited to.
    #[wrap_incoming(InvitedRoom)]
    pub invite: HashMap<RoomId, InvitedRoom>,
}

/// Historical updates to left rooms.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct LeftRoom {
    /// The timeline of messages and state changes in the room up to the point when the user
    /// left.
    #[wrap_incoming]
    pub timeline: Timeline,
    /// The state updates for the room up to the start of the timeline.
    #[wrap_incoming]
    pub state: State,
}

/// Updates to joined rooms.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct JoinedRoom {
    /// Counts of unread notifications for this room.
    pub unread_notifications: UnreadNotificationsCount,
    /// The timeline of messages and state changes in the room.
    #[wrap_incoming]
    pub timeline: Timeline,
    /// Updates to the state, between the time indicated by the `since` parameter, and the start
    /// of the `timeline` (or all state up to the start of the `timeline`, if `since` is not
    /// given, or `full_state` is true).
    #[wrap_incoming]
    pub state: State,
    /// The private data that this user has attached to this room.
    #[wrap_incoming]
    pub account_data: AccountData,
    /// The ephemeral events in the room that aren't recorded in the timeline or state of the
    /// room. e.g. typing.
    #[wrap_incoming]
    pub ephemeral: Ephemeral,
}

/// unread notifications count
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
pub struct UnreadNotificationsCount {
    /// The number of unread notifications for this room with the highlight flag set.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub highlight_count: Option<UInt>,
    /// The total number of unread notifications for this room.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub notification_count: Option<UInt>,
}

/// Events in the room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Timeline {
    /// True if the number of events returned was limited by the `limit` on the filter.
    pub limited: bool,
    /// A token that can be supplied to to the `from` parameter of the
    /// `/rooms/{roomId}/messages` endpoint.
    pub prev_batch: String,
    /// A list of events.
    #[wrap_incoming(RoomEvent with EventResult)]
    pub events: Vec<RoomEvent>,
}

/// State events in the room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct State {
    /// A list of state events.
    #[wrap_incoming(StateEvent with EventResult)]
    pub events: Vec<StateEvent>,
}

/// The private data that this user has attached to this room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct AccountData {
    /// A list of events.
    // TODO: Create
    #[wrap_incoming(NonRoomEvent with EventResult)]
    pub events: Vec<NonRoomEvent>,
}

/// Ephemeral events not recorded in the timeline or state of the room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Ephemeral {
    /// A list of events.
    #[wrap_incoming(NonRoomEvent with EventResult)]
    pub events: Vec<NonRoomEvent>,
}

/// Updates to the rooms that the user has been invited to.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct InvitedRoom {
    /// The state of a room that the user has been invited to.
    #[wrap_incoming]
    pub invite_state: InviteState,
}

/// The state of a room that the user has been invited to.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct InviteState {
    /// A list of state events.
    #[wrap_incoming(AnyStrippedStateEvent with EventResult)]
    pub events: Vec<AnyStrippedStateEvent>,
}

/// Updates to the presence status of other users.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Presence {
    /// A list of events.
    #[wrap_incoming(PresenceEvent with EventResult)]
    pub events: Vec<PresenceEvent>,
}
<commit_msg>sync_events: Parse the to-device events.<commit_after>//! [GET /_matrix/client/r0/sync](https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-sync)

use std::collections::HashMap;

use js_int::UInt;
use ruma_api::{ruma_api, Outgoing};
use ruma_events::{
    collections::{
        all::{RoomEvent, StateEvent},
        only::Event as NonRoomEvent,
    },
    presence::PresenceEvent,
    stripped::AnyStrippedStateEvent,
    to_device::AnyToDeviceEvent,
    EventResult,
};
use ruma_identifiers::RoomId;
use serde::{Deserialize, Serialize};

use crate::r0::filter::FilterDefinition;

ruma_api! {
    metadata {
        description: "Get all new events from all rooms since the last sync or a given point of time.",
        method: GET,
        name: "sync",
        path: "/_matrix/client/r0/sync",
        rate_limited: false,
        requires_authentication: true,
    }

    request {
        /// A filter represented either as its full JSON definition or the ID of a saved filter.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub filter: Option<Filter>,
        /// A point in time to continue a sync from.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub since: Option<String>,
        /// Controls whether to include the full state for all rooms the user is a member of.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub full_state: Option<bool>,
        /// Controls whether the client is automatically marked as online by polling this API.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub set_presence: Option<SetPresence>,
        /// The maximum time to poll in milliseconds before returning this request.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub timeout: Option<UInt>,
    }

    response {
        /// The batch token to supply in the `since` param of the next `/sync` request.
        pub next_batch: String,
        /// Updates to rooms.
        #[wrap_incoming]
        pub rooms: Rooms,
        /// Updates to the presence status of other users.
        #[wrap_incoming]
        pub presence: Presence,
        /// Messages sent dirrectly between devices.
        #[wrap_incoming]
        pub to_device: ToDevice,
    }
}

/// Whether to set presence or not during sync.
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
pub enum SetPresence {
    /// Do not set the presence of the user calling this API.
    #[serde(rename = "offline")]
    Offline,
}

/// A filter represented either as its full JSON definition or the ID of a saved filter.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[allow(clippy::large_enum_variant)]
#[serde(untagged)]
pub enum Filter {
    // The filter definition needs to be (de)serialized twice because it is a URL-encoded JSON
    // string. Since #[ruma_api(query)] only does the latter and this is a very uncommon
    // setup, we implement it through custom serde logic for this specific enum variant rather than
    // adding another ruma_api attribute.
    //
    // On the deserialization side, because this is an enum with #[serde(untagged)], serde will
    // try the variants in order (https://serde.rs/enum-representations.html). That means because
    // FilterDefinition is the first variant, JSON decoding is attempted first which is almost
    // functionally equivalent to looking at whether the first symbol is a '{' as the spec says.
    // (there are probably some corner cases like leading whitespace)
    #[serde(with = "filter_def_serde")]
    /// A complete filter definition serialized to JSON.
    FilterDefinition(FilterDefinition),
    /// The ID of a filter saved on the server.
    FilterId(String),
}

/// Serialization and deserialization logic for filter definitions.
mod filter_def_serde {
    use serde::{de::Error as _, ser::Error as _, Deserialize, Deserializer, Serializer};

    use crate::r0::filter::FilterDefinition;

    /// Serialization logic for filter definitions.
    pub fn serialize<S>(filter_def: &FilterDefinition, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let string = serde_json::to_string(filter_def).map_err(S::Error::custom)?;
        serializer.serialize_str(&string)
    }

    /// Deserialization logic for filter definitions.
    pub fn deserialize<'de, D>(deserializer: D) -> Result<FilterDefinition, D::Error>
    where
        D: Deserializer<'de>,
    {
        let filter_str = <&str>::deserialize(deserializer)?;

        serde_json::from_str(filter_str).map_err(D::Error::custom)
    }
}

/// Updates to rooms.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Rooms {
    /// The rooms that the user has left or been banned from.
    #[wrap_incoming(LeftRoom)]
    pub leave: HashMap<RoomId, LeftRoom>,
    /// The rooms that the user has joined.
    #[wrap_incoming(JoinedRoom)]
    pub join: HashMap<RoomId, JoinedRoom>,
    /// The rooms that the user has been invited to.
    #[wrap_incoming(InvitedRoom)]
    pub invite: HashMap<RoomId, InvitedRoom>,
}

/// Historical updates to left rooms.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct LeftRoom {
    /// The timeline of messages and state changes in the room up to the point when the user
    /// left.
    #[wrap_incoming]
    pub timeline: Timeline,
    /// The state updates for the room up to the start of the timeline.
    #[wrap_incoming]
    pub state: State,
}

/// Updates to joined rooms.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct JoinedRoom {
    /// Counts of unread notifications for this room.
    pub unread_notifications: UnreadNotificationsCount,
    /// The timeline of messages and state changes in the room.
    #[wrap_incoming]
    pub timeline: Timeline,
    /// Updates to the state, between the time indicated by the `since` parameter, and the start
    /// of the `timeline` (or all state up to the start of the `timeline`, if `since` is not
    /// given, or `full_state` is true).
    #[wrap_incoming]
    pub state: State,
    /// The private data that this user has attached to this room.
    #[wrap_incoming]
    pub account_data: AccountData,
    /// The ephemeral events in the room that aren't recorded in the timeline or state of the
    /// room. e.g. typing.
    #[wrap_incoming]
    pub ephemeral: Ephemeral,
}

/// unread notifications count
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
pub struct UnreadNotificationsCount {
    /// The number of unread notifications for this room with the highlight flag set.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub highlight_count: Option<UInt>,
    /// The total number of unread notifications for this room.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub notification_count: Option<UInt>,
}

/// Events in the room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Timeline {
    /// True if the number of events returned was limited by the `limit` on the filter.
    pub limited: bool,
    /// A token that can be supplied to to the `from` parameter of the
    /// `/rooms/{roomId}/messages` endpoint.
    pub prev_batch: String,
    /// A list of events.
    #[wrap_incoming(RoomEvent with EventResult)]
    pub events: Vec<RoomEvent>,
}

/// State events in the room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct State {
    /// A list of state events.
    #[wrap_incoming(StateEvent with EventResult)]
    pub events: Vec<StateEvent>,
}

/// The private data that this user has attached to this room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct AccountData {
    /// A list of events.
    // TODO: Create
    #[wrap_incoming(NonRoomEvent with EventResult)]
    pub events: Vec<NonRoomEvent>,
}

/// Ephemeral events not recorded in the timeline or state of the room.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Ephemeral {
    /// A list of events.
    #[wrap_incoming(NonRoomEvent with EventResult)]
    pub events: Vec<NonRoomEvent>,
}

/// Updates to the rooms that the user has been invited to.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct InvitedRoom {
    /// The state of a room that the user has been invited to.
    #[wrap_incoming]
    pub invite_state: InviteState,
}

/// The state of a room that the user has been invited to.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct InviteState {
    /// A list of state events.
    #[wrap_incoming(AnyStrippedStateEvent with EventResult)]
    pub events: Vec<AnyStrippedStateEvent>,
}

/// Updates to the presence status of other users.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct Presence {
    /// A list of events.
    #[wrap_incoming(PresenceEvent with EventResult)]
    pub events: Vec<PresenceEvent>,
}

/// Messages sent dirrectly between devices.
#[derive(Clone, Debug, Serialize, Outgoing)]
pub struct ToDevice {
    /// A list of to-device events.
    #[wrap_incoming(AnyToDeviceEvent with EventResult)]
    pub events: Vec<AnyToDeviceEvent>,
}
<|endoftext|>


<commit_before>//! This module handles auto-magic editing actions applied together with users
//! edits. For example, if the user typed
//!
//! ```text
//!     foo
//!         .bar()
//!         .baz()
//!     |   // <- cursor is here
//! ```
//!
//! and types `.` next, we want to indent the dot.
//!
//! Language server executes such typing assists synchronously. That is, they
//! block user's typing and should be pretty fast for this reason!

mod on_enter;

use ide_db::{
    base_db::{FilePosition, SourceDatabase},
    RootDatabase,
};
use syntax::{
    algo::find_node_at_offset,
    ast::{self, edit::IndentLevel, AstToken},
    AstNode, SourceFile,
    SyntaxKind::{FIELD_EXPR, METHOD_CALL_EXPR},
    TextRange, TextSize,
};

use text_edit::TextEdit;

use crate::SourceChange;

pub(crate) use on_enter::on_enter;

// Don't forget to add new trigger characters to `server_capabilities` in `caps.rs`.
pub(crate) const TRIGGER_CHARS: &str = ".=>{";

// Feature: On Typing Assists
//
// Some features trigger on typing certain characters:
//
// - typing `let =` tries to smartly add `;` if `=` is followed by an existing expression
// - typing `.` in a chain method call auto-indents
//
// VS Code::
//
// Add the following to `settings.json`:
// [source,json]
// ----
// "editor.formatOnType": true,
// ----
//
// image::https://user-images.githubusercontent.com/48062697/113166163-69758500-923a-11eb-81ee-eb33ec380399.gif[]
// image::https://user-images.githubusercontent.com/48062697/113171066-105c2000-923f-11eb-87ab-f4a263346567.gif[]
pub(crate) fn on_char_typed(
    db: &RootDatabase,
    position: FilePosition,
    char_typed: char,
) -> Option<SourceChange> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    let file = &db.parse(position.file_id).tree();
    assert_eq!(file.syntax().text().char_at(position.offset), Some(char_typed));
    let edit = on_char_typed_inner(file, position.offset, char_typed)?;
    Some(SourceChange::from_text_edit(position.file_id, edit))
}

fn on_char_typed_inner(file: &SourceFile, offset: TextSize, char_typed: char) -> Option<TextEdit> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    match char_typed {
        '.' => on_dot_typed(file, offset),
        '=' => on_eq_typed(file, offset),
        '>' => on_arrow_typed(file, offset),
        '{' => on_opening_brace_typed(file, offset),
        _ => unreachable!(),
    }
}

/// Inserts a closing `}` when the user types an opening `{`, wrapping an existing expression in a
/// block.
fn on_opening_brace_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    assert_eq!(file.syntax().text().char_at(offset), Some('{'));
    let brace_token = file.syntax().token_at_offset(offset).right_biased()?;
    let block = ast::BlockExpr::cast(brace_token.parent()?)?;

    // We expect a block expression enclosing exactly 1 preexisting expression. It can be parsed as
    // either the trailing expr or an ExprStmt.
    let offset = {
        match block.tail_expr() {
            Some(expr) => {
                if block.statements().next().is_some() {
                    return None;
                }
                expr.syntax().text_range().end()
            }
            None => {
                if block.statements().count() != 1 {
                    return None;
                }

                match block.statements().next()? {
                    ast::Stmt::ExprStmt(it) => {
                        // Use the expression span to place `}` before the `;`
                        it.expr()?.syntax().text_range().end()
                    }
                    _ => return None,
                }
            }
        }
    };

    Some(TextEdit::insert(offset, "}".to_string()))
}

/// Returns an edit which should be applied after `=` was typed. Primarily,
/// this works when adding `let =`.
// FIXME: use a snippet completion instead of this hack here.
fn on_eq_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    assert_eq!(file.syntax().text().char_at(offset), Some('='));
    let let_stmt: ast::LetStmt = find_node_at_offset(file.syntax(), offset)?;
    if let_stmt.semicolon_token().is_some() {
        return None;
    }
    if let Some(expr) = let_stmt.initializer() {
        let expr_range = expr.syntax().text_range();
        if expr_range.contains(offset) && offset != expr_range.start() {
            return None;
        }
        if file.syntax().text().slice(offset..expr_range.start()).contains_char('\n') {
            return None;
        }
    } else {
        return None;
    }
    let offset = let_stmt.syntax().text_range().end();
    Some(TextEdit::insert(offset, ";".to_string()))
}

/// Returns an edit which should be applied when a dot ('.') is typed on a blank line, indenting the line appropriately.
fn on_dot_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    assert_eq!(file.syntax().text().char_at(offset), Some('.'));
    let whitespace =
        file.syntax().token_at_offset(offset).left_biased().and_then(ast::Whitespace::cast)?;

    let current_indent = {
        let text = whitespace.text();
        let newline = text.rfind('\n')?;
        &text[newline + 1..]
    };
    let current_indent_len = TextSize::of(current_indent);

    let parent = whitespace.syntax().parent()?;
    // Make sure dot is a part of call chain
    if !matches!(parent.kind(), FIELD_EXPR | METHOD_CALL_EXPR) {
        return None;
    }
    let prev_indent = IndentLevel::from_node(&parent);
    let target_indent = format!("    {}", prev_indent);
    let target_indent_len = TextSize::of(&target_indent);
    if current_indent_len == target_indent_len {
        return None;
    }

    Some(TextEdit::replace(TextRange::new(offset - current_indent_len, offset), target_indent))
}

/// Adds a space after an arrow when `fn foo() { ... }` is turned into `fn foo() -> { ... }`
fn on_arrow_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    let file_text = file.syntax().text();
    assert_eq!(file_text.char_at(offset), Some('>'));
    let after_arrow = offset + TextSize::of('>');
    if file_text.char_at(after_arrow) != Some('{') {
        return None;
    }
    if find_node_at_offset::<ast::RetType>(file.syntax(), offset).is_none() {
        return None;
    }

    Some(TextEdit::insert(after_arrow, " ".to_string()))
}

#[cfg(test)]
mod tests {
    use test_utils::{assert_eq_text, extract_offset};

    use super::*;

    fn do_type_char(char_typed: char, before: &str) -> Option<String> {
        let (offset, mut before) = extract_offset(before);
        let edit = TextEdit::insert(offset, char_typed.to_string());
        edit.apply(&mut before);
        let parse = SourceFile::parse(&before);
        on_char_typed_inner(&parse.tree(), offset, char_typed).map(|it| {
            it.apply(&mut before);
            before.to_string()
        })
    }

    fn type_char(char_typed: char, ra_fixture_before: &str, ra_fixture_after: &str) {
        let actual = do_type_char(char_typed, ra_fixture_before)
            .unwrap_or_else(|| panic!("typing `{}` did nothing", char_typed));

        assert_eq_text!(ra_fixture_after, &actual);
    }

    fn type_char_noop(char_typed: char, before: &str) {
        let file_change = do_type_char(char_typed, before);
        assert!(file_change.is_none())
    }

    #[test]
    fn test_on_eq_typed() {
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        // }
        // ");
        type_char(
            '=',
            r"
fn foo() {
    let foo $0 1 + 1
}
",
            r"
fn foo() {
    let foo = 1 + 1;
}
",
        );
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        //     let bar = 1;
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        //     let bar = 1;
        // }
        // ");
    }

    #[test]
    fn indents_new_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_semi() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0;
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .;
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0;
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_let() {
        type_char(
            '.',
            r#"
fn main() {
    let _ = foo
    $0
    bar()
}
"#,
            r#"
fn main() {
    let _ = foo
        .
    bar()
}
"#,
        );
    }

    #[test]
    fn indents_continued_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .first()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                    $0
            }
            ",
        );
    }

    #[test]
    fn indents_middle_of_chain_call() {
        type_char(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                $0
                    .nth(92)
                    .unwrap();
            }
            ",
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    .
                    .nth(92)
                    .unwrap();
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    $0
                    .nth(92)
                    .unwrap();
            }
            ",
        );
    }

    #[test]
    fn dont_indent_freestanding_dot() {
        type_char_noop(
            '.',
            r"
            fn main() {
                $0
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
            $0
            }
            ",
        );
    }

    #[test]
    fn adds_space_after_return_type() {
        type_char('>', "fn foo() -$0{ 92 }", "fn foo() -> { 92 }")
    }

    #[test]
    fn adds_closing_brace() {
        type_char('{', r"fn f() { match () { _ => $0() } }", r"fn f() { match () { _ => {()} } }");
        type_char('{', r"fn f() { $0(); }", r"fn f() { {()}; }");
        type_char('{', r"fn f() { let x = $0(); }", r"fn f() { let x = {()}; }");
    }
}
<commit_msg>Complete braces more aggressively<commit_after>//! This module handles auto-magic editing actions applied together with users
//! edits. For example, if the user typed
//!
//! ```text
//!     foo
//!         .bar()
//!         .baz()
//!     |   // <- cursor is here
//! ```
//!
//! and types `.` next, we want to indent the dot.
//!
//! Language server executes such typing assists synchronously. That is, they
//! block user's typing and should be pretty fast for this reason!

mod on_enter;

use ide_db::{
    base_db::{FilePosition, SourceDatabase},
    RootDatabase,
};
use syntax::{
    algo::find_node_at_offset,
    ast::{self, edit::IndentLevel, AstToken},
    AstNode, SourceFile,
    SyntaxKind::{FIELD_EXPR, METHOD_CALL_EXPR},
    TextRange, TextSize,
};

use text_edit::TextEdit;

use crate::SourceChange;

pub(crate) use on_enter::on_enter;

// Don't forget to add new trigger characters to `server_capabilities` in `caps.rs`.
pub(crate) const TRIGGER_CHARS: &str = ".=>{";

// Feature: On Typing Assists
//
// Some features trigger on typing certain characters:
//
// - typing `let =` tries to smartly add `;` if `=` is followed by an existing expression
// - typing `.` in a chain method call auto-indents
//
// VS Code::
//
// Add the following to `settings.json`:
// [source,json]
// ----
// "editor.formatOnType": true,
// ----
//
// image::https://user-images.githubusercontent.com/48062697/113166163-69758500-923a-11eb-81ee-eb33ec380399.gif[]
// image::https://user-images.githubusercontent.com/48062697/113171066-105c2000-923f-11eb-87ab-f4a263346567.gif[]
pub(crate) fn on_char_typed(
    db: &RootDatabase,
    position: FilePosition,
    char_typed: char,
) -> Option<SourceChange> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    let file = &db.parse(position.file_id).tree();
    assert_eq!(file.syntax().text().char_at(position.offset), Some(char_typed));
    let edit = on_char_typed_inner(file, position.offset, char_typed)?;
    Some(SourceChange::from_text_edit(position.file_id, edit))
}

fn on_char_typed_inner(file: &SourceFile, offset: TextSize, char_typed: char) -> Option<TextEdit> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    match char_typed {
        '.' => on_dot_typed(file, offset),
        '=' => on_eq_typed(file, offset),
        '>' => on_arrow_typed(file, offset),
        '{' => on_opening_brace_typed(file, offset),
        _ => unreachable!(),
    }
}

/// Inserts a closing `}` when the user types an opening `{`, wrapping an existing expression in a
/// block.
fn on_opening_brace_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    assert_eq!(file.syntax().text().char_at(offset), Some('{'));
    let brace_token = file.syntax().token_at_offset(offset).right_biased()?;
    let block = ast::BlockExpr::cast(brace_token.parent()?)?;

    // We expect a block expression enclosing exactly 1 preexisting expression. It can be parsed as
    // either the trailing expr or an ExprStmt.
    let offset = {
        match block.statements().next() {
            Some(ast::Stmt::ExprStmt(it)) => {
                // Use the expression span to place `}` before the `;`
                it.expr()?.syntax().text_range().end()
            },
            None => block.tail_expr()?.syntax().text_range().end(),
            _ => return None,
        }
    };

    Some(TextEdit::insert(offset, "}".to_string()))
}

/// Returns an edit which should be applied after `=` was typed. Primarily,
/// this works when adding `let =`.
// FIXME: use a snippet completion instead of this hack here.
fn on_eq_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    assert_eq!(file.syntax().text().char_at(offset), Some('='));
    let let_stmt: ast::LetStmt = find_node_at_offset(file.syntax(), offset)?;
    if let_stmt.semicolon_token().is_some() {
        return None;
    }
    if let Some(expr) = let_stmt.initializer() {
        let expr_range = expr.syntax().text_range();
        if expr_range.contains(offset) && offset != expr_range.start() {
            return None;
        }
        if file.syntax().text().slice(offset..expr_range.start()).contains_char('\n') {
            return None;
        }
    } else {
        return None;
    }
    let offset = let_stmt.syntax().text_range().end();
    Some(TextEdit::insert(offset, ";".to_string()))
}

/// Returns an edit which should be applied when a dot ('.') is typed on a blank line, indenting the line appropriately.
fn on_dot_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    assert_eq!(file.syntax().text().char_at(offset), Some('.'));
    let whitespace =
        file.syntax().token_at_offset(offset).left_biased().and_then(ast::Whitespace::cast)?;

    let current_indent = {
        let text = whitespace.text();
        let newline = text.rfind('\n')?;
        &text[newline + 1..]
    };
    let current_indent_len = TextSize::of(current_indent);

    let parent = whitespace.syntax().parent()?;
    // Make sure dot is a part of call chain
    if !matches!(parent.kind(), FIELD_EXPR | METHOD_CALL_EXPR) {
        return None;
    }
    let prev_indent = IndentLevel::from_node(&parent);
    let target_indent = format!("    {}", prev_indent);
    let target_indent_len = TextSize::of(&target_indent);
    if current_indent_len == target_indent_len {
        return None;
    }

    Some(TextEdit::replace(TextRange::new(offset - current_indent_len, offset), target_indent))
}

/// Adds a space after an arrow when `fn foo() { ... }` is turned into `fn foo() -> { ... }`
fn on_arrow_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    let file_text = file.syntax().text();
    assert_eq!(file_text.char_at(offset), Some('>'));
    let after_arrow = offset + TextSize::of('>');
    if file_text.char_at(after_arrow) != Some('{') {
        return None;
    }
    if find_node_at_offset::<ast::RetType>(file.syntax(), offset).is_none() {
        return None;
    }

    Some(TextEdit::insert(after_arrow, " ".to_string()))
}

#[cfg(test)]
mod tests {
    use test_utils::{assert_eq_text, extract_offset};

    use super::*;

    fn do_type_char(char_typed: char, before: &str) -> Option<String> {
        let (offset, mut before) = extract_offset(before);
        let edit = TextEdit::insert(offset, char_typed.to_string());
        edit.apply(&mut before);
        let parse = SourceFile::parse(&before);
        on_char_typed_inner(&parse.tree(), offset, char_typed).map(|it| {
            it.apply(&mut before);
            before.to_string()
        })
    }

    fn type_char(char_typed: char, ra_fixture_before: &str, ra_fixture_after: &str) {
        let actual = do_type_char(char_typed, ra_fixture_before)
            .unwrap_or_else(|| panic!("typing `{}` did nothing", char_typed));

        assert_eq_text!(ra_fixture_after, &actual);
    }

    fn type_char_noop(char_typed: char, before: &str) {
        let file_change = do_type_char(char_typed, before);
        assert!(file_change.is_none())
    }

    #[test]
    fn test_on_eq_typed() {
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        // }
        // ");
        type_char(
            '=',
            r"
fn foo() {
    let foo $0 1 + 1
}
",
            r"
fn foo() {
    let foo = 1 + 1;
}
",
        );
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        //     let bar = 1;
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        //     let bar = 1;
        // }
        // ");
    }

    #[test]
    fn indents_new_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_semi() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0;
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .;
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0;
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_let() {
        type_char(
            '.',
            r#"
fn main() {
    let _ = foo
    $0
    bar()
}
"#,
            r#"
fn main() {
    let _ = foo
        .
    bar()
}
"#,
        );
    }

    #[test]
    fn indents_continued_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .first()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                    $0
            }
            ",
        );
    }

    #[test]
    fn indents_middle_of_chain_call() {
        type_char(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                $0
                    .nth(92)
                    .unwrap();
            }
            ",
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    .
                    .nth(92)
                    .unwrap();
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    $0
                    .nth(92)
                    .unwrap();
            }
            ",
        );
    }

    #[test]
    fn dont_indent_freestanding_dot() {
        type_char_noop(
            '.',
            r"
            fn main() {
                $0
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
            $0
            }
            ",
        );
    }

    #[test]
    fn adds_space_after_return_type() {
        type_char('>', "fn foo() -$0{ 92 }", "fn foo() -> { 92 }")
    }

    #[test]
    fn adds_closing_brace() {
        type_char('{', r"fn f() { match () { _ => $0() } }", r"fn f() { match () { _ => {()} } }");
        type_char('{', r"fn f() { $0(); }", r"fn f() { {()}; }");
        type_char('{', r"fn f() { let x = $0(); }", r"fn f() { let x = {()}; }");
        type_char(
            '{',
            r"
            const S: () = $0();
            fn f() {}
            ",
            r"
            const S: () = {()};
            fn f() {}
            ",
        );
        type_char(
            '{',
            r"
            fn f() {
                match x {
                    0 => $0(),
                    1 => (),
                }
            }",
            r"
            fn f() {
                match x {
                    0 => {()},
                    1 => (),
                }
            }",
        );
    }
}
<|endoftext|>


<commit_before>//! VFS stands for Virtual File System.
//!
//! When doing analysis, we don't want to do any IO, we want to keep all source
//! code in memory. However, the actual source code is stored on disk, so you
//! component which does this.
//! need to get it into the memory in the first place somehow. VFS is the
//!
//! It also is responsible for watching the disk for changes, and for merging
//! editor state (modified, unsaved files) with disk state.
//!
//! VFS is based on a concept of roots: a set of directories on the file system
//! whihc are watched for changes. Typically, there will be a root for each
//! Cargo package.
mod arena;
mod io;

use std::{
    fmt,
    mem,
    thread,
    cmp::Reverse,
    path::{Path, PathBuf},
    ffi::OsStr,
    sync::Arc,
    fs,
};

use rustc_hash::{FxHashMap, FxHashSet};
use relative_path::RelativePathBuf;
use crossbeam_channel::Receiver;
use walkdir::DirEntry;
use thread_worker::{WorkerHandle};

use crate::{
    arena::{ArenaId, Arena},
};

pub use crate::io::TaskResult as VfsTask;

/// `RootFilter` is a predicate that checks if a file can belong to a root. If
/// several filters match a file (nested dirs), the most nested one wins.
struct RootFilter {
    root: PathBuf,
    file_filter: fn(&Path) -> bool,
}

impl RootFilter {
    fn new(root: PathBuf) -> RootFilter {
        RootFilter {
            root,
            file_filter: has_rs_extension,
        }
    }
    /// Check if this root can contain `path`. NB: even if this returns
    /// true, the `path` might actually be conained in some nested root.
    fn can_contain(&self, path: &Path) -> Option<RelativePathBuf> {
        if !(self.file_filter)(path) {
            return None;
        }
        if !(path.starts_with(&self.root)) {
            return None;
        }
        let path = path.strip_prefix(&self.root).unwrap();
        let path = RelativePathBuf::from_path(path).unwrap();
        Some(path)
    }
}

fn has_rs_extension(p: &Path) -> bool {
    p.extension() == Some(OsStr::new("rs"))
}

#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub struct VfsRoot(pub u32);

impl ArenaId for VfsRoot {
    fn from_u32(idx: u32) -> VfsRoot {
        VfsRoot(idx)
    }
    fn to_u32(self) -> u32 {
        self.0
    }
}

#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub struct VfsFile(pub u32);

impl ArenaId for VfsFile {
    fn from_u32(idx: u32) -> VfsFile {
        VfsFile(idx)
    }
    fn to_u32(self) -> u32 {
        self.0
    }
}

struct VfsFileData {
    root: VfsRoot,
    path: RelativePathBuf,
    text: Arc<String>,
}

pub struct Vfs {
    roots: Arena<VfsRoot, RootFilter>,
    files: Arena<VfsFile, VfsFileData>,
    root2files: FxHashMap<VfsRoot, FxHashSet<VfsFile>>,
    pending_changes: Vec<VfsChange>,
    worker: io::Worker,
    worker_handle: WorkerHandle,
}

impl fmt::Debug for Vfs {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str("Vfs { ... }")
    }
}

impl Vfs {
    pub fn new(mut roots: Vec<PathBuf>) -> (Vfs, Vec<VfsRoot>) {
        let (worker, worker_handle) = io::start();

        let mut res = Vfs {
            roots: Arena::default(),
            files: Arena::default(),
            root2files: FxHashMap::default(),
            worker,
            worker_handle,
            pending_changes: Vec::new(),
        };

        // A hack to make nesting work.
        roots.sort_by_key(|it| Reverse(it.as_os_str().len()));
        for (i, path) in roots.iter().enumerate() {
            let root = res.roots.alloc(RootFilter::new(path.clone()));
            res.root2files.insert(root, Default::default());
            let nested = roots[..i]
                .iter()
                .filter(|it| it.starts_with(path))
                .map(|it| it.clone())
                .collect::<Vec<_>>();
            let filter = move |entry: &DirEntry| {
                if entry.file_type().is_file() {
                    has_rs_extension(entry.path())
                } else {
                    nested.iter().all(|it| it != entry.path())
                }
            };
            let task = io::Task {
                root,
                path: path.clone(),
                filter: Box::new(filter),
            };
            res.worker.inp.send(task);
        }
        let roots = res.roots.iter().map(|(id, _)| id).collect();
        (res, roots)
    }

    pub fn root2path(&self, root: VfsRoot) -> PathBuf {
        self.roots[root].root.clone()
    }

    pub fn path2file(&self, path: &Path) -> Option<VfsFile> {
        if let Some((_root, _path, Some(file))) = self.find_root(path) {
            return Some(file);
        }
        None
    }

    pub fn file2path(&self, file: VfsFile) -> PathBuf {
        let rel_path = &self.files[file].path;
        let root_path = &self.roots[self.files[file].root].root;
        rel_path.to_path(root_path)
    }

    pub fn file_for_path(&self, path: &Path) -> Option<VfsFile> {
        if let Some((_root, _path, Some(file))) = self.find_root(path) {
            return Some(file);
        }
        None
    }

    pub fn load(&mut self, path: &Path) -> Option<VfsFile> {
        if let Some((root, rel_path, file)) = self.find_root(path) {
            return if let Some(file) = file {
                Some(file)
            } else {
                let text = fs::read_to_string(path).unwrap_or_default();
                let text = Arc::new(text);
                let file = self.add_file(root, rel_path.clone(), Arc::clone(&text));
                let change = VfsChange::AddFile {
                    file,
                    text,
                    root,
                    path: rel_path,
                };
                self.pending_changes.push(change);
                Some(file)
            };
        }
        None
    }

    pub fn task_receiver(&self) -> &Receiver<io::TaskResult> {
        &self.worker.out
    }

    pub fn handle_task(&mut self, task: io::TaskResult) {
        let mut files = Vec::new();
        for (path, text) in task.files {
            let text = Arc::new(text);
            let file = self.add_file(task.root, path.clone(), Arc::clone(&text));
            files.push((file, path, text));
        }
        let change = VfsChange::AddRoot {
            root: task.root,
            files,
        };
        self.pending_changes.push(change);
    }

    pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option<VfsFile> {
        let mut res = None;
        if let Some((root, path, file)) = self.find_root(path) {
            let text = Arc::new(text);
            let change = if let Some(file) = file {
                res = Some(file);
                self.change_file(file, Arc::clone(&text));
                VfsChange::ChangeFile { file, text }
            } else {
                let file = self.add_file(root, path.clone(), Arc::clone(&text));
                res = Some(file);
                VfsChange::AddFile {
                    file,
                    text,
                    root,
                    path,
                }
            };
            self.pending_changes.push(change);
        }
        res
    }

    pub fn change_file_overlay(&mut self, path: &Path, new_text: String) {
        if let Some((_root, _path, file)) = self.find_root(path) {
            let file = file.expect("can't change a file which wasn't added");
            let text = Arc::new(new_text);
            self.change_file(file, Arc::clone(&text));
            let change = VfsChange::ChangeFile { file, text };
            self.pending_changes.push(change);
        }
    }

    pub fn remove_file_overlay(&mut self, path: &Path) -> Option<VfsFile> {
        let mut res = None;
        if let Some((root, path, file)) = self.find_root(path) {
            let file = file.expect("can't remove a file which wasn't added");
            res = Some(file);
            let full_path = path.to_path(&self.roots[root].root);
            let change = if let Ok(text) = fs::read_to_string(&full_path) {
                let text = Arc::new(text);
                self.change_file(file, Arc::clone(&text));
                VfsChange::ChangeFile { file, text }
            } else {
                self.remove_file(file);
                VfsChange::RemoveFile { root, file, path }
            };
            self.pending_changes.push(change);
        }
        res
    }

    pub fn commit_changes(&mut self) -> Vec<VfsChange> {
        mem::replace(&mut self.pending_changes, Vec::new())
    }

    /// Sutdown the VFS and terminate the background watching thread.
    pub fn shutdown(self) -> thread::Result<()> {
        let _ = self.worker.shutdown();
        self.worker_handle.shutdown()
    }

    fn add_file(&mut self, root: VfsRoot, path: RelativePathBuf, text: Arc<String>) -> VfsFile {
        let data = VfsFileData { root, path, text };
        let file = self.files.alloc(data);
        self.root2files.get_mut(&root).unwrap().insert(file);
        file
    }

    fn change_file(&mut self, file: VfsFile, new_text: Arc<String>) {
        self.files[file].text = new_text;
    }

    fn remove_file(&mut self, file: VfsFile) {
        //FIXME: use arena with removal
        self.files[file].text = Default::default();
        self.files[file].path = Default::default();
        let root = self.files[file].root;
        let removed = self.root2files.get_mut(&root).unwrap().remove(&file);
        assert!(removed);
    }

    fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option<VfsFile>)> {
        let (root, path) = self
            .roots
            .iter()
            .find_map(|(root, data)| data.can_contain(path).map(|it| (root, it)))?;
        let file = self.root2files[&root]
            .iter()
            .map(|&it| it)
            .find(|&file| self.files[file].path == path);
        Some((root, path, file))
    }
}

#[derive(Debug, Clone)]
pub enum VfsChange {
    AddRoot {
        root: VfsRoot,
        files: Vec<(VfsFile, RelativePathBuf, Arc<String>)>,
    },
    AddFile {
        root: VfsRoot,
        file: VfsFile,
        path: RelativePathBuf,
        text: Arc<String>,
    },
    RemoveFile {
        root: VfsRoot,
        file: VfsFile,
        path: RelativePathBuf,
    },
    ChangeFile {
        file: VfsFile,
        text: Arc<String>,
    },
}
<commit_msg>File can be opened before the root is scanned<commit_after>//! VFS stands for Virtual File System.
//!
//! When doing analysis, we don't want to do any IO, we want to keep all source
//! code in memory. However, the actual source code is stored on disk, so you
//! component which does this.
//! need to get it into the memory in the first place somehow. VFS is the
//!
//! It also is responsible for watching the disk for changes, and for merging
//! editor state (modified, unsaved files) with disk state.
//!
//! VFS is based on a concept of roots: a set of directories on the file system
//! whihc are watched for changes. Typically, there will be a root for each
//! Cargo package.
mod arena;
mod io;

use std::{
    fmt,
    mem,
    thread,
    cmp::Reverse,
    path::{Path, PathBuf},
    ffi::OsStr,
    sync::Arc,
    fs,
};

use rustc_hash::{FxHashMap, FxHashSet};
use relative_path::RelativePathBuf;
use crossbeam_channel::Receiver;
use walkdir::DirEntry;
use thread_worker::{WorkerHandle};

use crate::{
    arena::{ArenaId, Arena},
};

pub use crate::io::TaskResult as VfsTask;

/// `RootFilter` is a predicate that checks if a file can belong to a root. If
/// several filters match a file (nested dirs), the most nested one wins.
struct RootFilter {
    root: PathBuf,
    file_filter: fn(&Path) -> bool,
}

impl RootFilter {
    fn new(root: PathBuf) -> RootFilter {
        RootFilter {
            root,
            file_filter: has_rs_extension,
        }
    }
    /// Check if this root can contain `path`. NB: even if this returns
    /// true, the `path` might actually be conained in some nested root.
    fn can_contain(&self, path: &Path) -> Option<RelativePathBuf> {
        if !(self.file_filter)(path) {
            return None;
        }
        if !(path.starts_with(&self.root)) {
            return None;
        }
        let path = path.strip_prefix(&self.root).unwrap();
        let path = RelativePathBuf::from_path(path).unwrap();
        Some(path)
    }
}

fn has_rs_extension(p: &Path) -> bool {
    p.extension() == Some(OsStr::new("rs"))
}

#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub struct VfsRoot(pub u32);

impl ArenaId for VfsRoot {
    fn from_u32(idx: u32) -> VfsRoot {
        VfsRoot(idx)
    }
    fn to_u32(self) -> u32 {
        self.0
    }
}

#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub struct VfsFile(pub u32);

impl ArenaId for VfsFile {
    fn from_u32(idx: u32) -> VfsFile {
        VfsFile(idx)
    }
    fn to_u32(self) -> u32 {
        self.0
    }
}

struct VfsFileData {
    root: VfsRoot,
    path: RelativePathBuf,
    text: Arc<String>,
}

pub struct Vfs {
    roots: Arena<VfsRoot, RootFilter>,
    files: Arena<VfsFile, VfsFileData>,
    root2files: FxHashMap<VfsRoot, FxHashSet<VfsFile>>,
    pending_changes: Vec<VfsChange>,
    worker: io::Worker,
    worker_handle: WorkerHandle,
}

impl fmt::Debug for Vfs {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str("Vfs { ... }")
    }
}

impl Vfs {
    pub fn new(mut roots: Vec<PathBuf>) -> (Vfs, Vec<VfsRoot>) {
        let (worker, worker_handle) = io::start();

        let mut res = Vfs {
            roots: Arena::default(),
            files: Arena::default(),
            root2files: FxHashMap::default(),
            worker,
            worker_handle,
            pending_changes: Vec::new(),
        };

        // A hack to make nesting work.
        roots.sort_by_key(|it| Reverse(it.as_os_str().len()));
        for (i, path) in roots.iter().enumerate() {
            let root = res.roots.alloc(RootFilter::new(path.clone()));
            res.root2files.insert(root, Default::default());
            let nested = roots[..i]
                .iter()
                .filter(|it| it.starts_with(path))
                .map(|it| it.clone())
                .collect::<Vec<_>>();
            let filter = move |entry: &DirEntry| {
                if entry.file_type().is_file() {
                    has_rs_extension(entry.path())
                } else {
                    nested.iter().all(|it| it != entry.path())
                }
            };
            let task = io::Task {
                root,
                path: path.clone(),
                filter: Box::new(filter),
            };
            res.worker.inp.send(task);
        }
        let roots = res.roots.iter().map(|(id, _)| id).collect();
        (res, roots)
    }

    pub fn root2path(&self, root: VfsRoot) -> PathBuf {
        self.roots[root].root.clone()
    }

    pub fn path2file(&self, path: &Path) -> Option<VfsFile> {
        if let Some((_root, _path, Some(file))) = self.find_root(path) {
            return Some(file);
        }
        None
    }

    pub fn file2path(&self, file: VfsFile) -> PathBuf {
        let rel_path = &self.files[file].path;
        let root_path = &self.roots[self.files[file].root].root;
        rel_path.to_path(root_path)
    }

    pub fn file_for_path(&self, path: &Path) -> Option<VfsFile> {
        if let Some((_root, _path, Some(file))) = self.find_root(path) {
            return Some(file);
        }
        None
    }

    pub fn load(&mut self, path: &Path) -> Option<VfsFile> {
        if let Some((root, rel_path, file)) = self.find_root(path) {
            return if let Some(file) = file {
                Some(file)
            } else {
                let text = fs::read_to_string(path).unwrap_or_default();
                let text = Arc::new(text);
                let file = self.add_file(root, rel_path.clone(), Arc::clone(&text));
                let change = VfsChange::AddFile {
                    file,
                    text,
                    root,
                    path: rel_path,
                };
                self.pending_changes.push(change);
                Some(file)
            };
        }
        None
    }

    pub fn task_receiver(&self) -> &Receiver<io::TaskResult> {
        &self.worker.out
    }

    pub fn handle_task(&mut self, task: io::TaskResult) {
        let mut files = Vec::new();
        // While we were scanning the root in the backgound, a file might have
        // been open in the editor, so we need to account for that.
        let exising = self.root2files[&task.root]
            .iter()
            .map(|&file| (self.files[file].path.clone(), file))
            .collect::<FxHashMap<_, _>>();
        for (path, text) in task.files {
            if let Some(&file) = exising.get(&path) {
                let text = Arc::clone(&self.files[file].text);
                files.push((file, path, text));
                continue;
            }
            let text = Arc::new(text);
            let file = self.add_file(task.root, path.clone(), Arc::clone(&text));
            files.push((file, path, text));
        }

        let change = VfsChange::AddRoot {
            root: task.root,
            files,
        };
        self.pending_changes.push(change);
    }

    pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option<VfsFile> {
        let mut res = None;
        if let Some((root, path, file)) = self.find_root(path) {
            let text = Arc::new(text);
            let change = if let Some(file) = file {
                res = Some(file);
                self.change_file(file, Arc::clone(&text));
                VfsChange::ChangeFile { file, text }
            } else {
                let file = self.add_file(root, path.clone(), Arc::clone(&text));
                res = Some(file);
                VfsChange::AddFile {
                    file,
                    text,
                    root,
                    path,
                }
            };
            self.pending_changes.push(change);
        }
        res
    }

    pub fn change_file_overlay(&mut self, path: &Path, new_text: String) {
        if let Some((_root, _path, file)) = self.find_root(path) {
            let file = file.expect("can't change a file which wasn't added");
            let text = Arc::new(new_text);
            self.change_file(file, Arc::clone(&text));
            let change = VfsChange::ChangeFile { file, text };
            self.pending_changes.push(change);
        }
    }

    pub fn remove_file_overlay(&mut self, path: &Path) -> Option<VfsFile> {
        let mut res = None;
        if let Some((root, path, file)) = self.find_root(path) {
            let file = file.expect("can't remove a file which wasn't added");
            res = Some(file);
            let full_path = path.to_path(&self.roots[root].root);
            let change = if let Ok(text) = fs::read_to_string(&full_path) {
                let text = Arc::new(text);
                self.change_file(file, Arc::clone(&text));
                VfsChange::ChangeFile { file, text }
            } else {
                self.remove_file(file);
                VfsChange::RemoveFile { root, file, path }
            };
            self.pending_changes.push(change);
        }
        res
    }

    pub fn commit_changes(&mut self) -> Vec<VfsChange> {
        mem::replace(&mut self.pending_changes, Vec::new())
    }

    /// Sutdown the VFS and terminate the background watching thread.
    pub fn shutdown(self) -> thread::Result<()> {
        let _ = self.worker.shutdown();
        self.worker_handle.shutdown()
    }

    fn add_file(&mut self, root: VfsRoot, path: RelativePathBuf, text: Arc<String>) -> VfsFile {
        let data = VfsFileData { root, path, text };
        let file = self.files.alloc(data);
        self.root2files.get_mut(&root).unwrap().insert(file);
        file
    }

    fn change_file(&mut self, file: VfsFile, new_text: Arc<String>) {
        self.files[file].text = new_text;
    }

    fn remove_file(&mut self, file: VfsFile) {
        //FIXME: use arena with removal
        self.files[file].text = Default::default();
        self.files[file].path = Default::default();
        let root = self.files[file].root;
        let removed = self.root2files.get_mut(&root).unwrap().remove(&file);
        assert!(removed);
    }

    fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option<VfsFile>)> {
        let (root, path) = self
            .roots
            .iter()
            .find_map(|(root, data)| data.can_contain(path).map(|it| (root, it)))?;
        let file = self.root2files[&root]
            .iter()
            .map(|&it| it)
            .find(|&file| self.files[file].path == path);
        Some((root, path, file))
    }
}

#[derive(Debug, Clone)]
pub enum VfsChange {
    AddRoot {
        root: VfsRoot,
        files: Vec<(VfsFile, RelativePathBuf, Arc<String>)>,
    },
    AddFile {
        root: VfsRoot,
        file: VfsFile,
        path: RelativePathBuf,
        text: Arc<String>,
    },
    RemoveFile {
        root: VfsRoot,
        file: VfsFile,
        path: RelativePathBuf,
    },
    ChangeFile {
        file: VfsFile,
        text: Arc<String>,
    },
}
<|endoftext|>


<commit_before>#[cfg(test)]
extern crate assert_cli;
extern crate boxcars;
extern crate failure;
extern crate globset;
extern crate rayon;
extern crate serde_json;
#[macro_use]
extern crate structopt;

use boxcars::{CrcCheck, NetworkParse, ParserBuilder, Replay};
use failure::{err_msg, Error, ResultExt};
use globset::Glob;
use rayon::prelude::*;
use std::fs::{File, OpenOptions};
use std::io::prelude::*;
use std::io::{self, BufWriter};
use std::path::Path;
use structopt::StructOpt;

#[derive(StructOpt, Debug, Clone, PartialEq)]
#[structopt(name = "rrrocket",
            about = "Parses Rocket League replay files and outputs JSON with decoded information")]
struct Opt {
    #[structopt(short = "c", long = "crc-check",
                help = "forces a crc check for corruption even when replay was successfully parsed")]
    crc: bool,

    #[structopt(short = "n", long = "network-parse",
                help = "parses the network data of a replay instead of skipping it")]
    body: bool,

    #[structopt(short = "m", long = "multiple",
                help = "parse multiple replays, instead of writing JSON to stdout, write to a sibling JSON file")]
    multiple: bool,

    #[structopt(long = "dry-run", help = "parses but does not write JSON output")]
    dry_run: bool,

    #[structopt(help = "Rocket League replay files")]
    input: Vec<String>,
}

fn read_file(input: &str) -> Result<Vec<u8>, Error> {
    let mut f = File::open(input)
        .with_context(|_e| format!("Could not open rocket league file: {}", input))?;
    let mut buffer = vec![];
    f.read_to_end(&mut buffer)
        .with_context(|_e| format!("Could not read rocket league file: {}", input))?;
    Ok(buffer)
}

fn parse_replay<'a>(opt: &Opt, data: &'a [u8]) -> Result<Replay<'a>, Error> {
    ParserBuilder::new(&data[..])
        .with_crc_check(if opt.crc {
            CrcCheck::Always
        } else {
            CrcCheck::OnError
        })
        .with_network_parse(if opt.body {
            NetworkParse::Always
        } else {
            NetworkParse::Never
        })
        .parse()
}

/// Each file argument that we get could be a directory so we need to expand that directory and
/// find all the *.replay files. File arguments turn into single element vectors.
fn expand_paths(files: &[String]) -> Result<Vec<Vec<String>>, Error> {
    let glob = Glob::new("*.replay")?.compile_matcher();

    files
        .iter()
        .map(|x| {
            let p = Path::new(x);
            if p.is_dir() {
                // If the commandline argument is a directory we look for all files that match
                // *.replay. A file that does not match the pattern because of an error reading the
                // directory / file will not be filtered and will cause the error to bubble up. In
                // the future, we could get smart and ignore directories / files we don't have
                // permission that wouldn't match the pattern anyways
                let files: Result<Vec<_>, _> = p.read_dir()?
                    .filter_map(|entry| {
                        match entry {
                            Ok(y) => {
                                if glob.is_match(y.path()) {
                                    // Force UTF-8. There is a special place in the fourth circle
                                    // of inferno for people who rename their rocket league replays
                                    // to not contain UTF-8. We won't panic, but will cause an
                                    // error when the file is attempted to be read.
                                    Some(Ok(y.path().to_string_lossy().into_owned()))
                                } else {
                                    None
                                }
                            }
                            Err(e) => Some(Err(e)),
                        }
                    })
                    .collect();
                Ok(files?)
            } else {
                Ok(vec![x.clone()])
            }
        })
        .collect()
}

fn parse_multiple_replays(opt: &Opt) -> Result<(), Error> {
    let res: Result<Vec<()>, Error> = expand_paths(&opt.input)?
        .into_iter()
        .flat_map(|x| x)
        .collect::<Vec<_>>()
        .par_iter()
        .map(|file| {
            let data = read_file(file)?;
            let replay = parse_replay(opt, &data[..])
                .with_context(|_e| format!("Could not parse: {}", file))?;

            if !opt.dry_run {
                let outfile = format!("{}.json", file);
                let fout = OpenOptions::new()
                    .write(true)
                    .create(true)
                    .truncate(true)
                    .open(&outfile)
                    .with_context(|_e| format!("Could not open json output file: {}", outfile))?;

                let mut writer = BufWriter::new(fout);
                serde_json::to_writer(&mut writer, &replay)
                    .with_context(|_e| format!("Could not serialize replay {}", file))?;
            }
            Ok(())
        })
        .collect();
    res?;
    Ok(())
}

fn run() -> Result<(), Error> {
    let opt = Opt::from_args();
    if opt.multiple {
        parse_multiple_replays(&opt)
    } else if opt.input.len() != 1 {
        Err(err_msg(
            "Expected one input file if --multiple is not specified",
        ))
    } else {
        let file = &opt.input[0];
        let data = read_file(file)?;
        let replay = parse_replay(&opt, &data[..]).context("Could not parse replay")?;
        if !opt.dry_run {
            serde_json::to_writer(&mut io::stdout(), &replay)
                .context("Could not serialize replay")?;
        }
        Ok(())
    }
}

fn main() {
    if let Err(ref e) = run() {
        let mut stderr = io::stderr();
        for fail in e.causes() {
            let _ = writeln!(stderr, "{}", fail);
        }

        ::std::process::exit(1);
    }
}

#[cfg(test)]
mod tests {
    use assert_cli::Assert;
    use std::io::Write;

    #[test]
    fn test_error_output() {
        let mut w = Vec::new();
        writeln!(
            &mut w,
            "Could not open rocket league file: assets/fuzz-string-too-long.replay"
        ).unwrap();

        Assert::cargo_binary("rrrocket")
            .with_args(&[
                "-n",
                "-c",
                "--dry-run",
                "assets/fuzz-string-too-long.replay",
            ])
            .fails_with(1)
            .stderr()
            .contains(String::from_utf8(w).unwrap())
            .unwrap();
    }

    #[test]
    fn test_error_output2() {
        let mut w = Vec::new();
        writeln!(
            &mut w,
            "Could not parse: ../assets/fuzz-string-too-long.replay"
        ).unwrap();
        writeln!(
            &mut w,
            "Crc mismatch. Expected 3765941959 but received 1825689991"
        ).unwrap();

        Assert::cargo_binary("rrrocket")
            .with_args(&[
                "-n",
                "-c",
                "--dry-run",
                "-m",
                "../assets/fuzz-string-too-long.replay",
            ])
            .fails_with(1)
            .stderr()
            .contains(String::from_utf8(w).unwrap())
            .unwrap();
    }
}
<commit_msg>Rustfmt rrrocket<commit_after>#[cfg(test)]
extern crate assert_cli;
extern crate boxcars;
extern crate failure;
extern crate globset;
extern crate rayon;
extern crate serde_json;
#[macro_use]
extern crate structopt;

use boxcars::{CrcCheck, NetworkParse, ParserBuilder, Replay};
use failure::{err_msg, Error, ResultExt};
use globset::Glob;
use rayon::prelude::*;
use std::fs::{File, OpenOptions};
use std::io::prelude::*;
use std::io::{self, BufWriter};
use std::path::Path;
use structopt::StructOpt;

#[derive(StructOpt, Debug, Clone, PartialEq)]
#[structopt(
    name = "rrrocket",
    about = "Parses Rocket League replay files and outputs JSON with decoded information"
)]
struct Opt {
    #[structopt(
        short = "c",
        long = "crc-check",
        help = "forces a crc check for corruption even when replay was successfully parsed"
    )]
    crc: bool,

    #[structopt(
        short = "n",
        long = "network-parse",
        help = "parses the network data of a replay instead of skipping it"
    )]
    body: bool,

    #[structopt(
        short = "m",
        long = "multiple",
        help = "parse multiple replays, instead of writing JSON to stdout, write to a sibling JSON file"
    )]
    multiple: bool,

    #[structopt(
        long = "dry-run",
        help = "parses but does not write JSON output"
    )]
    dry_run: bool,

    #[structopt(help = "Rocket League replay files")]
    input: Vec<String>,
}

fn read_file(input: &str) -> Result<Vec<u8>, Error> {
    let mut f = File::open(input)
        .with_context(|_e| format!("Could not open rocket league file: {}", input))?;
    let mut buffer = vec![];
    f.read_to_end(&mut buffer)
        .with_context(|_e| format!("Could not read rocket league file: {}", input))?;
    Ok(buffer)
}

fn parse_replay<'a>(opt: &Opt, data: &'a [u8]) -> Result<Replay<'a>, Error> {
    ParserBuilder::new(&data[..])
        .with_crc_check(if opt.crc {
            CrcCheck::Always
        } else {
            CrcCheck::OnError
        }).with_network_parse(if opt.body {
            NetworkParse::Always
        } else {
            NetworkParse::Never
        }).parse()
}

/// Each file argument that we get could be a directory so we need to expand that directory and
/// find all the *.replay files. File arguments turn into single element vectors.
fn expand_paths(files: &[String]) -> Result<Vec<Vec<String>>, Error> {
    let glob = Glob::new("*.replay")?.compile_matcher();

    files
        .iter()
        .map(|x| {
            let p = Path::new(x);
            if p.is_dir() {
                // If the commandline argument is a directory we look for all files that match
                // *.replay. A file that does not match the pattern because of an error reading the
                // directory / file will not be filtered and will cause the error to bubble up. In
                // the future, we could get smart and ignore directories / files we don't have
                // permission that wouldn't match the pattern anyways
                let files: Result<Vec<_>, _> = p
                    .read_dir()?
                    .filter_map(|entry| {
                        match entry {
                            Ok(y) => {
                                if glob.is_match(y.path()) {
                                    // Force UTF-8. There is a special place in the fourth circle
                                    // of inferno for people who rename their rocket league replays
                                    // to not contain UTF-8. We won't panic, but will cause an
                                    // error when the file is attempted to be read.
                                    Some(Ok(y.path().to_string_lossy().into_owned()))
                                } else {
                                    None
                                }
                            }
                            Err(e) => Some(Err(e)),
                        }
                    }).collect();
                Ok(files?)
            } else {
                Ok(vec![x.clone()])
            }
        }).collect()
}

fn parse_multiple_replays(opt: &Opt) -> Result<(), Error> {
    let res: Result<Vec<()>, Error> = expand_paths(&opt.input)?
        .into_iter()
        .flat_map(|x| x)
        .collect::<Vec<_>>()
        .par_iter()
        .map(|file| {
            let data = read_file(file)?;
            let replay = parse_replay(opt, &data[..])
                .with_context(|_e| format!("Could not parse: {}", file))?;

            if !opt.dry_run {
                let outfile = format!("{}.json", file);
                let fout = OpenOptions::new()
                    .write(true)
                    .create(true)
                    .truncate(true)
                    .open(&outfile)
                    .with_context(|_e| format!("Could not open json output file: {}", outfile))?;

                let mut writer = BufWriter::new(fout);
                serde_json::to_writer(&mut writer, &replay)
                    .with_context(|_e| format!("Could not serialize replay {}", file))?;
            }
            Ok(())
        }).collect();
    res?;
    Ok(())
}

fn run() -> Result<(), Error> {
    let opt = Opt::from_args();
    if opt.multiple {
        parse_multiple_replays(&opt)
    } else if opt.input.len() != 1 {
        Err(err_msg(
            "Expected one input file if --multiple is not specified",
        ))
    } else {
        let file = &opt.input[0];
        let data = read_file(file)?;
        let replay = parse_replay(&opt, &data[..]).context("Could not parse replay")?;
        if !opt.dry_run {
            serde_json::to_writer(&mut io::stdout(), &replay)
                .context("Could not serialize replay")?;
        }
        Ok(())
    }
}

fn main() {
    if let Err(ref e) = run() {
        let mut stderr = io::stderr();
        for fail in e.causes() {
            let _ = writeln!(stderr, "{}", fail);
        }

        ::std::process::exit(1);
    }
}

#[cfg(test)]
mod tests {
    use assert_cli::Assert;
    use std::io::Write;

    #[test]
    fn test_error_output() {
        let mut w = Vec::new();
        writeln!(
            &mut w,
            "Could not open rocket league file: assets/fuzz-string-too-long.replay"
        ).unwrap();

        Assert::cargo_binary("rrrocket")
            .with_args(&[
                "-n",
                "-c",
                "--dry-run",
                "assets/fuzz-string-too-long.replay",
            ]).fails_with(1)
            .stderr()
            .contains(String::from_utf8(w).unwrap())
            .unwrap();
    }

    #[test]
    fn test_error_output2() {
        let mut w = Vec::new();
        writeln!(
            &mut w,
            "Could not parse: ../assets/fuzz-string-too-long.replay"
        ).unwrap();
        writeln!(
            &mut w,
            "Crc mismatch. Expected 3765941959 but received 1825689991"
        ).unwrap();

        Assert::cargo_binary("rrrocket")
            .with_args(&[
                "-n",
                "-c",
                "--dry-run",
                "-m",
                "../assets/fuzz-string-too-long.replay",
            ]).fails_with(1)
            .stderr()
            .contains(String::from_utf8(w).unwrap())
            .unwrap();
    }
}
<|endoftext|>


<commit_before>pub mod ngram_generator;
pub mod lucene_asciifold;
pub mod registry;
pub mod tokenizers;
pub mod filters;

use std::cmp;

use unicode_segmentation::UnicodeSegmentation;

use search::term::Term;
use search::token::Token;

#[derive(Debug, PartialEq)]
pub enum Analyzer {
    None,
    Standard,
    EdgeNGram,
}

impl Analyzer {
    pub fn run(&self, input: String) -> Vec<Token> {
        match *self {
            Analyzer::None => vec![Token{term: Term::String(input), position: 1}],
            Analyzer::Standard => {
                // Lowercase
                let input = input.to_lowercase();

                // Convert string to ascii (not standard in Elasticsearch, but Wagtail needs it)
                let input = lucene_asciifold::fold_to_ascii(&input);

                // Tokenise
                let mut position = 0;
                let tokens = input.unicode_words()
                                  .map(|s| {
                                      position += 1;

                                      Token {
                                          term: Term::String(s.to_string()),
                                          position: position,
                                      }
                                  })
                                  .collect();

                tokens
            }
            Analyzer::EdgeNGram => {
                // Analyze with standard analyzer
                let tokens = Analyzer::Standard.run(input);

                // Generate ngrams
                let mut ngrams = Vec::new();
                let min_gram = 2;
                let max_gram = Some(15);

                for token in tokens {
                    if let Term::String(s) = token.term {
                        let max_gram = match max_gram {
                            Some(max_gram) => cmp::min(max_gram, s.len()),
                            None => s.len(),
                        };
                        for last_char in (0 + min_gram)..(0 + max_gram + 1) {
                            // TODO: Currently breaks on non-ascii code points
                            ngrams.push(Token {
                                term: Term::String(s[0..last_char].to_string()),
                                position: token.position,
                            });
                        }
                    }
                }

                ngrams
            }
        }
    }
}
<commit_msg>Update analyzers to use new tokeniser/filter implementations<commit_after>pub mod ngram_generator;
pub mod lucene_asciifold;
pub mod registry;
pub mod tokenizers;
pub mod filters;

use std::cmp;

use unicode_segmentation::UnicodeSegmentation;

use search::term::Term;
use search::token::Token;

use search::analysis::ngram_generator::Edge;
use search::analysis::tokenizers::standard::StandardTokenizer;
use search::analysis::filters::lowercase::LowercaseFilter;
use search::analysis::filters::asciifolding::ASCIIFoldingFilter;
use search::analysis::filters::ngram::NGramFilter;

#[derive(Debug, PartialEq)]
pub enum Analyzer {
    None,
    Standard,
    EdgeNGram,
}

impl Analyzer {
    pub fn run(&self, input: String) -> Vec<Token> {
        match *self {
            Analyzer::None => vec![Token{term: Term::String(input), position: 1}],
            Analyzer::Standard => {
                let tokens = Box::new(StandardTokenizer::new(&input));

                // Lowercase
                let tokens = Box::new(LowercaseFilter::new(tokens));

                // ASCII Folding (not standard in Elasticsearch, but Wagtail needs it)
                let tokens = Box::new(ASCIIFoldingFilter::new(tokens));

                tokens.collect::<Vec<Token>>()
            }
            Analyzer::EdgeNGram => {
                let tokens = Box::new(StandardTokenizer::new(&input));

                // Lowercase
                let tokens = Box::new(LowercaseFilter::new(tokens));

                // ASCII Folding (not standard in Elasticsearch, but Wagtail needs it)
                let tokens = Box::new(ASCIIFoldingFilter::new(tokens));

                // Ngrams
                let tokens = Box::new(NGramFilter::new(tokens, 2, 15, Edge::Left));

                tokens.collect::<Vec<Token>>()
            }
        }
    }
}
<|endoftext|>