<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! Definitions and utilities for working with sinks.
//!
//! Sinks ("well-known flows" in terms of GRR nomenclature) are ever-existing
//! sessions on the GRR server that listen for various kinds of data. They are
//! a way to break away from the usual request-response workflow.
//!
//! For example, sinks can be used to notify the server about agent startup
//! (which is clearly not a response to a particular request) or to transfer
//! file blobs to a specialized storage.

use crate::action;
use crate::session;

/// Data sent to a server not belonging to a particular action call.
///
/// Sometimes the agent should send data to the server that is not connected
/// with a particular flow request. An example can be the agent startup data
/// sent to the server regardless of whether there was a request for it or not.
/// Another example can be file contents that are delivered to the server not
/// as part of the flow results but go to a separate store that has logic for
/// deduplicating data that we already collected in the past.
pub trait Parcel {
    /// Low-level Protocol Buffers type representing the parcel data.
    type Proto: protobuf::Message;

    /// A name of the corresponding RDF class in GRR.
    const RDF_NAME: &'static str;

    /// Converts the parcel to its low-level representation.
    fn into_proto(self) -> Self::Proto;
}

/// A parcel addressed to a particular server-side sink.
pub struct AddressedParcel<P: Parcel> {
    /// Destination of the parcel.
    sink: Sink,
    /// Actual parcel.
    parcel: P,
}

impl<P: Parcel> AddressedParcel<P> {

    /// Sink to which the parcel should be delivered to.
    pub fn sink(&self) -> Sink {
        self.sink
    }
}

impl<P: Parcel> std::convert::TryInto<rrg_proto::jobs::GrrMessage> for AddressedParcel<P> {

    type Error = protobuf::ProtobufError;

    fn try_into(self) -> Result<rrg_proto::jobs::GrrMessage, protobuf::ProtobufError> {
        let mut message = rrg_proto::jobs::GrrMessage::new();
        message.set_session_id(String::from(self.sink.id));
        message.set_field_type(rrg_proto::jobs::GrrMessage_Type::MESSAGE);
        message.set_args_rdf_name(String::from(P::RDF_NAME));

        let proto = self.parcel.into_proto();
        message.set_args(protobuf::Message::write_to_bytes(&proto)?);

        Ok(message)
    }
}

/// Handle to a specific sink.
#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub struct Sink {
    /// An underlying identifier of the sink.
    id: &'static str,
}

impl Sink {

    // TODO: Refactor sinks to use custom `Message` type rather than `Response`
    // and make response always have `request_id` and `response_id` fields.

    /// Adresses a given `parcel` to this sink.
    pub fn address<P: Parcel>(&self, parcel: P) -> AddressedParcel<P> {
        AddressedParcel {
            sink: *self,
            parcel,
        }
    }

    // TODO: Remove the function below once only parcels are used.
    /// Wraps an action response to a sink-specific session response.
    pub fn wrap<R>(&self, response: R) -> session::Response<R>
    where
        R: action::Response,
    {
        session::Response {
            session_id: String::from(self.id),
            request_id: None,
            response_id: None,
            data: response,
        }
    }
}

/// A handle to the sink expecting startup information.
pub const STARTUP: Sink = Sink { id: "/flows/F:Startup" };

/// A handle to the transfer store sink.
pub const TRANSFER_STORE: Sink = Sink { id: "/flows/F:TransferStore" };
<commit_msg>Provide an implementation for empty parcels.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! Definitions and utilities for working with sinks.
//!
//! Sinks ("well-known flows" in terms of GRR nomenclature) are ever-existing
//! sessions on the GRR server that listen for various kinds of data. They are
//! a way to break away from the usual request-response workflow.
//!
//! For example, sinks can be used to notify the server about agent startup
//! (which is clearly not a response to a particular request) or to transfer
//! file blobs to a specialized storage.

use crate::action;
use crate::session;

/// Data sent to a server not belonging to a particular action call.
///
/// Sometimes the agent should send data to the server that is not connected
/// with a particular flow request. An example can be the agent startup data
/// sent to the server regardless of whether there was a request for it or not.
/// Another example can be file contents that are delivered to the server not
/// as part of the flow results but go to a separate store that has logic for
/// deduplicating data that we already collected in the past.
pub trait Parcel {
    /// Low-level Protocol Buffers type representing the parcel data.
    type Proto: protobuf::Message;

    /// A name of the corresponding RDF class in GRR.
    const RDF_NAME: &'static str;

    /// Converts the parcel to its low-level representation.
    fn into_proto(self) -> Self::Proto;
}

impl Parcel for () {

    type Proto = protobuf::well_known_types::Empty;

    // This implementation is intended to be used only in tests and we do not
    // really care about the `RDF_NAME` field there. And since GRR does not have
    // any RDF wrapper for empty type (except maybe `EmptyFlowArgs`, but this is
    // semantically different), we just leave it blank.
    const RDF_NAME: &'static str = "";

    fn into_proto(self) -> protobuf::well_known_types::Empty {
        protobuf::well_known_types::Empty::new()
    }
}

/// A parcel addressed to a particular server-side sink.
pub struct AddressedParcel<P: Parcel> {
    /// Destination of the parcel.
    sink: Sink,
    /// Actual parcel.
    parcel: P,
}

impl<P: Parcel> AddressedParcel<P> {

    /// Sink to which the parcel should be delivered to.
    pub fn sink(&self) -> Sink {
        self.sink
    }
}

impl<P: Parcel> std::convert::TryInto<rrg_proto::jobs::GrrMessage> for AddressedParcel<P> {

    type Error = protobuf::ProtobufError;

    fn try_into(self) -> Result<rrg_proto::jobs::GrrMessage, protobuf::ProtobufError> {
        let mut message = rrg_proto::jobs::GrrMessage::new();
        message.set_session_id(String::from(self.sink.id));
        message.set_field_type(rrg_proto::jobs::GrrMessage_Type::MESSAGE);
        message.set_args_rdf_name(String::from(P::RDF_NAME));

        let proto = self.parcel.into_proto();
        message.set_args(protobuf::Message::write_to_bytes(&proto)?);

        Ok(message)
    }
}

/// Handle to a specific sink.
#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub struct Sink {
    /// An underlying identifier of the sink.
    id: &'static str,
}

impl Sink {

    // TODO: Refactor sinks to use custom `Message` type rather than `Response`
    // and make response always have `request_id` and `response_id` fields.

    /// Adresses a given `parcel` to this sink.
    pub fn address<P: Parcel>(&self, parcel: P) -> AddressedParcel<P> {
        AddressedParcel {
            sink: *self,
            parcel,
        }
    }

    // TODO: Remove the function below once only parcels are used.
    /// Wraps an action response to a sink-specific session response.
    pub fn wrap<R>(&self, response: R) -> session::Response<R>
    where
        R: action::Response,
    {
        session::Response {
            session_id: String::from(self.id),
            request_id: None,
            response_id: None,
            data: response,
        }
    }
}

/// A handle to the sink expecting startup information.
pub const STARTUP: Sink = Sink { id: "/flows/F:Startup" };

/// A handle to the transfer store sink.
pub const TRANSFER_STORE: Sink = Sink { id: "/flows/F:TransferStore" };
<|endoftext|>


<commit_before>mod build_lex_table;
mod build_parse_table;
mod coincident_tokens;
mod item;
mod item_set_builder;
mod minimize_parse_table;
mod token_conflicts;

use self::build_lex_table::build_lex_table;
use self::build_parse_table::build_parse_table;
use self::coincident_tokens::CoincidentTokenIndex;
use self::item::TokenSet;
use self::minimize_parse_table::minimize_parse_table;
use self::token_conflicts::TokenConflictMap;
use crate::error::Result;
use crate::generate::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar};
use crate::generate::nfa::{CharacterSet, NfaCursor};
use crate::generate::rules::{AliasMap, Symbol, SymbolType};
use crate::generate::tables::{LexTable, ParseAction, ParseTable, ParseTableEntry};
use log::info;

pub(crate) fn build_tables(
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    simple_aliases: &AliasMap,
    inlines: &InlinedProductionMap,
    minimize: bool,
    state_ids_to_log: Vec<usize>,
) -> Result<(ParseTable, LexTable, LexTable, Option<Symbol>)> {
    let (mut parse_table, following_tokens) =
        build_parse_table(syntax_grammar, lexical_grammar, inlines, state_ids_to_log)?;
    let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
    let keywords = identify_keywords(
        lexical_grammar,
        &parse_table,
        syntax_grammar.word_token,
        &token_conflict_map,
        &coincident_token_index,
    );
    populate_error_state(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &coincident_token_index,
        &token_conflict_map,
        &keywords,
    );
    populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
    if minimize {
        minimize_parse_table(
            &mut parse_table,
            syntax_grammar,
            lexical_grammar,
            simple_aliases,
            &token_conflict_map,
            &keywords,
        );
    }
    let (main_lex_table, keyword_lex_table) = build_lex_table(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &keywords,
        &coincident_token_index,
        &token_conflict_map,
        minimize,
    );
    Ok((
        parse_table,
        main_lex_table,
        keyword_lex_table,
        syntax_grammar.word_token,
    ))
}

fn populate_error_state(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    coincident_token_index: &CoincidentTokenIndex,
    token_conflict_map: &TokenConflictMap,
    keywords: &TokenSet,
) {
    let state = &mut parse_table.states[0];
    let n = lexical_grammar.variables.len();

    // First identify the *conflict-free tokens*: tokens that do not overlap with
    // any other token in any way, besides matching exactly the same string.
    let conflict_free_tokens: TokenSet = (0..n)
        .into_iter()
        .filter_map(|i| {
            let conflicts_with_other_tokens = (0..n).into_iter().any(|j| {
                j != i
                    && !coincident_token_index.contains(Symbol::terminal(i), Symbol::terminal(j))
                    && token_conflict_map.does_match_shorter_or_longer(i, j)
            });
            if conflicts_with_other_tokens {
                None
            } else {
                info!(
                    "error recovery - token {} has no conflicts",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            }
        })
        .collect();

    let recover_entry = ParseTableEntry {
        reusable: false,
        actions: vec![ParseAction::Recover],
    };

    // Exclude from the error-recovery state any token that conflicts with one of
    // the *conflict-free tokens* identified above.
    for i in 0..n {
        let symbol = Symbol::terminal(i);
        if !conflict_free_tokens.contains(&symbol) && !keywords.contains(&symbol) {
            if syntax_grammar.word_token != Some(symbol) {
                if let Some(t) = conflict_free_tokens.iter().find(|t| {
                    !coincident_token_index.contains(symbol, *t)
                        && token_conflict_map.does_conflict(symbol.index, t.index)
                }) {
                    info!(
                        "error recovery - exclude token {} because of conflict with {}",
                        lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
                    );
                    continue;
                }
            }
        }
        info!(
            "error recovery - include token {}",
            lexical_grammar.variables[i].name
        );
        state
            .terminal_entries
            .entry(symbol)
            .or_insert_with(|| recover_entry.clone());
    }

    for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
        if external_token.corresponding_internal_token.is_none() {
            state
                .terminal_entries
                .entry(Symbol::external(i))
                .or_insert_with(|| recover_entry.clone());
        }
    }

    state.terminal_entries.insert(Symbol::end(), recover_entry);
}

fn populate_used_symbols(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
) {
    let mut terminal_usages = vec![false; lexical_grammar.variables.len()];
    let mut non_terminal_usages = vec![false; syntax_grammar.variables.len()];
    let mut external_usages = vec![false; syntax_grammar.external_tokens.len()];
    for state in &parse_table.states {
        for symbol in state.terminal_entries.keys() {
            match symbol.kind {
                SymbolType::Terminal => terminal_usages[symbol.index] = true,
                SymbolType::External => external_usages[symbol.index] = true,
                _ => {}
            }
        }
        for symbol in state.nonterminal_entries.keys() {
            non_terminal_usages[symbol.index] = true;
        }
    }
    parse_table.symbols.push(Symbol::end());
    for (i, value) in terminal_usages.into_iter().enumerate() {
        if value {
            // Assign the grammar's word token a low numerical index. This ensures that
            // it can be stored in a subtree with no heap allocations, even for grammars with
            // very large numbers of tokens. This is an optimization, but it's also important to
            // ensure that a subtree's symbol can be successfully reassigned to the word token
            // without having to move the subtree to the heap.
            // See https://github.com/tree-sitter/tree-sitter/issues/258
            if syntax_grammar.word_token.map_or(false, |t| t.index == i) {
                parse_table.symbols.insert(1, Symbol::terminal(i));
            } else {
                parse_table.symbols.push(Symbol::terminal(i));
            }
        }
    }
    for (i, value) in external_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::external(i));
        }
    }
    for (i, value) in non_terminal_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::non_terminal(i));
        }
    }
}

fn identify_keywords(
    lexical_grammar: &LexicalGrammar,
    parse_table: &ParseTable,
    word_token: Option<Symbol>,
    token_conflict_map: &TokenConflictMap,
    coincident_token_index: &CoincidentTokenIndex,
) -> TokenSet {
    if word_token.is_none() {
        return TokenSet::new();
    }

    let word_token = word_token.unwrap();
    let mut cursor = NfaCursor::new(&lexical_grammar.nfa, Vec::new());

    // First find all of the candidate keyword tokens: tokens that start with
    // letters or underscore and can match the same string as a word token.
    let keyword_candidates: TokenSet = lexical_grammar
        .variables
        .iter()
        .enumerate()
        .filter_map(|(i, variable)| {
            cursor.reset(vec![variable.start_state]);
            if all_chars_are_alphabetical(&cursor)
                && token_conflict_map.does_match_same_string(i, word_token.index)
            {
                info!(
                    "Keywords - add candidate {}",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            } else {
                None
            }
        })
        .collect();

    // Exclude keyword candidates that shadow another keyword candidate.
    let keywords: TokenSet = keyword_candidates
        .iter()
        .filter(|token| {
            for other_token in keyword_candidates.iter() {
                if other_token != *token
                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
                {
                    info!(
                        "Keywords - exclude {} because it matches the same string as {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_token.index].name
                    );
                    return false;
                }
            }
            true
        })
        .collect();

    // Exclude keyword candidates for which substituting the keyword capture
    // token would introduce new lexical conflicts with other tokens.
    let keywords = keywords
        .iter()
        .filter(|token| {
            for other_index in 0..lexical_grammar.variables.len() {
                if keyword_candidates.contains(&Symbol::terminal(other_index)) {
                    continue;
                }

                // If the word token was already valid in every state containing
                // this keyword candidate, then substituting the word token won't
                // introduce any new lexical conflicts.
                if coincident_token_index
                    .states_with(*token, Symbol::terminal(other_index))
                    .iter()
                    .all(|state_id| {
                        parse_table.states[*state_id]
                            .terminal_entries
                            .contains_key(&word_token)
                    })
                {
                    continue;
                }

                if !token_conflict_map.has_same_conflict_status(
                    token.index,
                    word_token.index,
                    other_index,
                ) {
                    info!(
                        "Keywords - exclude {} because of conflict with {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_index].name
                    );
                    return false;
                }
            }

            info!(
                "Keywords - include {}",
                lexical_grammar.variables[token.index].name,
            );
            true
        })
        .collect();

    keywords
}

fn mark_fragile_tokens(
    parse_table: &mut ParseTable,
    lexical_grammar: &LexicalGrammar,
    token_conflict_map: &TokenConflictMap,
) {
    let n = lexical_grammar.variables.len();
    let mut valid_tokens_mask = Vec::with_capacity(n);
    for state in parse_table.states.iter_mut() {
        valid_tokens_mask.clear();
        valid_tokens_mask.resize(n, false);
        for token in state.terminal_entries.keys() {
            if token.is_terminal() {
                valid_tokens_mask[token.index] = true;
            }
        }
        for (token, entry) in state.terminal_entries.iter_mut() {
            if token.is_terminal() {
                for i in 0..n {
                    if token_conflict_map.does_overlap(i, token.index) {
                        if valid_tokens_mask[i] {
                            entry.reusable = false;
                            break;
                        }
                    }
                }
            }
        }
    }
}

fn all_chars_are_alphabetical(cursor: &NfaCursor) -> bool {
    cursor.transition_chars().all(|(chars, is_sep)| {
        if is_sep {
            true
        } else if let CharacterSet::Include(chars) = chars {
            chars.iter().all(|c| c.is_alphabetic() || *c == '_')
        } else {
            false
        }
    })
}
<commit_msg>cli: Compute fragile tokens *after* merging compatibile parse states<commit_after>mod build_lex_table;
mod build_parse_table;
mod coincident_tokens;
mod item;
mod item_set_builder;
mod minimize_parse_table;
mod token_conflicts;

use self::build_lex_table::build_lex_table;
use self::build_parse_table::build_parse_table;
use self::coincident_tokens::CoincidentTokenIndex;
use self::item::TokenSet;
use self::minimize_parse_table::minimize_parse_table;
use self::token_conflicts::TokenConflictMap;
use crate::error::Result;
use crate::generate::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar};
use crate::generate::nfa::{CharacterSet, NfaCursor};
use crate::generate::rules::{AliasMap, Symbol, SymbolType};
use crate::generate::tables::{LexTable, ParseAction, ParseTable, ParseTableEntry};
use log::info;

pub(crate) fn build_tables(
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    simple_aliases: &AliasMap,
    inlines: &InlinedProductionMap,
    minimize: bool,
    state_ids_to_log: Vec<usize>,
) -> Result<(ParseTable, LexTable, LexTable, Option<Symbol>)> {
    let (mut parse_table, following_tokens) =
        build_parse_table(syntax_grammar, lexical_grammar, inlines, state_ids_to_log)?;
    let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
    let keywords = identify_keywords(
        lexical_grammar,
        &parse_table,
        syntax_grammar.word_token,
        &token_conflict_map,
        &coincident_token_index,
    );
    populate_error_state(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &coincident_token_index,
        &token_conflict_map,
        &keywords,
    );
    populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
    if minimize {
        minimize_parse_table(
            &mut parse_table,
            syntax_grammar,
            lexical_grammar,
            simple_aliases,
            &token_conflict_map,
            &keywords,
        );
    }
    let (main_lex_table, keyword_lex_table) = build_lex_table(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &keywords,
        &coincident_token_index,
        &token_conflict_map,
        minimize,
    );
    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
    Ok((
        parse_table,
        main_lex_table,
        keyword_lex_table,
        syntax_grammar.word_token,
    ))
}

fn populate_error_state(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    coincident_token_index: &CoincidentTokenIndex,
    token_conflict_map: &TokenConflictMap,
    keywords: &TokenSet,
) {
    let state = &mut parse_table.states[0];
    let n = lexical_grammar.variables.len();

    // First identify the *conflict-free tokens*: tokens that do not overlap with
    // any other token in any way, besides matching exactly the same string.
    let conflict_free_tokens: TokenSet = (0..n)
        .into_iter()
        .filter_map(|i| {
            let conflicts_with_other_tokens = (0..n).into_iter().any(|j| {
                j != i
                    && !coincident_token_index.contains(Symbol::terminal(i), Symbol::terminal(j))
                    && token_conflict_map.does_match_shorter_or_longer(i, j)
            });
            if conflicts_with_other_tokens {
                None
            } else {
                info!(
                    "error recovery - token {} has no conflicts",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            }
        })
        .collect();

    let recover_entry = ParseTableEntry {
        reusable: false,
        actions: vec![ParseAction::Recover],
    };

    // Exclude from the error-recovery state any token that conflicts with one of
    // the *conflict-free tokens* identified above.
    for i in 0..n {
        let symbol = Symbol::terminal(i);
        if !conflict_free_tokens.contains(&symbol) && !keywords.contains(&symbol) {
            if syntax_grammar.word_token != Some(symbol) {
                if let Some(t) = conflict_free_tokens.iter().find(|t| {
                    !coincident_token_index.contains(symbol, *t)
                        && token_conflict_map.does_conflict(symbol.index, t.index)
                }) {
                    info!(
                        "error recovery - exclude token {} because of conflict with {}",
                        lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
                    );
                    continue;
                }
            }
        }
        info!(
            "error recovery - include token {}",
            lexical_grammar.variables[i].name
        );
        state
            .terminal_entries
            .entry(symbol)
            .or_insert_with(|| recover_entry.clone());
    }

    for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
        if external_token.corresponding_internal_token.is_none() {
            state
                .terminal_entries
                .entry(Symbol::external(i))
                .or_insert_with(|| recover_entry.clone());
        }
    }

    state.terminal_entries.insert(Symbol::end(), recover_entry);
}

fn populate_used_symbols(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
) {
    let mut terminal_usages = vec![false; lexical_grammar.variables.len()];
    let mut non_terminal_usages = vec![false; syntax_grammar.variables.len()];
    let mut external_usages = vec![false; syntax_grammar.external_tokens.len()];
    for state in &parse_table.states {
        for symbol in state.terminal_entries.keys() {
            match symbol.kind {
                SymbolType::Terminal => terminal_usages[symbol.index] = true,
                SymbolType::External => external_usages[symbol.index] = true,
                _ => {}
            }
        }
        for symbol in state.nonterminal_entries.keys() {
            non_terminal_usages[symbol.index] = true;
        }
    }
    parse_table.symbols.push(Symbol::end());
    for (i, value) in terminal_usages.into_iter().enumerate() {
        if value {
            // Assign the grammar's word token a low numerical index. This ensures that
            // it can be stored in a subtree with no heap allocations, even for grammars with
            // very large numbers of tokens. This is an optimization, but it's also important to
            // ensure that a subtree's symbol can be successfully reassigned to the word token
            // without having to move the subtree to the heap.
            // See https://github.com/tree-sitter/tree-sitter/issues/258
            if syntax_grammar.word_token.map_or(false, |t| t.index == i) {
                parse_table.symbols.insert(1, Symbol::terminal(i));
            } else {
                parse_table.symbols.push(Symbol::terminal(i));
            }
        }
    }
    for (i, value) in external_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::external(i));
        }
    }
    for (i, value) in non_terminal_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::non_terminal(i));
        }
    }
}

fn identify_keywords(
    lexical_grammar: &LexicalGrammar,
    parse_table: &ParseTable,
    word_token: Option<Symbol>,
    token_conflict_map: &TokenConflictMap,
    coincident_token_index: &CoincidentTokenIndex,
) -> TokenSet {
    if word_token.is_none() {
        return TokenSet::new();
    }

    let word_token = word_token.unwrap();
    let mut cursor = NfaCursor::new(&lexical_grammar.nfa, Vec::new());

    // First find all of the candidate keyword tokens: tokens that start with
    // letters or underscore and can match the same string as a word token.
    let keyword_candidates: TokenSet = lexical_grammar
        .variables
        .iter()
        .enumerate()
        .filter_map(|(i, variable)| {
            cursor.reset(vec![variable.start_state]);
            if all_chars_are_alphabetical(&cursor)
                && token_conflict_map.does_match_same_string(i, word_token.index)
            {
                info!(
                    "Keywords - add candidate {}",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            } else {
                None
            }
        })
        .collect();

    // Exclude keyword candidates that shadow another keyword candidate.
    let keywords: TokenSet = keyword_candidates
        .iter()
        .filter(|token| {
            for other_token in keyword_candidates.iter() {
                if other_token != *token
                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
                {
                    info!(
                        "Keywords - exclude {} because it matches the same string as {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_token.index].name
                    );
                    return false;
                }
            }
            true
        })
        .collect();

    // Exclude keyword candidates for which substituting the keyword capture
    // token would introduce new lexical conflicts with other tokens.
    let keywords = keywords
        .iter()
        .filter(|token| {
            for other_index in 0..lexical_grammar.variables.len() {
                if keyword_candidates.contains(&Symbol::terminal(other_index)) {
                    continue;
                }

                // If the word token was already valid in every state containing
                // this keyword candidate, then substituting the word token won't
                // introduce any new lexical conflicts.
                if coincident_token_index
                    .states_with(*token, Symbol::terminal(other_index))
                    .iter()
                    .all(|state_id| {
                        parse_table.states[*state_id]
                            .terminal_entries
                            .contains_key(&word_token)
                    })
                {
                    continue;
                }

                if !token_conflict_map.has_same_conflict_status(
                    token.index,
                    word_token.index,
                    other_index,
                ) {
                    info!(
                        "Keywords - exclude {} because of conflict with {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_index].name
                    );
                    return false;
                }
            }

            info!(
                "Keywords - include {}",
                lexical_grammar.variables[token.index].name,
            );
            true
        })
        .collect();

    keywords
}

fn mark_fragile_tokens(
    parse_table: &mut ParseTable,
    lexical_grammar: &LexicalGrammar,
    token_conflict_map: &TokenConflictMap,
) {
    let n = lexical_grammar.variables.len();
    let mut valid_tokens_mask = Vec::with_capacity(n);
    for state in parse_table.states.iter_mut() {
        valid_tokens_mask.clear();
        valid_tokens_mask.resize(n, false);
        for token in state.terminal_entries.keys() {
            if token.is_terminal() {
                valid_tokens_mask[token.index] = true;
            }
        }
        for (token, entry) in state.terminal_entries.iter_mut() {
            if token.is_terminal() {
                for (i, is_valid) in valid_tokens_mask.iter().enumerate() {
                    if *is_valid {
                        if token_conflict_map.does_overlap(i, token.index) {
                            entry.reusable = false;
                            break;
                        }
                    }
                }
            }
        }
    }
}

fn all_chars_are_alphabetical(cursor: &NfaCursor) -> bool {
    cursor.transition_chars().all(|(chars, is_sep)| {
        if is_sep {
            true
        } else if let CharacterSet::Include(chars) = chars {
            chars.iter().all(|c| c.is_alphabetic() || *c == '_')
        } else {
            false
        }
    })
}
<|endoftext|>


<commit_before>use countminsketch::CountMinSketch;
use std::collections::{BTreeSet, HashMap};
use std::collections::hash_map::Entry;
use std::cmp::Ordering;
use std::hash::Hash;
use std::rc::Rc;

struct TreeEntry<T>
where
    T: Eq + Ord,
{
    obj: Rc<T>,
    n: usize,
}

impl<T> Clone for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn clone(&self) -> Self {
        TreeEntry {
            obj: Rc::clone(&self.obj),
            n: self.n,
        }
    }
}

impl<T> PartialEq for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn eq(&self, other: &TreeEntry<T>) -> bool {
        self.obj == other.obj
    }
}

impl<T> Eq for TreeEntry<T>
where
    T: Eq + Ord,
{
}

impl<T> PartialOrd for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn partial_cmp(&self, other: &TreeEntry<T>) -> Option<Ordering> {
        match self.n.cmp(&other.n) {
            Ordering::Greater => Some(Ordering::Greater),
            Ordering::Less => Some(Ordering::Less),
            Ordering::Equal => Some(self.obj.cmp(&other.obj)),
        }
    }
}

impl<T> Ord for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn cmp(&self, other: &TreeEntry<T>) -> Ordering {
        match self.n.cmp(&other.n) {
            Ordering::Greater => Ordering::Greater,
            Ordering::Less => Ordering::Less,
            Ordering::Equal => self.obj.cmp(&other.obj),
        }
    }
}

/// Top-K implementation.
///
/// This data structure keeps the `k` most frequent data points of a stream.
pub struct TopK<T>
where
    T: Clone + Eq + Hash + Ord,
{
    cms: CountMinSketch,
    obj2count: HashMap<Rc<T>, usize>,
    tree: BTreeSet<TreeEntry<T>>,
    k: usize,
}

impl<T> TopK<T>
where
    T: Clone + Eq + Hash + Ord,
{
    /// Create new Top-K data structure.
    ///
    /// - `k`: number of elements to remember
    /// - `cms`: CountMinSketch that is used for guessing the frequency of elements not currently
    ///   hold. Refer to its documentation about parameter selection.
    ///
    /// Panics if `k == 0`.
    pub fn new(k: usize, cms: CountMinSketch) -> TopK<T> {
        assert!(k > 0);
        TopK {
            cms: cms,
            obj2count: HashMap::new(),
            tree: BTreeSet::new(),
            k: k,
        }
    }

    /// Number of data points to remember.
    pub fn k(&self) -> usize {
        return self.k;
    }

    /// Observe a data point.
    pub fn add(&mut self, obj: T) {
        // create Rc so we can use obj in multiple data structs
        let rc = Rc::new(obj);

        // always increase CountMinSketch counts
        self.cms.add(&rc);

        // check if entry is in top K
        let size = self.obj2count.len();
        let mut update_obj2count: Option<(usize, Rc<T>)> = None;
        match self.obj2count.entry(Rc::clone(&rc)) {
            Entry::Occupied(mut o) => {
                // it is => increase exact counter
                let mut n = o.get_mut();
                *n += 1;

                // update tree data
                let mut entry = TreeEntry {
                    obj: Rc::clone(&rc),
                    n: *n - 1,
                };
                self.tree.remove(&entry);
                entry.n += 1;
                self.tree.insert(entry);
            }
            Entry::Vacant(v) => {
                // it's not => check capicity
                if size < self.k {
                    // space left => add to top k
                    v.insert(1);
                    self.tree.insert(TreeEntry {
                        obj: Rc::clone(&rc),
                        n: 1,
                    });
                } else {
                    // not enough space => check if it would be a top k element
                    let count = self.cms.query_point(&rc);
                    let min: TreeEntry<T> = (*self.tree.iter().next().unwrap()).clone();
                    if count > min.n {
                        // => kick out minimal element of top k
                        self.tree.remove(&min);
                        self.tree.insert(TreeEntry {
                            obj: Rc::clone(&rc),
                            n: count,
                        });

                        // delay obj2count changes, because it is currently borrowed
                        update_obj2count = Some((count, min.obj));
                    }
                }
            }
        }

        // trick the borrow checker
        if let Some((count, obj)) = update_obj2count {
            self.obj2count.insert(rc, count);
            self.obj2count.remove(&obj);
        }
    }

    /// Returns top-k values.
    ///
    /// The result may contain less than `k` values if less than `k` unique data points where
    /// observed.
    pub fn values(&self) -> Vec<T> {
        self.tree.iter().map(|x| (*x.obj).clone()).collect()
    }

    /// Check whether the sampler is empty (i.e. no data points observer so far)
    pub fn is_empty(&self) -> bool {
        self.obj2count.is_empty()
    }

    /// Clear sampler, so that it behaves like it never observed any data points.
    pub fn clear(&mut self) {
        self.cms.clear();
        self.tree.clear();
        self.obj2count.clear();
    }
}

#[cfg(test)]
mod tests {
    use countminsketch::CountMinSketch;
    use super::TopK;

    #[test]
    fn getter() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let tk: TopK<usize> = TopK::new(2, cms);

        assert_eq!(tk.k(), 2);
    }

    #[test]
    fn add_1() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        tk.add(1);
        assert_eq!(tk.values(), vec![1]);
    }

    #[test]
    fn add_2_same() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        tk.add(1);
        tk.add(1);
        assert_eq!(tk.values(), vec![1]);
    }

    #[test]
    fn add_2_different() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        tk.add(1);
        tk.add(2);
        assert_eq!(tk.values(), vec![1, 2]);
    }

    #[test]
    fn add_n() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        for i in 0..5 {
            tk.add(i);
        }
        for _ in 0..100 {
            tk.add(99);
        }
        for _ in 0..100 {
            tk.add(100);
        }
        for i in 0..5 {
            tk.add(i);
        }
        assert_eq!(tk.values(), vec![99, 100]);
    }

    #[test]
    fn is_empty() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);
        assert_eq!(tk.is_empty(), true);

        tk.add(0);
        assert_eq!(tk.is_empty(), false);
    }

    #[test]
    fn clear() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);
        tk.add(0);

        tk.clear();
        assert_eq!(tk.is_empty(), true);

        tk.add(1);
        assert_eq!(tk.values(), vec![1]);
    }
}
<commit_msg>topk: impl clone<commit_after>use countminsketch::CountMinSketch;
use std::collections::{BTreeSet, HashMap};
use std::collections::hash_map::Entry;
use std::cmp::Ordering;
use std::hash::Hash;
use std::rc::Rc;

struct TreeEntry<T>
where
    T: Eq + Ord,
{
    obj: Rc<T>,
    n: usize,
}

impl<T> Clone for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn clone(&self) -> Self {
        TreeEntry {
            obj: Rc::clone(&self.obj),
            n: self.n,
        }
    }
}

impl<T> PartialEq for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn eq(&self, other: &TreeEntry<T>) -> bool {
        self.obj == other.obj
    }
}

impl<T> Eq for TreeEntry<T>
where
    T: Eq + Ord,
{
}

impl<T> PartialOrd for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn partial_cmp(&self, other: &TreeEntry<T>) -> Option<Ordering> {
        match self.n.cmp(&other.n) {
            Ordering::Greater => Some(Ordering::Greater),
            Ordering::Less => Some(Ordering::Less),
            Ordering::Equal => Some(self.obj.cmp(&other.obj)),
        }
    }
}

impl<T> Ord for TreeEntry<T>
where
    T: Eq + Ord,
{
    fn cmp(&self, other: &TreeEntry<T>) -> Ordering {
        match self.n.cmp(&other.n) {
            Ordering::Greater => Ordering::Greater,
            Ordering::Less => Ordering::Less,
            Ordering::Equal => self.obj.cmp(&other.obj),
        }
    }
}

/// Top-K implementation.
///
/// This data structure keeps the `k` most frequent data points of a stream.
#[derive(Clone)]
pub struct TopK<T>
where
    T: Clone + Eq + Hash + Ord,
{
    cms: CountMinSketch,
    obj2count: HashMap<Rc<T>, usize>,
    tree: BTreeSet<TreeEntry<T>>,
    k: usize,
}

impl<T> TopK<T>
where
    T: Clone + Eq + Hash + Ord,
{
    /// Create new Top-K data structure.
    ///
    /// - `k`: number of elements to remember
    /// - `cms`: CountMinSketch that is used for guessing the frequency of elements not currently
    ///   hold. Refer to its documentation about parameter selection.
    ///
    /// Panics if `k == 0`.
    pub fn new(k: usize, cms: CountMinSketch) -> TopK<T> {
        assert!(k > 0);
        TopK {
            cms: cms,
            obj2count: HashMap::new(),
            tree: BTreeSet::new(),
            k: k,
        }
    }

    /// Number of data points to remember.
    pub fn k(&self) -> usize {
        return self.k;
    }

    /// Observe a data point.
    pub fn add(&mut self, obj: T) {
        // create Rc so we can use obj in multiple data structs
        let rc = Rc::new(obj);

        // always increase CountMinSketch counts
        self.cms.add(&rc);

        // check if entry is in top K
        let size = self.obj2count.len();
        let mut update_obj2count: Option<(usize, Rc<T>)> = None;
        match self.obj2count.entry(Rc::clone(&rc)) {
            Entry::Occupied(mut o) => {
                // it is => increase exact counter
                let mut n = o.get_mut();
                *n += 1;

                // update tree data
                let mut entry = TreeEntry {
                    obj: Rc::clone(&rc),
                    n: *n - 1,
                };
                self.tree.remove(&entry);
                entry.n += 1;
                self.tree.insert(entry);
            }
            Entry::Vacant(v) => {
                // it's not => check capicity
                if size < self.k {
                    // space left => add to top k
                    v.insert(1);
                    self.tree.insert(TreeEntry {
                        obj: Rc::clone(&rc),
                        n: 1,
                    });
                } else {
                    // not enough space => check if it would be a top k element
                    let count = self.cms.query_point(&rc);
                    let min: TreeEntry<T> = (*self.tree.iter().next().unwrap()).clone();
                    if count > min.n {
                        // => kick out minimal element of top k
                        self.tree.remove(&min);
                        self.tree.insert(TreeEntry {
                            obj: Rc::clone(&rc),
                            n: count,
                        });

                        // delay obj2count changes, because it is currently borrowed
                        update_obj2count = Some((count, min.obj));
                    }
                }
            }
        }

        // trick the borrow checker
        if let Some((count, obj)) = update_obj2count {
            self.obj2count.insert(rc, count);
            self.obj2count.remove(&obj);
        }
    }

    /// Returns top-k values.
    ///
    /// The result may contain less than `k` values if less than `k` unique data points where
    /// observed.
    pub fn values(&self) -> Vec<T> {
        self.tree.iter().map(|x| (*x.obj).clone()).collect()
    }

    /// Check whether the sampler is empty (i.e. no data points observer so far)
    pub fn is_empty(&self) -> bool {
        self.obj2count.is_empty()
    }

    /// Clear sampler, so that it behaves like it never observed any data points.
    pub fn clear(&mut self) {
        self.cms.clear();
        self.tree.clear();
        self.obj2count.clear();
    }
}

#[cfg(test)]
mod tests {
    use countminsketch::CountMinSketch;
    use super::TopK;

    #[test]
    fn getter() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let tk: TopK<usize> = TopK::new(2, cms);

        assert_eq!(tk.k(), 2);
    }

    #[test]
    fn add_1() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        tk.add(1);
        assert_eq!(tk.values(), vec![1]);
    }

    #[test]
    fn add_2_same() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        tk.add(1);
        tk.add(1);
        assert_eq!(tk.values(), vec![1]);
    }

    #[test]
    fn add_2_different() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        tk.add(1);
        tk.add(2);
        assert_eq!(tk.values(), vec![1, 2]);
    }

    #[test]
    fn add_n() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);

        for i in 0..5 {
            tk.add(i);
        }
        for _ in 0..100 {
            tk.add(99);
        }
        for _ in 0..100 {
            tk.add(100);
        }
        for i in 0..5 {
            tk.add(i);
        }
        assert_eq!(tk.values(), vec![99, 100]);
    }

    #[test]
    fn is_empty() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);
        assert_eq!(tk.is_empty(), true);

        tk.add(0);
        assert_eq!(tk.is_empty(), false);
    }

    #[test]
    fn clear() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk = TopK::new(2, cms);
        tk.add(0);

        tk.clear();
        assert_eq!(tk.is_empty(), true);

        tk.add(1);
        assert_eq!(tk.values(), vec![1]);
    }

    #[test]
    fn clone() {
        let cms = CountMinSketch::<usize>::with_params(10, 20);
        let mut tk1 = TopK::new(2, cms);
        tk1.add(0);

        let mut tk2 = tk1.clone();
        tk2.add(1);

        assert_eq!(tk1.values(), vec![0]);
        assert_eq!(tk2.values(), vec![0, 1]);
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

use crate::context::LayoutContext;
use crate::dom_traversal::{Contents, NodeExt};
use crate::formatting_contexts::IndependentFormattingContext;
use crate::fragments::{AnonymousFragment, BoxFragment, CollapsedBlockMargins, Fragment};
use crate::geom::flow_relative::{Rect, Sides, Vec2};
use crate::sizing::ContentSizesRequest;
use crate::style_ext::{ComputedValuesExt, DisplayInside};
use crate::{ContainingBlock, DefiniteContainingBlock};
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use servo_arc::Arc;
use style::properties::ComputedValues;
use style::values::computed::{Length, LengthOrAuto, LengthPercentage, LengthPercentageOrAuto};
use style::Zero;

#[derive(Debug)]
pub(crate) struct AbsolutelyPositionedBox {
    pub contents: IndependentFormattingContext,
}

#[derive(Debug)]
pub(crate) struct AbsolutelyPositionedFragment<'box_> {
    absolutely_positioned_box: &'box_ AbsolutelyPositionedBox,

    /// The rank of the child from which this absolutely positioned fragment
    /// came from, when doing the layout of a block container. Used to compute
    /// static positions when going up the tree.
    pub(crate) tree_rank: usize,

    pub(crate) inline_start: AbsoluteBoxOffsets<LengthPercentage>,
    pub(crate) block_start: AbsoluteBoxOffsets<LengthPercentage>,
}

#[derive(Clone, Copy, Debug)]
pub(crate) enum AbsoluteBoxOffsets<NonStatic> {
    StaticStart { start: Length },
    Start { start: NonStatic },
    End { end: NonStatic },
    Both { start: NonStatic, end: NonStatic },
}

impl AbsolutelyPositionedBox {
    pub fn construct<'dom>(
        context: &LayoutContext,
        style: Arc<ComputedValues>,
        display_inside: DisplayInside,
        contents: Contents<impl NodeExt<'dom>>,
    ) -> Self {
        // "Shrink-to-fit" in https://drafts.csswg.org/css2/visudet.html#abs-non-replaced-width
        let content_sizes = ContentSizesRequest::inline_if(
            // If inline-size is non-auto, that value is used without shrink-to-fit
            style.inline_size_is_auto() &&
            // If it is, then the only case where shrink-to-fit is *not* used is
            // if both offsets are non-auto, leaving inline-size as the only variable
            // in the constraint equation.
            !style.inline_box_offsets_are_both_non_auto(),
        );
        Self {
            contents: IndependentFormattingContext::construct(
                context,
                style,
                display_inside,
                contents,
                content_sizes,
            ),
        }
    }

    pub(crate) fn layout<'a>(
        &'a self,
        initial_start_corner: Vec2<Length>,
        tree_rank: usize,
    ) -> AbsolutelyPositionedFragment {
        let style = &self.contents.style;
        let box_offsets = style.box_offsets();

        fn absolute_box_offsets(
            initial_static_start: Length,
            start: LengthPercentageOrAuto,
            end: LengthPercentageOrAuto,
        ) -> AbsoluteBoxOffsets<LengthPercentage> {
            match (start.non_auto(), end.non_auto()) {
                (None, None) => AbsoluteBoxOffsets::StaticStart {
                    start: initial_static_start,
                },
                (Some(start), Some(end)) => AbsoluteBoxOffsets::Both { start, end },
                (None, Some(end)) => AbsoluteBoxOffsets::End { end },
                (Some(start), None) => AbsoluteBoxOffsets::Start { start },
            }
        }

        let inline_start = absolute_box_offsets(
            initial_start_corner.inline,
            box_offsets.inline_start,
            box_offsets.inline_end,
        );
        let block_start = absolute_box_offsets(
            initial_start_corner.block,
            box_offsets.block_start,
            box_offsets.block_end,
        );

        AbsolutelyPositionedFragment {
            absolutely_positioned_box: self,
            tree_rank,
            inline_start,
            block_start,
        }
    }
}

impl<'a> AbsolutelyPositionedFragment<'a> {
    pub(crate) fn in_positioned_containing_block(
        layout_context: &LayoutContext,
        absolute: &[Self],
        fragments: &mut Vec<Fragment>,
        content_rect_size: &Vec2<Length>,
        padding: &Sides<Length>,
        style: &ComputedValues,
    ) {
        if absolute.is_empty() {
            return;
        }
        let padding_rect = Rect {
            size: content_rect_size.clone(),
            // Ignore the content rect’s position in its own containing block:
            start_corner: Vec2::zero(),
        }
        .inflate(&padding);
        let containing_block = DefiniteContainingBlock {
            size: padding_rect.size.clone(),
            style,
        };
        fragments.push(Fragment::Anonymous(AnonymousFragment {
            children: absolute
                .par_iter()
                .map(|a| a.layout(layout_context, &containing_block))
                .collect(),
            rect: padding_rect,
            mode: style.writing_mode,
        }))
    }

    pub(crate) fn layout(
        &self,
        layout_context: &LayoutContext,
        containing_block: &DefiniteContainingBlock,
    ) -> Fragment {
        let style = &self.absolutely_positioned_box.contents.style;
        let cbis = containing_block.size.inline;
        let cbbs = containing_block.size.block;

        let box_size = style.box_size();
        let padding = style.padding().percentages_relative_to(cbis);
        let border = style.border_width();
        let computed_margin = style.margin().percentages_relative_to(cbis);
        let pb = &padding + &border;

        enum Anchor {
            Start(Length),
            End(Length),
        }

        fn solve_axis(
            containing_size: Length,
            padding_border_sum: Length,
            computed_margin_start: LengthOrAuto,
            computed_margin_end: LengthOrAuto,
            solve_margins: impl FnOnce(Length) -> (Length, Length),
            box_offsets: AbsoluteBoxOffsets<LengthPercentage>,
            size: LengthPercentageOrAuto,
        ) -> (Anchor, LengthOrAuto, Length, Length) {
            let size = size.percentage_relative_to(containing_size);
            match box_offsets {
                AbsoluteBoxOffsets::StaticStart { start } => (
                    Anchor::Start(start),
                    size,
                    computed_margin_start.auto_is(Length::zero),
                    computed_margin_end.auto_is(Length::zero),
                ),
                AbsoluteBoxOffsets::Start { start } => (
                    Anchor::Start(start.percentage_relative_to(containing_size)),
                    size,
                    computed_margin_start.auto_is(Length::zero),
                    computed_margin_end.auto_is(Length::zero),
                ),
                AbsoluteBoxOffsets::End { end } => (
                    Anchor::End(end.percentage_relative_to(containing_size)),
                    size,
                    computed_margin_start.auto_is(Length::zero),
                    computed_margin_end.auto_is(Length::zero),
                ),
                AbsoluteBoxOffsets::Both { start, end } => {
                    let start = start.percentage_relative_to(containing_size);
                    let end = end.percentage_relative_to(containing_size);

                    let mut margin_start = computed_margin_start.auto_is(Length::zero);
                    let mut margin_end = computed_margin_end.auto_is(Length::zero);

                    let size = if let LengthOrAuto::LengthPercentage(size) = size {
                        let margins = containing_size - start - end - padding_border_sum - size;
                        match (computed_margin_start, computed_margin_end) {
                            (LengthOrAuto::Auto, LengthOrAuto::Auto) => {
                                let (s, e) = solve_margins(margins);
                                margin_start = s;
                                margin_end = e;
                            },
                            (LengthOrAuto::Auto, LengthOrAuto::LengthPercentage(end)) => {
                                margin_start = margins - end;
                            },
                            (LengthOrAuto::LengthPercentage(start), LengthOrAuto::Auto) => {
                                margin_end = margins - start;
                            },
                            (
                                LengthOrAuto::LengthPercentage(_),
                                LengthOrAuto::LengthPercentage(_),
                            ) => {},
                        }
                        size
                    } else {
                        // FIXME(nox): What happens if that is negative?
                        containing_size -
                            start -
                            end -
                            padding_border_sum -
                            margin_start -
                            margin_end
                    };
                    (
                        Anchor::Start(start),
                        LengthOrAuto::LengthPercentage(size),
                        margin_start,
                        margin_end,
                    )
                },
            }
        }

        let (inline_anchor, inline_size, margin_inline_start, margin_inline_end) = solve_axis(
            cbis,
            pb.inline_sum(),
            computed_margin.inline_start,
            computed_margin.inline_end,
            |margins| {
                if margins.px() >= 0. {
                    (margins / 2., margins / 2.)
                } else {
                    (Length::zero(), margins)
                }
            },
            self.inline_start,
            box_size.inline,
        );

        let (block_anchor, block_size, margin_block_start, margin_block_end) = solve_axis(
            cbis,
            pb.block_sum(),
            computed_margin.block_start,
            computed_margin.block_end,
            |margins| (margins / 2., margins / 2.),
            self.block_start,
            box_size.block,
        );

        let margin = Sides {
            inline_start: margin_inline_start,
            inline_end: margin_inline_end,
            block_start: margin_block_start,
            block_end: margin_block_end,
        };

        let inline_size = inline_size.auto_is(|| {
            let available_size = match inline_anchor {
                Anchor::Start(start) => cbis - start - pb.inline_sum() - margin.inline_sum(),
                Anchor::End(end) => cbis - end - pb.inline_sum() - margin.inline_sum(),
            };

            if self
                .absolutely_positioned_box
                .contents
                .as_replaced()
                .is_ok()
            {
                // FIXME: implement https://drafts.csswg.org/css2/visudet.html#abs-replaced-width
                available_size
            } else {
                self.absolutely_positioned_box
                    .contents
                    .content_sizes
                    .shrink_to_fit(available_size)
            }
        });

        let mut absolutely_positioned_fragments = Vec::new();
        let mut independent_layout = match self.absolutely_positioned_box.contents.as_replaced() {
            Ok(replaced) => {
                // FIXME: implement https://drafts.csswg.org/css2/visudet.html#abs-replaced-width
                // and https://drafts.csswg.org/css2/visudet.html#abs-replaced-height
                let block_size = block_size.auto_is(Length::zero);
                let fragments = replaced.make_fragments(
                    &self.absolutely_positioned_box.contents.style,
                    Vec2 {
                        inline: inline_size,
                        block: block_size,
                    },
                );
                crate::formatting_contexts::IndependentLayout {
                    fragments,
                    content_block_size: block_size,
                }
            },
            Err(non_replaced) => {
                let containing_block_for_children = ContainingBlock {
                    inline_size,
                    block_size,
                    style,
                };
                // https://drafts.csswg.org/css-writing-modes/#orthogonal-flows
                assert_eq!(
                    containing_block.style.writing_mode,
                    containing_block_for_children.style.writing_mode,
                    "Mixed writing modes are not supported yet"
                );
                let dummy_tree_rank = 0;
                non_replaced.layout(
                    layout_context,
                    &containing_block_for_children,
                    dummy_tree_rank,
                    &mut absolutely_positioned_fragments,
                )
            },
        };

        let inline_start = match inline_anchor {
            Anchor::Start(start) => start + pb.inline_start + margin.inline_start,
            Anchor::End(end) => cbbs - end - pb.inline_end - margin.inline_end - inline_size,
        };

        let block_size = block_size.auto_is(|| independent_layout.content_block_size);
        let block_start = match block_anchor {
            Anchor::Start(start) => start + pb.block_start + margin.block_start,
            Anchor::End(end) => cbbs - end - pb.block_end - margin.block_end - block_size,
        };

        let content_rect = Rect {
            start_corner: Vec2 {
                inline: inline_start,
                block: block_start,
            },
            size: Vec2 {
                inline: inline_size,
                block: block_size,
            },
        };

        AbsolutelyPositionedFragment::in_positioned_containing_block(
            layout_context,
            &absolutely_positioned_fragments,
            &mut independent_layout.fragments,
            &content_rect.size,
            &padding,
            style,
        );

        Fragment::Box(BoxFragment {
            style: style.clone(),
            children: independent_layout.fragments,
            content_rect,
            padding,
            border,
            margin,
            block_margins_collapsed_with_children: CollapsedBlockMargins::zero(),
        })
    }
}

pub(crate) fn adjust_static_positions(
    absolutely_positioned_fragments: &mut [AbsolutelyPositionedFragment],
    child_fragments: &mut [Fragment],
    tree_rank_in_parent: usize,
) {
    for abspos_fragment in absolutely_positioned_fragments {
        let child_fragment_rect = match &child_fragments[abspos_fragment.tree_rank] {
            Fragment::Box(b) => &b.content_rect,
            Fragment::Anonymous(a) => &a.rect,
            _ => unreachable!(),
        };

        abspos_fragment.tree_rank = tree_rank_in_parent;

        if let AbsoluteBoxOffsets::StaticStart { start } = &mut abspos_fragment.inline_start {
            *start += child_fragment_rect.start_corner.inline;
        }

        if let AbsoluteBoxOffsets::StaticStart { start } = &mut abspos_fragment.block_start {
            *start += child_fragment_rect.start_corner.block;
        }
    }
}
<commit_msg>Remove AbsoluteBoxOffsets’s type parameter<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

use crate::context::LayoutContext;
use crate::dom_traversal::{Contents, NodeExt};
use crate::formatting_contexts::IndependentFormattingContext;
use crate::fragments::{AnonymousFragment, BoxFragment, CollapsedBlockMargins, Fragment};
use crate::geom::flow_relative::{Rect, Sides, Vec2};
use crate::sizing::ContentSizesRequest;
use crate::style_ext::{ComputedValuesExt, DisplayInside};
use crate::{ContainingBlock, DefiniteContainingBlock};
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use servo_arc::Arc;
use style::properties::ComputedValues;
use style::values::computed::{Length, LengthOrAuto, LengthPercentage, LengthPercentageOrAuto};
use style::Zero;

#[derive(Debug)]
pub(crate) struct AbsolutelyPositionedBox {
    pub contents: IndependentFormattingContext,
}

#[derive(Debug)]
pub(crate) struct AbsolutelyPositionedFragment<'box_> {
    absolutely_positioned_box: &'box_ AbsolutelyPositionedBox,

    /// The rank of the child from which this absolutely positioned fragment
    /// came from, when doing the layout of a block container. Used to compute
    /// static positions when going up the tree.
    pub(crate) tree_rank: usize,

    inline_start: AbsoluteBoxOffsets,
    block_start: AbsoluteBoxOffsets,
}

#[derive(Clone, Copy, Debug)]
pub(crate) enum AbsoluteBoxOffsets {
    StaticStart {
        start: Length,
    },
    Start {
        start: LengthPercentage,
    },
    End {
        end: LengthPercentage,
    },
    Both {
        start: LengthPercentage,
        end: LengthPercentage,
    },
}

impl AbsolutelyPositionedBox {
    pub fn construct<'dom>(
        context: &LayoutContext,
        style: Arc<ComputedValues>,
        display_inside: DisplayInside,
        contents: Contents<impl NodeExt<'dom>>,
    ) -> Self {
        // "Shrink-to-fit" in https://drafts.csswg.org/css2/visudet.html#abs-non-replaced-width
        let content_sizes = ContentSizesRequest::inline_if(
            // If inline-size is non-auto, that value is used without shrink-to-fit
            style.inline_size_is_auto() &&
            // If it is, then the only case where shrink-to-fit is *not* used is
            // if both offsets are non-auto, leaving inline-size as the only variable
            // in the constraint equation.
            !style.inline_box_offsets_are_both_non_auto(),
        );
        Self {
            contents: IndependentFormattingContext::construct(
                context,
                style,
                display_inside,
                contents,
                content_sizes,
            ),
        }
    }

    pub(crate) fn layout<'a>(
        &'a self,
        initial_start_corner: Vec2<Length>,
        tree_rank: usize,
    ) -> AbsolutelyPositionedFragment {
        let style = &self.contents.style;
        let box_offsets = style.box_offsets();

        fn absolute_box_offsets(
            initial_static_start: Length,
            start: LengthPercentageOrAuto,
            end: LengthPercentageOrAuto,
        ) -> AbsoluteBoxOffsets {
            match (start.non_auto(), end.non_auto()) {
                (None, None) => AbsoluteBoxOffsets::StaticStart {
                    start: initial_static_start,
                },
                (Some(start), Some(end)) => AbsoluteBoxOffsets::Both { start, end },
                (None, Some(end)) => AbsoluteBoxOffsets::End { end },
                (Some(start), None) => AbsoluteBoxOffsets::Start { start },
            }
        }

        let inline_start = absolute_box_offsets(
            initial_start_corner.inline,
            box_offsets.inline_start,
            box_offsets.inline_end,
        );
        let block_start = absolute_box_offsets(
            initial_start_corner.block,
            box_offsets.block_start,
            box_offsets.block_end,
        );

        AbsolutelyPositionedFragment {
            absolutely_positioned_box: self,
            tree_rank,
            inline_start,
            block_start,
        }
    }
}

impl<'a> AbsolutelyPositionedFragment<'a> {
    pub(crate) fn in_positioned_containing_block(
        layout_context: &LayoutContext,
        absolute: &[Self],
        fragments: &mut Vec<Fragment>,
        content_rect_size: &Vec2<Length>,
        padding: &Sides<Length>,
        style: &ComputedValues,
    ) {
        if absolute.is_empty() {
            return;
        }
        let padding_rect = Rect {
            size: content_rect_size.clone(),
            // Ignore the content rect’s position in its own containing block:
            start_corner: Vec2::zero(),
        }
        .inflate(&padding);
        let containing_block = DefiniteContainingBlock {
            size: padding_rect.size.clone(),
            style,
        };
        fragments.push(Fragment::Anonymous(AnonymousFragment {
            children: absolute
                .par_iter()
                .map(|a| a.layout(layout_context, &containing_block))
                .collect(),
            rect: padding_rect,
            mode: style.writing_mode,
        }))
    }

    pub(crate) fn layout(
        &self,
        layout_context: &LayoutContext,
        containing_block: &DefiniteContainingBlock,
    ) -> Fragment {
        let style = &self.absolutely_positioned_box.contents.style;
        let cbis = containing_block.size.inline;
        let cbbs = containing_block.size.block;

        let box_size = style.box_size();
        let padding = style.padding().percentages_relative_to(cbis);
        let border = style.border_width();
        let computed_margin = style.margin().percentages_relative_to(cbis);
        let pb = &padding + &border;

        enum Anchor {
            Start(Length),
            End(Length),
        }

        fn solve_axis(
            containing_size: Length,
            padding_border_sum: Length,
            computed_margin_start: LengthOrAuto,
            computed_margin_end: LengthOrAuto,
            solve_margins: impl FnOnce(Length) -> (Length, Length),
            box_offsets: AbsoluteBoxOffsets,
            size: LengthPercentageOrAuto,
        ) -> (Anchor, LengthOrAuto, Length, Length) {
            let size = size.percentage_relative_to(containing_size);
            match box_offsets {
                AbsoluteBoxOffsets::StaticStart { start } => (
                    Anchor::Start(start),
                    size,
                    computed_margin_start.auto_is(Length::zero),
                    computed_margin_end.auto_is(Length::zero),
                ),
                AbsoluteBoxOffsets::Start { start } => (
                    Anchor::Start(start.percentage_relative_to(containing_size)),
                    size,
                    computed_margin_start.auto_is(Length::zero),
                    computed_margin_end.auto_is(Length::zero),
                ),
                AbsoluteBoxOffsets::End { end } => (
                    Anchor::End(end.percentage_relative_to(containing_size)),
                    size,
                    computed_margin_start.auto_is(Length::zero),
                    computed_margin_end.auto_is(Length::zero),
                ),
                AbsoluteBoxOffsets::Both { start, end } => {
                    let start = start.percentage_relative_to(containing_size);
                    let end = end.percentage_relative_to(containing_size);

                    let mut margin_start = computed_margin_start.auto_is(Length::zero);
                    let mut margin_end = computed_margin_end.auto_is(Length::zero);

                    let size = if let LengthOrAuto::LengthPercentage(size) = size {
                        let margins = containing_size - start - end - padding_border_sum - size;
                        match (computed_margin_start, computed_margin_end) {
                            (LengthOrAuto::Auto, LengthOrAuto::Auto) => {
                                let (s, e) = solve_margins(margins);
                                margin_start = s;
                                margin_end = e;
                            },
                            (LengthOrAuto::Auto, LengthOrAuto::LengthPercentage(end)) => {
                                margin_start = margins - end;
                            },
                            (LengthOrAuto::LengthPercentage(start), LengthOrAuto::Auto) => {
                                margin_end = margins - start;
                            },
                            (
                                LengthOrAuto::LengthPercentage(_),
                                LengthOrAuto::LengthPercentage(_),
                            ) => {},
                        }
                        size
                    } else {
                        // FIXME(nox): What happens if that is negative?
                        containing_size -
                            start -
                            end -
                            padding_border_sum -
                            margin_start -
                            margin_end
                    };
                    (
                        Anchor::Start(start),
                        LengthOrAuto::LengthPercentage(size),
                        margin_start,
                        margin_end,
                    )
                },
            }
        }

        let (inline_anchor, inline_size, margin_inline_start, margin_inline_end) = solve_axis(
            cbis,
            pb.inline_sum(),
            computed_margin.inline_start,
            computed_margin.inline_end,
            |margins| {
                if margins.px() >= 0. {
                    (margins / 2., margins / 2.)
                } else {
                    (Length::zero(), margins)
                }
            },
            self.inline_start,
            box_size.inline,
        );

        let (block_anchor, block_size, margin_block_start, margin_block_end) = solve_axis(
            cbis,
            pb.block_sum(),
            computed_margin.block_start,
            computed_margin.block_end,
            |margins| (margins / 2., margins / 2.),
            self.block_start,
            box_size.block,
        );

        let margin = Sides {
            inline_start: margin_inline_start,
            inline_end: margin_inline_end,
            block_start: margin_block_start,
            block_end: margin_block_end,
        };

        let inline_size = inline_size.auto_is(|| {
            let available_size = match inline_anchor {
                Anchor::Start(start) => cbis - start - pb.inline_sum() - margin.inline_sum(),
                Anchor::End(end) => cbis - end - pb.inline_sum() - margin.inline_sum(),
            };

            if self
                .absolutely_positioned_box
                .contents
                .as_replaced()
                .is_ok()
            {
                // FIXME: implement https://drafts.csswg.org/css2/visudet.html#abs-replaced-width
                available_size
            } else {
                self.absolutely_positioned_box
                    .contents
                    .content_sizes
                    .shrink_to_fit(available_size)
            }
        });

        let mut absolutely_positioned_fragments = Vec::new();
        let mut independent_layout = match self.absolutely_positioned_box.contents.as_replaced() {
            Ok(replaced) => {
                // FIXME: implement https://drafts.csswg.org/css2/visudet.html#abs-replaced-width
                // and https://drafts.csswg.org/css2/visudet.html#abs-replaced-height
                let block_size = block_size.auto_is(Length::zero);
                let fragments = replaced.make_fragments(
                    &self.absolutely_positioned_box.contents.style,
                    Vec2 {
                        inline: inline_size,
                        block: block_size,
                    },
                );
                crate::formatting_contexts::IndependentLayout {
                    fragments,
                    content_block_size: block_size,
                }
            },
            Err(non_replaced) => {
                let containing_block_for_children = ContainingBlock {
                    inline_size,
                    block_size,
                    style,
                };
                // https://drafts.csswg.org/css-writing-modes/#orthogonal-flows
                assert_eq!(
                    containing_block.style.writing_mode,
                    containing_block_for_children.style.writing_mode,
                    "Mixed writing modes are not supported yet"
                );
                let dummy_tree_rank = 0;
                non_replaced.layout(
                    layout_context,
                    &containing_block_for_children,
                    dummy_tree_rank,
                    &mut absolutely_positioned_fragments,
                )
            },
        };

        let inline_start = match inline_anchor {
            Anchor::Start(start) => start + pb.inline_start + margin.inline_start,
            Anchor::End(end) => cbbs - end - pb.inline_end - margin.inline_end - inline_size,
        };

        let block_size = block_size.auto_is(|| independent_layout.content_block_size);
        let block_start = match block_anchor {
            Anchor::Start(start) => start + pb.block_start + margin.block_start,
            Anchor::End(end) => cbbs - end - pb.block_end - margin.block_end - block_size,
        };

        let content_rect = Rect {
            start_corner: Vec2 {
                inline: inline_start,
                block: block_start,
            },
            size: Vec2 {
                inline: inline_size,
                block: block_size,
            },
        };

        AbsolutelyPositionedFragment::in_positioned_containing_block(
            layout_context,
            &absolutely_positioned_fragments,
            &mut independent_layout.fragments,
            &content_rect.size,
            &padding,
            style,
        );

        Fragment::Box(BoxFragment {
            style: style.clone(),
            children: independent_layout.fragments,
            content_rect,
            padding,
            border,
            margin,
            block_margins_collapsed_with_children: CollapsedBlockMargins::zero(),
        })
    }
}

pub(crate) fn adjust_static_positions(
    absolutely_positioned_fragments: &mut [AbsolutelyPositionedFragment],
    child_fragments: &mut [Fragment],
    tree_rank_in_parent: usize,
) {
    for abspos_fragment in absolutely_positioned_fragments {
        let child_fragment_rect = match &child_fragments[abspos_fragment.tree_rank] {
            Fragment::Box(b) => &b.content_rect,
            Fragment::Anonymous(a) => &a.rect,
            _ => unreachable!(),
        };

        abspos_fragment.tree_rank = tree_rank_in_parent;

        if let AbsoluteBoxOffsets::StaticStart { start } = &mut abspos_fragment.inline_start {
            *start += child_fragment_rect.start_corner.inline;
        }

        if let AbsoluteBoxOffsets::StaticStart { start } = &mut abspos_fragment.block_start {
            *start += child_fragment_rect.start_corner.block;
        }
    }
}
<|endoftext|>


<commit_before>
// Copyright 2017 The gltf Library Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! The reference loader implementation for the `gltf` crate.
//!
//! # Examples
//!
//! ### Importing some `glTF` 2.0
//!
//! ```rust
//! use gltf_importer::import;
//! # #[allow(unused_variables)]
//! let path = "path/to/asset.gltf";
//! # let path = "../examples/Box.gltf";
//! match import(path) {
//!     Ok((gltf, _)) => println!("{:#?}", gltf.as_json()),
//!     Err(err) => println!("error: {:?}", err),
//! }
//! ```

extern crate base64;
extern crate gltf;
extern crate gltf_utils;

use gltf::json::{self, validation};
use std::{fmt, fs, io};

use gltf::Gltf;
use gltf_utils::Source;
use std::error::Error as StdError;
use std::path::Path;

/// Contains parameters for import configuration.
pub mod config;

pub use self::config::Config;
pub use self::config::ValidationStrategy;

/// Error encountered when importing a glTF 2.0 asset.
#[derive(Debug)]
pub enum Error {
    /// A loaded glTF buffer is not of the required length.
    BufferLength(json::Path),

    /// Base 64 decoding error.
    Base64Decoding(base64::DecodeError),

    /// A glTF extension required by the asset has not been enabled by the user.
    ExtensionDisabled(String),

    /// A glTF extension required by the asset is not supported by the library.
    ExtensionUnsupported(String),

    /// The glTF version of the asset is incompatible with the importer.
    IncompatibleVersion(String),

    /// Standard I/O error.
    Io(std::io::Error),

    /// `gltf` crate error.
    Gltf(gltf::Error),

    /// Failure when deserializing .gltf or .glb JSON.
    MalformedJson(json::Error),

    /// The .gltf data is invalid.
    Validation(Vec<(json::Path, validation::Error)>),
}

/// Buffer data returned from `import`.
#[derive(Clone, Debug)]
pub struct Buffers(Vec<Vec<u8>>);

impl Source for Buffers {
    fn source_buffer(&self, buffer: &gltf::Buffer) -> &[u8] {
        &self.0[buffer.index()]
    }
}

impl Buffers {
    /// Obtain the contents of a loaded buffer.
    pub fn buffer(&self, buffer: &gltf::Buffer) -> Option<&[u8]> {
        self.0.get(buffer.index()).map(Vec::as_slice)
    }

    /// Obtain the contents of a loaded buffer view.
    pub fn view(&self, view: &gltf::buffer::View) -> Option<&[u8]> {
        self.buffer(&view.buffer())
            .map(|data| {
                let begin = view.offset();
                let end = begin + view.length();
                &data[begin..end]
            })
    }

    /// Take the loaded buffer data.
    pub fn take(self) -> Vec<Vec<u8>> {
        self.0
    }
}

fn import_impl(path: &Path, config: Config) -> Result<(Gltf, Buffers), Error> {
    let data = read_to_end(path)?;
    if data.starts_with(b"glTF") {
        import_binary(&data, &config, path)
    } else {
        import_standard(&data, &config, path)
    }
}

/// Imports glTF 2.0 with custom configuration.
pub fn import_with_config<P>(path: P, config: Config) -> Result<(Gltf, Buffers), Error>
    where P: AsRef<Path>
{
    import_impl(path.as_ref(), config)
}

/// Imports glTF 2.0 with default configuration.
pub fn import<P>(path: P) -> Result<(Gltf, Buffers), Error>
    where P: AsRef<Path>
{
    import_impl(path.as_ref(), Default::default())
}

fn read_to_end_impl(path: &Path) -> Result<Vec<u8>, Error> {
    use io::Read;
    let file = fs::File::open(path)?;
    let mut reader = io::BufReader::new(file);
    let mut buffer = vec![];
    let _ = reader.read_to_end(&mut buffer)?;
    Ok(buffer)
}

fn read_to_end<P: AsRef<Path>>(path: P) -> Result<Vec<u8>, Error> {
    read_to_end_impl(path.as_ref())
}

fn parse_data_uri(uri: &str) -> Result<Vec<u8>, Error> {
    let encoded = uri.split(",").nth(1).unwrap();
    let decoded = base64::decode(&encoded)?;
    Ok(decoded)
}

fn load_external_buffers(
    base_path: &Path,
    gltf: &Gltf,
    mut bin: Option<Vec<u8>>,
) -> Result<Vec<Vec<u8>>, Error> {
    let mut buffers = vec![];
    for (index, buffer) in gltf.buffers().enumerate() {
        let uri = buffer.uri();
        let data = if uri == "#bin" {
            Ok(bin.take().unwrap())
        } else if uri.starts_with("data:") {
            Ok(parse_data_uri(uri)?)
        } else {
            let path = base_path.parent().unwrap_or(Path::new("./")).join(uri);
            read_to_end(&path)
        }?;
        if data.len() != buffer.length() {
            let path = json::Path::new().field("buffers").index(index);
            return Err(Error::BufferLength(path));
        }
        buffers.push(data);
    }
    Ok(buffers)
}

fn validate_standard(
    unvalidated: gltf::Unvalidated,
    config: &Config,
) -> Result<Gltf, Error> {
    use config::ValidationStrategy;
    Ok(match config.validation_strategy {
        ValidationStrategy::Skip => unsafe { unvalidated.skip_validation() },
        ValidationStrategy::Minimal => unvalidated.validate_minimally()?,
        ValidationStrategy::Complete => unvalidated.validate_completely()?,
    })
}

fn validate_binary(
    unvalidated: gltf::Unvalidated,
    config: &Config,
    has_bin: bool,
) -> Result<Gltf, Error> {
    use config::ValidationStrategy;
    use json::validation::Error as Reason;

    if config.validation_strategy == ValidationStrategy::Skip {
        return Ok(unsafe { unvalidated.skip_validation() });
    }

    let mut errs = vec![];
    {
        let json = unvalidated.as_json();
        for (index, buffer) in json.buffers.iter().enumerate() {
            let path = || {
                json::Path::new()
                    .field("buffers")
                    .index(index)
                    .field("uri")
            };
            match index {
                0 if has_bin => if buffer.uri.is_some() {
                    errs.push((path(), Reason::Missing));
                },
                _ if buffer.uri.is_none() => {
                    errs.push((path(), Reason::Missing));
                },
                _ => {},
            }
        }
    }

    if errs.is_empty() {
        Ok(match config.validation_strategy {
            ValidationStrategy::Minimal => unvalidated.validate_minimally()?,
            ValidationStrategy::Complete => unvalidated.validate_completely()?,
            ValidationStrategy::Skip => unreachable!(),
        })
    } else {
        Err(Error::Validation(errs))
    }
}

fn import_standard<'a>(
    data: &'a [u8],
    config: &Config,
    base_path: &Path,
) -> Result<(Gltf, Buffers), Error> {
    let unvalidated = Gltf::from_slice(data)?;
    let gltf = validate_standard(unvalidated, &config)?;
    let bin = None;
    let mut buffers = Buffers(vec![]);
    for buffer in load_external_buffers(base_path, &gltf, bin)? {
        buffers.0.push(buffer);
    }
    Ok((gltf, buffers))
}

fn import_binary<'a>(
    data: &'a [u8],
    config: &Config,
    base_path: &Path,
) -> Result<(Gltf, Buffers), Error> {
    let gltf::Glb { header: _, json, bin } = gltf::Glb::from_slice(data)?;
    let unvalidated = Gltf::from_slice(json)?;
    let bin = bin.map(|x| x.to_vec());
    let gltf = validate_binary(unvalidated, &config, bin.is_some())?;
    let mut buffers = Buffers(vec![]);
    for buffer in load_external_buffers(base_path, &gltf, bin)? {
        buffers.0.push(buffer);
    }
    Ok((gltf, buffers))
}

impl From<json::Error> for Error {
    fn from(err: json::Error) -> Error {
        Error::MalformedJson(err)
    }
}

impl From<std::io::Error> for Error {
    fn from(err: std::io::Error) -> Error {
        Error::Io(err)
    }
}

impl From<Vec<(json::Path, validation::Error)>> for Error {
    fn from(errs: Vec<(json::Path, validation::Error)>) -> Error {
        Error::Validation(errs)
    }
}

impl From<gltf::Error> for Error {
    fn from(err: gltf::Error) -> Error {
        match err {
            gltf::Error::Validation(errs) => Error::Validation(errs),
            _ => Error::Gltf(err),
        }
    }
}

impl From<base64::DecodeError> for Error {
    fn from(err: base64::DecodeError) -> Error {
        Error::Base64Decoding(err)
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use std::error::Error;
        write!(f, "{}", self.description())
    }
}

impl StdError for Error {
    fn description(&self) -> &str {
        use self::Error::*;
        match *self {
            Base64Decoding(_) => "Base 64 decoding failed",
            BufferLength(_) => "Loaded buffer does not match required length",
            ExtensionDisabled(_) => "Asset requires a disabled extension",
            ExtensionUnsupported(_) => "Assets requires an unsupported extension",
            IncompatibleVersion(_) => "Asset is not glTF version 2.0",
            Io(_) => "I/O error",
            Gltf(_) => "Error from gltf crate",
            MalformedJson(_) => "Malformed .gltf / .glb JSON",
            Validation(_) => "Asset failed validation tests",
        }
    }

    fn cause(&self) -> Option<&StdError> {
        use self::Error::*;
        match *self {
            MalformedJson(ref err) => Some(err),
            Io(ref err) => Some(err),
            _ => None,
        }
    }
}
<commit_msg>Allow external buffer data to be larger than expected<commit_after>
// Copyright 2017 The gltf Library Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! The reference loader implementation for the `gltf` crate.
//!
//! # Examples
//!
//! ### Importing some `glTF` 2.0
//!
//! ```rust
//! use gltf_importer::import;
//! # #[allow(unused_variables)]
//! let path = "path/to/asset.gltf";
//! # let path = "../examples/Box.gltf";
//! match import(path) {
//!     Ok((gltf, _)) => println!("{:#?}", gltf.as_json()),
//!     Err(err) => println!("error: {:?}", err),
//! }
//! ```

extern crate base64;
extern crate gltf;
extern crate gltf_utils;

use gltf::json::{self, validation};
use std::{fmt, fs, io};

use gltf::Gltf;
use gltf_utils::Source;
use std::error::Error as StdError;
use std::path::Path;

/// Contains parameters for import configuration.
pub mod config;

pub use self::config::Config;
pub use self::config::ValidationStrategy;

/// Error encountered when importing a glTF 2.0 asset.
#[derive(Debug)]
pub enum Error {
    /// A loaded glTF buffer is not of the required length.
    BufferLength(json::Path),

    /// Base 64 decoding error.
    Base64Decoding(base64::DecodeError),

    /// A glTF extension required by the asset has not been enabled by the user.
    ExtensionDisabled(String),

    /// A glTF extension required by the asset is not supported by the library.
    ExtensionUnsupported(String),

    /// The glTF version of the asset is incompatible with the importer.
    IncompatibleVersion(String),

    /// Standard I/O error.
    Io(std::io::Error),

    /// `gltf` crate error.
    Gltf(gltf::Error),

    /// Failure when deserializing .gltf or .glb JSON.
    MalformedJson(json::Error),

    /// The .gltf data is invalid.
    Validation(Vec<(json::Path, validation::Error)>),
}

/// Buffer data returned from `import`.
#[derive(Clone, Debug)]
pub struct Buffers(Vec<Vec<u8>>);

impl Source for Buffers {
    fn source_buffer(&self, buffer: &gltf::Buffer) -> &[u8] {
        &self.0[buffer.index()]
    }
}

impl Buffers {
    /// Obtain the contents of a loaded buffer.
    pub fn buffer(&self, buffer: &gltf::Buffer) -> Option<&[u8]> {
        self.0.get(buffer.index()).map(Vec::as_slice)
    }

    /// Obtain the contents of a loaded buffer view.
    pub fn view(&self, view: &gltf::buffer::View) -> Option<&[u8]> {
        self.buffer(&view.buffer())
            .map(|data| {
                let begin = view.offset();
                let end = begin + view.length();
                &data[begin..end]
            })
    }

    /// Take the loaded buffer data.
    pub fn take(self) -> Vec<Vec<u8>> {
        self.0
    }
}

fn import_impl(path: &Path, config: Config) -> Result<(Gltf, Buffers), Error> {
    let data = read_to_end(path)?;
    if data.starts_with(b"glTF") {
        import_binary(&data, &config, path)
    } else {
        import_standard(&data, &config, path)
    }
}

/// Imports glTF 2.0 with custom configuration.
pub fn import_with_config<P>(path: P, config: Config) -> Result<(Gltf, Buffers), Error>
    where P: AsRef<Path>
{
    import_impl(path.as_ref(), config)
}

/// Imports glTF 2.0 with default configuration.
pub fn import<P>(path: P) -> Result<(Gltf, Buffers), Error>
    where P: AsRef<Path>
{
    import_impl(path.as_ref(), Default::default())
}

fn read_to_end_impl(path: &Path) -> Result<Vec<u8>, Error> {
    use io::Read;
    let file = fs::File::open(path)?;
    let mut reader = io::BufReader::new(file);
    let mut buffer = vec![];
    let _ = reader.read_to_end(&mut buffer)?;
    Ok(buffer)
}

fn read_to_end<P: AsRef<Path>>(path: P) -> Result<Vec<u8>, Error> {
    read_to_end_impl(path.as_ref())
}

fn parse_data_uri(uri: &str) -> Result<Vec<u8>, Error> {
    let encoded = uri.split(",").nth(1).unwrap();
    let decoded = base64::decode(&encoded)?;
    Ok(decoded)
}

fn load_external_buffers(
    base_path: &Path,
    gltf: &Gltf,
    mut bin: Option<Vec<u8>>,
) -> Result<Vec<Vec<u8>>, Error> {
    let mut buffers = vec![];
    for (index, buffer) in gltf.buffers().enumerate() {
        let uri = buffer.uri();
        let data = if uri == "#bin" {
            Ok(bin.take().unwrap())
        } else if uri.starts_with("data:") {
            Ok(parse_data_uri(uri)?)
        } else {
            let path = base_path.parent().unwrap_or(Path::new("./")).join(uri);
            read_to_end(&path)
        }?;
        if data.len() < buffer.length() {
            let path = json::Path::new().field("buffers").index(index);
            return Err(Error::BufferLength(path));
        }
        buffers.push(data);
    }
    Ok(buffers)
}

fn validate_standard(
    unvalidated: gltf::Unvalidated,
    config: &Config,
) -> Result<Gltf, Error> {
    use config::ValidationStrategy;
    Ok(match config.validation_strategy {
        ValidationStrategy::Skip => unsafe { unvalidated.skip_validation() },
        ValidationStrategy::Minimal => unvalidated.validate_minimally()?,
        ValidationStrategy::Complete => unvalidated.validate_completely()?,
    })
}

fn validate_binary(
    unvalidated: gltf::Unvalidated,
    config: &Config,
    has_bin: bool,
) -> Result<Gltf, Error> {
    use config::ValidationStrategy;
    use json::validation::Error as Reason;

    if config.validation_strategy == ValidationStrategy::Skip {
        return Ok(unsafe { unvalidated.skip_validation() });
    }

    let mut errs = vec![];
    {
        let json = unvalidated.as_json();
        for (index, buffer) in json.buffers.iter().enumerate() {
            let path = || {
                json::Path::new()
                    .field("buffers")
                    .index(index)
                    .field("uri")
            };
            match index {
                0 if has_bin => if buffer.uri.is_some() {
                    errs.push((path(), Reason::Missing));
                },
                _ if buffer.uri.is_none() => {
                    errs.push((path(), Reason::Missing));
                },
                _ => {},
            }
        }
    }

    if errs.is_empty() {
        Ok(match config.validation_strategy {
            ValidationStrategy::Minimal => unvalidated.validate_minimally()?,
            ValidationStrategy::Complete => unvalidated.validate_completely()?,
            ValidationStrategy::Skip => unreachable!(),
        })
    } else {
        Err(Error::Validation(errs))
    }
}

fn import_standard<'a>(
    data: &'a [u8],
    config: &Config,
    base_path: &Path,
) -> Result<(Gltf, Buffers), Error> {
    let unvalidated = Gltf::from_slice(data)?;
    let gltf = validate_standard(unvalidated, &config)?;
    let bin = None;
    let mut buffers = Buffers(vec![]);
    for buffer in load_external_buffers(base_path, &gltf, bin)? {
        buffers.0.push(buffer);
    }
    Ok((gltf, buffers))
}

fn import_binary<'a>(
    data: &'a [u8],
    config: &Config,
    base_path: &Path,
) -> Result<(Gltf, Buffers), Error> {
    let gltf::Glb { header: _, json, bin } = gltf::Glb::from_slice(data)?;
    let unvalidated = Gltf::from_slice(json)?;
    let bin = bin.map(|x| x.to_vec());
    let gltf = validate_binary(unvalidated, &config, bin.is_some())?;
    let mut buffers = Buffers(vec![]);
    for buffer in load_external_buffers(base_path, &gltf, bin)? {
        buffers.0.push(buffer);
    }
    Ok((gltf, buffers))
}

impl From<json::Error> for Error {
    fn from(err: json::Error) -> Error {
        Error::MalformedJson(err)
    }
}

impl From<std::io::Error> for Error {
    fn from(err: std::io::Error) -> Error {
        Error::Io(err)
    }
}

impl From<Vec<(json::Path, validation::Error)>> for Error {
    fn from(errs: Vec<(json::Path, validation::Error)>) -> Error {
        Error::Validation(errs)
    }
}

impl From<gltf::Error> for Error {
    fn from(err: gltf::Error) -> Error {
        match err {
            gltf::Error::Validation(errs) => Error::Validation(errs),
            _ => Error::Gltf(err),
        }
    }
}

impl From<base64::DecodeError> for Error {
    fn from(err: base64::DecodeError) -> Error {
        Error::Base64Decoding(err)
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use std::error::Error;
        write!(f, "{}", self.description())
    }
}

impl StdError for Error {
    fn description(&self) -> &str {
        use self::Error::*;
        match *self {
            Base64Decoding(_) => "Base 64 decoding failed",
            BufferLength(_) => "Loaded buffer does not match required length",
            ExtensionDisabled(_) => "Asset requires a disabled extension",
            ExtensionUnsupported(_) => "Assets requires an unsupported extension",
            IncompatibleVersion(_) => "Asset is not glTF version 2.0",
            Io(_) => "I/O error",
            Gltf(_) => "Error from gltf crate",
            MalformedJson(_) => "Malformed .gltf / .glb JSON",
            Validation(_) => "Asset failed validation tests",
        }
    }

    fn cause(&self) -> Option<&StdError> {
        use self::Error::*;
        match *self {
            MalformedJson(ref err) => Some(err),
            Io(ref err) => Some(err),
            _ => None,
        }
    }
}
<|endoftext|>