<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::attr::AttrHelpers;
use dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
use dom::bindings::codegen::InheritTypes::{NodeCast, ElementCast, HTMLScriptElementCast};
use dom::bindings::js::{JS, JSRef, Temporary, OptionalRootable, Root};
use dom::document::{Document, DocumentHelpers};
use dom::element::{Element, AttributeHandlers, ElementHelpers, ParserCreated};
use dom::htmlscriptelement::HTMLScriptElementHelpers;
use dom::node::{Node, NodeHelpers, TrustedNodeAddress};
use dom::servohtmlparser;
use dom::servohtmlparser::ServoHTMLParser;
use dom::types::*;
use page::Page;
use parse::Parser;

use encoding::all::UTF_8;
use encoding::types::{Encoding, DecodeReplace};

use servo_net::resource_task::{Load, LoadData, Payload, Done, ResourceTask};
use servo_msg::constellation_msg::LoadData as MsgLoadData;
use servo_util::task_state;
use servo_util::task_state::InHTMLParser;
use std::ascii::StrAsciiExt;
use std::comm::channel;
use std::str::MaybeOwned;
use url::Url;
use http::headers::HeaderEnum;
use time;
use html5ever::Attribute;
use html5ever::tree_builder::{TreeSink, QuirksMode, NodeOrText, AppendNode, AppendText};
use string_cache::QualName;

pub enum HTMLInput {
    InputString(String),
    InputUrl(Url),
}

// Parses an RFC 2616 compliant date/time string, and returns a localized
// date/time string in a format suitable for document.lastModified.
fn parse_last_modified(timestamp: &str) -> String {
    let format = "%m/%d/%Y %H:%M:%S";

    // RFC 822, updated by RFC 1123
    match time::strptime(timestamp, "%a, %d %b %Y %T %Z") {
        Ok(t) => return t.to_local().strftime(format),
        Err(_) => ()
    }

    // RFC 850, obsoleted by RFC 1036
    match time::strptime(timestamp, "%A, %d-%b-%y %T %Z") {
        Ok(t) => return t.to_local().strftime(format),
        Err(_) => ()
    }

    // ANSI C's asctime() format
    match time::strptime(timestamp, "%c") {
        Ok(t) => t.to_local().strftime(format),
        Err(_) => String::from_str("")
    }
}

trait SinkHelpers {
    fn get_or_create(&self, child: NodeOrText<TrustedNodeAddress>) -> Temporary<Node>;
}

impl SinkHelpers for servohtmlparser::Sink {
    fn get_or_create(&self, child: NodeOrText<TrustedNodeAddress>) -> Temporary<Node> {
        match child {
            AppendNode(n) => Temporary::new(unsafe { JS::from_trusted_node_address(n) }),
            AppendText(t) => {
                let doc = self.document.root();
                let text = Text::new(t, *doc);
                NodeCast::from_temporary(text)
            }
        }
    }
}

impl<'a> TreeSink<TrustedNodeAddress> for servohtmlparser::Sink {
    fn get_document(&mut self) -> TrustedNodeAddress {
        let doc = self.document.root();
        let node: JSRef<Node> = NodeCast::from_ref(*doc);
        node.to_trusted_node_address()
    }

    fn same_node(&self, x: TrustedNodeAddress, y: TrustedNodeAddress) -> bool {
        x == y
    }

    fn elem_name(&self, target: TrustedNodeAddress) -> QualName {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(target).root() };
        let elem: JSRef<Element> = ElementCast::to_ref(*node)
            .expect("tried to get name of non-Element in HTML parsing");
        QualName {
            ns: elem.get_namespace().clone(),
            local: elem.get_local_name().clone(),
        }
    }

    fn create_element(&mut self, name: QualName, attrs: Vec<Attribute>)
            -> TrustedNodeAddress {
        let doc = self.document.root();
        let elem = Element::create(name, None, *doc, ParserCreated).root();

        for attr in attrs.into_iter() {
            elem.set_attribute_from_parser(attr.name, attr.value, None);
        }

        let node: JSRef<Node> = NodeCast::from_ref(*elem);
        node.to_trusted_node_address()
    }

    fn create_comment(&mut self, text: String) -> TrustedNodeAddress {
        let doc = self.document.root();
        let comment = Comment::new(text, *doc);
        let node: Root<Node> = NodeCast::from_temporary(comment).root();
        node.to_trusted_node_address()
    }

    fn append_before_sibling(&mut self,
            sibling: TrustedNodeAddress,
            new_node: NodeOrText<TrustedNodeAddress>) -> Result<(), NodeOrText<TrustedNodeAddress>> {
        // If there is no parent, return the node to the parser.
        let sibling: Root<Node> = unsafe { JS::from_trusted_node_address(sibling).root() };
        let parent = match sibling.parent_node() {
            Some(p) => p.root(),
            None => return Err(new_node),
        };

        let child = self.get_or_create(new_node).root();
        assert!(parent.InsertBefore(*child, Some(*sibling)).is_ok());
        Ok(())
    }

    fn parse_error(&mut self, msg: MaybeOwned<'static>) {
        error!("Parse error: {:s}", msg);
    }

    fn set_quirks_mode(&mut self, mode: QuirksMode) {
        let doc = self.document.root();
        doc.set_quirks_mode(mode);
    }

    fn append(&mut self, parent: TrustedNodeAddress, child: NodeOrText<TrustedNodeAddress>) {
        let parent: Root<Node> = unsafe { JS::from_trusted_node_address(parent).root() };
        let child = self.get_or_create(child).root();

        // FIXME(#3701): Use a simpler algorithm and merge adjacent text nodes
        assert!(parent.AppendChild(*child).is_ok());
    }

    fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) {
        let doc = self.document.root();
        let doc_node: JSRef<Node> = NodeCast::from_ref(*doc);
        let doctype = DocumentType::new(name, Some(public_id), Some(system_id), *doc);
        let node: Root<Node> = NodeCast::from_temporary(doctype).root();

        assert!(doc_node.AppendChild(*node).is_ok());
    }

    fn add_attrs_if_missing(&mut self, target: TrustedNodeAddress, attrs: Vec<Attribute>) {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(target).root() };
        let elem: JSRef<Element> = ElementCast::to_ref(*node)
            .expect("tried to set attrs on non-Element in HTML parsing");
        for attr in attrs.into_iter() {
            elem.set_attribute_from_parser(attr.name, attr.value, None);
        }
    }

    fn remove_from_parent(&mut self, _target: TrustedNodeAddress) {
        error!("remove_from_parent not implemented!");
    }

    fn mark_script_already_started(&mut self, node: TrustedNodeAddress) {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(node).root() };
        let script: Option<JSRef<HTMLScriptElement>> = HTMLScriptElementCast::to_ref(*node);
        script.map(|script| script.mark_already_started());
    }

    fn complete_script(&mut self, node: TrustedNodeAddress) {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(node).root() };
        let script: Option<JSRef<HTMLScriptElement>> = HTMLScriptElementCast::to_ref(*node);
        script.map(|script| script.prepare());
    }
}

// The url from msg_load_data is ignored here
pub fn parse_html(page: &Page,
                  document: JSRef<Document>,
                  input: HTMLInput,
                  resource_task: ResourceTask,
                  msg_load_data: Option<MsgLoadData>) {
    let (base_url, load_response) = match input {
        InputUrl(ref url) => {
            // Wait for the LoadResponse so that the parser knows the final URL.
            let (input_chan, input_port) = channel();
            let mut load_data = LoadData::new(url.clone());
            msg_load_data.map(|m| {
                load_data.headers = m.headers;
                load_data.method = m.method;
                load_data.data = m.data;
            });
            resource_task.send(Load(load_data, input_chan));

            let load_response = input_port.recv();

            debug!("Fetched page; metadata is {:?}", load_response.metadata);

            load_response.metadata.headers.as_ref().map(|headers| {
                let header = headers.iter().find(|h|
                    h.header_name().as_slice().to_ascii_lower() == "last-modified".to_string()
                );

                match header {
                    Some(h) => document.set_last_modified(
                        parse_last_modified(h.header_value().as_slice())),
                    None => {},
                };
            });

            let base_url = load_response.metadata.final_url.clone();

            {
                // Store the final URL before we start parsing, so that DOM routines
                // (e.g. HTMLImageElement::update_image) can resolve relative URLs
                // correctly.
                *page.mut_url() = Some((base_url.clone(), true));
            }

            (Some(base_url), Some(load_response))
        },
        InputString(_) => {
            match *page.url() {
                Some((ref page_url, _)) => (Some(page_url.clone()), None),
                None => (None, None),
            }
        },
    };

    let parser = ServoHTMLParser::new(base_url.clone(), document).root();
    let parser: JSRef<ServoHTMLParser> = *parser;

    task_state::enter(InHTMLParser);

    match input {
        InputString(s) => {
            parser.parse_chunk(s);
        }
        InputUrl(url) => {
            let load_response = load_response.unwrap();
            match load_response.metadata.content_type {
                Some((ref t, _)) if t.as_slice().eq_ignore_ascii_case("image") => {
                    let page = format!("", base_url.as_ref().unwrap().serialize());
                    parser.parse_chunk(page);
                },
                _ => {
                    for msg in load_response.progress_port.iter() {
                        match msg {
                            Payload(data) => {
                                // FIXME: use Vec<u8> (html5ever #34)
                                let data = UTF_8.decode(data.as_slice(), DecodeReplace).unwrap();
                                parser.parse_chunk(data);
                            }
                            Done(Err(err)) => {
                                fail!("Failed to load page URL {:s}, error: {:s}", url.serialize(), err);
                            }
                            Done(Ok(())) => break,
                        }
                    }
                }
            }
        }
    }

    parser.finish();

    task_state::exit(InHTMLParser);

    debug!("finished parsing");
}
<commit_msg>auto merge of #3900 : kmcallister/servo/parse-error, r=Ms2ger<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::attr::AttrHelpers;
use dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
use dom::bindings::codegen::InheritTypes::{NodeCast, ElementCast, HTMLScriptElementCast};
use dom::bindings::js::{JS, JSRef, Temporary, OptionalRootable, Root};
use dom::document::{Document, DocumentHelpers};
use dom::element::{Element, AttributeHandlers, ElementHelpers, ParserCreated};
use dom::htmlscriptelement::HTMLScriptElementHelpers;
use dom::node::{Node, NodeHelpers, TrustedNodeAddress};
use dom::servohtmlparser;
use dom::servohtmlparser::ServoHTMLParser;
use dom::types::*;
use page::Page;
use parse::Parser;

use encoding::all::UTF_8;
use encoding::types::{Encoding, DecodeReplace};

use servo_net::resource_task::{Load, LoadData, Payload, Done, ResourceTask};
use servo_msg::constellation_msg::LoadData as MsgLoadData;
use servo_util::task_state;
use servo_util::task_state::InHTMLParser;
use std::ascii::StrAsciiExt;
use std::comm::channel;
use std::str::MaybeOwned;
use url::Url;
use http::headers::HeaderEnum;
use time;
use html5ever::Attribute;
use html5ever::tree_builder::{TreeSink, QuirksMode, NodeOrText, AppendNode, AppendText};
use string_cache::QualName;

pub enum HTMLInput {
    InputString(String),
    InputUrl(Url),
}

// Parses an RFC 2616 compliant date/time string, and returns a localized
// date/time string in a format suitable for document.lastModified.
fn parse_last_modified(timestamp: &str) -> String {
    let format = "%m/%d/%Y %H:%M:%S";

    // RFC 822, updated by RFC 1123
    match time::strptime(timestamp, "%a, %d %b %Y %T %Z") {
        Ok(t) => return t.to_local().strftime(format),
        Err(_) => ()
    }

    // RFC 850, obsoleted by RFC 1036
    match time::strptime(timestamp, "%A, %d-%b-%y %T %Z") {
        Ok(t) => return t.to_local().strftime(format),
        Err(_) => ()
    }

    // ANSI C's asctime() format
    match time::strptime(timestamp, "%c") {
        Ok(t) => t.to_local().strftime(format),
        Err(_) => String::from_str("")
    }
}

trait SinkHelpers {
    fn get_or_create(&self, child: NodeOrText<TrustedNodeAddress>) -> Temporary<Node>;
}

impl SinkHelpers for servohtmlparser::Sink {
    fn get_or_create(&self, child: NodeOrText<TrustedNodeAddress>) -> Temporary<Node> {
        match child {
            AppendNode(n) => Temporary::new(unsafe { JS::from_trusted_node_address(n) }),
            AppendText(t) => {
                let doc = self.document.root();
                let text = Text::new(t, *doc);
                NodeCast::from_temporary(text)
            }
        }
    }
}

impl<'a> TreeSink<TrustedNodeAddress> for servohtmlparser::Sink {
    fn get_document(&mut self) -> TrustedNodeAddress {
        let doc = self.document.root();
        let node: JSRef<Node> = NodeCast::from_ref(*doc);
        node.to_trusted_node_address()
    }

    fn same_node(&self, x: TrustedNodeAddress, y: TrustedNodeAddress) -> bool {
        x == y
    }

    fn elem_name(&self, target: TrustedNodeAddress) -> QualName {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(target).root() };
        let elem: JSRef<Element> = ElementCast::to_ref(*node)
            .expect("tried to get name of non-Element in HTML parsing");
        QualName {
            ns: elem.get_namespace().clone(),
            local: elem.get_local_name().clone(),
        }
    }

    fn create_element(&mut self, name: QualName, attrs: Vec<Attribute>)
            -> TrustedNodeAddress {
        let doc = self.document.root();
        let elem = Element::create(name, None, *doc, ParserCreated).root();

        for attr in attrs.into_iter() {
            elem.set_attribute_from_parser(attr.name, attr.value, None);
        }

        let node: JSRef<Node> = NodeCast::from_ref(*elem);
        node.to_trusted_node_address()
    }

    fn create_comment(&mut self, text: String) -> TrustedNodeAddress {
        let doc = self.document.root();
        let comment = Comment::new(text, *doc);
        let node: Root<Node> = NodeCast::from_temporary(comment).root();
        node.to_trusted_node_address()
    }

    fn append_before_sibling(&mut self,
            sibling: TrustedNodeAddress,
            new_node: NodeOrText<TrustedNodeAddress>) -> Result<(), NodeOrText<TrustedNodeAddress>> {
        // If there is no parent, return the node to the parser.
        let sibling: Root<Node> = unsafe { JS::from_trusted_node_address(sibling).root() };
        let parent = match sibling.parent_node() {
            Some(p) => p.root(),
            None => return Err(new_node),
        };

        let child = self.get_or_create(new_node).root();
        assert!(parent.InsertBefore(*child, Some(*sibling)).is_ok());
        Ok(())
    }

    fn parse_error(&mut self, msg: MaybeOwned<'static>) {
        debug!("Parse error: {:s}", msg);
    }

    fn set_quirks_mode(&mut self, mode: QuirksMode) {
        let doc = self.document.root();
        doc.set_quirks_mode(mode);
    }

    fn append(&mut self, parent: TrustedNodeAddress, child: NodeOrText<TrustedNodeAddress>) {
        let parent: Root<Node> = unsafe { JS::from_trusted_node_address(parent).root() };
        let child = self.get_or_create(child).root();

        // FIXME(#3701): Use a simpler algorithm and merge adjacent text nodes
        assert!(parent.AppendChild(*child).is_ok());
    }

    fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) {
        let doc = self.document.root();
        let doc_node: JSRef<Node> = NodeCast::from_ref(*doc);
        let doctype = DocumentType::new(name, Some(public_id), Some(system_id), *doc);
        let node: Root<Node> = NodeCast::from_temporary(doctype).root();

        assert!(doc_node.AppendChild(*node).is_ok());
    }

    fn add_attrs_if_missing(&mut self, target: TrustedNodeAddress, attrs: Vec<Attribute>) {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(target).root() };
        let elem: JSRef<Element> = ElementCast::to_ref(*node)
            .expect("tried to set attrs on non-Element in HTML parsing");
        for attr in attrs.into_iter() {
            elem.set_attribute_from_parser(attr.name, attr.value, None);
        }
    }

    fn remove_from_parent(&mut self, _target: TrustedNodeAddress) {
        error!("remove_from_parent not implemented!");
    }

    fn mark_script_already_started(&mut self, node: TrustedNodeAddress) {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(node).root() };
        let script: Option<JSRef<HTMLScriptElement>> = HTMLScriptElementCast::to_ref(*node);
        script.map(|script| script.mark_already_started());
    }

    fn complete_script(&mut self, node: TrustedNodeAddress) {
        let node: Root<Node> = unsafe { JS::from_trusted_node_address(node).root() };
        let script: Option<JSRef<HTMLScriptElement>> = HTMLScriptElementCast::to_ref(*node);
        script.map(|script| script.prepare());
    }
}

// The url from msg_load_data is ignored here
pub fn parse_html(page: &Page,
                  document: JSRef<Document>,
                  input: HTMLInput,
                  resource_task: ResourceTask,
                  msg_load_data: Option<MsgLoadData>) {
    let (base_url, load_response) = match input {
        InputUrl(ref url) => {
            // Wait for the LoadResponse so that the parser knows the final URL.
            let (input_chan, input_port) = channel();
            let mut load_data = LoadData::new(url.clone());
            msg_load_data.map(|m| {
                load_data.headers = m.headers;
                load_data.method = m.method;
                load_data.data = m.data;
            });
            resource_task.send(Load(load_data, input_chan));

            let load_response = input_port.recv();

            debug!("Fetched page; metadata is {:?}", load_response.metadata);

            load_response.metadata.headers.as_ref().map(|headers| {
                let header = headers.iter().find(|h|
                    h.header_name().as_slice().to_ascii_lower() == "last-modified".to_string()
                );

                match header {
                    Some(h) => document.set_last_modified(
                        parse_last_modified(h.header_value().as_slice())),
                    None => {},
                };
            });

            let base_url = load_response.metadata.final_url.clone();

            {
                // Store the final URL before we start parsing, so that DOM routines
                // (e.g. HTMLImageElement::update_image) can resolve relative URLs
                // correctly.
                *page.mut_url() = Some((base_url.clone(), true));
            }

            (Some(base_url), Some(load_response))
        },
        InputString(_) => {
            match *page.url() {
                Some((ref page_url, _)) => (Some(page_url.clone()), None),
                None => (None, None),
            }
        },
    };

    let parser = ServoHTMLParser::new(base_url.clone(), document).root();
    let parser: JSRef<ServoHTMLParser> = *parser;

    task_state::enter(InHTMLParser);

    match input {
        InputString(s) => {
            parser.parse_chunk(s);
        }
        InputUrl(url) => {
            let load_response = load_response.unwrap();
            match load_response.metadata.content_type {
                Some((ref t, _)) if t.as_slice().eq_ignore_ascii_case("image") => {
                    let page = format!("", base_url.as_ref().unwrap().serialize());
                    parser.parse_chunk(page);
                },
                _ => {
                    for msg in load_response.progress_port.iter() {
                        match msg {
                            Payload(data) => {
                                // FIXME: use Vec<u8> (html5ever #34)
                                let data = UTF_8.decode(data.as_slice(), DecodeReplace).unwrap();
                                parser.parse_chunk(data);
                            }
                            Done(Err(err)) => {
                                fail!("Failed to load page URL {:s}, error: {:s}", url.serialize(), err);
                            }
                            Done(Ok(())) => break,
                        }
                    }
                }
            }
        }
    }

    parser.finish();

    task_state::exit(InHTMLParser);

    debug!("finished parsing");
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::config::MAX_APPLICATION_MESSAGE_SIZE;
use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncConfig {
    pub data_streaming_service: DataStreamingServiceConfig,
    pub aptos_data_client: AptosDataClientConfig,
    pub state_sync_driver: StateSyncDriverConfig,
    pub storage_service: StorageServiceConfig,
}

/// The bootstrapping mode determines how the node will bootstrap to the latest
/// blockchain state, e.g., directly download the latest states.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum BootstrappingMode {
    ApplyTransactionOutputsFromGenesis, // Applies transaction outputs (starting at genesis)
    DownloadLatestStates, // Downloads the state keys and values (at the latest version)
    ExecuteTransactionsFromGenesis, // Executes transactions (starting at genesis)
}

impl BootstrappingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            BootstrappingMode::ApplyTransactionOutputsFromGenesis => {
                "apply_transaction_outputs_from_genesis"
            }
            BootstrappingMode::DownloadLatestStates => "download_latest_states",
            BootstrappingMode::ExecuteTransactionsFromGenesis => {
                "execute_transactions_from_genesis"
            }
        }
    }
}

/// The continuous syncing mode determines how the node will stay up-to-date
/// once it has bootstrapped and the blockchain continues to grow, e.g.,
/// continuously executing all transactions.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum ContinuousSyncingMode {
    ApplyTransactionOutputs, // Applies transaction outputs to stay up-to-date
    ExecuteTransactions,     // Executes transactions to stay up-to-date
}

impl ContinuousSyncingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            ContinuousSyncingMode::ApplyTransactionOutputs => "apply_transaction_outputs",
            ContinuousSyncingMode::ExecuteTransactions => "execute_transactions",
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncDriverConfig {
    pub bootstrapping_mode: BootstrappingMode, // The mode by which to bootstrap
    pub commit_notification_timeout_ms: u64, // The max time taken to process a commit notification
    pub continuous_syncing_mode: ContinuousSyncingMode, // The mode by which to sync after bootstrapping
    pub progress_check_interval_ms: u64, // The interval (ms) at which to check state sync progress
    pub max_connection_deadline_secs: u64, // The max time (secs) to wait for connections from peers
    pub max_consecutive_stream_notifications: u64, // The max number of notifications to process per driver loop
    pub max_pending_data_chunks: u64, // The max number of data chunks pending execution or commit
    pub max_stream_wait_time_ms: u64, // The max time (ms) to wait for a data stream notification
    pub num_versions_to_skip_snapshot_sync: u64, // The version lag we'll tolerate before snapshot syncing
}

/// The default state sync driver config will be the one that gets (and keeps)
/// the node up-to-date as quickly and cheaply as possible.
impl Default for StateSyncDriverConfig {
    fn default() -> Self {
        Self {
            bootstrapping_mode: BootstrappingMode::ApplyTransactionOutputsFromGenesis,
            commit_notification_timeout_ms: 5000,
            continuous_syncing_mode: ContinuousSyncingMode::ApplyTransactionOutputs,
            progress_check_interval_ms: 100,
            max_connection_deadline_secs: 10,
            max_consecutive_stream_notifications: 10,
            max_pending_data_chunks: 100,
            max_stream_wait_time_ms: 5000,
            num_versions_to_skip_snapshot_sync: 100_000_000, // At 5k TPS, this allows a node to fail for about 6 hours.
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageServiceConfig {
    pub max_concurrent_requests: u64, // Max num of concurrent storage server tasks
    pub max_epoch_chunk_size: u64,    // Max num of epoch ending ledger infos per chunk
    pub max_lru_cache_size: u64,      // Max num of items in the lru cache before eviction
    pub max_network_channel_size: u64, // Max num of pending network messages
    pub max_network_chunk_bytes: u64, // Max num of bytes to send per network message
    pub max_state_chunk_size: u64,    // Max num of state keys and values per chunk
    pub max_subscription_period_ms: u64, // Max period (ms) of pending subscription requests
    pub max_transaction_chunk_size: u64, // Max num of transactions per chunk
    pub max_transaction_output_chunk_size: u64, // Max num of transaction outputs per chunk
    pub storage_summary_refresh_interval_ms: u64, // The interval (ms) to refresh the storage summary
}

impl Default for StorageServiceConfig {
    fn default() -> Self {
        Self {
            max_concurrent_requests: 4000,
            max_epoch_chunk_size: 100,
            max_lru_cache_size: 500, // At ~0.6MiB per chunk, this should take no more than 0.5GiB
            max_network_channel_size: 4000,
            max_network_chunk_bytes: MAX_APPLICATION_MESSAGE_SIZE as u64,
            max_state_chunk_size: 2000,
            max_subscription_period_ms: 5000,
            max_transaction_chunk_size: 2000,
            max_transaction_output_chunk_size: 2000,
            storage_summary_refresh_interval_ms: 50,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct DataStreamingServiceConfig {
    // The interval (milliseconds) at which to refresh the global data summary.
    pub global_summary_refresh_interval_ms: u64,

    // Maximum number of concurrent data client requests (per stream).
    pub max_concurrent_requests: u64,

    // Maximum number of concurrent data client requests (per stream) for state keys/values.
    pub max_concurrent_state_requests: u64,

    // Maximum channel sizes for each data stream listener. If messages are not
    // consumed, they will be dropped (oldest messages first). The remaining
    // messages will be retrieved using FIFO ordering.
    pub max_data_stream_channel_sizes: u64,

    // Maximum number of retries for a single client request before a data
    // stream will terminate.
    pub max_request_retry: u64,

    // Maximum number of notification ID to response context mappings held in
    // memory. Once the number grows beyond this value, garbage collection occurs.
    pub max_notification_id_mappings: u64,

    // The interval (milliseconds) at which to check the progress of each stream.
    pub progress_check_interval_ms: u64,
}

impl Default for DataStreamingServiceConfig {
    fn default() -> Self {
        Self {
            global_summary_refresh_interval_ms: 50,
            max_concurrent_requests: 3,
            max_concurrent_state_requests: 6,
            max_data_stream_channel_sizes: 300,
            max_request_retry: 3,
            max_notification_id_mappings: 300,
            progress_check_interval_ms: 100,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct AptosDataClientConfig {
    pub max_num_in_flight_priority_polls: u64, // Max num of in-flight polls for priority peers
    pub max_num_in_flight_regular_polls: u64,  // Max num of in-flight polls for regular peers
    pub response_timeout_ms: u64, // Timeout (in milliseconds) when waiting for a response
    pub summary_poll_interval_ms: u64, // Interval (in milliseconds) between data summary polls
    pub use_compression: bool,    // Whether or not to request compression for incoming data
}

impl Default for AptosDataClientConfig {
    fn default() -> Self {
        Self {
            max_num_in_flight_priority_polls: 10,
            max_num_in_flight_regular_polls: 10,
            response_timeout_ms: 5000,
            summary_poll_interval_ms: 200,
            use_compression: true,
        }
    }
}
<commit_msg>[State Sync] Improve the default config values.<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

// The maximum message size per state sync message
pub const MAX_MESSAGE_SIZE: usize = 12 * 1024 * 1024; /* 12 MiB */

#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncConfig {
    pub data_streaming_service: DataStreamingServiceConfig,
    pub aptos_data_client: AptosDataClientConfig,
    pub state_sync_driver: StateSyncDriverConfig,
    pub storage_service: StorageServiceConfig,
}

/// The bootstrapping mode determines how the node will bootstrap to the latest
/// blockchain state, e.g., directly download the latest states.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum BootstrappingMode {
    ApplyTransactionOutputsFromGenesis, // Applies transaction outputs (starting at genesis)
    DownloadLatestStates, // Downloads the state keys and values (at the latest version)
    ExecuteTransactionsFromGenesis, // Executes transactions (starting at genesis)
}

impl BootstrappingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            BootstrappingMode::ApplyTransactionOutputsFromGenesis => {
                "apply_transaction_outputs_from_genesis"
            }
            BootstrappingMode::DownloadLatestStates => "download_latest_states",
            BootstrappingMode::ExecuteTransactionsFromGenesis => {
                "execute_transactions_from_genesis"
            }
        }
    }
}

/// The continuous syncing mode determines how the node will stay up-to-date
/// once it has bootstrapped and the blockchain continues to grow, e.g.,
/// continuously executing all transactions.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum ContinuousSyncingMode {
    ApplyTransactionOutputs, // Applies transaction outputs to stay up-to-date
    ExecuteTransactions,     // Executes transactions to stay up-to-date
}

impl ContinuousSyncingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            ContinuousSyncingMode::ApplyTransactionOutputs => "apply_transaction_outputs",
            ContinuousSyncingMode::ExecuteTransactions => "execute_transactions",
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncDriverConfig {
    pub bootstrapping_mode: BootstrappingMode, // The mode by which to bootstrap
    pub commit_notification_timeout_ms: u64, // The max time taken to process a commit notification
    pub continuous_syncing_mode: ContinuousSyncingMode, // The mode by which to sync after bootstrapping
    pub progress_check_interval_ms: u64, // The interval (ms) at which to check state sync progress
    pub max_connection_deadline_secs: u64, // The max time (secs) to wait for connections from peers
    pub max_consecutive_stream_notifications: u64, // The max number of notifications to process per driver loop
    pub max_pending_data_chunks: u64, // The max number of data chunks pending execution or commit
    pub max_stream_wait_time_ms: u64, // The max time (ms) to wait for a data stream notification
    pub num_versions_to_skip_snapshot_sync: u64, // The version lag we'll tolerate before snapshot syncing
}

/// The default state sync driver config will be the one that gets (and keeps)
/// the node up-to-date as quickly and cheaply as possible.
impl Default for StateSyncDriverConfig {
    fn default() -> Self {
        Self {
            bootstrapping_mode: BootstrappingMode::ExecuteTransactionsFromGenesis,
            commit_notification_timeout_ms: 5000,
            continuous_syncing_mode: ContinuousSyncingMode::ExecuteTransactions,
            progress_check_interval_ms: 100,
            max_connection_deadline_secs: 10,
            max_consecutive_stream_notifications: 10,
            max_pending_data_chunks: 100,
            max_stream_wait_time_ms: 5000,
            num_versions_to_skip_snapshot_sync: 100_000_000, // At 5k TPS, this allows a node to fail for about 6 hours.
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageServiceConfig {
    pub max_concurrent_requests: u64, // Max num of concurrent storage server tasks
    pub max_epoch_chunk_size: u64,    // Max num of epoch ending ledger infos per chunk
    pub max_lru_cache_size: u64,      // Max num of items in the lru cache before eviction
    pub max_network_channel_size: u64, // Max num of pending network messages
    pub max_network_chunk_bytes: u64, // Max num of bytes to send per network message
    pub max_state_chunk_size: u64,    // Max num of state keys and values per chunk
    pub max_subscription_period_ms: u64, // Max period (ms) of pending subscription requests
    pub max_transaction_chunk_size: u64, // Max num of transactions per chunk
    pub max_transaction_output_chunk_size: u64, // Max num of transaction outputs per chunk
    pub storage_summary_refresh_interval_ms: u64, // The interval (ms) to refresh the storage summary
}

impl Default for StorageServiceConfig {
    fn default() -> Self {
        Self {
            max_concurrent_requests: 4000,
            max_epoch_chunk_size: 100,
            max_lru_cache_size: 500, // At ~0.6MiB per chunk, this should take no more than 0.5GiB
            max_network_channel_size: 4000,
            max_network_chunk_bytes: MAX_MESSAGE_SIZE as u64,
            max_state_chunk_size: 2000,
            max_subscription_period_ms: 5000,
            max_transaction_chunk_size: 2000,
            max_transaction_output_chunk_size: 2000,
            storage_summary_refresh_interval_ms: 50,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct DataStreamingServiceConfig {
    // The interval (milliseconds) at which to refresh the global data summary.
    pub global_summary_refresh_interval_ms: u64,

    // Maximum number of concurrent data client requests (per stream).
    pub max_concurrent_requests: u64,

    // Maximum number of concurrent data client requests (per stream) for state keys/values.
    pub max_concurrent_state_requests: u64,

    // Maximum channel sizes for each data stream listener. If messages are not
    // consumed, they will be dropped (oldest messages first). The remaining
    // messages will be retrieved using FIFO ordering.
    pub max_data_stream_channel_sizes: u64,

    // Maximum number of retries for a single client request before a data
    // stream will terminate.
    pub max_request_retry: u64,

    // Maximum number of notification ID to response context mappings held in
    // memory. Once the number grows beyond this value, garbage collection occurs.
    pub max_notification_id_mappings: u64,

    // The interval (milliseconds) at which to check the progress of each stream.
    pub progress_check_interval_ms: u64,
}

impl Default for DataStreamingServiceConfig {
    fn default() -> Self {
        Self {
            global_summary_refresh_interval_ms: 50,
            max_concurrent_requests: 3,
            max_concurrent_state_requests: 6,
            max_data_stream_channel_sizes: 300,
            max_request_retry: 3,
            max_notification_id_mappings: 300,
            progress_check_interval_ms: 100,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct AptosDataClientConfig {
    pub max_num_in_flight_priority_polls: u64, // Max num of in-flight polls for priority peers
    pub max_num_in_flight_regular_polls: u64,  // Max num of in-flight polls for regular peers
    pub response_timeout_ms: u64, // Timeout (in milliseconds) when waiting for a response
    pub summary_poll_interval_ms: u64, // Interval (in milliseconds) between data summary polls
    pub use_compression: bool,    // Whether or not to request compression for incoming data
}

impl Default for AptosDataClientConfig {
    fn default() -> Self {
        Self {
            max_num_in_flight_priority_polls: 10,
            max_num_in_flight_regular_polls: 10,
            response_timeout_ms: 10000,
            summary_poll_interval_ms: 200,
            use_compression: true,
        }
    }
}
<|endoftext|>


<commit_before>use traits::*;
use bf_lexer::*;
extern crate std;
use std::io;
use std::io::Write;
use std::io::Read;

// Struct for our Virtual Machine that interprets
//  the Brainfuck tokens
pub struct BFVM {

    // Memory storage for the BF programs to use
    //  Note: 60000 bytes is double standard BF size
    mem : [i8; 60000],

    // Current location BF program is looking at
    //  in memory.
    data_ptr : usize,

    // Current location the VM is running instructions
    //  from (using it's token list, not 'mem' memory)
    inst_ptr : usize,

    // Settings that can be changed via input args
    settings : VMSettings
}

// Extra settings for the VM
//  Should these just in the BFVM struct?
pub struct VMSettings {

    // Controls if we show the prompt or not.
    // Input from a file should naturally work by
    //  controlling stdin via a pipe could use
    //  isatty to determine this automatically
    //  later on.
    pub prompt_for_input : bool,

}

impl VMSettings {
    pub fn new() -> VMSettings {
        VMSettings {
            prompt_for_input : false,
        }
    }
}

impl VM<BFToken> for BFVM {

    // This will loop through the tokens until
    //  the instruction pointer reaches the end
    //  of the token vector.
    fn run(&mut self, data : Vec<BFToken>) -> VMResult {

        //println!("Tokens: {:?}", data);

        let mut reader = io::stdin();
        let mut writer = io::stdout();

        while self.inst_ptr < data.len() {
            let result = self.step(&data, &mut reader, &mut writer);
            match result {
                VMResult::Success => continue,
                _ => return result,
            }
        }

        VMResult::Success
    }
}

impl BFVM {
    // Creates a new BFVM.
    pub fn new(settings : VMSettings) -> BFVM {
        BFVM {

            // Initialise memory to 0
            mem : [0; 60000],

            // Start data pointer in middle of memory, so it can
            //  go backwards.
            data_ptr : 30000,

            // Instruction pointer also needs to be 0
            inst_ptr : 0,

            settings : settings
        }
    }

    // Interprets the current token.
    pub fn step(&mut self, data : &Vec<BFToken>, reader : &mut io::Stdin, writer : &mut io::Stdout) -> VMResult {
        // Import enum -> Allows for using enum values without
        //  BFTokenType:: prefix
        use bf_lexer::BFTokenType::*;

        // Get the current token
        let token = &data[self.inst_ptr];

        match token.token_type {

            // >    Increments data pointer
            IncrementPtr(x) => self.data_ptr+=x,

            // <    Decrements data pointer
            DecrementPtr(x) => self.data_ptr-=x,

            // +    Wrapping adds 1 to cell that data pointer is pointing to
            IncrementData(x) => self.mem[self.data_ptr] = self.mem[self.data_ptr].wrapping_add((x % 255) as i8),

            // -    Wrapping subtracts 1 from cell
            DecrementData(x) => self.mem[self.data_ptr] = self.mem[self.data_ptr].wrapping_sub((x % 255) as i8),

            // .    Prints the current cell as a character to stdout (65 - A)
            Output => {
                // Write current cell to stdout as a byte
                let data = &[self.mem[self.data_ptr] as u8];
                writer.write(data).expect("Unable to write to STDOUT");
            },

            // ,    Reads input from stdin and puts it into current cell
            Input => {
                if self.settings.prompt_for_input {
                    print!("\n> ");
                    io::stdout().flush().ok().expect("Could not flush stdout");
                }
                // Read one byte from stdin as a signed byte and store it
                let mut buffer = [0u8; 1];
                reader.read(&mut buffer[..]).expect("Unable to read from STDIN");
                self.mem[self.data_ptr] = buffer[0] as i8;
            },

            // [     If current data cell is 0 skip to matching ]
            LoopStart(x) => {
                if self.mem[self.data_ptr] == 0 {
                    self.inst_ptr = x;
                }
            },

            // ]     If current data cell isn't 0 skip to matching [
            LoopEnd(x) => {
                if self.mem[self.data_ptr] != 0 {
                    self.inst_ptr = x;
                }
            },

            // Optimisation - Sets current cell to 0
            SetCurrent(x) => {
                self.mem[self.data_ptr] = x;
            },

            // Optimisation - Adds current cell contents to cell offset by +x
            AddCurrentUp(x) => {
                self.mem[self.data_ptr + x] += self.mem[self.data_ptr];
                self.mem[self.data_ptr] = 0;
            },

            // Optimisation - Adds current cell contents to cell offset by -x
            AddCurrentDown(x) => {
                self.mem[self.data_ptr - x] += self.mem[self.data_ptr];
                self.mem[self.data_ptr] = 0;
            }
        }

        self.inst_ptr+=1;

        VMResult::Success
    }
}<commit_msg>Fixed issue #2. Adding cells now wraps.<commit_after>use traits::*;
use bf_lexer::*;
extern crate std;
use std::io;
use std::io::Write;
use std::io::Read;

// Struct for our Virtual Machine that interprets
//  the Brainfuck tokens
pub struct BFVM {

    // Memory storage for the BF programs to use
    //  Note: 60000 bytes is double standard BF size
    mem : [i8; 60000],

    // Current location BF program is looking at
    //  in memory.
    data_ptr : usize,

    // Current location the VM is running instructions
    //  from (using it's token list, not 'mem' memory)
    inst_ptr : usize,

    // Settings that can be changed via input args
    settings : VMSettings
}

// Extra settings for the VM
//  Should these just in the BFVM struct?
pub struct VMSettings {

    // Controls if we show the prompt or not.
    // Input from a file should naturally work by
    //  controlling stdin via a pipe could use
    //  isatty to determine this automatically
    //  later on.
    pub prompt_for_input : bool,

}

impl VMSettings {
    pub fn new() -> VMSettings {
        VMSettings {
            prompt_for_input : false,
        }
    }
}

impl VM<BFToken> for BFVM {

    // This will loop through the tokens until
    //  the instruction pointer reaches the end
    //  of the token vector.
    fn run(&mut self, data : Vec<BFToken>) -> VMResult {

        //println!("Tokens: {:?}", data);

        let mut reader = io::stdin();
        let mut writer = io::stdout();

        while self.inst_ptr < data.len() {
            let result = self.step(&data, &mut reader, &mut writer);
            match result {
                VMResult::Success => continue,
                _ => return result,
            }
        }

        VMResult::Success
    }
}

impl BFVM {
    // Creates a new BFVM.
    pub fn new(settings : VMSettings) -> BFVM {
        BFVM {

            // Initialise memory to 0
            mem : [0; 60000],

            // Start data pointer in middle of memory, so it can
            //  go backwards.
            data_ptr : 30000,

            // Instruction pointer also needs to be 0
            inst_ptr : 0,

            settings : settings
        }
    }

    // Interprets the current token.
    pub fn step(&mut self, data : &Vec<BFToken>, reader : &mut io::Stdin, writer : &mut io::Stdout) -> VMResult {
        // Import enum -> Allows for using enum values without
        //  BFTokenType:: prefix
        use bf_lexer::BFTokenType::*;

        // Get the current token
        let token = &data[self.inst_ptr];

        match token.token_type {

            // >    Increments data pointer
            IncrementPtr(x) => self.data_ptr+=x,

            // <    Decrements data pointer
            DecrementPtr(x) => self.data_ptr-=x,

            // +    Wrapping adds 1 to cell that data pointer is pointing to
            IncrementData(x) => self.mem[self.data_ptr] = self.mem[self.data_ptr].wrapping_add((x % 255) as i8),

            // -    Wrapping subtracts 1 from cell
            DecrementData(x) => self.mem[self.data_ptr] = self.mem[self.data_ptr].wrapping_sub((x % 255) as i8),

            // .    Prints the current cell as a character to stdout (65 - A)
            Output => {
                // Write current cell to stdout as a byte
                let data = &[self.mem[self.data_ptr] as u8];
                writer.write(data).expect("Unable to write to STDOUT");
            },

            // ,    Reads input from stdin and puts it into current cell
            Input => {
                if self.settings.prompt_for_input {
                    print!("\n> ");
                    io::stdout().flush().ok().expect("Could not flush stdout");
                }
                // Read one byte from stdin as a signed byte and store it
                let mut buffer = [0u8; 1];
                reader.read(&mut buffer[..]).expect("Unable to read from STDIN");
                self.mem[self.data_ptr] = buffer[0] as i8;
            },

            // [     If current data cell is 0 skip to matching ]
            LoopStart(x) => {
                if self.mem[self.data_ptr] == 0 {
                    self.inst_ptr = x;
                }
            },

            // ]     If current data cell isn't 0 skip to matching [
            LoopEnd(x) => {
                if self.mem[self.data_ptr] != 0 {
                    self.inst_ptr = x;
                }
            },

            // Optimisation - Sets current cell to 0
            SetCurrent(x) => {
                self.mem[self.data_ptr] = x;
            },

            // Optimisation - Adds current cell contents to cell offset by +x
            AddCurrentUp(x) => {
                self.mem[self.data_ptr + x] = self.mem[self.data_ptr + x].wrapping_add(self.mem[self.data_ptr]);
                self.mem[self.data_ptr] = 0;
            },

            // Optimisation - Adds current cell contents to cell offset by -x
            AddCurrentDown(x) => {
                self.mem[self.data_ptr - x] = self.mem[self.data_ptr - x].wrapping_add(self.mem[self.data_ptr]);
                self.mem[self.data_ptr] = 0;
            }
        }

        self.inst_ptr+=1;

        VMResult::Success
    }
}<|endoftext|>


<commit_before>use super::*;

// test param_list
// fn a() {}
// fn b(x: i32) {}
// fn c(x: i32, ) {}
// fn d(x: i32, y: ()) {}
pub(super) fn param_list(p: &mut Parser) {
    list_(p, Flavor::Normal)
}

// test param_list_opt_patterns
// fn foo<F: FnMut(&mut Foo<'a>)>(){}
pub(super) fn param_list_opt_patterns(p: &mut Parser) {
    list_(p, Flavor::OptionalPattern)
}

pub(super) fn param_list_opt_types(p: &mut Parser) {
    list_(p, Flavor::OptionalType)
}

#[derive(Clone, Copy, Eq, PartialEq)]
enum Flavor {
    OptionalType,
    OptionalPattern,
    Normal,
}

impl Flavor {
    fn type_required(self) -> bool {
        match self {
            Flavor::OptionalType => false,
            _ => true,
        }
    }
}

fn list_(p: &mut Parser, flavor: Flavor) {
    let (bra, ket) = if flavor.type_required() {
        (L_PAREN, R_PAREN)
    } else {
        (PIPE, PIPE)
    };
    assert!(p.at(bra));
    let m = p.start();
    p.bump();
    if flavor.type_required() {
        opt_self_param(p);
    }
    while !p.at(EOF) && !p.at(ket) {
        if !p.at_ts(VALUE_PARAMETER_FIRST) {
            p.error("expected value parameter");
            break;
        }
        value_parameter(p, flavor);
        if !p.at(ket) {
            p.expect(COMMA);
        }
    }
    p.expect(ket);
    m.complete(p, PARAM_LIST);
}

const VALUE_PARAMETER_FIRST: TokenSet = patterns::PATTERN_FIRST.union(types::TYPE_FIRST);

fn value_parameter(p: &mut Parser, flavor: Flavor) {
    let m = p.start();
    match flavor {
        Flavor::OptionalType | Flavor::Normal => {
            patterns::pattern(p);
            if p.at(COLON) || flavor.type_required() {
                types::ascription(p)
            }
        }
        // test value_parameters_no_patterns
        // type F = Box<Fn(a: i32, &b: &i32, &mut c: &i32, ())>;
        Flavor::OptionalPattern => {
            let la0 = p.current();
            let la1 = p.nth(1);
            let la2 = p.nth(2);
            let la3 = p.nth(3);
            if la0 == IDENT && la1 == COLON
                || la0 == AMP && la1 == IDENT && la2 == COLON
                || la0 == AMP && la1 == MUT_KW && la2 == IDENT && la3 == COLON
            {
                patterns::pattern(p);
                types::ascription(p);
            } else {
                types::type_(p);
            }
        }
    }
    m.complete(p, PARAM);
}

// test self_param
// impl S {
//     fn a(self) {}
//     fn b(&self,) {}
//     fn c(&'a self,) {}
//     fn d(&'a mut self, x: i32) {}
//     fn e(mut self) {}
// }
fn opt_self_param(p: &mut Parser) {
    let m;
    if p.at(SELF_KW) || p.at(MUT_KW) && p.nth(1) == SELF_KW {
        m = p.start();
        p.eat(MUT_KW);
        p.eat(SELF_KW);
        // test arb_self_types
        // impl S {
        //     fn a(self: &Self) {}
        //     fn b(mut self: Box<Self>) {}
        // }
        if p.at(COLON) {
            types::ascription(p);
        }
    } else {
        let la1 = p.nth(1);
        let la2 = p.nth(2);
        let la3 = p.nth(3);
        let n_toks = match (p.current(), la1, la2, la3) {
            (AMP, SELF_KW, _, _) => 2,
            (AMP, MUT_KW, SELF_KW, _) => 3,
            (AMP, LIFETIME, SELF_KW, _) => 3,
            (AMP, LIFETIME, MUT_KW, SELF_KW) => 4,
            _ => return,
        };
        m = p.start();
        for _ in 0..n_toks {
            p.bump();
        }
    }
    m.complete(p, SELF_PARAM);
    if !p.at(R_PAREN) {
        p.expect(COMMA);
    }
}
<commit_msg>Allow placeholder parameters in trait fn defs<commit_after>use super::*;

// test param_list
// fn a() {}
// fn b(x: i32) {}
// fn c(x: i32, ) {}
// fn d(x: i32, y: ()) {}
pub(super) fn param_list(p: &mut Parser) {
    list_(p, Flavor::Normal)
}

// test param_list_opt_patterns
// fn foo<F: FnMut(&mut Foo<'a>)>(){}
pub(super) fn param_list_opt_patterns(p: &mut Parser) {
    list_(p, Flavor::OptionalPattern)
}

pub(super) fn param_list_opt_types(p: &mut Parser) {
    list_(p, Flavor::OptionalType)
}

#[derive(Clone, Copy, Eq, PartialEq)]
enum Flavor {
    OptionalType,
    OptionalPattern,
    Normal,
}

impl Flavor {
    fn type_required(self) -> bool {
        match self {
            Flavor::OptionalType => false,
            _ => true,
        }
    }
}

fn list_(p: &mut Parser, flavor: Flavor) {
    let (bra, ket) = if flavor.type_required() {
        (L_PAREN, R_PAREN)
    } else {
        (PIPE, PIPE)
    };
    assert!(p.at(bra));
    let m = p.start();
    p.bump();
    if flavor.type_required() {
        opt_self_param(p);
    }
    while !p.at(EOF) && !p.at(ket) {
        if !p.at_ts(VALUE_PARAMETER_FIRST) {
            p.error("expected value parameter");
            break;
        }
        value_parameter(p, flavor);
        if !p.at(ket) {
            p.expect(COMMA);
        }
    }
    p.expect(ket);
    m.complete(p, PARAM_LIST);
}

const VALUE_PARAMETER_FIRST: TokenSet = patterns::PATTERN_FIRST.union(types::TYPE_FIRST);

fn value_parameter(p: &mut Parser, flavor: Flavor) {
    let m = p.start();
    match flavor {
        Flavor::OptionalType | Flavor::Normal => {
            patterns::pattern(p);
            if p.at(COLON) || flavor.type_required() {
                types::ascription(p)
            }
        }
        // test value_parameters_no_patterns
        // type F = Box<Fn(a: i32, &b: &i32, &mut c: &i32, ())>;
        Flavor::OptionalPattern => {
            let la0 = p.current();
            let la1 = p.nth(1);
            let la2 = p.nth(2);
            let la3 = p.nth(3);
            if (la0 == IDENT || la0 == UNDERSCORE) && la1 == COLON
                || la0 == AMP && la1 == IDENT && la2 == COLON
                || la0 == AMP && la1 == MUT_KW && la2 == IDENT && la3 == COLON
            {
                patterns::pattern(p);
                types::ascription(p);
            } else {
                types::type_(p);
            }
        }
    }
    m.complete(p, PARAM);
}

// test self_param
// impl S {
//     fn a(self) {}
//     fn b(&self,) {}
//     fn c(&'a self,) {}
//     fn d(&'a mut self, x: i32) {}
//     fn e(mut self) {}
// }
fn opt_self_param(p: &mut Parser) {
    let m;
    if p.at(SELF_KW) || p.at(MUT_KW) && p.nth(1) == SELF_KW {
        m = p.start();
        p.eat(MUT_KW);
        p.eat(SELF_KW);
        // test arb_self_types
        // impl S {
        //     fn a(self: &Self) {}
        //     fn b(mut self: Box<Self>) {}
        // }
        if p.at(COLON) {
            types::ascription(p);
        }
    } else {
        let la1 = p.nth(1);
        let la2 = p.nth(2);
        let la3 = p.nth(3);
        let n_toks = match (p.current(), la1, la2, la3) {
            (AMP, SELF_KW, _, _) => 2,
            (AMP, MUT_KW, SELF_KW, _) => 3,
            (AMP, LIFETIME, SELF_KW, _) => 3,
            (AMP, LIFETIME, MUT_KW, SELF_KW) => 4,
            _ => return,
        };
        m = p.start();
        for _ in 0..n_toks {
            p.bump();
        }
    }
    m.complete(p, SELF_PARAM);
    if !p.at(R_PAREN) {
        p.expect(COMMA);
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;

fn get_sorted_chars(string : &str) -> String {
    let mut chars : Vec<char> = string.chars().collect();
    chars.sort();
    let mut sorted_string = String::new();
    for c in chars {
        sorted_string.push(c);
    }
    sorted_string
}

pub fn get_anagrams<'a>(strings : &[&'a str]) -> HashMap<String, Vec<&'a str>> {
    let mut anagram_map = HashMap::new();
    for string in strings {
        let sorted_string = get_sorted_chars(string);
        let string_vec = anagram_map.entry(sorted_string).or_insert(Vec::new());
        string_vec.push(*string);
    }
    anagram_map
}

#[test]
fn get_sorted_chars_works() {
    let string = "waffles";
    let sorted = get_sorted_chars(string);
    assert_eq!(sorted, "aefflsw");
}
<commit_msg>Kata06: Add simple test for anagram method<commit_after>use std::collections::HashMap;

fn get_sorted_chars(string : &str) -> String {
    let mut chars : Vec<char> = string.chars().collect();
    chars.sort();
    let mut sorted_string = String::new();
    for c in chars {
        sorted_string.push(c);
    }
    sorted_string
}

pub fn get_anagrams<'a>(strings : &[&'a str]) -> HashMap<String, Vec<&'a str>> {
    let mut anagram_map = HashMap::new();
    for string in strings {
        let sorted_string = get_sorted_chars(string);
        let string_vec = anagram_map.entry(sorted_string).or_insert(Vec::new());
        string_vec.push(*string);
    }
    anagram_map
}

#[test]
fn get_sorted_chars_works() {
    let string = "waffles";
    let sorted = get_sorted_chars(string);
    assert_eq!(sorted, "aefflsw");
}

#[test]
fn get_anagrams_works_on_simple_data() {
    let strings = ["ate", "eat"];
    let mut expected_anagrams = Vec::new();
    expected_anagrams.push("ate");
    expected_anagrams.push("eat");
    let anagram_map = get_anagrams(&strings);
    assert_eq!(anagram_map.keys().len(), 1);
    assert_eq!(anagram_map["aet"], expected_anagrams);
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::io::Write;
use std::collections::{HashMap, HashSet};
use std::cmp;
use tipb::schema::ColumnInfo;

use super::number::{NumberDecoder, NumberEncoder};
use super::bytes::BytesDecoder;
use super::datum::DatumDecoder;
use super::{Result, Datum, datum};
use super::mysql::{self, types, Duration};
use util::escape;

// handle or index id
pub const ID_LEN: usize = 8;
pub const PREFIX_LEN: usize = TABLE_PREFIX_LEN + ID_LEN /*table_id*/ + SEP_LEN;
pub const RECORD_ROW_KEY_LEN: usize = PREFIX_LEN + ID_LEN;
pub const TABLE_PREFIX: &'static [u8] = b"t";
pub const RECORD_PREFIX_SEP: &'static [u8] = b"_r";
pub const INDEX_PREFIX_SEP: &'static [u8] = b"_i";
pub const SEP_LEN: usize = 2;
pub const TABLE_PREFIX_LEN: usize = 1;

trait TableEncoder: NumberEncoder {
    fn append_table_record_prefix(&mut self, table_id: i64) -> Result<()> {
        try!(self.write_all(TABLE_PREFIX));
        try!(self.encode_i64(table_id));
        self.write_all(RECORD_PREFIX_SEP).map_err(From::from)
    }

    fn append_table_index_prefix(&mut self, table_id: i64) -> Result<()> {
        try!(self.write_all(TABLE_PREFIX));
        try!(self.encode_i64(table_id));
        self.write_all(INDEX_PREFIX_SEP).map_err(From::from)
    }
}

impl<T: Write> TableEncoder for T {}

fn flatten(data: Datum) -> Result<Datum> {
    match data {
        Datum::Dur(d) => Ok(Datum::I64(d.to_nanos())),
        Datum::Dec(d) => Ok(Datum::Bytes(format!("{}", d).into_bytes())),
        _ => Ok(data),
    }
}

// `encode_row` encodes row data and column ids into a slice of byte.
// Row layout: colID1, value1, colID2, value2, .....
pub fn encode_row(row: Vec<Datum>, col_ids: &[i64]) -> Result<Vec<u8>> {
    if row.len() != col_ids.len() {
        return Err(box_err!("data and columnID count not match {} vs {}",
                            row.len(),
                            col_ids.len()));
    }
    let mut values = Vec::with_capacity(cmp::max(row.len() * 2, 1));
    for (&id, col) in col_ids.into_iter().zip(row) {
        values.push(Datum::I64(id));
        let fc = try!(flatten(col));
        values.push(fc);
    }
    if values.is_empty() {
        values.push(Datum::Null);
    }
    datum::encode_value(&values)
}

/// `encode_row_key` encodes the table id and record handle into a byte array.
pub fn encode_row_key(table_id: i64, encoded_handle: &[u8]) -> Vec<u8> {
    let mut key = Vec::with_capacity(RECORD_ROW_KEY_LEN);
    // can't panic
    key.append_table_record_prefix(table_id).unwrap();
    key.write_all(encoded_handle).unwrap();
    key
}

/// `encode_column_key` encodes the table id, row handle and column id into a byte array.
pub fn encode_column_key(table_id: i64, handle: i64, column_id: i64) -> Vec<u8> {
    let mut key = Vec::with_capacity(RECORD_ROW_KEY_LEN + ID_LEN);
    key.append_table_record_prefix(table_id).unwrap();
    key.encode_i64(handle).unwrap();
    key.encode_i64(column_id).unwrap();
    key
}

/// `decode_handle` decodes the key and gets the handle.
pub fn decode_handle(encoded: &[u8]) -> Result<i64> {
    if !encoded.starts_with(TABLE_PREFIX) {
        return Err(invalid_type!("record key expected, but got {}", escape(encoded)));
    }

    let mut remaining = &encoded[TABLE_PREFIX.len()..];
    try!(remaining.decode_i64());

    if !remaining.starts_with(RECORD_PREFIX_SEP) {
        return Err(invalid_type!("record key expected, but got {}", escape(encoded)));
    }

    remaining = &remaining[RECORD_PREFIX_SEP.len()..];
    remaining.decode_i64()
}

/// `truncate_as_row_key` truncate extra part of a tidb key and just keep the row key part.
pub fn truncate_as_row_key(key: &[u8]) -> Result<&[u8]> {
    try!(decode_handle(key));
    Ok(&key[..RECORD_ROW_KEY_LEN])
}

/// `encode_index_seek_key` encodes an index value to byte array.
pub fn encode_index_seek_key(table_id: i64, idx_id: i64, encoded: &[u8]) -> Vec<u8> {
    let mut key = Vec::with_capacity(PREFIX_LEN + ID_LEN + encoded.len());
    key.append_table_index_prefix(table_id).unwrap();
    key.encode_i64(idx_id).unwrap();
    key.write_all(encoded).unwrap();
    key
}

// `decode_index_key` decodes datums from an index key.
pub fn decode_index_key(encoded: &[u8]) -> Result<Vec<Datum>> {
    (&encoded[PREFIX_LEN + ID_LEN..]).decode()
}

/// `unflatten` converts a raw datum to a column datum.
fn unflatten(datum: Datum, col: &ColumnInfo) -> Result<Datum> {
    if let Datum::Null = datum {
        return Ok(datum);
    }
    match col.get_tp() {
        types::FLOAT => Ok(Datum::F64(datum.f64() as f32 as f64)),
        types::TINY |
        types::SHORT |
        types::YEAR |
        types::INT24 |
        types::LONG |
        types::LONG_LONG |
        types::DOUBLE |
        types::TINY_BLOB |
        types::MEDIUM_BLOB |
        types::BLOB |
        types::LONG_BLOB |
        types::VARCHAR |
        types::STRING => Ok(datum),
        types::DATE | types::DATETIME | types::TIMESTAMP | types::ENUM | types::SET |
        types::BIT => unimplemented!(),
        types::DURATION => Duration::from_nanos(datum.i64(), mysql::MAX_FSP).map(Datum::Dur),
        types::NEW_DECIMAL => {
            if let Datum::Dec(_) = datum {
                return Ok(datum);
            }
            datum.into_string().and_then(|s| s.parse()).map(Datum::Dec)
        }
        t => {
            error!("unknown type {} {:?}", t, datum);
            Ok(datum)
        }
    }
}

pub trait TableDecoder: DatumDecoder {
    // `decode_col_value` decodes data to a Datum according to the column info.
    fn decode_col_value(&mut self, col: &ColumnInfo) -> Result<Datum> {
        let d = try!(self.decode_datum());
        unflatten(d, col)
    }

    // `decode_row` decodes a byte slice into datums.
    // TODO: We should only decode columns in the cols map.
    // Row layout: colID1, value1, colID2, value2, .....
    fn decode_row(&mut self, cols: &HashMap<i64, ColumnInfo>) -> Result<HashMap<i64, Datum>> {
        let mut values = try!(self.decode());
        if values.get(0).map_or(true, |d| *d == Datum::Null) {
            return Ok(map![]);
        }
        if values.len() & 1 == 1 {
            return Err(box_err!("decoded row values' length should be even!"));
        }
        let mut row = HashMap::with_capacity(cols.len());
        let mut drain = values.drain(..);
        loop {
            let id = match drain.next() {
                None => return Ok(row),
                Some(id) => id.i64(),
            };
            let v = drain.next().unwrap();
            if let Some(ci) = cols.get(&id) {
                let v = try!(unflatten(v, ci));
                row.insert(id, v);
            }
        }
    }
}

impl<T: BytesDecoder> TableDecoder for T {}

// `cut_row` cut encoded row into byte slices and return interested columns' byte slice.
// Row layout: colID1, value1, colID2, value2, .....
pub fn cut_row<'a>(mut data: &'a [u8], cols: &HashSet<i64>) -> Result<HashMap<i64, &'a [u8]>> {
    let mut res = HashMap::with_capacity(cols.len());
    if data.is_empty() || data.len() == 1 && data[0] == datum::NIL_FLAG {
        return Ok(res);
    }
    while !data.is_empty() && res.len() < cols.len() {
        let id = try!(data.decode_datum()).i64();
        let (val, rem) = try!(datum::split_datum(data, false));
        if cols.contains(&id) {
            res.insert(id, val);
        }
        data = rem;
    }
    Ok(res)
}

#[cfg(test)]
mod test {
    use super::*;
    use util::codec::mysql::*;
    use util::codec::datum::{self, Datum};
    use util::codec::number::NumberEncoder;
    use tipb::schema::ColumnInfo;
    use std::i64;
    use std::collections::{HashSet, HashMap};

    #[test]
    fn test_row_key_codec() {
        let tests = vec![i64::MIN, i64::MAX, -1, 0, 2, 3, 1024];
        for &t in &tests {
            let mut buf = vec![];
            buf.encode_i64(t).unwrap();
            let k = encode_row_key(1, &buf);
            assert_eq!(t, decode_handle(&k).unwrap());
        }
    }

    #[test]
    fn test_index_key_codec() {
        let tests = vec![Datum::U64(1), Datum::Bytes(b"123".to_vec()), Datum::I64(-1)];
        let buf = datum::encode_key(&tests).unwrap();
        let encoded = encode_index_seek_key(1, 2, &buf);
        assert_eq!(tests, decode_index_key(&encoded).unwrap());
    }

    fn new_col_info(tp: i32) -> ColumnInfo {
        let mut col_info = ColumnInfo::new();
        col_info.set_tp(tp);
        col_info
    }

    fn cut_row_as_owned(bs: &[u8], col_id_set: &HashSet<i64>) -> HashMap<i64, Vec<u8>> {
        let res = cut_row(bs, col_id_set).unwrap();
        res.iter().map(|(k, v)| (*k, v.to_vec())).collect()
    }

    #[test]
    fn test_row_codec() {
        let mut cols = map![
            1 => new_col_info(types::LONG_LONG),
            2 => new_col_info(types::VARCHAR),
            3 => new_col_info(types::NEW_DECIMAL)
        ];

        let mut row = map![
            1 => Datum::I64(100),
            2 => Datum::Bytes(b"abc".to_vec()),
            3 => Datum::Dec(Decimal::new(1.into(), 1, MAX_FSP))
        ];

        let col_ids: Vec<_> = row.iter().map(|(&id, _)| id).collect();
        let col_values: Vec<_> = row.iter().map(|(_, v)| v.clone()).collect();
        let mut col_encoded: HashMap<_, _> = row.iter()
            .map(|(k, v)| {
                let f = super::flatten(v.clone()).unwrap();
                (*k, datum::encode_value(&[f]).unwrap())
            })
            .collect();
        let mut col_id_set: HashSet<_> = col_ids.iter().cloned().collect();

        let bs = encode_row(col_values, &col_ids).unwrap();
        assert!(!bs.is_empty());

        let r = bs.as_slice().decode_row(&cols).unwrap();
        assert_eq!(row, r);

        let mut datums: HashMap<_, _>;
        datums = cut_row_as_owned(&bs, &col_id_set);
        assert_eq!(col_encoded, datums);

        cols.insert(4, new_col_info(types::FLOAT));
        let r = bs.as_slice().decode_row(&cols).unwrap();
        assert_eq!(row, r);
        col_id_set.insert(4);
        datums = cut_row_as_owned(&bs, &col_id_set);
        assert_eq!(col_encoded, datums);

        cols.remove(&4);
        cols.remove(&3);
        let r = bs.as_slice().decode_row(&cols).unwrap();
        row.remove(&3);
        assert_eq!(row, r);
        col_id_set.remove(&3);
        col_id_set.remove(&4);
        datums = cut_row_as_owned(&bs, &col_id_set);
        col_encoded.remove(&3);
        assert_eq!(col_encoded, datums);

        let bs = encode_row(vec![], &[]).unwrap();
        assert!(!bs.is_empty());
        assert!(bs.as_slice().decode_row(&cols).unwrap().is_empty());
        datums = cut_row_as_owned(&bs, &col_id_set);
        assert!(datums.is_empty());
    }
}
<commit_msg>copr: take it easy when encounter unsupported col (#894)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::io::Write;
use std::collections::{HashMap, HashSet};
use std::cmp;
use tipb::schema::ColumnInfo;

use super::number::{NumberDecoder, NumberEncoder};
use super::bytes::BytesDecoder;
use super::datum::DatumDecoder;
use super::{Result, Datum, datum};
use super::mysql::{self, types, Duration};
use util::escape;

// handle or index id
pub const ID_LEN: usize = 8;
pub const PREFIX_LEN: usize = TABLE_PREFIX_LEN + ID_LEN /*table_id*/ + SEP_LEN;
pub const RECORD_ROW_KEY_LEN: usize = PREFIX_LEN + ID_LEN;
pub const TABLE_PREFIX: &'static [u8] = b"t";
pub const RECORD_PREFIX_SEP: &'static [u8] = b"_r";
pub const INDEX_PREFIX_SEP: &'static [u8] = b"_i";
pub const SEP_LEN: usize = 2;
pub const TABLE_PREFIX_LEN: usize = 1;

trait TableEncoder: NumberEncoder {
    fn append_table_record_prefix(&mut self, table_id: i64) -> Result<()> {
        try!(self.write_all(TABLE_PREFIX));
        try!(self.encode_i64(table_id));
        self.write_all(RECORD_PREFIX_SEP).map_err(From::from)
    }

    fn append_table_index_prefix(&mut self, table_id: i64) -> Result<()> {
        try!(self.write_all(TABLE_PREFIX));
        try!(self.encode_i64(table_id));
        self.write_all(INDEX_PREFIX_SEP).map_err(From::from)
    }
}

impl<T: Write> TableEncoder for T {}

fn flatten(data: Datum) -> Result<Datum> {
    match data {
        Datum::Dur(d) => Ok(Datum::I64(d.to_nanos())),
        Datum::Dec(d) => Ok(Datum::Bytes(format!("{}", d).into_bytes())),
        _ => Ok(data),
    }
}

// `encode_row` encodes row data and column ids into a slice of byte.
// Row layout: colID1, value1, colID2, value2, .....
pub fn encode_row(row: Vec<Datum>, col_ids: &[i64]) -> Result<Vec<u8>> {
    if row.len() != col_ids.len() {
        return Err(box_err!("data and columnID count not match {} vs {}",
                            row.len(),
                            col_ids.len()));
    }
    let mut values = Vec::with_capacity(cmp::max(row.len() * 2, 1));
    for (&id, col) in col_ids.into_iter().zip(row) {
        values.push(Datum::I64(id));
        let fc = try!(flatten(col));
        values.push(fc);
    }
    if values.is_empty() {
        values.push(Datum::Null);
    }
    datum::encode_value(&values)
}

/// `encode_row_key` encodes the table id and record handle into a byte array.
pub fn encode_row_key(table_id: i64, encoded_handle: &[u8]) -> Vec<u8> {
    let mut key = Vec::with_capacity(RECORD_ROW_KEY_LEN);
    // can't panic
    key.append_table_record_prefix(table_id).unwrap();
    key.write_all(encoded_handle).unwrap();
    key
}

/// `encode_column_key` encodes the table id, row handle and column id into a byte array.
pub fn encode_column_key(table_id: i64, handle: i64, column_id: i64) -> Vec<u8> {
    let mut key = Vec::with_capacity(RECORD_ROW_KEY_LEN + ID_LEN);
    key.append_table_record_prefix(table_id).unwrap();
    key.encode_i64(handle).unwrap();
    key.encode_i64(column_id).unwrap();
    key
}

/// `decode_handle` decodes the key and gets the handle.
pub fn decode_handle(encoded: &[u8]) -> Result<i64> {
    if !encoded.starts_with(TABLE_PREFIX) {
        return Err(invalid_type!("record key expected, but got {}", escape(encoded)));
    }

    let mut remaining = &encoded[TABLE_PREFIX.len()..];
    try!(remaining.decode_i64());

    if !remaining.starts_with(RECORD_PREFIX_SEP) {
        return Err(invalid_type!("record key expected, but got {}", escape(encoded)));
    }

    remaining = &remaining[RECORD_PREFIX_SEP.len()..];
    remaining.decode_i64()
}

/// `truncate_as_row_key` truncate extra part of a tidb key and just keep the row key part.
pub fn truncate_as_row_key(key: &[u8]) -> Result<&[u8]> {
    try!(decode_handle(key));
    Ok(&key[..RECORD_ROW_KEY_LEN])
}

/// `encode_index_seek_key` encodes an index value to byte array.
pub fn encode_index_seek_key(table_id: i64, idx_id: i64, encoded: &[u8]) -> Vec<u8> {
    let mut key = Vec::with_capacity(PREFIX_LEN + ID_LEN + encoded.len());
    key.append_table_index_prefix(table_id).unwrap();
    key.encode_i64(idx_id).unwrap();
    key.write_all(encoded).unwrap();
    key
}

// `decode_index_key` decodes datums from an index key.
pub fn decode_index_key(encoded: &[u8]) -> Result<Vec<Datum>> {
    (&encoded[PREFIX_LEN + ID_LEN..]).decode()
}

/// `unflatten` converts a raw datum to a column datum.
fn unflatten(datum: Datum, col: &ColumnInfo) -> Result<Datum> {
    if let Datum::Null = datum {
        return Ok(datum);
    }
    match col.get_tp() {
        types::FLOAT => Ok(Datum::F64(datum.f64() as f32 as f64)),
        types::TINY |
        types::SHORT |
        types::YEAR |
        types::INT24 |
        types::LONG |
        types::LONG_LONG |
        types::DOUBLE |
        types::TINY_BLOB |
        types::MEDIUM_BLOB |
        types::BLOB |
        types::LONG_BLOB |
        types::VARCHAR |
        types::STRING => Ok(datum),
        types::DATE | types::DATETIME | types::TIMESTAMP | types::ENUM | types::SET |
        types::BIT => Err(box_err!("unflatten column {:?} is not supported yet.", col)),
        types::DURATION => Duration::from_nanos(datum.i64(), mysql::MAX_FSP).map(Datum::Dur),
        types::NEW_DECIMAL => {
            if let Datum::Dec(_) = datum {
                return Ok(datum);
            }
            datum.into_string().and_then(|s| s.parse()).map(Datum::Dec)
        }
        t => {
            error!("unknown type {} {:?}", t, datum);
            Ok(datum)
        }
    }
}

pub trait TableDecoder: DatumDecoder {
    // `decode_col_value` decodes data to a Datum according to the column info.
    fn decode_col_value(&mut self, col: &ColumnInfo) -> Result<Datum> {
        let d = try!(self.decode_datum());
        unflatten(d, col)
    }

    // `decode_row` decodes a byte slice into datums.
    // TODO: We should only decode columns in the cols map.
    // Row layout: colID1, value1, colID2, value2, .....
    fn decode_row(&mut self, cols: &HashMap<i64, ColumnInfo>) -> Result<HashMap<i64, Datum>> {
        let mut values = try!(self.decode());
        if values.get(0).map_or(true, |d| *d == Datum::Null) {
            return Ok(map![]);
        }
        if values.len() & 1 == 1 {
            return Err(box_err!("decoded row values' length should be even!"));
        }
        let mut row = HashMap::with_capacity(cols.len());
        let mut drain = values.drain(..);
        loop {
            let id = match drain.next() {
                None => return Ok(row),
                Some(id) => id.i64(),
            };
            let v = drain.next().unwrap();
            if let Some(ci) = cols.get(&id) {
                let v = try!(unflatten(v, ci));
                row.insert(id, v);
            }
        }
    }
}

impl<T: BytesDecoder> TableDecoder for T {}

// `cut_row` cut encoded row into byte slices and return interested columns' byte slice.
// Row layout: colID1, value1, colID2, value2, .....
pub fn cut_row<'a>(mut data: &'a [u8], cols: &HashSet<i64>) -> Result<HashMap<i64, &'a [u8]>> {
    let mut res = HashMap::with_capacity(cols.len());
    if data.is_empty() || data.len() == 1 && data[0] == datum::NIL_FLAG {
        return Ok(res);
    }
    while !data.is_empty() && res.len() < cols.len() {
        let id = try!(data.decode_datum()).i64();
        let (val, rem) = try!(datum::split_datum(data, false));
        if cols.contains(&id) {
            res.insert(id, val);
        }
        data = rem;
    }
    Ok(res)
}

#[cfg(test)]
mod test {
    use super::*;
    use util::codec::mysql::*;
    use util::codec::datum::{self, Datum};
    use util::codec::number::NumberEncoder;
    use tipb::schema::ColumnInfo;
    use std::i64;
    use std::collections::{HashSet, HashMap};

    #[test]
    fn test_row_key_codec() {
        let tests = vec![i64::MIN, i64::MAX, -1, 0, 2, 3, 1024];
        for &t in &tests {
            let mut buf = vec![];
            buf.encode_i64(t).unwrap();
            let k = encode_row_key(1, &buf);
            assert_eq!(t, decode_handle(&k).unwrap());
        }
    }

    #[test]
    fn test_index_key_codec() {
        let tests = vec![Datum::U64(1), Datum::Bytes(b"123".to_vec()), Datum::I64(-1)];
        let buf = datum::encode_key(&tests).unwrap();
        let encoded = encode_index_seek_key(1, 2, &buf);
        assert_eq!(tests, decode_index_key(&encoded).unwrap());
    }

    fn new_col_info(tp: i32) -> ColumnInfo {
        let mut col_info = ColumnInfo::new();
        col_info.set_tp(tp);
        col_info
    }

    fn cut_row_as_owned(bs: &[u8], col_id_set: &HashSet<i64>) -> HashMap<i64, Vec<u8>> {
        let res = cut_row(bs, col_id_set).unwrap();
        res.iter().map(|(k, v)| (*k, v.to_vec())).collect()
    }

    #[test]
    fn test_row_codec() {
        let mut cols = map![
            1 => new_col_info(types::LONG_LONG),
            2 => new_col_info(types::VARCHAR),
            3 => new_col_info(types::NEW_DECIMAL)
        ];

        let mut row = map![
            1 => Datum::I64(100),
            2 => Datum::Bytes(b"abc".to_vec()),
            3 => Datum::Dec(Decimal::new(1.into(), 1, MAX_FSP))
        ];

        let col_ids: Vec<_> = row.iter().map(|(&id, _)| id).collect();
        let col_values: Vec<_> = row.iter().map(|(_, v)| v.clone()).collect();
        let mut col_encoded: HashMap<_, _> = row.iter()
            .map(|(k, v)| {
                let f = super::flatten(v.clone()).unwrap();
                (*k, datum::encode_value(&[f]).unwrap())
            })
            .collect();
        let mut col_id_set: HashSet<_> = col_ids.iter().cloned().collect();

        let bs = encode_row(col_values, &col_ids).unwrap();
        assert!(!bs.is_empty());

        let r = bs.as_slice().decode_row(&cols).unwrap();
        assert_eq!(row, r);

        let mut datums: HashMap<_, _>;
        datums = cut_row_as_owned(&bs, &col_id_set);
        assert_eq!(col_encoded, datums);

        cols.insert(4, new_col_info(types::FLOAT));
        let r = bs.as_slice().decode_row(&cols).unwrap();
        assert_eq!(row, r);
        col_id_set.insert(4);
        datums = cut_row_as_owned(&bs, &col_id_set);
        assert_eq!(col_encoded, datums);

        cols.remove(&4);
        cols.remove(&3);
        let r = bs.as_slice().decode_row(&cols).unwrap();
        row.remove(&3);
        assert_eq!(row, r);
        col_id_set.remove(&3);
        col_id_set.remove(&4);
        datums = cut_row_as_owned(&bs, &col_id_set);
        col_encoded.remove(&3);
        assert_eq!(col_encoded, datums);

        let bs = encode_row(vec![], &[]).unwrap();
        assert!(!bs.is_empty());
        assert!(bs.as_slice().decode_row(&cols).unwrap().is_empty());
        datums = cut_row_as_owned(&bs, &col_id_set);
        assert!(datums.is_empty());
    }
}
<|endoftext|>


<commit_before>// compile-flags: -C relocation-model=pic -Z plt=no

#![crate_type = "lib"]

// We need a function which is normally called through the PLT.
extern "C" {
    // CHECK: Function Attrs: nounwind nonlazybind
    fn getenv(name: *const u8) -> *mut u8;
}

// Ensure the function gets referenced.
pub unsafe fn call_through_plt() -> *mut u8 {
    getenv(b"\0".as_ptr())
}

// Ensure intrinsics also skip the PLT
// CHECK: !"RtLibUseGOT"
<commit_msg>Relax a codegen test<commit_after>// compile-flags: -C relocation-model=pic -Z plt=no

#![crate_type = "lib"]

// We need a function which is normally called through the PLT.
extern "C" {
    // CHECK: Function Attrs:{{.*}}nonlazybind
    fn getenv(name: *const u8) -> *mut u8;
}

// Ensure the function gets referenced.
pub unsafe fn call_through_plt() -> *mut u8 {
    getenv(b"\0".as_ptr())
}

// Ensure intrinsics also skip the PLT
// CHECK: !"RtLibUseGOT"
<|endoftext|>


<commit_before>extern crate crates_index_diff;
extern crate tempdir;

use crates_index_diff::*;
use tempdir::TempDir;
use std::env;
use std::path::PathBuf;

const REV_ONE_ADDED: &'static str = "615c9c41942a3ba13e088fbcb1470c61b169a187";
const REV_ONE_YANKED: &'static str = "8cf8fbad7876586ced34c4b778f6a80fadd2a59b";
const REV_ONE_UNYANKED: &'static str = "f8cb00181";

#[test]
fn clone_if_needed() {
    let tmp = TempDir::new("new-index").unwrap();
    Index::at_path(tmp.path()).expect("successful clone to be created");
    Index::at_path(tmp.path()).expect("second instance re-uses existing clone");
}

fn make_index() -> (Index, TempDir) {
    let tmp = TempDir::new("new-index").unwrap();
    let index = Index::at_path(env::var("CRATES_INDEX_DIFF_TEST_EXISTING_INDEX")
            .map(PathBuf::from)
            .unwrap_or(tmp.path().to_owned()))
        .expect("successful clone");
    (index, tmp)
}

fn changes_of(index: &Index, commit: &str) -> Vec<Crate> {
    index.changes(format!("{}~1^{{tree}}", commit), format!("{}", commit))
        .expect("id to be valid and diff OK")
}

#[test]
fn quick_traverse_unyanked_crates() {
    let (index, _) = make_index();

    let crates = changes_of(&index, REV_ONE_UNYANKED);
    assert_eq!(crates,
    vec![Crate {
        name: "gfx_text".to_owned(),
        state: ChangeType::Added,
        version: "0.13.2".to_owned(),
    }]);
}

#[test]
fn quick_traverse_yanked_crates() {
    let (index, _) = make_index();

    let crates = changes_of(&index, REV_ONE_YANKED);
    assert_eq!(crates,
               vec![Crate {
                        name: "sha3".to_owned(),
                        state: ChangeType::Yanked,
                        version: "0.0.0".to_owned(),
                    }]);
}

#[test]
fn quick_traverse_added_crate() {
    let (index, _) = make_index();
    assert_eq!(index.changes("foo", REV_ONE_ADDED).is_err(), true);
    assert_eq!(index.changes(REV_ONE_ADDED, "bar").is_err(), true);

    let crates = changes_of(&index, REV_ONE_ADDED);
    assert_eq!(crates,
               vec![Crate {
                        name: "rpwg".to_owned(),
                        state: ChangeType::Added,
                        version: "0.1.0".to_owned(),
                    }]);
}
<commit_msg>test(index): assure we can handle the entire index<commit_after>extern crate crates_index_diff;
extern crate tempdir;

use crates_index_diff::*;
use tempdir::TempDir;
use std::env;
use std::path::PathBuf;

const REV_ONE_ADDED: &'static str = "615c9c41942a3ba13e088fbcb1470c61b169a187";
const REV_RECENT_COMMIT: &'static str = REV_ONE_ADDED;
const REV_ONE_YANKED: &'static str = "8cf8fbad7876586ced34c4b778f6a80fadd2a59b";
const REV_ONE_UNYANKED: &'static str = "f8cb00181";
const REV_FIRST_COMMIT: &'static str = "a33de1c98";

#[test]
fn clone_if_needed() {
    let tmp = TempDir::new("new-index").unwrap();
    Index::at_path(tmp.path()).expect("successful clone to be created");
    Index::at_path(tmp.path()).expect("second instance re-uses existing clone");
}

fn make_index() -> (Index, TempDir) {
    let tmp = TempDir::new("new-index").unwrap();
    let index = Index::at_path(env::var("CRATES_INDEX_DIFF_TEST_EXISTING_INDEX")
            .map(PathBuf::from)
            .unwrap_or(tmp.path().to_owned()))
        .expect("successful clone");
    (index, tmp)
}

fn changes_of(index: &Index, commit: &str) -> Vec<Crate> {
    index.changes(format!("{}~1^{{tree}}", commit), format!("{}", commit))
        .expect("id to be valid and diff OK")
}

#[test]
fn quick_traverse_unyanked_crates() {
    let (index, _) = make_index();

    let crates = changes_of(&index, REV_ONE_UNYANKED);
    assert_eq!(crates,
    vec![Crate {
        name: "gfx_text".to_owned(),
        state: ChangeType::Added,
        version: "0.13.2".to_owned(),
    }]);
}

#[test]
fn quick_traverse_yanked_crates() {
    let (index, _) = make_index();

    let crates = changes_of(&index, REV_ONE_YANKED);
    assert_eq!(crates,
               vec![Crate {
                        name: "sha3".to_owned(),
                        state: ChangeType::Yanked,
                        version: "0.0.0".to_owned(),
                    }]);
}

#[test]
fn quick_traverse_all_crates() {
    let (index, _) = make_index();
    let changes = index.changes(format!("{}", REV_FIRST_COMMIT), format!("{}", REV_RECENT_COMMIT))
         .expect("id to be valid and diff OK");
    assert_eq!(changes.len(), 39752);
}

#[test]
fn quick_traverse_added_crates() {
    let (index, _) = make_index();
    assert_eq!(index.changes("foo", REV_ONE_ADDED).is_err(), true);
    assert_eq!(index.changes(REV_ONE_ADDED, "bar").is_err(), true);

    let crates = changes_of(&index, REV_ONE_ADDED);
    assert_eq!(crates,
               vec![Crate {
                        name: "rpwg".to_owned(),
                        state: ChangeType::Added,
                        version: "0.1.0".to_owned(),
                    }]);
}
<|endoftext|>


<commit_before>#![cfg(feature = "serde")]

use hashbrown::{HashMap, HashSet};
use serde_test::{assert_tokens, Token};
use rustc_hash::FxHasher;
use core::hash::BuildHasherDefault;

// We use FxHash for this test because we rely on the ordering
type FxHashMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
type FxHashSet<T> = HashSet<T, BuildHasherDefault<FxHasher>>;

#[test]
fn map_serde_tokens_empty() {
    let map = FxHashMap::<char, u32>::default();

    assert_tokens(&map, &[Token::Map { len: Some(0) }, Token::MapEnd]);
}

#[test]
fn map_serde_tokens() {
    let mut map = FxHashMap::default();
    map.insert('b', 20);
    map.insert('a', 10);
    map.insert('c', 30);

    assert_tokens(
        &map,
        &[
            Token::Map { len: Some(3) },
            Token::Char('a'),
            Token::I32(10),
            Token::Char('b'),
            Token::I32(20),
            Token::Char('c'),
            Token::I32(30),
            Token::MapEnd,
        ],
    );
}

#[test]
fn set_serde_tokens_empty() {
    let set = FxHashSet::<u32>::default();

    assert_tokens(&set, &[Token::Seq { len: Some(0) }, Token::SeqEnd]);
}

#[test]
fn set_serde_tokens() {
    let mut set = FxHashSet::default();
    set.insert(20);
    set.insert(10);
    set.insert(30);

    assert_tokens(
        &set,
        &[
            Token::Seq { len: Some(3) },
            Token::I32(20),
            Token::I32(10),
            Token::I32(30),
            Token::SeqEnd,
        ],
    );
}
<commit_msg>Rustfmt<commit_after>#![cfg(feature = "serde")]

use core::hash::BuildHasherDefault;
use hashbrown::{HashMap, HashSet};
use rustc_hash::FxHasher;
use serde_test::{assert_tokens, Token};

// We use FxHash for this test because we rely on the ordering
type FxHashMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
type FxHashSet<T> = HashSet<T, BuildHasherDefault<FxHasher>>;

#[test]
fn map_serde_tokens_empty() {
    let map = FxHashMap::<char, u32>::default();

    assert_tokens(&map, &[Token::Map { len: Some(0) }, Token::MapEnd]);
}

#[test]
fn map_serde_tokens() {
    let mut map = FxHashMap::default();
    map.insert('b', 20);
    map.insert('a', 10);
    map.insert('c', 30);

    assert_tokens(
        &map,
        &[
            Token::Map { len: Some(3) },
            Token::Char('a'),
            Token::I32(10),
            Token::Char('b'),
            Token::I32(20),
            Token::Char('c'),
            Token::I32(30),
            Token::MapEnd,
        ],
    );
}

#[test]
fn set_serde_tokens_empty() {
    let set = FxHashSet::<u32>::default();

    assert_tokens(&set, &[Token::Seq { len: Some(0) }, Token::SeqEnd]);
}

#[test]
fn set_serde_tokens() {
    let mut set = FxHashSet::default();
    set.insert(20);
    set.insert(10);
    set.insert(30);

    assert_tokens(
        &set,
        &[
            Token::Seq { len: Some(3) },
            Token::I32(20),
            Token::I32(10),
            Token::I32(30),
            Token::SeqEnd,
        ],
    );
}
<|endoftext|>