<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::{
    data_stream::{DataStream, DataStreamId, DataStreamListener},
    error::Error,
    logging::{LogEntry, LogEvent, LogSchema},
    metrics,
    streaming_client::{
        StreamRequest, StreamRequestMessage, StreamingServiceListener, TerminateStreamRequest,
    },
};
use aptos_config::config::DataStreamingServiceConfig;
use aptos_data_client::{AptosDataClient, GlobalDataSummary, OptimalChunkSizes};
use aptos_id_generator::{IdGenerator, U64IdGenerator};
use aptos_logger::prelude::*;
use futures::StreamExt;
use std::{collections::HashMap, sync::Arc, time::Duration};
use tokio::time::interval;
use tokio_stream::wrappers::IntervalStream;

// Useful constants for the Data Streaming Service
const GLOBAL_DATA_REFRESH_LOG_FREQ_SECS: u64 = 3;
const NO_DATA_TO_FETCH_LOG_FREQ_SECS: u64 = 3;

/// The data streaming service that responds to data stream requests.
pub struct DataStreamingService<T> {
    // The configuration for this streaming service.
    config: DataStreamingServiceConfig,

    // The data client through which to fetch data from the Aptos network
    aptos_data_client: T,

    // Cached global data summary
    global_data_summary: GlobalDataSummary,

    // All requested data streams from clients
    data_streams: HashMap<DataStreamId, DataStream<T>>,

    // The listener through which to hear new client stream requests
    stream_requests: StreamingServiceListener,

    // Unique ID generators to maintain unique IDs across streams
    stream_id_generator: U64IdGenerator,
    notification_id_generator: Arc<U64IdGenerator>,
}

impl<T: AptosDataClient + Send + Clone + 'static> DataStreamingService<T> {
    pub fn new(
        config: DataStreamingServiceConfig,
        aptos_data_client: T,
        stream_requests: StreamingServiceListener,
    ) -> Self {
        Self {
            config,
            aptos_data_client,
            global_data_summary: GlobalDataSummary::empty(),
            data_streams: HashMap::new(),
            stream_requests,
            stream_id_generator: U64IdGenerator::new(),
            notification_id_generator: Arc::new(U64IdGenerator::new()),
        }
    }

    /// Starts the dedicated streaming service
    pub async fn start_service(mut self) {
        let mut data_refresh_interval = IntervalStream::new(interval(Duration::from_millis(
            self.config.global_summary_refresh_interval_ms,
        )))
        .fuse();
        let mut progress_check_interval = IntervalStream::new(interval(Duration::from_millis(
            self.config.progress_check_interval_ms,
        )))
        .fuse();

        loop {
            ::futures::select! {
                stream_request = self.stream_requests.select_next_some() => {
                    self.handle_stream_request_message(stream_request);
                }
                _ = data_refresh_interval.select_next_some() => {
                    self.refresh_global_data_summary();
                }
                _ = progress_check_interval.select_next_some() => {
                    self.check_progress_of_all_data_streams();
                }
            }
        }
    }

    /// Handles new stream request messages from clients
    fn handle_stream_request_message(&mut self, request_message: StreamRequestMessage) {
        if let StreamRequest::TerminateStream(request) = request_message.stream_request {
            // Process the feedback request
            if let Err(error) = self.process_terminate_stream_request(&request) {
                error!(LogSchema::new(LogEntry::HandleTerminateRequest)
                    .event(LogEvent::Error)
                    .error(&error));
            }
            return;
        }

        // Process the stream request
        let response = self.process_new_stream_request(&request_message);
        if let Err(error) = &response {
            error!(LogSchema::new(LogEntry::HandleStreamRequest)
                .event(LogEvent::Error)
                .error(error));
        }

        // Send the response to the client
        if let Err(error) = request_message.response_sender.send(response) {
            error!(LogSchema::new(LogEntry::RespondToStreamRequest)
                .event(LogEvent::Error)
                .message(&format!(
                    "Failed to send response for stream request: {:?}",
                    error
                )));
        }
    }

    /// Processes a request for terminating the stream that sent a specific
    /// notification ID.
    fn process_terminate_stream_request(
        &mut self,
        terminate_request: &TerminateStreamRequest,
    ) -> Result<(), Error> {
        // Increment the stream termination counter
        let notification_feedback = &terminate_request.notification_feedback;
        metrics::increment_counter(
            &metrics::TERMINATE_DATA_STREAM,
            notification_feedback.get_label().into(),
        );

        // Find the data stream that sent the notification
        let notification_id = &terminate_request.notification_id;
        let data_stream_ids = self.get_all_data_stream_ids();
        for data_stream_id in &data_stream_ids {
            let data_stream = self.get_data_stream(data_stream_id);
            if data_stream.sent_notification(notification_id) {
                info!(LogSchema::new(LogEntry::HandleTerminateRequest)
                    .stream_id(*data_stream_id)
                    .event(LogEvent::Success)
                    .message(&format!(
                        "Terminating the stream that sent notification ID: {:?}. Feedback: {:?}",
                        notification_id, notification_feedback,
                    )));
                data_stream.handle_notification_feedback(notification_id, notification_feedback)?;
                self.data_streams.remove(notification_id);
                return Ok(());
            }
        }

        panic!(
            "Unable to find the stream that sent notification ID: {:?}. Feedback: {:?}",
            notification_id, notification_feedback,
        );
    }

    /// Creates a new stream and ensures the data for that stream is available
    fn process_new_stream_request(
        &mut self,
        request_message: &StreamRequestMessage,
    ) -> Result<DataStreamListener, Error> {
        // Increment the stream creation counter
        metrics::increment_counter(
            &metrics::CREATE_DATA_STREAM,
            request_message.stream_request.get_label().into(),
        );

        // Refresh the cached global data summary
        self.refresh_global_data_summary();

        // Create a new data stream
        let stream_id = self.stream_id_generator.next();
        let (data_stream, stream_listener) = DataStream::new(
            self.config,
            stream_id,
            &request_message.stream_request,
            self.aptos_data_client.clone(),
            self.notification_id_generator.clone(),
            &self.global_data_summary.advertised_data,
        )?;

        // Verify the data stream can be fulfilled using the currently advertised data
        data_stream.ensure_data_is_available(&self.global_data_summary.advertised_data)?;

        // Store the data stream internally
        if self.data_streams.insert(stream_id, data_stream).is_some() {
            panic!(
                "Duplicate data stream found! This should not occur! ID: {:?}",
                stream_id,
            );
        }
        info!(LogSchema::new(LogEntry::HandleStreamRequest)
            .stream_id(stream_id)
            .event(LogEvent::Success)
            .message(&format!(
                "Stream created for request: {:?}",
                request_message
            )));

        // Return the listener
        Ok(stream_listener)
    }

    /// Refreshes the global data summary by communicating with the Aptos data client
    fn refresh_global_data_summary(&mut self) {
        if let Err(error) = self.fetch_global_data_summary() {
            metrics::increment_counter(
                &metrics::GLOBAL_DATA_SUMMARY_ERROR,
                error.get_label().into(),
            );
            sample!(
                SampleRate::Duration(Duration::from_secs(GLOBAL_DATA_REFRESH_LOG_FREQ_SECS)),
                error!(LogSchema::new(LogEntry::RefreshGlobalData)
                    .event(LogEvent::Error)
                    .error(&error))
            );
        }
    }

    fn fetch_global_data_summary(&mut self) -> Result<(), Error> {
        let global_data_summary = self.aptos_data_client.get_global_data_summary();
        if global_data_summary.is_empty() {
            sample!(
                SampleRate::Duration(Duration::from_secs(GLOBAL_DATA_REFRESH_LOG_FREQ_SECS)),
                info!(LogSchema::new(LogEntry::RefreshGlobalData)
                    .message("Latest global data summary is empty."))
            );
        } else {
            verify_optimal_chunk_sizes(&global_data_summary.optimal_chunk_sizes)?;
            self.global_data_summary = global_data_summary;
        }

        Ok(())
    }

    /// Ensures that all existing data streams are making progress
    fn check_progress_of_all_data_streams(&mut self) {
        let data_stream_ids = self.get_all_data_stream_ids();
        for data_stream_id in &data_stream_ids {
            if let Err(error) = self.update_progress_of_data_stream(data_stream_id) {
                if matches!(error, Error::NoDataToFetch(_)) {
                    sample!(
                        SampleRate::Duration(Duration::from_secs(NO_DATA_TO_FETCH_LOG_FREQ_SECS)),
                        info!(LogSchema::new(LogEntry::CheckStreamProgress)
                            .stream_id(*data_stream_id)
                            .event(LogEvent::Pending)
                            .error(&error))
                    );
                } else {
                    metrics::increment_counter(
                        &metrics::CHECK_STREAM_PROGRESS_ERROR,
                        error.get_label().into(),
                    );
                    error!(LogSchema::new(LogEntry::CheckStreamProgress)
                        .stream_id(*data_stream_id)
                        .event(LogEvent::Error)
                        .error(&error));
                }
            }
        }
    }

    /// Ensures that a data stream has in-flight data requests and handles
    /// any new responses that have arrived since we last checked.
    fn update_progress_of_data_stream(
        &mut self,
        data_stream_id: &DataStreamId,
    ) -> Result<(), Error> {
        let global_data_summary = self.global_data_summary.clone();

        let data_stream = self.get_data_stream_mut(data_stream_id);
        if !data_stream.data_requests_initialized() {
            // Initialize the request batch by sending out data client requests
            data_stream.initialize_data_requests(global_data_summary)?;
            info!(
                (LogSchema::new(LogEntry::InitializeStream)
                    .stream_id(*data_stream_id)
                    .event(LogEvent::Success)
                    .message("Data stream initialized."))
            );
        } else {
            // Process any data client requests that have received responses
            data_stream.process_data_responses(global_data_summary)?;
        }

        Ok(())
    }

    fn get_all_data_stream_ids(&self) -> Vec<DataStreamId> {
        self.data_streams
            .keys()
            .cloned()
            .collect::<Vec<DataStreamId>>()
    }

    /// Returns the data stream associated with the given `data_stream_id`.
    /// Note: this method assumes the caller has already verified the stream exists.
    fn get_data_stream(&self, data_stream_id: &DataStreamId) -> &DataStream<T> {
        self.data_streams.get(data_stream_id).unwrap_or_else(|| {
            panic!(
                "Expected a data stream with ID: {:?}, but found None!",
                data_stream_id
            )
        })
    }

    /// Returns the data stream associated with the given `data_stream_id`.
    /// Note: this method assumes the caller has already verified the stream exists.
    fn get_data_stream_mut(&mut self, data_stream_id: &DataStreamId) -> &mut DataStream<T> {
        self.data_streams
            .get_mut(data_stream_id)
            .unwrap_or_else(|| {
                panic!(
                    "Expected a data stream with ID: {:?}, but found None!",
                    data_stream_id
                )
            })
    }
}

/// Verifies that all optimal chunk sizes are valid (i.e., not zero). Returns an
/// error if a chunk size is 0.
fn verify_optimal_chunk_sizes(optimal_chunk_sizes: &OptimalChunkSizes) -> Result<(), Error> {
    if optimal_chunk_sizes.account_states_chunk_size == 0
        || optimal_chunk_sizes.epoch_chunk_size == 0
        || optimal_chunk_sizes.transaction_chunk_size == 0
        || optimal_chunk_sizes.transaction_output_chunk_size == 0
    {
        Err(Error::AptosDataClientResponseIsInvalid(format!(
            "Found at least one optimal chunk size of zero: {:?}",
            optimal_chunk_sizes
        )))
    } else {
        Ok(())
    }
}
<commit_msg>[State Sync] Sampling around stream creation errors.<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::{
    data_stream::{DataStream, DataStreamId, DataStreamListener},
    error::Error,
    logging::{LogEntry, LogEvent, LogSchema},
    metrics,
    streaming_client::{
        StreamRequest, StreamRequestMessage, StreamingServiceListener, TerminateStreamRequest,
    },
};
use aptos_config::config::DataStreamingServiceConfig;
use aptos_data_client::{AptosDataClient, GlobalDataSummary, OptimalChunkSizes};
use aptos_id_generator::{IdGenerator, U64IdGenerator};
use aptos_logger::prelude::*;
use futures::StreamExt;
use std::{collections::HashMap, sync::Arc, time::Duration};
use tokio::time::interval;
use tokio_stream::wrappers::IntervalStream;

// Useful constants for the Data Streaming Service
const GLOBAL_DATA_REFRESH_LOG_FREQ_SECS: u64 = 3;
const NO_DATA_TO_FETCH_LOG_FREQ_SECS: u64 = 3;
const STREAM_REQUEST_ERROR_LOG_FREQ_SECS: u64 = 3;

/// The data streaming service that responds to data stream requests.
pub struct DataStreamingService<T> {
    // The configuration for this streaming service.
    config: DataStreamingServiceConfig,

    // The data client through which to fetch data from the Aptos network
    aptos_data_client: T,

    // Cached global data summary
    global_data_summary: GlobalDataSummary,

    // All requested data streams from clients
    data_streams: HashMap<DataStreamId, DataStream<T>>,

    // The listener through which to hear new client stream requests
    stream_requests: StreamingServiceListener,

    // Unique ID generators to maintain unique IDs across streams
    stream_id_generator: U64IdGenerator,
    notification_id_generator: Arc<U64IdGenerator>,
}

impl<T: AptosDataClient + Send + Clone + 'static> DataStreamingService<T> {
    pub fn new(
        config: DataStreamingServiceConfig,
        aptos_data_client: T,
        stream_requests: StreamingServiceListener,
    ) -> Self {
        Self {
            config,
            aptos_data_client,
            global_data_summary: GlobalDataSummary::empty(),
            data_streams: HashMap::new(),
            stream_requests,
            stream_id_generator: U64IdGenerator::new(),
            notification_id_generator: Arc::new(U64IdGenerator::new()),
        }
    }

    /// Starts the dedicated streaming service
    pub async fn start_service(mut self) {
        let mut data_refresh_interval = IntervalStream::new(interval(Duration::from_millis(
            self.config.global_summary_refresh_interval_ms,
        )))
        .fuse();
        let mut progress_check_interval = IntervalStream::new(interval(Duration::from_millis(
            self.config.progress_check_interval_ms,
        )))
        .fuse();

        loop {
            ::futures::select! {
                stream_request = self.stream_requests.select_next_some() => {
                    self.handle_stream_request_message(stream_request);
                }
                _ = data_refresh_interval.select_next_some() => {
                    self.refresh_global_data_summary();
                }
                _ = progress_check_interval.select_next_some() => {
                    self.check_progress_of_all_data_streams();
                }
            }
        }
    }

    /// Handles new stream request messages from clients
    fn handle_stream_request_message(&mut self, request_message: StreamRequestMessage) {
        if let StreamRequest::TerminateStream(request) = request_message.stream_request {
            // Process the feedback request
            if let Err(error) = self.process_terminate_stream_request(&request) {
                error!(LogSchema::new(LogEntry::HandleTerminateRequest)
                    .event(LogEvent::Error)
                    .error(&error));
            }
            return;
        }

        // Process the stream request
        let response = self.process_new_stream_request(&request_message);
        if let Err(error) = &response {
            sample!(
                SampleRate::Duration(Duration::from_secs(STREAM_REQUEST_ERROR_LOG_FREQ_SECS)),
                error!(LogSchema::new(LogEntry::HandleStreamRequest)
                    .event(LogEvent::Error)
                    .error(error));
            );
        }

        // Send the response to the client
        if let Err(error) = request_message.response_sender.send(response) {
            error!(LogSchema::new(LogEntry::RespondToStreamRequest)
                .event(LogEvent::Error)
                .message(&format!(
                    "Failed to send response for stream request: {:?}",
                    error
                )));
        }
    }

    /// Processes a request for terminating the stream that sent a specific
    /// notification ID.
    fn process_terminate_stream_request(
        &mut self,
        terminate_request: &TerminateStreamRequest,
    ) -> Result<(), Error> {
        // Increment the stream termination counter
        let notification_feedback = &terminate_request.notification_feedback;
        metrics::increment_counter(
            &metrics::TERMINATE_DATA_STREAM,
            notification_feedback.get_label().into(),
        );

        // Find the data stream that sent the notification
        let notification_id = &terminate_request.notification_id;
        let data_stream_ids = self.get_all_data_stream_ids();
        for data_stream_id in &data_stream_ids {
            let data_stream = self.get_data_stream(data_stream_id);
            if data_stream.sent_notification(notification_id) {
                info!(LogSchema::new(LogEntry::HandleTerminateRequest)
                    .stream_id(*data_stream_id)
                    .event(LogEvent::Success)
                    .message(&format!(
                        "Terminating the stream that sent notification ID: {:?}. Feedback: {:?}",
                        notification_id, notification_feedback,
                    )));
                data_stream.handle_notification_feedback(notification_id, notification_feedback)?;
                self.data_streams.remove(notification_id);
                return Ok(());
            }
        }

        panic!(
            "Unable to find the stream that sent notification ID: {:?}. Feedback: {:?}",
            notification_id, notification_feedback,
        );
    }

    /// Creates a new stream and ensures the data for that stream is available
    fn process_new_stream_request(
        &mut self,
        request_message: &StreamRequestMessage,
    ) -> Result<DataStreamListener, Error> {
        // Increment the stream creation counter
        metrics::increment_counter(
            &metrics::CREATE_DATA_STREAM,
            request_message.stream_request.get_label().into(),
        );

        // Refresh the cached global data summary
        self.refresh_global_data_summary();

        // Create a new data stream
        let stream_id = self.stream_id_generator.next();
        let (data_stream, stream_listener) = DataStream::new(
            self.config,
            stream_id,
            &request_message.stream_request,
            self.aptos_data_client.clone(),
            self.notification_id_generator.clone(),
            &self.global_data_summary.advertised_data,
        )?;

        // Verify the data stream can be fulfilled using the currently advertised data
        data_stream.ensure_data_is_available(&self.global_data_summary.advertised_data)?;

        // Store the data stream internally
        if self.data_streams.insert(stream_id, data_stream).is_some() {
            panic!(
                "Duplicate data stream found! This should not occur! ID: {:?}",
                stream_id,
            );
        }
        info!(LogSchema::new(LogEntry::HandleStreamRequest)
            .stream_id(stream_id)
            .event(LogEvent::Success)
            .message(&format!(
                "Stream created for request: {:?}",
                request_message
            )));

        // Return the listener
        Ok(stream_listener)
    }

    /// Refreshes the global data summary by communicating with the Aptos data client
    fn refresh_global_data_summary(&mut self) {
        if let Err(error) = self.fetch_global_data_summary() {
            metrics::increment_counter(
                &metrics::GLOBAL_DATA_SUMMARY_ERROR,
                error.get_label().into(),
            );
            sample!(
                SampleRate::Duration(Duration::from_secs(GLOBAL_DATA_REFRESH_LOG_FREQ_SECS)),
                error!(LogSchema::new(LogEntry::RefreshGlobalData)
                    .event(LogEvent::Error)
                    .error(&error))
            );
        }
    }

    fn fetch_global_data_summary(&mut self) -> Result<(), Error> {
        let global_data_summary = self.aptos_data_client.get_global_data_summary();
        if global_data_summary.is_empty() {
            sample!(
                SampleRate::Duration(Duration::from_secs(GLOBAL_DATA_REFRESH_LOG_FREQ_SECS)),
                info!(LogSchema::new(LogEntry::RefreshGlobalData)
                    .message("Latest global data summary is empty."))
            );
        } else {
            verify_optimal_chunk_sizes(&global_data_summary.optimal_chunk_sizes)?;
            self.global_data_summary = global_data_summary;
        }

        Ok(())
    }

    /// Ensures that all existing data streams are making progress
    fn check_progress_of_all_data_streams(&mut self) {
        let data_stream_ids = self.get_all_data_stream_ids();
        for data_stream_id in &data_stream_ids {
            if let Err(error) = self.update_progress_of_data_stream(data_stream_id) {
                if matches!(error, Error::NoDataToFetch(_)) {
                    sample!(
                        SampleRate::Duration(Duration::from_secs(NO_DATA_TO_FETCH_LOG_FREQ_SECS)),
                        info!(LogSchema::new(LogEntry::CheckStreamProgress)
                            .stream_id(*data_stream_id)
                            .event(LogEvent::Pending)
                            .error(&error))
                    );
                } else {
                    metrics::increment_counter(
                        &metrics::CHECK_STREAM_PROGRESS_ERROR,
                        error.get_label().into(),
                    );
                    error!(LogSchema::new(LogEntry::CheckStreamProgress)
                        .stream_id(*data_stream_id)
                        .event(LogEvent::Error)
                        .error(&error));
                }
            }
        }
    }

    /// Ensures that a data stream has in-flight data requests and handles
    /// any new responses that have arrived since we last checked.
    fn update_progress_of_data_stream(
        &mut self,
        data_stream_id: &DataStreamId,
    ) -> Result<(), Error> {
        let global_data_summary = self.global_data_summary.clone();

        let data_stream = self.get_data_stream_mut(data_stream_id);
        if !data_stream.data_requests_initialized() {
            // Initialize the request batch by sending out data client requests
            data_stream.initialize_data_requests(global_data_summary)?;
            info!(
                (LogSchema::new(LogEntry::InitializeStream)
                    .stream_id(*data_stream_id)
                    .event(LogEvent::Success)
                    .message("Data stream initialized."))
            );
        } else {
            // Process any data client requests that have received responses
            data_stream.process_data_responses(global_data_summary)?;
        }

        Ok(())
    }

    fn get_all_data_stream_ids(&self) -> Vec<DataStreamId> {
        self.data_streams
            .keys()
            .cloned()
            .collect::<Vec<DataStreamId>>()
    }

    /// Returns the data stream associated with the given `data_stream_id`.
    /// Note: this method assumes the caller has already verified the stream exists.
    fn get_data_stream(&self, data_stream_id: &DataStreamId) -> &DataStream<T> {
        self.data_streams.get(data_stream_id).unwrap_or_else(|| {
            panic!(
                "Expected a data stream with ID: {:?}, but found None!",
                data_stream_id
            )
        })
    }

    /// Returns the data stream associated with the given `data_stream_id`.
    /// Note: this method assumes the caller has already verified the stream exists.
    fn get_data_stream_mut(&mut self, data_stream_id: &DataStreamId) -> &mut DataStream<T> {
        self.data_streams
            .get_mut(data_stream_id)
            .unwrap_or_else(|| {
                panic!(
                    "Expected a data stream with ID: {:?}, but found None!",
                    data_stream_id
                )
            })
    }
}

/// Verifies that all optimal chunk sizes are valid (i.e., not zero). Returns an
/// error if a chunk size is 0.
fn verify_optimal_chunk_sizes(optimal_chunk_sizes: &OptimalChunkSizes) -> Result<(), Error> {
    if optimal_chunk_sizes.account_states_chunk_size == 0
        || optimal_chunk_sizes.epoch_chunk_size == 0
        || optimal_chunk_sizes.transaction_chunk_size == 0
        || optimal_chunk_sizes.transaction_output_chunk_size == 0
    {
        Err(Error::AptosDataClientResponseIsInvalid(format!(
            "Found at least one optimal chunk size of zero: {:?}",
            optimal_chunk_sizes
        )))
    } else {
        Ok(())
    }
}
<|endoftext|>


<commit_before>use url::Url;
use serde::de::{Deserialize, Deserializer, Error as DeserializeError};
use serde::ser::{Serialize, Serializer};

/// Type which can be used to serialize a Url.
///
/// `#[serde(with = "UrlDef")]`
pub struct UrlDef(Url);

impl UrlDef {
    pub fn deserialize<'de, D>(deserializer: D) -> Result<Url, D::Error>
            where D: Deserializer<'de> {
        <&str>::deserialize(deserializer).and_then(|s| {
            s.parse().map_err(|e| DeserializeError::custom(format!("invalid URL: {}", e)))
        })
    }

    pub fn serialize<S>(url: &Url, serializer: S) -> Result<S::Ok, S::Error>
            where S: Serializer {
        url.as_str().serialize(serializer)
    }
}

impl From<UrlDef> for Url {
    fn from(def: UrlDef) -> Url {
        def.0
    }
}
<commit_msg>Remove unused parts of UrlDef<commit_after>use url::Url;
use serde::de::{Deserialize, Deserializer, Error as DeserializeError};
use serde::ser::{Serialize, Serializer};

/// Unit struct which can be used to serialize a Url.
///
/// `#[serde(with = "UrlDef")]`
pub struct UrlDef;
impl UrlDef {
    pub fn deserialize<'de, D>(deserializer: D) -> Result<Url, D::Error>
            where D: Deserializer<'de> {
        <&str>::deserialize(deserializer).and_then(|s| {
            s.parse().map_err(|e| DeserializeError::custom(format!("invalid URL: {}", e)))
        })
    }

    pub fn serialize<S>(url: &Url, serializer: S) -> Result<S::Ok, S::Error>
            where S: Serializer {
        url.as_str().serialize(serializer)
    }
}
<|endoftext|>


<commit_before>use std::fs::{File, OpenOptions, metadata};
use std::path::{Path, PathBuf};

use bincode::SizeLimit;
use bincode::rustc_serialize::{encode_into, decode_from};

use super::Error;
use super::database::Database;
use super::row::Row;

pub struct Table<'a> {
    database: &'a Database,
    name: String,
    meta_data: TableMetadata,
    rows: Vec<Row>
}

#[derive(Clone, RustcDecodable, RustcEncodable)]
struct TableMetadata {
    EngineID: u8,
}

/// Table representation
impl<'a> Table<'a> {
    /// Create table in database
    pub fn create<'b>(name: &str, engineID: u8, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("creating table: {}", name);

        let metadata = TableMetadata { EngineID: engineID };

        let mut buf = try!(File::create(db.get_path().clone().join(&name).with_extension("tbl")));

        encode_into(&metadata, &mut buf, SizeLimit::Infinite).unwrap();

        Ok(Table {
            database: &db,
            name: name.to_string(),
            meta_data: metadata.clone(),
            rows: Vec::new()
        })
    }

    /// Load table from database
    pub fn load<'b>(name: &str, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("loading table: {}", &name);
        let path = db.get_path().clone().join(&name).with_extension("tbl");
        if !try!(metadata(&path)).is_dir()
        {
            let mut file = try!(OpenOptions::new()
                        .read(true)
                        .open(&path));

            let metadata: TableMetadata = try!(decode_from(&mut file, SizeLimit::Infinite));
            info!("loaded table: {}", &name);

            Ok(Table {
                database: db,
                name: name.to_string(),
                meta_data: metadata,
                rows: Vec::new() // TODO: actually encode rows
            })
        }else {
            error!("could not load table: {} at {}", &name, &path.to_str().unwrap());
            Err(Error::LoadTable)
        }
    }

    /// Get full file path including filename and ext
    pub fn get_path(&self) -> PathBuf {
        self.database.get_path().clone().join(&self.name).with_extension("tbl")
    }

    pub fn get_engine_id(&self) -> u8 {
        self.meta_data.EngineID
    }

}

#[cfg(test)]
mod test {
    use super::Table;
    use super::super::database::*;
    use std::fs::metadata;

    #[test]
    fn create_table() {
        let path = "/tmp/test1/";
        let name = "test_db3";

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 0, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());
    }

    #[test]
    fn load_table() {
        let path = "/tmp/test1/";
        let name = "test_db4";

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 34, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());

        let table2 = Table::load("test_table1", &db).unwrap();

        assert_eq!(table.get_engine_id(), table2.get_engine_id());
    }
}
<commit_msg>updated table test for column<commit_after>use std::fs::{File, OpenOptions, metadata};
use std::path::{Path, PathBuf};

use bincode::SizeLimit;
use bincode::rustc_serialize::{encode_into, decode_from};

use super::Error;
use super::database::Database;
use super::column::Column;
use super::row::Row;

pub struct Table<'a> {
    database: &'a Database,
    name: String,
    meta_data: TableMetadata
}

#[derive(Clone, RustcDecodable, RustcEncodable)]
struct TableMetadata {
    engine_id: u8,
    columns: Vec<Column>
}

/// Table representation
impl<'a> Table<'a> {
    /// Create table in database
    pub fn create<'b>(name: &str, engine_id: u8, columns: Vec<Column>, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("creating table: {}", name);

        let metadata = TableMetadata {
            engine_id: engine_id,
            columns: columns
        };

        let mut buf = try!(File::create(db.get_path().clone().join(&name).with_extension("tbl")));

        encode_into(&metadata, &mut buf, SizeLimit::Infinite).unwrap();

        Ok(Table {
            database: &db,
            name: name.to_string(),
            meta_data: metadata.clone()
        })
    }

    /// Load table from database
    pub fn load<'b>(name: &str, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("loading table: {}", &name);
        let path = db.get_path().clone().join(&name).with_extension("tbl");
        if !try!(metadata(&path)).is_dir()
        {
            let mut file = try!(OpenOptions::new()
                        .read(true)
                        .open(&path));

            let metadata: TableMetadata = try!(decode_from(&mut file, SizeLimit::Infinite));
            info!("loaded table: {}", &name);

            Ok(Table {
                database: db,
                name: name.to_string(),
                meta_data: metadata
            })
        }else {
            error!("could not load table: {} at {}", &name, &path.to_str().unwrap());
            Err(Error::LoadTable)
        }
    }

    pub fn get_cols_sizes(&self) -> Vec<u32> {
        let mut column_sizes = Vec::<u32>::new();

        for v in &self.meta_data.columns {
            column_sizes.push(v.size());
        }

        column_sizes
    }

    /// Get full file path including filename and ext
    pub fn get_path(&self) -> PathBuf {
        self.database.get_path().clone().join(&self.name).with_extension("tbl")
    }

    pub fn get_engine_id(&self) -> u8 {
        self.meta_data.engine_id
    }

}

#[cfg(test)]
mod test {
    use super::Table;
    use super::super::database::*;
    use std::fs::metadata;
    use super::super::column::Column;
    use super::super::types::Types;

    #[test]
    fn create_table() {
        let path = "/tmp/test1/";
        let name = "test_db3";
        let mut columns = Vec::<Column>::new();
        columns.push(Column::new("Name", Types::Char(10)));
        columns.push(Column::new("Age", Types::Int));

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 0, columns, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());
    }

    #[test]
    fn load_table() {
        let path = "/tmp/test1/";
        let name = "test_db4";

        let mut columns = Vec::<Column>::new();
        columns.push(Column::new("Name", Types::Char(10)));
        columns.push(Column::new("Age", Types::Int));

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 34, columns, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());

        let table2 = Table::load("test_table1", &db).unwrap();

        assert_eq!(table.get_engine_id(), table2.get_engine_id());
    }
}
<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A symbol reader reads one symbol at a time from a source
//!

use std::slice::Iter;
use std::io::Read;
use std::io::Bytes;
use std::str::Chars;
use std::marker::PhantomData;

///
/// A symbol reader reads one symbol at a time from a source
///
pub trait SymbolReader<Symbol> {
    ///
    /// Reads the next symbol from this reader
    ///
    fn next_symbol(&mut self) -> Option<Symbol>;
}

///
/// Trait that can be implemented by things that can construct a symbol reader
///
pub trait SymbolSource<'a, Symbol> {
    /// The type of symbol reader that will be generated by this call
    type SymbolReader: SymbolReader<Symbol>+'a;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader;
}

impl<'a, Symbol: Clone+'a> SymbolSource<'a, Symbol> for &'a Vec<Symbol> {
    type SymbolReader = Iter<'a, Symbol>;

    fn read_symbols(self) -> Self::SymbolReader {
        self.iter()
    }
}

impl<'a, Symbol: Clone+'a> SymbolReader<Symbol> for Iter<'a, Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        if let Some(sym) = self.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

// Can read from streams

//
// TODO: implementing SymbolSource for Read would be nice
// It's just a matter of calling bytes() on the appropriate type, but Rust's type system is simultaneously too clever and not clever enough
// to allow that in any sensible way. You can't do it an an arbitrary Read because the Bytes object is not sized without knowing its real
// type; I couldn't figure out a trick using the Self type to get around that, though those do work sometimes.
// `impl<X: Read> SymbolReader for X` sounds like a nice idea but Rust thinks X can be anything and won't allow any other implementations
// of the interface, which makes that useless too.
// Screwing around with lifetimes and references like below seems like it should work, but Rust doesn't seem to see the implementations on
// actual streams.
//

/*
impl<'a> SymbolSource<'a, u8> for &'a mut Read {
    type SymbolReader = ByteSymbolReader<Self>;

    fn read_symbols(self) -> Self::SymbolReader {
        ByteSymbolReader::new(self.bytes())
    }
}

impl<'a> SymbolSource<'a, u8> for &'a mut BufRead {
    type SymbolReader = ByteSymbolReader<Self>;

    fn read_symbols(self) -> Self::SymbolReader {
        ByteSymbolReader::new(self.bytes())
    }
}
*/

///
/// The ByteSymbolReader turns a `std::io::Bytes` object into a symbol reader
///
pub struct ByteSymbolReader<Reader: Read> {
    bytes: Bytes<Reader>
}

impl<Reader: Read> ByteSymbolReader<Reader> {
    pub fn new(bytes: Bytes<Reader>) -> ByteSymbolReader<Reader> {
        ByteSymbolReader { bytes: bytes }
    }

    pub fn from(reader: Reader) -> ByteSymbolReader<Reader> {
        Self::new(reader.bytes())
    }
}

impl<Reader: Read> SymbolReader<u8> for ByteSymbolReader<Reader> {
    fn next_symbol(&mut self) -> Option<u8> {
        if let Some(Ok(sym)) = self.bytes.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

//
// Can read from strings
//
impl<'a> SymbolSource<'a, char> for &'a str {
    type SymbolReader = Chars<'a>;

    fn read_symbols(self) -> Self::SymbolReader {
        self.chars()
    }
}

impl<'a> SymbolReader<char> for Chars<'a> {
    fn next_symbol(&mut self) -> Option<char> {
        self.next()
    }
}

//
// Boxed symbol readers act like normal symbol readers
//
// Note the lifetime specifier here (boxes are static by default which can cause a super confusing error message)
//
impl<'a, Symbol> SymbolReader<Symbol> for Box<SymbolReader<Symbol>+'a> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        (**self).next_symbol()
    }
}

///
/// Converts symbol streams to vectors
///
pub trait ReaderToVector<Symbol> {
    /// Reads every symbol in this object and returns a vector
    fn to_vec(&mut self) -> Vec<Symbol>;
}

impl<Symbol, Reader: SymbolReader<Symbol>> ReaderToVector<Symbol> for Reader {
    #[inline]
    fn to_vec(&mut self) -> Vec<Symbol> {
        let mut result = vec![];

        while let Some(symbol) = self.next_symbol() {
            result.push(symbol);
        }

        result
    }
}

///
/// A symbol stream that performs mapping of symbols from a source stream
///
pub struct MappedStream<InputSymbol, MapFunction, Reader>
where Reader: SymbolReader<InputSymbol> {
    /// The source stream
    source_stream: Reader,

    /// A function used to map a symbol from the source stream to a symbol in the output stream
    mapping_function: MapFunction,

    /// Can't see how to eliminate the input symbol reference due to rust's type system. InputSymbol is used by source_stream, but rust can't see that
    #[allow(dead_code)]
    phantom: PhantomData<InputSymbol>
}

///
/// Provides a way to map symbol streams to streams of other types
///
pub trait MapSymbolReader<Symbol> : SymbolReader<Symbol>+Sized {
    /// Maps symbols in this stream to symbols in a new stream
    fn map_symbols<OutputSymbol, MapFunction>(self, mapping_function: MapFunction) -> MappedStream<Symbol, MapFunction, Self>
    where MapFunction: FnMut(Symbol) -> OutputSymbol;
}

impl<Symbol, Reader: SymbolReader<Symbol>> MapSymbolReader<Symbol> for Reader {
    fn map_symbols<OutputSymbol, MapFunction>(self, mapping_function: MapFunction) -> MappedStream<Symbol, MapFunction, Self>
    where MapFunction: FnMut(Symbol) -> OutputSymbol {
        MappedStream {
            source_stream:    self,
            mapping_function: mapping_function,
            phantom:          PhantomData
        }
    }
}

impl<InputSymbol, OutputSymbol, MapFunction, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for MappedStream<InputSymbol, MapFunction, Reader>
where MapFunction: FnMut(InputSymbol) -> OutputSymbol {
    // TODO: would like to eliminate InputSymbol as phantom data from the structure
    // This means removing the constraint on Reader. That doesn't work because rust then thinks InputSymbol is unconstrained here,
    // even though it's constrained by the function type, the reader type *and* the structure type

    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        if let Some(input_symbol) = self.source_stream.next_symbol() {
            Some((self.mapping_function)(input_symbol))
        } else {
            None
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn can_read_from_vec() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == None);
    }

    #[test]
    fn can_convert_to_vec() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();
        let result      = reader.to_vec();

        assert!(result == vec![1, 2, 3]);
    }

    #[test]
    fn can_map_stream() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();
        let result      = reader.map_symbols(|sym| sym+1).to_vec();

        assert!(result == vec![2, 3, 4]);
    }

    #[test]
    fn can_read_from_bytes_reader() {
        let array: [u8; 3] = [1, 2, 3];
        let slice = &array[..];
        let mut reader = ByteSymbolReader::from(slice);

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == None);
    }

    #[test]
    fn can_read_from_string_reader() {
        let mut reader = "abc".read_symbols();

        assert!(reader.next_symbol() == Some('a'));
        assert!(reader.next_symbol() == Some('b'));
        assert!(reader.next_symbol() == Some('c'));
        assert!(reader.next_symbol() == None);
    }
}
<commit_msg>Add a new struct that can be used to convert a vec directly to a symbol reader.<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A symbol reader reads one symbol at a time from a source
//!

use std::slice::Iter;
use std::io::Read;
use std::io::Bytes;
use std::str::Chars;
use std::marker::PhantomData;

///
/// A symbol reader reads one symbol at a time from a source
///
pub trait SymbolReader<Symbol> {
    ///
    /// Reads the next symbol from this reader
    ///
    fn next_symbol(&mut self) -> Option<Symbol>;
}

///
/// Trait that can be implemented by things that can construct a symbol reader
///
pub trait SymbolSource<'a, Symbol> {
    /// The type of symbol reader that will be generated by this call
    type SymbolReader: SymbolReader<Symbol>+'a;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader;
}

impl<'a, Symbol: Clone+'a> SymbolSource<'a, Symbol> for &'a Vec<Symbol> {
    type SymbolReader = Iter<'a, Symbol>;

    fn read_symbols(self) -> Self::SymbolReader {
        self.iter()
    }
}

impl<'a, Symbol: Clone+'a> SymbolReader<Symbol> for Iter<'a, Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        if let Some(sym) = self.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

// Can read from streams

//
// TODO: implementing SymbolSource for Read would be nice
// It's just a matter of calling bytes() on the appropriate type, but Rust's type system is simultaneously too clever and not clever enough
// to allow that in any sensible way. You can't do it an an arbitrary Read because the Bytes object is not sized without knowing its real
// type; I couldn't figure out a trick using the Self type to get around that, though those do work sometimes.
// `impl<X: Read> SymbolReader for X` sounds like a nice idea but Rust thinks X can be anything and won't allow any other implementations
// of the interface, which makes that useless too.
// Screwing around with lifetimes and references like below seems like it should work, but Rust doesn't seem to see the implementations on
// actual streams.
//

/*
impl<'a> SymbolSource<'a, u8> for &'a mut Read {
    type SymbolReader = ByteSymbolReader<Self>;

    fn read_symbols(self) -> Self::SymbolReader {
        ByteSymbolReader::new(self.bytes())
    }
}

impl<'a> SymbolSource<'a, u8> for &'a mut BufRead {
    type SymbolReader = ByteSymbolReader<Self>;

    fn read_symbols(self) -> Self::SymbolReader {
        ByteSymbolReader::new(self.bytes())
    }
}
*/

///
/// The ByteSymbolReader turns a `std::io::Bytes` object into a symbol reader
///
pub struct ByteSymbolReader<Reader: Read> {
    bytes: Bytes<Reader>
}

impl<Reader: Read> ByteSymbolReader<Reader> {
    pub fn new(bytes: Bytes<Reader>) -> ByteSymbolReader<Reader> {
        ByteSymbolReader { bytes: bytes }
    }

    pub fn from(reader: Reader) -> ByteSymbolReader<Reader> {
        Self::new(reader.bytes())
    }
}

impl<Reader: Read> SymbolReader<u8> for ByteSymbolReader<Reader> {
    fn next_symbol(&mut self) -> Option<u8> {
        if let Some(Ok(sym)) = self.bytes.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

//
// Can read from strings
//
impl<'a> SymbolSource<'a, char> for &'a str {
    type SymbolReader = Chars<'a>;

    fn read_symbols(self) -> Self::SymbolReader {
        self.chars()
    }
}

impl<'a> SymbolReader<char> for Chars<'a> {
    fn next_symbol(&mut self) -> Option<char> {
        self.next()
    }
}

//
// Boxed symbol readers act like normal symbol readers
//
// Note the lifetime specifier here (boxes are static by default which can cause a super confusing error message)
//
impl<'a, Symbol> SymbolReader<Symbol> for Box<SymbolReader<Symbol>+'a> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        (**self).next_symbol()
    }
}

///
/// Converts symbol streams to vectors
///
pub trait ReaderToVector<Symbol> {
    /// Reads every symbol in this object and returns a vector
    fn to_vec(&mut self) -> Vec<Symbol>;
}

impl<Symbol, Reader: SymbolReader<Symbol>> ReaderToVector<Symbol> for Reader {
    #[inline]
    fn to_vec(&mut self) -> Vec<Symbol> {
        let mut result = vec![];

        while let Some(symbol) = self.next_symbol() {
            result.push(symbol);
        }

        result
    }
}

///
/// A symbol stream that performs mapping of symbols from a source stream
///
pub struct MappedStream<InputSymbol, MapFunction, Reader>
where Reader: SymbolReader<InputSymbol> {
    /// The source stream
    source_stream: Reader,

    /// A function used to map a symbol from the source stream to a symbol in the output stream
    mapping_function: MapFunction,

    /// Can't see how to eliminate the input symbol reference due to rust's type system. InputSymbol is used by source_stream, but rust can't see that
    #[allow(dead_code)]
    phantom: PhantomData<InputSymbol>
}

///
/// Provides a way to map symbol streams to streams of other types
///
pub trait MapSymbolReader<Symbol> : SymbolReader<Symbol>+Sized {
    /// Maps symbols in this stream to symbols in a new stream
    fn map_symbols<OutputSymbol, MapFunction>(self, mapping_function: MapFunction) -> MappedStream<Symbol, MapFunction, Self>
    where MapFunction: FnMut(Symbol) -> OutputSymbol;
}

impl<Symbol, Reader: SymbolReader<Symbol>> MapSymbolReader<Symbol> for Reader {
    fn map_symbols<OutputSymbol, MapFunction>(self, mapping_function: MapFunction) -> MappedStream<Symbol, MapFunction, Self>
    where MapFunction: FnMut(Symbol) -> OutputSymbol {
        MappedStream {
            source_stream:    self,
            mapping_function: mapping_function,
            phantom:          PhantomData
        }
    }
}

impl<InputSymbol, OutputSymbol, MapFunction, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for MappedStream<InputSymbol, MapFunction, Reader>
where MapFunction: FnMut(InputSymbol) -> OutputSymbol {
    // TODO: would like to eliminate InputSymbol as phantom data from the structure
    // This means removing the constraint on Reader. That doesn't work because rust then thinks InputSymbol is unconstrained here,
    // even though it's constrained by the function type, the reader type *and* the structure type

    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        if let Some(input_symbol) = self.source_stream.next_symbol() {
            Some((self.mapping_function)(input_symbol))
        } else {
            None
        }
    }
}

///
/// A VecReader consumes a vector, which can be read out using the SymbolReader trait
///
pub struct VecReader<Symbol> {
    symbols: Vec<Symbol>
}

impl<Symbol> VecReader<Symbol> {
    ///
    /// Creates a new VecReader by consuming a Vec
    ///
    pub fn from_vec(vec: Vec<Symbol>) -> VecReader<Symbol> {
        let mut reversed = vec;
        reversed.reverse();
        VecReader { symbols: reversed }
    }
}

impl<Symbol> SymbolReader<Symbol> for VecReader<Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        self.symbols.pop()
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn can_read_from_vec() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == None);
    }

    #[test]
    fn can_read_from_vecreader() {
        let source      = vec![1, 2, 3];
        let mut reader  = VecReader::from_vec(source);

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == None);
    }

    #[test]
    fn can_convert_to_vec() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();
        let result      = reader.to_vec();

        assert!(result == vec![1, 2, 3]);
    }

    #[test]
    fn can_map_stream() {
        let source = vec![1, 2, 3];
        let reader = source.read_symbols();
        let result = reader.map_symbols(|sym| sym+1).to_vec();

        assert!(result == vec![2, 3, 4]);
    }

    #[test]
    fn can_read_from_bytes_reader() {
        let array: [u8; 3] = [1, 2, 3];
        let slice = &array[..];
        let mut reader = ByteSymbolReader::from(slice);

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == None);
    }

    #[test]
    fn can_read_from_string_reader() {
        let mut reader = "abc".read_symbols();

        assert!(reader.next_symbol() == Some('a'));
        assert!(reader.next_symbol() == Some('b'));
        assert!(reader.next_symbol() == Some('c'));
        assert!(reader.next_symbol() == None);
    }
}
<|endoftext|>


<commit_before>//! Provide helpers for making ioctl system calls
//!
//! Currently supports Linux on all architectures. Other platforms welcome!
//!
//! This library is pretty low-level and messy. `ioctl` is not fun.
//!
//! What is an `ioctl`?
//! ===================
//!
//! The `ioctl` syscall is the grab-bag syscall on POSIX systems. Don't want
//! to add a new syscall? Make it an `ioctl`! `ioctl` refers to both the syscall,
//! and the commands that can be send with it. `ioctl` stands for "IO control",
//! and the commands are always sent to a file descriptor.
//!
//! It is common to see `ioctl`s used for the following purposes:
//!
//! * Provide read/write access to out-of-band data related
//!   to a device such as configuration (for instance, setting
//!   serial port options)
//! * Provide a mechanism for performing full-duplex data
//!   transfers (for instance, xfer on SPI devices).
//! * Provide access to control functions on a device (for example,
//!   on Linux you can send commands like pause, resume, and eject
//!   to the CDROM device.
//! * Do whatever else the device driver creator thought made most sense.
//!
//! `ioctl`s are synchronous system calls and are similar to read and
//! write calls in that regard.
//!
//! What does this module support?
//! ===============================
//!
//! This library provides the `ioctl!` macro, for binding `ioctl`s.
//! Here's a few examples of how that can work for SPI under Linux
//! from [rust-spidev](https://github.com/posborne/rust-spidev).
//!
//! ```
//! #[allow(non_camel_case_types)]
//! pub struct spi_ioc_transfer {
//!     pub tx_buf: u64,
//!     pub rx_buf: u64,
//!     pub len: u32,
//!
//!     // optional overrides
//!     pub speed_hz: u32,
//!     pub delay_usecs: u16,
//!     pub bits_per_word: u8,
//!     pub cs_change: u8,
//!     pub pad: u32,
//! }
//!
//! #[cfg(linux)]
//! mod ioctl {
//!     use super::*;
//!
//!     const SPI_IOC_MAGIC: u8 = 'k' as u8;
//!     const SPI_IOC_NR_TRANSFER: u8 = 0;
//!     const SPI_IOC_NR_MODE: u8 = 1;
//!     const SPI_IOC_NR_LSB_FIRST: u8 = 2;
//!     const SPI_IOC_NR_BITS_PER_WORD: u8 = 3;
//!     const SPI_IOC_NR_MAX_SPEED_HZ: u8 = 4;
//!     const SPI_IOC_NR_MODE32: u8 = 5;
//!
//!     ioctl!(read  get_mode_u8 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE; u8);
//!     ioctl!(read  get_mode_u32 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE; u32);
//!     ioctl!(write set_mode_u8 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE; u8);
//!     ioctl!(write set_mode_u32 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE32; u32);
//!     ioctl!(read  get_lsb_first with SPI_IOC_MAGIC, SPI_IOC_NR_LSB_FIRST; u8);
//!     ioctl!(write set_lsb_first with SPI_IOC_MAGIC, SPI_IOC_NR_LSB_FIRST; u8);
//!     ioctl!(read  get_bits_per_word with SPI_IOC_MAGIC, SPI_IOC_NR_BITS_PER_WORD; u8);
//!     ioctl!(write set_bits_per_word with SPI_IOC_MAGIC, SPI_IOC_NR_BITS_PER_WORD; u8);
//!     ioctl!(read  get_max_speed_hz with SPI_IOC_MAGIC, SPI_IOC_NR_MAX_SPEED_HZ; u32);
//!     ioctl!(write set_max_speed_hz with SPI_IOC_MAGIC, SPI_IOC_NR_MAX_SPEED_HZ; u32);
//!     ioctl!(write spidev_transfer with SPI_IOC_MAGIC, SPI_IOC_NR_TRANSFER; spi_ioc_transfer);
//!     ioctl!(write buf spidev_transfer_buf with SPI_IOC_MAGIC, SPI_IOC_NR_TRANSFER; spi_ioc_transfer);
//! }
//!
//! // doctest workaround
//! fn main() {}
//! ```
//!
//! Spidev uses the `_IOC` macros that are encouraged (as far as
//! `ioctl` can be encouraged at all) for newer drivers.  Many
//! drivers, however, just use magic numbers with no attached
//! semantics.  For those, the `ioctl!(bad ...)` variant should be
//! used (the "bad" terminology is from the Linux kernel).
//!
//! How do I get the magic numbers?
//! ===============================
//!
//! For Linux, look at your system's headers. For example, `/usr/include/linux/input.h` has a lot
//! of lines defining macros which use `_IOR`, `_IOW`, `_IOC`, and `_IORW`.  These macros
//! correspond to the `ior!`, `iow!`, `ioc!`, and `iorw!` macros defined in this crate.
//! Additionally, there is the `ioctl!` macro for creating a wrapper around `ioctl` that is
//! somewhat more type-safe.
//!
//! Most `ioctl`s have no or little documentation. You'll need to scrounge through
//! the source to figure out what they do and how they should be used.
//!
#[cfg(any(target_os = "linux", target_os = "android"))]
#[path = "platform/linux.rs"]
#[macro_use]
mod platform;

#[cfg(any(target_os = "macos",
          target_os = "ios",
          target_os = "netbsd",
          target_os = "openbsd",
          target_os = "freebsd",
          target_os = "dragonfly"))]
#[path = "platform/bsd.rs"]
#[macro_use]
mod platform;

pub use self::platform::*;

/// A hack to get the macros to work nicely.
#[doc(hidden)]
pub use ::libc as libc;

/// Convert raw ioctl return value to a Nix result
#[macro_export]
#[doc(hidden)]
macro_rules! convert_ioctl_res {
    ($w:expr) => (
        {
            $crate::Errno::result($w)
        }
    );
}

#[macro_export]
macro_rules! ioctl {
    (bad $name:ident with $nr:expr) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            data: *mut u8)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, $nr as $crate::sys::ioctl::libc::c_ulong, data))
        }
        );
    (bad none $name:ident with $nr:expr) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, $nr as $crate::sys::ioctl::libc::c_ulong))
        }
        );
    (none $name:ident with $ioty:expr, $nr:expr) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, io!($ioty, $nr) as $crate::sys::ioctl::libc::c_ulong))
        }
        );
    (read $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            val: *mut $ty)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, ior!($ioty, $nr, ::std::mem::size_of::<$ty>()) as $crate::sys::ioctl::libc::c_ulong, val))
        }
        );
    (write $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            val: $ty)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, iow!($ioty, $nr, ::std::mem::size_of::<$ty>()) as $crate::sys::ioctl::libc::c_ulong, val))
        }
        );
    (readwrite $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            val: *mut $ty)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, iorw!($ioty, $nr, ::std::mem::size_of::<$ty>()) as $crate::sys::ioctl::libc::c_ulong, val))
        }
        );
    (read buf $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            val: *mut $ty,
                            len: usize)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, ior!($ioty, $nr, len) as $crate::sys::ioctl::libc::c_ulong, val))
        }
        );
    (write buf $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            val: *const $ty,
                            len: usize) -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, iow!($ioty, $nr, len) as $crate::sys::ioctl::libc::c_ulong, val))
        }
        );
    (readwrite buf $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::sys::ioctl::libc::c_int,
                            val: *mut $ty,
                            len: usize)
                            -> $crate::Result<$crate::sys::ioctl::libc::c_int> {
            convert_ioctl_res!($crate::sys::ioctl::libc::ioctl(fd, iorw!($ioty, $nr, len) as $crate::sys::ioctl::libc::c_ulong, val))
        }
        );
}
<commit_msg>Remove unnecessary path aliasing<commit_after>//! Provide helpers for making ioctl system calls
//!
//! Currently supports Linux on all architectures. Other platforms welcome!
//!
//! This library is pretty low-level and messy. `ioctl` is not fun.
//!
//! What is an `ioctl`?
//! ===================
//!
//! The `ioctl` syscall is the grab-bag syscall on POSIX systems. Don't want
//! to add a new syscall? Make it an `ioctl`! `ioctl` refers to both the syscall,
//! and the commands that can be send with it. `ioctl` stands for "IO control",
//! and the commands are always sent to a file descriptor.
//!
//! It is common to see `ioctl`s used for the following purposes:
//!
//! * Provide read/write access to out-of-band data related
//!   to a device such as configuration (for instance, setting
//!   serial port options)
//! * Provide a mechanism for performing full-duplex data
//!   transfers (for instance, xfer on SPI devices).
//! * Provide access to control functions on a device (for example,
//!   on Linux you can send commands like pause, resume, and eject
//!   to the CDROM device.
//! * Do whatever else the device driver creator thought made most sense.
//!
//! `ioctl`s are synchronous system calls and are similar to read and
//! write calls in that regard.
//!
//! What does this module support?
//! ===============================
//!
//! This library provides the `ioctl!` macro, for binding `ioctl`s.
//! Here's a few examples of how that can work for SPI under Linux
//! from [rust-spidev](https://github.com/posborne/rust-spidev).
//!
//! ```
//! #[allow(non_camel_case_types)]
//! pub struct spi_ioc_transfer {
//!     pub tx_buf: u64,
//!     pub rx_buf: u64,
//!     pub len: u32,
//!
//!     // optional overrides
//!     pub speed_hz: u32,
//!     pub delay_usecs: u16,
//!     pub bits_per_word: u8,
//!     pub cs_change: u8,
//!     pub pad: u32,
//! }
//!
//! #[cfg(linux)]
//! mod ioctl {
//!     use super::*;
//!
//!     const SPI_IOC_MAGIC: u8 = 'k' as u8;
//!     const SPI_IOC_NR_TRANSFER: u8 = 0;
//!     const SPI_IOC_NR_MODE: u8 = 1;
//!     const SPI_IOC_NR_LSB_FIRST: u8 = 2;
//!     const SPI_IOC_NR_BITS_PER_WORD: u8 = 3;
//!     const SPI_IOC_NR_MAX_SPEED_HZ: u8 = 4;
//!     const SPI_IOC_NR_MODE32: u8 = 5;
//!
//!     ioctl!(read  get_mode_u8 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE; u8);
//!     ioctl!(read  get_mode_u32 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE; u32);
//!     ioctl!(write set_mode_u8 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE; u8);
//!     ioctl!(write set_mode_u32 with SPI_IOC_MAGIC, SPI_IOC_NR_MODE32; u32);
//!     ioctl!(read  get_lsb_first with SPI_IOC_MAGIC, SPI_IOC_NR_LSB_FIRST; u8);
//!     ioctl!(write set_lsb_first with SPI_IOC_MAGIC, SPI_IOC_NR_LSB_FIRST; u8);
//!     ioctl!(read  get_bits_per_word with SPI_IOC_MAGIC, SPI_IOC_NR_BITS_PER_WORD; u8);
//!     ioctl!(write set_bits_per_word with SPI_IOC_MAGIC, SPI_IOC_NR_BITS_PER_WORD; u8);
//!     ioctl!(read  get_max_speed_hz with SPI_IOC_MAGIC, SPI_IOC_NR_MAX_SPEED_HZ; u32);
//!     ioctl!(write set_max_speed_hz with SPI_IOC_MAGIC, SPI_IOC_NR_MAX_SPEED_HZ; u32);
//!     ioctl!(write spidev_transfer with SPI_IOC_MAGIC, SPI_IOC_NR_TRANSFER; spi_ioc_transfer);
//!     ioctl!(write buf spidev_transfer_buf with SPI_IOC_MAGIC, SPI_IOC_NR_TRANSFER; spi_ioc_transfer);
//! }
//!
//! // doctest workaround
//! fn main() {}
//! ```
//!
//! Spidev uses the `_IOC` macros that are encouraged (as far as
//! `ioctl` can be encouraged at all) for newer drivers.  Many
//! drivers, however, just use magic numbers with no attached
//! semantics.  For those, the `ioctl!(bad ...)` variant should be
//! used (the "bad" terminology is from the Linux kernel).
//!
//! How do I get the magic numbers?
//! ===============================
//!
//! For Linux, look at your system's headers. For example, `/usr/include/linux/input.h` has a lot
//! of lines defining macros which use `_IOR`, `_IOW`, `_IOC`, and `_IORW`.  These macros
//! correspond to the `ior!`, `iow!`, `ioc!`, and `iorw!` macros defined in this crate.
//! Additionally, there is the `ioctl!` macro for creating a wrapper around `ioctl` that is
//! somewhat more type-safe.
//!
//! Most `ioctl`s have no or little documentation. You'll need to scrounge through
//! the source to figure out what they do and how they should be used.
//!
#[cfg(any(target_os = "linux", target_os = "android"))]
#[path = "platform/linux.rs"]
#[macro_use]
mod platform;

#[cfg(any(target_os = "macos",
          target_os = "ios",
          target_os = "netbsd",
          target_os = "openbsd",
          target_os = "freebsd",
          target_os = "dragonfly"))]
#[path = "platform/bsd.rs"]
#[macro_use]
mod platform;

pub use self::platform::*;

/// Convert raw ioctl return value to a Nix result
#[macro_export]
#[doc(hidden)]
macro_rules! convert_ioctl_res {
    ($w:expr) => (
        {
            $crate::Errno::result($w)
        }
    );
}

/// Generates ioctl functions. See [::sys::ioctl](sys/ioctl/index.html).
#[macro_export]
macro_rules! ioctl {
    (bad $name:ident with $nr:expr) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            data: *mut u8)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, $nr as $crate::libc::c_ulong, data))
        }
        );
    (bad none $name:ident with $nr:expr) => (
        pub unsafe fn $name(fd: $crate::libc::c_int)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, $nr as $crate::libc::c_ulong))
        }
        );
    (none $name:ident with $ioty:expr, $nr:expr) => (
        pub unsafe fn $name(fd: $crate::libc::c_int)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, io!($ioty, $nr) as $crate::libc::c_ulong))
        }
        );
    (read $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            val: *mut $ty)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, ior!($ioty, $nr, ::std::mem::size_of::<$ty>()) as $crate::libc::c_ulong, val))
        }
        );
    (write $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            val: $ty)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, iow!($ioty, $nr, ::std::mem::size_of::<$ty>()) as $crate::libc::c_ulong, val))
        }
        );
    (readwrite $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            val: *mut $ty)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, iorw!($ioty, $nr, ::std::mem::size_of::<$ty>()) as $crate::libc::c_ulong, val))
        }
        );
    (read buf $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            val: *mut $ty,
                            len: usize)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, ior!($ioty, $nr, len) as $crate::libc::c_ulong, val))
        }
        );
    (write buf $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            val: *const $ty,
                            len: usize) -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, iow!($ioty, $nr, len) as $crate::libc::c_ulong, val))
        }
        );
    (readwrite buf $name:ident with $ioty:expr, $nr:expr; $ty:ty) => (
        pub unsafe fn $name(fd: $crate::libc::c_int,
                            val: *mut $ty,
                            len: usize)
                            -> $crate::Result<$crate::libc::c_int> {
            convert_ioctl_res!($crate::libc::ioctl(fd, iorw!($ioty, $nr, len) as $crate::libc::c_ulong, val))
        }
        );
}
<|endoftext|>


<commit_before>
use types::bit;
use types::hash::FNVHash;
use world::block;

use std::collections::HashMap;
use std::hash::BuildHasherDefault;

pub struct BlockStorage {
    blocks: bit::Map,
    block_map: Vec<(block::Block, u32)>,
    rev_block_map: HashMap<block::Block, usize, BuildHasherDefault<FNVHash>>,
}

impl BlockStorage {
    pub fn new(size: usize) -> BlockStorage {
        Self::new_default(size, block::Air{})
    }

    pub fn new_default(size: usize, def: block::Block) -> BlockStorage {
        let mut storage = BlockStorage {
            blocks: bit::Map::new(size, 4),
            block_map: vec![
                (def, size as u32)
            ],
            rev_block_map: HashMap::with_hasher(BuildHasherDefault::default()),
        };
        storage.rev_block_map.insert(def, 0);
        storage
    }

    pub fn get(&self, idx: usize) -> block::Block {
        let idx = self.blocks.get(idx);
        self.block_map.get(idx).map_or(block::Missing{}, |v| v.0)
    }

    pub fn set(&mut self, idx: usize, b: block::Block) -> bool {
        use std::collections::hash_map::Entry;
        let old = self.get(idx);
        if old == b {
            return false;
        }
        // Clean up old block
        {
            let idx = self.rev_block_map[&old];
            let info = &mut self.block_map[idx];
            info.1 -= 1;
            if info.1 == 0 { // None left of this type
                self.rev_block_map.remove(&old);
            }
        }

        if let Entry::Vacant(entry) = self.rev_block_map.entry(b) {
            let mut found = false;
            let id = entry.insert(self.block_map.len());
            for (i, ref mut info) in self.block_map.iter_mut().enumerate() {
                if info.1 == 0 {
                    info.0 = b;
                    *id = i;
                    found = true;
                    break;
                }
            }
            if !found {
                if self.block_map.len() >= 1 << self.blocks.bit_size {
                    let new_size = self.blocks.bit_size + 1;
                    self.blocks = self.blocks.resize(new_size);
                }
                self.block_map.push((b, 0));
            }
        }

        {
            let b_idx = self.rev_block_map[&b];
            let info = &mut self.block_map[b_idx];
            info.1 += 1;
            self.blocks.set(idx, b_idx);
        }
        true
    }

    // Quick chunk loading support helpers

    pub fn clear(&mut self) {
        self.block_map.clear();
        self.rev_block_map.clear();
    }

    pub fn force_mapping(&mut self, idx: usize, b: block::Block) {
        if self.block_map.len() < idx {
            self.block_map[idx] = (b, 0);
        } else if self.block_map.len() == idx {
            self.block_map.push((b, 0));
        } else {
            panic!("Out of bounds force mapping")
        }
        self.rev_block_map.insert(b, idx);
    }

    pub fn use_raw(&mut self, data: bit::Map) {
        self.blocks = data;
        // Recount blocks
        for bi in 0 .. 4096 {
            let bl_id = self.blocks.get(bi);
            if self.blocks.bit_size == 13 { // Global palette
                if self.block_map.get(bl_id)
                        .map(|v| v.1)
                        .unwrap_or(0) == 0 {
                    if bl_id >= self.block_map.len() {
                        self.block_map.resize(bl_id + 1, (block::Air{}, 0xFFFFF)); // Impossible to reach this value normally
                    }
                    let bl = block::Block::by_vanilla_id(bl_id as usize);
                    self.block_map[bl_id] = (bl, 0);
                    self.rev_block_map.insert(bl, bl_id);
                }
            }
            let bmap = self.block_map.get_mut(bl_id).unwrap();
            if bmap.1 == 0xFFFFF {
                bmap.1 = 0;
            }
            bmap.1 += 1;
        }
        self.gc_entries();
    }

    fn gc_entries(&mut self) {
        for entry in &mut self.block_map {
            if entry.1 == 0xFFFFF {
                println!("GC'd block");
                self.rev_block_map.remove(&entry.0);
                entry.1 = 0;
            }
        }
    }
}
<commit_msg>Fix a crash caused by duplicate block mappings<commit_after>
use types::bit;
use types::hash::FNVHash;
use world::block;

use std::collections::HashMap;
use std::hash::BuildHasherDefault;

pub struct BlockStorage {
    blocks: bit::Map,
    block_map: Vec<(block::Block, u32)>,
    rev_block_map: HashMap<block::Block, usize, BuildHasherDefault<FNVHash>>,
}

impl BlockStorage {
    pub fn new(size: usize) -> BlockStorage {
        Self::new_default(size, block::Air{})
    }

    pub fn new_default(size: usize, def: block::Block) -> BlockStorage {
        let mut storage = BlockStorage {
            blocks: bit::Map::new(size, 4),
            block_map: vec![
                (def, size as u32)
            ],
            rev_block_map: HashMap::with_hasher(BuildHasherDefault::default()),
        };
        storage.rev_block_map.insert(def, 0);
        storage
    }

    pub fn get(&self, idx: usize) -> block::Block {
        let idx = self.blocks.get(idx);
        self.block_map[idx].0
    }

    pub fn set(&mut self, idx: usize, b: block::Block) -> bool {
        use std::collections::hash_map::Entry;
        let old = self.get(idx);
        if old == b {
            return false;
        }
        // Clean up old block
        {
            let idx = self.rev_block_map[&old];
            let info = &mut self.block_map[idx];
            info.1 -= 1;
            if info.1 == 0 { // None left of this type
                self.rev_block_map.remove(&old);
            }
        }

        if let Entry::Vacant(entry) = self.rev_block_map.entry(b) {
            let mut found = false;
            let id = entry.insert(self.block_map.len());
            for (i, ref mut info) in self.block_map.iter_mut().enumerate() {
                if info.1 == 0 {
                    info.0 = b;
                    *id = i;
                    found = true;
                    break;
                }
            }
            if !found {
                if self.block_map.len() >= 1 << self.blocks.bit_size {
                    let new_size = self.blocks.bit_size + 1;
                    self.blocks = self.blocks.resize(new_size);
                }
                self.block_map.push((b, 0));
            }
        }

        {
            let b_idx = self.rev_block_map[&b];
            let info = &mut self.block_map[b_idx];
            info.1 += 1;
            self.blocks.set(idx, b_idx);
        }
        true
    }

    // Quick chunk loading support helpers

    pub fn clear(&mut self) {
        self.block_map.clear();
        self.rev_block_map.clear();
    }

    pub fn force_mapping(&mut self, idx: usize, b: block::Block) {
        if self.block_map.len() < idx {
            self.block_map[idx] = (b, 0);
        } else if self.block_map.len() == idx {
            self.block_map.push((b, 0));
        } else {
            panic!("Out of bounds force mapping")
        }
        self.rev_block_map.insert(b, idx);
    }

    pub fn use_raw(&mut self, data: bit::Map) {
        self.blocks = data;
        // Recount blocks
        for bi in 0 .. 4096 {
            let mut bl_id = self.blocks.get(bi);
            if self.blocks.bit_size == 13 { // Global palette
                if self.block_map.get(bl_id)
                        .map(|v| v.1)
                        .unwrap_or(0) == 0 {
                    if bl_id >= self.block_map.len() {
                        self.block_map.resize(bl_id + 1, (block::Air{}, 0xFFFFF)); // Impossible to reach this value normally
                    }
                    let bl = block::Block::by_vanilla_id(bl_id as usize);
                    self.block_map[bl_id] = (bl, 0);
                    self.rev_block_map.insert(bl, bl_id);
                }
            }
            {
                // Sometimes we can have two mappings for the same block
                // (either due to a server bug or unimplemented blocks
                // in steven). This attempts to repair this by replacing
                // duplicate mappings with the last one.
                let bmap = self.block_map.get_mut(bl_id).unwrap();
                let rev = *self.rev_block_map.get(&bmap.0).unwrap();
                if rev != bl_id {
                    bl_id = rev;
                    self.blocks.set(bi, bl_id);
                }
            }
            let mut bmap = self.block_map.get_mut(bl_id).unwrap();
            if bmap.1 == 0xFFFFF {
                bmap.1 = 0;
            }
            bmap.1 += 1;
        }
        self.gc_entries();
    }

    fn gc_entries(&mut self) {
        for entry in &mut self.block_map {
            if entry.1 == 0xFFFFF {
                println!("GC'd block");
                self.rev_block_map.remove(&entry.0);
                entry.1 = 0;
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;

use timely::PartialOrder;
use timely::dataflow::Scope;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::dataflow::operators::Operator;

use timely_sort::Unsigned;

use differential_dataflow::{ExchangeData, Collection, AsCollection, Hashable};
use differential_dataflow::difference::Monoid;
use differential_dataflow::lattice::Lattice;
use differential_dataflow::operators::arrange::Arranged;
use differential_dataflow::trace::{Cursor, TraceReader, BatchReader};

/// Proposes extensions to a stream of prefixes.
///
/// This method takes a stream of prefixes and for each determines a
/// key with `key_selector` and then proposes all pair af the prefix
/// and values associated with the key in `arrangement`.
pub fn lookup_map<G, D, R, Tr, F, DOut, ROut, S>(
    prefixes: &Collection<G, D, R>,
    arrangement: Arranged<G, Tr>,
    key_selector: F,
    output_func: S,
) -> Collection<G, DOut, ROut>
where
    G: Scope,
    G::Timestamp: Lattice,
    Tr: TraceReader<Time=G::Timestamp>+Clone+'static,
    Tr::Key: Ord+Hashable+Default,
    Tr::Val: Clone,
    Tr::Batch: BatchReader<Tr::Key, Tr::Val, Tr::Time, Tr::R>,
    Tr::Cursor: Cursor<Tr::Key, Tr::Val, Tr::Time, Tr::R>,
    Tr::R: Monoid+ExchangeData,
    F: Fn(&D, &mut Tr::Key)+Clone+'static,
    D: ExchangeData,
    R: ExchangeData+Monoid,
    DOut: Clone+'static,
    ROut: Monoid,
    S: Fn(&D, &R, &Tr::Val, &Tr::R)->(DOut, ROut)+'static,
{
    let propose_stream = arrangement.stream;
    let mut propose_trace = Some(arrangement.trace);

    let mut stash = HashMap::new();
    let logic1 = key_selector.clone();
    let logic2 = key_selector.clone();

    let mut buffer1 = Vec::new();
    let mut buffer2 = Vec::new();

    let mut key: Tr::Key = Default::default();
    let exchange = Exchange::new(move |update: &(D,G::Timestamp,R)| {
        logic1(&update.0, &mut key);
        key.hashed().as_u64()
    });

    let mut key1: Tr::Key = Default::default();
    let mut key2: Tr::Key = Default::default();
    prefixes.inner.binary_frontier(&propose_stream, exchange, Pipeline, "Propose", move |_,_| move |input1, input2, output| {

        // drain the first input, stashing requests.
        input1.for_each(|capability, data| {
            data.swap(&mut buffer1);
            stash.entry(capability.retain())
                 .or_insert(Vec::new())
                 .extend(buffer1.drain(..))
        });

        // advance the `distinguish_since` frontier to allow all merges.
        input2.for_each(|_, batches| {
            batches.swap(&mut buffer2);
            for batch in buffer2.drain(..) {
                if let Some(ref mut trace) = propose_trace {
                    trace.distinguish_since(batch.upper());
                }
            }
        });

        if let Some(ref mut trace) = propose_trace {

            for (capability, prefixes) in stash.iter_mut() {

                // defer requests at incomplete times.
                // NOTE: not all updates may be at complete times, but if this test fails then none of them are.
                if !input2.frontier.less_equal(capability.time()) {

                    let mut session = output.session(capability);

                    // sort requests for in-order cursor traversal. could consolidate?
                    prefixes.sort_by(|x,y| {
                        logic2(&x.0, &mut key1);
                        logic2(&y.0, &mut key2);
                        key1.cmp(&key2)
                    });

                    let (mut cursor, storage) = trace.cursor();

                    for &mut (ref prefix, ref time, ref mut diff) in prefixes.iter_mut() {
                        if !input2.frontier.less_equal(time) {
                            logic2(prefix, &mut key1);
                            // let key = logic2(prefix);
                            cursor.seek_key(&storage, &key1);
                            if cursor.get_key(&storage) == Some(&key1) {
                                while let Some(value) = cursor.get_val(&storage) {
                                    let mut count = Tr::R::zero();
                                    cursor.map_times(&storage, |t, d| if t.less_equal(time) { count += d; });
                                    if !count.is_zero() {
                                        let (dout, rout) = output_func(prefix, diff, value, &count);
                                        if !rout.is_zero() {
                                            session.give((dout, time.clone(), rout));
                                        }
                                    }
                                    cursor.step_val(&storage);
                                }
                                cursor.rewind_vals(&storage);
                            }
                            *diff = R::zero();
                        }
                    }

                    prefixes.retain(|ptd| !ptd.2.is_zero());
                }
            }
        }

        // drop fully processed capabilities.
        stash.retain(|_,prefixes| !prefixes.is_empty());

        // advance the consolidation frontier (TODO: wierd lexicographic times!)
        propose_trace.as_mut().map(|trace| trace.advance_by(&input1.frontier().frontier()));

        if input1.frontier().is_empty() && stash.is_empty() {
            propose_trace = None;
        }

    }).as_collection()
}<commit_msg>better name<commit_after>use std::collections::HashMap;

use timely::PartialOrder;
use timely::dataflow::Scope;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::dataflow::operators::Operator;

use timely_sort::Unsigned;

use differential_dataflow::{ExchangeData, Collection, AsCollection, Hashable};
use differential_dataflow::difference::Monoid;
use differential_dataflow::lattice::Lattice;
use differential_dataflow::operators::arrange::Arranged;
use differential_dataflow::trace::{Cursor, TraceReader, BatchReader};

/// Proposes extensions to a stream of prefixes.
///
/// This method takes a stream of prefixes and for each determines a
/// key with `key_selector` and then proposes all pair af the prefix
/// and values associated with the key in `arrangement`.
pub fn lookup_map<G, D, R, Tr, F, DOut, ROut, S>(
    prefixes: &Collection<G, D, R>,
    arrangement: Arranged<G, Tr>,
    key_selector: F,
    output_func: S,
) -> Collection<G, DOut, ROut>
where
    G: Scope,
    G::Timestamp: Lattice,
    Tr: TraceReader<Time=G::Timestamp>+Clone+'static,
    Tr::Key: Ord+Hashable+Default,
    Tr::Val: Clone,
    Tr::Batch: BatchReader<Tr::Key, Tr::Val, Tr::Time, Tr::R>,
    Tr::Cursor: Cursor<Tr::Key, Tr::Val, Tr::Time, Tr::R>,
    Tr::R: Monoid+ExchangeData,
    F: Fn(&D, &mut Tr::Key)+Clone+'static,
    D: ExchangeData,
    R: ExchangeData+Monoid,
    DOut: Clone+'static,
    ROut: Monoid,
    S: Fn(&D, &R, &Tr::Val, &Tr::R)->(DOut, ROut)+'static,
{
    let propose_stream = arrangement.stream;
    let mut propose_trace = Some(arrangement.trace);

    let mut stash = HashMap::new();
    let logic1 = key_selector.clone();
    let logic2 = key_selector.clone();

    let mut buffer1 = Vec::new();
    let mut buffer2 = Vec::new();

    let mut key: Tr::Key = Default::default();
    let exchange = Exchange::new(move |update: &(D,G::Timestamp,R)| {
        logic1(&update.0, &mut key);
        key.hashed().as_u64()
    });

    let mut key1: Tr::Key = Default::default();
    let mut key2: Tr::Key = Default::default();
    prefixes.inner.binary_frontier(&propose_stream, exchange, Pipeline, "LookupMap", move |_,_| move |input1, input2, output| {

        // drain the first input, stashing requests.
        input1.for_each(|capability, data| {
            data.swap(&mut buffer1);
            stash.entry(capability.retain())
                 .or_insert(Vec::new())
                 .extend(buffer1.drain(..))
        });

        // advance the `distinguish_since` frontier to allow all merges.
        input2.for_each(|_, batches| {
            batches.swap(&mut buffer2);
            for batch in buffer2.drain(..) {
                if let Some(ref mut trace) = propose_trace {
                    trace.distinguish_since(batch.upper());
                }
            }
        });

        if let Some(ref mut trace) = propose_trace {

            for (capability, prefixes) in stash.iter_mut() {

                // defer requests at incomplete times.
                // NOTE: not all updates may be at complete times, but if this test fails then none of them are.
                if !input2.frontier.less_equal(capability.time()) {

                    let mut session = output.session(capability);

                    // sort requests for in-order cursor traversal. could consolidate?
                    prefixes.sort_by(|x,y| {
                        logic2(&x.0, &mut key1);
                        logic2(&y.0, &mut key2);
                        key1.cmp(&key2)
                    });

                    let (mut cursor, storage) = trace.cursor();

                    for &mut (ref prefix, ref time, ref mut diff) in prefixes.iter_mut() {
                        if !input2.frontier.less_equal(time) {
                            logic2(prefix, &mut key1);
                            // let key = logic2(prefix);
                            cursor.seek_key(&storage, &key1);
                            if cursor.get_key(&storage) == Some(&key1) {
                                while let Some(value) = cursor.get_val(&storage) {
                                    let mut count = Tr::R::zero();
                                    cursor.map_times(&storage, |t, d| if t.less_equal(time) { count += d; });
                                    if !count.is_zero() {
                                        let (dout, rout) = output_func(prefix, diff, value, &count);
                                        if !rout.is_zero() {
                                            session.give((dout, time.clone(), rout));
                                        }
                                    }
                                    cursor.step_val(&storage);
                                }
                                cursor.rewind_vals(&storage);
                            }
                            *diff = R::zero();
                        }
                    }

                    prefixes.retain(|ptd| !ptd.2.is_zero());
                }
            }
        }

        // drop fully processed capabilities.
        stash.retain(|_,prefixes| !prefixes.is_empty());

        // advance the consolidation frontier (TODO: wierd lexicographic times!)
        propose_trace.as_mut().map(|trace| trace.advance_by(&input1.frontier().frontier()));

        if input1.frontier().is_empty() && stash.is_empty() {
            propose_trace = None;
        }

    }).as_collection()
}<|endoftext|>