<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

//! This module is responsible to build a single project. It does not handle
//! watch mode or other state.

mod apply_transforms;
mod artifact_content;
pub mod artifact_writer;
mod build_ir;
mod build_schema;
mod generate_artifacts;
pub mod generate_extra_artifacts;
mod persist_operations;
mod source_control;
mod validate;

use crate::compiler_state::{ArtifactMapKind, CompilerState, ProjectName, SourceSetName};
use crate::config::{Config, ProjectConfig};
use crate::errors::BuildProjectError;
use crate::{artifact_map::ArtifactMap, graphql_asts::GraphQLAsts};
pub use apply_transforms::apply_transforms;
pub use apply_transforms::Programs;
use build_ir::BuildIRResult;
pub use build_ir::SourceHashes;
pub use build_schema::build_schema;
use common::{PerfLogEvent, PerfLogger};
use fnv::{FnvHashMap, FnvHashSet};
pub use generate_artifacts::{
    create_path_for_artifact, generate_artifacts, Artifact, ArtifactContent,
};
use generate_extra_artifacts::generate_extra_artifacts;
use graphql_ir::Program;
use interner::StringKey;
use log::info;
use relay_codegen::Printer;
use schema::Schema;
pub use source_control::add_to_mercurial;
use std::{collections::hash_map::Entry, path::PathBuf, sync::Arc};
pub use validate::validate;

fn build_programs(
    config: &Config,
    project_config: &ProjectConfig,
    compiler_state: &CompilerState,
    graphql_asts: &FnvHashMap<SourceSetName, GraphQLAsts>,
    schema: Arc<Schema>,
    log_event: &impl PerfLogEvent,
    perf_logger: Arc<impl PerfLogger + 'static>,
) -> Result<(Programs, Arc<SourceHashes>), BuildProjectError> {
    let project_name = project_config.name;
    let is_incremental_build =
        compiler_state.has_processed_changes() && !compiler_state.has_breaking_schema_change();

    // Build a type aware IR.
    let BuildIRResult {
        ir,
        base_fragment_names,
        source_hashes,
    } = log_event.time("build_ir_time", || {
        build_ir::build_ir(project_config, &schema, graphql_asts, is_incremental_build)
            .map_err(|errors| BuildProjectError::ValidationErrors { errors })
    })?;

    // Turn the IR into a base Program.
    let program = log_event.time("build_program_time", || {
        Program::from_definitions(schema, ir)
    });

    // Call validation rules that go beyond type checking.
    log_event.time("validate_time", || {
        // TODO(T63482263): Pass connection interface from configuration
        validate(&program, &config.connection_interface)
            .map_err(|errors| BuildProjectError::ValidationErrors { errors })
    })?;

    // Apply various chains of transforms to create a set of output programs.
    let programs = log_event.time("apply_transforms_time", || {
        apply_transforms(
            project_name,
            Arc::new(program),
            Arc::new(base_fragment_names),
            &config.connection_interface,
            Arc::new(config.feature_flags),
            perf_logger,
        )
        .map_err(|errors| BuildProjectError::ValidationErrors { errors })
    })?;

    Ok((programs, Arc::new(source_hashes)))
}

pub fn check_project(
    config: &Config,
    project_config: &ProjectConfig,
    compiler_state: &CompilerState,
    graphql_asts: &FnvHashMap<SourceSetName, GraphQLAsts>,
    schema: Arc<Schema>,
    perf_logger: Arc<impl PerfLogger + 'static>,
) -> Result<Programs, BuildProjectError> {
    let log_event = perf_logger.create_event("check_project");
    let build_time = log_event.start("check_time");
    let project_name = project_config.name.lookup();
    log_event.string("project", project_name.to_string());

    let (programs, _) = build_programs(
        config,
        project_config,
        compiler_state,
        graphql_asts,
        schema,
        &log_event,
        Arc::clone(&perf_logger),
    )?;

    log_event.stop(build_time);
    perf_logger.complete_event(log_event);

    Ok(programs)
}

pub fn build_project(
    config: &Config,
    project_config: &ProjectConfig,
    compiler_state: &CompilerState,
    graphql_asts: &FnvHashMap<SourceSetName, GraphQLAsts>,
    perf_logger: Arc<impl PerfLogger + 'static>,
) -> Result<(ProjectName, Arc<Schema>, Programs, Vec<Artifact>), BuildProjectError> {
    let log_event = perf_logger.create_event("build_project");
    let build_time = log_event.start("build_project_time");
    let project_name = project_config.name.lookup();
    log_event.string("project", project_name.to_string());

    // Construct a schema instance including project specific extensions.
    let schema = log_event.time("build_schema_time", || {
        Arc::new(build_schema(compiler_state, project_config))
    });

    // Apply different transform pipelines to produce the `Programs`.
    let (programs, source_hashes) = build_programs(
        config,
        project_config,
        compiler_state,
        graphql_asts,
        Arc::clone(&schema),
        &log_event,
        Arc::clone(&perf_logger),
    )?;

    // Generate artifacts by collecting information from the `Programs`.
    let artifacts_timer = log_event.start("generate_artifacts_time");
    let artifacts = generate_artifacts(project_config, &programs, Arc::clone(&source_hashes))?;
    log_event.stop(artifacts_timer);

    log_event.number(
        "generated_artifacts",
        programs.reader.document_count() + programs.normalization.document_count(),
    );

    log_event.stop(build_time);
    perf_logger.complete_event(log_event);
    Ok((project_config.name, schema, programs, artifacts))
}

pub async fn commit_project(
    config: &Config,
    project_config: &ProjectConfig,
    perf_logger: Arc<impl PerfLogger + 'static>,
    schema: &Schema,
    programs: Programs,
    mut artifacts: Vec<Artifact>,
    artifact_map: Arc<ArtifactMapKind>,
    removed_definition_names: Vec<StringKey>,
    mut dirty_artifact_paths: FnvHashSet<PathBuf>,
) -> Result<ArtifactMap, BuildProjectError> {
    let log_event = perf_logger.create_event("commit_project");
    let commit_time = log_event.start("commit_project_time");

    if let Some(ref artifact_persister) = config.artifact_persister {
        if let Some(ref persist_config) = project_config.persist {
            let persist_operations_timer = log_event.start("persist_operations_time");
            persist_operations::persist_operations(
                &mut artifacts,
                &config.root_dir,
                &persist_config,
                &artifact_persister,
            )
            .await?;
            log_event.stop(persist_operations_timer);
        }
    }

    // In some cases we need to create additional (platform specific) artifacts
    // For that, we will use `generate_extra_operation_artifacts` from the configs
    if let Some(generate_extra_operation_artifacts_fn) = &config.generate_extra_operation_artifacts
    {
        log_event.time("generate_extra_operation_artifacts_time", || {
            generate_extra_artifacts(
                schema,
                project_config,
                &mut artifacts,
                generate_extra_operation_artifacts_fn,
            )
        });
    }

    // Write the generated artifacts to disk. This step is separate from
    // generating artifacts or persisting to avoid partial writes in case of
    // errors as much as possible.
    let next_artifact_map = match Arc::as_ref(&artifact_map) {
        ArtifactMapKind::Unconnected(existing_artifacts) => {
            let mut existing_artifacts = existing_artifacts.clone();
            let mut printer = Printer::default();

            log_event.time("write_artifacts_time", || {
                for artifact in &artifacts {
                    if !existing_artifacts.remove(&artifact.path) {
                        info!(
                            "[{}] NEW: {:?} -> {:?}",
                            project_config.name, &artifact.source_definition_names, &artifact.path
                        );
                    }

                    let path = config.root_dir.join(&artifact.path);
                    let content =
                        artifact
                            .content
                            .as_bytes(config, project_config, &mut printer, schema);
                    config.artifact_writer.write_if_changed(path, content)?;
                }
                Ok(())
            })?;

            log_event.time("delete_artifacts_time", || {
                for remaining_artifact in &existing_artifacts {
                    let path = config.root_dir.join(remaining_artifact);
                    config.artifact_writer.remove(path)?;
                }
                Ok(())
            })?;

            ArtifactMap::from(artifacts)
        }
        ArtifactMapKind::Mapping(artifact_map) => {
            let mut printer = Printer::default();
            let mut artifact_map = artifact_map.clone();

            // Delete all generated paths for removed definitions
            log_event.time("delete_artifacts_time", || {
                for name in &removed_definition_names {
                    if let Some(artifact) = artifact_map.0.remove(&name) {
                        for (path, _) in artifact {
                            dirty_artifact_paths.remove(&path);
                            let path = config.root_dir.join(path);
                            config.artifact_writer.remove(path)?;
                        }
                    }
                }
                Ok(())
            })?;

            log_event.time("write_artifacts_time", || {
                let mut current_paths_map = ArtifactMap::default();
                // Write or update artifacts
                for artifact in artifacts {
                    dirty_artifact_paths.remove(&artifact.path);
                    let path = config.root_dir.join(&artifact.path);
                    let content =
                        artifact
                            .content
                            .as_bytes(config, project_config, &mut printer, schema);
                    config.artifact_writer.write_if_changed(path, content)?;
                    current_paths_map.insert(artifact);
                }
                // Check previous artifact map and delete any removed artifacts
                for (definition_name, artifact_tuples) in current_paths_map.0 {
                    match artifact_map.0.entry(definition_name) {
                        Entry::Occupied(mut entry) => {
                            let prev_tuples = entry.get_mut();
                            for (prev_path, _) in prev_tuples.drain(..) {
                                if !artifact_tuples.iter().any(|t| t.0 == prev_path) {
                                    dirty_artifact_paths.remove(&prev_path);
                                    config
                                        .artifact_writer
                                        .remove(config.root_dir.join(prev_path))?;
                                }
                            }
                            prev_tuples.extend(artifact_tuples.into_iter());
                        }
                        Entry::Vacant(entry) => {
                            entry.insert(artifact_tuples);
                        }
                    }
                }
                Ok(())
            })?;

            // All artifacts are committed, the remaining dirty artifacts are no longer required
            for path in dirty_artifact_paths {
                config.artifact_writer.remove(config.root_dir.join(path))?;
            }

            artifact_map
        }
    };

    info!(
        "[{}] compiled documents: {} reader, {} normalization, {} operation text",
        project_config.name,
        programs.reader.document_count(),
        programs.normalization.document_count(),
        programs.operation_text.document_count()
    );
    log_event.stop(commit_time);
    perf_logger.complete_event(log_event);

    Ok(next_artifact_map)
}
<commit_msg>Avoid remove a file if it's going to be written in the incremental mode<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

//! This module is responsible to build a single project. It does not handle
//! watch mode or other state.

mod apply_transforms;
mod artifact_content;
pub mod artifact_writer;
mod build_ir;
mod build_schema;
mod generate_artifacts;
pub mod generate_extra_artifacts;
mod persist_operations;
mod source_control;
mod validate;

use crate::compiler_state::{ArtifactMapKind, CompilerState, ProjectName, SourceSetName};
use crate::config::{Config, ProjectConfig};
use crate::errors::BuildProjectError;
use crate::{artifact_map::ArtifactMap, graphql_asts::GraphQLAsts};
pub use apply_transforms::apply_transforms;
pub use apply_transforms::Programs;
use build_ir::BuildIRResult;
pub use build_ir::SourceHashes;
pub use build_schema::build_schema;
use common::{PerfLogEvent, PerfLogger};
use fnv::{FnvHashMap, FnvHashSet};
pub use generate_artifacts::{
    create_path_for_artifact, generate_artifacts, Artifact, ArtifactContent,
};
use generate_extra_artifacts::generate_extra_artifacts;
use graphql_ir::Program;
use interner::StringKey;
use log::info;
use relay_codegen::Printer;
use schema::Schema;
pub use source_control::add_to_mercurial;
use std::{collections::hash_map::Entry, path::PathBuf, sync::Arc};
pub use validate::validate;

fn build_programs(
    config: &Config,
    project_config: &ProjectConfig,
    compiler_state: &CompilerState,
    graphql_asts: &FnvHashMap<SourceSetName, GraphQLAsts>,
    schema: Arc<Schema>,
    log_event: &impl PerfLogEvent,
    perf_logger: Arc<impl PerfLogger + 'static>,
) -> Result<(Programs, Arc<SourceHashes>), BuildProjectError> {
    let project_name = project_config.name;
    let is_incremental_build =
        compiler_state.has_processed_changes() && !compiler_state.has_breaking_schema_change();

    // Build a type aware IR.
    let BuildIRResult {
        ir,
        base_fragment_names,
        source_hashes,
    } = log_event.time("build_ir_time", || {
        build_ir::build_ir(project_config, &schema, graphql_asts, is_incremental_build)
            .map_err(|errors| BuildProjectError::ValidationErrors { errors })
    })?;

    // Turn the IR into a base Program.
    let program = log_event.time("build_program_time", || {
        Program::from_definitions(schema, ir)
    });

    // Call validation rules that go beyond type checking.
    log_event.time("validate_time", || {
        // TODO(T63482263): Pass connection interface from configuration
        validate(&program, &config.connection_interface)
            .map_err(|errors| BuildProjectError::ValidationErrors { errors })
    })?;

    // Apply various chains of transforms to create a set of output programs.
    let programs = log_event.time("apply_transforms_time", || {
        apply_transforms(
            project_name,
            Arc::new(program),
            Arc::new(base_fragment_names),
            &config.connection_interface,
            Arc::new(config.feature_flags),
            perf_logger,
        )
        .map_err(|errors| BuildProjectError::ValidationErrors { errors })
    })?;

    Ok((programs, Arc::new(source_hashes)))
}

pub fn check_project(
    config: &Config,
    project_config: &ProjectConfig,
    compiler_state: &CompilerState,
    graphql_asts: &FnvHashMap<SourceSetName, GraphQLAsts>,
    schema: Arc<Schema>,
    perf_logger: Arc<impl PerfLogger + 'static>,
) -> Result<Programs, BuildProjectError> {
    let log_event = perf_logger.create_event("check_project");
    let build_time = log_event.start("check_time");
    let project_name = project_config.name.lookup();
    log_event.string("project", project_name.to_string());

    let (programs, _) = build_programs(
        config,
        project_config,
        compiler_state,
        graphql_asts,
        schema,
        &log_event,
        Arc::clone(&perf_logger),
    )?;

    log_event.stop(build_time);
    perf_logger.complete_event(log_event);

    Ok(programs)
}

pub fn build_project(
    config: &Config,
    project_config: &ProjectConfig,
    compiler_state: &CompilerState,
    graphql_asts: &FnvHashMap<SourceSetName, GraphQLAsts>,
    perf_logger: Arc<impl PerfLogger + 'static>,
) -> Result<(ProjectName, Arc<Schema>, Programs, Vec<Artifact>), BuildProjectError> {
    let log_event = perf_logger.create_event("build_project");
    let build_time = log_event.start("build_project_time");
    let project_name = project_config.name.lookup();
    log_event.string("project", project_name.to_string());

    // Construct a schema instance including project specific extensions.
    let schema = log_event.time("build_schema_time", || {
        Arc::new(build_schema(compiler_state, project_config))
    });

    // Apply different transform pipelines to produce the `Programs`.
    let (programs, source_hashes) = build_programs(
        config,
        project_config,
        compiler_state,
        graphql_asts,
        Arc::clone(&schema),
        &log_event,
        Arc::clone(&perf_logger),
    )?;

    // Generate artifacts by collecting information from the `Programs`.
    let artifacts_timer = log_event.start("generate_artifacts_time");
    let artifacts = generate_artifacts(project_config, &programs, Arc::clone(&source_hashes))?;
    log_event.stop(artifacts_timer);

    log_event.number(
        "generated_artifacts",
        programs.reader.document_count() + programs.normalization.document_count(),
    );

    log_event.stop(build_time);
    perf_logger.complete_event(log_event);
    Ok((project_config.name, schema, programs, artifacts))
}

pub async fn commit_project(
    config: &Config,
    project_config: &ProjectConfig,
    perf_logger: Arc<impl PerfLogger + 'static>,
    schema: &Schema,
    programs: Programs,
    mut artifacts: Vec<Artifact>,
    artifact_map: Arc<ArtifactMapKind>,
    // Definitions that are removed from the previous artifact map
    removed_definition_names: Vec<StringKey>,
    // Dirty artrifacts that should be removed
    mut artifacts_to_remove: FnvHashSet<PathBuf>,
) -> Result<ArtifactMap, BuildProjectError> {
    let log_event = perf_logger.create_event("commit_project");
    let commit_time = log_event.start("commit_project_time");

    if let Some(ref artifact_persister) = config.artifact_persister {
        if let Some(ref persist_config) = project_config.persist {
            let persist_operations_timer = log_event.start("persist_operations_time");
            persist_operations::persist_operations(
                &mut artifacts,
                &config.root_dir,
                &persist_config,
                &artifact_persister,
            )
            .await?;
            log_event.stop(persist_operations_timer);
        }
    }

    // In some cases we need to create additional (platform specific) artifacts
    // For that, we will use `generate_extra_operation_artifacts` from the configs
    if let Some(generate_extra_operation_artifacts_fn) = &config.generate_extra_operation_artifacts
    {
        log_event.time("generate_extra_operation_artifacts_time", || {
            generate_extra_artifacts(
                schema,
                project_config,
                &mut artifacts,
                generate_extra_operation_artifacts_fn,
            )
        });
    }

    // Write the generated artifacts to disk. This step is separate from
    // generating artifacts or persisting to avoid partial writes in case of
    // errors as much as possible.
    let next_artifact_map = match Arc::as_ref(&artifact_map) {
        ArtifactMapKind::Unconnected(existing_artifacts) => {
            let mut existing_artifacts = existing_artifacts.clone();
            let mut printer = Printer::default();

            log_event.time("write_artifacts_time", || {
                for artifact in &artifacts {
                    if !existing_artifacts.remove(&artifact.path) {
                        info!(
                            "[{}] NEW: {:?} -> {:?}",
                            project_config.name, &artifact.source_definition_names, &artifact.path
                        );
                    }

                    let path = config.root_dir.join(&artifact.path);
                    let content =
                        artifact
                            .content
                            .as_bytes(config, project_config, &mut printer, schema);
                    config.artifact_writer.write_if_changed(path, content)?;
                }
                Ok(())
            })?;

            log_event.time("delete_artifacts_time", || {
                for remaining_artifact in &existing_artifacts {
                    let path = config.root_dir.join(remaining_artifact);
                    config.artifact_writer.remove(path)?;
                }
                Ok(())
            })?;

            ArtifactMap::from(artifacts)
        }
        ArtifactMapKind::Mapping(artifact_map) => {
            let mut printer = Printer::default();
            let mut artifact_map = artifact_map.clone();

            // All generated paths for removed definitions should be removed
            log_event.time("delete_artifacts_time", || {
                for name in &removed_definition_names {
                    if let Some(artifact) = artifact_map.0.remove(&name) {
                        for (path, _) in artifact {
                            artifacts_to_remove.insert(path);
                        }
                    }
                }
                Ok(())
            })?;

            log_event.time("write_artifacts_time", || {
                let mut current_paths_map = ArtifactMap::default();
                // Write or update artifacts
                for artifact in artifacts {
                    // If the path is written, stop removing it
                    artifacts_to_remove.remove(&artifact.path);
                    let path = config.root_dir.join(&artifact.path);
                    let content =
                        artifact
                            .content
                            .as_bytes(config, project_config, &mut printer, schema);
                    config.artifact_writer.write_if_changed(path, content)?;
                    current_paths_map.insert(artifact);
                }
                // Check previous artifact map and delete any removed artifacts
                for (definition_name, artifact_tuples) in current_paths_map.0 {
                    match artifact_map.0.entry(definition_name) {
                        Entry::Occupied(mut entry) => {
                            let prev_tuples = entry.get_mut();
                            for (prev_path, _) in prev_tuples.drain(..) {
                                if !artifact_tuples.iter().any(|t| t.0 == prev_path) {
                                    artifacts_to_remove.insert(prev_path);
                                }
                            }
                            prev_tuples.extend(artifact_tuples.into_iter());
                        }
                        Entry::Vacant(entry) => {
                            entry.insert(artifact_tuples);
                        }
                    }
                }
                Ok(())
            })?;

            // All artifacts are committed, the remaining dirty artifacts are no longer required
            for path in artifacts_to_remove {
                config.artifact_writer.remove(config.root_dir.join(path))?;
            }

            artifact_map
        }
    };

    info!(
        "[{}] compiled documents: {} reader, {} normalization, {} operation text",
        project_config.name,
        programs.reader.document_count(),
        programs.normalization.document_count(),
        programs.operation_text.document_count()
    );
    log_event.stop(commit_time);
    perf_logger.complete_event(log_event);

    Ok(next_artifact_map)
}
<|endoftext|>


<commit_before>//! The data that we will serialize and deserialize.

use super::query::DepGraphQuery;
use super::{DepKind, DepNode, DepNodeIndex};
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::{AtomicU32, Lock, Lrc, Ordering};
use rustc_index::vec::IndexVec;
use rustc_serialize::opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize};
use rustc_serialize::{Decodable, Decoder, Encodable};
use smallvec::SmallVec;
use std::convert::TryInto;

#[cfg(parallel_compiler)]
use {
    rustc_data_structures::sync::WorkerLocal, rustc_index::vec::Idx, std::sync::mpsc, std::thread,
};

// The maximum value of `SerializedDepNodeIndex` leaves the upper two bits
// unused so that we can store multiple index types in `CompressedHybridIndex`,
// and use those bits to encode which index type it contains.
rustc_index::newtype_index! {
    pub struct SerializedDepNodeIndex {
        MAX = 0x7FFF_FFFF
    }
}

/// Data for use when recompiling the **current crate**.
#[derive(Debug)]
pub struct SerializedDepGraph<K: DepKind> {
    /// The set of all DepNodes in the graph
    pub nodes: IndexVec<SerializedDepNodeIndex, DepNode<K>>,
    /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to
    /// the DepNode at the same index in the nodes vector.
    pub fingerprints: IndexVec<SerializedDepNodeIndex, Fingerprint>,
    /// For each DepNode, stores the list of edges originating from that
    /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
    /// which holds the actual DepNodeIndices of the target nodes.
    pub edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>,
    /// A flattened list of all edge targets in the graph. Edge sources are
    /// implicit in edge_list_indices.
    pub edge_list_data: Vec<SerializedDepNodeIndex>,
}

impl<K: DepKind> Default for SerializedDepGraph<K> {
    fn default() -> Self {
        SerializedDepGraph {
            nodes: Default::default(),
            fingerprints: Default::default(),
            edge_list_indices: Default::default(),
            edge_list_data: Default::default(),
        }
    }
}

impl<K: DepKind> SerializedDepGraph<K> {
    #[inline]
    pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] {
        let targets = self.edge_list_indices[source];
        &self.edge_list_data[targets.0 as usize..targets.1 as usize]
    }
}

impl<'a, K: DepKind + Decodable<opaque::Decoder<'a>>> Decodable<opaque::Decoder<'a>>
    for SerializedDepGraph<K>
{
    #[instrument(skip(d))]
    fn decode(d: &mut opaque::Decoder<'a>) -> Result<SerializedDepGraph<K>, String> {
        let position = d.position();

        // The last 16 bytes are the node count and edge count.
        debug!("position: {:?}", d.position());
        d.set_position(d.data.len() - 2 * IntEncodedWithFixedSize::ENCODED_SIZE);
        debug!("position: {:?}", d.position());

        let node_count = IntEncodedWithFixedSize::decode(d)?.0 as usize;
        let edge_count = IntEncodedWithFixedSize::decode(d)?.0 as usize;
        debug!(?node_count, ?edge_count);

        debug!("position: {:?}", d.position());
        d.set_position(position);
        debug!("position: {:?}", d.position());

        let mut nodes = IndexVec::with_capacity(node_count);
        let mut fingerprints = IndexVec::with_capacity(node_count);
        let mut edge_list_indices = IndexVec::with_capacity(node_count);
        let mut edge_list_data = Vec::with_capacity(edge_count);

        for _index in 0..node_count {
            d.read_struct("NodeInfo", 3, |d| {
                let dep_node: DepNode<K> = d.read_struct_field("node", 0, Decodable::decode)?;
                let _i: SerializedDepNodeIndex = nodes.push(dep_node);
                debug_assert_eq!(_i.index(), _index);

                let fingerprint: Fingerprint =
                    d.read_struct_field("fingerprint", 1, Decodable::decode)?;
                let _i: SerializedDepNodeIndex = fingerprints.push(fingerprint);
                debug_assert_eq!(_i.index(), _index);

                d.read_struct_field("edges", 2, |d| {
                    d.read_seq(|d, len| {
                        let start = edge_list_data.len().try_into().unwrap();
                        for e in 0..len {
                            let edge = d.read_seq_elt(e, Decodable::decode)?;
                            edge_list_data.push(edge);
                        }
                        let end = edge_list_data.len().try_into().unwrap();
                        let _i: SerializedDepNodeIndex = edge_list_indices.push((start, end));
                        debug_assert_eq!(_i.index(), _index);
                        Ok(())
                    })
                })
            })?;
        }

        Ok(SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data })
    }
}

#[derive(Debug, Encodable, Decodable)]
pub struct NodeInfo<K: DepKind> {
    node: DepNode<K>,
    fingerprint: Fingerprint,
    edges: SmallVec<[DepNodeIndex; 8]>,
}

struct Stat<K: DepKind> {
    kind: K,
    node_counter: u64,
    edge_counter: u64,
}

struct Stats<K: DepKind> {
    stats: FxHashMap<K, Stat<K>>,
    total_node_count: usize,
    total_edge_count: usize,
}

#[instrument(skip(encoder, _record_graph, record_stats))]
fn encode_node<K: DepKind>(
    encoder: &mut FileEncoder,
    _index: DepNodeIndex,
    node: &NodeInfo<K>,
    _record_graph: &Option<Lrc<Lock<DepGraphQuery<K>>>>,
    record_stats: &Option<Lrc<Lock<Stats<K>>>>,
) -> FileEncodeResult {
    #[cfg(debug_assertions)]
    if let Some(record_graph) = &_record_graph {
        if let Some(record_graph) = &mut if cfg!(parallel_compiler) {
            Some(record_graph.lock())
        } else {
            // Do not ICE when a query is called from within `with_query`.
            record_graph.try_lock()
        } {
            record_graph.push(_index, node.node, &node.edges);
        }
    }

    if let Some(record_stats) = &record_stats {
        let mut stats = record_stats.lock();
        let kind = node.node.kind;
        let edge_count = node.edges.len();

        let stat =
            stats.stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
        stat.node_counter += 1;
        stat.edge_counter += edge_count as u64;
        stats.total_node_count += 1;
        stats.total_edge_count += edge_count;
    }

    debug!(?_index, ?node);
    node.encode(encoder)
}

fn encode_counts(
    mut encoder: FileEncoder,
    node_count: usize,
    edge_count: usize,
) -> FileEncodeResult {
    let node_count = node_count.try_into().unwrap();
    let edge_count = edge_count.try_into().unwrap();

    debug!(?node_count, ?edge_count);
    debug!("position: {:?}", encoder.position());
    IntEncodedWithFixedSize(node_count).encode(&mut encoder)?;
    IntEncodedWithFixedSize(edge_count).encode(&mut encoder)?;
    debug!("position: {:?}", encoder.position());
    // Drop the encoder so that nothing is written after the counts.
    encoder.flush()
}

#[cfg(not(parallel_compiler))]
pub struct GraphEncoder<K: DepKind> {
    status: Lock<(FileEncoder, usize, FileEncodeResult)>,
    counter: AtomicU32,
    record_graph: Option<Lrc<Lock<DepGraphQuery<K>>>>,
    record_stats: Option<Lrc<Lock<Stats<K>>>>,
}

#[cfg(parallel_compiler)]
pub struct GraphEncoder<K: DepKind> {
    send: WorkerLocal<mpsc::Sender<(DepNodeIndex, NodeInfo<K>)>>,
    thread: thread::JoinHandle<FileEncodeResult>,
    counter: AtomicU32,
    record_graph: Option<Lrc<Lock<DepGraphQuery<K>>>>,
    record_stats: Option<Lrc<Lock<Stats<K>>>>,
}

impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
    pub fn new(
        encoder: FileEncoder,
        prev_node_count: usize,
        record_graph: bool,
        record_stats: bool,
    ) -> Self {
        let record_graph = if cfg!(debug_assertions) && record_graph {
            Some(Lrc::new(Lock::new(DepGraphQuery::new(prev_node_count))))
        } else {
            None
        };
        let record_stats = if record_stats {
            Some(Lrc::new(Lock::new(Stats {
                stats: FxHashMap::default(),
                total_node_count: 0,
                total_edge_count: 0,
            })))
        } else {
            None
        };
        let counter = AtomicU32::new(0);

        #[cfg(not(parallel_compiler))]
        {
            let status = Lock::new((encoder, 0, Ok(())));
            GraphEncoder { status, counter, record_graph, record_stats }
        }
        #[cfg(parallel_compiler)]
        {
            let (send, recv) = mpsc::channel();
            let thread = {
                let record_graph = record_graph.clone();
                let record_stats = record_stats.clone();
                thread::spawn(move || {
                    encode_graph(encoder, recv, |encoder, index, node| {
                        encode_node(encoder, index, node, &record_graph, &record_stats)
                    })
                })
            };
            let send = WorkerLocal::new(move |_| send.clone());

            GraphEncoder { send, thread, counter, record_graph, record_stats }
        }
    }

    pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery<K>)) {
        if let Some(record_graph) = &self.record_graph {
            f(&record_graph.lock())
        }
    }

    pub(crate) fn print_incremental_info(
        &self,
        total_read_count: u64,
        total_duplicate_read_count: u64,
    ) {
        if let Some(record_stats) = &self.record_stats {
            let record_stats = record_stats.lock();

            let mut stats: Vec<_> = record_stats.stats.values().collect();
            stats.sort_by_key(|s| -(s.node_counter as i64));

            const SEPARATOR: &str = "[incremental] --------------------------------\
                                     ----------------------------------------------\
                                     ------------";

            eprintln!("[incremental]");
            eprintln!("[incremental] DepGraph Statistics");
            eprintln!("{}", SEPARATOR);
            eprintln!("[incremental]");
            eprintln!("[incremental] Total Node Count: {}", record_stats.total_node_count);
            eprintln!("[incremental] Total Edge Count: {}", record_stats.total_edge_count);

            if cfg!(debug_assertions) {
                eprintln!("[incremental] Total Edge Reads: {}", total_read_count);
                eprintln!(
                    "[incremental] Total Duplicate Edge Reads: {}",
                    total_duplicate_read_count
                );
            }

            eprintln!("[incremental]");
            eprintln!(
                "[incremental]  {:<36}| {:<17}| {:<12}| {:<17}|",
                "Node Kind", "Node Frequency", "Node Count", "Avg. Edge Count"
            );
            eprintln!("{}", SEPARATOR);

            for stat in stats {
                let node_kind_ratio =
                    (100.0 * (stat.node_counter as f64)) / (record_stats.total_node_count as f64);
                let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64);

                eprintln!(
                    "[incremental]  {:<36}|{:>16.1}% |{:>12} |{:>17.1} |",
                    format!("{:?}", stat.kind),
                    node_kind_ratio,
                    stat.node_counter,
                    node_kind_avg_edges,
                );
            }

            eprintln!("{}", SEPARATOR);
            eprintln!("[incremental]");
        }
    }
}

#[cfg(not(parallel_compiler))]
impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
    pub(crate) fn send(
        &self,
        node: DepNode<K>,
        fingerprint: Fingerprint,
        edges: SmallVec<[DepNodeIndex; 8]>,
    ) -> DepNodeIndex {
        let index = self.counter.fetch_add(1, Ordering::SeqCst);
        let index = DepNodeIndex::from_u32(index);
        let &mut (ref mut encoder, ref mut edge_count, ref mut result) = &mut *self.status.lock();
        *edge_count += edges.len();
        *result = std::mem::replace(result, Ok(())).and_then(|()| {
            let node = NodeInfo { node, fingerprint, edges };
            encode_node(encoder, index, &node, &self.record_graph, &self.record_stats)
        });
        index
    }

    pub fn finish(self) -> FileEncodeResult {
        let (encoder, edge_count, result) = self.status.into_inner();
        let () = result?;
        let node_count = self.counter.into_inner() as usize;

        encode_counts(encoder, node_count, edge_count)
    }
}

#[cfg(parallel_compiler)]
impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
    pub(crate) fn send(
        &self,
        node: DepNode<K>,
        fingerprint: Fingerprint,
        edges: SmallVec<[DepNodeIndex; 8]>,
    ) -> DepNodeIndex {
        let node = NodeInfo { node, fingerprint, edges };
        let index = self.counter.fetch_add(1, Ordering::SeqCst);
        let index = DepNodeIndex::from_u32(index);
        self.send.send((index, node)).unwrap();
        index
    }

    pub fn finish(self) -> FileEncodeResult {
        std::mem::drop(self.send);
        self.thread.join().unwrap()
    }
}

#[cfg(parallel_compiler)]
#[instrument(skip(encoder, recv, process))]
fn encode_graph<K: DepKind + Encodable<FileEncoder>>(
    mut encoder: FileEncoder,
    recv: mpsc::Receiver<(DepNodeIndex, NodeInfo<K>)>,
    process: impl Fn(&mut FileEncoder, DepNodeIndex, &NodeInfo<K>) -> FileEncodeResult,
) -> FileEncodeResult {
    let mut edge_count: usize = 0;
    let node_count: usize = ordered_recv(recv, |index, node| {
        edge_count += node.edges.len();
        process(&mut encoder, index, node)
    })?;

    encode_counts(encoder, node_count, edge_count)
}

/// Since there are multiple producers assigning the DepNodeIndex using an atomic,
/// the messages may not arrive in order. This function sorts them as they come.
#[cfg(parallel_compiler)]
fn ordered_recv<K: DepKind + Encodable<opaque::FileEncoder>>(
    recv: mpsc::Receiver<(DepNodeIndex, NodeInfo<K>)>,
    mut f: impl FnMut(DepNodeIndex, &NodeInfo<K>) -> FileEncodeResult,
) -> Result<usize, std::io::Error> {
    let mut pending = Vec::<(DepNodeIndex, _)>::new();
    let mut expected = DepNodeIndex::new(0);

    // INVARIANT: No message can arrive with an index less than `expected`.
    'outer: loop {
        pending.sort_by_key(|n| n.0);
        for (index, node) in pending.drain_filter(|(index, _)| {
            if *index == expected {
                expected.increment_by(1);
                true
            } else {
                false
            }
        }) {
            f(index, &node)?;
        }

        while let Ok((index, node)) = recv.recv() {
            if index > expected {
                pending.push((index, node));
            } else if index == expected {
                f(index, &node)?;
                expected.increment_by(1);
                continue 'outer;
            } else {
                panic!("Unexpected index {:?} while waiting for {:?}", index, expected);
            }
        }

        break;
    }

    Ok(expected.as_u32() as usize)
}
<commit_msg>Remove the parallel version.<commit_after>//! The data that we will serialize and deserialize.

use super::query::DepGraphQuery;
use super::{DepKind, DepNode, DepNodeIndex};
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::{Lock, Lrc};
use rustc_index::vec::{Idx, IndexVec};
use rustc_serialize::opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize};
use rustc_serialize::{Decodable, Decoder, Encodable};
use smallvec::SmallVec;
use std::convert::TryInto;

// The maximum value of `SerializedDepNodeIndex` leaves the upper two bits
// unused so that we can store multiple index types in `CompressedHybridIndex`,
// and use those bits to encode which index type it contains.
rustc_index::newtype_index! {
    pub struct SerializedDepNodeIndex {
        MAX = 0x7FFF_FFFF
    }
}

/// Data for use when recompiling the **current crate**.
#[derive(Debug)]
pub struct SerializedDepGraph<K: DepKind> {
    /// The set of all DepNodes in the graph
    pub nodes: IndexVec<SerializedDepNodeIndex, DepNode<K>>,
    /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to
    /// the DepNode at the same index in the nodes vector.
    pub fingerprints: IndexVec<SerializedDepNodeIndex, Fingerprint>,
    /// For each DepNode, stores the list of edges originating from that
    /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
    /// which holds the actual DepNodeIndices of the target nodes.
    pub edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>,
    /// A flattened list of all edge targets in the graph. Edge sources are
    /// implicit in edge_list_indices.
    pub edge_list_data: Vec<SerializedDepNodeIndex>,
}

impl<K: DepKind> Default for SerializedDepGraph<K> {
    fn default() -> Self {
        SerializedDepGraph {
            nodes: Default::default(),
            fingerprints: Default::default(),
            edge_list_indices: Default::default(),
            edge_list_data: Default::default(),
        }
    }
}

impl<K: DepKind> SerializedDepGraph<K> {
    #[inline]
    pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] {
        let targets = self.edge_list_indices[source];
        &self.edge_list_data[targets.0 as usize..targets.1 as usize]
    }
}

impl<'a, K: DepKind + Decodable<opaque::Decoder<'a>>> Decodable<opaque::Decoder<'a>>
    for SerializedDepGraph<K>
{
    #[instrument(skip(d))]
    fn decode(d: &mut opaque::Decoder<'a>) -> Result<SerializedDepGraph<K>, String> {
        let position = d.position();

        // The last 16 bytes are the node count and edge count.
        debug!("position: {:?}", d.position());
        d.set_position(d.data.len() - 2 * IntEncodedWithFixedSize::ENCODED_SIZE);
        debug!("position: {:?}", d.position());

        let node_count = IntEncodedWithFixedSize::decode(d)?.0 as usize;
        let edge_count = IntEncodedWithFixedSize::decode(d)?.0 as usize;
        debug!(?node_count, ?edge_count);

        debug!("position: {:?}", d.position());
        d.set_position(position);
        debug!("position: {:?}", d.position());

        let mut nodes = IndexVec::with_capacity(node_count);
        let mut fingerprints = IndexVec::with_capacity(node_count);
        let mut edge_list_indices = IndexVec::with_capacity(node_count);
        let mut edge_list_data = Vec::with_capacity(edge_count);

        for _index in 0..node_count {
            d.read_struct("NodeInfo", 3, |d| {
                let dep_node: DepNode<K> = d.read_struct_field("node", 0, Decodable::decode)?;
                let _i: SerializedDepNodeIndex = nodes.push(dep_node);
                debug_assert_eq!(_i.index(), _index);

                let fingerprint: Fingerprint =
                    d.read_struct_field("fingerprint", 1, Decodable::decode)?;
                let _i: SerializedDepNodeIndex = fingerprints.push(fingerprint);
                debug_assert_eq!(_i.index(), _index);

                d.read_struct_field("edges", 2, |d| {
                    d.read_seq(|d, len| {
                        let start = edge_list_data.len().try_into().unwrap();
                        for e in 0..len {
                            let edge = d.read_seq_elt(e, Decodable::decode)?;
                            edge_list_data.push(edge);
                        }
                        let end = edge_list_data.len().try_into().unwrap();
                        let _i: SerializedDepNodeIndex = edge_list_indices.push((start, end));
                        debug_assert_eq!(_i.index(), _index);
                        Ok(())
                    })
                })
            })?;
        }

        Ok(SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data })
    }
}

#[derive(Debug, Encodable, Decodable)]
pub struct NodeInfo<K: DepKind> {
    node: DepNode<K>,
    fingerprint: Fingerprint,
    edges: SmallVec<[DepNodeIndex; 8]>,
}

struct Stat<K: DepKind> {
    kind: K,
    node_counter: u64,
    edge_counter: u64,
}

struct Stats<K: DepKind> {
    stats: FxHashMap<K, Stat<K>>,
    total_node_count: usize,
    total_edge_count: usize,
}

#[instrument(skip(encoder, _record_graph, record_stats))]
fn encode_node<K: DepKind>(
    encoder: &mut FileEncoder,
    _index: DepNodeIndex,
    node: &NodeInfo<K>,
    _record_graph: &Option<Lrc<Lock<DepGraphQuery<K>>>>,
    record_stats: &Option<Lrc<Lock<Stats<K>>>>,
) -> FileEncodeResult {
    #[cfg(debug_assertions)]
    if let Some(record_graph) = &_record_graph {
        // Do not ICE when a query is called from within `with_query`.
        if let Some(record_graph) = &mut record_graph.try_lock() {
            record_graph.push(_index, node.node, &node.edges);
        }
    }

    if let Some(record_stats) = &record_stats {
        let mut stats = record_stats.lock();
        let kind = node.node.kind;
        let edge_count = node.edges.len();

        let stat =
            stats.stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
        stat.node_counter += 1;
        stat.edge_counter += edge_count as u64;
        stats.total_node_count += 1;
        stats.total_edge_count += edge_count;
    }

    debug!(?_index, ?node);
    node.encode(encoder)
}

fn encode_counts(
    mut encoder: FileEncoder,
    node_count: usize,
    edge_count: usize,
) -> FileEncodeResult {
    let node_count = node_count.try_into().unwrap();
    let edge_count = edge_count.try_into().unwrap();

    debug!(?node_count, ?edge_count);
    debug!("position: {:?}", encoder.position());
    IntEncodedWithFixedSize(node_count).encode(&mut encoder)?;
    IntEncodedWithFixedSize(edge_count).encode(&mut encoder)?;
    debug!("position: {:?}", encoder.position());
    // Drop the encoder so that nothing is written after the counts.
    encoder.flush()
}

pub struct GraphEncoder<K: DepKind> {
    status: Lock<(FileEncoder, DepNodeIndex, usize, FileEncodeResult)>,
    record_graph: Option<Lrc<Lock<DepGraphQuery<K>>>>,
    record_stats: Option<Lrc<Lock<Stats<K>>>>,
}

impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
    pub fn new(
        encoder: FileEncoder,
        prev_node_count: usize,
        record_graph: bool,
        record_stats: bool,
    ) -> Self {
        let record_graph = if cfg!(debug_assertions) && record_graph {
            Some(Lrc::new(Lock::new(DepGraphQuery::new(prev_node_count))))
        } else {
            None
        };
        let record_stats = if record_stats {
            Some(Lrc::new(Lock::new(Stats {
                stats: FxHashMap::default(),
                total_node_count: 0,
                total_edge_count: 0,
            })))
        } else {
            None
        };
        let status = Lock::new((encoder, DepNodeIndex::new(0), 0, Ok(())));
        GraphEncoder { status, record_graph, record_stats }
    }

    pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery<K>)) {
        if let Some(record_graph) = &self.record_graph {
            f(&record_graph.lock())
        }
    }

    pub(crate) fn print_incremental_info(
        &self,
        total_read_count: u64,
        total_duplicate_read_count: u64,
    ) {
        if let Some(record_stats) = &self.record_stats {
            let record_stats = record_stats.lock();

            let mut stats: Vec<_> = record_stats.stats.values().collect();
            stats.sort_by_key(|s| -(s.node_counter as i64));

            const SEPARATOR: &str = "[incremental] --------------------------------\
                                     ----------------------------------------------\
                                     ------------";

            eprintln!("[incremental]");
            eprintln!("[incremental] DepGraph Statistics");
            eprintln!("{}", SEPARATOR);
            eprintln!("[incremental]");
            eprintln!("[incremental] Total Node Count: {}", record_stats.total_node_count);
            eprintln!("[incremental] Total Edge Count: {}", record_stats.total_edge_count);

            if cfg!(debug_assertions) {
                eprintln!("[incremental] Total Edge Reads: {}", total_read_count);
                eprintln!(
                    "[incremental] Total Duplicate Edge Reads: {}",
                    total_duplicate_read_count
                );
            }

            eprintln!("[incremental]");
            eprintln!(
                "[incremental]  {:<36}| {:<17}| {:<12}| {:<17}|",
                "Node Kind", "Node Frequency", "Node Count", "Avg. Edge Count"
            );
            eprintln!("{}", SEPARATOR);

            for stat in stats {
                let node_kind_ratio =
                    (100.0 * (stat.node_counter as f64)) / (record_stats.total_node_count as f64);
                let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64);

                eprintln!(
                    "[incremental]  {:<36}|{:>16.1}% |{:>12} |{:>17.1} |",
                    format!("{:?}", stat.kind),
                    node_kind_ratio,
                    stat.node_counter,
                    node_kind_avg_edges,
                );
            }

            eprintln!("{}", SEPARATOR);
            eprintln!("[incremental]");
        }
    }

    pub(crate) fn send(
        &self,
        node: DepNode<K>,
        fingerprint: Fingerprint,
        edges: SmallVec<[DepNodeIndex; 8]>,
    ) -> DepNodeIndex {
        let &mut (ref mut encoder, ref mut next_index, ref mut edge_count, ref mut result) =
            &mut *self.status.lock();
        let index = next_index.clone();
        next_index.increment_by(1);
        *edge_count += edges.len();
        *result = std::mem::replace(result, Ok(())).and_then(|()| {
            let node = NodeInfo { node, fingerprint, edges };
            encode_node(encoder, index, &node, &self.record_graph, &self.record_stats)
        });
        index
    }

    pub fn finish(self) -> FileEncodeResult {
        let (encoder, node_count, edge_count, result) = self.status.into_inner();
        let () = result?;
        let node_count = node_count.index();

        encode_counts(encoder, node_count, edge_count)
    }
}
<|endoftext|>


<commit_before>///
/// Text Scanner
///

use alloc::heap;
use std::marker::PhantomData;
use std::mem;
use std::ptr::copy_nonoverlapping;
use std::result::Result;
use std::slice;

use common::constant::ROWBLOCK_SIZE;
use common::err::*;
use exec::Executor;
use io::stream::*;
use schema::Schema;
use rows::RowBlock;
use rows::vrows::BorrowedVRowBlock;
use types::*;
use util::str::{StrSlice,split_str_slice};

const BUF_SIZE: usize = 65536;

pub struct DelimTextScanner<'a> {
  data_schema : Schema,
  read_fields : Option<Schema>,
  line_delim  : u8,
  field_delim : u8,
  reader      : Box<StreamReader>,
  marker      : PhantomData<&'a ()>,

  // variable
  readbuf_ptr          : *mut u8,
  line_slices_idx      : usize,
  read_line_num        : usize,    // number of read lines for each next
  line_slices_ptr      : *mut StrSlice,
  line_slices          : &'a mut [StrSlice],
  fields_slices_ptr    : *mut StrSlice,
  fields_slices        : &'a mut [StrSlice],
  parsed_batch_len     : usize,
  read_len             : usize,
}

impl<'a> DelimTextScanner<'a> {
  pub fn new(
        data_schema: Schema,
        read_fields: Option<Schema>,
        stream: Box<StreamReader>,
        field_delim: u8) -> DelimTextScanner<'a> {

    let mut line_slices_ptr = unsafe {
      heap::allocate(mem::size_of::<StrSlice>() * 1024, mem::min_align_of::<StrSlice>())
    } as *mut StrSlice;

    let mut line_slices: &mut [StrSlice] = unsafe {
      slice::from_raw_parts_mut(line_slices_ptr as *mut StrSlice, 1024)
    };

    let mut fields_slices_ptr = unsafe {
      heap::allocate(
        mem::size_of::<StrSlice>() * data_schema.size(),
        mem::min_align_of::<StrSlice>()
      )
    } as *mut StrSlice;

    let mut fields_slices: &mut [StrSlice] = unsafe {
      slice::from_raw_parts_mut(fields_slices_ptr as *mut StrSlice, data_schema.size())
    };

    DelimTextScanner {
      data_schema: data_schema,
      read_fields: read_fields,

      line_delim: '\n' as u8,
      field_delim: field_delim,

      reader: stream,
      marker: PhantomData,

      readbuf_ptr: unsafe { heap::allocate(BUF_SIZE, 16) },
      line_slices_idx: 0,
      read_line_num: 0,
      line_slices_ptr: line_slices_ptr,
      line_slices: line_slices,
      fields_slices_ptr: fields_slices_ptr,
      fields_slices: fields_slices,
      parsed_batch_len: 0,
      read_len: 0
    }
  }

  fn read_line_num(&self) -> usize {
    self.read_line_num
  }

  fn line_slices(&self) -> &[StrSlice] {
    self.line_slices
  }

  fn find_first_record_index(&self, bytes: &[u8]) -> Option<usize> {
    //let bytes : &[u8] = unsafe { mem::transmute(text) };

    let mut pos : usize = 0;
    let mut found : bool = false;

    for c in bytes {
      if self.line_delim == *c {
        found = true;
        break;
      }

      pos = pos + 1;
    }

    match found {
      true => Some(pos + 1),
      false => None
    }
  }

  /// Return the last index of fields, which will be used for the following call.
  /// It will return a tuple consisting of found line number and last delim index.
  fn fill_line_slices(&mut self) {

    self.read_line_num      = 0; // read line counter
    let mut last_pos: usize = 0; // keep the start offset
    let mut cur_pos : usize = 0; // the current offset

    while (cur_pos < self.read_len && self.read_line_num < ROWBLOCK_SIZE) {
      // for each character
      let c: u8  = unsafe { *self.readbuf_ptr.offset(cur_pos as isize) };

      // check if the character is line delimiter
      if c == self.line_delim {

        // if found, set each StrSlice with the start offset and the current position
        self.line_slices[self.read_line_num].set_ptr(unsafe {self.readbuf_ptr.offset(last_pos as isize)});
        self.line_slices[self.read_line_num].set_len((cur_pos - last_pos) as i32);

        last_pos = cur_pos + 1; // to skip the delimiter character
        self.read_line_num = self.read_line_num + 1; // increase the line number
      }

      cur_pos = cur_pos + 1;
    }

    self.parsed_batch_len = last_pos;
    self.line_slices_idx = 0;
  }

  fn add_row(&mut self, row_idx: usize, split_num: usize) {
  }

  /// move the remain bytes into the header of buffer and fill the buffer.
  fn fill_readbuf(&mut self) -> Void {
    let remain_len = self.read_len - self.parsed_batch_len;
    debug_assert!(remain_len >= 0, "remain length must be positive number.");

    // check if buffer should be compact.
    // If true, move the remain bytes into the head of the readbuf.
    if remain_len > 0 {
      unsafe {
        copy_nonoverlapping(
          self.readbuf_ptr.offset(self.parsed_batch_len as isize),
          self.readbuf_ptr,
          remain_len);
      }
    }

    // fill readbuf
    self.read_len = try!(
      self.reader.read(
        unsafe {
          slice::from_raw_parts_mut(
            self.readbuf_ptr.offset(remain_len as isize),
            BUF_SIZE - remain_len
          )
        },
      )
    );

    self.read_len = self.read_len + remain_len;

    void_ok()
  }
}

impl<'a> Executor for DelimTextScanner<'a> {

  fn init(&mut self) -> Void {
    void_ok()
  }

  fn next(&mut self, rowblock: &mut RowBlock) -> TResult<bool> {

    // check if all data are consumed
    if (try!(self.reader.pos()) >= try!(self.reader.len())) {
      rowblock.set_row_num(0);
      return Ok(false); // return false to stop iteration
    }

    let mut row_num: usize = 0;
    let mut field_parse_res: (usize, usize);
    let mut need_more_rows = true;
    loop { // this loop continues until rowblock is filled or

      if self.line_slices_idx >= self.read_line_num { // need more line slices
        // compact and fill read buffer
        self.fill_readbuf();
        if self.read_len < 1 { // there is no readable buffer
          break;
        }
        self.fill_line_slices();
      }

      unsafe {
        let mut cur_line_idx = self.line_slices_idx;
        while cur_line_idx < self.read_line_num && need_more_rows {

          // split each line slice into field slices
          field_parse_res = split_str_slice(
            &mut self.line_slices[cur_line_idx],
            self.fields_slices,
            self.field_delim
          );

          cur_line_idx = cur_line_idx + 1;
          row_num = row_num + 1;
          need_more_rows = row_num < ROWBLOCK_SIZE;
        }

        // record parsed line index
        self.line_slices_idx = cur_line_idx;
      }

      if !need_more_rows {
        break;
      }

    } // outmost loop

    println!("pos: {}, file_len: {}", try!(self.reader.pos()), try!(self.reader.len()));
    rowblock.set_row_num(row_num);
    Ok(row_num > 0)
  }

  fn close(&mut self) -> Void {
    void_ok()
  }

  fn in_schema(&self) -> &Schema {
    &self.data_schema
  }

  fn out_schema(&self) -> &Schema {
    match self.read_fields {
      Some(ref s) => s,
      None => &self.data_schema
    }
  }
}

#[test]
fn test_find_first_record_index() {
  let mut s = Schema::new();
  s.add_column("c1", *TEXT_TY);
  s.add_column("c2", *TEXT_TY);

  let fin = Box::new(FileInputStream::new("/home/hyunsik/tpch/lineitem/lineitem.tbl".to_string()));
  let s = DelimTextScanner::new(s, None, fin, '\n' as u8);

  assert_eq!(4, s.find_first_record_index("abc\nbb".as_bytes()).unwrap());
  assert_eq!(1, s.find_first_record_index("\nabc\nbb".as_bytes()).unwrap());
  assert_eq!(2, s.find_first_record_index("\r\nabc\nbb".as_bytes()).unwrap());
  assert!(s.find_first_record_index("aaaaabcabbb".as_bytes()).is_none());

}

#[test]
fn test_str_array() {
  let a: [&str;1024];

  //let mut slices: Vec<&str> = Vec::with_capacity(12);
  //unsafe { slices.set_len(12) };
  //slices.get_unchecked_mut(x)
  //slices[1] = "abc";

  let mut ptr: *mut &str = unsafe {
    heap::allocate(
      mem::size_of::<&str>() * 1024,
      mem::min_align_of::<&str>()
    ) as *mut &str
  };

  let mut slices: &mut [&str] = unsafe {
    slice::from_raw_parts_mut(ptr as *mut &str, 1024)
  };

  slices[0] = "abc";

  //slices[0] = "abc";
}

#[test]
fn test_read_line_batch() {
  let mut schema = Schema::new();
  schema.add_column("c1", *TEXT_TY);
  schema.add_column("c2", *TEXT_TY);

  let mut rowblock = BorrowedVRowBlock::new(&schema);

  let mut fin = Box::new(FileInputStream::new("/Users/hyunsik/tpch/lineitem/lineitem.tbl".to_string()));
  assert!(fin.open().is_ok());
  let mut s = DelimTextScanner::new(schema, None, fin, '\n' as u8);
  assert!(s.init().is_ok());

  let mut sum = 0;
  while(s.next(&mut rowblock).unwrap()) {
    sum = sum + rowblock.row_num();
    println!("acc: {}, rows: {}", sum, rowblock.row_num());
  }

  println!("{}", sum);
}
  // let mut delim_indexes:Vec<usize> = Vec::new();
  // let r1 =
  //   s.next_line_indexes("abc\nbb\nabcdef\nabcd".as_ptr(), &mut delim_indexes);
  // assert_eq!(3, r1.0);
  // assert_eq!(13, r1.1);

//   delim_indexes.clear();

//   let r2 =
//     s.next_line_indexes("a\nb\nabcde\n".as_ptr(), &mut delim_indexes);
//   assert_eq!(3, r2.0);
//   assert_eq!(9, r2.1);
// }<commit_msg>Removed print statements.<commit_after>///
/// Delimited Text Scanner for comma/tab separated value formats
///

use alloc::heap;
use std::marker::PhantomData;
use std::mem;
use std::ptr::copy_nonoverlapping;
use std::result::Result;
use std::slice;

use common::constant::ROWBLOCK_SIZE;
use common::err::*;
use exec::Executor;
use io::stream::*;
use schema::Schema;
use rows::RowBlock;
use rows::vrows::BorrowedVRowBlock;
use types::*;
use util::str::{StrSlice,split_str_slice};

/// default buffer size
const BUF_SIZE: usize = 65536;

pub struct DelimTextScanner<'a> {
  // constants
  data_schema : Schema,
  read_fields : Option<Schema>,
  line_delim  : u8,
  field_delim : u8,
  reader      : Box<StreamReader>,
  marker      : PhantomData<&'a ()>,

  // variable
  readbuf_ptr      : *mut u8,           // readbuf
  read_len         : usize,             // length of read bytes
  consumed_len     : usize,             // actually consumed bytes length for line slices

  line_slices_idx  : usize,             // current line slice for reading
  line_slices_num  : usize,             // number of filled line slices
  line_slices_ptr  : *mut StrSlice,     // ptr for line slice array
  line_slices      : &'a mut [StrSlice],// line slice array

  fields_slices_ptr: *mut StrSlice,     // ptr for field slice array
  fields_slices    : &'a mut [StrSlice] // field slice array
}

impl<'a> DelimTextScanner<'a> {
  pub fn new(
        data_schema: Schema,
        read_fields: Option<Schema>,
        stream: Box<StreamReader>,
        field_delim: u8) -> DelimTextScanner<'a> {

    let mut line_slices_ptr = unsafe {
      heap::allocate(mem::size_of::<StrSlice>() * 1024, mem::min_align_of::<StrSlice>())
    } as *mut StrSlice;

    let mut line_slices: &mut [StrSlice] = unsafe {
      slice::from_raw_parts_mut(line_slices_ptr as *mut StrSlice, 1024)
    };

    let mut fields_slices_ptr = unsafe {
      heap::allocate(
        mem::size_of::<StrSlice>() * data_schema.size(),
        mem::min_align_of::<StrSlice>()
      )
    } as *mut StrSlice;

    let mut fields_slices: &mut [StrSlice] = unsafe {
      slice::from_raw_parts_mut(fields_slices_ptr as *mut StrSlice, data_schema.size())
    };

    DelimTextScanner {
      data_schema: data_schema,
      read_fields: read_fields,

      line_delim: '\n' as u8,
      field_delim: field_delim,

      reader: stream,
      marker: PhantomData,

      readbuf_ptr: unsafe { heap::allocate(BUF_SIZE, 16) },
      consumed_len: 0,
      read_len: 0,

      line_slices_idx: 0,
      line_slices_num: 0,
      line_slices_ptr: line_slices_ptr,
      line_slices: line_slices,

      fields_slices_ptr: fields_slices_ptr,
      fields_slices: fields_slices
    }
  }

  fn line_slices_num(&self) -> usize {
    self.line_slices_num
  }

  fn line_slices(&self) -> &[StrSlice] {
    self.line_slices
  }

  fn find_first_record_index(&self, bytes: &[u8]) -> Option<usize> {
    let mut pos : usize = 0;
    let mut found : bool = false;

    for c in bytes {
      if self.line_delim == *c {
        found = true;
        break;
      }

      pos = pos + 1;
    }

    match found {
      true => Some(pos + 1),
      false => None
    }
  }

  /// Return the last index of fields, which will be used for the following call.
  /// It will return a tuple consisting of found line number and last delim index.
  fn fill_line_slices(&mut self) {

    let mut line_num: usize = 0;
    let mut last_pos: usize = 0; // keep the start offset
    let mut cur_pos : usize = 0; // the current offset

    while (cur_pos < self.read_len && line_num < ROWBLOCK_SIZE) {
      // for each character
      let c: u8  = unsafe { *self.readbuf_ptr.offset(cur_pos as isize) };

      // check if the character is line delimiter
      if c == self.line_delim {

        // if found, set each StrSlice with the start offset and the current position
        self.line_slices[line_num].set_ptr(unsafe {self.readbuf_ptr.offset(last_pos as isize)});
        self.line_slices[line_num].set_len((cur_pos - last_pos) as i32);

        last_pos = cur_pos + 1; // to skip the delimiter character
        line_num = line_num + 1; // increase the line number
      }

      cur_pos = cur_pos + 1;
    }

    self.line_slices_num  = line_num; // read line counter
    self.consumed_len = last_pos;
    self.line_slices_idx = 0;
  }

  fn add_row(&mut self, row_idx: usize, split_num: usize) {
  }

  /// move the remain bytes into the header of buffer and fill the buffer.
  fn fill_readbuf(&mut self) -> Void {
    let remain_len = self.read_len - self.consumed_len;
    debug_assert!(remain_len >= 0, "remain length must be positive number.");

    // check if buffer should be compact.
    // If true, move the remain bytes into the head of the readbuf.
    if remain_len > 0 {
      unsafe {
        copy_nonoverlapping(
          self.readbuf_ptr.offset(self.consumed_len as isize),
          self.readbuf_ptr,
          remain_len);
      }
    }

    // fill readbuf
    self.read_len = try!(
      self.reader.read(
        unsafe {
          slice::from_raw_parts_mut(
            self.readbuf_ptr.offset(remain_len as isize),
            BUF_SIZE - remain_len
          )
        },
      )
    );

    self.read_len = self.read_len + remain_len;

    void_ok()
  }
}

impl<'a> Executor for DelimTextScanner<'a> {

  fn init(&mut self) -> Void {
    void_ok()
  }

  fn next(&mut self, rowblock: &mut RowBlock) -> TResult<bool> {

    // check if all data are consumed
    if (try!(self.reader.pos()) >= try!(self.reader.len())) {
      rowblock.set_row_num(0);
      return Ok(false); // return false to stop iteration
    }

    let mut row_num: usize = 0; // number of rows which are set to rowblock.
    let mut need_more_rows = true; // indicate whether rowblock is not full or not.
    let mut parse_result: (usize, usize);

    loop { // this loop continues until rowblock is filled or EOS

      if self.line_slices_idx >= self.line_slices_num { // need more line slices
        // compact and fill read buffer
        self.fill_readbuf();
        if self.read_len < 1 { // there is no readable buffer
          break;
        }
        self.fill_line_slices();
      }

      unsafe {
        let mut cur_line_idx = self.line_slices_idx;

        while cur_line_idx < self.line_slices_num && need_more_rows {

          // split each line slice into field slices
          parse_result = split_str_slice(
            &mut self.line_slices[cur_line_idx],
            self.fields_slices,
            self.field_delim
          );

          // the second value of parse result tuple is number of fields
          self.add_row(row_num, parse_result.1);

          cur_line_idx   = cur_line_idx + 1;
          row_num        = row_num + 1;
          need_more_rows = row_num < ROWBLOCK_SIZE;
        }

        // record parsed line index
        self.line_slices_idx = cur_line_idx;
      }

      if !need_more_rows {
        break;
      }
    } // outmost loop

    rowblock.set_row_num(row_num);
    Ok(row_num > 0)
  }

  fn close(&mut self) -> Void {
    void_ok()
  }

  fn in_schema(&self) -> &Schema {
    &self.data_schema
  }

  fn out_schema(&self) -> &Schema {
    match self.read_fields {
      Some(ref s) => s,
      None => &self.data_schema
    }
  }
}

#[test]
fn test_find_first_record_index() {
  let mut s = Schema::new();
  s.add_column("c1", *TEXT_TY);
  s.add_column("c2", *TEXT_TY);

  let fin = Box::new(FileInputStream::new("/home/hyunsik/tpch/lineitem/lineitem.tbl".to_string()));
  let s = DelimTextScanner::new(s, None, fin, '\n' as u8);

  assert_eq!(4, s.find_first_record_index("abc\nbb".as_bytes()).unwrap());
  assert_eq!(1, s.find_first_record_index("\nabc\nbb".as_bytes()).unwrap());
  assert_eq!(2, s.find_first_record_index("\r\nabc\nbb".as_bytes()).unwrap());
  assert!(s.find_first_record_index("aaaaabcabbb".as_bytes()).is_none());

}

#[test]
fn test_str_array() {
  let a: [&str;1024];

  //let mut slices: Vec<&str> = Vec::with_capacity(12);
  //unsafe { slices.set_len(12) };
  //slices.get_unchecked_mut(x)
  //slices[1] = "abc";

  let mut ptr: *mut &str = unsafe {
    heap::allocate(
      mem::size_of::<&str>() * 1024,
      mem::min_align_of::<&str>()
    ) as *mut &str
  };

  let mut slices: &mut [&str] = unsafe {
    slice::from_raw_parts_mut(ptr as *mut &str, 1024)
  };

  slices[0] = "abc";
}

#[test]
fn test_read_line_batch() {
  let mut schema = Schema::new();
  schema.add_column("c1", *TEXT_TY);
  schema.add_column("c2", *TEXT_TY);

  let mut rowblock = BorrowedVRowBlock::new(&schema);

  let mut fin = Box::new(FileInputStream::new("/Users/hyunsik/tpch/lineitem/lineitem.tbl".to_string()));
  assert!(fin.open().is_ok());
  let mut s = DelimTextScanner::new(schema, None, fin, '\n' as u8);
  assert!(s.init().is_ok());

  let mut sum = 0;
  while(s.next(&mut rowblock).unwrap()) {
    sum = sum + rowblock.row_num();
    //println!("acc: {}, rows: {}", sum, rowblock.row_num());
  }

  assert_eq!(6001216, sum);
}
  // let mut delim_indexes:Vec<usize> = Vec::new();
  // let r1 =
  //   s.next_line_indexes("abc\nbb\nabcdef\nabcd".as_ptr(), &mut delim_indexes);
  // assert_eq!(3, r1.0);
  // assert_eq!(13, r1.1);

//   delim_indexes.clear();

//   let r2 =
//     s.next_line_indexes("a\nb\nabcde\n".as_ptr(), &mut delim_indexes);
//   assert_eq!(3, r2.0);
//   assert_eq!(9, r2.1);
// }<|endoftext|>


<commit_before>// A port of the simplistic benchmark from
//
//    http://github.com/PaulKeeble/ScalaVErlangAgents
//
// I *think* it's the same, more or less.

extern mod std;
extern mod extra;
extern mod native;
extern mod zmq;

use std::comm;
use std::os;

fn server(mut pull_socket: zmq::Socket, mut push_socket: zmq::Socket, mut workers: uint) {
    let mut count = 0u;
    let mut msg = zmq::Message::new();

    while workers != 0 {
        match pull_socket.recv(&mut msg, 0) {
            Err(e) => fail!(e.to_str()),
            Ok(()) => {
                msg.with_str(|s| {
                    if s == "" {
                        workers -= 1;
                    } else {
                        count += from_str::<uint>(s).unwrap();
                    }
                })
            }
        }
    }

    match push_socket.send_str(count.to_str(), 0) {
        Ok(()) => { }
        Err(e) => fail!(e.to_str()),
    }
}

fn spawn_server(ctx: &mut zmq::Context, workers: uint) -> comm::Chan<()> {
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    pull_socket.bind("inproc://server-pull").unwrap();
    push_socket.bind("inproc://server-push").unwrap();

    // Spawn the server.
    let (ready_po, ready_ch) = comm::Chan::new();
    let (start_po, start_ch) = comm::Chan::new();

    // Mutable sockets cannot be implicitly captured.
    let pull_socket = pull_socket;
    let push_socket = push_socket;

    do native::task::spawn {
        // Let the main thread know we're ready.
        ready_ch.send(());

        // Wait until we need to start.
        start_po.recv();

        server(pull_socket, push_socket, workers);
    }

    // Wait for the server to start.
    ready_po.recv();

    start_ch
}

fn worker(mut push_socket: zmq::Socket, count: uint) {
    for _ in range(0, count) {
        push_socket.send_str(100u.to_str(), 0).unwrap();
    }

    // Let the server know we're done.
    push_socket.send_str("", 0).unwrap();
}

fn spawn_worker(ctx: &mut zmq::Context, count: uint) -> comm::Port<()> {
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();

    // Mutable sockets cannot be implicitly captured.
    let push_socket = push_socket;

    // Spawn the worker.
    let (po, ch) = comm::Chan::new();
    do native::task::spawn {
        // Let the main thread we're ready.
        ch.send(());

        worker(push_socket, count);

        ch.send(());
    }

    // Wait for the worker to start.
    po.recv();

    po
}

fn run(ctx: &mut zmq::Context, size: uint, workers: uint) {
    let start_ch = spawn_server(ctx, workers);

    // Create some command/control sockets.
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    pull_socket.connect("inproc://server-push").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();
    //pull_socket.connect("tcp://127.0.0.1:3457").unwrap();

    // Spawn all the workers.
    let mut worker_results = ~[];
    for _ in range(0, workers) {
        worker_results.push(spawn_worker(ctx, size / workers));
    }

    let start = extra::time::precise_time_s();

    start_ch.send(());

    // Block until all the workers finish.
    for po in worker_results.iter() {
        po.recv();
    }

    // Receive the final count.
    let result = match pull_socket.recv_msg(0) {
        Ok(msg) => msg.with_str(|s| from_str::<uint>(s).unwrap()),
        Err(e) => fail!(e.to_str()),
    };

    let end = extra::time::precise_time_s();
    let elapsed = end - start;

    println!("Count is {}", result);
    println!("Test took {} seconds", elapsed);
    let thruput = ((size / workers * workers) as f64) / (elapsed as f64);
    println!("Throughput={:f} per sec", thruput);
}

fn main() {
    let args = os::args();

    let args = if os::getenv("RUST_BENCH").is_some() {
        ~[~"", ~"1000000", ~"10000"]
    } else if args.len() <= 1u {
        ~[~"", ~"10000", ~"4"]
    } else {
        args
    };

    let size = from_str::<uint>(args[1]).unwrap();
    let workers = from_str::<uint>(args[2]).unwrap();

    let mut ctx = zmq::Context::new();

    run(&mut ctx, size, workers);
}
<commit_msg>"do" is gone<commit_after>// A port of the simplistic benchmark from
//
//    http://github.com/PaulKeeble/ScalaVErlangAgents
//
// I *think* it's the same, more or less.

extern mod std;
extern mod extra;
extern mod native;
extern mod zmq;

use std::comm;
use std::os;

fn server(mut pull_socket: zmq::Socket, mut push_socket: zmq::Socket, mut workers: uint) {
    let mut count = 0u;
    let mut msg = zmq::Message::new();

    while workers != 0 {
        match pull_socket.recv(&mut msg, 0) {
            Err(e) => fail!(e.to_str()),
            Ok(()) => {
                msg.with_str(|s| {
                    if s == "" {
                        workers -= 1;
                    } else {
                        count += from_str::<uint>(s).unwrap();
                    }
                })
            }
        }
    }

    match push_socket.send_str(count.to_str(), 0) {
        Ok(()) => { }
        Err(e) => fail!(e.to_str()),
    }
}

fn spawn_server(ctx: &mut zmq::Context, workers: uint) -> comm::Chan<()> {
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    pull_socket.bind("inproc://server-pull").unwrap();
    push_socket.bind("inproc://server-push").unwrap();

    // Spawn the server.
    let (ready_po, ready_ch) = comm::Chan::new();
    let (start_po, start_ch) = comm::Chan::new();

    // Mutable sockets cannot be implicitly captured.
    let pull_socket = pull_socket;
    let push_socket = push_socket;

    native::task::spawn(proc() {
        // Let the main thread know we're ready.
        ready_ch.send(());

        // Wait until we need to start.
        start_po.recv();

        server(pull_socket, push_socket, workers);
    });

    // Wait for the server to start.
    ready_po.recv();

    start_ch
}

fn worker(mut push_socket: zmq::Socket, count: uint) {
    for _ in range(0, count) {
        push_socket.send_str(100u.to_str(), 0).unwrap();
    }

    // Let the server know we're done.
    push_socket.send_str("", 0).unwrap();
}

fn spawn_worker(ctx: &mut zmq::Context, count: uint) -> comm::Port<()> {
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();

    // Mutable sockets cannot be implicitly captured.
    let push_socket = push_socket;

    // Spawn the worker.
    let (po, ch) = comm::Chan::new();
    native::task::spawn(proc() {
        // Let the main thread we're ready.
        ch.send(());

        worker(push_socket, count);

        ch.send(());
    });

    // Wait for the worker to start.
    po.recv();

    po
}

fn run(ctx: &mut zmq::Context, size: uint, workers: uint) {
    let start_ch = spawn_server(ctx, workers);

    // Create some command/control sockets.
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    pull_socket.connect("inproc://server-push").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();
    //pull_socket.connect("tcp://127.0.0.1:3457").unwrap();

    // Spawn all the workers.
    let mut worker_results = ~[];
    for _ in range(0, workers) {
        worker_results.push(spawn_worker(ctx, size / workers));
    }

    let start = extra::time::precise_time_s();

    start_ch.send(());

    // Block until all the workers finish.
    for po in worker_results.iter() {
        po.recv();
    }

    // Receive the final count.
    let result = match pull_socket.recv_msg(0) {
        Ok(msg) => msg.with_str(|s| from_str::<uint>(s).unwrap()),
        Err(e) => fail!(e.to_str()),
    };

    let end = extra::time::precise_time_s();
    let elapsed = end - start;

    println!("Count is {}", result);
    println!("Test took {} seconds", elapsed);
    let thruput = ((size / workers * workers) as f64) / (elapsed as f64);
    println!("Throughput={:f} per sec", thruput);
}

fn main() {
    let args = os::args();

    let args = if os::getenv("RUST_BENCH").is_some() {
        ~[~"", ~"1000000", ~"10000"]
    } else if args.len() <= 1u {
        ~[~"", ~"10000", ~"4"]
    } else {
        args
    };

    let size = from_str::<uint>(args[1]).unwrap();
    let workers = from_str::<uint>(args[2]).unwrap();

    let mut ctx = zmq::Context::new();

    run(&mut ctx, size, workers);
}
<|endoftext|>


<commit_before>use euclid::{Vector2D, vec2};
use hex::Dir6;
use num::Integer;

/// User data for field of view cells.
pub trait FovValue: PartialEq + Clone {
    /// Construct a new FovValue for a position based on the previous one along the line of sight.
    fn advance(&self, offset: Vector2D<i32>) -> Option<Self>;
}

/// Field of view iterator for a hexagonal map.
pub struct HexFov<T> {
    stack: Vec<Arc<T>>,
    /// Extra values generated by special cases.
    side_channel: Vec<(Vector2D<i32>, T)>,
}

impl<T: FovValue> HexFov<T> {
    /// Create a new field of view iterator with a seed userdata for the origin position.
    pub fn new(init: T) -> HexFov<T> {
        // We could run f for (0, 0) here, but the traditional way for the FOV to work is to only
        // consider your surroundings, not the origin site itself.
        HexFov {
            stack: vec![
                Arc::new(
                    PolarPoint::new(0.0, 1),
                    PolarPoint::new(6.0, 1),
                    init.clone()
                ),
            ],
            // The FOV algorithm will not generate the origin point, so we use
            // the side channel to explicitly add it in the beginning.
            side_channel: vec![(vec2(0, 0), init)],
        }
    }
}

impl<T: FovValue> Iterator for HexFov<T> {
    type Item = (Vector2D<i32>, T);
    fn next(&mut self) -> Option<(Vector2D<i32>, T)> {
        // Empty the side channel before proceeding to the algorithm proper.
        if let Some(ret) = self.side_channel.pop() {
            return Some(ret);
        }

        // Start processing the next arc in the stack.
        if let Some(current) = self.stack.pop() {
            if current.arc_has_split(&mut self.stack) {
                return self.next();
            }

            debug_assert!(current.group_value == current.prev_value.advance(current.pt.to_v2()));

            let pos = current.pt.to_v2();
            let ret = current.group_value.clone();

            current.advance(&mut self.stack);

            if let Some(ret) = ret {
                return Some((pos, ret));
            } else {
                return self.next();
            }
        } else {
            None
        }
    }
}

struct Arc<T> {
    /// Start point of current arc.
    begin: PolarPoint,
    /// Point currently being processed.
    pt: PolarPoint,
    /// End point of current arc.
    end: PolarPoint,
    /// The user value from previous iteration.
    prev_value: T,
    /// The user value for this group.
    group_value: Option<T>,
}

impl<T: FovValue> Arc<T> {
    pub fn new(begin: PolarPoint, end: PolarPoint, prev_value: T) -> Arc<T> {
        let group_value = prev_value.advance(begin.to_v2());
        Arc {
            begin: begin,
            pt: begin,
            end: end,
            prev_value: prev_value,
            group_value: group_value,
        }
    }

    /// Consume the given arc and add its descendent, if any, to the stack.
    pub fn advance(mut self, stack: &mut Vec<Arc<T>>) {
        self.pt = self.pt.next();
        if self.pt.is_below(self.end) {
            stack.push(self);
        } else if let Some(group_value) = self.group_value {
            stack.push(Arc::new(
                self.begin.further(),
                self.end.further(),
                group_value,
            ));
        }
    }

    /// If the arc has advanced into a different value group, split it into the given stack.
    ///
    /// Return true if arc was split, false otherwise.
    pub fn arc_has_split(&self, stack: &mut Vec<Arc<T>>) -> bool {
        let next_value = self.prev_value.advance(self.pt.to_v2());
        if next_value != self.group_value {
            // Using the literal instead of the constructor to avoid recomputing next_value.
            stack.push(Arc {
                begin: self.pt,
                pt: self.pt,
                end: self.end,
                prev_value: self.prev_value.clone(),
                group_value: next_value,
            });

            // Extend current arc if it has a group value.
            if let Some(ref group_value) = self.group_value {
                stack.push(Arc::new(
                    self.begin.further(),
                    self.pt.further(),
                    group_value.clone(),
                ));
            }

            true
        } else {
            false
        }
    }
}

/// Points on a hex circle expressed in polar coordinates.
#[derive(Copy, Clone, PartialEq)]
struct PolarPoint {
    pos: f32,
    radius: u32,
}

impl PolarPoint {
    pub fn new(pos: f32, radius: u32) -> PolarPoint { PolarPoint { pos, radius } }

    /// Index of the discrete hex cell along the circle that corresponds to this point.
    fn winding_index(self) -> i32 { (self.pos + 0.5).floor() as i32 }

    pub fn is_below(self, other: PolarPoint) -> bool { self.winding_index() < other.end_index() }

    fn end_index(self) -> i32 { (self.pos + 0.5).ceil() as i32 }

    pub fn to_v2(self) -> Vector2D<i32> {
        if self.radius == 0 {
            return vec2(0, 0);
        }
        let index = self.winding_index();
        let sector = index.mod_floor(&(self.radius as i32 * 6)) / self.radius as i32;
        let offset = index.mod_floor(&(self.radius as i32));

        let rod = Dir6::from_int(sector).to_v2();
        let tangent = Dir6::from_int(sector + 2).to_v2();

        rod * (self.radius as i32) + tangent * offset
    }

    /// The point corresponding to this one on the hex circle with radius +1.
    pub fn further(self) -> PolarPoint {
        PolarPoint::new(
            self.pos * (self.radius + 1) as f32 / self.radius as f32,
            self.radius + 1,
        )
    }

    /// The point next to this one along the hex circle.
    pub fn next(self) -> PolarPoint { PolarPoint::new((self.pos + 0.5).floor() + 0.5, self.radius) }
}

#[cfg(test)]
mod test {
    use super::{FovValue, HexFov};
    use euclid::{Vector2D, vec2};
    use hex::HexGeom;
    use std::collections::HashMap;
    use std::iter::FromIterator;

    #[derive(PartialEq, Eq, Clone)]
    struct Cell1 {
        range: i32,
    }

    impl FovValue for Cell1 {
        fn advance(&self, offset: Vector2D<i32>) -> Option<Self> {
            if offset.hex_dist() < self.range {
                Some(self.clone())
            } else {
                None
            }
        }
    }

    #[test]
    fn trivial_fov() {
        // Just draw a small circle.
        let field: HashMap<Vector2D<i32>, Cell1> =
            HashMap::from_iter(HexFov::new(Cell1 { range: 2 }));
        assert!(field.contains_key(&vec2(1, 0)));
        assert!(!field.contains_key(&vec2(1, -1)));
    }
}
<commit_msg>PolarPoint trait<commit_after>use euclid::{Vector2D, vec2};
use hex::Dir6;
use num::Integer;

/// User data for field of view cells.
pub trait FovValue: PartialEq + Clone {
    /// Construct a new FovValue for a position based on the previous one along the line of sight.
    fn advance(&self, offset: Vector2D<i32>) -> Option<Self>;
}

/// Field of view iterator for a hexagonal map.
pub struct HexFov<T> {
    stack: Vec<Arc<T>>,
    /// Extra values generated by special cases.
    side_channel: Vec<(Vector2D<i32>, T)>,
}

impl<T: FovValue> HexFov<T> {
    /// Create a new field of view iterator with a seed userdata for the origin position.
    pub fn new(init: T) -> HexFov<T> {
        // We could run f for (0, 0) here, but the traditional way for the FOV to work is to only
        // consider your surroundings, not the origin site itself.
        HexFov {
            stack: vec![
                Arc::new(
                    HexPolarPoint::new(0.0, 1),
                    HexPolarPoint::new(6.0, 1),
                    init.clone()
                ),
            ],
            // The FOV algorithm will not generate the origin point, so we use
            // the side channel to explicitly add it in the beginning.
            side_channel: vec![(vec2(0, 0), init)],
        }
    }
}

impl<T: FovValue> Iterator for HexFov<T> {
    type Item = (Vector2D<i32>, T);
    fn next(&mut self) -> Option<(Vector2D<i32>, T)> {
        // Empty the side channel before proceeding to the algorithm proper.
        if let Some(ret) = self.side_channel.pop() {
            return Some(ret);
        }

        // Start processing the next arc in the stack.
        if let Some(current) = self.stack.pop() {
            if current.arc_has_split(&mut self.stack) {
                return self.next();
            }

            debug_assert!(current.group_value == current.prev_value.advance(current.pt.to_v2()));

            let pos = current.pt.to_v2();
            let ret = current.group_value.clone();

            current.advance(&mut self.stack);

            if let Some(ret) = ret {
                return Some((pos, ret));
            } else {
                return self.next();
            }
        } else {
            None
        }
    }
}

struct Arc<T> {
    /// Start point of current arc.
    begin: HexPolarPoint,
    /// Point currently being processed.
    pt: HexPolarPoint,
    /// End point of current arc.
    end: HexPolarPoint,
    /// The user value from previous iteration.
    prev_value: T,
    /// The user value for this group.
    group_value: Option<T>,
}

impl<T: FovValue> Arc<T> {
    pub fn new(begin: HexPolarPoint, end: HexPolarPoint, prev_value: T) -> Arc<T> {
        let group_value = prev_value.advance(begin.to_v2());
        Arc {
            begin: begin,
            pt: begin,
            end: end,
            prev_value: prev_value,
            group_value: group_value,
        }
    }

    /// Consume the given arc and add its descendent, if any, to the stack.
    pub fn advance(mut self, stack: &mut Vec<Arc<T>>) {
        self.pt = self.pt.next();
        if self.pt.is_below(self.end) {
            stack.push(self);
        } else if let Some(group_value) = self.group_value {
            stack.push(Arc::new(
                self.begin.further(),
                self.end.further(),
                group_value,
            ));
        }
    }

    /// If the arc has advanced into a different value group, split it into the given stack.
    ///
    /// Return true if arc was split, false otherwise.
    pub fn arc_has_split(&self, stack: &mut Vec<Arc<T>>) -> bool {
        let next_value = self.prev_value.advance(self.pt.to_v2());
        if next_value != self.group_value {
            // Using the literal instead of the constructor to avoid recomputing next_value.
            stack.push(Arc {
                begin: self.pt,
                pt: self.pt,
                end: self.end,
                prev_value: self.prev_value.clone(),
                group_value: next_value,
            });

            // Extend current arc if it has a group value.
            if let Some(ref group_value) = self.group_value {
                stack.push(Arc::new(
                    self.begin.further(),
                    self.pt.further(),
                    group_value.clone(),
                ));
            }

            true
        } else {
            false
        }
    }
}

/// Point in FOV arc polar coordinates for the desired geometry.
pub trait PolarPoint: Sized {
    /// Create the beginning and end points for radius 1 unit circle.
    ///
    /// These are the initial cells checked by the FOV algorithm.
    fn unit_circle_endpoints() -> (Self, Self);

    /// Return whether this point is earlier in the circle than the other.
    fn is_below(&self, other: &Self) -> bool;

    /// Return the vector offset for this point.
    fn to_v2(&self) -> Vector2D<i32>;

    /// Return the point corresponding to this one on the next circle with +1 radius.
    fn expand(&self) -> Self;

    /// Step forward on the current radius.
    fn advance(&mut self);
}

/// Points on a hex circle expressed in polar coordinates.
#[derive(Copy, Clone, PartialEq)]
struct HexPolarPoint {
    pos: f32,
    radius: u32,
}

impl HexPolarPoint {
    pub fn new(pos: f32, radius: u32) -> HexPolarPoint { HexPolarPoint { pos, radius } }

    /// Index of the discrete hex cell along the circle that corresponds to this point.
    fn winding_index(self) -> i32 { (self.pos + 0.5).floor() as i32 }

    pub fn is_below(self, other: HexPolarPoint) -> bool { self.winding_index() < other.end_index() }

    fn end_index(self) -> i32 { (self.pos + 0.5).ceil() as i32 }

    pub fn to_v2(self) -> Vector2D<i32> {
        if self.radius == 0 {
            return vec2(0, 0);
        }
        let index = self.winding_index();
        let sector = index.mod_floor(&(self.radius as i32 * 6)) / self.radius as i32;
        let offset = index.mod_floor(&(self.radius as i32));

        let rod = Dir6::from_int(sector).to_v2();
        let tangent = Dir6::from_int(sector + 2).to_v2();

        rod * (self.radius as i32) + tangent * offset
    }

    /// The point corresponding to this one on the hex circle with radius +1.
    pub fn further(self) -> HexPolarPoint {
        HexPolarPoint::new(
            self.pos * (self.radius + 1) as f32 / self.radius as f32,
            self.radius + 1,
        )
    }

    /// The point next to this one along the hex circle.
    pub fn next(self) -> HexPolarPoint { HexPolarPoint::new((self.pos + 0.5).floor() + 0.5, self.radius) }
}

#[cfg(test)]
mod test {
    use super::{FovValue, HexFov};
    use euclid::{Vector2D, vec2};
    use hex::HexGeom;
    use std::collections::HashMap;
    use std::iter::FromIterator;

    #[derive(PartialEq, Eq, Clone)]
    struct Cell1 {
        range: i32,
    }

    impl FovValue for Cell1 {
        fn advance(&self, offset: Vector2D<i32>) -> Option<Self> {
            if offset.hex_dist() < self.range {
                Some(self.clone())
            } else {
                None
            }
        }
    }

    #[test]
    fn trivial_fov() {
        // Just draw a small circle.
        let field: HashMap<Vector2D<i32>, Cell1> =
            HashMap::from_iter(HexFov::new(Cell1 { range: 2 }));
        assert!(field.contains_key(&vec2(1, 0)));
        assert!(!field.contains_key(&vec2(1, -1)));
    }
}
<|endoftext|>