<commit_before>use core as _core;

#[doc(hidden)]
pub trait ImplementedByBitFlagsMacro {}

/// A trait that is automatically implemented for all bitflags.
///
/// It should not be implemented manually.
pub trait BitFlags: ImplementedByBitFlagsMacro {
    type Bits;
    /// Returns an empty set of flags.
    fn empty() -> Self;
    /// Returns the set containing all flags.
    fn all() -> Self;
    /// Returns the raw value of the flags currently stored.
    fn bits(&self) -> Self::Bits;
    /// Convert from underlying bit representation, unless that
    /// representation contains bits that do not correspond to a flag.
    fn from_bits(bits: Self::Bits) -> _core::option::Option<Self>
    where Self: Sized;
    /// Convert from underlying bit representation, dropping any bits
    /// that do not correspond to flags.
    fn from_bits_truncate(bits: Self::Bits) -> Self;
    /// Convert from underlying bit representation, preserving all
    /// bits (even those not corresponding to a defined flag).
    ///
    /// # Safety
    ///
    /// The caller of the `bitflags!` macro can chose to allow or
    /// disallow extra bits for their bitflags type.
    ///
    /// The caller of `from_bits_unchecked()` has to ensure that
    /// all bits correspond to a defined flag or that extra bits
    /// are valid for this bitflags type.
    unsafe fn from_bits_unchecked(bits: Self::Bits) -> Self;
    /// Returns `true` if no flags are currently stored.
    fn is_empty(&self) -> bool;
    /// Returns `true` if all flags are currently set.
    fn is_all(&self) -> bool;
    /// Returns `true` if there are flags common to both `self` and `other`.
    fn intersects(&self, other: Self) -> bool;
    /// Returns `true` all of the flags in `other` are contained within `self`.
    fn contains(&self, other: Self) -> bool;
    /// Inserts the specified flags in-place.
    fn insert(&mut self, other: Self);
    /// Removes the specified flags in-place.
    fn remove(&mut self, other: Self);
    /// Toggles the specified flags in-place.
    fn toggle(&mut self, other: Self);
    /// Inserts or removes the specified flags depending on the passed value.
    fn set(&mut self, other: Self, value: bool);
}
<commit_msg>Remove unnecessary core use statement<commit_after>#[doc(hidden)]
pub trait ImplementedByBitFlagsMacro {}

/// A trait that is automatically implemented for all bitflags.
///
/// It should not be implemented manually.
pub trait BitFlags: ImplementedByBitFlagsMacro {
    type Bits;
    /// Returns an empty set of flags.
    fn empty() -> Self;
    /// Returns the set containing all flags.
    fn all() -> Self;
    /// Returns the raw value of the flags currently stored.
    fn bits(&self) -> Self::Bits;
    /// Convert from underlying bit representation, unless that
    /// representation contains bits that do not correspond to a flag.
    fn from_bits(bits: Self::Bits) -> _core::option::Option<Self>
    where Self: Sized;
    /// Convert from underlying bit representation, dropping any bits
    /// that do not correspond to flags.
    fn from_bits_truncate(bits: Self::Bits) -> Self;
    /// Convert from underlying bit representation, preserving all
    /// bits (even those not corresponding to a defined flag).
    ///
    /// # Safety
    ///
    /// The caller of the `bitflags!` macro can chose to allow or
    /// disallow extra bits for their bitflags type.
    ///
    /// The caller of `from_bits_unchecked()` has to ensure that
    /// all bits correspond to a defined flag or that extra bits
    /// are valid for this bitflags type.
    unsafe fn from_bits_unchecked(bits: Self::Bits) -> Self;
    /// Returns `true` if no flags are currently stored.
    fn is_empty(&self) -> bool;
    /// Returns `true` if all flags are currently set.
    fn is_all(&self) -> bool;
    /// Returns `true` if there are flags common to both `self` and `other`.
    fn intersects(&self, other: Self) -> bool;
    /// Returns `true` all of the flags in `other` are contained within `self`.
    fn contains(&self, other: Self) -> bool;
    /// Inserts the specified flags in-place.
    fn insert(&mut self, other: Self);
    /// Removes the specified flags in-place.
    fn remove(&mut self, other: Self);
    /// Toggles the specified flags in-place.
    fn toggle(&mut self, other: Self);
    /// Inserts or removes the specified flags depending on the passed value.
    fn set(&mut self, other: Self, value: bool);
}
<|endoftext|>


<commit_before>fn check_bit(left: &u8, right: &u8, bit: u8) -> u16 {
	if left & ((1 << bit) as u8) ^ right & ((1 << bit) as u8) > 0 {
		1
	} else {
		0
	}
}

pub fn hamming_distance(a: &Vec<u8>, b: &Vec<u8>) -> u16 {
	let mut count: u16 = 0;

	for (left, right) in a.iter().zip(b.iter()) {
		for bit in 0..9 {
			count += check_bit(&left, &right, bit);
			println!("{}", bit);
		}
	}

	count
}

#[cfg(test)]
mod test {
	use super::*;

	#[test]
	fn hamming_distance_no_distance() {
		assert_eq!(0, hamming_distance(&vec![5], &vec![5]));
	}

	#[test]
	fn hamming_distance_one_distance_1() {
		assert_eq!(1, hamming_distance(&vec![0], &vec![1]));
	}

	#[test]
	fn hamming_distance_one_distance_2() {
		assert_eq!(1, hamming_distance(&vec![4], &vec![5]));
	}

	#[test]
	fn hamming_distance_one_distance_3() {
		assert_eq!(1, hamming_distance(&vec![4], &vec![6]));
	}

	#[test]
	fn hamming_distance_two_distance_1() {
		assert_eq!(2, hamming_distance(&vec![4], &vec![7]));
	}

	#[test]
	fn hamming_distance_example() {
		use byte_conversion::*;
		let left = readable_text_to_bytes("this is a test");
		let right = readable_text_to_bytes("wokka wokka!!!");

		assert_eq!(37, hamming_distance(&left, &right));
	}
}<commit_msg>Hamming readability<commit_after>fn check_bit(left: &u8, right: &u8, bit: u8) -> u16 {
	let left_bit = left & ((1 << bit) as u8);
	let right_bit = right & ((1 << bit) as u8);

	if (left_bit ^ right_bit) > 0 {
		1
	} else {
		0
	}
}

pub fn hamming_distance(a: &Vec<u8>, b: &Vec<u8>) -> u16 {
	let mut count: u16 = 0;

	for (left, right) in a.iter().zip(b.iter()) {
		for bit in 0..8
		{
			count += check_bit(&left, &right, bit);
		}
	}

	count
}

#[cfg(test)]
mod test {
	use super::*;

	#[test]
	fn hamming_distance_no_distance() {
		assert_eq!(0, hamming_distance(&vec![5], &vec![5]));
	}

	#[test]
	fn hamming_distance_one_distance_1() {
		assert_eq!(1, hamming_distance(&vec![0], &vec![1]));
	}

	#[test]
	fn hamming_distance_one_distance_2() {
		assert_eq!(1, hamming_distance(&vec![4], &vec![5]));
	}

	#[test]
	fn hamming_distance_one_distance_3() {
		assert_eq!(1, hamming_distance(&vec![4], &vec![6]));
	}

	#[test]
	fn hamming_distance_two_distance_1() {
		assert_eq!(2, hamming_distance(&vec![4], &vec![7]));
	}

	#[test]
	fn hamming_distance_example() {
		use byte_conversion::*;
		let left = readable_text_to_bytes("this is a test");
		let right = readable_text_to_bytes("wokka wokka!!!");

		assert_eq!(37, hamming_distance(&left, &right));
	}
}<|endoftext|>


<commit_before>use std::process::{Command, Stdio};
use std::path::{Path, PathBuf};
use std::io::{BufRead, BufReader};
use std::ffi::OsString;
use std::env;
use std::thread;

use cargo_metadata;
use serde_json;

mod cargo_output;
mod rustc_diagnostic;
mod diagnostic_formatter;

use self::cargo_output::CargoOutput;

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum BuildType {
    Debug,
    Release
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Profile {
    Main,
    Test,
    Bench
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum TargetKind {
    Lib,
    Bin,
    Example,
    Test,
    Bench
}

#[derive(Clone, Debug)]
pub struct CargoProject {
    pub packages: Vec< CargoPackage >
}

#[derive(Clone, Debug)]
pub struct CargoPackage {
    pub name: String,
    pub manifest_path: PathBuf,
    pub crate_root: PathBuf,
    pub targets: Vec< CargoTarget >
}

#[derive(Clone, Debug)]
pub struct CargoTarget {
    pub name: String,
    pub kind: TargetKind,
    pub source_directory: PathBuf
}

impl CargoProject {
    pub fn new( manifest_path: Option< &str > ) -> CargoProject {
        let metadata = cargo_metadata::metadata( manifest_path.map( |path| Path::new( path ) ) ).unwrap();
        CargoProject {
            packages: metadata.packages.into_iter().map( |package| {
                let manifest_path: PathBuf = package.manifest_path.into();
                CargoPackage {
                    name: package.name,
                    crate_root: manifest_path.parent().unwrap().into(),
                    manifest_path: manifest_path,
                    targets: package.targets.into_iter().filter_map( |target| {
                        Some( CargoTarget {
                            name: target.name,
                            kind: match target.kind[ 0 ].as_str() {
                                "lib" => TargetKind::Lib,
                                "bin" => TargetKind::Bin,
                                "example" => TargetKind::Example,
                                "test" => TargetKind::Test,
                                "bench" => TargetKind::Bench,
                                "custom-build" => return None,
                                _ => panic!( "Unknown target kind: '{}'", target.kind[ 0 ] )
                            },
                            source_directory: Into::< PathBuf >::into( target.src_path ).parent().unwrap().into()
                        })
                    }).collect()
                }
            }).collect()
        }
    }

    pub fn default_package( &self ) -> &CargoPackage {
        &self.packages[ 0 ]
    }
}

#[derive(Clone, Debug)]
pub enum BuildTarget {
    Lib( String, Profile ),
    Bin( String, Profile ),
    ExampleBin( String ),
    IntegrationTest( String ),
    IntegrationBench( String )
}

#[derive(Copy, Clone, PartialEq, Debug)]
pub enum MessageFormat {
    Human,
    Json
}

#[derive(Clone, Debug)]
pub struct BuildConfig {
    pub build_target: BuildTarget,
    pub build_type: BuildType,
    pub triplet: Option< String >,
    pub package: Option< String >,
    pub features: Vec< String >,
    pub no_default_features: bool,
    pub enable_all_features: bool,
    pub extra_paths: Vec< PathBuf >,
    pub extra_rustflags: Vec< String >,
    pub extra_environment: Vec< (String, String) >,
    pub message_format: MessageFormat,
    pub is_verbose: bool
}

fn profile_to_arg( profile: Profile ) -> &'static str {
    match profile {
        Profile::Main => "dev",
        Profile::Test => "test",
        Profile::Bench => "bench"
    }
}

pub fn target_to_build_target( target: &CargoTarget, profile: Profile ) -> BuildTarget {
    match target.kind {
        TargetKind::Lib => BuildTarget::Lib( target.name.clone(), profile ),
        TargetKind::Bin => BuildTarget::Bin( target.name.clone(), profile ),
        TargetKind::Example => BuildTarget::ExampleBin( target.name.clone() ),
        TargetKind::Test => BuildTarget::IntegrationTest( target.name.clone() ),
        TargetKind::Bench => BuildTarget::IntegrationBench( target.name.clone() )
    }
}

impl BuildConfig {
    fn as_command( &self ) -> Command {
        let mut command = Command::new( "cargo" );
        command.arg( "rustc" );
        command.arg( "--message-format" );
        command.arg( "json" );
        command.arg( "--color" );
        command.arg( "always" );

        if let Some( ref triplet ) = self.triplet {
            command.arg( "--target" ).arg( triplet.as_str() );
        }

        if let Some( ref package ) = self.package {
            command.arg( "--package" ).arg( package.as_str() );
        }

        match self.build_type {
            BuildType::Debug => {},
            BuildType::Release => {
                command.arg( "--release" );
            }
        }

        match self.build_target {
            BuildTarget::Lib( _, profile ) => {
                command
                    .arg( "--profile" ).arg( profile_to_arg( profile ) )
                    .arg( "--lib" );
            },
            BuildTarget::Bin( ref name, profile ) => {
                command
                    .arg( "--profile" ).arg( profile_to_arg( profile ) )
                    .arg( "--bin" ).arg( name.as_str() );
            },
            BuildTarget::ExampleBin( ref name ) => {
                command.arg( "--example" ).arg( name.as_str() );
            },
            BuildTarget::IntegrationTest( ref name ) => {
                command.arg( "--test" ).arg( name.as_str() );
            },
            BuildTarget::IntegrationBench( ref name ) => {
                command.arg( "--bench" ).arg( name.as_str() );
            }
        }

        if self.no_default_features {
            command.arg( "--no-default-features" );
        }

        if self.enable_all_features {
            command.arg( "--all-features" );
        }

        if !self.features.is_empty() {
            command.arg( "--features" );
            command.arg( &self.features.join( " " ) );
        }

        if self.is_verbose {
            command.arg( "--verbose" );
        }

        command
    }

    pub fn build< F >( &self, mut extra_artifact_generator: Option< F > ) -> CargoResult
        where F: for <'a> FnMut( &'a Path ) -> Vec< PathBuf >
    {
        let mut result = self.build_internal( &mut extra_artifact_generator );
        if result.is_ok() == false {
            return result;
        }

        // HACK: For some reason when you install emscripten for the first time
        // the first build is always a dud (it produces no artifacts), so we retry once.
        let is_emscripten = self.triplet.as_ref().map( |triplet| {
            triplet == "wasm32-unknown-emscripten" || triplet == "asmjs-unknown-emscripten"
        }).unwrap_or( false );

        if is_emscripten {
            let no_js_generated = result
                .artifacts()
                .iter()
                .find( |artifact| artifact.extension().map( |ext| ext == "js" ).unwrap_or( false ) )
                .is_none();

            if no_js_generated {
                debug!( "No artifacts were generated yet build succeeded; retrying..." );
                result = self.build_internal( &mut extra_artifact_generator );
            }
        }

        return result;
    }

    fn build_internal< F >( &self, extra_artifact_generator: &mut Option< F > ) -> CargoResult
        where F: for <'a> FnMut( &'a Path ) -> Vec< PathBuf >
    {
        let mut command = self.as_command();

        let env_paths = env::var_os( "PATH" )
            .map( |paths| env::split_paths( &paths ).collect() )
            .unwrap_or( Vec::new() );

        let mut paths = Vec::new();
        paths.extend( self.extra_paths.clone().into_iter() );
        paths.extend( env_paths.into_iter() );

        let new_paths = env::join_paths( paths ).unwrap();
        debug!( "Will launch cargo with PATH: {:?}", new_paths );
        command.env( "PATH", new_paths );

        let mut rustflags = env::var_os( "RUSTFLAGS" ).unwrap_or( OsString::new() );
        for flag in &self.extra_rustflags {
            if !rustflags.is_empty() {
                rustflags.push( " " );
            }
            rustflags.push( flag );
        }
        debug!( "Will launch cargo with RUSTFLAGS: {:?}", rustflags );
        command.env( "RUSTFLAGS", rustflags );

        for &(ref key, ref value) in &self.extra_environment {
            debug!( "Will launch cargo with variable \"{}\" set to \"{}\"", key, value );
            command.env( key, value );
        }

        command.stdout( Stdio::piped() );
        command.stderr( Stdio::piped() );

        debug!( "Launching cargo: {:?}", command );
        let mut child = match command.spawn() {
            Ok( child ) => child,
            Err( _ ) => {
                return CargoResult {
                    status: None,
                    artifacts: Vec::new()
                };
            }
        };

        let stderr = BufReader::new( child.stderr.take().unwrap() );
        let stdout = BufReader::new( child.stdout.take().unwrap() );

        let is_verbose = self.is_verbose;
        thread::spawn( move || {
            let mut skip = 0;
            for line in stderr.lines() {
                let line = match line {
                    Ok( line ) => line,
                    Err( _ ) => break
                };

                if skip > 0 {
                    skip -= 1;
                    continue;
                }

                // This is really ugly, so let's skip it.
                if line.trim() == "Caused by:" && !is_verbose {
                    skip += 1;
                    continue;
                }

                eprintln!( "{}", line );
            }
        });

        let mut artifacts = Vec::new();
        for line in stdout.lines() {
            let line = match line {
                Ok( line ) => line,
                Err( _ ) => break
            };

            let line = line.trim();
            if line.is_empty() {
                continue;
            }

            let json: serde_json::Value = serde_json::from_str( &line ).expect( "failed to parse cargo output" );
            let line = serde_json::to_string_pretty( &json ).unwrap();
            if let Some( output ) = CargoOutput::parse( &line ) {
                match output {
                    CargoOutput::Message( message ) => {
                        match self.message_format {
                            MessageFormat::Human => diagnostic_formatter::print( &message ),
                            MessageFormat::Json => {
                                println!( "{}", serde_json::to_string( &message.to_json_value() ).unwrap() );
                            }
                        }
                    },
                    CargoOutput::Artifact( artifact ) => {
                        for filename in &artifact.filenames {
                            debug!( "Built artifact: {}", filename );
                        }

                        artifacts.push( artifact );
                    },
                    CargoOutput::BuildScriptExecuted( executed ) => {
                        match self.message_format {
                            MessageFormat::Human => {},
                            MessageFormat::Json => {
                                println!( "{}", serde_json::to_string( &executed.to_json_value() ).unwrap() );
                            }
                        }
                    }
                }
            }
        }

        let result = child.wait();
        let status = result.unwrap().code().expect( "failed to grab cargo status code" );
        debug!( "Cargo finished with status: {}", status );

        fn has_extension< P: AsRef< Path > >( path: P, extension: &str ) -> bool {
            path.as_ref().extension().map( |ext| ext == extension ).unwrap_or( false )
        }

        fn find_artifact( artifacts: &[cargo_output::Artifact], extension: &str ) -> Option< (usize, usize) > {
            artifacts.iter().enumerate().filter_map( |(artifact_index, artifact)| {
                if let Some( filename_index ) = artifact.filenames.iter().position( |filename| has_extension( filename, extension ) ) {
                    Some( (artifact_index, filename_index) )
                } else {
                    None
                }
            }).next()
        }

        // For some reason when building tests cargo doesn't treat
        // the `.wasm` file as an artifact.
        if status == 0 && self.triplet.as_ref().map( |triplet| triplet == "wasm32-unknown-emscripten" ).unwrap_or( false ) {
            match self.build_target {
                BuildTarget::Bin( _, Profile::Test ) | BuildTarget::Lib( _, Profile::Test ) => {
                    if find_artifact( &artifacts, "wasm" ).is_none() {
                        if let Some( (artifact_index, filename_index) ) = find_artifact( &artifacts, "js" ) {
                            let wasm_path = {
                                let main_artifact = Path::new( &artifacts[ artifact_index ].filenames[ filename_index ] );
                                let filename = main_artifact.file_name().unwrap();
                                main_artifact.parent().unwrap().join( "deps" ).join( filename ).with_extension( "wasm" )
                            };

                            assert!( wasm_path.exists(), "internal error: wasm doesn't exist where I expected it to be" );
                            artifacts[ artifact_index ].filenames.push( wasm_path.to_str().unwrap().to_owned() );
                        }
                    }
                },
                _ => {}
            }
        }

        let mut artifact_paths = Vec::new();
        for mut artifact in artifacts {
            if let Some( ref mut callback ) = extra_artifact_generator.as_mut() {
                let mut extra_filenames = Vec::new();
                for filename in &artifact.filenames {
                    extra_filenames.extend(
                        callback( Path::new( &filename ) ).into_iter().map( |artifact| artifact.to_str().unwrap().to_owned() )
                    );
                }
                artifact.filenames.extend( extra_filenames );
            }

            match self.message_format {
                MessageFormat::Human => {},
                MessageFormat::Json => {
                    println!( "{}", serde_json::to_string( &artifact.to_json_value() ).unwrap() );
                }
            }

            for filename in artifact.filenames {
                // NOTE: Since we extract the paths from the JSON
                //       we get a list of artifacts as `String`s instead of `PathBuf`s.
                artifact_paths.push( filename.into() )
            }
        }

        CargoResult {
            status: Some( status ),
            artifacts: artifact_paths
        }
    }
}

pub struct CargoResult {
    status: Option< i32 >,
    artifacts: Vec< PathBuf >
}

impl CargoResult {
    pub fn is_ok( &self ) -> bool {
        self.status == Some( 0 )
    }

    pub fn artifacts( &self ) -> &[PathBuf] {
        &self.artifacts
    }
}
<commit_msg>Treat `cdylib` targets as libs for now<commit_after>use std::process::{Command, Stdio};
use std::path::{Path, PathBuf};
use std::io::{BufRead, BufReader};
use std::ffi::OsString;
use std::env;
use std::thread;

use cargo_metadata;
use serde_json;

mod cargo_output;
mod rustc_diagnostic;
mod diagnostic_formatter;

use self::cargo_output::CargoOutput;

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum BuildType {
    Debug,
    Release
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Profile {
    Main,
    Test,
    Bench
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum TargetKind {
    Lib,
    Bin,
    Example,
    Test,
    Bench
}

#[derive(Clone, Debug)]
pub struct CargoProject {
    pub packages: Vec< CargoPackage >
}

#[derive(Clone, Debug)]
pub struct CargoPackage {
    pub name: String,
    pub manifest_path: PathBuf,
    pub crate_root: PathBuf,
    pub targets: Vec< CargoTarget >
}

#[derive(Clone, Debug)]
pub struct CargoTarget {
    pub name: String,
    pub kind: TargetKind,
    pub source_directory: PathBuf
}

impl CargoProject {
    pub fn new( manifest_path: Option< &str > ) -> CargoProject {
        let metadata = cargo_metadata::metadata( manifest_path.map( |path| Path::new( path ) ) ).unwrap();
        CargoProject {
            packages: metadata.packages.into_iter().map( |package| {
                let manifest_path: PathBuf = package.manifest_path.into();
                CargoPackage {
                    name: package.name,
                    crate_root: manifest_path.parent().unwrap().into(),
                    manifest_path: manifest_path,
                    targets: package.targets.into_iter().filter_map( |target| {
                        Some( CargoTarget {
                            name: target.name,
                            kind: match target.kind[ 0 ].as_str() {
                                "lib" => TargetKind::Lib,
                                "cdylib" => TargetKind::Lib,
                                "bin" => TargetKind::Bin,
                                "example" => TargetKind::Example,
                                "test" => TargetKind::Test,
                                "bench" => TargetKind::Bench,
                                "custom-build" => return None,
                                _ => panic!( "Unknown target kind: '{}'", target.kind[ 0 ] )
                            },
                            source_directory: Into::< PathBuf >::into( target.src_path ).parent().unwrap().into()
                        })
                    }).collect()
                }
            }).collect()
        }
    }

    pub fn default_package( &self ) -> &CargoPackage {
        &self.packages[ 0 ]
    }
}

#[derive(Clone, Debug)]
pub enum BuildTarget {
    Lib( String, Profile ),
    Bin( String, Profile ),
    ExampleBin( String ),
    IntegrationTest( String ),
    IntegrationBench( String )
}

#[derive(Copy, Clone, PartialEq, Debug)]
pub enum MessageFormat {
    Human,
    Json
}

#[derive(Clone, Debug)]
pub struct BuildConfig {
    pub build_target: BuildTarget,
    pub build_type: BuildType,
    pub triplet: Option< String >,
    pub package: Option< String >,
    pub features: Vec< String >,
    pub no_default_features: bool,
    pub enable_all_features: bool,
    pub extra_paths: Vec< PathBuf >,
    pub extra_rustflags: Vec< String >,
    pub extra_environment: Vec< (String, String) >,
    pub message_format: MessageFormat,
    pub is_verbose: bool
}

fn profile_to_arg( profile: Profile ) -> &'static str {
    match profile {
        Profile::Main => "dev",
        Profile::Test => "test",
        Profile::Bench => "bench"
    }
}

pub fn target_to_build_target( target: &CargoTarget, profile: Profile ) -> BuildTarget {
    match target.kind {
        TargetKind::Lib => BuildTarget::Lib( target.name.clone(), profile ),
        TargetKind::Bin => BuildTarget::Bin( target.name.clone(), profile ),
        TargetKind::Example => BuildTarget::ExampleBin( target.name.clone() ),
        TargetKind::Test => BuildTarget::IntegrationTest( target.name.clone() ),
        TargetKind::Bench => BuildTarget::IntegrationBench( target.name.clone() )
    }
}

impl BuildConfig {
    fn as_command( &self ) -> Command {
        let mut command = Command::new( "cargo" );
        command.arg( "rustc" );
        command.arg( "--message-format" );
        command.arg( "json" );
        command.arg( "--color" );
        command.arg( "always" );

        if let Some( ref triplet ) = self.triplet {
            command.arg( "--target" ).arg( triplet.as_str() );
        }

        if let Some( ref package ) = self.package {
            command.arg( "--package" ).arg( package.as_str() );
        }

        match self.build_type {
            BuildType::Debug => {},
            BuildType::Release => {
                command.arg( "--release" );
            }
        }

        match self.build_target {
            BuildTarget::Lib( _, profile ) => {
                command
                    .arg( "--profile" ).arg( profile_to_arg( profile ) )
                    .arg( "--lib" );
            },
            BuildTarget::Bin( ref name, profile ) => {
                command
                    .arg( "--profile" ).arg( profile_to_arg( profile ) )
                    .arg( "--bin" ).arg( name.as_str() );
            },
            BuildTarget::ExampleBin( ref name ) => {
                command.arg( "--example" ).arg( name.as_str() );
            },
            BuildTarget::IntegrationTest( ref name ) => {
                command.arg( "--test" ).arg( name.as_str() );
            },
            BuildTarget::IntegrationBench( ref name ) => {
                command.arg( "--bench" ).arg( name.as_str() );
            }
        }

        if self.no_default_features {
            command.arg( "--no-default-features" );
        }

        if self.enable_all_features {
            command.arg( "--all-features" );
        }

        if !self.features.is_empty() {
            command.arg( "--features" );
            command.arg( &self.features.join( " " ) );
        }

        if self.is_verbose {
            command.arg( "--verbose" );
        }

        command
    }

    pub fn build< F >( &self, mut extra_artifact_generator: Option< F > ) -> CargoResult
        where F: for <'a> FnMut( &'a Path ) -> Vec< PathBuf >
    {
        let mut result = self.build_internal( &mut extra_artifact_generator );
        if result.is_ok() == false {
            return result;
        }

        // HACK: For some reason when you install emscripten for the first time
        // the first build is always a dud (it produces no artifacts), so we retry once.
        let is_emscripten = self.triplet.as_ref().map( |triplet| {
            triplet == "wasm32-unknown-emscripten" || triplet == "asmjs-unknown-emscripten"
        }).unwrap_or( false );

        if is_emscripten {
            let no_js_generated = result
                .artifacts()
                .iter()
                .find( |artifact| artifact.extension().map( |ext| ext == "js" ).unwrap_or( false ) )
                .is_none();

            if no_js_generated {
                debug!( "No artifacts were generated yet build succeeded; retrying..." );
                result = self.build_internal( &mut extra_artifact_generator );
            }
        }

        return result;
    }

    fn build_internal< F >( &self, extra_artifact_generator: &mut Option< F > ) -> CargoResult
        where F: for <'a> FnMut( &'a Path ) -> Vec< PathBuf >
    {
        let mut command = self.as_command();

        let env_paths = env::var_os( "PATH" )
            .map( |paths| env::split_paths( &paths ).collect() )
            .unwrap_or( Vec::new() );

        let mut paths = Vec::new();
        paths.extend( self.extra_paths.clone().into_iter() );
        paths.extend( env_paths.into_iter() );

        let new_paths = env::join_paths( paths ).unwrap();
        debug!( "Will launch cargo with PATH: {:?}", new_paths );
        command.env( "PATH", new_paths );

        let mut rustflags = env::var_os( "RUSTFLAGS" ).unwrap_or( OsString::new() );
        for flag in &self.extra_rustflags {
            if !rustflags.is_empty() {
                rustflags.push( " " );
            }
            rustflags.push( flag );
        }
        debug!( "Will launch cargo with RUSTFLAGS: {:?}", rustflags );
        command.env( "RUSTFLAGS", rustflags );

        for &(ref key, ref value) in &self.extra_environment {
            debug!( "Will launch cargo with variable \"{}\" set to \"{}\"", key, value );
            command.env( key, value );
        }

        command.stdout( Stdio::piped() );
        command.stderr( Stdio::piped() );

        debug!( "Launching cargo: {:?}", command );
        let mut child = match command.spawn() {
            Ok( child ) => child,
            Err( _ ) => {
                return CargoResult {
                    status: None,
                    artifacts: Vec::new()
                };
            }
        };

        let stderr = BufReader::new( child.stderr.take().unwrap() );
        let stdout = BufReader::new( child.stdout.take().unwrap() );

        let is_verbose = self.is_verbose;
        thread::spawn( move || {
            let mut skip = 0;
            for line in stderr.lines() {
                let line = match line {
                    Ok( line ) => line,
                    Err( _ ) => break
                };

                if skip > 0 {
                    skip -= 1;
                    continue;
                }

                // This is really ugly, so let's skip it.
                if line.trim() == "Caused by:" && !is_verbose {
                    skip += 1;
                    continue;
                }

                eprintln!( "{}", line );
            }
        });

        let mut artifacts = Vec::new();
        for line in stdout.lines() {
            let line = match line {
                Ok( line ) => line,
                Err( _ ) => break
            };

            let line = line.trim();
            if line.is_empty() {
                continue;
            }

            let json: serde_json::Value = serde_json::from_str( &line ).expect( "failed to parse cargo output" );
            let line = serde_json::to_string_pretty( &json ).unwrap();
            if let Some( output ) = CargoOutput::parse( &line ) {
                match output {
                    CargoOutput::Message( message ) => {
                        match self.message_format {
                            MessageFormat::Human => diagnostic_formatter::print( &message ),
                            MessageFormat::Json => {
                                println!( "{}", serde_json::to_string( &message.to_json_value() ).unwrap() );
                            }
                        }
                    },
                    CargoOutput::Artifact( artifact ) => {
                        for filename in &artifact.filenames {
                            debug!( "Built artifact: {}", filename );
                        }

                        artifacts.push( artifact );
                    },
                    CargoOutput::BuildScriptExecuted( executed ) => {
                        match self.message_format {
                            MessageFormat::Human => {},
                            MessageFormat::Json => {
                                println!( "{}", serde_json::to_string( &executed.to_json_value() ).unwrap() );
                            }
                        }
                    }
                }
            }
        }

        let result = child.wait();
        let status = result.unwrap().code().expect( "failed to grab cargo status code" );
        debug!( "Cargo finished with status: {}", status );

        fn has_extension< P: AsRef< Path > >( path: P, extension: &str ) -> bool {
            path.as_ref().extension().map( |ext| ext == extension ).unwrap_or( false )
        }

        fn find_artifact( artifacts: &[cargo_output::Artifact], extension: &str ) -> Option< (usize, usize) > {
            artifacts.iter().enumerate().filter_map( |(artifact_index, artifact)| {
                if let Some( filename_index ) = artifact.filenames.iter().position( |filename| has_extension( filename, extension ) ) {
                    Some( (artifact_index, filename_index) )
                } else {
                    None
                }
            }).next()
        }

        // For some reason when building tests cargo doesn't treat
        // the `.wasm` file as an artifact.
        if status == 0 && self.triplet.as_ref().map( |triplet| triplet == "wasm32-unknown-emscripten" ).unwrap_or( false ) {
            match self.build_target {
                BuildTarget::Bin( _, Profile::Test ) | BuildTarget::Lib( _, Profile::Test ) => {
                    if find_artifact( &artifacts, "wasm" ).is_none() {
                        if let Some( (artifact_index, filename_index) ) = find_artifact( &artifacts, "js" ) {
                            let wasm_path = {
                                let main_artifact = Path::new( &artifacts[ artifact_index ].filenames[ filename_index ] );
                                let filename = main_artifact.file_name().unwrap();
                                main_artifact.parent().unwrap().join( "deps" ).join( filename ).with_extension( "wasm" )
                            };

                            assert!( wasm_path.exists(), "internal error: wasm doesn't exist where I expected it to be" );
                            artifacts[ artifact_index ].filenames.push( wasm_path.to_str().unwrap().to_owned() );
                        }
                    }
                },
                _ => {}
            }
        }

        let mut artifact_paths = Vec::new();
        for mut artifact in artifacts {
            if let Some( ref mut callback ) = extra_artifact_generator.as_mut() {
                let mut extra_filenames = Vec::new();
                for filename in &artifact.filenames {
                    extra_filenames.extend(
                        callback( Path::new( &filename ) ).into_iter().map( |artifact| artifact.to_str().unwrap().to_owned() )
                    );
                }
                artifact.filenames.extend( extra_filenames );
            }

            match self.message_format {
                MessageFormat::Human => {},
                MessageFormat::Json => {
                    println!( "{}", serde_json::to_string( &artifact.to_json_value() ).unwrap() );
                }
            }

            for filename in artifact.filenames {
                // NOTE: Since we extract the paths from the JSON
                //       we get a list of artifacts as `String`s instead of `PathBuf`s.
                artifact_paths.push( filename.into() )
            }
        }

        CargoResult {
            status: Some( status ),
            artifacts: artifact_paths
        }
    }
}

pub struct CargoResult {
    status: Option< i32 >,
    artifacts: Vec< PathBuf >
}

impl CargoResult {
    pub fn is_ok( &self ) -> bool {
        self.status == Some( 0 )
    }

    pub fn artifacts( &self ) -> &[PathBuf] {
        &self.artifacts
    }
}
<|endoftext|>


<commit_before>use core::sync::atomic::Ordering;

use context::{arch, contexts, Context, Status, CONTEXT_ID};
use context::signal::signal_handler;
use gdt;
use interrupt;
use interrupt::irq::PIT_TICKS;
use time;

unsafe fn update(context: &mut Context, cpu_id: usize) {
    // Take ownership if not already owned
    if context.cpu_id == None {
        context.cpu_id = Some(cpu_id);
        // println!("{}: take {} {}", cpu_id, context.id, ::core::str::from_utf8_unchecked(&context.name.lock()));
    }

    // Restore from signal, must only be done from another context to avoid overwriting the stack!
    if context.ksig_restore && ! context.running {
        let ksig = context.ksig.take().expect("context::switch: ksig not set with ksig_restore");
        context.arch = ksig.0;

        if let Some(ref mut kfx) = context.kfx {
            kfx.clone_from_slice(&ksig.1.expect("context::switch: ksig kfx not set with ksig_restore"));
        } else {
            panic!("context::switch: kfx not set with ksig_restore");
        }

        if let Some(ref mut kstack) = context.kstack {
            kstack.clone_from_slice(&ksig.2.expect("context::switch: ksig kstack not set with ksig_restore"));
        } else {
            panic!("context::switch: kstack not set with ksig_restore");
        }

        context.ksig_restore = false;

        context.unblock();
    }

    // Unblock when there are pending signals
    if context.status == Status::Blocked && !context.pending.is_empty() {
        context.unblock();
    }

    // Wake from sleep
    if context.status == Status::Blocked && context.wake.is_some() {
        let wake = context.wake.expect("context::switch: wake not set");

        let current = time::monotonic();
        if current.0 > wake.0 || (current.0 == wake.0 && current.1 >= wake.1) {
            context.wake = None;
            context.unblock();
        }
    }
}

unsafe fn runnable(context: &Context, cpu_id: usize) -> bool {
    // Switch to context if it needs to run, is not currently running, and is owned by the current CPU
    !context.running && context.status == Status::Runnable && context.cpu_id == Some(cpu_id)
}

/// Switch to the next context
///
/// # Safety
///
/// Do not call this while holding locks!
pub unsafe fn switch() -> bool {
    use core::ops::DerefMut;

    //set PIT Interrupt counter to 0, giving each process same amount of PIT ticks
    PIT_TICKS.store(0, Ordering::SeqCst);

    // Set the global lock to avoid the unsafe operations below from causing issues
    while arch::CONTEXT_SWITCH_LOCK.compare_and_swap(false, true, Ordering::SeqCst) {
        interrupt::pause();
    }

    let cpu_id = ::cpu_id();

    let from_ptr;
    let mut to_ptr = 0 as *mut Context;
    let mut to_sig = None;
    {
        let contexts = contexts();
        {
            let context_lock = contexts
                .current()
                .expect("context::switch: not inside of context");
            let mut context = context_lock.write();
            from_ptr = context.deref_mut() as *mut Context;
        }

        for (pid, context_lock) in contexts.iter() {
            let mut context = context_lock.write();
            update(&mut context, cpu_id);
        }

        for (pid, context_lock) in contexts.iter() {
            if *pid > (*from_ptr).id {
                let mut context = context_lock.write();
                if runnable(&mut context, cpu_id) {
                    to_ptr = context.deref_mut() as *mut Context;
                    if (&mut *to_ptr).ksig.is_none() {
                        to_sig = context.pending.pop_front();
                    }
                    break;
                }
            }
        }

        if to_ptr as usize == 0 {
            for (pid, context_lock) in contexts.iter() {
                if *pid < (*from_ptr).id {
                    let mut context = context_lock.write();
                    if runnable(&mut context, cpu_id) {
                        to_ptr = context.deref_mut() as *mut Context;
                        if (&mut *to_ptr).ksig.is_none() {
                            to_sig = context.pending.pop_front();
                        }
                        break;
                    }
                }
            }
        }
    };

    // Switch process states, TSS stack pointer, and store new context ID
    if to_ptr as usize != 0 {
        (&mut *from_ptr).running = false;
        (&mut *to_ptr).running = true;
        if let Some(ref stack) = (*to_ptr).kstack {
            gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
        }
        CONTEXT_ID.store((&mut *to_ptr).id, Ordering::SeqCst);
    }

    // Unset global lock before switch, as arch is only usable by the current CPU at this time
    arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);

    if to_ptr as usize == 0 {
        // No target was found, return

        false
    } else {
        if let Some(sig) = to_sig {
            // Signal was found, run signal handler

            //TODO: Allow nested signals
            assert!((&mut *to_ptr).ksig.is_none());

            let arch = (&mut *to_ptr).arch.clone();
            let kfx = (&mut *to_ptr).kfx.clone();
            let kstack = (&mut *to_ptr).kstack.clone();
            (&mut *to_ptr).ksig = Some((arch, kfx, kstack));
            (&mut *to_ptr).arch.signal_stack(signal_handler, sig);
        }

        (&mut *from_ptr).arch.switch_to(&mut (&mut *to_ptr).arch);

        true
    }
}
<commit_msg>Fix warning in context::switch<commit_after>use core::sync::atomic::Ordering;

use context::{arch, contexts, Context, Status, CONTEXT_ID};
use context::signal::signal_handler;
use gdt;
use interrupt;
use interrupt::irq::PIT_TICKS;
use time;

unsafe fn update(context: &mut Context, cpu_id: usize) {
    // Take ownership if not already owned
    if context.cpu_id == None {
        context.cpu_id = Some(cpu_id);
        // println!("{}: take {} {}", cpu_id, context.id, ::core::str::from_utf8_unchecked(&context.name.lock()));
    }

    // Restore from signal, must only be done from another context to avoid overwriting the stack!
    if context.ksig_restore && ! context.running {
        let ksig = context.ksig.take().expect("context::switch: ksig not set with ksig_restore");
        context.arch = ksig.0;

        if let Some(ref mut kfx) = context.kfx {
            kfx.clone_from_slice(&ksig.1.expect("context::switch: ksig kfx not set with ksig_restore"));
        } else {
            panic!("context::switch: kfx not set with ksig_restore");
        }

        if let Some(ref mut kstack) = context.kstack {
            kstack.clone_from_slice(&ksig.2.expect("context::switch: ksig kstack not set with ksig_restore"));
        } else {
            panic!("context::switch: kstack not set with ksig_restore");
        }

        context.ksig_restore = false;

        context.unblock();
    }

    // Unblock when there are pending signals
    if context.status == Status::Blocked && !context.pending.is_empty() {
        context.unblock();
    }

    // Wake from sleep
    if context.status == Status::Blocked && context.wake.is_some() {
        let wake = context.wake.expect("context::switch: wake not set");

        let current = time::monotonic();
        if current.0 > wake.0 || (current.0 == wake.0 && current.1 >= wake.1) {
            context.wake = None;
            context.unblock();
        }
    }
}

unsafe fn runnable(context: &Context, cpu_id: usize) -> bool {
    // Switch to context if it needs to run, is not currently running, and is owned by the current CPU
    !context.running && context.status == Status::Runnable && context.cpu_id == Some(cpu_id)
}

/// Switch to the next context
///
/// # Safety
///
/// Do not call this while holding locks!
pub unsafe fn switch() -> bool {
    use core::ops::DerefMut;

    //set PIT Interrupt counter to 0, giving each process same amount of PIT ticks
    PIT_TICKS.store(0, Ordering::SeqCst);

    // Set the global lock to avoid the unsafe operations below from causing issues
    while arch::CONTEXT_SWITCH_LOCK.compare_and_swap(false, true, Ordering::SeqCst) {
        interrupt::pause();
    }

    let cpu_id = ::cpu_id();

    let from_ptr;
    let mut to_ptr = 0 as *mut Context;
    let mut to_sig = None;
    {
        let contexts = contexts();
        {
            let context_lock = contexts
                .current()
                .expect("context::switch: not inside of context");
            let mut context = context_lock.write();
            from_ptr = context.deref_mut() as *mut Context;
        }

        for (_pid, context_lock) in contexts.iter() {
            let mut context = context_lock.write();
            update(&mut context, cpu_id);
        }

        for (pid, context_lock) in contexts.iter() {
            if *pid > (*from_ptr).id {
                let mut context = context_lock.write();
                if runnable(&mut context, cpu_id) {
                    to_ptr = context.deref_mut() as *mut Context;
                    if (&mut *to_ptr).ksig.is_none() {
                        to_sig = context.pending.pop_front();
                    }
                    break;
                }
            }
        }

        if to_ptr as usize == 0 {
            for (pid, context_lock) in contexts.iter() {
                if *pid < (*from_ptr).id {
                    let mut context = context_lock.write();
                    if runnable(&mut context, cpu_id) {
                        to_ptr = context.deref_mut() as *mut Context;
                        if (&mut *to_ptr).ksig.is_none() {
                            to_sig = context.pending.pop_front();
                        }
                        break;
                    }
                }
            }
        }
    };

    // Switch process states, TSS stack pointer, and store new context ID
    if to_ptr as usize != 0 {
        (&mut *from_ptr).running = false;
        (&mut *to_ptr).running = true;
        if let Some(ref stack) = (*to_ptr).kstack {
            gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
        }
        CONTEXT_ID.store((&mut *to_ptr).id, Ordering::SeqCst);
    }

    // Unset global lock before switch, as arch is only usable by the current CPU at this time
    arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);

    if to_ptr as usize == 0 {
        // No target was found, return

        false
    } else {
        if let Some(sig) = to_sig {
            // Signal was found, run signal handler

            //TODO: Allow nested signals
            assert!((&mut *to_ptr).ksig.is_none());

            let arch = (&mut *to_ptr).arch.clone();
            let kfx = (&mut *to_ptr).kfx.clone();
            let kstack = (&mut *to_ptr).kstack.clone();
            (&mut *to_ptr).ksig = Some((arch, kfx, kstack));
            (&mut *to_ptr).arch.signal_stack(signal_handler, sig);
        }

        (&mut *from_ptr).arch.switch_to(&mut (&mut *to_ptr).arch);

        true
    }
}
<|endoftext|>


<commit_before>extern mod OpenCL;
use OpenCL::CL::*;
use OpenCL::hl::*;

fn main() {
    let platforms = get_platforms();

    error!("%?", platforms);

    do platforms.map |platform| { get_devices(platform);}
}
<commit_msg>fixed issue dereferencing pointer.<commit_after>extern mod OpenCL;
use OpenCL::CL::*;
use OpenCL::hl::*;

fn main() {
    let platforms = get_platforms();

    error!("%?", platforms);

    do platforms.map |platform| { get_devices(*platform); }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Implements parallel traversal over the DOM tree.
//!
//! This traversal is based on Rayon, and therefore its safety is largely
//! verified by the type system.
//!
//! The primary trickiness and fine print for the above relates to the
//! thread safety of the DOM nodes themselves. Accessing a DOM element
//! concurrently on multiple threads is actually mostly "safe", since all
//! the mutable state is protected by an AtomicRefCell, and so we'll
//! generally panic if something goes wrong. Still, we try to to enforce our
//! thread invariants at compile time whenever possible. As such, TNode and
//! TElement are not Send, so ordinary style system code cannot accidentally
//! share them with other threads. In the parallel traversal, we explicitly
//! invoke |unsafe { SendNode::new(n) }| to put nodes in containers that may
//! be sent to other threads. This occurs in only a handful of places and is
//! easy to grep for. At the time of this writing, there is no other unsafe
//! code in the parallel traversal.

#![deny(missing_docs)]

use context::TraversalStatistics;
use dom::{OpaqueNode, SendNode, TElement, TNode};
use rayon;
use scoped_tls::ScopedTLS;
use smallvec::SmallVec;
use std::borrow::Borrow;
use std::mem;
use time;
use traversal::{DomTraversal, PerLevelTraversalData, PreTraverseToken};

/// The maximum number of child nodes that we will process as a single unit.
///
/// Larger values will increase style sharing cache hits and general DOM locality
/// at the expense of decreased opportunities for parallelism. This value has not
/// been measured and could potentially be tuned.
pub const WORK_UNIT_MAX: usize = 16;

/// A list of node pointers.
///
/// We make the inline storage size WORK_UNIT_MAX so that we can collect chunks
/// into this structure without heap-allocating.
type NodeList<N> = SmallVec<[SendNode<N>; WORK_UNIT_MAX]>;

/// Entry point for the parallel traversal.
#[allow(unsafe_code)]
pub fn traverse_dom<E, D>(traversal: &D,
                          root: E,
                          token: PreTraverseToken,
                          pool: &rayon::ThreadPool)
    where E: TElement,
          D: DomTraversal<E>,
{
    let dump_stats = traversal.shared_context().options.dump_style_statistics;
    let start_time = if dump_stats { Some(time::precise_time_s()) } else { None };
    let mut nodes = NodeList::<E::ConcreteNode>::new();

    debug_assert!(traversal.is_parallel());
    // Handle Gecko's eager initial styling. We don't currently support it
    // in conjunction with bottom-up traversal. If we did, we'd need to put
    // it on the context to make it available to the bottom-up phase.
    let depth = if token.traverse_unstyled_children_only() {
        debug_assert!(!D::needs_postorder_traversal());
        for kid in root.as_node().traversal_children() {
            if kid.as_element().map_or(false, |el| el.get_data().is_none()) {
                nodes.push(unsafe { SendNode::new(kid) });
            }
        }
        root.depth() + 1
    } else {
        nodes.push(unsafe { SendNode::new(root.as_node()) });
        root.depth()
    };

    if nodes.is_empty() {
        return;
    }

    let traversal_data = PerLevelTraversalData {
        current_dom_depth: depth,
    };
    let tls = ScopedTLS::<D::ThreadLocalContext>::new(pool);
    let root = root.as_node().opaque();

    pool.install(|| {
        rayon::scope(|scope| {
            traverse_nodes(nodes,
                           DispatchMode::TailCall,
                           root,
                           traversal_data,
                           scope,
                           pool,
                           traversal,
                           &tls);
        });
    });

    // Dump statistics to stdout if requested.
    if dump_stats {
        let slots = unsafe { tls.unsafe_get() };
        let mut aggregate = slots.iter().fold(TraversalStatistics::default(), |acc, t| {
            match *t.borrow() {
                None => acc,
                Some(ref cx) => &cx.borrow().statistics + &acc,
            }
        });
        aggregate.finish(traversal, start_time.unwrap());
        if aggregate.is_large_traversal() {
            println!("{}", aggregate);
        }
    }
}

/// A callback to create our thread local context.  This needs to be
/// out of line so we don't allocate stack space for the entire struct
/// in the caller.
#[inline(never)]
fn create_thread_local_context<'scope, E, D>(
    traversal: &'scope D,
    slot: &mut Option<D::ThreadLocalContext>)
    where E: TElement + 'scope,
          D: DomTraversal<E>
{
    *slot = Some(traversal.create_thread_local_context())
}

/// A parallel top-down DOM traversal.
///
/// This algorithm traverses the DOM in a breadth-first, top-down manner. The
/// goals are:
/// * Never process a child before its parent (since child style depends on
///   parent style). If this were to happen, the styling algorithm would panic.
/// * Prioritize discovering nodes as quickly as possible to maximize
///   opportunities for parallelism.
/// * Style all the children of a given node (i.e. all sibling nodes) on
///   a single thread (with an upper bound to handle nodes with an
///   abnormally large number of children). This is important because we use
///   a thread-local cache to share styles between siblings.
#[inline(always)]
#[allow(unsafe_code)]
fn top_down_dom<'a, 'scope, E, D>(nodes: &'a [SendNode<E::ConcreteNode>],
                                  root: OpaqueNode,
                                  mut traversal_data: PerLevelTraversalData,
                                  scope: &'a rayon::Scope<'scope>,
                                  pool: &'scope rayon::ThreadPool,
                                  traversal: &'scope D,
                                  tls: &'scope ScopedTLS<'scope, D::ThreadLocalContext>)
    where E: TElement + 'scope,
          D: DomTraversal<E>,
{
    debug_assert!(nodes.len() <= WORK_UNIT_MAX);
    let mut discovered_child_nodes = NodeList::<E::ConcreteNode>::new();
    {
        // Scope the borrow of the TLS so that the borrow is dropped before
        // a potential recursive call when we pass TailCall.
        let mut tlc = tls.ensure(
            |slot: &mut Option<D::ThreadLocalContext>| create_thread_local_context(traversal, slot));

        for n in nodes {
            // If the last node we processed produced children, spawn them off
            // into a work item. We do this at the beginning of the loop (rather
            // than at the end) so that we can traverse the children of the last
            // sibling directly on this thread without a spawn call.
            //
            // This has the important effect of removing the allocation and
            // context-switching overhead of the parallel traversal for perfectly
            // linear regions of the DOM, i.e.:
            //
            // <russian><doll><tag><nesting></nesting></tag></doll></russian>
            //
            // Which are not at all uncommon.
            if !discovered_child_nodes.is_empty() {
                let children = mem::replace(&mut discovered_child_nodes, Default::default());
                let mut traversal_data_copy = traversal_data.clone();
                traversal_data_copy.current_dom_depth += 1;
                traverse_nodes(children,
                               DispatchMode::NotTailCall,
                               root,
                               traversal_data_copy,
                               scope,
                               pool,
                               traversal,
                               tls);
            }

            let node = **n;
            let mut children_to_process = 0isize;
            traversal.process_preorder(&traversal_data, &mut *tlc, node);
            if let Some(el) = node.as_element() {
                traversal.traverse_children(&mut *tlc, el, |_tlc, kid| {
                    children_to_process += 1;
                    discovered_child_nodes.push(unsafe { SendNode::new(kid) })
                });
            }

            traversal.handle_postorder_traversal(&mut *tlc, root, node,
                                                 children_to_process);
        }
    }

    // Handle the children of the last element in this work unit. If any exist,
    // we can process them (or at least one work unit's worth of them) directly
    // on this thread by passing TailCall.
    if !discovered_child_nodes.is_empty() {
        traversal_data.current_dom_depth += 1;
        traverse_nodes(discovered_child_nodes,
                       DispatchMode::TailCall,
                       root,
                       traversal_data,
                       scope,
                       pool,
                       traversal,
                       tls);
    }
}

/// Controls whether traverse_nodes may make a recursive call to continue
/// doing work, or whether it should always dispatch work asynchronously.
#[derive(Clone, Copy, PartialEq)]
enum DispatchMode {
    TailCall,
    NotTailCall,
}

impl DispatchMode {
    fn is_tail_call(&self) -> bool { matches!(*self, DispatchMode::TailCall) }
}

#[inline]
fn traverse_nodes<'a, 'scope, E, D>(nodes: NodeList<E::ConcreteNode>,
                                    mode: DispatchMode,
                                    root: OpaqueNode,
                                    traversal_data: PerLevelTraversalData,
                                    scope: &'a rayon::Scope<'scope>,
                                    pool: &'scope rayon::ThreadPool,
                                    traversal: &'scope D,
                                    tls: &'scope ScopedTLS<'scope, D::ThreadLocalContext>)
    where E: TElement + 'scope,
          D: DomTraversal<E>,
{
    debug_assert!(!nodes.is_empty());

    // This is a tail call from the perspective of the caller. However, we only
    // want to actually dispatch the job as a tail call if there's nothing left
    // in our local queue. Otherwise we need to return to it to maintain proper
    // breadth-first ordering.
    let may_dispatch_tail = mode.is_tail_call() &&
        !pool.current_thread_has_pending_tasks().unwrap();

    // In the common case, our children fit within a single work unit, in which
    // case we can pass the SmallVec directly and avoid extra allocation.
    if nodes.len() <= WORK_UNIT_MAX {
        if may_dispatch_tail {
            top_down_dom(&nodes, root, traversal_data, scope, pool, traversal, tls);
        } else {
            scope.spawn(move |scope| {
                let nodes = nodes;
                top_down_dom(&nodes, root, traversal_data, scope, pool, traversal, tls);
            });
        }
    } else {
        for chunk in nodes.chunks(WORK_UNIT_MAX) {
            let nodes = chunk.iter().cloned().collect::<NodeList<E::ConcreteNode>>();
            debug_assert!(!nodes.spilled());
            let traversal_data_copy = traversal_data.clone();
            scope.spawn(move |scope| {
                let n = nodes;
                top_down_dom(&*n, root, traversal_data_copy, scope, pool, traversal, tls)
            });
        }
    }
}
<commit_msg>Use ArrayVec and tweak the SmallVec sizes.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Implements parallel traversal over the DOM tree.
//!
//! This traversal is based on Rayon, and therefore its safety is largely
//! verified by the type system.
//!
//! The primary trickiness and fine print for the above relates to the
//! thread safety of the DOM nodes themselves. Accessing a DOM element
//! concurrently on multiple threads is actually mostly "safe", since all
//! the mutable state is protected by an AtomicRefCell, and so we'll
//! generally panic if something goes wrong. Still, we try to to enforce our
//! thread invariants at compile time whenever possible. As such, TNode and
//! TElement are not Send, so ordinary style system code cannot accidentally
//! share them with other threads. In the parallel traversal, we explicitly
//! invoke |unsafe { SendNode::new(n) }| to put nodes in containers that may
//! be sent to other threads. This occurs in only a handful of places and is
//! easy to grep for. At the time of this writing, there is no other unsafe
//! code in the parallel traversal.

#![deny(missing_docs)]

use arrayvec::ArrayVec;
use context::TraversalStatistics;
use dom::{OpaqueNode, SendNode, TElement, TNode};
use rayon;
use scoped_tls::ScopedTLS;
use smallvec::SmallVec;
use std::borrow::Borrow;
use std::mem;
use time;
use traversal::{DomTraversal, PerLevelTraversalData, PreTraverseToken};

/// The maximum number of child nodes that we will process as a single unit.
///
/// Larger values will increase style sharing cache hits and general DOM locality
/// at the expense of decreased opportunities for parallelism. This value has not
/// been measured and could potentially be tuned.
pub const WORK_UNIT_MAX: usize = 16;

/// A set of nodes, sized to the work unit. This gets copied when sent to other
/// threads, so we keep it compact.
type WorkUnit<N> = ArrayVec<[SendNode<N>; WORK_UNIT_MAX]>;

/// Entry point for the parallel traversal.
#[allow(unsafe_code)]
pub fn traverse_dom<E, D>(traversal: &D,
                          root: E,
                          token: PreTraverseToken,
                          pool: &rayon::ThreadPool)
    where E: TElement,
          D: DomTraversal<E>,
{
    let dump_stats = traversal.shared_context().options.dump_style_statistics;
    let start_time = if dump_stats { Some(time::precise_time_s()) } else { None };

    // Set up the SmallVec. We need to move this, and in most cases this is just
    // one node, so keep it small.
    let mut nodes = SmallVec::<[SendNode<E::ConcreteNode>; 8]>::new();

    debug_assert!(traversal.is_parallel());
    // Handle Gecko's eager initial styling. We don't currently support it
    // in conjunction with bottom-up traversal. If we did, we'd need to put
    // it on the context to make it available to the bottom-up phase.
    let depth = if token.traverse_unstyled_children_only() {
        debug_assert!(!D::needs_postorder_traversal());
        for kid in root.as_node().traversal_children() {
            if kid.as_element().map_or(false, |el| el.get_data().is_none()) {
                nodes.push(unsafe { SendNode::new(kid) });
            }
        }
        root.depth() + 1
    } else {
        nodes.push(unsafe { SendNode::new(root.as_node()) });
        root.depth()
    };

    if nodes.is_empty() {
        return;
    }

    let traversal_data = PerLevelTraversalData {
        current_dom_depth: depth,
    };
    let tls = ScopedTLS::<D::ThreadLocalContext>::new(pool);
    let root = root.as_node().opaque();

    pool.install(|| {
        rayon::scope(|scope| {
            let nodes = nodes;
            traverse_nodes(&*nodes,
                           DispatchMode::TailCall,
                           root,
                           traversal_data,
                           scope,
                           pool,
                           traversal,
                           &tls);
        });
    });

    // Dump statistics to stdout if requested.
    if dump_stats {
        let slots = unsafe { tls.unsafe_get() };
        let mut aggregate = slots.iter().fold(TraversalStatistics::default(), |acc, t| {
            match *t.borrow() {
                None => acc,
                Some(ref cx) => &cx.borrow().statistics + &acc,
            }
        });
        aggregate.finish(traversal, start_time.unwrap());
        if aggregate.is_large_traversal() {
            println!("{}", aggregate);
        }
    }
}

/// A callback to create our thread local context.  This needs to be
/// out of line so we don't allocate stack space for the entire struct
/// in the caller.
#[inline(never)]
fn create_thread_local_context<'scope, E, D>(
    traversal: &'scope D,
    slot: &mut Option<D::ThreadLocalContext>)
    where E: TElement + 'scope,
          D: DomTraversal<E>
{
    *slot = Some(traversal.create_thread_local_context())
}

/// A parallel top-down DOM traversal.
///
/// This algorithm traverses the DOM in a breadth-first, top-down manner. The
/// goals are:
/// * Never process a child before its parent (since child style depends on
///   parent style). If this were to happen, the styling algorithm would panic.
/// * Prioritize discovering nodes as quickly as possible to maximize
///   opportunities for parallelism.
/// * Style all the children of a given node (i.e. all sibling nodes) on
///   a single thread (with an upper bound to handle nodes with an
///   abnormally large number of children). This is important because we use
///   a thread-local cache to share styles between siblings.
#[inline(always)]
#[allow(unsafe_code)]
fn top_down_dom<'a, 'scope, E, D>(nodes: &'a [SendNode<E::ConcreteNode>],
                                  root: OpaqueNode,
                                  mut traversal_data: PerLevelTraversalData,
                                  scope: &'a rayon::Scope<'scope>,
                                  pool: &'scope rayon::ThreadPool,
                                  traversal: &'scope D,
                                  tls: &'scope ScopedTLS<'scope, D::ThreadLocalContext>)
    where E: TElement + 'scope,
          D: DomTraversal<E>,
{
    debug_assert!(nodes.len() <= WORK_UNIT_MAX);

    // Collect all the children of the elements in our work unit. This will
    // contain the combined children of up to WORK_UNIT_MAX nodes, which may
    // be numerous. As such, we store it in a large SmallVec to minimize heap-
    // spilling, and never move it.
    let mut discovered_child_nodes = SmallVec::<[SendNode<E::ConcreteNode>; 128]>::new();
    {
        // Scope the borrow of the TLS so that the borrow is dropped before
        // a potential recursive call when we pass TailCall.
        let mut tlc = tls.ensure(
            |slot: &mut Option<D::ThreadLocalContext>| create_thread_local_context(traversal, slot));

        for n in nodes {
            // If the last node we processed produced children, spawn them off
            // into a work item. We do this at the beginning of the loop (rather
            // than at the end) so that we can traverse the children of the last
            // sibling directly on this thread without a spawn call.
            //
            // This has the important effect of removing the allocation and
            // context-switching overhead of the parallel traversal for perfectly
            // linear regions of the DOM, i.e.:
            //
            // <russian><doll><tag><nesting></nesting></tag></doll></russian>
            //
            // Which are not at all uncommon.
            if !discovered_child_nodes.is_empty() {
                let children = mem::replace(&mut discovered_child_nodes, Default::default());
                let mut traversal_data_copy = traversal_data.clone();
                traversal_data_copy.current_dom_depth += 1;
                traverse_nodes(&*children,
                               DispatchMode::NotTailCall,
                               root,
                               traversal_data_copy,
                               scope,
                               pool,
                               traversal,
                               tls);
            }

            let node = **n;
            let mut children_to_process = 0isize;
            traversal.process_preorder(&traversal_data, &mut *tlc, node);
            if let Some(el) = node.as_element() {
                traversal.traverse_children(&mut *tlc, el, |_tlc, kid| {
                    children_to_process += 1;
                    discovered_child_nodes.push(unsafe { SendNode::new(kid) })
                });
            }

            traversal.handle_postorder_traversal(&mut *tlc, root, node,
                                                 children_to_process);
        }
    }

    // Handle the children of the last element in this work unit. If any exist,
    // we can process them (or at least one work unit's worth of them) directly
    // on this thread by passing TailCall.
    if !discovered_child_nodes.is_empty() {
        traversal_data.current_dom_depth += 1;
        traverse_nodes(&discovered_child_nodes,
                       DispatchMode::TailCall,
                       root,
                       traversal_data,
                       scope,
                       pool,
                       traversal,
                       tls);
    }
}

/// Controls whether traverse_nodes may make a recursive call to continue
/// doing work, or whether it should always dispatch work asynchronously.
#[derive(Clone, Copy, PartialEq)]
enum DispatchMode {
    TailCall,
    NotTailCall,
}

impl DispatchMode {
    fn is_tail_call(&self) -> bool { matches!(*self, DispatchMode::TailCall) }
}

#[inline]
fn traverse_nodes<'a, 'scope, E, D>(nodes: &[SendNode<E::ConcreteNode>],
                                    mode: DispatchMode,
                                    root: OpaqueNode,
                                    traversal_data: PerLevelTraversalData,
                                    scope: &'a rayon::Scope<'scope>,
                                    pool: &'scope rayon::ThreadPool,
                                    traversal: &'scope D,
                                    tls: &'scope ScopedTLS<'scope, D::ThreadLocalContext>)
    where E: TElement + 'scope,
          D: DomTraversal<E>,
{
    debug_assert!(!nodes.is_empty());

    // This is a tail call from the perspective of the caller. However, we only
    // want to actually dispatch the job as a tail call if there's nothing left
    // in our local queue. Otherwise we need to return to it to maintain proper
    // breadth-first ordering.
    let may_dispatch_tail = mode.is_tail_call() &&
        !pool.current_thread_has_pending_tasks().unwrap();

    // In the common case, our children fit within a single work unit, in which
    // case we can pass the SmallVec directly and avoid extra allocation.
    if nodes.len() <= WORK_UNIT_MAX {
        let work = nodes.iter().cloned().collect::<WorkUnit<E::ConcreteNode>>();
        if may_dispatch_tail {
            top_down_dom(&work, root, traversal_data, scope, pool, traversal, tls);
        } else {
            scope.spawn(move |scope| {
                let work = work;
                top_down_dom(&work, root, traversal_data, scope, pool, traversal, tls);
            });
        }
    } else {
        for chunk in nodes.chunks(WORK_UNIT_MAX) {
            let nodes = chunk.iter().cloned().collect::<WorkUnit<E::ConcreteNode>>();
            let traversal_data_copy = traversal_data.clone();
            scope.spawn(move |scope| {
                let n = nodes;
                top_down_dom(&*n, root, traversal_data_copy, scope, pool, traversal, tls)
            });
        }
    }
}
<|endoftext|>


<commit_before>use std::io::Result as IoResult;
use std::io::Write;
use nom::IError;

#[derive(Debug)]
enum Frequency {
    Optional,
    Repeated,
    Required,
}

#[derive(Debug)]
struct Field<'a> {
    name: &'a str,
    frequency: Frequency,
    typ: &'a str,
    number: i32,
    default: Option<&'a str>,
    packed: bool,
}

impl<'a> Field<'a> {

    fn rust_type(&self) -> &str {
        match self.typ {
            "int32" | "sint32" | "sfixed32" => "i32",
            "int64" | "sint64" | "sfixed64" => "i64",
            "uint32" | "fixed32" => "u32",
            "uint64" | "fixed64" => "u64",
            "float" => "f32",
            "double" => "f64",
            "string" => "String",
            "bytes" => "Vec<u8>",
            t => t,
        }
    }

    fn wire_type_num(&self, enums: &[&str]) -> u32 {
        match self.typ {
            "int32" | "sint32" | "int64" | "sint64" |
                "uint32" | "uint64" | "bool" | "enum" => 0,
            "fixed64" | "sfixed64" | "double" => 1,
            "fixed32" | "sfixed32" | "float" => 5,
            t if enums.contains(&t) => 0,
            _ => 2,
        }
    }

    fn read_fn(&self, enums: &[&str]) -> &str {
        match self.typ {
            "int32" | "sint32" | "int64" | "sint64" |
                "uint32" | "uint64" | "bool" | "fixed64" |
                "sfixed64" | "double" | "fixed32" | "sfixed32" |
                "float" | "string" | "bytes" => self.typ,
            t if enums.contains(&t) => "enum",
            _ => "message",
        }
    }

    fn tag(&self, enums: &[&str]) -> u32 {
        (self.number as u32) << 3 | self.wire_type_num(enums)
    }

    fn write_definition<W: Write>(&self, w: &mut W) -> IoResult<()> {
        match self.frequency {
            Frequency::Optional => writeln!(w, "    pub {}: Option<{}>,", self.name, self.rust_type()),
            Frequency::Repeated => writeln!(w, "    pub {}: Vec<{}>,", self.name, self.rust_type()),
            Frequency::Required => writeln!(w, "    pub {}: {},", self.name, self.rust_type()),
        }
    }

    fn write_match_tag<W: Write>(&self, w: &mut W, enums: &[&str]) -> IoResult<()> {
        match self.frequency {
            Frequency::Optional => writeln!(w, "{} => msg.{} = Some(r.read_{}()?),",
                                            self.tag(enums), self.name, self.read_fn(enums)),
            Frequency::Repeated => {
                if self.packed {
                    writeln!(w, "{} => msg.{} = r.read_packed_repeated_field(|r| r.read_{}())?,",
                             self.tag(enums), self.name, self.read_fn(enums))
                } else {
                    writeln!(w, "{} => msg.{}.push(r.read_{}()?),",
                             self.tag(enums), self.name, self.read_fn(enums))
                }
            }
            Frequency::Required => writeln!(w, "{} => msg.{} = r.read_{}()?,",
                                            self.tag(enums), self.name, self.read_fn(enums)),
        }
    }
}

#[derive(Debug)]
pub struct Message<'a> {
    name: &'a str,
    fields: Vec<Field<'a>>,
}

impl<'a> Message<'a> {
    fn write_definition<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "#[derive(Debug, Default, PartialEq, Clone)]")?;
        writeln!(w, "pub struct {} {{", self.name)?;
        for f in &self.fields {
            f.write_definition(w)?;
        }
        writeln!(w, "}}")
    }

    fn write_impl_message<W: Write>(&self, w: &mut W, enums: &[&str]) -> IoResult<()> {
        writeln!(w, "impl Message for {} {{", self.name)?;
        writeln!(w, "    fn from_reader<R: Read>(mut r: &mut Reader<R>) -> Result<Self> {{")?;
        writeln!(w, "        let mut msg = Self::default();")?;
        writeln!(w, "        loop {{")?;
        writeln!(w, "            let tag = match r.next_tag_value() {{")?;
        writeln!(w, "                None => break,")?;
        writeln!(w, "                Some(Err(e)) => return Err(e),")?;
        writeln!(w, "                Some(Ok(tag)) => tag,")?;
        writeln!(w, "            }};")?;
        writeln!(w, "            match tag {{")?;
        for f in &self.fields {
            write!(w, "                ")?;
            f.write_match_tag(w, enums)?;
        }
        writeln!(w, "                t => {{ r.read_unknown(t)?; }}")?;
        writeln!(w, "            }}")?;
        writeln!(w, "        }}")?;
        writeln!(w, "        Ok(msg)")?;
        writeln!(w, "    }}")?;
        writeln!(w, "}}")
    }
}

#[derive(Debug)]
pub struct Enumerator<'a> {
    name: &'a str,
    fields: Vec<(&'a str, i32)>,
}

impl<'a> Enumerator<'a> {
    fn write_definition<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "#[derive(Debug, PartialEq, Eq, Clone)]")?;
        writeln!(w, "pub enum {} {{", self.name)?;
        for &(f, _) in &self.fields {
            writeln!(w, "    {},", f)?;
        }
        writeln!(w, "}}")
    }

    fn write_impl_default<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "impl Default for {} {{", self.name)?;
        writeln!(w, "    fn default() -> Self {{")?;
        // TODO: check with default field and return error if there is no field
        writeln!(w, "        {}::{}", self.name, self.fields[0].0)?;
        writeln!(w, "    }}")?;
        writeln!(w, "}}")
    }

    fn write_from_u64<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "impl From<u64> for {} {{", self.name)?;
        writeln!(w, "    fn from(i: u64) -> Self {{")?;
        writeln!(w, "        match i {{")?;
        for &(f, number) in &self.fields {
            writeln!(w, "            {} => {}::{},", number, self.name, f)?;
        }
        writeln!(w, "            _ => Self::default(),")?;
        writeln!(w, "        }}")?;
        writeln!(w, "    }}")?;
        writeln!(w, "}}")
    }
}

#[derive(Debug)]
pub enum MessageOrEnum<'a> {
    Msg(Message<'a>),
    Enum(Enumerator<'a>),
    Ignore,
}

#[derive(Debug)]
pub struct FileDescriptor<'a> {
    message_and_enums: Vec<MessageOrEnum<'a>>,
}

impl<'a> FileDescriptor<'a> {
    pub fn from_str(f: &'a str) -> Result<FileDescriptor<'a>, IError<u32>> {
        FileDescriptor::from_bytes(f.as_bytes())

    }
    pub fn from_bytes(b: &'a [u8]) -> Result<FileDescriptor<'a>, IError<u32>> {
        self::parser::file_descriptor(b).to_full_result()
    }

    pub fn write<W: Write>(&self, w: &mut W, filename: &str) -> IoResult<()> {

        writeln!(w, "//! Automatically generated rust module for '{}' file", filename)?;
        writeln!(w, "")?;
        writeln!(w, "#![allow(non_snake_case)]")?;
        writeln!(w, "#![allow(non_upper_case_globals)]")?;
        writeln!(w, "")?;
        writeln!(w, "use std::io::Read;")?;
        writeln!(w, "use quick_protobuf::errors::Result;")?;
        writeln!(w, "use quick_protobuf::message::Message;")?;
        writeln!(w, "use quick_protobuf::reader::Reader;")?;

        let enums = self.message_and_enums.iter().filter_map(|m| {
            if let &MessageOrEnum::Enum(ref e) = m {
                Some(e.name)
            } else {
                None
            }
        }).collect::<Vec<_>>();
        for m in &self.message_and_enums {
            match m {
                &MessageOrEnum::Msg(ref m) => {
                    writeln!(w, "")?;
                    m.write_definition(w)?;
                    writeln!(w, "")?;
                    m.write_impl_message(w, &enums)?;
                },
                &MessageOrEnum::Enum(ref m) => {
                    writeln!(w, "")?;
                    m.write_definition(w)?;
                    writeln!(w, "")?;
                    m.write_impl_default(w)?;
                    writeln!(w, "")?;
                    m.write_from_u64(w)?;
                },
                &MessageOrEnum::Ignore => continue,
            }
        }
        Ok(())
    }
}

mod parser {

    use std::str;
    use parser::{Frequency, Field, Message, Enumerator, MessageOrEnum, FileDescriptor};
    use nom::{alphanumeric, multispace, digit};

    named!(comment<()>, do_parse!(tag!("//") >> take_until_and_consume!("\n") >> ()));

    /// word break: multispace or comment
    named!(br<()>, alt!(map!(multispace, |_| ()) | comment));

    named!(default_value<&str>, do_parse!(
        tag!("[") >> many0!(br) >> tag!("default") >> many0!(br) >> tag!("=") >> many0!(br) >>
        default: map_res!(alphanumeric, str::from_utf8) >> many0!(br) >> tag!("]") >>
        (default)));

    named!(packed<bool>, do_parse!(
        tag!("[") >> many0!(br) >> tag!("packed") >> many0!(br) >> tag!("=") >> many0!(br) >>
        packed: map_res!(map_res!(alphanumeric, str::from_utf8), str::FromStr::from_str) >> many0!(br) >> tag!("]") >>
        (packed)));

    named!(frequency<Frequency>,
           alt!(tag!("optional") => { |_| Frequency::Optional } |
                tag!("repeated") => { |_| Frequency::Repeated } |
                tag!("required") => { |_| Frequency::Required } ));

    named!(message_field<Field>, do_parse!(
        frequency: frequency >> many1!(br) >>
        typ: map_res!(alphanumeric, str::from_utf8) >> many1!(br) >>
        name: map_res!(alphanumeric, str::from_utf8) >> many0!(br) >>
        tag!("=") >> many0!(br) >>
        number: map_res!(map_res!(digit, str::from_utf8), str::FromStr::from_str) >> many0!(br) >>
        default: opt!(default_value) >> many0!(br) >>
        packed: opt!(packed) >> many0!(br) >> tag!(";") >> many0!(br) >>
        (Field {
           name: name,
           frequency: frequency,
           typ: typ,
           number: number,
           default: default,
           packed: packed.unwrap_or(false),
        })));

    named!(message<Message>, do_parse!(
        tag!("message") >> many0!(br) >>
        name: map_res!(alphanumeric, str::from_utf8) >> many0!(br) >>
        tag!("{") >> many0!(br) >>
        fields: many0!(message_field) >>
        tag!("}") >> many0!(br) >>
        (Message { name: name, fields: fields })));

    named!(enum_field<(&str, i32)>, do_parse!(
        name: map_res!(alphanumeric, str::from_utf8) >> many0!(br) >>
        tag!("=") >> many0!(br) >>
        number: map_res!(map_res!(digit, str::from_utf8), str::FromStr::from_str) >> many0!(br) >>
        tag!(";") >> many0!(br) >>
        ((name, number))));

    named!(enumerator<Enumerator>, do_parse!(
        tag!("enum") >> many1!(br) >>
        name: map_res!(alphanumeric, str::from_utf8) >> many0!(br) >>
        tag!("{") >> many0!(br) >>
        fields: many0!(enum_field) >>
        tag!("}") >> many0!(br) >>
        (Enumerator { name: name, fields: fields })));

    named!(ignore<()>, do_parse!(
        alt!(tag!("package") | tag!("option")) >> many1!(br) >>
        take_until_and_consume!(";") >> many0!(br) >> ()));

    named!(message_or_enum<MessageOrEnum>, alt!(
             message => { |m| MessageOrEnum::Msg(m) } |
             enumerator => { |e| MessageOrEnum::Enum(e) } |
             ignore => { |_| MessageOrEnum::Ignore } ));

    named!(pub file_descriptor<FileDescriptor>, do_parse!(
        many0!(br) >>
        message_and_enums: many0!(message_or_enum) >>
        (FileDescriptor {
            message_and_enums: message_and_enums
        })));

    #[test]
    fn test_message() {
        let msg = r#"message ReferenceData
    {
        repeated ScenarioInfo  scenarioSet = 1;
        repeated CalculatedObjectInfo calculatedObjectSet = 2;
        repeated RiskFactorList riskFactorListSet = 3;
        repeated RiskMaturityInfo riskMaturitySet = 4;
        repeated IndicatorInfo indicatorSet = 5;
        repeated RiskStrikeInfo riskStrikeSet = 6;
        repeated FreeProjectionList freeProjectionListSet = 7;
        repeated ValidationProperty ValidationSet = 8;
        repeated CalcProperties calcPropertiesSet = 9;
        repeated MaturityInfo maturitySet = 10;
    }"#;

        let mess = message(msg.as_bytes());
        if let ::nom::IResult::Done(_, mess) = mess {
            assert_eq!(10, mess.fields.len());
        }
    }

    #[test]
    fn test_enum() {
        let msg = r#"enum PairingStatus {
                DEALPAIRED        = 0;
                INVENTORYORPHAN   = 1;
                CALCULATEDORPHAN  = 2;
                CANCELED          = 3;
    }"#;

        let mess = enumerator(msg.as_bytes());
        if let ::nom::IResult::Done(_, mess) = mess {
            assert_eq!(4, mess.fields.len());
        }
    }

    #[test]
    fn test_ignore() {
        let msg = r#"package com.test.v0;

    option optimize_for = SPEED;
    "#;

        match ignore(msg.as_bytes()) {
            ::nom::IResult::Done(_, _) => (),
            e => panic!("Expecting done {:?}", e),
        }
    }
}
<commit_msg>recognize underscores in words<commit_after>use std::io::Result as IoResult;
use std::io::Write;
use nom::IError;

#[derive(Debug)]
enum Frequency {
    Optional,
    Repeated,
    Required,
}

#[derive(Debug)]
struct Field<'a> {
    name: &'a str,
    frequency: Frequency,
    typ: &'a str,
    number: i32,
    default: Option<&'a str>,
    packed: bool,
}

impl<'a> Field<'a> {

    fn rust_type(&self) -> &str {
        match self.typ {
            "int32" | "sint32" | "sfixed32" => "i32",
            "int64" | "sint64" | "sfixed64" => "i64",
            "uint32" | "fixed32" => "u32",
            "uint64" | "fixed64" => "u64",
            "float" => "f32",
            "double" => "f64",
            "string" => "String",
            "bytes" => "Vec<u8>",
            t => t,
        }
    }

    fn wire_type_num(&self, enums: &[&str]) -> u32 {
        match self.typ {
            "int32" | "sint32" | "int64" | "sint64" |
                "uint32" | "uint64" | "bool" | "enum" => 0,
            "fixed64" | "sfixed64" | "double" => 1,
            "fixed32" | "sfixed32" | "float" => 5,
            t if enums.contains(&t) => 0,
            _ => 2,
        }
    }

    fn read_fn(&self, enums: &[&str]) -> &str {
        match self.typ {
            "int32" | "sint32" | "int64" | "sint64" |
                "uint32" | "uint64" | "bool" | "fixed64" |
                "sfixed64" | "double" | "fixed32" | "sfixed32" |
                "float" | "string" | "bytes" => self.typ,
            t if enums.contains(&t) => "enum",
            _ => "message",
        }
    }

    fn tag(&self, enums: &[&str]) -> u32 {
        (self.number as u32) << 3 | self.wire_type_num(enums)
    }

    fn write_definition<W: Write>(&self, w: &mut W) -> IoResult<()> {
        match self.frequency {
            Frequency::Optional => writeln!(w, "    pub {}: Option<{}>,", self.name, self.rust_type()),
            Frequency::Repeated => writeln!(w, "    pub {}: Vec<{}>,", self.name, self.rust_type()),
            Frequency::Required => writeln!(w, "    pub {}: {},", self.name, self.rust_type()),
        }
    }

    fn write_match_tag<W: Write>(&self, w: &mut W, enums: &[&str]) -> IoResult<()> {
        match self.frequency {
            Frequency::Optional => writeln!(w, "{} => msg.{} = Some(r.read_{}()?),",
                                            self.tag(enums), self.name, self.read_fn(enums)),
            Frequency::Repeated => {
                if self.packed {
                    writeln!(w, "{} => msg.{} = r.read_packed_repeated_field(|r| r.read_{}())?,",
                             self.tag(enums), self.name, self.read_fn(enums))
                } else {
                    writeln!(w, "{} => msg.{}.push(r.read_{}()?),",
                             self.tag(enums), self.name, self.read_fn(enums))
                }
            }
            Frequency::Required => writeln!(w, "{} => msg.{} = r.read_{}()?,",
                                            self.tag(enums), self.name, self.read_fn(enums)),
        }
    }
}

#[derive(Debug)]
pub struct Message<'a> {
    name: &'a str,
    fields: Vec<Field<'a>>,
}

impl<'a> Message<'a> {
    fn write_definition<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "#[derive(Debug, Default, PartialEq, Clone)]")?;
        writeln!(w, "pub struct {} {{", self.name)?;
        for f in &self.fields {
            f.write_definition(w)?;
        }
        writeln!(w, "}}")
    }

    fn write_impl_message<W: Write>(&self, w: &mut W, enums: &[&str]) -> IoResult<()> {
        writeln!(w, "impl Message for {} {{", self.name)?;
        writeln!(w, "    fn from_reader<R: Read>(mut r: &mut Reader<R>) -> Result<Self> {{")?;
        writeln!(w, "        let mut msg = Self::default();")?;
        writeln!(w, "        loop {{")?;
        writeln!(w, "            let tag = match r.next_tag_value() {{")?;
        writeln!(w, "                None => break,")?;
        writeln!(w, "                Some(Err(e)) => return Err(e),")?;
        writeln!(w, "                Some(Ok(tag)) => tag,")?;
        writeln!(w, "            }};")?;
        writeln!(w, "            match tag {{")?;
        for f in &self.fields {
            write!(w, "                ")?;
            f.write_match_tag(w, enums)?;
        }
        writeln!(w, "                t => {{ r.read_unknown(t)?; }}")?;
        writeln!(w, "            }}")?;
        writeln!(w, "        }}")?;
        writeln!(w, "        Ok(msg)")?;
        writeln!(w, "    }}")?;
        writeln!(w, "}}")
    }
}

#[derive(Debug)]
pub struct Enumerator<'a> {
    name: &'a str,
    fields: Vec<(&'a str, i32)>,
}

impl<'a> Enumerator<'a> {
    fn write_definition<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "#[derive(Debug, PartialEq, Eq, Clone)]")?;
        writeln!(w, "pub enum {} {{", self.name)?;
        for &(f, _) in &self.fields {
            writeln!(w, "    {},", f)?;
        }
        writeln!(w, "}}")
    }

    fn write_impl_default<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "impl Default for {} {{", self.name)?;
        writeln!(w, "    fn default() -> Self {{")?;
        // TODO: check with default field and return error if there is no field
        writeln!(w, "        {}::{}", self.name, self.fields[0].0)?;
        writeln!(w, "    }}")?;
        writeln!(w, "}}")
    }

    fn write_from_u64<W: Write>(&self, w: &mut W) -> IoResult<()> {
        writeln!(w, "impl From<u64> for {} {{", self.name)?;
        writeln!(w, "    fn from(i: u64) -> Self {{")?;
        writeln!(w, "        match i {{")?;
        for &(f, number) in &self.fields {
            writeln!(w, "            {} => {}::{},", number, self.name, f)?;
        }
        writeln!(w, "            _ => Self::default(),")?;
        writeln!(w, "        }}")?;
        writeln!(w, "    }}")?;
        writeln!(w, "}}")
    }
}

#[derive(Debug)]
pub enum MessageOrEnum<'a> {
    Msg(Message<'a>),
    Enum(Enumerator<'a>),
    Ignore,
}

#[derive(Debug)]
pub struct FileDescriptor<'a> {
    message_and_enums: Vec<MessageOrEnum<'a>>,
}

impl<'a> FileDescriptor<'a> {
    pub fn from_str(f: &'a str) -> Result<FileDescriptor<'a>, IError<u32>> {
        FileDescriptor::from_bytes(f.as_bytes())

    }
    pub fn from_bytes(b: &'a [u8]) -> Result<FileDescriptor<'a>, IError<u32>> {
        self::parser::file_descriptor(b).to_full_result()
    }

    pub fn write<W: Write>(&self, w: &mut W, filename: &str) -> IoResult<()> {

        println!("Found {} messages, enums or ignored expressions (imports ...)", self.message_and_enums.len());

        writeln!(w, "//! Automatically generated rust module for '{}' file", filename)?;
        writeln!(w, "")?;
        writeln!(w, "#![allow(non_snake_case)]")?;
        writeln!(w, "#![allow(non_upper_case_globals)]")?;
        writeln!(w, "")?;
        writeln!(w, "use std::io::Read;")?;
        writeln!(w, "use quick_protobuf::errors::Result;")?;
        writeln!(w, "use quick_protobuf::message::Message;")?;
        writeln!(w, "use quick_protobuf::reader::Reader;")?;

        let enums = self.message_and_enums.iter().filter_map(|m| {
            if let &MessageOrEnum::Enum(ref e) = m {
                Some(e.name)
            } else {
                None
            }
        }).collect::<Vec<_>>();
        for m in &self.message_and_enums {
            match m {
                &MessageOrEnum::Msg(ref m) => {
                    writeln!(w, "")?;
                    m.write_definition(w)?;
                    writeln!(w, "")?;
                    m.write_impl_message(w, &enums)?;
                },
                &MessageOrEnum::Enum(ref m) => {
                    writeln!(w, "")?;
                    m.write_definition(w)?;
                    writeln!(w, "")?;
                    m.write_impl_default(w)?;
                    writeln!(w, "")?;
                    m.write_from_u64(w)?;
                },
                &MessageOrEnum::Ignore => continue,
            }
        }
        Ok(())
    }
}

mod parser {

    use std::str;
    use parser::{Frequency, Field, Message, Enumerator, MessageOrEnum, FileDescriptor};
    use nom::{multispace, digit};

    fn is_word(b: u8) -> bool {
        match b {
            b'a'...b'z' | b'A'...b'Z' | b'0'...b'9' | b'_' => true,
            _ => false
        }
    }

    named!(word<&str>, map_res!(take_while!(is_word), str::from_utf8));

    named!(comment<()>, do_parse!(tag!("//") >> take_until_and_consume!("\n") >> ()));

    /// word break: multispace or comment
    named!(br<()>, alt!(map!(multispace, |_| ()) | comment));

    named!(default_value<&str>, do_parse!(
        tag!("[") >> many0!(br) >> tag!("default") >> many0!(br) >> tag!("=") >> many0!(br) >>
        default: word >> many0!(br) >> tag!("]") >>
        (default)));

    named!(packed<bool>, do_parse!(
        tag!("[") >> many0!(br) >> tag!("packed") >> many0!(br) >> tag!("=") >> many0!(br) >>
        packed: map_res!(word, str::FromStr::from_str) >> many0!(br) >> tag!("]") >>
        (packed)));

    named!(frequency<Frequency>,
           alt!(tag!("optional") => { |_| Frequency::Optional } |
                tag!("repeated") => { |_| Frequency::Repeated } |
                tag!("required") => { |_| Frequency::Required } ));

    named!(message_field<Field>, do_parse!(
        frequency: frequency >> many1!(br) >>
        typ: word >> many1!(br) >>
        name: word >> many0!(br) >>
        tag!("=") >> many0!(br) >>
        number: map_res!(map_res!(digit, str::from_utf8), str::FromStr::from_str) >> many0!(br) >>
        default: opt!(default_value) >> many0!(br) >>
        packed: opt!(packed) >> many0!(br) >> tag!(";") >> many0!(br) >>
        (Field {
           name: name,
           frequency: frequency,
           typ: typ,
           number: number,
           default: default,
           packed: packed.unwrap_or(false),
        })));

    named!(message<Message>, do_parse!(
        tag!("message") >> many0!(br) >>
        name: word >> many0!(br) >>
        tag!("{") >> many0!(br) >>
        fields: many0!(message_field) >>
        tag!("}") >> many0!(br) >>
        (Message { name: name, fields: fields })));

    named!(enum_field<(&str, i32)>, do_parse!(
        name: word >> many0!(br) >>
        tag!("=") >> many0!(br) >>
        number: map_res!(map_res!(digit, str::from_utf8), str::FromStr::from_str) >> many0!(br) >>
        tag!(";") >> many0!(br) >>
        ((name, number))));

    named!(enumerator<Enumerator>, do_parse!(
        tag!("enum") >> many1!(br) >>
        name: word >> many0!(br) >>
        tag!("{") >> many0!(br) >>
        fields: many0!(enum_field) >>
        tag!("}") >> many0!(br) >>
        (Enumerator { name: name, fields: fields })));

    named!(ignore<()>, do_parse!(
        alt!(tag!("package") | tag!("option")) >> many1!(br) >>
        take_until_and_consume!(";") >> many0!(br) >> ()));

    named!(message_or_enum<MessageOrEnum>, alt!(
             message => { |m| MessageOrEnum::Msg(m) } |
             enumerator => { |e| MessageOrEnum::Enum(e) } |
             ignore => { |_| MessageOrEnum::Ignore } ));

    named!(pub file_descriptor<FileDescriptor>, do_parse!(
        many0!(br) >>
        message_and_enums: many0!(message_or_enum) >>
        (FileDescriptor {
            message_and_enums: message_and_enums
        })));

    #[test]
    fn test_message() {
        let msg = r#"message ReferenceData
    {
        repeated ScenarioInfo  scenarioSet = 1;
        repeated CalculatedObjectInfo calculatedObjectSet = 2;
        repeated RiskFactorList riskFactorListSet = 3;
        repeated RiskMaturityInfo riskMaturitySet = 4;
        repeated IndicatorInfo indicatorSet = 5;
        repeated RiskStrikeInfo riskStrikeSet = 6;
        repeated FreeProjectionList freeProjectionListSet = 7;
        repeated ValidationProperty ValidationSet = 8;
        repeated CalcProperties calcPropertiesSet = 9;
        repeated MaturityInfo maturitySet = 10;
    }"#;

        let mess = message(msg.as_bytes());
        if let ::nom::IResult::Done(_, mess) = mess {
            assert_eq!(10, mess.fields.len());
        }
    }

    #[test]
    fn test_enum() {
        let msg = r#"enum PairingStatus {
                DEALPAIRED        = 0;
                INVENTORYORPHAN   = 1;
                CALCULATEDORPHAN  = 2;
                CANCELED          = 3;
    }"#;

        let mess = enumerator(msg.as_bytes());
        if let ::nom::IResult::Done(_, mess) = mess {
            assert_eq!(4, mess.fields.len());
        }
    }

    #[test]
    fn test_ignore() {
        let msg = r#"package com.test.v0;

    option optimize_for = SPEED;
    "#;

        match ignore(msg.as_bytes()) {
            ::nom::IResult::Done(_, _) => (),
            e => panic!("Expecting done {:?}", e),
        }
    }
}
<|endoftext|>