<commit_before>extern crate cargo;
extern crate reqwest;
#[macro_use]
extern crate lazy_static;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
extern crate toml;

use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::btree_map::Entry;
use std::fs::File;
use std::io::{Read, Write};

use cargo::core::{Dependency, Package, Source, SourceId, Summary};
use cargo::core::registry::PackageRegistry;
use cargo::core::resolver::{self, Method, Resolve};
use cargo::sources::RegistrySource;
use cargo::util::Config;

/// The list of crates from crates.io
#[derive(Debug, Deserialize)]
struct TopCrates {
    crates: Vec<Crate>,
}

/// A single crate from crates.io
#[derive(Debug, Deserialize)]
struct OneCrate {
    #[serde(rename="crate")]
    krate: Crate,
}

/// The shared description of a crate
#[derive(Debug, Deserialize)]
struct Crate {
    #[serde(rename="id")]
    name: String,
}

/// A Cargo.toml file.
#[derive(Serialize)]
struct TomlManifest {
    package: TomlPackage,
    profile: Profiles,
    #[serde(serialize_with = "toml::ser::tables_last")]
    dependencies: BTreeMap<String, DependencySpec>,
}

/// Header of Cargo.toml file.
#[derive(Serialize)]
struct TomlPackage {
    name: String,
    version: String,
    authors: Vec<String>,
}

/// A mapping of a crates name to its identifier used in source code
#[derive(Debug, Serialize)]
struct CrateInformation {
    name: String,
    version: String,
    id: String,
}

/// Hand-curated changes to the crate list
#[derive(Debug, Deserialize)]
struct Modifications {
    #[serde(default)]
    blacklist: Vec<String>,
    #[serde(default)]
    additions: BTreeSet<String>,
}

/// A profile section in a Cargo.toml file
#[derive(Serialize)]
#[serde(rename_all="kebab-case")]
struct Profile {
    codegen_units: u32,
    incremental: bool,
}

/// Available profile types
#[derive(Serialize)]
struct Profiles {
    dev: Profile,
    release: Profile,
}

/// `"1.0.0"` or `{ version = "1.0.0", features = ["..."] }`
#[derive(Serialize, Clone)]
#[serde(untagged)]
enum DependencySpec {
    String(String),
    #[serde(rename_all = "kebab-case")]
    Explicit {
        version: String,
        #[serde(skip_serializing_if = "Vec::is_empty")]
        features: Vec<String>,
        #[serde(skip_serializing_if = "is_true")]
        default_features: bool,
    }
}

impl DependencySpec {
    fn version(&self) -> String {
        match *self {
            DependencySpec::String(ref version) |
            DependencySpec::Explicit { ref version, .. } => version.clone(),
        }
    }
}

fn is_true(b: &bool) -> bool {
    *b
}

impl Modifications {
    fn blacklisted(&self, name: &str) -> bool {
        self.blacklist.iter().any(|n| n == name)
    }
}

lazy_static! {
    static ref MODIFICATIONS: Modifications = {
        let mut f = File::open("crate-modifications.toml")
            .expect("unable to open crate modifications file");

        let mut d = Vec::new();
        f.read_to_end(&mut d)
            .expect("unable to read crate modifications file");

        toml::from_slice(&d)
            .expect("unable to parse crate modifications file")
    };
}

impl TopCrates {
    /// List top 100 crates by number of downloads on crates.io.
    fn download() -> TopCrates {
        let resp =
            reqwest::get("https://crates.io/api/v1/crates?page=1&per_page=100&sort=downloads")
            .expect("Could not fetch top crates");
        assert!(resp.status().is_success());

        serde_json::from_reader(resp).expect("Invalid JSON")
    }

    /// Add crates that have been hand-picked
    fn add_curated_crates(&mut self) {
        self.crates.extend({
            MODIFICATIONS
                .additions
                .iter()
                .cloned()
                .map(|name| Crate { name })
        });
    }
}

fn decide_features(summary: &Summary) -> Method<'static> {
    // Enable `playground` feature if present.
    if summary.features().contains_key("playground") {
        // Would put this in a lazy_static, but its type is Vec<InternedString>
        // and InternedString is not publicly nameable.
        let features = Method::split_features(&["playground".to_owned()]);

        Method::Required {
            dev_deps: false,
            features: Box::leak(Box::new(features)),
            uses_default_features: false,
            all_features: false,
        }
    } else {
        Method::Required {
            dev_deps: false,
            features: &[],
            uses_default_features: true,
            all_features: false,
        }
    }
}

fn unique_latest_crates(resolve: Resolve) -> BTreeMap<String, DependencySpec> {
    let mut uniqs = BTreeMap::new();
    for pkg in resolve.iter() {
        if MODIFICATIONS.blacklisted(pkg.name().as_str()) {
            continue;
        }

        match uniqs.entry(pkg.name()) {
            Entry::Vacant(entry) => {
                // First time seeing this package.
                entry.insert(pkg.version());
            }
            Entry::Occupied(mut entry) => {
                // Seen before, keep the newest version.
                if &pkg.version() > entry.get() {
                    entry.insert(pkg.version());
                }
            }
        }
    }

    uniqs.into_iter()
         .map(|(name, version)| (
             name.to_string(),
             DependencySpec::String(version.to_string()),
         ))
         .collect()
}

/// Updates `dep` to include features specified by the custom metadata of `pkg`.
///
/// Our custom metadata format looks like:
///
///     [package.metadata.playground]
///     default-features = true
///     features = ["std", "extra-traits"]
///     all-features = false
///
/// All fields are optional.
fn fill_playground_metadata_features(dep: &mut DependencySpec, pkg: &Package) {
    let custom_metadata = match pkg.manifest().custom_metadata() {
        Some(custom_metadata) => custom_metadata,
        None => return,
    };

    let playground_metadata = match custom_metadata.get("playground") {
        Some(playground_metadata) => playground_metadata,
        None => return,
    };

    #[derive(Deserialize)]
    #[serde(default, rename_all = "kebab-case")]
    struct Metadata {
        features: Vec<String>,
        default_features: bool,
        all_features: bool,
    }

    impl Default for Metadata {
        fn default() -> Self {
            Metadata {
                features: Vec::new(),
                default_features: true,
                all_features: false,
            }
        }
    }

    let metadata = match playground_metadata.clone().try_into::<Metadata>() {
        Ok(metadata) => metadata,
        Err(err) => {
            eprintln!(
                "Failed to parse custom metadata for {} {}: {}",
                pkg.name(), pkg.version(), err);
            return;
        }
    };

    // If `all-features` is set then we ignore `features`.
    let summary = pkg.summary();
    let mut enabled_features: BTreeSet<String> = if metadata.all_features {
        summary.features().keys().cloned().collect()
    } else {
        metadata.features.into_iter().collect()
    };

    // If not opting out of default features, remove default features from the
    // explicit features list. This avoids ongoing spurious diffs in our
    // generated Cargo.toml as default features are added to a library.
    if metadata.default_features {
        if let Some(default_feature_names) = summary.features().get("default") {
            enabled_features.remove("default");
            for feature in default_feature_names {
                enabled_features.remove(&feature.to_string(summary));
            }
        }
    }

    if !enabled_features.is_empty() || !metadata.default_features {
        *dep = DependencySpec::Explicit {
            version: dep.version(),
            features: enabled_features.into_iter().collect(),
            default_features: metadata.default_features,
        };
    }
}

fn write_manifest(manifest: TomlManifest, path: &str) {
    let mut f = File::create(path).expect("Unable to create Cargo.toml");
    let content = toml::to_vec(&manifest).expect("Couldn't serialize TOML");
    f.write_all(&content).expect("Couldn't write Cargo.toml");
}

fn main() {
    // Setup to interact with cargo.
    let config = Config::default().expect("Unable to create default Cargo config");
    let crates_io = SourceId::crates_io(&config).expect("Unable to create crates.io source ID");
    let mut source = RegistrySource::remote(&crates_io, &config);
    source.update().expect("Unable to update registry");

    let mut top = TopCrates::download();
    top.add_curated_crates();

    // Find the newest (non-prerelease, non-yanked) versions of all
    // the interesting crates.
    let mut summaries = Vec::new();
    for Crate { ref name } in top.crates {
        if MODIFICATIONS.blacklisted(name) {
            continue;
        }

        // Query the registry for a summary of this crate.
        // Usefully, this doesn't seem to include yanked versions
        let dep = Dependency::parse_no_deprecated(name, None, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to parse dependency for {}: {}", name, e));

        let matches = source.query_vec(&dep).unwrap_or_else(|e| {
            panic!("Unable to query registry for {}: {}", name, e);
        });

        // Find the newest non-prelease version
        let summary = matches.into_iter()
            .filter(|summary| !summary.version().is_prerelease())
            .max_by_key(|summary| summary.version().clone())
            .unwrap_or_else(|| panic!("Registry has no viable versions of {}", name));

        // Add a dependency on this crate.
        let method = decide_features(&summary);
        summaries.push((summary, method));
    }

    // Resolve transitive dependencies.
    let mut registry = PackageRegistry::new(&config)
        .expect("Unable to create package registry");
    registry.lock_patches();
    let try_to_use = HashSet::new();
    let res = resolver::resolve(&summaries, &[], &mut registry, &try_to_use, None, true)
        .expect("Unable to resolve dependencies");

    // Construct playground's Cargo.toml.
    let mut unique_latest_crates = unique_latest_crates(res);

    let mut infos = Vec::new();

    for (name, spec) in &mut unique_latest_crates {
        let version = spec.version();

        let pkgid = cargo::core::PackageId::new(&name, &version, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to build PackageId for {} {}: {}", name, version, e));

        let pkg = source.download(&pkgid)
            .unwrap_or_else(|e| panic!("Unable to download {} {}: {}", name, version, e));

        for target in pkg.targets() {
            if let cargo::core::TargetKind::Lib(_) = *target.kind() {
                infos.push(CrateInformation {
                    name: name.clone(),
                    version: version.clone(),
                    id: target.crate_name()
                })
            }
        }

        fill_playground_metadata_features(spec, &pkg);
    }

    let manifest = TomlManifest {
        package: TomlPackage {
            name: "playground".to_owned(),
            version: "0.0.1".to_owned(),
            authors: vec!["The Rust Playground".to_owned()],
        },
        profile: Profiles {
            dev: Profile { codegen_units: 1, incremental: false },
            release: Profile { codegen_units: 1, incremental: false },
        },
        dependencies: unique_latest_crates,
    };

    // Write manifest file.
    let cargo_toml = "../compiler/base/Cargo.toml";
    write_manifest(manifest, cargo_toml);
    println!("wrote {}", cargo_toml);

    let path = "../compiler/base/crate-information.json";
    let mut f = File::create(path)
        .unwrap_or_else(|e| panic!("Unable to create {}: {}", path, e));
    serde_json::to_writer_pretty(&mut f, &infos)
        .unwrap_or_else(|e| panic!("Unable to write {}: {}", path, e));
    println!("Wrote {}", path);
}
<commit_msg>Remove enabling a "playground" feature<commit_after>extern crate cargo;
extern crate reqwest;
#[macro_use]
extern crate lazy_static;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
extern crate toml;

use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::btree_map::Entry;
use std::fs::File;
use std::io::{Read, Write};

use cargo::core::{Dependency, Package, Source, SourceId};
use cargo::core::registry::PackageRegistry;
use cargo::core::resolver::{self, Method, Resolve};
use cargo::sources::RegistrySource;
use cargo::util::Config;

/// The list of crates from crates.io
#[derive(Debug, Deserialize)]
struct TopCrates {
    crates: Vec<Crate>,
}

/// A single crate from crates.io
#[derive(Debug, Deserialize)]
struct OneCrate {
    #[serde(rename="crate")]
    krate: Crate,
}

/// The shared description of a crate
#[derive(Debug, Deserialize)]
struct Crate {
    #[serde(rename="id")]
    name: String,
}

/// A Cargo.toml file.
#[derive(Serialize)]
struct TomlManifest {
    package: TomlPackage,
    profile: Profiles,
    #[serde(serialize_with = "toml::ser::tables_last")]
    dependencies: BTreeMap<String, DependencySpec>,
}

/// Header of Cargo.toml file.
#[derive(Serialize)]
struct TomlPackage {
    name: String,
    version: String,
    authors: Vec<String>,
}

/// A mapping of a crates name to its identifier used in source code
#[derive(Debug, Serialize)]
struct CrateInformation {
    name: String,
    version: String,
    id: String,
}

/// Hand-curated changes to the crate list
#[derive(Debug, Deserialize)]
struct Modifications {
    #[serde(default)]
    blacklist: Vec<String>,
    #[serde(default)]
    additions: BTreeSet<String>,
}

/// A profile section in a Cargo.toml file
#[derive(Serialize)]
#[serde(rename_all="kebab-case")]
struct Profile {
    codegen_units: u32,
    incremental: bool,
}

/// Available profile types
#[derive(Serialize)]
struct Profiles {
    dev: Profile,
    release: Profile,
}

/// `"1.0.0"` or `{ version = "1.0.0", features = ["..."] }`
#[derive(Serialize, Clone)]
#[serde(untagged)]
enum DependencySpec {
    String(String),
    #[serde(rename_all = "kebab-case")]
    Explicit {
        version: String,
        #[serde(skip_serializing_if = "Vec::is_empty")]
        features: Vec<String>,
        #[serde(skip_serializing_if = "is_true")]
        default_features: bool,
    }
}

impl DependencySpec {
    fn version(&self) -> String {
        match *self {
            DependencySpec::String(ref version) |
            DependencySpec::Explicit { ref version, .. } => version.clone(),
        }
    }
}

fn is_true(b: &bool) -> bool {
    *b
}

impl Modifications {
    fn blacklisted(&self, name: &str) -> bool {
        self.blacklist.iter().any(|n| n == name)
    }
}

lazy_static! {
    static ref MODIFICATIONS: Modifications = {
        let mut f = File::open("crate-modifications.toml")
            .expect("unable to open crate modifications file");

        let mut d = Vec::new();
        f.read_to_end(&mut d)
            .expect("unable to read crate modifications file");

        toml::from_slice(&d)
            .expect("unable to parse crate modifications file")
    };
}

impl TopCrates {
    /// List top 100 crates by number of downloads on crates.io.
    fn download() -> TopCrates {
        let resp =
            reqwest::get("https://crates.io/api/v1/crates?page=1&per_page=100&sort=downloads")
            .expect("Could not fetch top crates");
        assert!(resp.status().is_success());

        serde_json::from_reader(resp).expect("Invalid JSON")
    }

    /// Add crates that have been hand-picked
    fn add_curated_crates(&mut self) {
        self.crates.extend({
            MODIFICATIONS
                .additions
                .iter()
                .cloned()
                .map(|name| Crate { name })
        });
    }
}

fn unique_latest_crates(resolve: Resolve) -> BTreeMap<String, DependencySpec> {
    let mut uniqs = BTreeMap::new();
    for pkg in resolve.iter() {
        if MODIFICATIONS.blacklisted(pkg.name().as_str()) {
            continue;
        }

        match uniqs.entry(pkg.name()) {
            Entry::Vacant(entry) => {
                // First time seeing this package.
                entry.insert(pkg.version());
            }
            Entry::Occupied(mut entry) => {
                // Seen before, keep the newest version.
                if &pkg.version() > entry.get() {
                    entry.insert(pkg.version());
                }
            }
        }
    }

    uniqs.into_iter()
         .map(|(name, version)| (
             name.to_string(),
             DependencySpec::String(version.to_string()),
         ))
         .collect()
}

/// Updates `dep` to include features specified by the custom metadata of `pkg`.
///
/// Our custom metadata format looks like:
///
///     [package.metadata.playground]
///     default-features = true
///     features = ["std", "extra-traits"]
///     all-features = false
///
/// All fields are optional.
fn fill_playground_metadata_features(dep: &mut DependencySpec, pkg: &Package) {
    let custom_metadata = match pkg.manifest().custom_metadata() {
        Some(custom_metadata) => custom_metadata,
        None => return,
    };

    let playground_metadata = match custom_metadata.get("playground") {
        Some(playground_metadata) => playground_metadata,
        None => return,
    };

    #[derive(Deserialize)]
    #[serde(default, rename_all = "kebab-case")]
    struct Metadata {
        features: Vec<String>,
        default_features: bool,
        all_features: bool,
    }

    impl Default for Metadata {
        fn default() -> Self {
            Metadata {
                features: Vec::new(),
                default_features: true,
                all_features: false,
            }
        }
    }

    let metadata = match playground_metadata.clone().try_into::<Metadata>() {
        Ok(metadata) => metadata,
        Err(err) => {
            eprintln!(
                "Failed to parse custom metadata for {} {}: {}",
                pkg.name(), pkg.version(), err);
            return;
        }
    };

    // If `all-features` is set then we ignore `features`.
    let summary = pkg.summary();
    let mut enabled_features: BTreeSet<String> = if metadata.all_features {
        summary.features().keys().cloned().collect()
    } else {
        metadata.features.into_iter().collect()
    };

    // If not opting out of default features, remove default features from the
    // explicit features list. This avoids ongoing spurious diffs in our
    // generated Cargo.toml as default features are added to a library.
    if metadata.default_features {
        if let Some(default_feature_names) = summary.features().get("default") {
            enabled_features.remove("default");
            for feature in default_feature_names {
                enabled_features.remove(&feature.to_string(summary));
            }
        }
    }

    if !enabled_features.is_empty() || !metadata.default_features {
        *dep = DependencySpec::Explicit {
            version: dep.version(),
            features: enabled_features.into_iter().collect(),
            default_features: metadata.default_features,
        };
    }
}

fn write_manifest(manifest: TomlManifest, path: &str) {
    let mut f = File::create(path).expect("Unable to create Cargo.toml");
    let content = toml::to_vec(&manifest).expect("Couldn't serialize TOML");
    f.write_all(&content).expect("Couldn't write Cargo.toml");
}

fn main() {
    // Setup to interact with cargo.
    let config = Config::default().expect("Unable to create default Cargo config");
    let crates_io = SourceId::crates_io(&config).expect("Unable to create crates.io source ID");
    let mut source = RegistrySource::remote(&crates_io, &config);
    source.update().expect("Unable to update registry");

    let mut top = TopCrates::download();
    top.add_curated_crates();

    // Find the newest (non-prerelease, non-yanked) versions of all
    // the interesting crates.
    let mut summaries = Vec::new();
    for Crate { ref name } in top.crates {
        if MODIFICATIONS.blacklisted(name) {
            continue;
        }

        // Query the registry for a summary of this crate.
        // Usefully, this doesn't seem to include yanked versions
        let dep = Dependency::parse_no_deprecated(name, None, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to parse dependency for {}: {}", name, e));

        let matches = source.query_vec(&dep).unwrap_or_else(|e| {
            panic!("Unable to query registry for {}: {}", name, e);
        });

        // Find the newest non-prelease version
        let summary = matches.into_iter()
            .filter(|summary| !summary.version().is_prerelease())
            .max_by_key(|summary| summary.version().clone())
            .unwrap_or_else(|| panic!("Registry has no viable versions of {}", name));

        // Add a dependency on this crate.
        summaries.push((summary, Method::Required {
            dev_deps: false,
            features: &[],
            uses_default_features: true,
            all_features: false,
        }));
    }

    // Resolve transitive dependencies.
    let mut registry = PackageRegistry::new(&config)
        .expect("Unable to create package registry");
    registry.lock_patches();
    let try_to_use = HashSet::new();
    let res = resolver::resolve(&summaries, &[], &mut registry, &try_to_use, None, true)
        .expect("Unable to resolve dependencies");

    // Construct playground's Cargo.toml.
    let mut unique_latest_crates = unique_latest_crates(res);

    let mut infos = Vec::new();

    for (name, spec) in &mut unique_latest_crates {
        let version = spec.version();

        let pkgid = cargo::core::PackageId::new(&name, &version, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to build PackageId for {} {}: {}", name, version, e));

        let pkg = source.download(&pkgid)
            .unwrap_or_else(|e| panic!("Unable to download {} {}: {}", name, version, e));

        for target in pkg.targets() {
            if let cargo::core::TargetKind::Lib(_) = *target.kind() {
                infos.push(CrateInformation {
                    name: name.clone(),
                    version: version.clone(),
                    id: target.crate_name()
                })
            }
        }

        fill_playground_metadata_features(spec, &pkg);
    }

    let manifest = TomlManifest {
        package: TomlPackage {
            name: "playground".to_owned(),
            version: "0.0.1".to_owned(),
            authors: vec!["The Rust Playground".to_owned()],
        },
        profile: Profiles {
            dev: Profile { codegen_units: 1, incremental: false },
            release: Profile { codegen_units: 1, incremental: false },
        },
        dependencies: unique_latest_crates,
    };

    // Write manifest file.
    let cargo_toml = "../compiler/base/Cargo.toml";
    write_manifest(manifest, cargo_toml);
    println!("wrote {}", cargo_toml);

    let path = "../compiler/base/crate-information.json";
    let mut f = File::create(path)
        .unwrap_or_else(|e| panic!("Unable to create {}: {}", path, e));
    serde_json::to_writer_pretty(&mut f, &infos)
        .unwrap_or_else(|e| panic!("Unable to write {}: {}", path, e));
    println!("Wrote {}", path);
}
<|endoftext|>


<commit_before>mod extractor;

extern crate num_cpus;

use clap;
use flate2::write::GzEncoder;
use rayon::prelude::*;
use std::fs;
use std::io::{BufRead, BufWriter};
use std::path::{Path, PathBuf};
use tree_sitter::{Language, Parser, Range};

enum TrapCompression {
    None,
    Gzip,
}

impl TrapCompression {
    fn from_env() -> TrapCompression {
        match std::env::var("CODEQL_RUBY_TRAP_COMPRESSION") {
            Ok(method) => match TrapCompression::from_string(&method) {
                Some(c) => c,
                None => {
                    tracing::error!("Unknown compression method '{}'; using gzip.", &method);
                    TrapCompression::Gzip
                }
            },
            // Default compression method if the env var isn't set:
            Err(_) => TrapCompression::Gzip,
        }
    }

    fn from_string(s: &str) -> Option<TrapCompression> {
        match s.to_lowercase().as_ref() {
            "none" => Some(TrapCompression::None),
            "gzip" => Some(TrapCompression::Gzip),
            _ => None,
        }
    }

    fn extension(&self) -> &str {
        match self {
            TrapCompression::None => "trap",
            TrapCompression::Gzip => "trap.gz",
        }
    }
}

/**
 * Gets the number of threads the extractor should use, by reading the
 * CODEQL_THREADS environment variable and using it as described in the
 * extractor spec:
 *
 * "If the number is positive, it indicates the number of threads that should
 * be used. If the number is negative or zero, it should be added to the number
 * of cores available on the machine to determine how many threads to use
 * (minimum of 1). If unspecified, should be considered as set to -1."
 */
fn num_codeql_threads() -> usize {
    let threads_str = std::env::var("CODEQL_THREADS").unwrap_or("-1".to_owned());
    match threads_str.parse::<i32>() {
        Ok(num) if num <= 0 => {
            let reduction = -num as usize;
            std::cmp::max(1, num_cpus::get() - reduction)
        }
        Ok(num) => num as usize,

        Err(_) => {
            tracing::error!(
                "Unable to parse CODEQL_THREADS value '{}'; defaulting to 1 thread.",
                &threads_str
            );
            1
        }
    }
}

fn main() -> std::io::Result<()> {
    tracing_subscriber::fmt()
        .with_target(false)
        .without_time()
        .with_level(true)
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
        .init();

    let num_threads = num_codeql_threads();
    tracing::info!(
        "Using {} {}",
        num_threads,
        if num_threads == 1 {
            "thread"
        } else {
            "threads"
        }
    );
    rayon::ThreadPoolBuilder::new()
        .num_threads(num_threads)
        .build_global()
        .unwrap();

    let matches = clap::App::new("Ruby extractor")
        .version("1.0")
        .author("GitHub")
        .about("CodeQL Ruby extractor")
        .args_from_usage(
            "--source-archive-dir=<DIR> 'Sets a custom source archive folder'
                    --output-dir=<DIR>         'Sets a custom trap folder'
                    --file-list=<FILE_LIST>    'A text files containing the paths of the files to extract'",
        )
        .get_matches();
    let src_archive_dir = matches
        .value_of("source-archive-dir")
        .expect("missing --source-archive-dir");
    let src_archive_dir = PathBuf::from(src_archive_dir);

    let trap_dir = matches
        .value_of("output-dir")
        .expect("missing --output-dir");
    let trap_dir = PathBuf::from(trap_dir);
    let trap_compression = TrapCompression::from_env();

    let file_list = matches.value_of("file-list").expect("missing --file-list");
    let file_list = fs::File::open(file_list)?;

    let language = tree_sitter_ruby::language();
    let erb = tree_sitter_embedded_template::language();
    let schema = node_types::read_node_types_str("ruby", tree_sitter_ruby::NODE_TYPES)?;
    let erb_schema =
        node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?;
    let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
    let lines = lines?;
    lines
        .par_iter()
        .try_for_each(|line| {
            let path = PathBuf::from(line).canonicalize()?;
            let src_archive_file = path_for(&src_archive_dir, &path, "");
            let mut source = std::fs::read(&path)?;
            let code_ranges;
            let mut trap_writer = extractor::new_trap_writer();
            if path.extension().map_or(false, |x| x == "erb") {
                tracing::info!("scanning: {}", path.display());
                extractor::extract(
                    erb,
                    "erb",
                    &erb_schema,
                    &mut trap_writer,
                    &path,
                    &source,
                    &[],
                )?;

                let (ranges, line_breaks) = scan_erb(erb, &source);
                for i in line_breaks {
                    if i < source.len() {
                        source[i] = b'\n';
                    }
                }
                code_ranges = ranges;
            } else {
                code_ranges = vec![];
            }
            extractor::extract(
                language,
                "ruby",
                &schema,
                &mut trap_writer,
                &path,
                &source,
                &code_ranges,
            )?;
            std::fs::create_dir_all(&src_archive_file.parent().unwrap())?;
            std::fs::copy(&path, &src_archive_file)?;
            write_trap(&trap_dir, path, trap_writer, &trap_compression)
        })
        .expect("failed to extract files");

    let path = PathBuf::from("extras");
    let mut trap_writer = extractor::new_trap_writer();
    trap_writer.populate_empty_location();
    write_trap(&trap_dir, path, trap_writer, &trap_compression)
}

fn write_trap(
    trap_dir: &PathBuf,
    path: PathBuf,
    trap_writer: extractor::TrapWriter,
    trap_compression: &TrapCompression,
) -> std::io::Result<()> {
    let trap_file = path_for(&trap_dir, &path, trap_compression.extension());
    std::fs::create_dir_all(&trap_file.parent().unwrap())?;
    let trap_file = std::fs::File::create(&trap_file)?;
    let mut trap_file = BufWriter::new(trap_file);
    match trap_compression {
        TrapCompression::None => trap_writer.output(&mut trap_file),
        TrapCompression::Gzip => {
            let mut compressed_writer = GzEncoder::new(trap_file, flate2::Compression::fast());
            trap_writer.output(&mut compressed_writer)
        }
    }
}

fn scan_erb(erb: Language, source: &std::vec::Vec<u8>) -> (Vec<Range>, Vec<usize>) {
    let mut parser = Parser::new();
    parser.set_language(erb).unwrap();
    let tree = parser.parse(&source, None).expect("Failed to parse file");
    let mut result = Vec::new();
    let mut line_breaks = vec![];

    for n in tree.root_node().children(&mut tree.walk()) {
        let kind = n.kind();
        if kind == "directive" || kind == "output_directive" {
            for c in n.children(&mut tree.walk()) {
                if c.kind() == "code" {
                    let mut range = c.range();
                    if range.end_byte < source.len() {
                        line_breaks.push(range.end_byte);
                        range.end_byte += 1;
                        range.end_point.column += 1;
                    }
                    result.push(range);
                }
            }
        }
    }
    if result.len() == 0 {
        let root = tree.root_node();
        // Add an empty range at the end of the file
        result.push(Range {
            start_byte: root.end_byte(),
            end_byte: root.end_byte(),
            start_point: root.end_position(),
            end_point: root.end_position(),
        });
    }
    (result, line_breaks)
}

fn path_for(dir: &Path, path: &Path, ext: &str) -> PathBuf {
    let mut result = PathBuf::from(dir);
    for component in path.components() {
        match component {
            std::path::Component::Prefix(prefix) => match prefix.kind() {
                std::path::Prefix::Disk(letter) | std::path::Prefix::VerbatimDisk(letter) => {
                    result.push(format!("{}_", letter as char))
                }
                std::path::Prefix::Verbatim(x) | std::path::Prefix::DeviceNS(x) => {
                    result.push(x);
                }
                std::path::Prefix::UNC(server, share)
                | std::path::Prefix::VerbatimUNC(server, share) => {
                    result.push("unc");
                    result.push(server);
                    result.push(share);
                }
            },
            std::path::Component::RootDir => {
                // skip
            }
            std::path::Component::Normal(_) => {
                result.push(component);
            }
            std::path::Component::CurDir => {
                // skip
            }
            std::path::Component::ParentDir => {
                result.pop();
            }
        }
    }
    if let Some(x) = result.extension() {
        let mut new_ext = x.to_os_string();
        new_ext.push(".");
        new_ext.push(ext);
        result.set_extension(new_ext);
    } else {
        result.set_extension(ext);
    }
    result
}
<commit_msg>Replace `if let` with `match`.<commit_after>mod extractor;

extern crate num_cpus;

use clap;
use flate2::write::GzEncoder;
use rayon::prelude::*;
use std::fs;
use std::io::{BufRead, BufWriter};
use std::path::{Path, PathBuf};
use tree_sitter::{Language, Parser, Range};

enum TrapCompression {
    None,
    Gzip,
}

impl TrapCompression {
    fn from_env() -> TrapCompression {
        match std::env::var("CODEQL_RUBY_TRAP_COMPRESSION") {
            Ok(method) => match TrapCompression::from_string(&method) {
                Some(c) => c,
                None => {
                    tracing::error!("Unknown compression method '{}'; using gzip.", &method);
                    TrapCompression::Gzip
                }
            },
            // Default compression method if the env var isn't set:
            Err(_) => TrapCompression::Gzip,
        }
    }

    fn from_string(s: &str) -> Option<TrapCompression> {
        match s.to_lowercase().as_ref() {
            "none" => Some(TrapCompression::None),
            "gzip" => Some(TrapCompression::Gzip),
            _ => None,
        }
    }

    fn extension(&self) -> &str {
        match self {
            TrapCompression::None => "trap",
            TrapCompression::Gzip => "trap.gz",
        }
    }
}

/**
 * Gets the number of threads the extractor should use, by reading the
 * CODEQL_THREADS environment variable and using it as described in the
 * extractor spec:
 *
 * "If the number is positive, it indicates the number of threads that should
 * be used. If the number is negative or zero, it should be added to the number
 * of cores available on the machine to determine how many threads to use
 * (minimum of 1). If unspecified, should be considered as set to -1."
 */
fn num_codeql_threads() -> usize {
    let threads_str = std::env::var("CODEQL_THREADS").unwrap_or("-1".to_owned());
    match threads_str.parse::<i32>() {
        Ok(num) if num <= 0 => {
            let reduction = -num as usize;
            std::cmp::max(1, num_cpus::get() - reduction)
        }
        Ok(num) => num as usize,

        Err(_) => {
            tracing::error!(
                "Unable to parse CODEQL_THREADS value '{}'; defaulting to 1 thread.",
                &threads_str
            );
            1
        }
    }
}

fn main() -> std::io::Result<()> {
    tracing_subscriber::fmt()
        .with_target(false)
        .without_time()
        .with_level(true)
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
        .init();

    let num_threads = num_codeql_threads();
    tracing::info!(
        "Using {} {}",
        num_threads,
        if num_threads == 1 {
            "thread"
        } else {
            "threads"
        }
    );
    rayon::ThreadPoolBuilder::new()
        .num_threads(num_threads)
        .build_global()
        .unwrap();

    let matches = clap::App::new("Ruby extractor")
        .version("1.0")
        .author("GitHub")
        .about("CodeQL Ruby extractor")
        .args_from_usage(
            "--source-archive-dir=<DIR> 'Sets a custom source archive folder'
                    --output-dir=<DIR>         'Sets a custom trap folder'
                    --file-list=<FILE_LIST>    'A text files containing the paths of the files to extract'",
        )
        .get_matches();
    let src_archive_dir = matches
        .value_of("source-archive-dir")
        .expect("missing --source-archive-dir");
    let src_archive_dir = PathBuf::from(src_archive_dir);

    let trap_dir = matches
        .value_of("output-dir")
        .expect("missing --output-dir");
    let trap_dir = PathBuf::from(trap_dir);
    let trap_compression = TrapCompression::from_env();

    let file_list = matches.value_of("file-list").expect("missing --file-list");
    let file_list = fs::File::open(file_list)?;

    let language = tree_sitter_ruby::language();
    let erb = tree_sitter_embedded_template::language();
    let schema = node_types::read_node_types_str("ruby", tree_sitter_ruby::NODE_TYPES)?;
    let erb_schema =
        node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?;
    let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
    let lines = lines?;
    lines
        .par_iter()
        .try_for_each(|line| {
            let path = PathBuf::from(line).canonicalize()?;
            let src_archive_file = path_for(&src_archive_dir, &path, "");
            let mut source = std::fs::read(&path)?;
            let code_ranges;
            let mut trap_writer = extractor::new_trap_writer();
            if path.extension().map_or(false, |x| x == "erb") {
                tracing::info!("scanning: {}", path.display());
                extractor::extract(
                    erb,
                    "erb",
                    &erb_schema,
                    &mut trap_writer,
                    &path,
                    &source,
                    &[],
                )?;

                let (ranges, line_breaks) = scan_erb(erb, &source);
                for i in line_breaks {
                    if i < source.len() {
                        source[i] = b'\n';
                    }
                }
                code_ranges = ranges;
            } else {
                code_ranges = vec![];
            }
            extractor::extract(
                language,
                "ruby",
                &schema,
                &mut trap_writer,
                &path,
                &source,
                &code_ranges,
            )?;
            std::fs::create_dir_all(&src_archive_file.parent().unwrap())?;
            std::fs::copy(&path, &src_archive_file)?;
            write_trap(&trap_dir, path, trap_writer, &trap_compression)
        })
        .expect("failed to extract files");

    let path = PathBuf::from("extras");
    let mut trap_writer = extractor::new_trap_writer();
    trap_writer.populate_empty_location();
    write_trap(&trap_dir, path, trap_writer, &trap_compression)
}

fn write_trap(
    trap_dir: &PathBuf,
    path: PathBuf,
    trap_writer: extractor::TrapWriter,
    trap_compression: &TrapCompression,
) -> std::io::Result<()> {
    let trap_file = path_for(&trap_dir, &path, trap_compression.extension());
    std::fs::create_dir_all(&trap_file.parent().unwrap())?;
    let trap_file = std::fs::File::create(&trap_file)?;
    let mut trap_file = BufWriter::new(trap_file);
    match trap_compression {
        TrapCompression::None => trap_writer.output(&mut trap_file),
        TrapCompression::Gzip => {
            let mut compressed_writer = GzEncoder::new(trap_file, flate2::Compression::fast());
            trap_writer.output(&mut compressed_writer)
        }
    }
}

fn scan_erb(erb: Language, source: &std::vec::Vec<u8>) -> (Vec<Range>, Vec<usize>) {
    let mut parser = Parser::new();
    parser.set_language(erb).unwrap();
    let tree = parser.parse(&source, None).expect("Failed to parse file");
    let mut result = Vec::new();
    let mut line_breaks = vec![];

    for n in tree.root_node().children(&mut tree.walk()) {
        let kind = n.kind();
        if kind == "directive" || kind == "output_directive" {
            for c in n.children(&mut tree.walk()) {
                if c.kind() == "code" {
                    let mut range = c.range();
                    if range.end_byte < source.len() {
                        line_breaks.push(range.end_byte);
                        range.end_byte += 1;
                        range.end_point.column += 1;
                    }
                    result.push(range);
                }
            }
        }
    }
    if result.len() == 0 {
        let root = tree.root_node();
        // Add an empty range at the end of the file
        result.push(Range {
            start_byte: root.end_byte(),
            end_byte: root.end_byte(),
            start_point: root.end_position(),
            end_point: root.end_position(),
        });
    }
    (result, line_breaks)
}

fn path_for(dir: &Path, path: &Path, ext: &str) -> PathBuf {
    let mut result = PathBuf::from(dir);
    for component in path.components() {
        match component {
            std::path::Component::Prefix(prefix) => match prefix.kind() {
                std::path::Prefix::Disk(letter) | std::path::Prefix::VerbatimDisk(letter) => {
                    result.push(format!("{}_", letter as char))
                }
                std::path::Prefix::Verbatim(x) | std::path::Prefix::DeviceNS(x) => {
                    result.push(x);
                }
                std::path::Prefix::UNC(server, share)
                | std::path::Prefix::VerbatimUNC(server, share) => {
                    result.push("unc");
                    result.push(server);
                    result.push(share);
                }
            },
            std::path::Component::RootDir => {
                // skip
            }
            std::path::Component::Normal(_) => {
                result.push(component);
            }
            std::path::Component::CurDir => {
                // skip
            }
            std::path::Component::ParentDir => {
                result.pop();
            }
        }
    }
    match result.extension() {
        Some(x) => {
            let mut new_ext = x.to_os_string();
            new_ext.push(".");
            new_ext.push(ext);
            result.set_extension(new_ext);
        }
        None => {
            result.set_extension(ext);
        }
    }
    result
}
<|endoftext|>


<commit_before>use super::*;

use dpi::LogicalPosition;
use event::{DeviceId as RootDI, ElementState, Event, KeyboardInput, StartCause, WindowEvent};
use event_loop::{ControlFlow, EventLoopWindowTarget as RootELW, EventLoopClosed};
use window::{WindowId as RootWI};
use stdweb::{
    traits::*,
    web::{
        document,
        event::*,
        html_element::CanvasElement,
    },
};
use std::cell::RefCell;
use std::collections::VecDeque;
use std::collections::vec_deque::IntoIter as VecDequeIter;
use std::marker::PhantomData;
use std::rc::Rc;

#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct DeviceId(i32);

impl DeviceId {
    pub unsafe fn dummy() -> Self {
        DeviceId(0)
    }
}

pub struct EventLoop<T: 'static> {
    elw: RootELW<T>,
}

pub struct EventLoopWindowTarget<T: 'static> {
    pub(crate) runner: EventLoopRunnerShared<T>,
}

impl<T> EventLoopWindowTarget<T> {
    fn new() -> Self {
        EventLoopWindowTarget {
            runner: Rc::new(ELRShared {
                runner: RefCell::new(None),
                events: RefCell::new(VecDeque::new())
            })
        }
    }
}

#[derive(Clone)]
pub struct EventLoopProxy<T> {
    runner: EventLoopRunnerShared<T>
}

impl<T> EventLoopProxy<T> {
    pub fn send_event(&self, event: T) -> Result<(), EventLoopClosed> {
        self.runner.send_event(Event::UserEvent(event));
        Ok(())
    }
}

pub type EventLoopRunnerShared<T> = Rc<ELRShared<T>>;

pub struct ELRShared<T> {
    runner: RefCell<Option<EventLoopRunner<T>>>,
    events: RefCell<VecDeque<Event<T>>>, // TODO: this may not be necessary?
}

struct EventLoopRunner<T> {
    control: ControlFlow,
    is_busy: bool,
    event_handler: Box<dyn FnMut(Event<T>, &mut ControlFlow)>,
}

impl<T> EventLoop<T> {
    pub fn new() -> Self {
        EventLoop {
            elw: RootELW {
                p: EventLoopWindowTarget::new(),
                _marker: PhantomData
            },
        }
    }

    pub fn get_available_monitors(&self) -> VecDequeIter<MonitorHandle> {
        VecDeque::new().into_iter()
    }

    pub fn get_primary_monitor(&self) -> MonitorHandle {
        MonitorHandle
    }

    pub fn run<F>(self, mut event_handler: F) -> !
        where F: 'static + FnMut(Event<T>, &RootELW<T>, &mut ControlFlow) {
        let runner = self.elw.p.runner;

        let relw = RootELW {
            p: EventLoopWindowTarget::new(),
            _marker: PhantomData
        };
        runner.set_listener(Box::new(move |evt, ctrl| event_handler(evt, &relw, ctrl)));

        let document = &document();
        add_event(&runner, document, |elrs, _: BlurEvent| {
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::Focused(false)
            });
        });
        add_event(&runner, document, |elrs, _: FocusEvent| {
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::Focused(true)
            });

        });
        add_event(&runner, document, |elrs, event: KeyDownEvent| {
            let key = event.key();
            let mut characters = key.chars();
            let first = characters.next();
            let second = characters.next();
            if let (Some(key), None) = (first, second) {
                elrs.send_event(Event::WindowEvent {
                    window_id: RootWI(WindowId),
                    event: WindowEvent::ReceivedCharacter(key)
                });
            }
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::KeyboardInput {
                    device_id: RootDI(unsafe { DeviceId::dummy() }),
                    input: KeyboardInput {
                        scancode: scancode(&event),
                        state: ElementState::Pressed,
                        virtual_keycode: button_mapping(&event),
                        modifiers: keyboard_modifiers_state(&event),
                    }
                }
            });
        });
        add_event(&runner, document, |elrs, event: KeyUpEvent| {
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::KeyboardInput {
                    device_id: RootDI(unsafe { DeviceId::dummy() }),
                    input: KeyboardInput {
                        scancode: scancode(&event),
                        state: ElementState::Released,
                        virtual_keycode: button_mapping(&event),
                        modifiers: keyboard_modifiers_state(&event),
                    }
                }
            });
        });
        stdweb::event_loop(); // TODO: this is only necessary for stdweb emscripten, should it be here?

        // Throw an exception to break out of Rust exceution and use unreachable to tell the
        // compiler this function won't return, giving it a return type of '!'
        js! {
            throw "Using exceptions for control flow, don't mind me";
        }
        unreachable!();
    }

    pub fn create_proxy(&self) -> EventLoopProxy<T> {
        EventLoopProxy {
            runner: self.elw.p.runner.clone()
        }
    }

    pub fn window_target(&self) -> &RootELW<T> {
        &self.elw
    }
}

pub fn register<T: 'static>(elrs: &EventLoopRunnerShared<T>, canvas: &CanvasElement) {
    add_event(elrs, canvas, |elrs, event: PointerOutEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::CursorLeft {
                device_id: RootDI(DeviceId(event.pointer_id()))
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerOverEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::CursorEntered {
                device_id: RootDI(DeviceId(event.pointer_id()))
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerMoveEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::CursorMoved {
                device_id: RootDI(DeviceId(event.pointer_id())),
                position: LogicalPosition {
                    x: event.offset_x(),
                    y: event.offset_y()
                },
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerUpEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::MouseInput {
                device_id: RootDI(DeviceId(event.pointer_id())),
                state: ElementState::Pressed,
                button: mouse_button(&event),
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerDownEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::MouseInput {
                device_id: RootDI(DeviceId(event.pointer_id())),
                state: ElementState::Released,
                button: mouse_button(&event),
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
}

fn add_event<T: 'static, E, F>(elrs: &EventLoopRunnerShared<T>, target: &impl IEventTarget, mut handler: F)
        where E: ConcreteEvent, F: FnMut(&EventLoopRunnerShared<T>, E) + 'static {
    let elrs = elrs.clone();

    target.add_event_listener(move |event: E| {
        // Don't capture the event if the events loop has been destroyed
        match &*elrs.runner.borrow() {
            Some(ref runner) if runner.control == ControlFlow::Exit => return,
            _ => ()
        }

        event.prevent_default();
        event.stop_propagation();
        event.cancel_bubble();

        handler(&elrs, event);
    });
}

impl<T> ELRShared<T> {
    // Set the event callback to use for the event loop runner
    // This the event callback is a fairly thin layer over the user-provided callback that closes
    // over a RootEventLoopWindowTarget reference
    fn set_listener(&self, event_handler: Box<dyn FnMut(Event<T>, &mut ControlFlow)>) {
        *self.runner.borrow_mut() = Some(EventLoopRunner {
            control: ControlFlow::Poll,
            is_busy: false,
            event_handler
        });
    }

    // Add an event to the event loop runner
    //
    // It will determine if the event should be immediately sent to the user or buffered for later
    pub fn send_event(&self, event: Event<T>) {
        // If the event loop is closed, it should discard any new events
        if self.closed() {
            return;
        }

        let start_cause = StartCause::Poll; // TODO: determine start cause

        // Determine if event handling is in process, and then release the borrow on the runner
        let is_busy = if let Some(ref runner) = *self.runner.borrow() {
            runner.is_busy
        } else {
            true // If there is no event runner yet, then there's no point in processing events
        };

        if is_busy {
            self.events.borrow_mut().push_back(event);
        } else {
            // Handle starting a new batch of events
            //
            // The user is informed via Event::NewEvents that there is a batch of events to process
            // However, there is only one of these per batch of events
            self.handle_event(Event::NewEvents(start_cause));
            self.handle_event(event);
            self.handle_event(Event::EventsCleared);

            // If the event loop is closed, it has been closed this iteration and now the closing
            // event should be emitted
            if self.closed() {
                self.handle_event(Event::LoopDestroyed);
            }
        }
    }

    // Check if the event loop is currntly closed
    fn closed(&self) -> bool {
        match *self.runner.borrow() {
            Some(ref runner) => runner.control == ControlFlow::Exit,
            None => false, // If the event loop is None, it has not been intialised yet, so it cannot be closed
        }
    }

    // handle_event takes in events and either queues them or applies a callback
    //
    // It should only ever be called from send_event
    fn handle_event(&self, event: Event<T>) {
        let closed = self.closed();

        match *self.runner.borrow_mut() {
            Some(ref mut runner) => {
                // An event is being processed, so the runner should be marked busy
                runner.is_busy = true;

                // TODO: bracket this in control flow events?
                (runner.event_handler)(event, &mut runner.control);

                // Maintain closed state, even if the callback changes it
                if closed {
                    runner.control = ControlFlow::Exit;
                }

                // An event is no longer being processed
                runner.is_busy = false;
            }
            // If an event is being handled without a runner somehow, add it to the event queue so
            // it will eventually be processed
            _ => self.events.borrow_mut().push_back(event)
        }

        // Don't take events out of the queue if the loop is closed or the runner doesn't exist
        // If the runner doesn't exist and this method recurses, it will recurse infinitely
        if !closed && self.runner.borrow().is_some() {
            // Take an event out of the queue and handle it
            if let Some(event) = self.events.borrow_mut().pop_front() {
                self.handle_event(event);
            }
        }
    }

}

<commit_msg>Add support for mouse wheel<commit_after>use super::*;

use dpi::LogicalPosition;
use event::{DeviceId as RootDI, ElementState, Event, KeyboardInput, MouseScrollDelta, StartCause, TouchPhase, WindowEvent};
use event_loop::{ControlFlow, EventLoopWindowTarget as RootELW, EventLoopClosed};
use window::{WindowId as RootWI};
use stdweb::{
    traits::*,
    web::{
        document,
        event::*,
        html_element::CanvasElement,
    },
};
use std::cell::RefCell;
use std::collections::VecDeque;
use std::collections::vec_deque::IntoIter as VecDequeIter;
use std::marker::PhantomData;
use std::rc::Rc;

#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct DeviceId(i32);

impl DeviceId {
    pub unsafe fn dummy() -> Self {
        DeviceId(0)
    }
}

pub struct EventLoop<T: 'static> {
    elw: RootELW<T>,
}

pub struct EventLoopWindowTarget<T: 'static> {
    pub(crate) runner: EventLoopRunnerShared<T>,
}

impl<T> EventLoopWindowTarget<T> {
    fn new() -> Self {
        EventLoopWindowTarget {
            runner: Rc::new(ELRShared {
                runner: RefCell::new(None),
                events: RefCell::new(VecDeque::new())
            })
        }
    }
}

#[derive(Clone)]
pub struct EventLoopProxy<T> {
    runner: EventLoopRunnerShared<T>
}

impl<T> EventLoopProxy<T> {
    pub fn send_event(&self, event: T) -> Result<(), EventLoopClosed> {
        self.runner.send_event(Event::UserEvent(event));
        Ok(())
    }
}

pub type EventLoopRunnerShared<T> = Rc<ELRShared<T>>;

pub struct ELRShared<T> {
    runner: RefCell<Option<EventLoopRunner<T>>>,
    events: RefCell<VecDeque<Event<T>>>, // TODO: this may not be necessary?
}

struct EventLoopRunner<T> {
    control: ControlFlow,
    is_busy: bool,
    event_handler: Box<dyn FnMut(Event<T>, &mut ControlFlow)>,
}

impl<T> EventLoop<T> {
    pub fn new() -> Self {
        EventLoop {
            elw: RootELW {
                p: EventLoopWindowTarget::new(),
                _marker: PhantomData
            },
        }
    }

    pub fn get_available_monitors(&self) -> VecDequeIter<MonitorHandle> {
        VecDeque::new().into_iter()
    }

    pub fn get_primary_monitor(&self) -> MonitorHandle {
        MonitorHandle
    }

    pub fn run<F>(self, mut event_handler: F) -> !
        where F: 'static + FnMut(Event<T>, &RootELW<T>, &mut ControlFlow) {
        let runner = self.elw.p.runner;

        let relw = RootELW {
            p: EventLoopWindowTarget::new(),
            _marker: PhantomData
        };
        runner.set_listener(Box::new(move |evt, ctrl| event_handler(evt, &relw, ctrl)));

        let document = &document();
        add_event(&runner, document, |elrs, _: BlurEvent| {
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::Focused(false)
            });
        });
        add_event(&runner, document, |elrs, _: FocusEvent| {
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::Focused(true)
            });

        });
        add_event(&runner, document, |elrs, event: KeyDownEvent| {
            let key = event.key();
            let mut characters = key.chars();
            let first = characters.next();
            let second = characters.next();
            if let (Some(key), None) = (first, second) {
                elrs.send_event(Event::WindowEvent {
                    window_id: RootWI(WindowId),
                    event: WindowEvent::ReceivedCharacter(key)
                });
            }
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::KeyboardInput {
                    device_id: RootDI(unsafe { DeviceId::dummy() }),
                    input: KeyboardInput {
                        scancode: scancode(&event),
                        state: ElementState::Pressed,
                        virtual_keycode: button_mapping(&event),
                        modifiers: keyboard_modifiers_state(&event),
                    }
                }
            });
        });
        add_event(&runner, document, |elrs, event: KeyUpEvent| {
            elrs.send_event(Event::WindowEvent {
                window_id: RootWI(WindowId),
                event: WindowEvent::KeyboardInput {
                    device_id: RootDI(unsafe { DeviceId::dummy() }),
                    input: KeyboardInput {
                        scancode: scancode(&event),
                        state: ElementState::Released,
                        virtual_keycode: button_mapping(&event),
                        modifiers: keyboard_modifiers_state(&event),
                    }
                }
            });
        });

        runner.send_event(Event::NewEvents(StartCause::Init));

        stdweb::event_loop(); // TODO: this is only necessary for stdweb emscripten, should it be here?

        // Throw an exception to break out of Rust exceution and use unreachable to tell the
        // compiler this function won't return, giving it a return type of '!'
        js! {
            throw "Using exceptions for control flow, don't mind me. This isn't actually an error!";
        }
        unreachable!();
    }

    pub fn create_proxy(&self) -> EventLoopProxy<T> {
        EventLoopProxy {
            runner: self.elw.p.runner.clone()
        }
    }

    pub fn window_target(&self) -> &RootELW<T> {
        &self.elw
    }
}

pub fn register<T: 'static>(elrs: &EventLoopRunnerShared<T>, canvas: &CanvasElement) {
    add_event(elrs, canvas, |elrs, event: PointerOutEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::CursorLeft {
                device_id: RootDI(DeviceId(event.pointer_id()))
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerOverEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::CursorEntered {
                device_id: RootDI(DeviceId(event.pointer_id()))
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerMoveEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::CursorMoved {
                device_id: RootDI(DeviceId(event.pointer_id())),
                position: LogicalPosition {
                    x: event.offset_x(),
                    y: event.offset_y()
                },
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerUpEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::MouseInput {
                device_id: RootDI(DeviceId(event.pointer_id())),
                state: ElementState::Pressed,
                button: mouse_button(&event),
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: PointerDownEvent| {
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::MouseInput {
                device_id: RootDI(DeviceId(event.pointer_id())),
                state: ElementState::Released,
                button: mouse_button(&event),
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
    add_event(elrs, canvas, |elrs, event: MouseWheelEvent| {
        let x = event.delta_x();
        let y = event.delta_y();
        let delta = match event.delta_mode() {
            MouseWheelDeltaMode::Line => MouseScrollDelta::LineDelta(x as f32, y as f32),
            MouseWheelDeltaMode::Pixel => MouseScrollDelta::PixelDelta(LogicalPosition { x, y }),
            MouseWheelDeltaMode::Page => return,
        };
        elrs.send_event(Event::WindowEvent {
            window_id: RootWI(WindowId),
            event: WindowEvent::MouseWheel {
                device_id: RootDI(DeviceId(0)),
                delta,
                phase: TouchPhase::Moved,
                modifiers: mouse_modifiers_state(&event)
            }
        });
    });
}

fn add_event<T: 'static, E, F>(elrs: &EventLoopRunnerShared<T>, target: &impl IEventTarget, mut handler: F)
        where E: ConcreteEvent, F: FnMut(&EventLoopRunnerShared<T>, E) + 'static {
    let elrs = elrs.clone();

    target.add_event_listener(move |event: E| {
        // Don't capture the event if the events loop has been destroyed
        match &*elrs.runner.borrow() {
            Some(ref runner) if runner.control == ControlFlow::Exit => return,
            _ => ()
        }

        event.prevent_default();
        event.stop_propagation();
        event.cancel_bubble();

        handler(&elrs, event);
    });
}

impl<T> ELRShared<T> {
    // Set the event callback to use for the event loop runner
    // This the event callback is a fairly thin layer over the user-provided callback that closes
    // over a RootEventLoopWindowTarget reference
    fn set_listener(&self, event_handler: Box<dyn FnMut(Event<T>, &mut ControlFlow)>) {
        *self.runner.borrow_mut() = Some(EventLoopRunner {
            control: ControlFlow::Poll,
            is_busy: false,
            event_handler
        });
    }

    // Add an event to the event loop runner
    //
    // It will determine if the event should be immediately sent to the user or buffered for later
    pub fn send_event(&self, event: Event<T>) {
        // If the event loop is closed, it should discard any new events
        if self.closed() {
            return;
        }

        let start_cause = StartCause::Poll; // TODO: determine start cause

        // Determine if event handling is in process, and then release the borrow on the runner
        let is_busy = if let Some(ref runner) = *self.runner.borrow() {
            runner.is_busy
        } else {
            true // If there is no event runner yet, then there's no point in processing events
        };

        if is_busy {
            self.events.borrow_mut().push_back(event);
        } else {
            // Handle starting a new batch of events
            //
            // The user is informed via Event::NewEvents that there is a batch of events to process
            // However, there is only one of these per batch of events
            self.handle_event(Event::NewEvents(start_cause));
            self.handle_event(event);
            self.handle_event(Event::EventsCleared);

            // If the event loop is closed, it has been closed this iteration and now the closing
            // event should be emitted
            if self.closed() {
                self.handle_event(Event::LoopDestroyed);
            }
        }
    }

    // Check if the event loop is currntly closed
    fn closed(&self) -> bool {
        match *self.runner.borrow() {
            Some(ref runner) => runner.control == ControlFlow::Exit,
            None => false, // If the event loop is None, it has not been intialised yet, so it cannot be closed
        }
    }

    // handle_event takes in events and either queues them or applies a callback
    //
    // It should only ever be called from send_event
    fn handle_event(&self, event: Event<T>) {
        let closed = self.closed();

        match *self.runner.borrow_mut() {
            Some(ref mut runner) => {
                // An event is being processed, so the runner should be marked busy
                runner.is_busy = true;

                // TODO: bracket this in control flow events?
                (runner.event_handler)(event, &mut runner.control);

                // Maintain closed state, even if the callback changes it
                if closed {
                    runner.control = ControlFlow::Exit;
                }

                // An event is no longer being processed
                runner.is_busy = false;
            }
            // If an event is being handled without a runner somehow, add it to the event queue so
            // it will eventually be processed
            _ => self.events.borrow_mut().push_back(event)
        }

        // Don't take events out of the queue if the loop is closed or the runner doesn't exist
        // If the runner doesn't exist and this method recurses, it will recurse infinitely
        if !closed && self.runner.borrow().is_some() {
            // Take an event out of the queue and handle it
            if let Some(event) = self.events.borrow_mut().pop_front() {
                self.handle_event(event);
            }
        }
    }

}

<|endoftext|>


<commit_before>//! The instruction selector
//!
//! Note: For now this will be the simplest instruction selector possible.
//!       I'll be able to improve it later, but I need something that works
//!       before I can get there.

// TODO: Finish Instruction Selection
// TODO: Write assembly pretty printer
// TODO: Add tests

use std::collections::HashMap;
use ::Ident;
use back::machine::{Address, Instruction, MachineCode, MachineRegister, NativeInt, REGISTER_COUNT};
use middle::ir;

struct InstructionSelector<'a> {
    ir: &'a ir::Program,
    code: MachineCode,
    globals: HashMap<Ident, usize>,
}

impl<'a> InstructionSelector<'a> {
    fn new(ir: &'a ir::Program) -> InstructionSelector<'a> {
        InstructionSelector {
            ir: ir,
            code: MachineCode::new(),
            globals: HashMap::new(),
        }
    }

    fn init_global(&mut self, name: &Ident, value: NativeInt, offset: usize) {
        let address = offset + REGISTER_COUNT - 1;

        // Store the offset
        self.globals.insert(*name, address);

        // Emit initialization
        self.code.emit(instruction!(MOV [Address::Immediate(address as NativeInt)] value))
    }

    fn trans_fn(&mut self, name: &Ident, body: &[ir::Block], args: &[Ident]) {
        // TODO: Generate the prologue

        for block in body {
            for inst in block.inst {
                // TODO: Translate instruction
                /* IDEA:
                let asm_instr = match *inst {
                    ir::Instruction::BinOp { op, lhs, rhs, dst } => {
                        match op {
                            ir::InfixOp::Add => instruction!(ADD ...),
                            ...
                        }
                    },
                    ir::Instruction::UnOp { op, item, dst } => { ... },
                    ...
                }
                */
            }

            // TODO: Translate closing instruction
        }

        // TODO: Generate the epilogue
    }

    fn translate(mut self) -> MachineCode {
        // First, initialize global variables
        for (offset, symbol) in self.ir.iter().enumerate() {
            if let ir::Symbol::Global { ref name, ref value } = *symbol {
                self.init_global(name, value.val() as NativeInt, offset);
            }
        }

        // Then initialize the stack management registers
        let tos = (REGISTER_COUNT + self.globals.len()) as NativeInt - 1;  // Top of stack
        self.code.emit(instruction!(MOV [Address::MachineRegister(MachineRegister::BP)] tos));
        self.code.emit(instruction!(MOV [Address::MachineRegister(MachineRegister::SP)] tos));

        // Execute the main method and then halt
        self.code.emit(instruction!(JMP Ident::new("main")));
        self.code.emit(instruction!(HALT));

        // Translate all functions
        for symbol in self.ir {
            if let ir::Symbol::Function { ref name, ref body, ref args } = *symbol {
                self.trans_fn(name, body, args);
            }
        }

        self.code
    }
}

pub fn select_instructions(ir: &ir::Program) -> MachineCode {
    let is = InstructionSelector::new(ir);
    is.translate()
}
<commit_msg>Small fix<commit_after>//! The instruction selector
//!
//! Note: For now this will be the simplest instruction selector possible.
//!       I'll be able to improve it later, but I need something that works
//!       before I can get there.

// TODO: Finish Instruction Selection
// TODO: Write assembly pretty printer
// TODO: Add tests

use std::collections::HashMap;
use ::Ident;
use back::machine::{Address, Instruction, MachineCode, MachineRegister, NativeInt, REGISTER_COUNT};
use middle::ir;

struct InstructionSelector<'a> {
    ir: &'a ir::Program,
    code: MachineCode,
    globals: HashMap<Ident, usize>,
}

impl<'a> InstructionSelector<'a> {
    fn new(ir: &'a ir::Program) -> InstructionSelector<'a> {
        InstructionSelector {
            ir: ir,
            code: MachineCode::new(),
            globals: HashMap::new(),
        }
    }

    fn init_global(&mut self, name: &Ident, value: NativeInt, offset: usize) {
        let address = offset + REGISTER_COUNT - 1;

        // Store the offset
        self.globals.insert(*name, address);

        // Emit initialization
        self.code.emit(instruction!(MOV [Address::Immediate(address as NativeInt)] value))
    }

    fn trans_fn(&mut self, name: &Ident, body: &[ir::Block], args: &[Ident]) {
        // TODO: Generate the prologue

        for block in body {
            /* IDEA:
            for inst in block.inst {
                // TODO: Translate instruction
                match *inst {
                    ir::Instruction::BinOp { op, lhs, rhs, dst } => {
                        match op {
                            ir::InfixOp::Add => {
                                self.code.emit(instruction!(MOV lhs dst));
                                self.code.emit(instruction!(ADD dst rhs));
                            }
                            ...
                        }
                    },
                    ir::Instruction::UnOp { op, item, dst } => { ... },
                    ...
                }
            }
            // */

            // TODO: Translate closing instruction
        }

        // TODO: Generate the epilogue
    }

    fn translate(mut self) -> MachineCode {
        // First, initialize global variables
        for (offset, symbol) in self.ir.iter().enumerate() {
            if let ir::Symbol::Global { ref name, ref value } = *symbol {
                self.init_global(name, value.val() as NativeInt, offset);
            }
        }

        // Then initialize the stack management registers
        let tos = (REGISTER_COUNT + self.globals.len()) as NativeInt - 1;  // Top of stack
        self.code.emit(instruction!(MOV [Address::MachineRegister(MachineRegister::BP)] tos));
        self.code.emit(instruction!(MOV [Address::MachineRegister(MachineRegister::SP)] tos));

        // Execute the main method and then halt
        self.code.emit(instruction!(JMP Ident::new("main")));
        self.code.emit(instruction!(HALT));

        // Translate all functions
        for symbol in self.ir {
            if let ir::Symbol::Function { ref name, ref body, ref args } = *symbol {
                self.trans_fn(name, body, args);
            }
        }

        self.code
    }
}

pub fn select_instructions(ir: &ir::Program) -> MachineCode {
    let is = InstructionSelector::new(ir);
    is.translate()
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::types::TypeName;
use itertools::Itertools;
use std::collections::HashMap;
use syn::{Ident, Type};

enum ArgumentConversionType {
    None,
    FromUniquePtrToValue,
    FromValueToUniquePtr,
}

pub(crate) struct ArgumentConversion {
    unwrapped_type: Type,
    conversion: ArgumentConversionType,
}

impl ArgumentConversion {
    pub(crate) fn new_unconverted(ty: Type) -> Self {
        ArgumentConversion {
            unwrapped_type: ty,
            conversion: ArgumentConversionType::None,
        }
    }

    pub(crate) fn new_to_unique_ptr(ty: Type) -> Self {
        ArgumentConversion {
            unwrapped_type: ty,
            conversion: ArgumentConversionType::FromValueToUniquePtr,
        }
    }

    pub(crate) fn new_from_unique_ptr(ty: Type) -> Self {
        ArgumentConversion {
            unwrapped_type: ty,
            conversion: ArgumentConversionType::FromUniquePtrToValue,
        }
    }

    pub(crate) fn work_needed(&self) -> bool {
        !matches!(self.conversion, ArgumentConversionType::None)
    }

    fn unconverted_type(&self) -> String {
        match self.conversion {
            ArgumentConversionType::FromUniquePtrToValue => self.wrapped_type(),
            _ => self.unwrapped_type_as_string(),
        }
    }

    fn converted_type(&self) -> String {
        match self.conversion {
            ArgumentConversionType::FromValueToUniquePtr => self.wrapped_type(),
            _ => self.unwrapped_type_as_string(),
        }
    }

    fn unwrapped_type_as_string(&self) -> String {
        crate::types::to_cpp_name(&self.unwrapped_type)
    }

    fn wrapped_type(&self) -> String {
        format!("std::unique_ptr<{}>", self.unwrapped_type_as_string())
    }

    fn conversion(&self, var_name: &str) -> String {
        match self.conversion {
            ArgumentConversionType::None => var_name.to_string(),
            ArgumentConversionType::FromUniquePtrToValue => format!("std::move(*{})", var_name),
            ArgumentConversionType::FromValueToUniquePtr => format!(
                "std::make_unique<{}>({})",
                self.unconverted_type(),
                var_name
            ),
        }
    }
}

pub(crate) struct ByValueWrapper {
    pub(crate) id: Ident,
    pub(crate) return_conversion: Option<ArgumentConversion>,
    pub(crate) argument_conversion: Vec<ArgumentConversion>,
    pub(crate) extra_blocklist: Option<String>,
    pub(crate) is_a_method: bool,
}

/// Instructions for new C++ which we need to generate.
pub(crate) enum AdditionalNeed {
    MakeStringConstructor,
    MakeUnique(TypeName, Vec<TypeName>),
    ByValueWrapper(ByValueWrapper),
}

struct AdditionalFunction {
    declaration: String,
    definition: String,
    name: String,
    suppress_older: Vec<String>,
    rename: Option<(String, String)>,
}

/// Details of additional generated C++.
pub(crate) struct AdditionalCpp {
    pub(crate) declarations: String,
    pub(crate) definitions: String,
    pub(crate) extra_allowlist: Vec<String>,
    pub(crate) extra_blocklist: Vec<String>,
    pub(crate) renames: HashMap<String, String>,
}

/// Generates additional C++ glue functions needed by autocxx.
/// At the moment, the only use here is for generating an ability
/// to do `make_unique` but more uses are expected in future.
/// In some ways it would be preferable to be able to pass snippets
/// of C++ through to `cxx` for inclusion in the C++ file which it
/// generates, and perhaps we'll explore that in future. But for now,
/// autocxx generates its own _additional_ C++ files which therefore
/// need to be built and included in linking procedures.
pub(crate) struct AdditionalCppGenerator {
    additional_functions: Vec<AdditionalFunction>,
    inclusions: String,
}

impl AdditionalCppGenerator {
    pub(crate) fn new(inclusions: String) -> Self {
        AdditionalCppGenerator {
            additional_functions: Vec::new(),
            inclusions,
        }
    }

    pub(crate) fn add_needs(&mut self, additions: Vec<AdditionalNeed>) {
        for need in additions {
            match need {
                AdditionalNeed::MakeStringConstructor => self.generate_string_constructor(),
                AdditionalNeed::MakeUnique(ty, args) => self.generate_make_unique(&ty, &args),
                AdditionalNeed::ByValueWrapper(by_value_wrapper) => {
                    self.generate_by_value_wrapper(by_value_wrapper)
                }
            }
        }
    }

    pub(crate) fn generate(&self) -> Option<AdditionalCpp> {
        if self.additional_functions.is_empty() {
            None
        } else {
            let declarations = self.concat_additional_items(|x| &x.declaration);
            let declarations = format!("#include <memory>\n{}\n{}", self.inclusions, declarations);
            let definitions = self.concat_additional_items(|x| &x.definition);
            let definitions = format!("#include \"autocxxgen.h\"\n{}", definitions);
            let extra_allowlist = self
                .additional_functions
                .iter()
                .map(|x| x.name.to_string())
                .collect();
            let extra_blocklist = self
                .additional_functions
                .iter()
                .map(|x| x.suppress_older.clone())
                .flatten()
                .collect();
            let renames = self
                .additional_functions
                .iter()
                .filter_map(|x| x.rename.clone())
                .collect();
            Some(AdditionalCpp {
                declarations,
                definitions,
                extra_allowlist,
                extra_blocklist,
                renames,
            })
        }
    }

    fn concat_additional_items<F>(&self, field_access: F) -> String
    where
        F: FnMut(&AdditionalFunction) -> &str,
    {
        let mut s = self
            .additional_functions
            .iter()
            .map(field_access)
            .collect::<Vec<&str>>()
            .join("\n\n");
        s.push('\n');
        s
    }

    fn generate_string_constructor(&mut self) {
        let name = "make_string";
        let declaration = format!(
            "std::unique_ptr<std::string> {}(const ::rust::Str& str)",
            name
        );
        let definition = format!(
            "{} {{ return std::make_unique<std::string>(std::string(str)); }}",
            declaration
        );
        let declaration = format!("{};", declaration);

        self.additional_functions.push(AdditionalFunction {
            name: name.into(),
            declaration,
            definition,
            suppress_older: Vec::new(),
            rename: None,
        })
    }

    fn generate_make_unique(&mut self, ty: &TypeName, constructor_arg_types: &[TypeName]) {
        let name = format!("{}_make_unique", ty.to_cpp_name());
        let constructor_args = constructor_arg_types
            .iter()
            .enumerate()
            .map(|(counter, ty)| format!("{} arg{}", ty.to_cpp_name(), counter))
            .join(", ");
        let declaration = format!("std::unique_ptr<{}> {}({})", ty, name, constructor_args);
        let arg_list = constructor_arg_types
            .iter()
            .enumerate()
            .map(|(counter, _)| format!("arg{}", counter))
            .join(", ");
        let definition = format!(
            "{} {{ return std::make_unique<{}>({}); }}",
            declaration, ty, arg_list
        );
        let declaration = format!("{};", declaration);
        self.additional_functions.push(AdditionalFunction {
            name,
            declaration,
            definition,
            suppress_older: Vec::new(),
            rename: None,
        })
    }

    fn generate_by_value_wrapper(&mut self, details: ByValueWrapper) {
        let ident = details.id;
        let is_a_method = details.is_a_method;
        let name = format!("{}_up_wrapper", ident.to_string());
        let get_arg_name = |counter: usize| -> String {
            if is_a_method && counter == 0 {
                // For method calls that we generate, the first
                // argument name needs to be such that we recognize
                // it as a method in the second invocation of
                // bridge_converter after it's flowed again through
                // bindgen.
                "autocxx_gen_this".to_string()
            } else {
                format!("arg{}", counter)
            }
        };
        let args = details
            .argument_conversion
            .iter()
            .enumerate()
            .map(|(counter, ty)| format!("{} {}", ty.unconverted_type(), get_arg_name(counter)))
            .join(", ");
        let ret_type = details
            .return_conversion
            .as_ref()
            .map_or("void".to_string(), |x| x.converted_type());
        let declaration = format!("{} {}({})", ret_type, name, args);
        let mut arg_list = details
            .argument_conversion
            .iter()
            .enumerate()
            .map(|(counter, conv)| conv.conversion(&get_arg_name(counter)));
        let receiver = if is_a_method { arg_list.next() } else { None };
        let arg_list = arg_list.join(", ");
        let mut underlying_function_call = format!("{}({})", ident.to_string(), arg_list);
        if let Some(receiver) = receiver {
            underlying_function_call = format!("{}.{}", receiver, underlying_function_call);
        }
        if let Some(ret) = details.return_conversion {
            underlying_function_call =
                format!("return {}", ret.conversion(&underlying_function_call));
        };
        let definition = format!("{} {{ {}; }}", declaration, underlying_function_call,);
        let declaration = format!("{};", declaration);
        let mut suppress_older = vec![ident.to_string()];
        suppress_older.extend(details.extra_blocklist.iter().cloned());
        self.additional_functions.push(AdditionalFunction {
            name: name.clone(),
            declaration,
            definition,
            suppress_older,
            rename: Some((name, ident.to_string())),
        })
    }
}
<commit_msg>Include correct headers in autogenerated C++<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::types::TypeName;
use itertools::Itertools;
use std::collections::{HashMap, HashSet};
use syn::{Ident, Type};

enum ArgumentConversionType {
    None,
    FromUniquePtrToValue,
    FromValueToUniquePtr,
}

pub(crate) struct ArgumentConversion {
    unwrapped_type: Type,
    conversion: ArgumentConversionType,
}

impl ArgumentConversion {
    pub(crate) fn new_unconverted(ty: Type) -> Self {
        ArgumentConversion {
            unwrapped_type: ty,
            conversion: ArgumentConversionType::None,
        }
    }

    pub(crate) fn new_to_unique_ptr(ty: Type) -> Self {
        ArgumentConversion {
            unwrapped_type: ty,
            conversion: ArgumentConversionType::FromValueToUniquePtr,
        }
    }

    pub(crate) fn new_from_unique_ptr(ty: Type) -> Self {
        ArgumentConversion {
            unwrapped_type: ty,
            conversion: ArgumentConversionType::FromUniquePtrToValue,
        }
    }

    pub(crate) fn work_needed(&self) -> bool {
        !matches!(self.conversion, ArgumentConversionType::None)
    }

    fn unconverted_type(&self) -> String {
        match self.conversion {
            ArgumentConversionType::FromUniquePtrToValue => self.wrapped_type(),
            _ => self.unwrapped_type_as_string(),
        }
    }

    fn converted_type(&self) -> String {
        match self.conversion {
            ArgumentConversionType::FromValueToUniquePtr => self.wrapped_type(),
            _ => self.unwrapped_type_as_string(),
        }
    }

    fn unwrapped_type_as_string(&self) -> String {
        crate::types::to_cpp_name(&self.unwrapped_type)
    }

    fn wrapped_type(&self) -> String {
        format!("std::unique_ptr<{}>", self.unwrapped_type_as_string())
    }

    fn conversion(&self, var_name: &str) -> String {
        match self.conversion {
            ArgumentConversionType::None => var_name.to_string(),
            ArgumentConversionType::FromUniquePtrToValue => format!("std::move(*{})", var_name),
            ArgumentConversionType::FromValueToUniquePtr => format!(
                "std::make_unique<{}>({})",
                self.unconverted_type(),
                var_name
            ),
        }
    }
}

pub(crate) struct ByValueWrapper {
    pub(crate) id: Ident,
    pub(crate) return_conversion: Option<ArgumentConversion>,
    pub(crate) argument_conversion: Vec<ArgumentConversion>,
    pub(crate) extra_blocklist: Option<String>,
    pub(crate) is_a_method: bool,
}

/// Instructions for new C++ which we need to generate.
pub(crate) enum AdditionalNeed {
    MakeStringConstructor,
    MakeUnique(TypeName, Vec<TypeName>),
    ByValueWrapper(ByValueWrapper),
}

#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Hash)]
struct Header {
    name: &'static str,
    system: bool,
}

impl Header {
    fn system(name: &'static str) -> Self {
        Header { name, system: true }
    }

    fn user(name: &'static str) -> Self {
        Header {
            name,
            system: false,
        }
    }

    fn include_stmt(&self) -> String {
        if self.system {
            format!("#include <{}>", self.name)
        } else {
            format!("#include \"{}\"", self.name)
        }
    }
}

struct AdditionalFunction {
    declaration: String,
    definition: String,
    name: String,
    suppress_older: Vec<String>,
    rename: Option<(String, String)>,
    headers: Vec<Header>,
}

/// Details of additional generated C++.
pub(crate) struct AdditionalCpp {
    pub(crate) declarations: String,
    pub(crate) definitions: String,
    pub(crate) extra_allowlist: Vec<String>,
    pub(crate) extra_blocklist: Vec<String>,
    pub(crate) renames: HashMap<String, String>,
}

/// Generates additional C++ glue functions needed by autocxx.
/// At the moment, the only use here is for generating an ability
/// to do `make_unique` but more uses are expected in future.
/// In some ways it would be preferable to be able to pass snippets
/// of C++ through to `cxx` for inclusion in the C++ file which it
/// generates, and perhaps we'll explore that in future. But for now,
/// autocxx generates its own _additional_ C++ files which therefore
/// need to be built and included in linking procedures.
pub(crate) struct AdditionalCppGenerator {
    additional_functions: Vec<AdditionalFunction>,
    inclusions: String,
}

impl AdditionalCppGenerator {
    pub(crate) fn new(inclusions: String) -> Self {
        AdditionalCppGenerator {
            additional_functions: Vec::new(),
            inclusions,
        }
    }

    pub(crate) fn add_needs(&mut self, additions: Vec<AdditionalNeed>) {
        for need in additions {
            match need {
                AdditionalNeed::MakeStringConstructor => self.generate_string_constructor(),
                AdditionalNeed::MakeUnique(ty, args) => self.generate_make_unique(&ty, &args),
                AdditionalNeed::ByValueWrapper(by_value_wrapper) => {
                    self.generate_by_value_wrapper(by_value_wrapper)
                }
            }
        }
    }

    pub(crate) fn generate(&self) -> Option<AdditionalCpp> {
        if self.additional_functions.is_empty() {
            None
        } else {
            let headers: HashSet<Header> = self
                .additional_functions
                .iter()
                .map(|x| x.headers.iter().cloned())
                .flatten()
                .collect();
            let headers = headers.iter().map(|x| x.include_stmt()).join("\n");
            let declarations = self.concat_additional_items(|x| &x.declaration);
            let declarations = format!("{}\n{}\n{}", headers, self.inclusions, declarations);
            let definitions = self.concat_additional_items(|x| &x.definition);
            let definitions = format!("#include \"autocxxgen.h\"\n{}", definitions);
            let extra_allowlist = self
                .additional_functions
                .iter()
                .map(|x| x.name.to_string())
                .collect();
            let extra_blocklist = self
                .additional_functions
                .iter()
                .map(|x| x.suppress_older.clone())
                .flatten()
                .collect();
            let renames = self
                .additional_functions
                .iter()
                .filter_map(|x| x.rename.clone())
                .collect();
            Some(AdditionalCpp {
                declarations,
                definitions,
                extra_allowlist,
                extra_blocklist,
                renames,
            })
        }
    }

    fn concat_additional_items<F>(&self, field_access: F) -> String
    where
        F: FnMut(&AdditionalFunction) -> &str,
    {
        let mut s = self
            .additional_functions
            .iter()
            .map(field_access)
            .collect::<Vec<&str>>()
            .join("\n\n");
        s.push('\n');
        s
    }

    fn generate_string_constructor(&mut self) {
        let name = "make_string";
        let declaration = format!(
            "std::unique_ptr<std::string> {}(const ::rust::Str& str)",
            name
        );
        let definition = format!(
            "{} {{ return std::make_unique<std::string>(std::string(str)); }}",
            declaration
        );
        let declaration = format!("{};", declaration);

        self.additional_functions.push(AdditionalFunction {
            name: name.into(),
            declaration,
            definition,
            suppress_older: Vec::new(),
            rename: None,
            headers: vec![
                Header::system("memory"),
                Header::system("string"),
                Header::user("cxx.h"),
            ],
        })
    }

    fn generate_make_unique(&mut self, ty: &TypeName, constructor_arg_types: &[TypeName]) {
        let name = format!("{}_make_unique", ty.to_cpp_name());
        let constructor_args = constructor_arg_types
            .iter()
            .enumerate()
            .map(|(counter, ty)| format!("{} arg{}", ty.to_cpp_name(), counter))
            .join(", ");
        let declaration = format!("std::unique_ptr<{}> {}({})", ty, name, constructor_args);
        let arg_list = constructor_arg_types
            .iter()
            .enumerate()
            .map(|(counter, _)| format!("arg{}", counter))
            .join(", ");
        let definition = format!(
            "{} {{ return std::make_unique<{}>({}); }}",
            declaration, ty, arg_list
        );
        let declaration = format!("{};", declaration);
        self.additional_functions.push(AdditionalFunction {
            name,
            declaration,
            definition,
            suppress_older: Vec::new(),
            rename: None,
            headers: vec![Header::system("memory")],
        })
    }

    fn generate_by_value_wrapper(&mut self, details: ByValueWrapper) {
        let ident = details.id;
        let is_a_method = details.is_a_method;
        let name = format!("{}_up_wrapper", ident.to_string());
        let get_arg_name = |counter: usize| -> String {
            if is_a_method && counter == 0 {
                // For method calls that we generate, the first
                // argument name needs to be such that we recognize
                // it as a method in the second invocation of
                // bridge_converter after it's flowed again through
                // bindgen.
                "autocxx_gen_this".to_string()
            } else {
                format!("arg{}", counter)
            }
        };
        let args = details
            .argument_conversion
            .iter()
            .enumerate()
            .map(|(counter, ty)| format!("{} {}", ty.unconverted_type(), get_arg_name(counter)))
            .join(", ");
        let ret_type = details
            .return_conversion
            .as_ref()
            .map_or("void".to_string(), |x| x.converted_type());
        let declaration = format!("{} {}({})", ret_type, name, args);
        let mut arg_list = details
            .argument_conversion
            .iter()
            .enumerate()
            .map(|(counter, conv)| conv.conversion(&get_arg_name(counter)));
        let receiver = if is_a_method { arg_list.next() } else { None };
        let arg_list = arg_list.join(", ");
        let mut underlying_function_call = format!("{}({})", ident.to_string(), arg_list);
        if let Some(receiver) = receiver {
            underlying_function_call = format!("{}.{}", receiver, underlying_function_call);
        }
        if let Some(ret) = details.return_conversion {
            underlying_function_call =
                format!("return {}", ret.conversion(&underlying_function_call));
        };
        let definition = format!("{} {{ {}; }}", declaration, underlying_function_call,);
        let declaration = format!("{};", declaration);
        let mut suppress_older = vec![ident.to_string()];
        suppress_older.extend(details.extra_blocklist.iter().cloned());
        self.additional_functions.push(AdditionalFunction {
            name: name.clone(),
            declaration,
            definition,
            suppress_older,
            rename: Some((name, ident.to_string())),
            headers: vec![Header::system("memory")],
        })
    }
}
<|endoftext|>


<commit_before>#[cfg(test)]
mod tests;

/// Uses a sorted slice `data: &[E]` as a kind of "multi-map". The
/// `key_fn` extracts a key of type `K` from the data, and this
/// function finds the range of elements that match the key. `data`
/// must have been sorted as if by a call to `sort_by_key` for this to
/// work.
pub fn binary_search_slice<E, K>(data: &'d [E], key_fn: impl Fn(&E) -> K, key: &K) -> &'d [E]
where
    K: Ord,
{
    let mid = match data.binary_search_by_key(key, &key_fn) {
        Ok(mid) => mid,
        Err(_) => return &[],
    };

    // We get back *some* element with the given key -- so
    // search backwards to find the *first* one.
    //
    // (It'd be more efficient to use a "galloping" search
    // here, but it's not really worth it for small-ish
    // amounts of data.)
    let mut start = mid;
    while start > 0 {
        if key_fn(&data[start - 1]) == *key {
            start -= 1;
        } else {
            break;
        }
    }

    // Now search forward to find the *last* one.
    //
    // (It'd be more efficient to use a "galloping" search
    // here, but it's not really worth it for small-ish
    // amounts of data.)
    let mut end = mid + 1;
    let max = data.len();
    while end < max {
        if key_fn(&data[end]) == *key {
            end += 1;
        } else {
            break;
        }
    }

    &data[start..end]
}
<commit_msg>Rollup merge of #67948 - llogiq:gallop, r=Mark-Simulacrum<commit_after>#[cfg(test)]
mod tests;

/// Uses a sorted slice `data: &[E]` as a kind of "multi-map". The
/// `key_fn` extracts a key of type `K` from the data, and this
/// function finds the range of elements that match the key. `data`
/// must have been sorted as if by a call to `sort_by_key` for this to
/// work.
pub fn binary_search_slice<E, K>(data: &'d [E], key_fn: impl Fn(&E) -> K, key: &K) -> &'d [E]
where
    K: Ord,
{
    let mid = match data.binary_search_by_key(key, &key_fn) {
        Ok(mid) => mid,
        Err(_) => return &[],
    };
    let size = data.len();

    // We get back *some* element with the given key -- so do
    // a galloping search backwards to find the *first* one.
    let mut start = mid;
    let mut previous = mid;
    let mut step = 1;
    loop {
        start = start.saturating_sub(step);
        if start == 0 || key_fn(&data[start]) != *key {
            break;
        }
        previous = start;
        step *= 2;
    }
    step = previous - start;
    while step > 1 {
        let half = step / 2;
        let mid = start + half;
        if key_fn(&data[mid]) != *key {
            start = mid;
        }
        step -= half;
    }
    // adjust by one if we have overshot
    if start < size && key_fn(&data[start]) != *key {
        start += 1;
    }

    // Now search forward to find the *last* one.
    let mut end = mid;
    let mut previous = mid;
    let mut step = 1;
    loop {
        end = end.saturating_add(step).min(size);
        if end == size || key_fn(&data[end]) != *key {
            break;
        }
        previous = end;
        step *= 2;
    }
    step = end - previous;
    while step > 1 {
        let half = step / 2;
        let mid = end - half;
        if key_fn(&data[mid]) != *key {
            end = mid;
        }
        step -= half;
    }

    &data[start..end]
}
<|endoftext|>