<commit_before>//! FIXME: write short doc here

use std::path::{Path, PathBuf};

use cargo_metadata::{CargoOpt, MetadataCommand};
use ra_arena::{impl_arena_id, Arena, RawId};
use ra_db::Edition;
use rustc_hash::FxHashMap;
use serde::Deserialize;

use crate::Result;

/// `CargoWorkspace` represents the logical structure of, well, a Cargo
/// workspace. It pretty closely mirrors `cargo metadata` output.
///
/// Note that internally, rust analyzer uses a different structure:
/// `CrateGraph`. `CrateGraph` is lower-level: it knows only about the crates,
/// while this knows about `Packages` & `Targets`: purely cargo-related
/// concepts.
#[derive(Debug, Clone)]
pub struct CargoWorkspace {
    packages: Arena<Package, PackageData>,
    targets: Arena<Target, TargetData>,
    workspace_root: PathBuf,
}

#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase", default)]
pub struct CargoFeatures {
    /// Do not activate the `default` feature.
    pub no_default_features: bool,

    /// Activate all available features
    pub all_features: bool,

    /// List of features to activate.
    /// This will be ignored if `cargo_all_features` is true.
    pub features: Vec<String>,
}

impl Default for CargoFeatures {
    fn default() -> Self {
        CargoFeatures { no_default_features: false, all_features: true, features: Vec::new() }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Package(RawId);
impl_arena_id!(Package);

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Target(RawId);
impl_arena_id!(Target);

#[derive(Debug, Clone)]
struct PackageData {
    name: String,
    manifest: PathBuf,
    targets: Vec<Target>,
    is_member: bool,
    dependencies: Vec<PackageDependency>,
    edition: Edition,
    features: Vec<String>,
}

#[derive(Debug, Clone)]
pub struct PackageDependency {
    pub pkg: Package,
    pub name: String,
}

#[derive(Debug, Clone)]
struct TargetData {
    pkg: Package,
    name: String,
    root: PathBuf,
    kind: TargetKind,
    is_proc_macro: bool,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TargetKind {
    Bin,
    /// Any kind of Cargo lib crate-type (dylib, rlib, proc-macro, ...).
    Lib,
    Example,
    Test,
    Bench,
    Other,
}

impl TargetKind {
    fn new(kinds: &[String]) -> TargetKind {
        for kind in kinds {
            return match kind.as_str() {
                "bin" => TargetKind::Bin,
                "test" => TargetKind::Test,
                "bench" => TargetKind::Bench,
                "example" => TargetKind::Example,
                "proc-macro" => TargetKind::Lib,
                _ if kind.contains("lib") => TargetKind::Lib,
                _ => continue,
            };
        }
        TargetKind::Other
    }
}

impl Package {
    pub fn name(self, ws: &CargoWorkspace) -> &str {
        ws.packages[self].name.as_str()
    }
    pub fn root(self, ws: &CargoWorkspace) -> &Path {
        ws.packages[self].manifest.parent().unwrap()
    }
    pub fn edition(self, ws: &CargoWorkspace) -> Edition {
        ws.packages[self].edition
    }
    pub fn features(self, ws: &CargoWorkspace) -> &[String] {
        &ws.packages[self].features
    }
    pub fn targets<'a>(self, ws: &'a CargoWorkspace) -> impl Iterator<Item = Target> + 'a {
        ws.packages[self].targets.iter().cloned()
    }
    #[allow(unused)]
    pub fn is_member(self, ws: &CargoWorkspace) -> bool {
        ws.packages[self].is_member
    }
    pub fn dependencies<'a>(
        self,
        ws: &'a CargoWorkspace,
    ) -> impl Iterator<Item = &'a PackageDependency> + 'a {
        ws.packages[self].dependencies.iter()
    }
}

impl Target {
    pub fn package(self, ws: &CargoWorkspace) -> Package {
        ws.targets[self].pkg
    }
    pub fn name(self, ws: &CargoWorkspace) -> &str {
        ws.targets[self].name.as_str()
    }
    pub fn root(self, ws: &CargoWorkspace) -> &Path {
        ws.targets[self].root.as_path()
    }
    pub fn kind(self, ws: &CargoWorkspace) -> TargetKind {
        ws.targets[self].kind
    }
    pub fn is_proc_macro(self, ws: &CargoWorkspace) -> bool {
        ws.targets[self].is_proc_macro
    }
}

impl CargoWorkspace {
    pub fn from_cargo_metadata(
        cargo_toml: &Path,
        cargo_features: &CargoFeatures,
    ) -> Result<CargoWorkspace> {
        let mut meta = MetadataCommand::new();
        meta.manifest_path(cargo_toml);
        if cargo_features.all_features {
            meta.features(CargoOpt::AllFeatures);
        } else if cargo_features.no_default_features {
            // FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
            // https://github.com/oli-obk/cargo_metadata/issues/79
            meta.features(CargoOpt::NoDefaultFeatures);
        } else if cargo_features.features.len() > 0 {
            meta.features(CargoOpt::SomeFeatures(cargo_features.features.clone()));
        }
        if let Some(parent) = cargo_toml.parent() {
            meta.current_dir(parent);
        }
        let meta = meta.exec().map_err(|e| format!("cargo metadata failed: {}", e))?;
        let mut pkg_by_id = FxHashMap::default();
        let mut packages = Arena::default();
        let mut targets = Arena::default();

        let ws_members = &meta.workspace_members;

        for meta_pkg in meta.packages {
            let cargo_metadata::Package { id, edition, name, manifest_path, .. } = meta_pkg;
            let is_member = ws_members.contains(&id);
            let edition = edition.parse::<Edition>()?;
            let pkg = packages.alloc(PackageData {
                name,
                manifest: manifest_path,
                targets: Vec::new(),
                is_member,
                edition,
                dependencies: Vec::new(),
                features: Vec::new(),
            });
            let pkg_data = &mut packages[pkg];
            pkg_by_id.insert(id, pkg);
            for meta_tgt in meta_pkg.targets {
                let is_proc_macro = meta_tgt.kind.as_slice() == &["proc-macro"];
                let tgt = targets.alloc(TargetData {
                    pkg,
                    name: meta_tgt.name,
                    root: meta_tgt.src_path.clone(),
                    kind: TargetKind::new(meta_tgt.kind.as_slice()),
                    is_proc_macro,
                });
                pkg_data.targets.push(tgt);
            }
        }
        let resolve = meta.resolve.expect("metadata executed with deps");
        for node in resolve.nodes {
            let source = match pkg_by_id.get(&node.id) {
                Some(&src) => src,
                None => {
                    log::error!("Node id do not match in cargo metadata, ignoring {}", node.id);
                    continue;
                }
            };
            for dep_node in node.deps {
                let pkg = match pkg_by_id.get(&dep_node.pkg) {
                    Some(&pkg) => pkg,
                    None => {
                        log::error!(
                            "Dep node id do not match in cargo metadata, ignoring {}",
                            dep_node.pkg
                        );
                        continue;
                    }
                };
                let dep = PackageDependency { name: dep_node.name, pkg };
                packages[source].dependencies.push(dep);
            }
            packages[source].features.extend(node.features);
        }

        Ok(CargoWorkspace { packages, targets, workspace_root: meta.workspace_root })
    }

    pub fn packages<'a>(&'a self) -> impl Iterator<Item = Package> + ExactSizeIterator + 'a {
        self.packages.iter().map(|(id, _pkg)| id)
    }

    pub fn target_by_root(&self, root: &Path) -> Option<Target> {
        self.packages().filter_map(|pkg| pkg.targets(self).find(|it| it.root(self) == root)).next()
    }

    pub fn workspace_root(&self) -> &Path {
        &self.workspace_root
    }
}
<commit_msg>Add a FIXME note<commit_after>//! FIXME: write short doc here

use std::path::{Path, PathBuf};

use cargo_metadata::{CargoOpt, MetadataCommand};
use ra_arena::{impl_arena_id, Arena, RawId};
use ra_db::Edition;
use rustc_hash::FxHashMap;
use serde::Deserialize;

use crate::Result;

/// `CargoWorkspace` represents the logical structure of, well, a Cargo
/// workspace. It pretty closely mirrors `cargo metadata` output.
///
/// Note that internally, rust analyzer uses a different structure:
/// `CrateGraph`. `CrateGraph` is lower-level: it knows only about the crates,
/// while this knows about `Packages` & `Targets`: purely cargo-related
/// concepts.
#[derive(Debug, Clone)]
pub struct CargoWorkspace {
    packages: Arena<Package, PackageData>,
    targets: Arena<Target, TargetData>,
    workspace_root: PathBuf,
}

#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase", default)]
pub struct CargoFeatures {
    /// Do not activate the `default` feature.
    pub no_default_features: bool,

    /// Activate all available features
    pub all_features: bool,

    /// List of features to activate.
    /// This will be ignored if `cargo_all_features` is true.
    pub features: Vec<String>,
}

impl Default for CargoFeatures {
    fn default() -> Self {
        CargoFeatures { no_default_features: false, all_features: true, features: Vec::new() }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Package(RawId);
impl_arena_id!(Package);

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Target(RawId);
impl_arena_id!(Target);

#[derive(Debug, Clone)]
struct PackageData {
    name: String,
    manifest: PathBuf,
    targets: Vec<Target>,
    is_member: bool,
    dependencies: Vec<PackageDependency>,
    edition: Edition,
    features: Vec<String>,
}

#[derive(Debug, Clone)]
pub struct PackageDependency {
    pub pkg: Package,
    pub name: String,
}

#[derive(Debug, Clone)]
struct TargetData {
    pkg: Package,
    name: String,
    root: PathBuf,
    kind: TargetKind,
    is_proc_macro: bool,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TargetKind {
    Bin,
    /// Any kind of Cargo lib crate-type (dylib, rlib, proc-macro, ...).
    Lib,
    Example,
    Test,
    Bench,
    Other,
}

impl TargetKind {
    fn new(kinds: &[String]) -> TargetKind {
        for kind in kinds {
            return match kind.as_str() {
                "bin" => TargetKind::Bin,
                "test" => TargetKind::Test,
                "bench" => TargetKind::Bench,
                "example" => TargetKind::Example,
                "proc-macro" => TargetKind::Lib,
                _ if kind.contains("lib") => TargetKind::Lib,
                _ => continue,
            };
        }
        TargetKind::Other
    }
}

impl Package {
    pub fn name(self, ws: &CargoWorkspace) -> &str {
        ws.packages[self].name.as_str()
    }
    pub fn root(self, ws: &CargoWorkspace) -> &Path {
        ws.packages[self].manifest.parent().unwrap()
    }
    pub fn edition(self, ws: &CargoWorkspace) -> Edition {
        ws.packages[self].edition
    }
    pub fn features(self, ws: &CargoWorkspace) -> &[String] {
        &ws.packages[self].features
    }
    pub fn targets<'a>(self, ws: &'a CargoWorkspace) -> impl Iterator<Item = Target> + 'a {
        ws.packages[self].targets.iter().cloned()
    }
    #[allow(unused)]
    pub fn is_member(self, ws: &CargoWorkspace) -> bool {
        ws.packages[self].is_member
    }
    pub fn dependencies<'a>(
        self,
        ws: &'a CargoWorkspace,
    ) -> impl Iterator<Item = &'a PackageDependency> + 'a {
        ws.packages[self].dependencies.iter()
    }
}

impl Target {
    pub fn package(self, ws: &CargoWorkspace) -> Package {
        ws.targets[self].pkg
    }
    pub fn name(self, ws: &CargoWorkspace) -> &str {
        ws.targets[self].name.as_str()
    }
    pub fn root(self, ws: &CargoWorkspace) -> &Path {
        ws.targets[self].root.as_path()
    }
    pub fn kind(self, ws: &CargoWorkspace) -> TargetKind {
        ws.targets[self].kind
    }
    pub fn is_proc_macro(self, ws: &CargoWorkspace) -> bool {
        ws.targets[self].is_proc_macro
    }
}

impl CargoWorkspace {
    pub fn from_cargo_metadata(
        cargo_toml: &Path,
        cargo_features: &CargoFeatures,
    ) -> Result<CargoWorkspace> {
        let mut meta = MetadataCommand::new();
        meta.manifest_path(cargo_toml);
        if cargo_features.all_features {
            meta.features(CargoOpt::AllFeatures);
        } else if cargo_features.no_default_features {
            // FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
            // https://github.com/oli-obk/cargo_metadata/issues/79
            meta.features(CargoOpt::NoDefaultFeatures);
        } else if cargo_features.features.len() > 0 {
            meta.features(CargoOpt::SomeFeatures(cargo_features.features.clone()));
        }
        if let Some(parent) = cargo_toml.parent() {
            meta.current_dir(parent);
        }
        let meta = meta.exec().map_err(|e| format!("cargo metadata failed: {}", e))?;
        let mut pkg_by_id = FxHashMap::default();
        let mut packages = Arena::default();
        let mut targets = Arena::default();

        let ws_members = &meta.workspace_members;

        for meta_pkg in meta.packages {
            let cargo_metadata::Package { id, edition, name, manifest_path, .. } = meta_pkg;
            let is_member = ws_members.contains(&id);
            let edition = edition.parse::<Edition>()?;
            let pkg = packages.alloc(PackageData {
                name,
                manifest: manifest_path,
                targets: Vec::new(),
                is_member,
                edition,
                dependencies: Vec::new(),
                features: Vec::new(),
            });
            let pkg_data = &mut packages[pkg];
            pkg_by_id.insert(id, pkg);
            for meta_tgt in meta_pkg.targets {
                let is_proc_macro = meta_tgt.kind.as_slice() == &["proc-macro"];
                let tgt = targets.alloc(TargetData {
                    pkg,
                    name: meta_tgt.name,
                    root: meta_tgt.src_path.clone(),
                    kind: TargetKind::new(meta_tgt.kind.as_slice()),
                    is_proc_macro,
                });
                pkg_data.targets.push(tgt);
            }
        }
        let resolve = meta.resolve.expect("metadata executed with deps");
        for node in resolve.nodes {
            let source = match pkg_by_id.get(&node.id) {
                Some(&src) => src,
                // FIXME: replace this and a similar branch below with `.unwrap`, once
                // https://github.com/rust-lang/cargo/issues/7841
                // is fixed and hits stable (around 1.43-is probably?).
                None => {
                    log::error!("Node id do not match in cargo metadata, ignoring {}", node.id);
                    continue;
                }
            };
            for dep_node in node.deps {
                let pkg = match pkg_by_id.get(&dep_node.pkg) {
                    Some(&pkg) => pkg,
                    None => {
                        log::error!(
                            "Dep node id do not match in cargo metadata, ignoring {}",
                            dep_node.pkg
                        );
                        continue;
                    }
                };
                let dep = PackageDependency { name: dep_node.name, pkg };
                packages[source].dependencies.push(dep);
            }
            packages[source].features.extend(node.features);
        }

        Ok(CargoWorkspace { packages, targets, workspace_root: meta.workspace_root })
    }

    pub fn packages<'a>(&'a self) -> impl Iterator<Item = Package> + ExactSizeIterator + 'a {
        self.packages.iter().map(|(id, _pkg)| id)
    }

    pub fn target_by_root(&self, root: &Path) -> Option<Target> {
        self.packages().filter_map(|pkg| pkg.targets(self).find(|it| it.root(self) == root)).next()
    }

    pub fn workspace_root(&self) -> &Path {
        &self.workspace_root
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::context::XContext;
use guppy::graph::summaries::{diff::SummaryDiff, Summary};
use std::{fs, path::PathBuf};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
pub struct Args {
    #[structopt(name = "BASE_SUMMARY")]
    /// Path to the base summary
    base_summary: PathBuf,
    #[structopt(name = "COMPARE_SUMMARY")]
    /// Path to the comparison summary
    compare_summary: PathBuf,
}

pub fn run(args: Args, _xctx: XContext) -> crate::Result<()> {
    let base_summary_text = fs::read_to_string(&args.base_summary)?;
    let base_summary = Summary::parse(&base_summary_text)?;
    let compare_summary_text = fs::read_to_string(&args.compare_summary)?;
    let compare_summary = Summary::parse(&compare_summary_text)?;

    let summary_diff = SummaryDiff::new(&base_summary, &compare_summary);
    println!("{}", summary_diff.report());
    Ok(())
}
<commit_msg>[x] adding json output for cargo x diff-summaries<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::context::XContext;
use guppy::graph::summaries::{diff::SummaryDiff, Summary};
use std::{fs, path::PathBuf};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
pub struct Args {
    #[structopt(name = "BASE_SUMMARY")]
    /// Path to the base summary
    base_summary: PathBuf,
    #[structopt(name = "COMPARE_SUMMARY")]
    /// Path to the comparison summary
    compare_summary: PathBuf,
    #[structopt(name = "OUTPUT_FORMAT")]
    /// optionally, output can be formated as json or toml
    output_format: Option<String>,
}

pub fn run(args: Args, _xctx: XContext) -> crate::Result<()> {
    let base_summary_text = fs::read_to_string(&args.base_summary)?;
    let base_summary = Summary::parse(&base_summary_text)?;
    let compare_summary_text = fs::read_to_string(&args.compare_summary)?;
    let compare_summary = Summary::parse(&compare_summary_text)?;

    let summary_diff = SummaryDiff::new(&base_summary, &compare_summary);

    match args.output_format.as_deref() {
        Some("json") => println!("{}", serde_json::to_string(&summary_diff)?),
        Some("toml") => println!("{}", toml::to_string(&summary_diff)?),
        _ => println!("{}", summary_diff.report()),
    };

    Ok(())
}
<|endoftext|>


<commit_before>//! This calculates the types which has storage which lives across a suspension point in a
//! generator from the perspective of typeck. The actual types used at runtime
//! is calculated in `rustc_mir::transform::generator` and may be a subset of the
//! types computed here.

use super::FnCtxt;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_hir as hir;
use rustc_hir::def::{CtorKind, DefKind, Res};
use rustc_hir::def_id::DefId;
use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
use rustc_hir::{Expr, ExprKind, Pat, PatKind};
use rustc_middle::middle::region::{self, YieldData};
use rustc_middle::ty::{self, Ty};
use rustc_span::Span;

struct InteriorVisitor<'a, 'tcx> {
    fcx: &'a FnCtxt<'a, 'tcx>,
    closure_def_id: DefId,
    types: FxHashMap<ty::GeneratorInteriorTypeCause<'tcx>, usize>,
    region_scope_tree: &'tcx region::ScopeTree,
    expr_count: usize,
    kind: hir::GeneratorKind,
    prev_unresolved_span: Option<Span>,
}

impl<'a, 'tcx> InteriorVisitor<'a, 'tcx> {
    fn record(
        &mut self,
        ty: Ty<'tcx>,
        scope: Option<region::Scope>,
        expr: Option<&'tcx Expr<'tcx>>,
        source_span: Span,
        is_upvar: bool,
    ) {
        use rustc_span::DUMMY_SP;

        debug!(
            "generator_interior: attempting to record type {:?} {:?} {:?} {:?}",
            ty, scope, expr, source_span
        );

        let live_across_yield = scope
            .map(|s| {
                self.region_scope_tree.yield_in_scope(s).and_then(|yield_data| {
                    // If we are recording an expression that is the last yield
                    // in the scope, or that has a postorder CFG index larger
                    // than the one of all of the yields, then its value can't
                    // be storage-live (and therefore live) at any of the yields.
                    //
                    // See the mega-comment at `yield_in_scope` for a proof.

                    debug!(
                        "comparing counts yield: {} self: {}, source_span = {:?}",
                        yield_data.expr_and_pat_count, self.expr_count, source_span
                    );

                    if yield_data.expr_and_pat_count >= self.expr_count {
                        Some(yield_data)
                    } else {
                        None
                    }
                })
            })
            .unwrap_or_else(|| {
                Some(YieldData { span: DUMMY_SP, expr_and_pat_count: 0, source: self.kind.into() })
            });

        if let Some(yield_data) = live_across_yield {
            let ty = self.fcx.resolve_vars_if_possible(&ty);
            debug!(
                "type in expr = {:?}, scope = {:?}, type = {:?}, count = {}, yield_span = {:?}",
                expr, scope, ty, self.expr_count, yield_data.span
            );

            if let Some((unresolved_type, unresolved_type_span)) =
                self.fcx.unresolved_type_vars(&ty)
            {
                let note = format!(
                    "the type is part of the {} because of this {}",
                    self.kind, yield_data.source
                );

                // If unresolved type isn't a ty_var then unresolved_type_span is None
                let span = self
                    .prev_unresolved_span
                    .unwrap_or_else(|| unresolved_type_span.unwrap_or(source_span));
                self.fcx
                    .need_type_info_err_in_generator(self.kind, span, unresolved_type)
                    .span_note(yield_data.span, &*note)
                    .emit();
            } else {
                // Map the type to the number of types added before it
                let entries = self.types.len();
                let scope_span = scope.map(|s| s.span(self.fcx.tcx, self.region_scope_tree));
                self.types
                    .entry(ty::GeneratorInteriorTypeCause {
                        span: source_span,
                        ty: &ty,
                        scope_span,
                        yield_span: Some(yield_data.span),
                        expr: expr.map(|e| e.hir_id),
                    })
                    .or_insert(entries);
            }
        } else {
            debug!(
                "no type in expr = {:?}, count = {:?}, span = {:?}",
                expr,
                self.expr_count,
                expr.map(|e| e.span)
            );
            let ty = self.fcx.resolve_vars_if_possible(&ty);
            if let Some((unresolved_type, unresolved_type_span)) =
                self.fcx.unresolved_type_vars(&ty)
            {
                debug!(
                    "remained unresolved_type = {:?}, unresolved_type_span: {:?}",
                    unresolved_type, unresolved_type_span
                );
                self.prev_unresolved_span = unresolved_type_span;
            } else {
                if is_upvar {
                    let entries = self.types.len();
                    let scope_span = scope.map(|s| s.span(self.fcx.tcx, self.region_scope_tree));
                    self.types
                        .entry(ty::GeneratorInteriorTypeCause {
                            span: source_span,
                            ty: &ty,
                            scope_span,
                            yield_span: None,
                            expr: expr.map(|e| e.hir_id),
                        })
                        .or_insert(entries);
                }
            }
        }
    }
}

pub fn resolve_interior<'a, 'tcx>(
    fcx: &'a FnCtxt<'a, 'tcx>,
    def_id: DefId,
    body_id: hir::BodyId,
    interior: Ty<'tcx>,
    kind: hir::GeneratorKind,
) {
    let body = fcx.tcx.hir().body(body_id);

    let closure_def_id = fcx.tcx.hir().body_owner_def_id(body_id).to_def_id();

    let mut visitor = InteriorVisitor {
        fcx,
        closure_def_id,
        types: FxHashMap::default(),
        region_scope_tree: fcx.tcx.region_scope_tree(def_id),
        expr_count: 0,
        kind,
        prev_unresolved_span: None,
    };
    intravisit::walk_body(&mut visitor, body);

    // Check that we visited the same amount of expressions and the RegionResolutionVisitor
    let region_expr_count = visitor.region_scope_tree.body_expr_count(body_id).unwrap();
    assert_eq!(region_expr_count, visitor.expr_count);

    let mut types: Vec<_> = visitor.types.drain().collect();

    // Sort types by insertion order
    types.sort_by_key(|t| t.1);

    // The types in the generator interior contain lifetimes local to the generator itself,
    // which should not be exposed outside of the generator. Therefore, we replace these
    // lifetimes with existentially-bound lifetimes, which reflect the exact value of the
    // lifetimes not being known by users.
    //
    // These lifetimes are used in auto trait impl checking (for example,
    // if a Sync generator contains an &'α T, we need to check whether &'α T: Sync),
    // so knowledge of the exact relationships between them isn't particularly important.

    debug!("types in generator {:?}, span = {:?}", types, body.value.span);

    let mut counter = 0;
    let mut captured_tys = FxHashSet::default();
    let type_causes: Vec<_> = types
        .into_iter()
        .filter_map(|(mut cause, _)| {
            // Erase regions and canonicalize late-bound regions to deduplicate as many types as we
            // can.
            let erased = fcx.tcx.erase_regions(&cause.ty);
            if captured_tys.insert(erased) {
                // Replace all regions inside the generator interior with late bound regions.
                // Note that each region slot in the types gets a new fresh late bound region,
                // which means that none of the regions inside relate to any other, even if
                // typeck had previously found constraints that would cause them to be related.
                let folded = fcx.tcx.fold_regions(&erased, &mut false, |_, current_depth| {
                    counter += 1;
                    fcx.tcx.mk_region(ty::ReLateBound(current_depth, ty::BrAnon(counter)))
                });

                cause.ty = folded;
                Some(cause)
            } else {
                None
            }
        })
        .collect();

    // Extract type components to build the witness type.
    let type_list = fcx.tcx.mk_type_list(type_causes.iter().map(|cause| cause.ty));
    let witness = fcx.tcx.mk_generator_witness(ty::Binder::bind(type_list));

    // Store the generator types and spans into the tables for this generator.
    visitor.fcx.inh.tables.borrow_mut().generator_interior_types = type_causes;

    debug!(
        "types in generator after region replacement {:?}, span = {:?}",
        witness, body.value.span
    );

    // Unify the type variable inside the generator with the new witness
    match fcx.at(&fcx.misc(body.value.span), fcx.param_env).eq(interior, witness) {
        Ok(ok) => fcx.register_infer_ok_obligations(ok),
        _ => bug!(),
    }
}

// This visitor has to have the same visit_expr calls as RegionResolutionVisitor in
// librustc_middle/middle/region.rs since `expr_count` is compared against the results
// there.
impl<'a, 'tcx> Visitor<'tcx> for InteriorVisitor<'a, 'tcx> {
    type Map = intravisit::ErasedMap<'tcx>;

    fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> {
        NestedVisitorMap::None
    }

    fn visit_pat(&mut self, pat: &'tcx Pat<'tcx>) {
        intravisit::walk_pat(self, pat);

        self.expr_count += 1;

        if let PatKind::Binding(..) = pat.kind {
            let scope = self.region_scope_tree.var_scope(pat.hir_id.local_id);
            let ty = self.fcx.tables.borrow().pat_ty(pat);
            self.record(ty, Some(scope), None, pat.span, false);
        }
    }

    fn visit_expr(&mut self, expr: &'tcx Expr<'tcx>) {
        match &expr.kind {
            ExprKind::Call(callee, args) => match &callee.kind {
                ExprKind::Path(qpath) => {
                    let res = self.fcx.tables.borrow().qpath_res(qpath, callee.hir_id);
                    match res {
                        // Direct calls never need to keep the callee `ty::FnDef`
                        // ZST in a temporary, so skip its type, just in case it
                        // can significantly complicate the generator type.
                        Res::Def(
                            DefKind::Fn | DefKind::AssocFn | DefKind::Ctor(_, CtorKind::Fn),
                            _,
                        ) => {
                            // NOTE(eddyb) this assumes a path expression has
                            // no nested expressions to keep track of.
                            self.expr_count += 1;

                            // Record the rest of the call expression normally.
                            for arg in *args {
                                self.visit_expr(arg);
                            }
                        }
                        _ => intravisit::walk_expr(self, expr),
                    }
                }
                _ => intravisit::walk_expr(self, expr),
            },
            _ => intravisit::walk_expr(self, expr),
        }

        self.expr_count += 1;

        let scope = self.region_scope_tree.temporary_scope(expr.hir_id.local_id);

        // If there are adjustments, then record the final type --
        // this is the actual value that is being produced.
        if let Some(adjusted_ty) = self.fcx.tables.borrow().expr_ty_adjusted_opt(expr) {
            self.record(adjusted_ty, scope, Some(expr), expr.span, false);
        }

        // Also record the unadjusted type (which is the only type if
        // there are no adjustments). The reason for this is that the
        // unadjusted value is sometimes a "temporary" that would wind
        // up in a MIR temporary.
        //
        // As an example, consider an expression like `vec![].push()`.
        // Here, the `vec![]` would wind up MIR stored into a
        // temporary variable `t` which we can borrow to invoke
        // `<Vec<_>>::push(&mut t)`.
        //
        // Note that an expression can have many adjustments, and we
        // are just ignoring those intermediate types. This is because
        // those intermediate values are always linearly "consumed" by
        // the other adjustments, and hence would never be directly
        // captured in the MIR.
        //
        // (Note that this partly relies on the fact that the `Deref`
        // traits always return references, which means their content
        // can be reborrowed without needing to spill to a temporary.
        // If this were not the case, then we could conceivably have
        // to create intermediate temporaries.)
        //
        // The type table might not have information for this expression
        // if it is in a malformed scope. (#66387)
        if let Some(ty) = self.fcx.tables.borrow().expr_ty_opt(expr) {
            self.record(ty, scope, Some(expr), expr.span, false);
        } else {
            self.fcx.tcx.sess.delay_span_bug(expr.span, "no type for node");
        }

        if let Some(upvars) = self.fcx.tcx.upvars(self.closure_def_id) {
            for (upvar_id, upvar) in upvars.iter() {
                let upvar_ty = self.fcx.tables.borrow().node_type(*upvar_id);
                debug!("type of upvar: {:?}", upvar_ty);
                self.record(upvar_ty, scope, Some(expr), upvar.span, true);
            }
        }
    }
}
<commit_msg>don't record upvars into generator interior<commit_after>//! This calculates the types which has storage which lives across a suspension point in a
//! generator from the perspective of typeck. The actual types used at runtime
//! is calculated in `rustc_mir::transform::generator` and may be a subset of the
//! types computed here.

use super::FnCtxt;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_hir as hir;
use rustc_hir::def::{CtorKind, DefKind, Res};
use rustc_hir::def_id::DefId;
use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
use rustc_hir::{Expr, ExprKind, Pat, PatKind};
use rustc_middle::middle::region::{self, YieldData};
use rustc_middle::ty::{self, Ty};
use rustc_span::Span;

struct InteriorVisitor<'a, 'tcx> {
    fcx: &'a FnCtxt<'a, 'tcx>,
    types: FxHashMap<ty::GeneratorInteriorTypeCause<'tcx>, usize>,
    region_scope_tree: &'tcx region::ScopeTree,
    expr_count: usize,
    kind: hir::GeneratorKind,
    prev_unresolved_span: Option<Span>,
}

impl<'a, 'tcx> InteriorVisitor<'a, 'tcx> {
    fn record(
        &mut self,
        ty: Ty<'tcx>,
        scope: Option<region::Scope>,
        expr: Option<&'tcx Expr<'tcx>>,
        source_span: Span,
    ) {
        use rustc_span::DUMMY_SP;

        debug!(
            "generator_interior: attempting to record type {:?} {:?} {:?} {:?}",
            ty, scope, expr, source_span
        );

        let live_across_yield = scope
            .map(|s| {
                self.region_scope_tree.yield_in_scope(s).and_then(|yield_data| {
                    // If we are recording an expression that is the last yield
                    // in the scope, or that has a postorder CFG index larger
                    // than the one of all of the yields, then its value can't
                    // be storage-live (and therefore live) at any of the yields.
                    //
                    // See the mega-comment at `yield_in_scope` for a proof.

                    debug!(
                        "comparing counts yield: {} self: {}, source_span = {:?}",
                        yield_data.expr_and_pat_count, self.expr_count, source_span
                    );

                    if yield_data.expr_and_pat_count >= self.expr_count {
                        Some(yield_data)
                    } else {
                        None
                    }
                })
            })
            .unwrap_or_else(|| {
                Some(YieldData { span: DUMMY_SP, expr_and_pat_count: 0, source: self.kind.into() })
            });

        if let Some(yield_data) = live_across_yield {
            let ty = self.fcx.resolve_vars_if_possible(&ty);
            debug!(
                "type in expr = {:?}, scope = {:?}, type = {:?}, count = {}, yield_span = {:?}",
                expr, scope, ty, self.expr_count, yield_data.span
            );

            if let Some((unresolved_type, unresolved_type_span)) =
                self.fcx.unresolved_type_vars(&ty)
            {
                let note = format!(
                    "the type is part of the {} because of this {}",
                    self.kind, yield_data.source
                );

                // If unresolved type isn't a ty_var then unresolved_type_span is None
                let span = self
                    .prev_unresolved_span
                    .unwrap_or_else(|| unresolved_type_span.unwrap_or(source_span));
                self.fcx
                    .need_type_info_err_in_generator(self.kind, span, unresolved_type)
                    .span_note(yield_data.span, &*note)
                    .emit();
            } else {
                // Map the type to the number of types added before it
                let entries = self.types.len();
                let scope_span = scope.map(|s| s.span(self.fcx.tcx, self.region_scope_tree));
                self.types
                    .entry(ty::GeneratorInteriorTypeCause {
                        span: source_span,
                        ty: &ty,
                        scope_span,
                        yield_span: Some(yield_data.span),
                        expr: expr.map(|e| e.hir_id),
                    })
                    .or_insert(entries);
            }
        } else {
            debug!(
                "no type in expr = {:?}, count = {:?}, span = {:?}",
                expr,
                self.expr_count,
                expr.map(|e| e.span)
            );
            let ty = self.fcx.resolve_vars_if_possible(&ty);
            if let Some((unresolved_type, unresolved_type_span)) =
                self.fcx.unresolved_type_vars(&ty)
            {
                debug!(
                    "remained unresolved_type = {:?}, unresolved_type_span: {:?}",
                    unresolved_type, unresolved_type_span
                );
                self.prev_unresolved_span = unresolved_type_span;
            }
        }
    }
}

pub fn resolve_interior<'a, 'tcx>(
    fcx: &'a FnCtxt<'a, 'tcx>,
    def_id: DefId,
    body_id: hir::BodyId,
    interior: Ty<'tcx>,
    kind: hir::GeneratorKind,
) {
    let body = fcx.tcx.hir().body(body_id);

    let mut visitor = InteriorVisitor {
        fcx,
        types: FxHashMap::default(),
        region_scope_tree: fcx.tcx.region_scope_tree(def_id),
        expr_count: 0,
        kind,
        prev_unresolved_span: None,
    };
    intravisit::walk_body(&mut visitor, body);

    // Check that we visited the same amount of expressions and the RegionResolutionVisitor
    let region_expr_count = visitor.region_scope_tree.body_expr_count(body_id).unwrap();
    assert_eq!(region_expr_count, visitor.expr_count);

    let mut types: Vec<_> = visitor.types.drain().collect();

    // Sort types by insertion order
    types.sort_by_key(|t| t.1);

    // The types in the generator interior contain lifetimes local to the generator itself,
    // which should not be exposed outside of the generator. Therefore, we replace these
    // lifetimes with existentially-bound lifetimes, which reflect the exact value of the
    // lifetimes not being known by users.
    //
    // These lifetimes are used in auto trait impl checking (for example,
    // if a Sync generator contains an &'α T, we need to check whether &'α T: Sync),
    // so knowledge of the exact relationships between them isn't particularly important.

    debug!("types in generator {:?}, span = {:?}", types, body.value.span);

    let mut counter = 0;
    let mut captured_tys = FxHashSet::default();
    let type_causes: Vec<_> = types
        .into_iter()
        .filter_map(|(mut cause, _)| {
            // Erase regions and canonicalize late-bound regions to deduplicate as many types as we
            // can.
            let erased = fcx.tcx.erase_regions(&cause.ty);
            if captured_tys.insert(erased) {
                // Replace all regions inside the generator interior with late bound regions.
                // Note that each region slot in the types gets a new fresh late bound region,
                // which means that none of the regions inside relate to any other, even if
                // typeck had previously found constraints that would cause them to be related.
                let folded = fcx.tcx.fold_regions(&erased, &mut false, |_, current_depth| {
                    counter += 1;
                    fcx.tcx.mk_region(ty::ReLateBound(current_depth, ty::BrAnon(counter)))
                });

                cause.ty = folded;
                Some(cause)
            } else {
                None
            }
        })
        .collect();

    // Extract type components to build the witness type.
    let type_list = fcx.tcx.mk_type_list(type_causes.iter().map(|cause| cause.ty));
    let witness = fcx.tcx.mk_generator_witness(ty::Binder::bind(type_list));

    // Store the generator types and spans into the tables for this generator.
    visitor.fcx.inh.tables.borrow_mut().generator_interior_types = type_causes;

    debug!(
        "types in generator after region replacement {:?}, span = {:?}",
        witness, body.value.span
    );

    // Unify the type variable inside the generator with the new witness
    match fcx.at(&fcx.misc(body.value.span), fcx.param_env).eq(interior, witness) {
        Ok(ok) => fcx.register_infer_ok_obligations(ok),
        _ => bug!(),
    }
}

// This visitor has to have the same visit_expr calls as RegionResolutionVisitor in
// librustc_middle/middle/region.rs since `expr_count` is compared against the results
// there.
impl<'a, 'tcx> Visitor<'tcx> for InteriorVisitor<'a, 'tcx> {
    type Map = intravisit::ErasedMap<'tcx>;

    fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> {
        NestedVisitorMap::None
    }

    fn visit_pat(&mut self, pat: &'tcx Pat<'tcx>) {
        intravisit::walk_pat(self, pat);

        self.expr_count += 1;

        if let PatKind::Binding(..) = pat.kind {
            let scope = self.region_scope_tree.var_scope(pat.hir_id.local_id);
            let ty = self.fcx.tables.borrow().pat_ty(pat);
            self.record(ty, Some(scope), None, pat.span);
        }
    }

    fn visit_expr(&mut self, expr: &'tcx Expr<'tcx>) {
        match &expr.kind {
            ExprKind::Call(callee, args) => match &callee.kind {
                ExprKind::Path(qpath) => {
                    let res = self.fcx.tables.borrow().qpath_res(qpath, callee.hir_id);
                    match res {
                        // Direct calls never need to keep the callee `ty::FnDef`
                        // ZST in a temporary, so skip its type, just in case it
                        // can significantly complicate the generator type.
                        Res::Def(
                            DefKind::Fn | DefKind::AssocFn | DefKind::Ctor(_, CtorKind::Fn),
                            _,
                        ) => {
                            // NOTE(eddyb) this assumes a path expression has
                            // no nested expressions to keep track of.
                            self.expr_count += 1;

                            // Record the rest of the call expression normally.
                            for arg in *args {
                                self.visit_expr(arg);
                            }
                        }
                        _ => intravisit::walk_expr(self, expr),
                    }
                }
                _ => intravisit::walk_expr(self, expr),
            },
            _ => intravisit::walk_expr(self, expr),
        }

        self.expr_count += 1;

        let scope = self.region_scope_tree.temporary_scope(expr.hir_id.local_id);

        // If there are adjustments, then record the final type --
        // this is the actual value that is being produced.
        if let Some(adjusted_ty) = self.fcx.tables.borrow().expr_ty_adjusted_opt(expr) {
            self.record(adjusted_ty, scope, Some(expr), expr.span);
        }

        // Also record the unadjusted type (which is the only type if
        // there are no adjustments). The reason for this is that the
        // unadjusted value is sometimes a "temporary" that would wind
        // up in a MIR temporary.
        //
        // As an example, consider an expression like `vec![].push()`.
        // Here, the `vec![]` would wind up MIR stored into a
        // temporary variable `t` which we can borrow to invoke
        // `<Vec<_>>::push(&mut t)`.
        //
        // Note that an expression can have many adjustments, and we
        // are just ignoring those intermediate types. This is because
        // those intermediate values are always linearly "consumed" by
        // the other adjustments, and hence would never be directly
        // captured in the MIR.
        //
        // (Note that this partly relies on the fact that the `Deref`
        // traits always return references, which means their content
        // can be reborrowed without needing to spill to a temporary.
        // If this were not the case, then we could conceivably have
        // to create intermediate temporaries.)
        //
        // The type table might not have information for this expression
        // if it is in a malformed scope. (#66387)
        if let Some(ty) = self.fcx.tables.borrow().expr_ty_opt(expr) {
            self.record(ty, scope, Some(expr), expr.span);
        } else {
            self.fcx.tcx.sess.delay_span_bug(expr.span, "no type for node");
        }
    }
}
<|endoftext|>


<commit_before>//! This module contains the `InterpCx` methods for executing a single step of the interpreter.
//!
//! The main entry point is the `step` method.

use rustc_middle::mir;
use rustc_middle::mir::interpret::{InterpResult, Scalar};
use rustc_middle::ty::layout::LayoutOf;

use super::{InterpCx, Machine};

/// Classify whether an operator is "left-homogeneous", i.e., the LHS has the
/// same type as the result.
#[inline]
fn binop_left_homogeneous(op: mir::BinOp) -> bool {
    use rustc_middle::mir::BinOp::*;
    match op {
        Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Offset | Shl | Shr => true,
        Eq | Ne | Lt | Le | Gt | Ge => false,
    }
}
/// Classify whether an operator is "right-homogeneous", i.e., the RHS has the
/// same type as the LHS.
#[inline]
fn binop_right_homogeneous(op: mir::BinOp) -> bool {
    use rustc_middle::mir::BinOp::*;
    match op {
        Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true,
        Offset | Shl | Shr => false,
    }
}

impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
    pub fn run(&mut self) -> InterpResult<'tcx> {
        while self.step()? {}
        Ok(())
    }

    /// Returns `true` as long as there are more things to do.
    ///
    /// This is used by [priroda](https://github.com/oli-obk/priroda)
    ///
    /// This is marked `#inline(always)` to work around adverserial codegen when `opt-level = 3`
    #[inline(always)]
    pub fn step(&mut self) -> InterpResult<'tcx, bool> {
        if self.stack().is_empty() {
            return Ok(false);
        }

        let loc = match self.frame().loc {
            Ok(loc) => loc,
            Err(_) => {
                // We are unwinding and this fn has no cleanup code.
                // Just go on unwinding.
                trace!("unwinding: skipping frame");
                self.pop_stack_frame(/* unwinding */ true)?;
                return Ok(true);
            }
        };
        let basic_block = &self.body().basic_blocks()[loc.block];

        let old_frames = self.frame_idx();

        if let Some(stmt) = basic_block.statements.get(loc.statement_index) {
            assert_eq!(old_frames, self.frame_idx());
            self.statement(stmt)?;
            return Ok(true);
        }

        M::before_terminator(self)?;

        let terminator = basic_block.terminator();
        assert_eq!(old_frames, self.frame_idx());
        self.terminator(terminator)?;
        Ok(true)
    }

    /// Runs the interpretation logic for the given `mir::Statement` at the current frame and
    /// statement counter. This also moves the statement counter forward.
    pub fn statement(&mut self, stmt: &mir::Statement<'tcx>) -> InterpResult<'tcx> {
        info!("{:?}", stmt);

        use rustc_middle::mir::StatementKind::*;

        // Some statements (e.g., box) push new stack frames.
        // We have to record the stack frame number *before* executing the statement.
        let frame_idx = self.frame_idx();

        match &stmt.kind {
            Assign(box (place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?,

            SetDiscriminant { place, variant_index } => {
                let dest = self.eval_place(**place)?;
                self.write_discriminant(*variant_index, &dest)?;
            }

            // Mark locals as alive
            StorageLive(local) => {
                self.storage_live(*local)?;
            }

            // Mark locals as dead
            StorageDead(local) => {
                self.storage_dead(*local)?;
            }

            // No dynamic semantics attached to `FakeRead`; MIR
            // interpreter is solely intended for borrowck'ed code.
            FakeRead(..) => {}

            // Stacked Borrows.
            Retag(kind, place) => {
                let dest = self.eval_place(**place)?;
                M::retag(self, *kind, &dest)?;
            }

            // Call CopyNonOverlapping
            CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
                let src = self.eval_operand(src, None)?;
                let dst = self.eval_operand(dst, None)?;
                let count = self.eval_operand(count, None)?;
                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
            }

            // Statements we do not track.
            AscribeUserType(..) => {}

            // Currently, Miri discards Coverage statements. Coverage statements are only injected
            // via an optional compile time MIR pass and have no side effects. Since Coverage
            // statements don't exist at the source level, it is safe for Miri to ignore them, even
            // for undefined behavior (UB) checks.
            //
            // A coverage counter inside a const expression (for example, a counter injected in a
            // const function) is discarded when the const is evaluated at compile time. Whether
            // this should change, and/or how to implement a const eval counter, is a subject of the
            // following issue:
            //
            // FIXME(#73156): Handle source code coverage in const eval
            Coverage(..) => {}

            // Defined to do nothing. These are added by optimization passes, to avoid changing the
            // size of MIR constantly.
            Nop => {}

            LlvmInlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"),
        }

        self.stack_mut()[frame_idx].loc.as_mut().unwrap().statement_index += 1;
        Ok(())
    }

    /// Evaluate an assignment statement.
    ///
    /// There is no separate `eval_rvalue` function. Instead, the code for handling each rvalue
    /// type writes its results directly into the memory specified by the place.
    pub fn eval_rvalue_into_place(
        &mut self,
        rvalue: &mir::Rvalue<'tcx>,
        place: mir::Place<'tcx>,
    ) -> InterpResult<'tcx> {
        let dest = self.eval_place(place)?;

        use rustc_middle::mir::Rvalue::*;
        match *rvalue {
            ThreadLocalRef(did) => {
                let ptr = M::thread_local_static_base_pointer(self, did)?;
                self.write_pointer(ptr, &dest)?;
            }

            Use(ref operand) => {
                // Avoid recomputing the layout
                let op = self.eval_operand(operand, Some(dest.layout))?;
                self.copy_op(&op, &dest)?;
            }

            BinaryOp(bin_op, box (ref left, ref right)) => {
                let layout = binop_left_homogeneous(bin_op).then_some(dest.layout);
                let left = self.read_immediate(&self.eval_operand(left, layout)?)?;
                let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                let right = self.read_immediate(&self.eval_operand(right, layout)?)?;
                self.binop_ignore_overflow(bin_op, &left, &right, &dest)?;
            }

            CheckedBinaryOp(bin_op, box (ref left, ref right)) => {
                // Due to the extra boolean in the result, we can never reuse the `dest.layout`.
                let left = self.read_immediate(&self.eval_operand(left, None)?)?;
                let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                let right = self.read_immediate(&self.eval_operand(right, layout)?)?;
                self.binop_with_overflow(bin_op, &left, &right, &dest)?;
            }

            UnaryOp(un_op, ref operand) => {
                // The operand always has the same type as the result.
                let val = self.read_immediate(&self.eval_operand(operand, Some(dest.layout))?)?;
                let val = self.unary_op(un_op, &val)?;
                assert_eq!(val.layout, dest.layout, "layout mismatch for result of {:?}", un_op);
                self.write_immediate(*val, &dest)?;
            }

            Aggregate(ref kind, ref operands) => {
                // active_field_index is for union initialization.
                let (dest, active_field_index) = match **kind {
                    mir::AggregateKind::Adt(adt_def, variant_index, _, _, active_field_index) => {
                        self.write_discriminant(variant_index, &dest)?;
                        if adt_def.is_enum() {
                            assert!(active_field_index.is_none());
                            (self.place_downcast(&dest, variant_index)?, None)
                        } else {
                            if active_field_index.is_some() {
                                assert_eq!(operands.len(), 1);
                            }
                            (dest, active_field_index)
                        }
                    }
                    _ => (dest, None),
                };

                for (i, operand) in operands.iter().enumerate() {
                    let op = self.eval_operand(operand, None)?;
                    let field_index = active_field_index.unwrap_or(i);
                    let field_dest = self.place_field(&dest, field_index)?;
                    self.copy_op(&op, &field_dest)?;
                }
            }

            Repeat(ref operand, _) => {
                let src = self.eval_operand(operand, None)?;
                assert!(!src.layout.is_unsized());
                let dest = self.force_allocation(&dest)?;
                let length = dest.len(self)?;

                if length == 0 {
                    // Nothing to copy... but let's still make sure that `dest` as a place is valid.
                    self.get_alloc_mut(&dest)?;
                } else {
                    // Write the src to the first element.
                    let first = self.mplace_field(&dest, 0)?;
                    self.copy_op(&src, &first.into())?;

                    // This is performance-sensitive code for big static/const arrays! So we
                    // avoid writing each operand individually and instead just make many copies
                    // of the first element.
                    let elem_size = first.layout.size;
                    let first_ptr = first.ptr;
                    let rest_ptr = first_ptr.offset(elem_size, self)?;
                    self.memory.copy_repeatedly(
                        first_ptr,
                        first.align,
                        rest_ptr,
                        first.align,
                        elem_size,
                        length - 1,
                        /*nonoverlapping:*/ true,
                    )?;
                }
            }

            Len(place) => {
                let src = self.eval_place(place)?;
                let mplace = self.force_allocation(&src)?;
                let len = mplace.len(self)?;
                self.write_scalar(Scalar::from_machine_usize(len, self), &dest)?;
            }

            AddressOf(_, place) | Ref(_, _, place) => {
                let src = self.eval_place(place)?;
                let place = self.force_allocation(&src)?;
                self.write_immediate(place.to_ref(self), &dest)?;
            }

            NullaryOp(mir::NullOp::Box, _) => {
                M::box_alloc(self, &dest)?;
            }

            NullaryOp(null_op, ty) => {
                let ty = self.subst_from_current_frame_and_normalize_erasing_regions(ty);
                let layout = self.layout_of(ty)?;
                if layout.is_unsized() {
                    // FIXME: This should be a span_bug (#80742)
                    self.tcx.sess.delay_span_bug(
                        self.frame().current_span(),
                        &format!("Nullary MIR operator called for unsized type {}", ty),
                    );
                    throw_inval!(SizeOfUnsizedType(ty));
                }
                let val = match null_op {
                    mir::NullOp::SizeOf => layout.size.bytes(),
                    mir::NullOp::AlignOf => layout.align.abi.bytes(),
                    mir::NullOp::Box => unreachable!(),
                };
                self.write_scalar(Scalar::from_machine_usize(val, self), &dest)?;
            }

            ShallowInitBox(ref operand, _) => {
                let src = self.eval_operand(operand, None)?;
                let v = self.read_immediate(&src)?;
                self.write_immediate(*v, &dest)?;
            }

            Cast(cast_kind, ref operand, cast_ty) => {
                let src = self.eval_operand(operand, None)?;
                let cast_ty = self.subst_from_current_frame_and_normalize_erasing_regions(cast_ty);
                self.cast(&src, cast_kind, cast_ty, &dest)?;
            }

            Discriminant(place) => {
                let op = self.eval_place_to_op(place, None)?;
                let discr_val = self.read_discriminant(&op)?.0;
                self.write_scalar(discr_val, &dest)?;
            }
        }

        trace!("{:?}", self.dump_place(*dest));

        Ok(())
    }

    fn terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> {
        info!("{:?}", terminator.kind);

        self.eval_terminator(terminator)?;
        if !self.stack().is_empty() {
            if let Ok(loc) = self.frame().loc {
                info!("// executing {:?}", loc.block);
            }
        }
        Ok(())
    }
}
<commit_msg>Miri: fix alignment check in array initialization<commit_after>//! This module contains the `InterpCx` methods for executing a single step of the interpreter.
//!
//! The main entry point is the `step` method.

use rustc_middle::mir;
use rustc_middle::mir::interpret::{InterpResult, Scalar};
use rustc_middle::ty::layout::LayoutOf;

use super::{InterpCx, Machine};

/// Classify whether an operator is "left-homogeneous", i.e., the LHS has the
/// same type as the result.
#[inline]
fn binop_left_homogeneous(op: mir::BinOp) -> bool {
    use rustc_middle::mir::BinOp::*;
    match op {
        Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Offset | Shl | Shr => true,
        Eq | Ne | Lt | Le | Gt | Ge => false,
    }
}
/// Classify whether an operator is "right-homogeneous", i.e., the RHS has the
/// same type as the LHS.
#[inline]
fn binop_right_homogeneous(op: mir::BinOp) -> bool {
    use rustc_middle::mir::BinOp::*;
    match op {
        Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true,
        Offset | Shl | Shr => false,
    }
}

impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
    pub fn run(&mut self) -> InterpResult<'tcx> {
        while self.step()? {}
        Ok(())
    }

    /// Returns `true` as long as there are more things to do.
    ///
    /// This is used by [priroda](https://github.com/oli-obk/priroda)
    ///
    /// This is marked `#inline(always)` to work around adverserial codegen when `opt-level = 3`
    #[inline(always)]
    pub fn step(&mut self) -> InterpResult<'tcx, bool> {
        if self.stack().is_empty() {
            return Ok(false);
        }

        let loc = match self.frame().loc {
            Ok(loc) => loc,
            Err(_) => {
                // We are unwinding and this fn has no cleanup code.
                // Just go on unwinding.
                trace!("unwinding: skipping frame");
                self.pop_stack_frame(/* unwinding */ true)?;
                return Ok(true);
            }
        };
        let basic_block = &self.body().basic_blocks()[loc.block];

        let old_frames = self.frame_idx();

        if let Some(stmt) = basic_block.statements.get(loc.statement_index) {
            assert_eq!(old_frames, self.frame_idx());
            self.statement(stmt)?;
            return Ok(true);
        }

        M::before_terminator(self)?;

        let terminator = basic_block.terminator();
        assert_eq!(old_frames, self.frame_idx());
        self.terminator(terminator)?;
        Ok(true)
    }

    /// Runs the interpretation logic for the given `mir::Statement` at the current frame and
    /// statement counter. This also moves the statement counter forward.
    pub fn statement(&mut self, stmt: &mir::Statement<'tcx>) -> InterpResult<'tcx> {
        info!("{:?}", stmt);

        use rustc_middle::mir::StatementKind::*;

        // Some statements (e.g., box) push new stack frames.
        // We have to record the stack frame number *before* executing the statement.
        let frame_idx = self.frame_idx();

        match &stmt.kind {
            Assign(box (place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?,

            SetDiscriminant { place, variant_index } => {
                let dest = self.eval_place(**place)?;
                self.write_discriminant(*variant_index, &dest)?;
            }

            // Mark locals as alive
            StorageLive(local) => {
                self.storage_live(*local)?;
            }

            // Mark locals as dead
            StorageDead(local) => {
                self.storage_dead(*local)?;
            }

            // No dynamic semantics attached to `FakeRead`; MIR
            // interpreter is solely intended for borrowck'ed code.
            FakeRead(..) => {}

            // Stacked Borrows.
            Retag(kind, place) => {
                let dest = self.eval_place(**place)?;
                M::retag(self, *kind, &dest)?;
            }

            // Call CopyNonOverlapping
            CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
                let src = self.eval_operand(src, None)?;
                let dst = self.eval_operand(dst, None)?;
                let count = self.eval_operand(count, None)?;
                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
            }

            // Statements we do not track.
            AscribeUserType(..) => {}

            // Currently, Miri discards Coverage statements. Coverage statements are only injected
            // via an optional compile time MIR pass and have no side effects. Since Coverage
            // statements don't exist at the source level, it is safe for Miri to ignore them, even
            // for undefined behavior (UB) checks.
            //
            // A coverage counter inside a const expression (for example, a counter injected in a
            // const function) is discarded when the const is evaluated at compile time. Whether
            // this should change, and/or how to implement a const eval counter, is a subject of the
            // following issue:
            //
            // FIXME(#73156): Handle source code coverage in const eval
            Coverage(..) => {}

            // Defined to do nothing. These are added by optimization passes, to avoid changing the
            // size of MIR constantly.
            Nop => {}

            LlvmInlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"),
        }

        self.stack_mut()[frame_idx].loc.as_mut().unwrap().statement_index += 1;
        Ok(())
    }

    /// Evaluate an assignment statement.
    ///
    /// There is no separate `eval_rvalue` function. Instead, the code for handling each rvalue
    /// type writes its results directly into the memory specified by the place.
    pub fn eval_rvalue_into_place(
        &mut self,
        rvalue: &mir::Rvalue<'tcx>,
        place: mir::Place<'tcx>,
    ) -> InterpResult<'tcx> {
        let dest = self.eval_place(place)?;

        use rustc_middle::mir::Rvalue::*;
        match *rvalue {
            ThreadLocalRef(did) => {
                let ptr = M::thread_local_static_base_pointer(self, did)?;
                self.write_pointer(ptr, &dest)?;
            }

            Use(ref operand) => {
                // Avoid recomputing the layout
                let op = self.eval_operand(operand, Some(dest.layout))?;
                self.copy_op(&op, &dest)?;
            }

            BinaryOp(bin_op, box (ref left, ref right)) => {
                let layout = binop_left_homogeneous(bin_op).then_some(dest.layout);
                let left = self.read_immediate(&self.eval_operand(left, layout)?)?;
                let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                let right = self.read_immediate(&self.eval_operand(right, layout)?)?;
                self.binop_ignore_overflow(bin_op, &left, &right, &dest)?;
            }

            CheckedBinaryOp(bin_op, box (ref left, ref right)) => {
                // Due to the extra boolean in the result, we can never reuse the `dest.layout`.
                let left = self.read_immediate(&self.eval_operand(left, None)?)?;
                let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                let right = self.read_immediate(&self.eval_operand(right, layout)?)?;
                self.binop_with_overflow(bin_op, &left, &right, &dest)?;
            }

            UnaryOp(un_op, ref operand) => {
                // The operand always has the same type as the result.
                let val = self.read_immediate(&self.eval_operand(operand, Some(dest.layout))?)?;
                let val = self.unary_op(un_op, &val)?;
                assert_eq!(val.layout, dest.layout, "layout mismatch for result of {:?}", un_op);
                self.write_immediate(*val, &dest)?;
            }

            Aggregate(ref kind, ref operands) => {
                // active_field_index is for union initialization.
                let (dest, active_field_index) = match **kind {
                    mir::AggregateKind::Adt(adt_def, variant_index, _, _, active_field_index) => {
                        self.write_discriminant(variant_index, &dest)?;
                        if adt_def.is_enum() {
                            assert!(active_field_index.is_none());
                            (self.place_downcast(&dest, variant_index)?, None)
                        } else {
                            if active_field_index.is_some() {
                                assert_eq!(operands.len(), 1);
                            }
                            (dest, active_field_index)
                        }
                    }
                    _ => (dest, None),
                };

                for (i, operand) in operands.iter().enumerate() {
                    let op = self.eval_operand(operand, None)?;
                    let field_index = active_field_index.unwrap_or(i);
                    let field_dest = self.place_field(&dest, field_index)?;
                    self.copy_op(&op, &field_dest)?;
                }
            }

            Repeat(ref operand, _) => {
                let src = self.eval_operand(operand, None)?;
                assert!(!src.layout.is_unsized());
                let dest = self.force_allocation(&dest)?;
                let length = dest.len(self)?;

                if length == 0 {
                    // Nothing to copy... but let's still make sure that `dest` as a place is valid.
                    self.get_alloc_mut(&dest)?;
                } else {
                    // Write the src to the first element.
                    let first = self.mplace_field(&dest, 0)?;
                    self.copy_op(&src, &first.into())?;

                    // This is performance-sensitive code for big static/const arrays! So we
                    // avoid writing each operand individually and instead just make many copies
                    // of the first element.
                    let elem_size = first.layout.size;
                    let first_ptr = first.ptr;
                    let rest_ptr = first_ptr.offset(elem_size, self)?;
                    // For the alignment of `rest_ptr`, we crucially do *not* use `first.align` as
                    // that place might be more aligned than its type mandates (a `u8` array could
                    // be 4-aligned if it sits at the right spot in a struct). Instead we use
                    // `first.layout.align`, i.e., the alignment given by the type.
                    self.memory.copy_repeatedly(
                        first_ptr,
                        first.align,
                        rest_ptr,
                        first.layout.align.abi,
                        elem_size,
                        length - 1,
                        /*nonoverlapping:*/ true,
                    )?;
                }
            }

            Len(place) => {
                let src = self.eval_place(place)?;
                let mplace = self.force_allocation(&src)?;
                let len = mplace.len(self)?;
                self.write_scalar(Scalar::from_machine_usize(len, self), &dest)?;
            }

            AddressOf(_, place) | Ref(_, _, place) => {
                let src = self.eval_place(place)?;
                let place = self.force_allocation(&src)?;
                self.write_immediate(place.to_ref(self), &dest)?;
            }

            NullaryOp(mir::NullOp::Box, _) => {
                M::box_alloc(self, &dest)?;
            }

            NullaryOp(null_op, ty) => {
                let ty = self.subst_from_current_frame_and_normalize_erasing_regions(ty);
                let layout = self.layout_of(ty)?;
                if layout.is_unsized() {
                    // FIXME: This should be a span_bug (#80742)
                    self.tcx.sess.delay_span_bug(
                        self.frame().current_span(),
                        &format!("Nullary MIR operator called for unsized type {}", ty),
                    );
                    throw_inval!(SizeOfUnsizedType(ty));
                }
                let val = match null_op {
                    mir::NullOp::SizeOf => layout.size.bytes(),
                    mir::NullOp::AlignOf => layout.align.abi.bytes(),
                    mir::NullOp::Box => unreachable!(),
                };
                self.write_scalar(Scalar::from_machine_usize(val, self), &dest)?;
            }

            ShallowInitBox(ref operand, _) => {
                let src = self.eval_operand(operand, None)?;
                let v = self.read_immediate(&src)?;
                self.write_immediate(*v, &dest)?;
            }

            Cast(cast_kind, ref operand, cast_ty) => {
                let src = self.eval_operand(operand, None)?;
                let cast_ty = self.subst_from_current_frame_and_normalize_erasing_regions(cast_ty);
                self.cast(&src, cast_kind, cast_ty, &dest)?;
            }

            Discriminant(place) => {
                let op = self.eval_place_to_op(place, None)?;
                let discr_val = self.read_discriminant(&op)?.0;
                self.write_scalar(discr_val, &dest)?;
            }
        }

        trace!("{:?}", self.dump_place(*dest));

        Ok(())
    }

    fn terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> {
        info!("{:?}", terminator.kind);

        self.eval_terminator(terminator)?;
        if !self.stack().is_empty() {
            if let Ok(loc) = self.frame().loc {
                info!("// executing {:?}", loc.block);
            }
        }
        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::fmt;
use std::path::{Path, PathBuf};

use async_trait::async_trait;
use bollard::container::LogOutput;
use bollard::Docker;
use futures::stream::BoxStream;
use futures::{StreamExt, TryFutureExt};
use nails::execution::ExitCode;
use store::Store;
use task_executor::Executor;
use workunit_store::{in_workunit, RunningWorkunit};

use crate::local::{
  apply_chroot, create_sandbox, prepare_workdir, setup_run_sh_script, CapturedWorkdir, ChildOutput,
  KeepSandboxes,
};
use crate::{
  Context, FallibleProcessResultWithPlatform, ImmutableInputs, NamedCaches, Platform, Process,
  ProcessError,
};

pub(crate) const SANDBOX_PATH_IN_CONTAINER: &str = "/pants-sandbox";
pub(crate) const NAMED_CACHES_PATH_IN_CONTAINER: &str = "/pants-named-caches";
pub(crate) const IMMUTABLE_INPUTS_PATH_IN_CONTAINER: &str = "/pants-immutable-inputs";

/// `CommandRunner` executes processes using a local Docker client.
pub struct CommandRunner {
  docker: Docker,
  store: Store,
  executor: Executor,
  work_dir_base: PathBuf,
  named_caches: NamedCaches,
  immutable_inputs: ImmutableInputs,
  keep_sandboxes: KeepSandboxes,
}

impl CommandRunner {
  pub fn new(
    store: Store,
    executor: Executor,
    work_dir_base: PathBuf,
    named_caches: NamedCaches,
    immutable_inputs: ImmutableInputs,
    keep_sandboxes: KeepSandboxes,
  ) -> Result<Self, String> {
    let docker = Docker::connect_with_local_defaults()
      .map_err(|err| format!("Failed to connect to local Docker: {err}"))?;
    Ok(CommandRunner {
      docker,
      store,
      executor,
      work_dir_base,
      named_caches,
      immutable_inputs,
      keep_sandboxes,
    })
  }
}

impl fmt::Debug for CommandRunner {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
    f.debug_struct("docker::CommandRunner")
      .finish_non_exhaustive()
  }
}

#[async_trait]
impl super::CommandRunner for CommandRunner {
  async fn run(
    &self,
    context: Context,
    _workunit: &mut RunningWorkunit,
    req: Process,
  ) -> Result<FallibleProcessResultWithPlatform, ProcessError> {
    let req_debug_repr = format!("{:#?}", req);
    in_workunit!(
      "run_local_process_via_docker",
      req.level,
      // NB: See engine::nodes::NodeKey::workunit_level for more information on why this workunit
      // renders at the Process's level.
      desc = Some(req.description.clone()),
      |_workunit| async move {
        let mut workdir = create_sandbox(
          self.executor.clone(),
          &self.work_dir_base,
          &req.description,
          self.keep_sandboxes,
        )?;

        // Start working on a mutable version of the process.
        let mut req = req;

        // Update env, replacing `{chroot}` placeholders with `/pants-sandbox`. This is the mount point
        // for the sandbox directory within the Docker container.
        //
        // DOCKER-TODO: With cached containers, the destination in the container
        // will need to be unique within that container to avoid conflicts.
        apply_chroot(SANDBOX_PATH_IN_CONTAINER, &mut req);

        // Prepare the workdir.
        // DOCKER-NOTE: The input root will be bind mounted into the container.
        let exclusive_spawn = prepare_workdir(
          workdir.path().to_owned(),
          &req,
          req.input_digests.input_files.clone(),
          self.store.clone(),
          self.executor.clone(),
          &self.named_caches,
          &self.immutable_inputs,
          Some(Path::new(NAMED_CACHES_PATH_IN_CONTAINER)),
          Some(Path::new(IMMUTABLE_INPUTS_PATH_IN_CONTAINER)),
        )
        .await?;

        // DOCKER-TODO: Add a metric for local docker execution?
        // workunit.increment_counter(Metric::LocalExecutionRequests, 1);

        let res = self
          .run_and_capture_workdir(
            req.clone(),
            context,
            self.store.clone(),
            self.executor.clone(),
            workdir.path().to_owned(),
            (
              self.immutable_inputs.workdir().to_path_buf(),
              self.named_caches.base_dir().to_path_buf(),
            ),
            exclusive_spawn,
            req
              .platform_constraint
              .unwrap_or_else(|| Platform::current().unwrap()),
          )
          .map_err(|msg| {
            // Processes that experience no infrastructure issues should result in an "Ok" return,
            // potentially with an exit code that indicates that they failed (with more information
            // on stderr). Actually failing at this level indicates a failure to start or otherwise
            // interact with the process, which would generally be an infrastructure or implementation
            // error (something missing from the sandbox, incorrect permissions, etc).
            //
            // Given that this is expected to be rare, we dump the entire process definition in the
            // error.
            ProcessError::Unclassified(format!("Failed to execute: {}\n\n{}", req_debug_repr, msg))
          })
          .await;

        if self.keep_sandboxes == KeepSandboxes::Always
          || self.keep_sandboxes == KeepSandboxes::OnFailure
            && res.as_ref().map(|r| r.exit_code).unwrap_or(1) != 0
        {
          workdir.keep(&req.description);
          setup_run_sh_script(&req.env, &req.working_directory, &req.argv, workdir.path())?;
        }

        res
      }
    )
    .await
  }
}

#[async_trait]
impl CapturedWorkdir for CommandRunner {
  type WorkdirToken = (PathBuf, PathBuf);

  async fn run_in_workdir<'a, 'b, 'c>(
    &'a self,
    workdir_path: &'b Path,
    (immutable_inputs_workdir, named_caches_workdir): Self::WorkdirToken,
    req: Process,
    _exclusive_spawn: bool,
  ) -> Result<BoxStream<'c, Result<ChildOutput, String>>, String> {
    let env = req
      .env
      .iter()
      .map(|(key, value)| format!("{}={}", key, value))
      .collect::<Vec<_>>();

    let workdir_path_as_string = workdir_path
      .to_path_buf()
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert workdir_path due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let immutable_inputs_workdir_as_string = immutable_inputs_workdir
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert immutable_inputs_workdir due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let named_caches_workdir_as_string = named_caches_workdir
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert named_caches_workdir due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let working_dir = req
      .working_directory
      .map(|relpath| Path::new(SANDBOX_PATH_IN_CONTAINER).join(&relpath))
      .unwrap_or_else(|| Path::new(SANDBOX_PATH_IN_CONTAINER).to_path_buf())
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert working directory due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let image = req
      .docker_image
      .ok_or("docker_image not set on the Process, but the Docker CommandRunner was used.")?;

    // DOCKER-TODO: Set creation options so we can set platform.

    let config = bollard::container::Config {
      env: Some(env),
      entrypoint: Some(req.argv),
      working_dir: Some(working_dir),
      // DOCKER-TODO: Is this necessary on linux hosts for allowing bind mount?
      // user: Some(format!("{}", unsafe { libc::geteuid() })),
      // user: Some("0".to_string()),
      host_config: Some(bollard_stubs::models::HostConfig {
        binds: Some(vec![
          format!("{}:{}", workdir_path_as_string, SANDBOX_PATH_IN_CONTAINER),
          // DOCKER-TODO: Consider making this bind mount read-only.
          format!(
            "{}:{}",
            immutable_inputs_workdir_as_string, IMMUTABLE_INPUTS_PATH_IN_CONTAINER
          ),
          format!(
            "{}:{}",
            named_caches_workdir_as_string, NAMED_CACHES_PATH_IN_CONTAINER
          ),
        ]),
        init: Some(true),
        ..bollard_stubs::models::HostConfig::default()
      }),
      image: Some(image),
      attach_stdout: Some(true),
      attach_stderr: Some(true),
      ..bollard::container::Config::default()
    };

    log::trace!("creating container with config: {:?}", &config);

    let container = self
      .docker
      .create_container::<&str, String>(None, config)
      .await
      .map_err(|err| format!("Failed to create Docker container: {:?}", err))?;

    log::trace!("created container {}", &container.id);

    self
      .docker
      .start_container::<String>(&container.id, None)
      .await
      .map_err(|err| {
        format!(
          "Failed to start Docker container `{}`: {:?}",
          &container.id, err
        )
      })?;

    log::trace!("started container {}", &container.id);

    let attach_options = bollard::container::AttachContainerOptions::<String> {
      stdout: Some(true),
      stderr: Some(true),
      logs: Some(true), // stream any output that was missed between the start_container call and now
      stream: Some(true),
      ..bollard::container::AttachContainerOptions::default()
    };

    let attach_result = self
      .docker
      .attach_container(&container.id, Some(attach_options))
      .await
      .map_err(|err| {
        format!(
          "Failed to attach to Docker container `{}`: {:?}",
          &container.id, err
        )
      })?;

    log::trace!("attached to container {}", &container.id);

    let mut output_stream = attach_result.output.boxed();

    let wait_options = bollard::container::WaitContainerOptions {
      condition: "not-running",
    };
    let mut wait_stream = self
      .docker
      .wait_container(&container.id, Some(wait_options))
      .boxed();

    let container_id = container.id.to_owned();
    let keep_sandboxes = self.keep_sandboxes;
    let docker = self.docker.clone();
    let result_stream = async_stream::stream! {
      let was_success = loop {
        tokio::select! {
          Some(output_msg) = output_stream.next() => {
            match output_msg {
              Ok(LogOutput::StdOut { message }) => {
                log::trace!("container {} wrote {} bytes to stdout", &container_id, message.len());
                yield Ok(ChildOutput::Stdout(message));
              }
              Ok(LogOutput::StdErr { message }) => {
                log::trace!("container {} wrote {} bytes to stderr", &container_id, message.len());
                yield Ok(ChildOutput::Stderr(message));
              }
              _ => (),
            }
          }
          Some(wait_msg) = wait_stream.next() => {
            log::trace!("wait_container stream ({}): {:?}", &container_id, wait_msg);
            match wait_msg {
              Ok(r) => {
                // DOCKER-TODO: How does Docker distinguish signal versus exit code? Improve
                // `ChildResults` to better support exit code vs signal vs error message?
                let status_code = r.status_code;
                yield Ok(ChildOutput::Exit(ExitCode(status_code as i32)));
                break status_code == 0;
              }
              Err(err) => {
                // DOCKER-TODO: Consider a way to pass error messages back to child status collector.
                log::error!("Docker wait failure for container {}: {:?}", &container_id, err);
                yield Err(format!("Docker wait_container failure for container {}: {:?}", &container_id, err));
                break false;
              }
            }
          }
        }
      };

      let do_remove_container = match keep_sandboxes {
        KeepSandboxes::Always => false,
        KeepSandboxes::Never => true,
        KeepSandboxes::OnFailure => !was_success,
      };

      if do_remove_container {
        let remove_options = bollard::container::RemoveContainerOptions {
          force: true,
          ..bollard::container::RemoveContainerOptions::default()
        };

        let remove_result = docker
          .remove_container(&container_id, Some(remove_options))
          .await
          .map_err(|err| format!("Failed to remove container `{}`: {:?}", &container_id, err));

        if let Err(err) = remove_result {
          log::warn!("{}", err);
        }
      }
    }
    .boxed();

    Ok(result_stream)
  }
}
<commit_msg>docker: lazily initialize docker connection (#16779)<commit_after>use std::fmt;
use std::path::{Path, PathBuf};

use async_oncecell::OnceCell;
use async_trait::async_trait;
use bollard::container::LogOutput;
use bollard::Docker;
use futures::stream::BoxStream;
use futures::{StreamExt, TryFutureExt};
use nails::execution::ExitCode;
use store::Store;
use task_executor::Executor;
use workunit_store::{in_workunit, RunningWorkunit};

use crate::local::{
  apply_chroot, create_sandbox, prepare_workdir, setup_run_sh_script, CapturedWorkdir, ChildOutput,
  KeepSandboxes,
};
use crate::{
  Context, FallibleProcessResultWithPlatform, ImmutableInputs, NamedCaches, Platform, Process,
  ProcessError,
};

pub(crate) const SANDBOX_PATH_IN_CONTAINER: &str = "/pants-sandbox";
pub(crate) const NAMED_CACHES_PATH_IN_CONTAINER: &str = "/pants-named-caches";
pub(crate) const IMMUTABLE_INPUTS_PATH_IN_CONTAINER: &str = "/pants-immutable-inputs";

/// `CommandRunner` executes processes using a local Docker client.
pub struct CommandRunner {
  docker: OnceCell<Docker>,
  store: Store,
  executor: Executor,
  work_dir_base: PathBuf,
  named_caches: NamedCaches,
  immutable_inputs: ImmutableInputs,
  keep_sandboxes: KeepSandboxes,
}

impl CommandRunner {
  pub fn new(
    store: Store,
    executor: Executor,
    work_dir_base: PathBuf,
    named_caches: NamedCaches,
    immutable_inputs: ImmutableInputs,
    keep_sandboxes: KeepSandboxes,
  ) -> Result<Self, String> {
    Ok(CommandRunner {
      docker: OnceCell::new(),
      store,
      executor,
      work_dir_base,
      named_caches,
      immutable_inputs,
      keep_sandboxes,
    })
  }

  async fn docker(&self) -> Result<&Docker, String> {
    self
      .docker
      .get_or_try_init(async move {
        let docker = Docker::connect_with_local_defaults()
          .map_err(|err| format!("Failed to connect to local Docker: {err}"))?;

        docker
          .ping()
          .await
          .map_err(|err| format!("Failed to receive response from local Docker: {err}"))?;

        Ok(docker)
      })
      .await
  }
}

impl fmt::Debug for CommandRunner {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
    f.debug_struct("docker::CommandRunner")
      .finish_non_exhaustive()
  }
}

#[async_trait]
impl super::CommandRunner for CommandRunner {
  async fn run(
    &self,
    context: Context,
    _workunit: &mut RunningWorkunit,
    req: Process,
  ) -> Result<FallibleProcessResultWithPlatform, ProcessError> {
    let req_debug_repr = format!("{:#?}", req);
    in_workunit!(
      "run_local_process_via_docker",
      req.level,
      // NB: See engine::nodes::NodeKey::workunit_level for more information on why this workunit
      // renders at the Process's level.
      desc = Some(req.description.clone()),
      |_workunit| async move {
        let mut workdir = create_sandbox(
          self.executor.clone(),
          &self.work_dir_base,
          &req.description,
          self.keep_sandboxes,
        )?;

        // Start working on a mutable version of the process.
        let mut req = req;

        // Update env, replacing `{chroot}` placeholders with `/pants-sandbox`. This is the mount point
        // for the sandbox directory within the Docker container.
        //
        // DOCKER-TODO: With cached containers, the destination in the container
        // will need to be unique within that container to avoid conflicts.
        apply_chroot(SANDBOX_PATH_IN_CONTAINER, &mut req);

        // Prepare the workdir.
        // DOCKER-NOTE: The input root will be bind mounted into the container.
        let exclusive_spawn = prepare_workdir(
          workdir.path().to_owned(),
          &req,
          req.input_digests.input_files.clone(),
          self.store.clone(),
          self.executor.clone(),
          &self.named_caches,
          &self.immutable_inputs,
          Some(Path::new(NAMED_CACHES_PATH_IN_CONTAINER)),
          Some(Path::new(IMMUTABLE_INPUTS_PATH_IN_CONTAINER)),
        )
        .await?;

        // DOCKER-TODO: Add a metric for local docker execution?
        // workunit.increment_counter(Metric::LocalExecutionRequests, 1);

        let res = self
          .run_and_capture_workdir(
            req.clone(),
            context,
            self.store.clone(),
            self.executor.clone(),
            workdir.path().to_owned(),
            (
              self.immutable_inputs.workdir().to_path_buf(),
              self.named_caches.base_dir().to_path_buf(),
            ),
            exclusive_spawn,
            req
              .platform_constraint
              .unwrap_or_else(|| Platform::current().unwrap()),
          )
          .map_err(|msg| {
            // Processes that experience no infrastructure issues should result in an "Ok" return,
            // potentially with an exit code that indicates that they failed (with more information
            // on stderr). Actually failing at this level indicates a failure to start or otherwise
            // interact with the process, which would generally be an infrastructure or implementation
            // error (something missing from the sandbox, incorrect permissions, etc).
            //
            // Given that this is expected to be rare, we dump the entire process definition in the
            // error.
            ProcessError::Unclassified(format!("Failed to execute: {}\n\n{}", req_debug_repr, msg))
          })
          .await;

        if self.keep_sandboxes == KeepSandboxes::Always
          || self.keep_sandboxes == KeepSandboxes::OnFailure
            && res.as_ref().map(|r| r.exit_code).unwrap_or(1) != 0
        {
          workdir.keep(&req.description);
          setup_run_sh_script(&req.env, &req.working_directory, &req.argv, workdir.path())?;
        }

        res
      }
    )
    .await
  }
}

#[async_trait]
impl CapturedWorkdir for CommandRunner {
  type WorkdirToken = (PathBuf, PathBuf);

  async fn run_in_workdir<'a, 'b, 'c>(
    &'a self,
    workdir_path: &'b Path,
    (immutable_inputs_workdir, named_caches_workdir): Self::WorkdirToken,
    req: Process,
    _exclusive_spawn: bool,
  ) -> Result<BoxStream<'c, Result<ChildOutput, String>>, String> {
    let docker = self.docker().await?;

    let env = req
      .env
      .iter()
      .map(|(key, value)| format!("{}={}", key, value))
      .collect::<Vec<_>>();

    let workdir_path_as_string = workdir_path
      .to_path_buf()
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert workdir_path due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let immutable_inputs_workdir_as_string = immutable_inputs_workdir
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert immutable_inputs_workdir due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let named_caches_workdir_as_string = named_caches_workdir
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert named_caches_workdir due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let working_dir = req
      .working_directory
      .map(|relpath| Path::new(SANDBOX_PATH_IN_CONTAINER).join(&relpath))
      .unwrap_or_else(|| Path::new(SANDBOX_PATH_IN_CONTAINER).to_path_buf())
      .into_os_string()
      .into_string()
      .map_err(|s| {
        format!(
          "Unable to convert working directory due to non UTF-8 characters: {:?}",
          s
        )
      })?;

    let image = req
      .docker_image
      .ok_or("docker_image not set on the Process, but the Docker CommandRunner was used.")?;

    // DOCKER-TODO: Set creation options so we can set platform.

    let config = bollard::container::Config {
      env: Some(env),
      entrypoint: Some(req.argv),
      working_dir: Some(working_dir),
      // DOCKER-TODO: Is this necessary on linux hosts for allowing bind mount?
      // user: Some(format!("{}", unsafe { libc::geteuid() })),
      // user: Some("0".to_string()),
      host_config: Some(bollard_stubs::models::HostConfig {
        binds: Some(vec![
          format!("{}:{}", workdir_path_as_string, SANDBOX_PATH_IN_CONTAINER),
          // DOCKER-TODO: Consider making this bind mount read-only.
          format!(
            "{}:{}",
            immutable_inputs_workdir_as_string, IMMUTABLE_INPUTS_PATH_IN_CONTAINER
          ),
          format!(
            "{}:{}",
            named_caches_workdir_as_string, NAMED_CACHES_PATH_IN_CONTAINER
          ),
        ]),
        init: Some(true),
        ..bollard_stubs::models::HostConfig::default()
      }),
      image: Some(image),
      attach_stdout: Some(true),
      attach_stderr: Some(true),
      ..bollard::container::Config::default()
    };

    log::trace!("creating container with config: {:?}", &config);

    let container = docker
      .create_container::<&str, String>(None, config)
      .await
      .map_err(|err| format!("Failed to create Docker container: {:?}", err))?;

    log::trace!("created container {}", &container.id);

    docker
      .start_container::<String>(&container.id, None)
      .await
      .map_err(|err| {
        format!(
          "Failed to start Docker container `{}`: {:?}",
          &container.id, err
        )
      })?;

    log::trace!("started container {}", &container.id);

    let attach_options = bollard::container::AttachContainerOptions::<String> {
      stdout: Some(true),
      stderr: Some(true),
      logs: Some(true), // stream any output that was missed between the start_container call and now
      stream: Some(true),
      ..bollard::container::AttachContainerOptions::default()
    };

    let attach_result = docker
      .attach_container(&container.id, Some(attach_options))
      .await
      .map_err(|err| {
        format!(
          "Failed to attach to Docker container `{}`: {:?}",
          &container.id, err
        )
      })?;

    log::trace!("attached to container {}", &container.id);

    let mut output_stream = attach_result.output.boxed();

    let wait_options = bollard::container::WaitContainerOptions {
      condition: "not-running",
    };
    let mut wait_stream = docker
      .wait_container(&container.id, Some(wait_options))
      .boxed();

    let container_id = container.id.to_owned();
    let keep_sandboxes = self.keep_sandboxes;
    let docker = docker.clone();
    let result_stream = async_stream::stream! {
      let was_success = loop {
        tokio::select! {
          Some(output_msg) = output_stream.next() => {
            match output_msg {
              Ok(LogOutput::StdOut { message }) => {
                log::trace!("container {} wrote {} bytes to stdout", &container_id, message.len());
                yield Ok(ChildOutput::Stdout(message));
              }
              Ok(LogOutput::StdErr { message }) => {
                log::trace!("container {} wrote {} bytes to stderr", &container_id, message.len());
                yield Ok(ChildOutput::Stderr(message));
              }
              _ => (),
            }
          }
          Some(wait_msg) = wait_stream.next() => {
            log::trace!("wait_container stream ({}): {:?}", &container_id, wait_msg);
            match wait_msg {
              Ok(r) => {
                // DOCKER-TODO: How does Docker distinguish signal versus exit code? Improve
                // `ChildResults` to better support exit code vs signal vs error message?
                let status_code = r.status_code;
                yield Ok(ChildOutput::Exit(ExitCode(status_code as i32)));
                break status_code == 0;
              }
              Err(err) => {
                // DOCKER-TODO: Consider a way to pass error messages back to child status collector.
                log::error!("Docker wait failure for container {}: {:?}", &container_id, err);
                yield Err(format!("Docker wait_container failure for container {}: {:?}", &container_id, err));
                break false;
              }
            }
          }
        }
      };

      let do_remove_container = match keep_sandboxes {
        KeepSandboxes::Always => false,
        KeepSandboxes::Never => true,
        KeepSandboxes::OnFailure => !was_success,
      };

      if do_remove_container {
        let remove_options = bollard::container::RemoveContainerOptions {
          force: true,
          ..bollard::container::RemoveContainerOptions::default()
        };

        let remove_result = docker
          .remove_container(&container_id, Some(remove_options))
          .await
          .map_err(|err| format!("Failed to remove container `{}`: {:?}", &container_id, err));

        if let Err(err) = remove_result {
          log::warn!("{}", err);
        }
      }
    }
    .boxed();

    Ok(result_stream)
  }
}
<|endoftext|>


<commit_before>use std::io::{stdin, File, timer};
use std::io::buffered::BufferedReader;
use std::str::{from_utf8};
use std::vec::ChunkIter;
use std::vec::OwnedStr;

fn main () {
  let p = Path::new("data/example.txt");
  let contents = File::open(&p).read_to_end();
  let mut string_iter = from_utf8(contents).split_str(" ");
  let mut reader = BufferedReader::new(stdin());
  println("Input WPM: ");
  let input = reader.read_line().unwrap();
  let WPM = from_str::<uint>(input.trim()).unwrap_or(300);

  println("Input the number of words to view at a time: ");
  let input = reader.read_line().unwrap();
  let words_at_a_time = from_str::<uint>(input.trim()).unwrap_or(1);

  let sleep_time = 60000 / WPM * words_at_a_time;

  let string_vec = string_iter.to_owned_vec();

  for words in string_vec.chunks(words_at_a_time) {
    println("\x1bc".into_owned().append(words.connect(" ")));
    // println();
    std::io::timer::sleep(sleep_time.to_u64().unwrap());
  }

  // for (i, word) in string_iter.enumerate() {
	 //  print(word);
  //   if i % words_at_a_time == (words_at_a_time - 1) {
  //     println("");
  //     println("\x1bc");
  //   }
  // }
}
<commit_msg>Removed newlines form input text<commit_after>use std::io::{stdin, File, timer};
use std::io::buffered::BufferedReader;
use std::str::{from_utf8};
use std::vec::ChunkIter;
use std::vec::OwnedStr;

fn main () {
  let p = Path::new("data/example.txt");
  let contents = File::open(&p).read_to_end();
  let string_contents = from_utf8(contents).replace("\n", " ");
  let mut string_iter = string_contents.split_str(" ");
  let mut reader = BufferedReader::new(stdin());
  println("Input WPM: ");
  let input = reader.read_line().unwrap();
  let WPM = from_str::<uint>(input.trim()).unwrap_or(300);

  println("Input the number of words to view at a time: ");
  let input = reader.read_line().unwrap();
  let words_at_a_time = from_str::<uint>(input.trim()).unwrap_or(1);

  let sleep_time = 60000 / WPM * words_at_a_time;

  let string_vec = string_iter.to_owned_vec();

  for words in string_vec.chunks(words_at_a_time) {
    println("\x1bc".into_owned().append(words.connect(" ")));
    std::io::timer::sleep(sleep_time.to_u64().unwrap());
  }

}
<|endoftext|>