<commit_before>use std::fmt;

#[allow(dead_code)]
pub enum Suit {
    Spades,
    Hearts,
    Diamonds,
    Clubs,
}

impl Suit {
    fn short_string(&self) -> &str {
        match self {
            &Suit::Spades => "s",
            &Suit::Hearts => "h",
            &Suit::Diamonds => "d",
            &Suit::Clubs => "c",
        }
    }
}

#[allow(dead_code)]
pub enum Value {
    Two,
    Three,
    Four,
    Five,
    Six,
    Seven,
    Eight,
    Nine,
    Ten,
    Jack,
    Queen,
    King,
    Ace,
    // no jokers
}

impl Value {
    fn short_string(&self) -> &str {
        match self {
            &Value::Two => "2",
            &Value::Three => "3",
            &Value::Four => "4",
            &Value::Five => "5",
            &Value::Six => "6",
            &Value::Seven => "7",
            &Value::Eight => "8",
            &Value::Nine => "9",
            &Value::Ten => "T",
            &Value::Jack => "J",
            &Value::Queen => "Q",
            &Value::King => "K",
            &Value::Ace => "A",
        }
    }
}

pub struct Card(pub Value, pub Suit);

impl Card {
    pub fn short_string(&self) -> String {
        let mut s = String::new();
        s.push_str(self.0.short_string());
        s.push_str(self.1.short_string());
        s
    }
}

impl fmt::Display for Card {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.short_string())
    }
}

<commit_msg>Make types implement Debug and PartialEq.<commit_after>use std::fmt;

#[derive(Debug, PartialEq)]
pub enum Suit {
    Spades,
    Hearts,
    Diamonds,
    Clubs,
}

impl Suit {
    fn short_string(&self) -> &str {
        match self {
            &Suit::Spades => "s",
            &Suit::Hearts => "h",
            &Suit::Diamonds => "d",
            &Suit::Clubs => "c",
        }
    }
}

#[derive(Debug, PartialEq)]
pub enum Value {
    Two,
    Three,
    Four,
    Five,
    Six,
    Seven,
    Eight,
    Nine,
    Ten,
    Jack,
    Queen,
    King,
    Ace,
    // no jokers
}

impl Value {
    fn short_string(&self) -> &str {
        match self {
            &Value::Two => "2",
            &Value::Three => "3",
            &Value::Four => "4",
            &Value::Five => "5",
            &Value::Six => "6",
            &Value::Seven => "7",
            &Value::Eight => "8",
            &Value::Nine => "9",
            &Value::Ten => "T",
            &Value::Jack => "J",
            &Value::Queen => "Q",
            &Value::King => "K",
            &Value::Ace => "A",
        }
    }
}

#[derive(Debug, PartialEq)]
pub struct Card(pub Value, pub Suit);

impl Card {
    pub fn short_string(&self) -> String {
        let mut s = String::new();
        s.push_str(self.0.short_string());
        s.push_str(self.1.short_string());
        s
    }
}

impl fmt::Display for Card {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.short_string())
    }
}

<|endoftext|>


<commit_before>//! Main evaluator loop and setting up the initial stack frame.

use rand::rngs::StdRng;
use rand::SeedableRng;

use rustc::hir::def_id::DefId;
use rustc::ty::layout::{LayoutOf, Size};
use rustc::ty::{self, TyCtxt};
use syntax::source_map::DUMMY_SP;

use crate::{
    EnvVars, Evaluator, FnVal, HelpersEvalContextExt, InterpCx, InterpError,
    InterpResult, MemoryExtra, MiriMemoryKind, Pointer, Scalar, StackPopCleanup, Tag,
    TlsEvalContextExt,
};

/// Configuration needed to spawn a Miri instance.
#[derive(Clone)]
pub struct MiriConfig {
    /// Determine if validity checking and Stacked Borrows are enabled.
    pub validate: bool,
    /// Determines if communication with the host environment is enabled.
    pub communicate: bool,
    /// Environment variables that should always be isolated from the host.
    pub excluded_env_vars: Vec<String>,
    /// Command-line arguments passed to the interpreted program.
    pub args: Vec<String>,
    /// The seed to use when non-determinism or randomness are required (e.g. ptr-to-int cast, `getrandom()`).
    pub seed: Option<u64>,
}

// Used by priroda.
pub fn create_ecx<'mir, 'tcx: 'mir>(
    tcx: TyCtxt<'tcx>,
    main_id: DefId,
    config: MiriConfig,
) -> InterpResult<'tcx, InterpCx<'mir, 'tcx, Evaluator<'tcx>>> {
    let mut ecx = InterpCx::new(
        tcx.at(syntax::source_map::DUMMY_SP),
        ty::ParamEnv::reveal_all(),
        Evaluator::new(config.communicate),
        MemoryExtra::new(
            StdRng::seed_from_u64(config.seed.unwrap_or(0)),
            config.validate,
        ),
    );
    // Complete initialization.
    EnvVars::init(&mut ecx, config.excluded_env_vars);

    // Setup first stack-frame
    let main_instance = ty::Instance::mono(tcx, main_id);
    let main_mir = ecx.load_mir(main_instance.def, None)?;

    if !main_mir.return_ty().is_unit() || main_mir.arg_count != 0 {
        throw_unsup_format!("miri does not support main functions without `fn()` type signatures");
    }

    let start_id = tcx.lang_items().start_fn().unwrap();
    let main_ret_ty = tcx.fn_sig(main_id).output();
    let main_ret_ty = main_ret_ty.no_bound_vars().unwrap();
    let start_instance = ty::Instance::resolve(
        tcx,
        ty::ParamEnv::reveal_all(),
        start_id,
        tcx.mk_substs(::std::iter::once(ty::subst::GenericArg::from(main_ret_ty))),
    )
    .unwrap();
    let start_mir = ecx.load_mir(start_instance.def, None)?;

    if start_mir.arg_count != 3 {
        bug!(
            "'start' lang item should have three arguments, but has {}",
            start_mir.arg_count
        );
    }

    // Return value (in static memory so that it does not count as leak).
    let ret = ecx.layout_of(start_mir.return_ty())?;
    let ret_ptr = ecx.allocate(ret, MiriMemoryKind::Static.into());

    // Push our stack frame.
    ecx.push_stack_frame(
        start_instance,
        // There is no call site.
        DUMMY_SP,
        start_mir,
        Some(ret_ptr.into()),
        StackPopCleanup::None { cleanup: true },
    )?;

    let mut args = ecx.frame().body.args_iter();

    // First argument: pointer to `main()`.
    let main_ptr = ecx
        .memory
        .create_fn_alloc(FnVal::Instance(main_instance));
    let dest = ecx.local_place(args.next().unwrap())?;
    ecx.write_scalar(Scalar::Ptr(main_ptr), dest)?;

    // Second argument (argc): `1`.
    let dest = ecx.local_place(args.next().unwrap())?;
    let argc = Scalar::from_uint(config.args.len() as u128, dest.layout.size);
    ecx.write_scalar(argc, dest)?;
    // Store argc for macOS's `_NSGetArgc`.
    {
        let argc_place = ecx.allocate(dest.layout, MiriMemoryKind::Env.into());
        ecx.write_scalar(argc, argc_place.into())?;
        ecx.machine.argc = Some(argc_place.ptr);
    }

    // Third argument (`argv`): created from `config.args`.
    let dest = ecx.local_place(args.next().unwrap())?;
    // For Windows, construct a command string with all the aguments.
    let mut cmd = String::new();
    for arg in config.args.iter() {
        if !cmd.is_empty() {
            cmd.push(' ');
        }
        cmd.push_str(&*shell_escape::windows::escape(arg.as_str().into()));
    }
    // Don't forget `0` terminator.
    cmd.push(std::char::from_u32(0).unwrap());
    // Collect the pointers to the individual strings.
    let mut argvs = Vec::<Pointer<Tag>>::new();
    for arg in config.args {
        // Add `0` terminator.
        let mut arg = arg.into_bytes();
        arg.push(0);
        argvs.push(
            ecx.memory
                .allocate_static_bytes(arg.as_slice(), MiriMemoryKind::Static.into()),
        );
    }
    // Make an array with all these pointers, in the Miri memory.
    let argvs_layout = ecx.layout_of(
        tcx.mk_array(tcx.mk_imm_ptr(tcx.types.u8), argvs.len() as u64),
    )?;
    let argvs_place = ecx.allocate(argvs_layout, MiriMemoryKind::Env.into());
    for (idx, arg) in argvs.into_iter().enumerate() {
        let place = ecx.mplace_field(argvs_place, idx as u64)?;
        ecx.write_scalar(Scalar::Ptr(arg), place.into())?;
    }
    ecx.memory
        .mark_immutable(argvs_place.ptr.assert_ptr().alloc_id)?;
    // Write a pointer to that place as the argument.
    let argv = argvs_place.ptr;
    ecx.write_scalar(argv, dest)?;
    // Store `argv` for macOS `_NSGetArgv`.
    {
        let argv_place = ecx.allocate(dest.layout, MiriMemoryKind::Env.into());
        ecx.write_scalar(argv, argv_place.into())?;
        ecx.machine.argv = Some(argv_place.ptr);
    }
    // Store command line as UTF-16 for Windows `GetCommandLineW`.
    {
        let cmd_utf16: Vec<u16> = cmd.encode_utf16().collect();
        let cmd_type = tcx.mk_array(tcx.types.u16, cmd_utf16.len() as u64);
        let cmd_place = ecx.allocate(ecx.layout_of(cmd_type)?, MiriMemoryKind::Env.into());
        ecx.machine.cmd_line = Some(cmd_place.ptr);
        // Store the UTF-16 string. We just allocated so we know the bounds are fine.
        let char_size = Size::from_bytes(2);
        for (idx, &c) in cmd_utf16.iter().enumerate() {
            let place = ecx.mplace_field(cmd_place, idx as u64)?;
            ecx.write_scalar(Scalar::from_uint(c, char_size), place.into())?;
        }
    }

    args.next().expect_none("start lang item has more arguments than expected");

    // Set the last_error to 0
    let errno_layout = ecx.layout_of(tcx.types.u32)?;
    let errno_place = ecx.allocate(errno_layout, MiriMemoryKind::Static.into());
    ecx.write_scalar(Scalar::from_u32(0), errno_place.into())?;
    ecx.machine.last_error = Some(errno_place);

    Ok(ecx)
}

pub fn eval_main<'tcx>(tcx: TyCtxt<'tcx>, main_id: DefId, config: MiriConfig) {
    let mut ecx = match create_ecx(tcx, main_id, config) {
        Ok(ecx) => ecx,
        Err(mut err) => {
            err.print_backtrace();
            panic!("Miri initialziation error: {}", err.kind)
        }
    };

    // Perform the main execution.
    let res: InterpResult<'_> = (|| {
        ecx.run()?;
        ecx.run_tls_dtors()
    })();

    // Process the result.
    match res {
        Ok(()) => {
            let leaks = ecx.memory.leak_report();
            // Disable the leak test on some platforms where we do not
            // correctly implement TLS destructors.
            let target_os = ecx.tcx.tcx.sess.target.target.target_os.to_lowercase();
            let ignore_leaks = target_os == "windows" || target_os == "macos";
            if !ignore_leaks && leaks != 0 {
                tcx.sess.err("the evaluated program leaked memory");
            }
        }
        Err(mut e) => {
            // Special treatment for some error kinds
            let msg = match e.kind {
                InterpError::Exit(code) => std::process::exit(code),
                err_unsup!(NoMirFor(..)) =>
                    format!("{}. Did you set `MIRI_SYSROOT` to a Miri-enabled sysroot? You can prepare one with `cargo miri setup`.", e),
                _ => e.to_string()
            };
            e.print_backtrace();
            if let Some(frame) = ecx.stack().last() {
                let block = &frame.body.basic_blocks()[frame.block];
                let span = if frame.stmt < block.statements.len() {
                    block.statements[frame.stmt].source_info.span
                } else {
                    block.terminator().source_info.span
                };

                let msg = format!("Miri evaluation error: {}", msg);
                let mut err = {
                    let new_tcx = ecx.tcx.tcx.at(span);
                    new_tcx.sess.struct_span_err(new_tcx.span, msg.as_str())
                };
                let frames = ecx.generate_stacktrace(None);
                err.span_label(span, msg);
                // We iterate with indices because we need to look at the next frame (the caller).
                for idx in 0..frames.len() {
                    let frame_info = &frames[idx];
                    let call_site_is_local = frames.get(idx + 1).map_or(false, |caller_info| {
                        caller_info.instance.def_id().is_local()
                    });
                    if call_site_is_local {
                        err.span_note(frame_info.call_site, &frame_info.to_string());
                    } else {
                        err.note(&frame_info.to_string());
                    }
                }
                err.emit();
            } else {
                ecx.tcx.sess.err(&msg);
            }

            for (i, frame) in ecx.stack().iter().enumerate() {
                trace!("-------------------");
                trace!("Frame {}", i);
                trace!("    return: {:?}", frame.return_place.map(|p| *p));
                for (i, local) in frame.locals.iter().enumerate() {
                    trace!("    local {}: {:?}", i, local.value);
                }
            }
        }
    }
}
<commit_msg>follow-up to review<commit_after>//! Main evaluator loop and setting up the initial stack frame.

use rand::rngs::StdRng;
use rand::SeedableRng;

use rustc::hir::def_id::DefId;
use rustc::ty::layout::{LayoutOf, Size};
use rustc::ty::{self, TyCtxt};
use syntax::source_map::DUMMY_SP;

use crate::{
    EnvVars, Evaluator, FnVal, HelpersEvalContextExt, InterpCx, InterpError,
    InterpResult, MemoryExtra, MiriMemoryKind, Pointer, Scalar, StackPopCleanup, Tag,
    TlsEvalContextExt,
};

/// Configuration needed to spawn a Miri instance.
#[derive(Clone)]
pub struct MiriConfig {
    /// Determine if validity checking and Stacked Borrows are enabled.
    pub validate: bool,
    /// Determines if communication with the host environment is enabled.
    pub communicate: bool,
    /// Environment variables that should always be isolated from the host.
    pub excluded_env_vars: Vec<String>,
    /// Command-line arguments passed to the interpreted program.
    pub args: Vec<String>,
    /// The seed to use when non-determinism or randomness are required (e.g. ptr-to-int cast, `getrandom()`).
    pub seed: Option<u64>,
}

// Used by priroda.
pub fn create_ecx<'mir, 'tcx: 'mir>(
    tcx: TyCtxt<'tcx>,
    main_id: DefId,
    config: MiriConfig,
) -> InterpResult<'tcx, InterpCx<'mir, 'tcx, Evaluator<'tcx>>> {
    let mut ecx = InterpCx::new(
        tcx.at(syntax::source_map::DUMMY_SP),
        ty::ParamEnv::reveal_all(),
        Evaluator::new(config.communicate),
        MemoryExtra::new(
            StdRng::seed_from_u64(config.seed.unwrap_or(0)),
            config.validate,
        ),
    );
    // Complete initialization.
    EnvVars::init(&mut ecx, config.excluded_env_vars);

    // Setup first stack-frame
    let main_instance = ty::Instance::mono(tcx, main_id);
    let main_mir = ecx.load_mir(main_instance.def, None)?;

    if !main_mir.return_ty().is_unit() || main_mir.arg_count != 0 {
        throw_unsup_format!("miri does not support main functions without `fn()` type signatures");
    }

    let start_id = tcx.lang_items().start_fn().unwrap();
    let main_ret_ty = tcx.fn_sig(main_id).output();
    let main_ret_ty = main_ret_ty.no_bound_vars().unwrap();
    let start_instance = ty::Instance::resolve(
        tcx,
        ty::ParamEnv::reveal_all(),
        start_id,
        tcx.mk_substs(::std::iter::once(ty::subst::GenericArg::from(main_ret_ty))),
    )
    .unwrap();
    let start_mir = ecx.load_mir(start_instance.def, None)?;

    if start_mir.arg_count != 3 {
        bug!(
            "'start' lang item should have three arguments, but has {}",
            start_mir.arg_count
        );
    }

    // Return value (in static memory so that it does not count as leak).
    let ret = ecx.layout_of(start_mir.return_ty())?;
    let ret_ptr = ecx.allocate(ret, MiriMemoryKind::Static.into());

    // Push our stack frame.
    ecx.push_stack_frame(
        start_instance,
        // There is no call site.
        DUMMY_SP,
        start_mir,
        Some(ret_ptr.into()),
        StackPopCleanup::None { cleanup: true },
    )?;

    let mut args = ecx.frame().body.args_iter();

    // First argument: pointer to `main()`.
    let main_ptr = ecx
        .memory
        .create_fn_alloc(FnVal::Instance(main_instance));
    let dest = ecx.local_place(args.next().unwrap())?;
    ecx.write_scalar(Scalar::Ptr(main_ptr), dest)?;

    // Second argument (argc): `1`.
    let dest = ecx.local_place(args.next().unwrap())?;
    let argc = Scalar::from_uint(config.args.len() as u128, dest.layout.size);
    ecx.write_scalar(argc, dest)?;
    // Store argc for macOS's `_NSGetArgc`.
    {
        let argc_place = ecx.allocate(dest.layout, MiriMemoryKind::Env.into());
        ecx.write_scalar(argc, argc_place.into())?;
        ecx.machine.argc = Some(argc_place.ptr);
    }

    // Third argument (`argv`): created from `config.args`.
    let dest = ecx.local_place(args.next().unwrap())?;
    // For Windows, construct a command string with all the aguments.
    let mut cmd = String::new();
    for arg in config.args.iter() {
        if !cmd.is_empty() {
            cmd.push(' ');
        }
        cmd.push_str(&*shell_escape::windows::escape(arg.as_str().into()));
    }
    // Don't forget `0` terminator.
    cmd.push(std::char::from_u32(0).unwrap());
    // Collect the pointers to the individual strings.
    let mut argvs = Vec::<Pointer<Tag>>::new();
    for arg in config.args {
        // Add `0` terminator.
        let mut arg = arg.into_bytes();
        arg.push(0);
        argvs.push(
            ecx.memory
                .allocate_static_bytes(arg.as_slice(), MiriMemoryKind::Static.into()),
        );
    }
    // Make an array with all these pointers, in the Miri memory.
    let argvs_layout = ecx.layout_of(
        tcx.mk_array(tcx.mk_imm_ptr(tcx.types.u8), argvs.len() as u64),
    )?;
    let argvs_place = ecx.allocate(argvs_layout, MiriMemoryKind::Env.into());
    for (idx, arg) in argvs.into_iter().enumerate() {
        let place = ecx.mplace_field(argvs_place, idx as u64)?;
        ecx.write_scalar(Scalar::Ptr(arg), place.into())?;
    }
    ecx.memory
        .mark_immutable(argvs_place.ptr.assert_ptr().alloc_id)?;
    // Write a pointer to that place as the argument.
    let argv = argvs_place.ptr;
    ecx.write_scalar(argv, dest)?;
    // Store `argv` for macOS `_NSGetArgv`.
    {
        let argv_place = ecx.allocate(dest.layout, MiriMemoryKind::Env.into());
        ecx.write_scalar(argv, argv_place.into())?;
        ecx.machine.argv = Some(argv_place.ptr);
    }
    // Store command line as UTF-16 for Windows `GetCommandLineW`.
    {
        let cmd_utf16: Vec<u16> = cmd.encode_utf16().collect();
        let cmd_type = tcx.mk_array(tcx.types.u16, cmd_utf16.len() as u64);
        let cmd_place = ecx.allocate(ecx.layout_of(cmd_type)?, MiriMemoryKind::Env.into());
        ecx.machine.cmd_line = Some(cmd_place.ptr);
        // Store the UTF-16 string. We just allocated so we know the bounds are fine.
        let char_size = Size::from_bytes(2);
        for (idx, &c) in cmd_utf16.iter().enumerate() {
            let place = ecx.mplace_field(cmd_place, idx as u64)?;
            ecx.write_scalar(Scalar::from_uint(c, char_size), place.into())?;
        }
    }

    args.next().expect_none("start lang item has more arguments than expected");

    // Set the last_error to 0
    let errno_layout = ecx.layout_of(tcx.types.u32)?;
    let errno_place = ecx.allocate(errno_layout, MiriMemoryKind::Static.into());
    ecx.write_scalar(Scalar::from_u32(0), errno_place.into())?;
    ecx.machine.last_error = Some(errno_place);

    Ok(ecx)
}

pub fn eval_main<'tcx>(tcx: TyCtxt<'tcx>, main_id: DefId, config: MiriConfig) {
    let mut ecx = match create_ecx(tcx, main_id, config) {
        Ok(ecx) => ecx,
        Err(mut err) => {
            err.print_backtrace();
            panic!("Miri initialziation error: {}", err.kind)
        }
    };

    // Perform the main execution.
    let res: InterpResult<'_> = (|| {
        ecx.run()?;
        ecx.run_tls_dtors()
    })();

    // Process the result.
    match res {
        Ok(()) => {
            let leaks = ecx.memory.leak_report();
            // Disable the leak test on some platforms where we do not
            // correctly implement TLS destructors.
            let target_os = ecx.tcx.tcx.sess.target.target.target_os.to_lowercase();
            let ignore_leaks = target_os == "windows" || target_os == "macos";
            if !ignore_leaks && leaks != 0 {
                tcx.sess.err("the evaluated program leaked memory");
            }
        }
        Err(mut e) => {
            // Special treatment for some error kinds
            let msg = match e.kind {
                InterpError::Exit(code) => std::process::exit(code),
                err_unsup!(NoMirFor(..)) =>
                    format!("{}. Did you set `MIRI_SYSROOT` to a Miri-enabled sysroot? You can prepare one with `cargo miri setup`.", e),
                _ => e.to_string()
            };
            e.print_backtrace();
            if let Some(frame) = ecx.stack().last() {
                let block = &frame.body.basic_blocks()[frame.block];
                let span = if frame.stmt < block.statements.len() {
                    block.statements[frame.stmt].source_info.span
                } else {
                    block.terminator().source_info.span
                };

                let msg = format!("Miri evaluation error: {}", msg);
                let mut err = ecx.tcx.sess.struct_span_err(span, msg.as_str());
                let frames = ecx.generate_stacktrace(None);
                err.span_label(span, msg);
                // We iterate with indices because we need to look at the next frame (the caller).
                for idx in 0..frames.len() {
                    let frame_info = &frames[idx];
                    let call_site_is_local = frames.get(idx + 1).map_or(false, |caller_info| {
                        caller_info.instance.def_id().is_local()
                    });
                    if call_site_is_local {
                        err.span_note(frame_info.call_site, &frame_info.to_string());
                    } else {
                        err.note(&frame_info.to_string());
                    }
                }
                err.emit();
            } else {
                ecx.tcx.sess.err(&msg);
            }

            for (i, frame) in ecx.stack().iter().enumerate() {
                trace!("-------------------");
                trace!("Frame {}", i);
                trace!("    return: {:?}", frame.return_place.map(|p| *p));
                for (i, local) in frame.locals.iter().enumerate() {
                    trace!("    local {}: {:?}", i, local.value);
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>mod extractor;

extern crate num_cpus;

use clap;
use flate2::write::GzEncoder;
use rayon::prelude::*;
use std::fs;
use std::io::{BufRead, BufWriter};
use std::path::{Path, PathBuf};
use tree_sitter::{Language, Parser, Range};

enum TrapCompression {
    None,
    Gzip,
}

impl TrapCompression {
    fn from_env() -> TrapCompression {
        match std::env::var("CODEQL_RUBY_TRAP_COMPRESSION") {
            Ok(method) => match TrapCompression::from_string(&method) {
                Some(c) => c,
                None => {
                    tracing::error!("Unknown compression method '{}'; using gzip.", &method);
                    TrapCompression::Gzip
                }
            },
            // Default compression method if the env var isn't set:
            Err(_) => TrapCompression::Gzip,
        }
    }

    fn from_string(s: &str) -> Option<TrapCompression> {
        match s.to_lowercase().as_ref() {
            "none" => Some(TrapCompression::None),
            "gzip" => Some(TrapCompression::Gzip),
            _ => None,
        }
    }

    fn extension(&self) -> &str {
        match self {
            TrapCompression::None => ".trap",
            TrapCompression::Gzip => ".trap.gz",
        }
    }
}

/**
 * Gets the number of threads the extractor should use, by reading the
 * CODEQL_THREADS environment variable and using it as described in the
 * extractor spec:
 *
 * "If the number is positive, it indicates the number of threads that should
 * be used. If the number is negative or zero, it should be added to the number
 * of cores available on the machine to determine how many threads to use
 * (minimum of 1). If unspecified, should be considered as set to -1."
 */
fn num_codeql_threads() -> usize {
    let threads_str = std::env::var("CODEQL_THREADS").unwrap_or("-1".to_owned());
    match threads_str.parse::<i32>() {
        Ok(num) if num <= 0 => {
            let reduction = -num as usize;
            std::cmp::max(1, num_cpus::get() - reduction)
        }
        Ok(num) => num as usize,

        Err(_) => {
            tracing::error!(
                "Unable to parse CODEQL_THREADS value '{}'; defaulting to 1 thread.",
                &threads_str
            );
            1
        }
    }
}

fn main() -> std::io::Result<()> {
    tracing_subscriber::fmt()
        .with_target(false)
        .without_time()
        .with_level(true)
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
        .init();

    let num_threads = num_codeql_threads();
    tracing::info!(
        "Using {} {}",
        num_threads,
        if num_threads == 1 {
            "thread"
        } else {
            "threads"
        }
    );
    rayon::ThreadPoolBuilder::new()
        .num_threads(num_threads)
        .build_global()
        .unwrap();

    let matches = clap::App::new("Ruby extractor")
        .version("1.0")
        .author("GitHub")
        .about("CodeQL Ruby extractor")
        .args_from_usage(
            "--source-archive-dir=<DIR> 'Sets a custom source archive folder'
                    --output-dir=<DIR>         'Sets a custom trap folder'
                    --file-list=<FILE_LIST>    'A text files containing the paths of the files to extract'",
        )
        .get_matches();
    let src_archive_dir = matches
        .value_of("source-archive-dir")
        .expect("missing --source-archive-dir");
    let src_archive_dir = PathBuf::from(src_archive_dir);

    let trap_dir = matches
        .value_of("output-dir")
        .expect("missing --output-dir");
    let trap_dir = PathBuf::from(trap_dir);
    let trap_compression = TrapCompression::from_env();

    let file_list = matches.value_of("file-list").expect("missing --file-list");
    let file_list = fs::File::open(file_list)?;

    let language = tree_sitter_ruby::language();
    let erb = tree_sitter_embedded_template::language();
    let schema = node_types::read_node_types_str("ruby", tree_sitter_ruby::NODE_TYPES)?;
    let erb_schema =
        node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?;
    let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
    let lines = lines?;
    lines
        .par_iter()
        .try_for_each(|line| {
            let path = PathBuf::from(line).canonicalize()?;
            let src_archive_file = path_for(&src_archive_dir, &path, "");
            let mut source = std::fs::read(&path)?;
            let code_ranges;
            let mut trap_writer = extractor::new_trap_writer();
            if path.extension().map_or(false, |x| x == "erb") {
                tracing::info!("scanning: {}", path.display());
                extractor::extract(
                    erb,
                    "erb",
                    &erb_schema,
                    &mut trap_writer,
                    &path,
                    &source,
                    &[],
                )?;

                let (ranges, line_breaks) = scan_erb(erb, &source);
                for i in line_breaks {
                    if i < source.len() {
                        source[i] = b'\n';
                    }
                }
                code_ranges = ranges;
            } else {
                code_ranges = vec![];
            }
            extractor::extract(
                language,
                "ruby",
                &schema,
                &mut trap_writer,
                &path,
                &source,
                &code_ranges,
            )?;
            std::fs::create_dir_all(&src_archive_file.parent().unwrap())?;
            std::fs::copy(&path, &src_archive_file)?;
            write_trap(&trap_dir, path, trap_writer, &trap_compression)
        })
        .expect("failed to extract files");

    let path = PathBuf::from("extras");
    let mut trap_writer = extractor::new_trap_writer();
    &trap_writer.populate_empty_location();
    write_trap(&trap_dir, path, trap_writer, &trap_compression)
}

fn write_trap(
    trap_dir: &PathBuf,
    path: PathBuf,
    trap_writer: extractor::TrapWriter,
    trap_compression: &TrapCompression,
) -> std::io::Result<()> {
    let trap_file = path_for(&trap_dir, &path, trap_compression.extension());
    std::fs::create_dir_all(&trap_file.parent().unwrap())?;
    let trap_file = std::fs::File::create(&trap_file)?;
    let mut trap_file = BufWriter::new(trap_file);
    match trap_compression {
        TrapCompression::None => trap_writer.output(&mut trap_file),
        TrapCompression::Gzip => {
            let mut compressed_writer = GzEncoder::new(trap_file, flate2::Compression::fast());
            trap_writer.output(&mut compressed_writer)
        }
    }
}

fn scan_erb(erb: Language, source: &std::vec::Vec<u8>) -> (Vec<Range>, Vec<usize>) {
    let mut parser = Parser::new();
    parser.set_language(erb).unwrap();
    let tree = parser.parse(&source, None).expect("Failed to parse file");
    let mut result = Vec::new();
    let mut line_breaks = vec![];

    for n in tree.root_node().children(&mut tree.walk()) {
        let kind = n.kind();
        if kind == "directive" || kind == "output_directive" {
            for c in n.children(&mut tree.walk()) {
                if c.kind() == "code" {
                    let mut range = c.range();
                    if range.end_byte < source.len() {
                        line_breaks.push(range.end_byte);
                        range.end_byte += 1;
                        range.end_point.column += 1;
                    }
                    result.push(range);
                }
            }
        }
    }
    if result.len() == 0 {
        let root = tree.root_node();
        // Add an empty range at the end of the file
        result.push(Range {
            start_byte: root.end_byte(),
            end_byte: root.end_byte(),
            start_point: root.end_position(),
            end_point: root.end_position(),
        });
    }
    (result, line_breaks)
}

fn path_for(dir: &Path, path: &Path, ext: &str) -> PathBuf {
    let mut result = PathBuf::from(dir);
    for component in path.components() {
        match component {
            std::path::Component::Prefix(prefix) => match prefix.kind() {
                std::path::Prefix::Disk(letter) | std::path::Prefix::VerbatimDisk(letter) => {
                    result.push(format!("{}_", letter as char))
                }
                std::path::Prefix::Verbatim(x) | std::path::Prefix::DeviceNS(x) => {
                    result.push(x);
                }
                std::path::Prefix::UNC(server, share)
                | std::path::Prefix::VerbatimUNC(server, share) => {
                    result.push("unc");
                    result.push(server);
                    result.push(share);
                }
            },
            std::path::Component::RootDir => {
                // skip
            }
            std::path::Component::Normal(_) => {
                result.push(component);
            }
            std::path::Component::CurDir => {
                // skip
            }
            std::path::Component::ParentDir => {
                result.pop();
            }
        }
    }
    if let Some(x) = result.extension() {
        let mut new_ext = x.to_os_string();
        new_ext.push(ext);
        result.set_extension(new_ext);
    } else {
        result.set_extension(ext);
    }
    result
}
<commit_msg>Update extractor/src/main.rs<commit_after>mod extractor;

extern crate num_cpus;

use clap;
use flate2::write::GzEncoder;
use rayon::prelude::*;
use std::fs;
use std::io::{BufRead, BufWriter};
use std::path::{Path, PathBuf};
use tree_sitter::{Language, Parser, Range};

enum TrapCompression {
    None,
    Gzip,
}

impl TrapCompression {
    fn from_env() -> TrapCompression {
        match std::env::var("CODEQL_RUBY_TRAP_COMPRESSION") {
            Ok(method) => match TrapCompression::from_string(&method) {
                Some(c) => c,
                None => {
                    tracing::error!("Unknown compression method '{}'; using gzip.", &method);
                    TrapCompression::Gzip
                }
            },
            // Default compression method if the env var isn't set:
            Err(_) => TrapCompression::Gzip,
        }
    }

    fn from_string(s: &str) -> Option<TrapCompression> {
        match s.to_lowercase().as_ref() {
            "none" => Some(TrapCompression::None),
            "gzip" => Some(TrapCompression::Gzip),
            _ => None,
        }
    }

    fn extension(&self) -> &str {
        match self {
            TrapCompression::None => ".trap",
            TrapCompression::Gzip => ".trap.gz",
        }
    }
}

/**
 * Gets the number of threads the extractor should use, by reading the
 * CODEQL_THREADS environment variable and using it as described in the
 * extractor spec:
 *
 * "If the number is positive, it indicates the number of threads that should
 * be used. If the number is negative or zero, it should be added to the number
 * of cores available on the machine to determine how many threads to use
 * (minimum of 1). If unspecified, should be considered as set to -1."
 */
fn num_codeql_threads() -> usize {
    let threads_str = std::env::var("CODEQL_THREADS").unwrap_or("-1".to_owned());
    match threads_str.parse::<i32>() {
        Ok(num) if num <= 0 => {
            let reduction = -num as usize;
            std::cmp::max(1, num_cpus::get() - reduction)
        }
        Ok(num) => num as usize,

        Err(_) => {
            tracing::error!(
                "Unable to parse CODEQL_THREADS value '{}'; defaulting to 1 thread.",
                &threads_str
            );
            1
        }
    }
}

fn main() -> std::io::Result<()> {
    tracing_subscriber::fmt()
        .with_target(false)
        .without_time()
        .with_level(true)
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
        .init();

    let num_threads = num_codeql_threads();
    tracing::info!(
        "Using {} {}",
        num_threads,
        if num_threads == 1 {
            "thread"
        } else {
            "threads"
        }
    );
    rayon::ThreadPoolBuilder::new()
        .num_threads(num_threads)
        .build_global()
        .unwrap();

    let matches = clap::App::new("Ruby extractor")
        .version("1.0")
        .author("GitHub")
        .about("CodeQL Ruby extractor")
        .args_from_usage(
            "--source-archive-dir=<DIR> 'Sets a custom source archive folder'
                    --output-dir=<DIR>         'Sets a custom trap folder'
                    --file-list=<FILE_LIST>    'A text files containing the paths of the files to extract'",
        )
        .get_matches();
    let src_archive_dir = matches
        .value_of("source-archive-dir")
        .expect("missing --source-archive-dir");
    let src_archive_dir = PathBuf::from(src_archive_dir);

    let trap_dir = matches
        .value_of("output-dir")
        .expect("missing --output-dir");
    let trap_dir = PathBuf::from(trap_dir);
    let trap_compression = TrapCompression::from_env();

    let file_list = matches.value_of("file-list").expect("missing --file-list");
    let file_list = fs::File::open(file_list)?;

    let language = tree_sitter_ruby::language();
    let erb = tree_sitter_embedded_template::language();
    let schema = node_types::read_node_types_str("ruby", tree_sitter_ruby::NODE_TYPES)?;
    let erb_schema =
        node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?;
    let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
    let lines = lines?;
    lines
        .par_iter()
        .try_for_each(|line| {
            let path = PathBuf::from(line).canonicalize()?;
            let src_archive_file = path_for(&src_archive_dir, &path, "");
            let mut source = std::fs::read(&path)?;
            let code_ranges;
            let mut trap_writer = extractor::new_trap_writer();
            if path.extension().map_or(false, |x| x == "erb") {
                tracing::info!("scanning: {}", path.display());
                extractor::extract(
                    erb,
                    "erb",
                    &erb_schema,
                    &mut trap_writer,
                    &path,
                    &source,
                    &[],
                )?;

                let (ranges, line_breaks) = scan_erb(erb, &source);
                for i in line_breaks {
                    if i < source.len() {
                        source[i] = b'\n';
                    }
                }
                code_ranges = ranges;
            } else {
                code_ranges = vec![];
            }
            extractor::extract(
                language,
                "ruby",
                &schema,
                &mut trap_writer,
                &path,
                &source,
                &code_ranges,
            )?;
            std::fs::create_dir_all(&src_archive_file.parent().unwrap())?;
            std::fs::copy(&path, &src_archive_file)?;
            write_trap(&trap_dir, path, trap_writer, &trap_compression)
        })
        .expect("failed to extract files");

    let path = PathBuf::from("extras");
    let mut trap_writer = extractor::new_trap_writer();
    trap_writer.populate_empty_location();
    write_trap(&trap_dir, path, trap_writer, &trap_compression)
}

fn write_trap(
    trap_dir: &PathBuf,
    path: PathBuf,
    trap_writer: extractor::TrapWriter,
    trap_compression: &TrapCompression,
) -> std::io::Result<()> {
    let trap_file = path_for(&trap_dir, &path, trap_compression.extension());
    std::fs::create_dir_all(&trap_file.parent().unwrap())?;
    let trap_file = std::fs::File::create(&trap_file)?;
    let mut trap_file = BufWriter::new(trap_file);
    match trap_compression {
        TrapCompression::None => trap_writer.output(&mut trap_file),
        TrapCompression::Gzip => {
            let mut compressed_writer = GzEncoder::new(trap_file, flate2::Compression::fast());
            trap_writer.output(&mut compressed_writer)
        }
    }
}

fn scan_erb(erb: Language, source: &std::vec::Vec<u8>) -> (Vec<Range>, Vec<usize>) {
    let mut parser = Parser::new();
    parser.set_language(erb).unwrap();
    let tree = parser.parse(&source, None).expect("Failed to parse file");
    let mut result = Vec::new();
    let mut line_breaks = vec![];

    for n in tree.root_node().children(&mut tree.walk()) {
        let kind = n.kind();
        if kind == "directive" || kind == "output_directive" {
            for c in n.children(&mut tree.walk()) {
                if c.kind() == "code" {
                    let mut range = c.range();
                    if range.end_byte < source.len() {
                        line_breaks.push(range.end_byte);
                        range.end_byte += 1;
                        range.end_point.column += 1;
                    }
                    result.push(range);
                }
            }
        }
    }
    if result.len() == 0 {
        let root = tree.root_node();
        // Add an empty range at the end of the file
        result.push(Range {
            start_byte: root.end_byte(),
            end_byte: root.end_byte(),
            start_point: root.end_position(),
            end_point: root.end_position(),
        });
    }
    (result, line_breaks)
}

fn path_for(dir: &Path, path: &Path, ext: &str) -> PathBuf {
    let mut result = PathBuf::from(dir);
    for component in path.components() {
        match component {
            std::path::Component::Prefix(prefix) => match prefix.kind() {
                std::path::Prefix::Disk(letter) | std::path::Prefix::VerbatimDisk(letter) => {
                    result.push(format!("{}_", letter as char))
                }
                std::path::Prefix::Verbatim(x) | std::path::Prefix::DeviceNS(x) => {
                    result.push(x);
                }
                std::path::Prefix::UNC(server, share)
                | std::path::Prefix::VerbatimUNC(server, share) => {
                    result.push("unc");
                    result.push(server);
                    result.push(share);
                }
            },
            std::path::Component::RootDir => {
                // skip
            }
            std::path::Component::Normal(_) => {
                result.push(component);
            }
            std::path::Component::CurDir => {
                // skip
            }
            std::path::Component::ParentDir => {
                result.pop();
            }
        }
    }
    if let Some(x) = result.extension() {
        let mut new_ext = x.to_os_string();
        new_ext.push(ext);
        result.set_extension(new_ext);
    } else {
        result.set_extension(ext);
    }
    result
}
<|endoftext|>


<commit_before>//! GC heap allocator.
//!
//! ### Safety
//!
//! First, a few general observations about safety in Rust.
//!
//! *   If a `mut` reference to a value exists, and other references to the value
//!     also exist, we can definitely crash.
//!
//!     One way it can happen is that the `mut` value is or contains an
//!     `enum`. Using the other reference, we can borrow a refrence to a
//!     current field of the `enum`. Then, using the `mut` reference, we can
//!     assign some other variant to the `enum`, invalidating the reference.
//!     Rust does not detect that the reference is invalid, so when we use it,
//!     we are accessing gibberish.
//!
//!     Another way is if a callback is called while we're in the middle of
//!     mutating a value, while it's in an invalid state. When you read
//!     "callback", think of `deref` and `drop` methods, which are called
//!     implicitly all over the place. These callbacks are normally safe,
//!     because they simply can't have a reference to the value that's in an
//!     intermediate state. But if another reference exists, it might be used
//!     to try to read from the half-mutated value.
//!
//! *   If a data structure contains two paths to a value, under Rust's usual
//!     rules, you can get two `mut` references to that value.
//!
//!     This is why you tend to build ownership trees in Rust and it's why GC
//!     is particularly challenging in Rust: GCs build an arbitrary graph of
//!     values and references.
//!
//! This GC takes the following approach to ensure safety.
//!
//! *   Minimize access to values stored in the heap. For the most part,
//!     application code *never* sees a direct reference to any value that is
//!     physically stored someplace where it's subject to GC.
//!
//! *   Minimize the times when direct references to in-heap values exist at
//!     all, and during these operations, prevent control from escaping to
//!     arbitrary application code.
//!
//! *   Ensure that when any direct references to in-heap values exist, they
//!     obey Rust's rules: for any given value, either only non-`mut`
//!     references, or at most one `mut` reference, exists at a time.
//!
//! Thus we are particularly keen to avoid the possibility of "reentering" the
//! heap, creating new references to in-heap values while others already exist.
//!
//! References to heap values therefore exist only during the following
//! operations:
//!
//! *   Allocation - That is, moving values into the heap. This is safe because
//!     it never triggers any user code at all while heap references exist.
//!
//! * - Heap reads and writes - The only way to do these is via macro-generated
//!     accessors which do not expose references.  Reads call `from_heap()` on
//!     in-heap values, which is dangerous because `from_heap()` receives a
//!     direct reference.  Writes call `drop()`, which is even more dangerous:
//!     (1) it receives a direct `mut` reference; and (2) it leaves in-heap
//!     values uninitialized.
//!
//! *   GC marking - The code for this is all macro-generated.
//!
//! *   GC sweeping - This calls `drop()`, which is dangerous for the reasons
//!     noted above.
//!
//! To make this scheme safe, `from_heap()` and `drop()` must be tightly controlled.
//! `from_heap()` is therefore in an unsafe trait; users are expected to use
//! the `gc_heap_type!` to autogenerate instances.
//!
//! However, **we leave it up to the user to exercise care with `drop()`.**
//! We suggest *never* implementing `Drop` for a heap type. If you must,
//! avoid reading pointer fields while dropping, and avoid calling into
//! arbitrary code.

use std::cell::RefCell;
use std::collections::HashMap;
use std::marker::PhantomData;
use std::ptr;
use traits::IntoHeapAllocation;
use pages::{heap_type_id, TypedPage, PageSet, PageSetRef};
use ptr::{Pointer, UntypedPointer};
use gcref::GcRef;
use marking::{mark, MarkingTracer};

pub struct Heap {
    page_sets: HashMap<usize, PageSet>,
    pins: RefCell<HashMap<UntypedPointer, usize>>,
    marking_tracer: Option<MarkingTracer>,
}

// What does this do? You'll never guess!
pub type HeapSessionId<'h> = PhantomData<::std::cell::Cell<&'h mut ()>>;

pub struct HeapSession<'h> {
    id: HeapSessionId<'h>,
    heap: &'h mut Heap,
}

/// Create a heap, pass it to a callback, then destroy the heap.
///
/// The heap's lifetime is directly tied to this function call, for safety. (So
/// the API is a little wonky --- but how many heaps were you planning on
/// creating?)
pub fn with_heap<R, F>(f: F) -> R
where
    F: for<'h> FnOnce(&mut HeapSession<'h>) -> R,
{
    Heap::new().enter(f)
}

impl Heap {
    /// Create a new, empty heap.
    pub fn new() -> Heap {
        Heap {
            page_sets: HashMap::new(),
            pins: RefCell::new(HashMap::new()),
            marking_tracer: Some(MarkingTracer::default()),
        }
    }

    /// Run some code using this Heap.
    ///
    /// # Example
    ///
    ///     use cell_gc::{Heap, GCLeaf};
    ///
    ///     let mut heap = Heap::new();
    ///     heap.enter(|hs| {
    ///         // ... hs.alloc(MyHeapStruct { ... }) ...
    ///         # hs.force_gc();
    ///     });
    ///
    pub fn enter<R, F>(&mut self, f: F) -> R
    where
        F: for<'h> FnOnce(&mut HeapSession<'h>) -> R,
    {
        f(&mut HeapSession::new(self))
    }

    /// Add the value `*p` to the root set, protecting it from GC.
    ///
    /// A value that has been pinned *n* times stays in the root set
    /// until it has been unpinned *n* times.
    ///
    /// # Safety
    ///
    /// `p` must point to a live allocation of type `T` in this heap.
    pub unsafe fn pin<'h, T: IntoHeapAllocation<'h>>(&self, p: Pointer<T::In>) {
        let mut pins = self.pins.borrow_mut();
        let entry = pins.entry(p.into()).or_insert(0);
        *entry += 1;
    }

    /// Unpin a heap-allocated value (see `pin`).
    ///
    /// # Safety
    ///
    /// `p` must point to a pinned allocation of type `T` in this heap.
    pub unsafe fn unpin<'h, T: IntoHeapAllocation<'h>>(&self, p: Pointer<T::In>) {
        let mut pins = self.pins.borrow_mut();
        let done = {
            let entry = pins.entry(p.into()).or_insert(0);
            assert!(*entry != 0);
            *entry -= 1;
            *entry == 0
        };
        if done {
            pins.remove(&p.into());
        }
    }

    /// Call the given function on each pinned root.
    pub fn each_pin<F>(&self, mut f: F)
    where
        F: FnMut(UntypedPointer),
    {
        for (&ptr, _) in self.pins.borrow().iter() {
            f(ptr);
        }
    }

    pub unsafe fn from_allocation<'h, T: IntoHeapAllocation<'h>>(ptr: Pointer<T::In>) -> *const Heap {
        (*TypedPage::find(ptr)).header.heap
    }

    pub unsafe fn get_mark_bit<'h, T: IntoHeapAllocation<'h>>(ptr: Pointer<T::In>) -> bool {
        (*TypedPage::find(ptr)).get_mark_bit(ptr)
    }

    pub unsafe fn set_mark_bit<'h, T: IntoHeapAllocation<'h>>(ptr: Pointer<T::In>) {
        (*TypedPage::find(ptr)).set_mark_bit(ptr);
    }

    fn take_marking_tracer(&mut self) -> MarkingTracer {
        self.marking_tracer.take().unwrap()
    }

    fn replace_marking_tracer(&mut self, tracer: MarkingTracer) {
        assert!(self.marking_tracer.is_none());
        assert!(tracer.mark_stack_is_empty());
        self.marking_tracer = Some(tracer);
    }

    /// Run the given function with the marking tracer.
    ///
    /// The marking tracer is taken out of the heap and replaced again so we can
    /// have two independent borrows of the heap and the marking tracer and the
    /// same time.
    pub(crate) fn with_marking_tracer<F, O>(&mut self, mut f: F) -> O
    where
        F: FnMut(&mut Self, &mut MarkingTracer) -> O,
    {
        let mut tracer = self.take_marking_tracer();
        let retval = f(self, &mut tracer);
        self.replace_marking_tracer(tracer);
        retval
    }

    /// Clear all mark bits in preparation for GC.
    ///
    /// # Safety
    ///
    /// This must be called only at the beginning of a GC cycle.
    pub(crate) unsafe fn clear_mark_bits(&mut self) {
        for page_set in self.page_sets.values_mut() {
            page_set.clear_mark_bits();
        }
    }

    fn gc(&mut self) {
        mark(self);

        // sweep phase
        for page_set in self.page_sets.values_mut() {
            unsafe {
                page_set.sweep();
            }
        }
    }
}

impl Drop for Heap {
    fn drop(&mut self) {
        // Perform a final GC to call destructors on any remaining allocations.
        assert!(self.pins.borrow().is_empty());
        self.gc();

        for page_set in self.page_sets.values() {
            page_set.assert_no_allocations();
        }
    }
}

impl<'h> HeapSession<'h> {
    fn new(heap: &'h mut Heap) -> HeapSession<'h> {
        HeapSession {
            id: PhantomData,
            heap,
        }
    }

    fn get_page_set<'a, T: IntoHeapAllocation<'h> + 'a>(&'a mut self) -> PageSetRef<'a, 'h, T> {
        let key = heap_type_id::<T>();
        let heap: *mut Heap = self.heap;
        self.heap
            .page_sets
            .entry(key)
            .or_insert_with(|| unsafe { PageSet::new::<T>(heap) })
            .downcast_mut()
    }

    /// Set (or unset) the limit on the number of pages that can be used to
    /// allocate values of type `T` in this heap. By default, no limit is set.
    ///
    /// See `try_alloc` for more.
    ///
    /// If there are already at least `limit` pages for `T` values, this may have no effect;
    /// it doesn't cause pages to be freed.
    pub fn set_page_limit<T: IntoHeapAllocation<'h>>(&mut self, limit: Option<usize>) {
        self.get_page_set::<T>().set_page_limit(limit);
    }

    /// Allocate memory, moving `value` into the heap.
    ///
    /// If a limit has previously been set using `set_page_limit`, and we run
    /// up against the limit (already have at least that many pages for `T`
    /// values, and they are all full of live values), `try_alloc` first
    /// attempts to free some memory by doing garbage collection. If that
    /// doesn't work, `try_alloc` returns `None`.
    pub fn try_alloc<T: IntoHeapAllocation<'h>>(&mut self, value: T) -> Option<T::Ref> {
        // For now, this is done very early, so that if it panics, the heap is
        // left in an OK state. Better wrapping of raw pointers would make it
        // possible to do this later, closer to the `ptr::write()` call. (And
        // the compiler might optimize away this temporary if we do it that
        // way.) Looking forward to placement new!
        let u = value.into_heap();
        unsafe {
            let alloc = self.get_page_set::<T>().try_alloc();
            alloc
                .or_else(|| {
                    self.heap.gc();
                    self.get_page_set::<T>().try_alloc()
                })
                .map(move |p| {
                    ptr::write(p.as_raw() as *mut _, u);
                    T::wrap_gcref(GcRef::new(p))
                })
        }
    }

    /// Allocate memory, moving `T` into the heap. This may cause garbage collection.
    ///
    /// # Panics
    ///
    /// If a page limit has been set, all pages are full, and GC fails to shake
    /// anything loose.
    pub fn alloc<T: IntoHeapAllocation<'h>>(&mut self, value: T) -> T::Ref {
        self.try_alloc(value)
            .expect("out of memory (gc did not collect anything)")
    }

    /// Do garbage collection.
    pub fn force_gc(&mut self) {
        self.heap.gc();
    }
}
<commit_msg>Tweak implementation of Heap::enter(). Comments.<commit_after>//! GC heap allocator.
//!
//! ### Safety
//!
//! First, a few general observations about safety in Rust.
//!
//! *   If a `mut` reference to a value exists, and other references to the value
//!     also exist, we can definitely crash.
//!
//!     One way it can happen is that the `mut` value is or contains an
//!     `enum`. Using the other reference, we can borrow a refrence to a
//!     current field of the `enum`. Then, using the `mut` reference, we can
//!     assign some other variant to the `enum`, invalidating the reference.
//!     Rust does not detect that the reference is invalid, so when we use it,
//!     we are accessing gibberish.
//!
//!     Another way is if a callback is called while we're in the middle of
//!     mutating a value, while it's in an invalid state. When you read
//!     "callback", think of `deref` and `drop` methods, which are called
//!     implicitly all over the place. These callbacks are normally safe,
//!     because they simply can't have a reference to the value that's in an
//!     intermediate state. But if another reference exists, it might be used
//!     to try to read from the half-mutated value.
//!
//! *   If a data structure contains two paths to a value, under Rust's usual
//!     rules, you can get two `mut` references to that value.
//!
//!     This is why you tend to build ownership trees in Rust and it's why GC
//!     is particularly challenging in Rust: GCs build an arbitrary graph of
//!     values and references.
//!
//! This GC takes the following approach to ensure safety.
//!
//! *   Minimize access to values stored in the heap. For the most part,
//!     application code *never* sees a direct reference to any value that is
//!     physically stored someplace where it's subject to GC.
//!
//! *   Minimize the times when direct references to in-heap values exist at
//!     all, and during these operations, prevent control from escaping to
//!     arbitrary application code.
//!
//! *   Ensure that when any direct references to in-heap values exist, they
//!     obey Rust's rules: for any given value, either only non-`mut`
//!     references, or at most one `mut` reference, exists at a time.
//!
//! Thus we are particularly keen to avoid the possibility of "reentering" the
//! heap, creating new references to in-heap values while others already exist.
//!
//! References to heap values therefore exist only during the following
//! operations:
//!
//! *   Allocation - That is, moving values into the heap. This is safe because
//!     it never triggers any user code at all while heap references exist.
//!
//! * - Heap reads and writes - The only way to do these is via macro-generated
//!     accessors which do not expose references.  Reads call `from_heap()` on
//!     in-heap values, which is dangerous because `from_heap()` receives a
//!     direct reference.  Writes call `drop()`, which is even more dangerous:
//!     (1) it receives a direct `mut` reference; and (2) it leaves in-heap
//!     values uninitialized.
//!
//! *   GC marking - The code for this is all macro-generated.
//!
//! *   GC sweeping - This calls `drop()`, which is dangerous for the reasons
//!     noted above.
//!
//! To make this scheme safe, `from_heap()` and `drop()` must be tightly controlled.
//! `from_heap()` is therefore in an unsafe trait; users are expected to use
//! the `gc_heap_type!` to autogenerate instances.
//!
//! However, **we leave it up to the user to exercise care with `drop()`.**
//! We suggest *never* implementing `Drop` for a heap type. If you must,
//! avoid reading pointer fields while dropping, and avoid calling into
//! arbitrary code.

use std::cell::RefCell;
use std::collections::HashMap;
use std::marker::PhantomData;
use std::ptr;
use traits::IntoHeapAllocation;
use pages::{heap_type_id, TypedPage, PageSet, PageSetRef};
use ptr::{Pointer, UntypedPointer};
use gcref::GcRef;
use marking::{mark, MarkingTracer};

pub struct Heap {
    page_sets: HashMap<usize, PageSet>,
    pins: RefCell<HashMap<UntypedPointer, usize>>,
    marking_tracer: Option<MarkingTracer>,
}

// What does this do? You'll never guess!
pub type HeapSessionId<'h> = PhantomData<::std::cell::Cell<&'h mut ()>>;

pub struct HeapSession<'h> {
    id: HeapSessionId<'h>,
    heap: &'h mut Heap,
}

/// Create a heap, pass it to a callback, then destroy the heap.
///
/// The heap's lifetime is directly tied to this function call, for safety. (So
/// the API is a little wonky --- but how many heaps were you planning on
/// creating?)
pub fn with_heap<R, F>(f: F) -> R
where
    F: for<'h> FnOnce(&mut HeapSession<'h>) -> R,
{
    Heap::new().enter(f)
}

impl Heap {
    /// Create a new, empty heap.
    pub fn new() -> Heap {
        Heap {
            page_sets: HashMap::new(),
            pins: RefCell::new(HashMap::new()),
            marking_tracer: Some(MarkingTracer::default()),
        }
    }

    /// Start a session to access this heap.
    ///
    /// You need a `HeapSession` in order to do anything interesting with a heap.
    /// Each heap has either 0 or 1 `HeapSession` at a time, and a `HeapSession`
    /// is bound to a stack lifetime, so Rust can enforce safety rules.
    ///
    /// It would be safe to make this method public, but it's a pain in
    /// practice. You'll want to pass a `&mut HeapSession<'h>` around, not a
    /// `HeapSession<'h>`, since `HeapSession` is not `Copy`. Use `enter`.
    fn open<'h>(&'h mut self) -> HeapSession<'h> {
        HeapSession {
            id: PhantomData,
            heap: self
        }
    }

    /// Run some code using this Heap.
    ///
    /// # Example
    ///
    ///     use cell_gc::{Heap, GCLeaf};
    ///
    ///     let mut heap = Heap::new();
    ///     heap.enter(|hs| {
    ///         // ... hs.alloc(MyHeapStruct { ... }) ...
    ///         # hs.force_gc();
    ///     });
    ///
    pub fn enter<R, F>(&mut self, f: F) -> R
    where
        F: for<'h> FnOnce(&mut HeapSession<'h>) -> R,
    {
        f(&mut self.open())
    }

    /// Add the value `*p` to the root set, protecting it from GC.
    ///
    /// A value that has been pinned *n* times stays in the root set
    /// until it has been unpinned *n* times.
    ///
    /// # Safety
    ///
    /// `p` must point to a live allocation of type `T` in this heap.
    pub unsafe fn pin<'h, T: IntoHeapAllocation<'h>>(&self, p: Pointer<T::In>) {
        let mut pins = self.pins.borrow_mut();
        let entry = pins.entry(p.into()).or_insert(0);
        *entry += 1;
    }

    /// Unpin a heap-allocated value (see `pin`).
    ///
    /// # Safety
    ///
    /// `p` must point to a pinned allocation of type `T` in this heap.
    pub unsafe fn unpin<'h, T: IntoHeapAllocation<'h>>(&self, p: Pointer<T::In>) {
        let mut pins = self.pins.borrow_mut();
        let done = {
            let entry = pins.entry(p.into()).or_insert(0);
            assert!(*entry != 0);
            *entry -= 1;
            *entry == 0
        };
        if done {
            pins.remove(&p.into());
        }
    }

    /// Call the given function on each pinned root.
    pub fn each_pin<F>(&self, mut f: F)
    where
        F: FnMut(UntypedPointer),
    {
        for (&ptr, _) in self.pins.borrow().iter() {
            f(ptr);
        }
    }

    pub unsafe fn from_allocation<'h, T: IntoHeapAllocation<'h>>(ptr: Pointer<T::In>) -> *const Heap {
        (*TypedPage::find(ptr)).header.heap
    }

    pub unsafe fn get_mark_bit<'h, T: IntoHeapAllocation<'h>>(ptr: Pointer<T::In>) -> bool {
        (*TypedPage::find(ptr)).get_mark_bit(ptr)
    }

    pub unsafe fn set_mark_bit<'h, T: IntoHeapAllocation<'h>>(ptr: Pointer<T::In>) {
        (*TypedPage::find(ptr)).set_mark_bit(ptr);
    }

    fn take_marking_tracer(&mut self) -> MarkingTracer {
        self.marking_tracer.take().unwrap()
    }

    fn replace_marking_tracer(&mut self, tracer: MarkingTracer) {
        assert!(self.marking_tracer.is_none());
        assert!(tracer.mark_stack_is_empty());
        self.marking_tracer = Some(tracer);
    }

    /// Run the given function with the marking tracer.
    ///
    /// The marking tracer is taken out of the heap and replaced again so we can
    /// have two independent borrows of the heap and the marking tracer and the
    /// same time.
    pub(crate) fn with_marking_tracer<F, O>(&mut self, mut f: F) -> O
    where
        F: FnMut(&mut Self, &mut MarkingTracer) -> O,
    {
        let mut tracer = self.take_marking_tracer();
        let retval = f(self, &mut tracer);
        self.replace_marking_tracer(tracer);
        retval
    }

    /// Clear all mark bits in preparation for GC.
    ///
    /// # Safety
    ///
    /// This must be called only at the beginning of a GC cycle.
    pub(crate) unsafe fn clear_mark_bits(&mut self) {
        for page_set in self.page_sets.values_mut() {
            page_set.clear_mark_bits();
        }
    }

    fn gc(&mut self) {
        mark(self);

        // sweep phase
        for page_set in self.page_sets.values_mut() {
            unsafe {
                page_set.sweep();
            }
        }
    }
}

impl Drop for Heap {
    fn drop(&mut self) {
        // Perform a final GC to call destructors on any remaining allocations.
        assert!(self.pins.borrow().is_empty());
        self.gc();

        for page_set in self.page_sets.values() {
            page_set.assert_no_allocations();
        }
    }
}

impl<'h> HeapSession<'h> {
    fn get_page_set<'a, T: IntoHeapAllocation<'h> + 'a>(&'a mut self) -> PageSetRef<'a, 'h, T> {
        let key = heap_type_id::<T>();
        let heap: *mut Heap = self.heap;
        self.heap
            .page_sets
            .entry(key)
            .or_insert_with(|| unsafe { PageSet::new::<T>(heap) })
            .downcast_mut()
    }

    /// Set (or unset) the limit on the number of pages that can be used to
    /// allocate values of type `T` in this heap. By default, no limit is set.
    ///
    /// See `try_alloc` for more.
    ///
    /// If there are already at least `limit` pages for `T` values, this may have no effect;
    /// it doesn't cause pages to be freed.
    pub fn set_page_limit<T: IntoHeapAllocation<'h>>(&mut self, limit: Option<usize>) {
        self.get_page_set::<T>().set_page_limit(limit);
    }

    /// Allocate memory, moving `value` into the heap.
    ///
    /// If a limit has previously been set using `set_page_limit`, and we run
    /// up against the limit (already have at least that many pages for `T`
    /// values, and they are all full of live values), `try_alloc` first
    /// attempts to free some memory by doing garbage collection. If that
    /// doesn't work, `try_alloc` returns `None`.
    pub fn try_alloc<T: IntoHeapAllocation<'h>>(&mut self, value: T) -> Option<T::Ref> {
        // For now, this is done very early, so that if it panics, the heap is
        // left in an OK state. Better wrapping of raw pointers would make it
        // possible to do this later, closer to the `ptr::write()` call. (And
        // the compiler might optimize away this temporary if we do it that
        // way.) Looking forward to placement new!
        let u = value.into_heap();
        unsafe {
            let alloc = self.get_page_set::<T>().try_alloc();
            alloc
                .or_else(|| {
                    self.heap.gc();
                    self.get_page_set::<T>().try_alloc()
                })
                .map(move |p| {
                    ptr::write(p.as_raw() as *mut _, u);
                    T::wrap_gcref(GcRef::new(p))
                })
        }
    }

    /// Allocate memory, moving `T` into the heap. This may cause garbage collection.
    ///
    /// # Panics
    ///
    /// If a page limit has been set, all pages are full, and GC fails to shake
    /// anything loose.
    pub fn alloc<T: IntoHeapAllocation<'h>>(&mut self, value: T) -> T::Ref {
        self.try_alloc(value)
            .expect("out of memory (gc did not collect anything)")
    }

    /// Do garbage collection.
    pub fn force_gc(&mut self) {
        self.heap.gc();
    }
}
<|endoftext|>


<commit_before>
use sdl2::keyboard::Scancode;
use rustc_serialize::json;
use std::path::Path;
use std::fs::File;
use std::io::{Read, Write};
use std::io;

const DEFAULT_CONSOLE_TOGGLE: Scancode = Scancode::Grave;

#[derive(RustcDecodable, RustcEncodable, Debug)]
pub struct Configuration {
    console_toggle: i32,
}

impl Configuration {
    pub fn default() -> Configuration {
        Configuration {
            console_toggle: DEFAULT_CONSOLE_TOGGLE as i32
        }
    }

    pub fn store(&self, target: &Path) -> Result<(), ConfigError> {
        let encoded = json::encode(&self)?;

        let mut file = File::create(target)?;
        write!(file, "{}", encoded)?;

        Ok(())
    }

    pub fn load(target: &Path) -> Result<Configuration, ConfigError> {
        let mut file = File::open(target)?;
        let mut buffer = String::new();
        file.read_to_string(&mut buffer)?;
        let config = json::decode(&buffer)?;
        Ok(config)
    }

    pub fn get_scancode(&self) -> Scancode {
        Scancode::from_i32(self.console_toggle).unwrap_or(DEFAULT_CONSOLE_TOGGLE)
    }
}

#[derive(Debug)]
pub enum ConfigError {
    FileError(io::Error),
    SerializationError(json::EncoderError),
    DeserializationError(json::DecoderError),
}

impl From<io::Error> for ConfigError {
    fn from(e: io::Error) -> ConfigError {
        ConfigError::FileError(e)
    }
}
impl From<json::EncoderError> for ConfigError {
    fn from(e: json::EncoderError) -> ConfigError {
        ConfigError::SerializationError(e)
    }
}
impl From<json::DecoderError> for ConfigError {
    fn from(e: json::DecoderError) -> ConfigError {
        ConfigError::DeserializationError(e)
    }
}

<commit_msg>Now pretty prints config<commit_after>
use sdl2::keyboard::Scancode;
use rustc_serialize::json;
use std::path::Path;
use std::fs::File;
use std::io::{Read, Write};
use std::io;

const DEFAULT_CONSOLE_TOGGLE: Scancode = Scancode::Grave;

#[derive(RustcDecodable, RustcEncodable, Debug)]
pub struct Configuration {
    console_toggle: i32,
}

impl Configuration {
    pub fn default() -> Configuration {
        Configuration {
            console_toggle: DEFAULT_CONSOLE_TOGGLE as i32
        }
    }

    pub fn store(&self, target: &Path) -> Result<(), ConfigError> {
        let encoded = json::as_pretty_json(&self);
        let mut file = File::create(target)?;
        write!(file, "{}", encoded)?;
        writeln!(file, "")?; // End file with newline
        Ok(())
    }

    pub fn load(target: &Path) -> Result<Configuration, ConfigError> {
        let mut file = File::open(target)?;
        let mut buffer = String::new();
        file.read_to_string(&mut buffer)?;
        let config = json::decode(&buffer)?;
        Ok(config)
    }

    pub fn get_scancode(&self) -> Scancode {
        Scancode::from_i32(self.console_toggle).unwrap_or(DEFAULT_CONSOLE_TOGGLE)
    }
}

#[derive(Debug)]
pub enum ConfigError {
    FileError(io::Error),
    SerializationError(json::EncoderError),
    DeserializationError(json::DecoderError),
}

impl From<io::Error> for ConfigError {
    fn from(e: io::Error) -> ConfigError {
        ConfigError::FileError(e)
    }
}
impl From<json::EncoderError> for ConfigError {
    fn from(e: json::EncoderError) -> ConfigError {
        ConfigError::SerializationError(e)
    }
}
impl From<json::DecoderError> for ConfigError {
    fn from(e: json::DecoderError) -> ConfigError {
        ConfigError::DeserializationError(e)
    }
}

<|endoftext|>


<commit_before>//! Read and unfold a line from a `BufRead`.
//!
//! Individual lines within vCard are delimited by the [RFC5322] line
//! break, which is a CRLF sequence (U+000D followed by U+000A).  Long
//! logical lines of text can be split into a multiple-physical-line
//! representation using the following folding technique.  Content lines
//! SHOULD be folded to a maximum width of 75 octets, excluding the line
//! break.  Multi-octet characters MUST remain contiguous.  The rationale
//! for this folding process can be found in [RFC5322], Section 2.1.1.
//!
//! A logical line MAY be continued on the next physical line anywhere
//! between two characters by inserting a CRLF immediately followed by a
//! single white space character (space (U+0020) or horizontal tab
//! (U+0009)).  The folded line MUST contain at least one character.  Any
//! sequence of CRLF followed immediately by a single white space
//! character is ignored (removed) when processing the content type.
//!
//! # Examples
//!
//! ```toml
//! [dependencies.ical]
//! version = "0.3.*"
//! default-features = false
//! features = ["line-reader"]
//! ```
//!
//! ```rust
//! extern crate ical;
//!
//! use std::io::BufReader;
//! use std::fs::File;
//!
//! let buf = BufReader::new(File::open("./tests/ressources/vcard_input.vcf").unwrap());
//!
//! let reader = ical::LineReader::new(buf);
//!
//! for line in reader {
//!     println!("{}", line);
//! }
//! ```

use std::fmt;
use std::io::BufRead;
use std::iter::Iterator;

/// An unfolded raw line.
///
/// Its inner is only a raw line from the file. No parsing or checking have
/// been made yet.
#[derive(Debug, Clone, Default)]
pub struct Line {
    inner: String,
    number: usize,
}

impl Line {
    /// Return a new `Line` object.
    pub fn new(line: String, line_number: usize) -> Line {
        Line {
            inner: line,
            number: line_number,
        }
    }

    /// Return a `&str`
    pub fn as_str(&self) -> &str {
        self.inner.as_str()
    }

    /// Return the line number.
    pub fn number(&self) -> usize {
        self.number
    }
}

impl fmt::Display for Line {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Line {}: {}", self.number, self.inner)
    }
}

/// A trait generic for implementing line reading use crate::by `PropertyParser`.
pub trait LineRead {
    /// Return the next line unwrapped and formated.
    fn next_line(&mut self) -> Option<Line>;
}

#[derive(Debug, Clone, Default)]
/// Take a `BufRead` and return the unfolded `Line`.
pub struct LineReader<B> {
    reader: B,
    saved: Option<String>,
    number: usize,
}

impl<B: BufRead> LineReader<B> {
    /// Return a new `LineReader` from a `Reader`.
    pub fn new(reader: B) -> LineReader<B> {
        LineReader {
            reader: reader,
            saved: None,
            number: 0,
        }
    }
}

impl<B: BufRead> LineRead for LineReader<B> {
    fn next_line(&mut self) -> Option<Line> {
        let mut next_line = String::new();
        let mut line_number: usize = 0;

        if let Some(start) = self.saved.take() {
            // If during the last iteration a new line have been saved, start with.
            next_line.push_str(start.as_str());
            self.number += 1;
            line_number = self.number;
        } else {
            // This is the first iteration, next_start isn't been filled yet.
            for line in self.reader.by_ref().lines() {
                let line = line.unwrap();
                self.number += 1;

                if !line.is_empty() {
                    next_line = line.trim_right().to_string();
                    line_number = self.number;
                    break;
                }
            }
        }

        for line in self.reader.by_ref().lines() {
            let mut line = line.unwrap();

            if line.is_empty() {
                self.number += 1;
            } else if line.starts_with(' ') || line.starts_with("  ") {
                // This is a multi-lines attribute.

                // Remove the ' ' charactere and join with the current line.
                line.remove(0);
                next_line.push_str(line.trim_right());
                self.number += 1;
            } else {
                // This is a new attribute so it need to be saved it for
                // the next iteration.
                self.saved = Some(line.trim().to_string());
                break;
            }
        }

        if next_line.is_empty() {
            None
        } else {
            Some(Line::new(next_line, line_number))
        }
    }
}

impl<B: BufRead> Iterator for LineReader<B> {
    type Item = Line;

    fn next(&mut self) -> Option<Line> {
        self.next_line()
    }
}
<commit_msg>Remove call to deprecated trim_right<commit_after>//! Read and unfold a line from a `BufRead`.
//!
//! Individual lines within vCard are delimited by the [RFC5322] line
//! break, which is a CRLF sequence (U+000D followed by U+000A).  Long
//! logical lines of text can be split into a multiple-physical-line
//! representation using the following folding technique.  Content lines
//! SHOULD be folded to a maximum width of 75 octets, excluding the line
//! break.  Multi-octet characters MUST remain contiguous.  The rationale
//! for this folding process can be found in [RFC5322], Section 2.1.1.
//!
//! A logical line MAY be continued on the next physical line anywhere
//! between two characters by inserting a CRLF immediately followed by a
//! single white space character (space (U+0020) or horizontal tab
//! (U+0009)).  The folded line MUST contain at least one character.  Any
//! sequence of CRLF followed immediately by a single white space
//! character is ignored (removed) when processing the content type.
//!
//! # Examples
//!
//! ```toml
//! [dependencies.ical]
//! version = "0.3.*"
//! default-features = false
//! features = ["line-reader"]
//! ```
//!
//! ```rust
//! extern crate ical;
//!
//! use std::io::BufReader;
//! use std::fs::File;
//!
//! let buf = BufReader::new(File::open("./tests/ressources/vcard_input.vcf").unwrap());
//!
//! let reader = ical::LineReader::new(buf);
//!
//! for line in reader {
//!     println!("{}", line);
//! }
//! ```

use std::fmt;
use std::io::BufRead;
use std::iter::Iterator;

/// An unfolded raw line.
///
/// Its inner is only a raw line from the file. No parsing or checking have
/// been made yet.
#[derive(Debug, Clone, Default)]
pub struct Line {
    inner: String,
    number: usize,
}

impl Line {
    /// Return a new `Line` object.
    pub fn new(line: String, line_number: usize) -> Line {
        Line {
            inner: line,
            number: line_number,
        }
    }

    /// Return a `&str`
    pub fn as_str(&self) -> &str {
        self.inner.as_str()
    }

    /// Return the line number.
    pub fn number(&self) -> usize {
        self.number
    }
}

impl fmt::Display for Line {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Line {}: {}", self.number, self.inner)
    }
}

/// A trait generic for implementing line reading use crate::by `PropertyParser`.
pub trait LineRead {
    /// Return the next line unwrapped and formated.
    fn next_line(&mut self) -> Option<Line>;
}

#[derive(Debug, Clone, Default)]
/// Take a `BufRead` and return the unfolded `Line`.
pub struct LineReader<B> {
    reader: B,
    saved: Option<String>,
    number: usize,
}

impl<B: BufRead> LineReader<B> {
    /// Return a new `LineReader` from a `Reader`.
    pub fn new(reader: B) -> LineReader<B> {
        LineReader {
            reader: reader,
            saved: None,
            number: 0,
        }
    }
}

impl<B: BufRead> LineRead for LineReader<B> {
    fn next_line(&mut self) -> Option<Line> {
        let mut next_line = String::new();
        let mut line_number: usize = 0;

        if let Some(start) = self.saved.take() {
            // If during the last iteration a new line have been saved, start with.
            next_line.push_str(start.as_str());
            self.number += 1;
            line_number = self.number;
        } else {
            // This is the first iteration, next_start isn't been filled yet.
            for line in self.reader.by_ref().lines() {
                let line = line.unwrap();
                self.number += 1;

                if !line.is_empty() {
                    next_line = line.trim_end().to_string();
                    line_number = self.number;
                    break;
                }
            }
        }

        for line in self.reader.by_ref().lines() {
            let mut line = line.unwrap();

            if line.is_empty() {
                self.number += 1;
            } else if line.starts_with(' ') || line.starts_with("  ") {
                // This is a multi-lines attribute.

                // Remove the ' ' charactere and join with the current line.
                line.remove(0);
                next_line.push_str(line.trim_end());
                self.number += 1;
            } else {
                // This is a new attribute so it need to be saved it for
                // the next iteration.
                self.saved = Some(line.trim().to_string());
                break;
            }
        }

        if next_line.is_empty() {
            None
        } else {
            Some(Line::new(next_line, line_number))
        }
    }
}

impl<B: BufRead> Iterator for LineReader<B> {
    type Item = Line;

    fn next(&mut self) -> Option<Line> {
        self.next_line()
    }
}
<|endoftext|>


<commit_before>//! Verify conventional SSA form.

use dbg::DisplayList;
use dominator_tree::DominatorTree;
use flowgraph::ControlFlowGraph;
use ir::Function;
use regalloc::liveness::Liveness;
use regalloc::virtregs::VirtRegs;
use timing;
use verifier::Result;

/// Verify conventional SSA form for `func`.
///
/// Conventional SSA form is represented in Cretonne with the help of virtual registers:
///
/// - Two values are said to be *PHI-related* if one is an EBB argument and the other is passed as
///   a branch argument in a location that matches the first value.
/// - PHI-related values must belong to the same virtual register.
/// - Two values in the same virtual register must not have overlapping live ranges.
///
/// Additionally, we verify this property of virtual registers:
///
/// - The values in a virtual register are topologically ordered w.r.t. dominance.
///
/// We don't verify that virtual registers are minimal. Minimal CSSA is not required.
pub fn verify_cssa(
    func: &Function,
    cfg: &ControlFlowGraph,
    domtree: &DominatorTree,
    liveness: &Liveness,
    virtregs: &VirtRegs,
) -> Result {
    let _tt = timing::verify_cssa();
    let verifier = CssaVerifier {
        func,
        cfg,
        domtree,
        virtregs,
        liveness,
    };
    verifier.check_virtregs()?;
    verifier.check_cssa()?;
    Ok(())
}

struct CssaVerifier<'a> {
    func: &'a Function,
    cfg: &'a ControlFlowGraph,
    domtree: &'a DominatorTree,
    virtregs: &'a VirtRegs,
    liveness: &'a Liveness,
}

impl<'a> CssaVerifier<'a> {
    fn check_virtregs(&self) -> Result {
        for vreg in self.virtregs.all_virtregs() {
            let values = self.virtregs.values(vreg);

            for (idx, &val) in values.iter().enumerate() {
                if !self.func.dfg.value_is_valid(val) {
                    return err!(val, "Invalid value in {}", vreg);
                }
                if !self.func.dfg.value_is_attached(val) {
                    return err!(val, "Detached value in {}", vreg);
                }
                if self.liveness.get(val).is_none() {
                    return err!(val, "Value in {} has no live range", vreg);
                };

                // Check topological ordering with the previous values in the virtual register.
                let def = self.func.dfg.value_def(val).into();
                let def_ebb = self.func.layout.pp_ebb(def);
                for &prev_val in &values[0..idx] {
                    let prev_def = self.func.dfg.value_def(prev_val);

                    // Enforce topological ordering of defs in the virtual register.
                    if self.domtree.dominates(def, prev_def, &self.func.layout) {
                        return err!(
                            val,
                            "Value in {} = {} def dominates previous {}",
                            vreg,
                            DisplayList(values),
                            prev_val
                        );
                    }

                    // Knowing that values are in topo order, we can check for interference this
                    // way.
                    let ctx = self.liveness.context(&self.func.layout);
                    if self.liveness[prev_val].overlaps_def(def, def_ebb, ctx) {
                        return err!(
                            val,
                            "Value def in {} = {} interferes with {}",
                            vreg,
                            DisplayList(values),
                            prev_val
                        );
                    }
                }
            }
        }

        Ok(())
    }

    fn check_cssa(&self) -> Result {
        for ebb in self.func.layout.ebbs() {
            let ebb_params = self.func.dfg.ebb_params(ebb);
            for (_, pred) in self.cfg.pred_iter(ebb) {
                let pred_args = self.func.dfg.inst_variable_args(pred);
                // This should have been caught by an earlier verifier pass.
                assert_eq!(
                    ebb_params.len(),
                    pred_args.len(),
                    "Wrong arguments on branch."
                );

                for (&ebb_param, &pred_arg) in ebb_params.iter().zip(pred_args) {
                    if !self.virtregs.same_class(ebb_param, pred_arg) {
                        return err!(
                            pred,
                            "{} and {} must be in the same virtual register",
                            ebb_param,
                            pred_arg
                        );
                    }
                }
            }
        }

        Ok(())
    }
}
<commit_msg>Better verifier error for coinciding defs.<commit_after>//! Verify conventional SSA form.

use dbg::DisplayList;
use dominator_tree::DominatorTree;
use flowgraph::ControlFlowGraph;
use ir::{Function, ExpandedProgramPoint};
use regalloc::liveness::Liveness;
use regalloc::virtregs::VirtRegs;
use timing;
use verifier::Result;

/// Verify conventional SSA form for `func`.
///
/// Conventional SSA form is represented in Cretonne with the help of virtual registers:
///
/// - Two values are said to be *PHI-related* if one is an EBB argument and the other is passed as
///   a branch argument in a location that matches the first value.
/// - PHI-related values must belong to the same virtual register.
/// - Two values in the same virtual register must not have overlapping live ranges.
///
/// Additionally, we verify this property of virtual registers:
///
/// - The values in a virtual register are topologically ordered w.r.t. dominance.
///
/// We don't verify that virtual registers are minimal. Minimal CSSA is not required.
pub fn verify_cssa(
    func: &Function,
    cfg: &ControlFlowGraph,
    domtree: &DominatorTree,
    liveness: &Liveness,
    virtregs: &VirtRegs,
) -> Result {
    let _tt = timing::verify_cssa();
    let verifier = CssaVerifier {
        func,
        cfg,
        domtree,
        virtregs,
        liveness,
    };
    verifier.check_virtregs()?;
    verifier.check_cssa()?;
    Ok(())
}

struct CssaVerifier<'a> {
    func: &'a Function,
    cfg: &'a ControlFlowGraph,
    domtree: &'a DominatorTree,
    virtregs: &'a VirtRegs,
    liveness: &'a Liveness,
}

impl<'a> CssaVerifier<'a> {
    fn check_virtregs(&self) -> Result {
        for vreg in self.virtregs.all_virtregs() {
            let values = self.virtregs.values(vreg);

            for (idx, &val) in values.iter().enumerate() {
                if !self.func.dfg.value_is_valid(val) {
                    return err!(val, "Invalid value in {}", vreg);
                }
                if !self.func.dfg.value_is_attached(val) {
                    return err!(val, "Detached value in {}", vreg);
                }
                if self.liveness.get(val).is_none() {
                    return err!(val, "Value in {} has no live range", vreg);
                };

                // Check topological ordering with the previous values in the virtual register.
                let def: ExpandedProgramPoint = self.func.dfg.value_def(val).into();
                let def_ebb = self.func.layout.pp_ebb(def);
                for &prev_val in &values[0..idx] {
                    let prev_def: ExpandedProgramPoint = self.func.dfg.value_def(prev_val).into();

                    if prev_def == def {
                        return err!(
                            val,
                            "Values {} and {} in {} = {} defined at the same program point",
                            prev_val,
                            val,
                            vreg,
                            DisplayList(values)
                        );
                    }

                    // Enforce topological ordering of defs in the virtual register.
                    if self.domtree.dominates(def, prev_def, &self.func.layout) {
                        return err!(
                            val,
                            "Value in {} = {} def dominates previous {}",
                            vreg,
                            DisplayList(values),
                            prev_val
                        );
                    }

                    // Knowing that values are in topo order, we can check for interference this
                    // way.
                    let ctx = self.liveness.context(&self.func.layout);
                    if self.liveness[prev_val].overlaps_def(def, def_ebb, ctx) {
                        return err!(
                            val,
                            "Value def in {} = {} interferes with {}",
                            vreg,
                            DisplayList(values),
                            prev_val
                        );
                    }
                }
            }
        }

        Ok(())
    }

    fn check_cssa(&self) -> Result {
        for ebb in self.func.layout.ebbs() {
            let ebb_params = self.func.dfg.ebb_params(ebb);
            for (_, pred) in self.cfg.pred_iter(ebb) {
                let pred_args = self.func.dfg.inst_variable_args(pred);
                // This should have been caught by an earlier verifier pass.
                assert_eq!(
                    ebb_params.len(),
                    pred_args.len(),
                    "Wrong arguments on branch."
                );

                for (&ebb_param, &pred_arg) in ebb_params.iter().zip(pred_args) {
                    if !self.virtregs.same_class(ebb_param, pred_arg) {
                        return err!(
                            pred,
                            "{} and {} must be in the same virtual register",
                            ebb_param,
                            pred_arg
                        );
                    }
                }
            }
        }

        Ok(())
    }
}
<|endoftext|>