<commit_before>use std::sync::atomic::{self, AtomicPtr};
use std::cell::Cell;

static GARBAGE: Stack<Box<Drop>> = Stack::new();
static READERS: Stack<RawReader> = Stack::new();
static STATE: State = State::new();
static TICKS_BEFORE_GC: u16 = 2000;

thread_local!(static CLOCK: Cell<u16> = 0);

pub fn gc() {
    // Start the garbage collection.
    if !STATE.start_gc() {
        // Another thread is garbage collecting, so we short-circuit.
        return;
    }

    // Initially, every garbage is marked unused.
    let mut unused = GARBAGE.collect();

    // Traverse the readers and update the reference counts.
    READERS.take_each(|reader| {
        if reader.active.load() {
            // The reader is not released yet, and is thus considered active.

            // Remove the reader from the unused set and insert it back into the log (if it
            // exists in the unused set), as the garbage is active.
            unused.remove(reader.ptr).map(|x| GARBAGE.insert(x));
            // Put the reader back in the structure.
            READERS.insert(reader);
        } else {
            // The reader was released. Destroy it.
            reader.destroy();
        }
    });

    // End the garbage collection cycle.
    STATE.end_gc();
}

pub fn tick() {
    let clock = CLOCK.get();
    if clock == TICKS_BEFORE_GC {
        CLOCK.set(0);
        gc();
    } else {
        CLOCK.set(clock + 1);
    }
}

pub struct Stack<T> {
    head: AtomicPtr<Node<T>>,
}

struct Node<T> {
    data: T,
    next: *mut Node<T>,
}

impl<T> Stack<T> {
    const fn new() -> Stack<T> {
        Stack {
            head: AtomicPtr::new(0 as *const T),
        }
    }

    fn push(&self, t: T) {
        // Allocate the node, and immediately turn it into a `*mut` pointer.
        let n = Box::into_raw(Box::new(Node {
            data: t,
            next: null_mut(),
        }));

        loop {
            // Snapshot current head.
            let head = self.head.load(atomic::Ordering::Relaxed);

            // Update `next` pointer with snapshot.
            unsafe { (*n).next = head; }

            // If snapshot is still good, link in new node.
            if self.head.compare_and_swap(head, n, atomic::Ordering::Release) == head {
                break
            }
        }
    }

    fn take_each(&self, f: F)
    where F: Fn(T) {
        // Replace the old head with a null pointer.
        let mut node = self.head.swap(AtomicPtr::default(), atomic::Ordering::Acquire);

        // We traverse every node until the pointer is null.
        while !node.is_null() {
            // Read the node into an owned box.
            let bx = unsafe { Box::from_raw(head) };
            // Apply the provided closure.
            f(bx.data);
            // Go to the next link.
            node = bx.next;
        }
    }

    fn collect(&self) -> HashSet<T> {
        let mut hs = HashSet::new();
        self.take_each(|x| hs.insert(x));

        hs
    }
}

struct RawReader {
    active: *const AtomicBool,
    ptr: *const T,
}

impl RawReader {
    unsafe fn destroy(self) {
        // Drop the atomic boolean stored on the heap.
        mem::drop_in_place(self.active);
    }
}

struct Reader<'a, T> {
    raw: RawReader,
    _marker: PhantomData<'a>,
}

impl<'a, T> Reader<'a, T> {
    fn drop(&mut self) {
        self.raw.active.store(true);
    }
}

#[derive(Default)]
struct State {
    flags: AtomicUsize,
}

impl State {
    fn start_gc(&self) -> bool {
        // Mark that a garbage collection is pending.
        if self.flags.fetch_or(1, atomic::Ordering::Relaxed) & 1 != 0 {
            // Another thread is pending to or currently garbage collecting, so we won't do the
            // same.
            return false;
        }

        // Spin until no thread is currently modifying the stacks. This prevents premature frees in
        // the thread which is currently pushing to `self.readers`.
        loop {
            // Read the flags, and if no readers or garbage collectors, activate garbage
            // collection.
            let flags = self.flags.compare_and_swap(1, 0b11, atomic::Ordering::Relaxed);
            if flags == 1 {
                // Currently, no one accesses the readers stack and the CAS above means that the
                // lowest bitflag have been set, indicating that a garbage collection is now
                // active.
                return true;
            }
        }
    }

    fn end_gc(&self) {
        self.flags.fetch_sub(1, atomic::Ordering::Relaxed);
    }

    fn start_read(&self) {
        // Increment the number of threads currently pushing to the readers stack. We add two to
        // account for the LSB being a separate bitflag.
        self.flags.fetch_add(2, atomic::Ordering::Relaxed);
    }

    fn end_read(&self) {
        self.flags.fetch_sub(2, atomic::Ordering::Relaxed);
    }
}

pub struct Reading;

impl Reading {
    pub fn obtain() -> Reading {
        STATE.start_read();

        Reading
    }

    pub fn load<T>(&self, a: &Atomic<T>) -> Reader<T> {
        // Construct the raw reader.
        let reader = RawReader {
            // Load a snapshot of the pointer.
            ptr: self.inner.load(atomic::Ordering::Relaxed),
            // We allocate the atomic boolean on the heap as it is shared between the returned RAII
            // guard and the reader stack.
            released: Box::into_raw(Box::new(AtomicBool::new(false))),
        };

        // Register the reader through the reader stack, ensuring that it is not freed before the
        // RAII guard drops (`reader.release` is set to `true`).
        READERS.push(reader);

        Reader {
            raw: reader,
            _marker: PhantomData,
        }
    }
}

impl Drop for Reading {
    fn drop(&mut self) {
        STATE.end_read();
        tick();
    }
}

pub struct Atomic<T> {
    inner: AtomicPtr<T>,
    snapshots: Stack<Box<T>>,
    readers: Stack<RawReader>,
    state: State,
}

impl<T> Atomic<T> {
    pub fn new(inner: T) -> Atomic<T> {
        Atomic {
            inner: AtomicPtr::new(Box::into_raw(Box::new(inner))),
            snapshots: Stack::new(),
            readers: Stack::new(),
            flags: State::default(),
        }
    }

    pub fn load(&self) -> Reader<T> {
        let guard = Reading::obtain();
        guard.load(self)
    }

    pub fn store(&self, new: Box<T>) {
        // Replace the inner by the new value.
        let old = self.inner.swap(Box::into_raw(new), atomic::Ordering::Relaxed);
        // Push the old pointer to the garbage stack.
        GARBAGE.push(Box::from_raw(old));

        tick();
    }
}
<commit_msg>Make reader reentrant.<commit_after>use std::sync::atomic::{self, AtomicPtr};
use std::cell::Cell;

static GARBAGE: Stack<Box<Drop>> = Stack::new();
static ACTIVE_SNAPSHOTS: Stack<RawSnapshot> = Stack::new();
static STATE: State = State::new();
static TICKS_BEFORE_GC: u16 = 2000;

thread_local!(static CLOCK: Cell<u16> = Cell::new(0));
thread_local!(static ACTIVE_READERS: Cell<usize> = Cell::new(0));

pub fn gc() {
    if ACTIVE_SNAPSHOTS.get() != 0 {
        return;
    }

    // Start the garbage collection.
    if !STATE.start_gc() {
        // Another thread is garbage collecting, so we short-circuit.
        return;
    }

    // Initially, every garbage is marked unused.
    let mut unused = GARBAGE.collect();

    // Traverse the active snapshots and update the reference counts.
    ACTIVE_SNAPSHOTS.take_each(|reader| {
        if reader.active.load() {
            // The reader is not released yet, and is thus considered active.

            // Remove the reader from the unused set and insert it back into the log (if it
            // exists in the unused set), as the garbage is active.
            unused.remove(reader.ptr).map(|x| GARBAGE.insert(x));
            // Put the reader back in the structure.
            ACTIVE_SNAPSHOTS.insert(reader);
        } else {
            // The reader was released. Destroy it.
            reader.destroy();
        }
    });

    // End the garbage collection cycle.
    STATE.end_gc();
}

pub fn tick() {
    let clock = CLOCK.get();
    if clock == TICKS_BEFORE_GC {
        CLOCK.set(0);
        gc();
    } else {
        CLOCK.set(clock + 1);
    }
}

pub fn read<T, F>(f: F) -> T
where F: Fn(Reader) -> T {
    let mut active = ACTIVE_READERS.get();
    ACTIVE_READERS.set(active + 1);
    if active == 0 {
        STATE.start_read();
    }

    let reader = Reader;
    let ret = f(&reader);

    active = ACTIVE_READERS.get();
    if active == 1 {
        STATE.end_read();
    }
    ACTIVE_READERS.set(active - 1);

    ret
}

pub struct Reader;

impl Reader {
    pub fn load<T>(&self, a: &Atomic<T>) -> Snapshot<T> {
        // Construct the raw reader.
        let reader = RawSnapshot {
            // Load a snapshot of the pointer.
            ptr: self.inner.load(atomic::Ordering::Relaxed),
            // We allocate the atomic boolean on the heap as it is shared between the returned RAII
            // guard and the reader stack.
            released: Box::into_raw(Box::new(AtomicBool::new(false))),
        };

        // Register the reader through the reader stack, ensuring that it is not freed before the
        // RAII guard drops (`reader.release` is set to `true`).
        ACTIVE_SNAPSHOTS.push(reader);

        Snapshot {
            raw: reader,
            _marker: PhantomData,
        }
    }
}

pub struct Stack<T> {
    head: AtomicPtr<Node<T>>,
}

struct Node<T> {
    data: T,
    next: *mut Node<T>,
}

impl<T> Stack<T> {
    const fn new() -> Stack<T> {
        Stack {
            head: AtomicPtr::new(0 as *const T),
        }
    }

    fn push(&self, t: T) {
        // Allocate the node, and immediately turn it into a `*mut` pointer.
        let n = Box::into_raw(Box::new(Node {
            data: t,
            next: null_mut(),
        }));

        loop {
            // Snapshot current head.
            let head = self.head.load(atomic::Ordering::Relaxed);

            // Update `next` pointer with snapshot.
            unsafe { (*n).next = head; }

            // If snapshot is still good, link in new node.
            if self.head.compare_and_swap(head, n, atomic::Ordering::Release) == head {
                break
            }
        }
    }

    fn take_each(&self, f: F)
    where F: Fn(T) {
        // Replace the old head with a null pointer.
        let mut node = self.head.swap(AtomicPtr::default(), atomic::Ordering::Acquire);

        // We traverse every node until the pointer is null.
        while !node.is_null() {
            // Read the node into an owned box.
            let bx = unsafe { Box::from_raw(head) };
            // Apply the provided closure.
            f(bx.data);
            // Go to the next link.
            node = bx.next;
        }
    }

    fn collect(&self) -> HashSet<T> {
        let mut hs = HashSet::new();
        self.take_each(|x| hs.insert(x));

        hs
    }
}

struct RawSnapshot<T> {
    active: *const AtomicBool,
    ptr: *const T,
}

impl RawSnapshot {
    unsafe fn destroy(self) {
        // Drop the atomic boolean stored on the heap.
        mem::drop_in_place(self.active);
    }
}

struct Snapshot<'a, T> {
    raw: RawSnapshot,
    _marker: PhantomData<'a>,
}

impl<'a, T> Snapshot<'a, T> {
    fn drop(&mut self) {
        self.raw.active.store(true);
    }
}

#[derive(Default)]
struct State {
    flags: AtomicUsize,
}

impl State {
    fn start_gc(&self) -> bool {
        // Mark that a garbage collection is pending.
        if self.flags.fetch_or(1, atomic::Ordering::Relaxed) & 1 != 0 {
            // Another thread is pending to or currently garbage collecting, so we won't do the
            // same.
            return false;
        }

        // Spin until no thread is currently modifying the stacks. This prevents premature frees in
        // the thread which is currently pushing to the active snapshots.
        loop {
            // Read the flags, and if no active snapshots or garbage collectors, activate garbage
            // collection.
            let flags = self.flags.compare_and_swap(1, 0b11, atomic::Ordering::Relaxed);
            if flags == 1 {
                // Currently, no one accesses the active snapshot stack and the CAS above means
                // that the lowest bitflag have been set, indicating that a garbage collection is
                // now active.
                return true;
            }
        }
    }

    fn end_gc(&self) {
        self.flags.fetch_sub(1, atomic::Ordering::Relaxed);
    }

    fn start_read(&self) {
        // Increment the number of threads currently pushing to the active snapshot stack. We add
        // two to account for the LSB being a separate bitflag.
        self.flags.fetch_add(2, atomic::Ordering::Relaxed);
    }

    fn end_read(&self) {
        self.flags.fetch_sub(2, atomic::Ordering::Relaxed);
    }
}

pub struct Atomic<T> {
    inner: AtomicPtr<T>,
}

impl<T> Atomic<T> {
    pub fn new(inner: T) -> Atomic<T> {
        Atomic {
            inner: AtomicPtr::new(Box::into_raw(Box::new(inner))),
        }
    }

    pub fn load(&self) -> Snapshot<T> {
        read(|r| r.load(self))
    }

    pub fn store(&self, new: Box<T>) {
        // Replace the inner by the new value.
        let old = self.inner.swap(Box::into_raw(new), atomic::Ordering::Relaxed);
        // Push the old pointer to the garbage stack.
        GARBAGE.push(Box::from_raw(old));

        tick();
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

#![allow(unsafe_code)]

//! Implementation of `iterable<...>` and `iterable<..., ...>` WebIDL declarations.

use crate::dom::bindings::codegen::Bindings::IterableIteratorBinding::IterableKeyAndValueResult;
use crate::dom::bindings::codegen::Bindings::IterableIteratorBinding::IterableKeyOrValueResult;
use crate::dom::bindings::error::Fallible;
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector};
use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::bindings::trace::{JSTraceable, RootedTraceableBox};
use crate::dom::globalscope::GlobalScope;
use crate::script_runtime::JSContext as SafeJSContext;
use dom_struct::dom_struct;
use js::conversions::ToJSValConvertible;
use js::jsapi::{Heap, JSContext, JSObject};
use js::jsval::UndefinedValue;
use js::rust::{HandleValue, MutableHandleObject};
use std::cell::Cell;
use std::ptr;
use std::ptr::NonNull;

/// The values that an iterator will iterate over.
#[derive(JSTraceable, MallocSizeOf)]
pub enum IteratorType {
    /// The keys of the iterable object.
    Keys,
    /// The values of the iterable object.
    Values,
    /// The keys and values of the iterable object combined.
    Entries,
}

/// A DOM object that can be iterated over using a pair value iterator.
pub trait Iterable {
    /// The type of the key of the iterator pair.
    type Key: ToJSValConvertible;
    /// The type of the value of the iterator pair.
    type Value: ToJSValConvertible;
    /// Return the number of entries that can be iterated over.
    fn get_iterable_length(&self) -> u32;
    /// Return the value at the provided index.
    fn get_value_at_index(&self, index: u32) -> Self::Value;
    /// Return the key at the provided index.
    fn get_key_at_index(&self, index: u32) -> Self::Key;
}

/// An iterator over the iterable entries of a given DOM interface.
//FIXME: #12811 prevents dom_struct with type parameters
#[dom_struct]
pub struct IterableIterator<T: DomObject + JSTraceable + Iterable> {
    reflector: Reflector,
    iterable: Dom<T>,
    type_: IteratorType,
    index: Cell<u32>,
}

impl<T: DomObject + JSTraceable + Iterable> IterableIterator<T> {
    /// Create a new iterator instance for the provided iterable DOM interface.
    pub fn new(
        iterable: &T,
        type_: IteratorType,
        wrap: unsafe fn(SafeJSContext, &GlobalScope, Box<IterableIterator<T>>) -> DomRoot<Self>,
    ) -> DomRoot<Self> {
        let iterator = Box::new(IterableIterator {
            reflector: Reflector::new(),
            type_: type_,
            iterable: Dom::from_ref(iterable),
            index: Cell::new(0),
        });
        reflect_dom_object(iterator, &*iterable.global(), wrap)
    }

    /// Return the next value from the iterable object.
    #[allow(non_snake_case)]
    pub fn Next(&self, cx: SafeJSContext) -> Fallible<NonNull<JSObject>> {
        let index = self.index.get();
        rooted!(in(*cx) let mut value = UndefinedValue());
        rooted!(in(*cx) let mut rval = ptr::null_mut::<JSObject>());
        let result = if index >= self.iterable.get_iterable_length() {
            dict_return(*cx, rval.handle_mut(), true, value.handle())
        } else {
            match self.type_ {
                IteratorType::Keys => {
                    unsafe {
                        self.iterable
                            .get_key_at_index(index)
                            .to_jsval(*cx, value.handle_mut());
                    }
                    dict_return(*cx, rval.handle_mut(), false, value.handle())
                },
                IteratorType::Values => {
                    unsafe {
                        self.iterable
                            .get_value_at_index(index)
                            .to_jsval(*cx, value.handle_mut());
                    }
                    dict_return(*cx, rval.handle_mut(), false, value.handle())
                },
                IteratorType::Entries => {
                    rooted!(in(*cx) let mut key = UndefinedValue());
                    unsafe {
                        self.iterable
                            .get_key_at_index(index)
                            .to_jsval(*cx, key.handle_mut());
                        self.iterable
                            .get_value_at_index(index)
                            .to_jsval(*cx, value.handle_mut());
                    }
                    key_and_value_return(*cx, rval.handle_mut(), key.handle(), value.handle())
                },
            }
        };
        self.index.set(index + 1);
        result.map(|_| NonNull::new(rval.get()).expect("got a null pointer"))
    }
}

fn dict_return(
    cx: *mut JSContext,
    mut result: MutableHandleObject,
    done: bool,
    value: HandleValue,
) -> Fallible<()> {
    let mut dict = IterableKeyOrValueResult::empty();
    dict.done = done;
    dict.value.set(value.get());
    rooted!(in(cx) let mut dict_value = UndefinedValue());
    unsafe {
        dict.to_jsval(cx, dict_value.handle_mut());
    }
    result.set(dict_value.to_object());
    Ok(())
}

fn key_and_value_return(
    cx: *mut JSContext,
    mut result: MutableHandleObject,
    key: HandleValue,
    value: HandleValue,
) -> Fallible<()> {
    let mut dict = IterableKeyAndValueResult::empty();
    dict.done = false;
    dict.value = Some(
        vec![key, value]
            .into_iter()
            .map(|handle| RootedTraceableBox::from_box(Heap::boxed(handle.get())))
            .collect(),
    );
    rooted!(in(cx) let mut dict_value = UndefinedValue());
    unsafe {
        dict.to_jsval(cx, dict_value.handle_mut());
    }
    result.set(dict_value.to_object());
    Ok(())
}
<commit_msg>Remove some usage of unsafe code in iterator.rs<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

#![allow(unsafe_code)]

//! Implementation of `iterable<...>` and `iterable<..., ...>` WebIDL declarations.

use crate::dom::bindings::codegen::Bindings::IterableIteratorBinding::IterableKeyAndValueResult;
use crate::dom::bindings::codegen::Bindings::IterableIteratorBinding::IterableKeyOrValueResult;
use crate::dom::bindings::error::Fallible;
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector};
use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::bindings::trace::{JSTraceable, RootedTraceableBox};
use crate::dom::globalscope::GlobalScope;
use crate::script_runtime::JSContext;
use dom_struct::dom_struct;
use js::conversions::ToJSValConvertible;
use js::jsapi::{Heap, JSObject};
use js::jsval::UndefinedValue;
use js::rust::{HandleValue, MutableHandleObject};
use std::cell::Cell;
use std::ptr;
use std::ptr::NonNull;

/// The values that an iterator will iterate over.
#[derive(JSTraceable, MallocSizeOf)]
pub enum IteratorType {
    /// The keys of the iterable object.
    Keys,
    /// The values of the iterable object.
    Values,
    /// The keys and values of the iterable object combined.
    Entries,
}

/// A DOM object that can be iterated over using a pair value iterator.
pub trait Iterable {
    /// The type of the key of the iterator pair.
    type Key: ToJSValConvertible;
    /// The type of the value of the iterator pair.
    type Value: ToJSValConvertible;
    /// Return the number of entries that can be iterated over.
    fn get_iterable_length(&self) -> u32;
    /// Return the value at the provided index.
    fn get_value_at_index(&self, index: u32) -> Self::Value;
    /// Return the key at the provided index.
    fn get_key_at_index(&self, index: u32) -> Self::Key;
}

/// An iterator over the iterable entries of a given DOM interface.
//FIXME: #12811 prevents dom_struct with type parameters
#[dom_struct]
pub struct IterableIterator<T: DomObject + JSTraceable + Iterable> {
    reflector: Reflector,
    iterable: Dom<T>,
    type_: IteratorType,
    index: Cell<u32>,
}

impl<T: DomObject + JSTraceable + Iterable> IterableIterator<T> {
    /// Create a new iterator instance for the provided iterable DOM interface.
    pub fn new(
        iterable: &T,
        type_: IteratorType,
        wrap: unsafe fn(JSContext, &GlobalScope, Box<IterableIterator<T>>) -> DomRoot<Self>,
    ) -> DomRoot<Self> {
        let iterator = Box::new(IterableIterator {
            reflector: Reflector::new(),
            type_: type_,
            iterable: Dom::from_ref(iterable),
            index: Cell::new(0),
        });
        reflect_dom_object(iterator, &*iterable.global(), wrap)
    }

    /// Return the next value from the iterable object.
    #[allow(non_snake_case)]
    pub fn Next(&self, cx: JSContext) -> Fallible<NonNull<JSObject>> {
        let index = self.index.get();
        rooted!(in(*cx) let mut value = UndefinedValue());
        rooted!(in(*cx) let mut rval = ptr::null_mut::<JSObject>());
        let result = if index >= self.iterable.get_iterable_length() {
            dict_return(cx, rval.handle_mut(), true, value.handle())
        } else {
            match self.type_ {
                IteratorType::Keys => {
                    unsafe {
                        self.iterable
                            .get_key_at_index(index)
                            .to_jsval(*cx, value.handle_mut());
                    }
                    dict_return(cx, rval.handle_mut(), false, value.handle())
                },
                IteratorType::Values => {
                    unsafe {
                        self.iterable
                            .get_value_at_index(index)
                            .to_jsval(*cx, value.handle_mut());
                    }
                    dict_return(cx, rval.handle_mut(), false, value.handle())
                },
                IteratorType::Entries => {
                    rooted!(in(*cx) let mut key = UndefinedValue());
                    unsafe {
                        self.iterable
                            .get_key_at_index(index)
                            .to_jsval(*cx, key.handle_mut());
                        self.iterable
                            .get_value_at_index(index)
                            .to_jsval(*cx, value.handle_mut());
                    }
                    key_and_value_return(cx, rval.handle_mut(), key.handle(), value.handle())
                },
            }
        };
        self.index.set(index + 1);
        result.map(|_| NonNull::new(rval.get()).expect("got a null pointer"))
    }
}

fn dict_return(
    cx: JSContext,
    mut result: MutableHandleObject,
    done: bool,
    value: HandleValue,
) -> Fallible<()> {
    let mut dict = IterableKeyOrValueResult::empty();
    dict.done = done;
    dict.value.set(value.get());
    rooted!(in(*cx) let mut dict_value = UndefinedValue());
    unsafe {
        dict.to_jsval(*cx, dict_value.handle_mut());
    }
    result.set(dict_value.to_object());
    Ok(())
}

fn key_and_value_return(
    cx: JSContext,
    mut result: MutableHandleObject,
    key: HandleValue,
    value: HandleValue,
) -> Fallible<()> {
    let mut dict = IterableKeyAndValueResult::empty();
    dict.done = false;
    dict.value = Some(
        vec![key, value]
            .into_iter()
            .map(|handle| RootedTraceableBox::from_box(Heap::boxed(handle.get())))
            .collect(),
    );
    rooted!(in(*cx) let mut dict_value = UndefinedValue());
    unsafe {
        dict.to_jsval(*cx, dict_value.handle_mut());
    }
    result.set(dict_value.to_object());
    Ok(())
}
<|endoftext|>


<commit_before>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Tidy check to ensure that unstable features are all in order
//!
//! This check will ensure properties like:
//!
//! * All stability attributes look reasonably well formed
//! * The set of library features is disjoint from the set of language features
//! * Library features have at most one stability level
//! * Library features have at most one `since` value
//! * All unstable lang features have tests to ensure they are actually unstable

use std::collections::HashMap;
use std::fmt;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

#[derive(Debug, PartialEq, Clone)]
pub enum Status {
    Stable,
    Removed,
    Unstable,
}

impl fmt::Display for Status {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let as_str = match *self {
            Status::Stable => "stable",
            Status::Unstable => "unstable",
            Status::Removed => "removed",
        };
        fmt::Display::fmt(as_str, f)
    }
}

#[derive(Debug, Clone)]
pub struct Feature {
    pub level: Status,
    pub since: String,
    pub has_gate_test: bool,
    pub tracking_issue: Option<u32>,
}

pub type Features = HashMap<String, Feature>;

pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
    let mut features = collect_lang_features(path);
    assert!(!features.is_empty());

    let lib_features = get_and_check_lib_features(path, bad, &features);
    assert!(!lib_features.is_empty());

    let mut contents = String::new();

    super::walk_many(&[&path.join("test/compile-fail"),
                       &path.join("test/compile-fail-fulldeps"),
                       &path.join("test/parse-fail"),],
                     &mut |path| super::filter_dirs(path),
                     &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        let filen_underscore = filename.replace("-","_").replace(".rs","");
        let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        for (i, line) in contents.lines().enumerate() {
            let mut err = |msg: &str| {
                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
            };

            let gate_test_str = "gate-test-";

            if !line.contains(gate_test_str) {
                continue;
            }

            let feature_name = match line.find(gate_test_str) {
                Some(i) => {
                    &line[i+gate_test_str.len()..line[i+1..].find(' ').unwrap_or(line.len())]
                },
                None => continue,
            };
            match features.get_mut(feature_name) {
                Some(f) => {
                    if filename_is_gate_test {
                        err(&format!("The file is already marked as gate test \
                                      through its name, no need for a \
                                      'gate-test-{}' comment",
                                     feature_name));
                    }
                    f.has_gate_test = true;
                }
                None => {
                    err(&format!("gate-test test found referencing a nonexistent feature '{}'",
                                 feature_name));
                }
            }
        }
    });

    // Only check the number of lang features.
    // Obligatory testing for library features is dumb.
    let gate_untested = features.iter()
                                .filter(|&(_, f)| f.level == Status::Unstable)
                                .filter(|&(_, f)| !f.has_gate_test)
                                .collect::<Vec<_>>();

    for &(name, _) in gate_untested.iter() {
        println!("Expected a gate test for the feature '{}'.", name);
        println!("Hint: create a file named 'feature-gate-{}.rs' in the compile-fail\
                \n      test suite, with its failures due to missing usage of\
                \n      #![feature({})].", name, name);
        println!("Hint: If you already have such a test and don't want to rename it,\
                \n      you can also add a // gate-test-{} line to the test file.",
                 name);
    }

    if gate_untested.len() > 0 {
        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
    }

    if *bad {
        return;
    }
    if quiet {
        println!("* {} features", features.len());
        return;
    }

    let mut lines = Vec::new();
    for (name, feature) in features.iter() {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lang",
                           feature.level,
                           feature.since));
    }
    for (name, feature) in lib_features {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lib",
                           feature.level,
                           feature.since));
    }

    lines.sort();
    for line in lines {
        println!("* {}", line);
    }
}

fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
    line.find(attr)
        .and_then(|i| line[i..].find('"').map(|j| i + j + 1))
        .and_then(|i| line[i..].find('"').map(|j| (i, i + j)))
        .map(|(i, j)| &line[i..j])
}

fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
    if filen_underscore.starts_with("feature_gate") {
        for (n, f) in features.iter_mut() {
            if filen_underscore == format!("feature_gate_{}", n) {
                f.has_gate_test = true;
                return true;
            }
        }
    }
    return false;
}

pub fn collect_lang_features(base_src_path: &Path) -> Features {
    let mut contents = String::new();
    let path = base_src_path.join("libsyntax/feature_gate.rs");
    t!(t!(File::open(path)).read_to_string(&mut contents));

    contents.lines()
        .filter_map(|line| {
            let mut parts = line.trim().split(",");
            let level = match parts.next().map(|l| l.trim().trim_left_matches('(')) {
                Some("active") => Status::Unstable,
                Some("removed") => Status::Removed,
                Some("accepted") => Status::Stable,
                _ => return None,
            };
            let name = parts.next().unwrap().trim();
            let since = parts.next().unwrap().trim().trim_matches('"');
            let issue_str = parts.next().unwrap().trim();
            let tracking_issue = if issue_str.starts_with("None") {
                None
            } else {
                let s = issue_str.split("(").nth(1).unwrap().split(")").nth(0).unwrap();
                Some(s.parse().unwrap())
            };
            Some((name.to_owned(),
                Feature {
                    level,
                    since: since.to_owned(),
                    has_gate_test: false,
                    tracking_issue,
                }))
        })
        .collect()
}

pub fn collect_lib_features(base_src_path: &Path) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, _, _| {
        match res {
            Ok((name, feature)) => {
                if lib_features.get(name).is_some() {
                    return;
                }
                lib_features.insert(name.to_owned(), feature);
            },
            Err(_) => (),
        }
    });
   lib_features
}

fn get_and_check_lib_features(base_src_path: &Path,
                              bad: &mut bool,
                              lang_features: &Features) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, file, line| {
            match res {
                Ok((name, f)) => {
                    let mut err = |msg: &str| {
                        tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                    };
                    if lang_features.contains_key(name) && name != "proc_macro" {
                        err("duplicating a lang feature");
                    }
                    if let Some(ref s) = lib_features.get(name) {
                        if s.level != f.level {
                            err("different stability level than before");
                        }
                        if s.since != f.since {
                            err("different `since` than before");
                        }
                        if s.tracking_issue != f.tracking_issue {
                            err("different `tracking_issue` than before");
                        }
                    }
                    lib_features.insert(name.to_owned(), f);
                },
                Err(msg) => {
                    tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                },
            }

    });
    lib_features
}

fn map_lib_features(base_src_path: &Path,
                    mf: &mut FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
    let mut contents = String::new();
    super::walk(base_src_path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        let mut becoming_feature: Option<(String, Feature)> = None;
        for (i, line) in contents.lines().enumerate() {
            macro_rules! err {
                ($msg:expr) => {{
                    mf(Err($msg), file, i + 1);
                    continue;
                }};
            };
            if let Some((ref name, ref mut f)) = becoming_feature {
                if f.tracking_issue.is_none() {
                    f.tracking_issue = find_attr_val(line, "issue")
                    .map(|s| s.parse().unwrap());
                }
                if line.ends_with("]") {
                    mf(Ok((name, f.clone())), file, i + 1);
                } else if !line.ends_with(",") && !line.ends_with("\\") {
                    // We need to bail here because we might have missed the
                    // end of a stability attribute above because the "]"
                    // might not have been at the end of the line.
                    // We could then get into the very unfortunate situation that
                    // we continue parsing the file assuming the current stability
                    // attribute has not ended, and ignoring possible feature
                    // attributes in the process.
                    err!("malformed stability attribute");
                } else {
                    continue;
                }
            }
            becoming_feature = None;
            let level = if line.contains("[unstable(") {
                Status::Unstable
            } else if line.contains("[stable(") {
                Status::Stable
            } else {
                continue;
            };
            let feature_name = match find_attr_val(line, "feature") {
                Some(name) => name,
                None => err!("malformed stability attribute"),
            };
            let since = match find_attr_val(line, "since") {
                Some(name) => name,
                None if level == Status::Stable => {
                    err!("malformed stability attribute");
                }
                None => "None",
            };
            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());

            let feature = Feature {
                level,
                since: since.to_owned(),
                has_gate_test: false,
                tracking_issue,
            };
            if line.contains("]") {
                mf(Ok((feature_name, feature)), file, i + 1);
            } else {
                becoming_feature = Some((feature_name.to_owned(), feature));
            }
        }
    });
}
<commit_msg>Tell `tidy` about `compiler_builtins_lib` feature<commit_after>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Tidy check to ensure that unstable features are all in order
//!
//! This check will ensure properties like:
//!
//! * All stability attributes look reasonably well formed
//! * The set of library features is disjoint from the set of language features
//! * Library features have at most one stability level
//! * Library features have at most one `since` value
//! * All unstable lang features have tests to ensure they are actually unstable

use std::collections::HashMap;
use std::fmt;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

#[derive(Debug, PartialEq, Clone)]
pub enum Status {
    Stable,
    Removed,
    Unstable,
}

impl fmt::Display for Status {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let as_str = match *self {
            Status::Stable => "stable",
            Status::Unstable => "unstable",
            Status::Removed => "removed",
        };
        fmt::Display::fmt(as_str, f)
    }
}

#[derive(Debug, Clone)]
pub struct Feature {
    pub level: Status,
    pub since: String,
    pub has_gate_test: bool,
    pub tracking_issue: Option<u32>,
}

pub type Features = HashMap<String, Feature>;

pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
    let mut features = collect_lang_features(path);
    assert!(!features.is_empty());

    let lib_features = get_and_check_lib_features(path, bad, &features);
    assert!(!lib_features.is_empty());

    let mut contents = String::new();

    super::walk_many(&[&path.join("test/compile-fail"),
                       &path.join("test/compile-fail-fulldeps"),
                       &path.join("test/parse-fail"),],
                     &mut |path| super::filter_dirs(path),
                     &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        let filen_underscore = filename.replace("-","_").replace(".rs","");
        let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        for (i, line) in contents.lines().enumerate() {
            let mut err = |msg: &str| {
                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
            };

            let gate_test_str = "gate-test-";

            if !line.contains(gate_test_str) {
                continue;
            }

            let feature_name = match line.find(gate_test_str) {
                Some(i) => {
                    &line[i+gate_test_str.len()..line[i+1..].find(' ').unwrap_or(line.len())]
                },
                None => continue,
            };
            match features.get_mut(feature_name) {
                Some(f) => {
                    if filename_is_gate_test {
                        err(&format!("The file is already marked as gate test \
                                      through its name, no need for a \
                                      'gate-test-{}' comment",
                                     feature_name));
                    }
                    f.has_gate_test = true;
                }
                None => {
                    err(&format!("gate-test test found referencing a nonexistent feature '{}'",
                                 feature_name));
                }
            }
        }
    });

    // Only check the number of lang features.
    // Obligatory testing for library features is dumb.
    let gate_untested = features.iter()
                                .filter(|&(_, f)| f.level == Status::Unstable)
                                .filter(|&(_, f)| !f.has_gate_test)
                                .collect::<Vec<_>>();

    for &(name, _) in gate_untested.iter() {
        println!("Expected a gate test for the feature '{}'.", name);
        println!("Hint: create a file named 'feature-gate-{}.rs' in the compile-fail\
                \n      test suite, with its failures due to missing usage of\
                \n      #![feature({})].", name, name);
        println!("Hint: If you already have such a test and don't want to rename it,\
                \n      you can also add a // gate-test-{} line to the test file.",
                 name);
    }

    if gate_untested.len() > 0 {
        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
    }

    if *bad {
        return;
    }
    if quiet {
        println!("* {} features", features.len());
        return;
    }

    let mut lines = Vec::new();
    for (name, feature) in features.iter() {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lang",
                           feature.level,
                           feature.since));
    }
    for (name, feature) in lib_features {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lib",
                           feature.level,
                           feature.since));
    }

    lines.sort();
    for line in lines {
        println!("* {}", line);
    }
}

fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
    line.find(attr)
        .and_then(|i| line[i..].find('"').map(|j| i + j + 1))
        .and_then(|i| line[i..].find('"').map(|j| (i, i + j)))
        .map(|(i, j)| &line[i..j])
}

fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
    if filen_underscore.starts_with("feature_gate") {
        for (n, f) in features.iter_mut() {
            if filen_underscore == format!("feature_gate_{}", n) {
                f.has_gate_test = true;
                return true;
            }
        }
    }
    return false;
}

pub fn collect_lang_features(base_src_path: &Path) -> Features {
    let mut contents = String::new();
    let path = base_src_path.join("libsyntax/feature_gate.rs");
    t!(t!(File::open(path)).read_to_string(&mut contents));

    contents.lines()
        .filter_map(|line| {
            let mut parts = line.trim().split(",");
            let level = match parts.next().map(|l| l.trim().trim_left_matches('(')) {
                Some("active") => Status::Unstable,
                Some("removed") => Status::Removed,
                Some("accepted") => Status::Stable,
                _ => return None,
            };
            let name = parts.next().unwrap().trim();
            let since = parts.next().unwrap().trim().trim_matches('"');
            let issue_str = parts.next().unwrap().trim();
            let tracking_issue = if issue_str.starts_with("None") {
                None
            } else {
                let s = issue_str.split("(").nth(1).unwrap().split(")").nth(0).unwrap();
                Some(s.parse().unwrap())
            };
            Some((name.to_owned(),
                Feature {
                    level,
                    since: since.to_owned(),
                    has_gate_test: false,
                    tracking_issue,
                }))
        })
        .collect()
}

pub fn collect_lib_features(base_src_path: &Path) -> Features {
    let mut lib_features = Features::new();

    // This library feature is defined in the `compiler_builtins` crate, which
    // has been moved out-of-tree. Now it can no longer be auto-discovered by
    // `tidy`, because we need to filter out its (submodule) directory. Manually
    // add it to the set of known library features so we can still generate docs.
    lib_features.insert("compiler_builtins_lib".to_owned(), Feature {
        level: Status::Unstable,
        since: "".to_owned(),
        has_gate_test: false,
        tracking_issue: None,
    });

    map_lib_features(base_src_path,
                     &mut |res, _, _| {
        match res {
            Ok((name, feature)) => {
                if lib_features.get(name).is_some() {
                    return;
                }
                lib_features.insert(name.to_owned(), feature);
            },
            Err(_) => (),
        }
    });
   lib_features
}

fn get_and_check_lib_features(base_src_path: &Path,
                              bad: &mut bool,
                              lang_features: &Features) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, file, line| {
            match res {
                Ok((name, f)) => {
                    let mut err = |msg: &str| {
                        tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                    };
                    if lang_features.contains_key(name) && name != "proc_macro" {
                        err("duplicating a lang feature");
                    }
                    if let Some(ref s) = lib_features.get(name) {
                        if s.level != f.level {
                            err("different stability level than before");
                        }
                        if s.since != f.since {
                            err("different `since` than before");
                        }
                        if s.tracking_issue != f.tracking_issue {
                            err("different `tracking_issue` than before");
                        }
                    }
                    lib_features.insert(name.to_owned(), f);
                },
                Err(msg) => {
                    tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                },
            }

    });
    lib_features
}

fn map_lib_features(base_src_path: &Path,
                    mf: &mut FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
    let mut contents = String::new();
    super::walk(base_src_path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        let mut becoming_feature: Option<(String, Feature)> = None;
        for (i, line) in contents.lines().enumerate() {
            macro_rules! err {
                ($msg:expr) => {{
                    mf(Err($msg), file, i + 1);
                    continue;
                }};
            };
            if let Some((ref name, ref mut f)) = becoming_feature {
                if f.tracking_issue.is_none() {
                    f.tracking_issue = find_attr_val(line, "issue")
                    .map(|s| s.parse().unwrap());
                }
                if line.ends_with("]") {
                    mf(Ok((name, f.clone())), file, i + 1);
                } else if !line.ends_with(",") && !line.ends_with("\\") {
                    // We need to bail here because we might have missed the
                    // end of a stability attribute above because the "]"
                    // might not have been at the end of the line.
                    // We could then get into the very unfortunate situation that
                    // we continue parsing the file assuming the current stability
                    // attribute has not ended, and ignoring possible feature
                    // attributes in the process.
                    err!("malformed stability attribute");
                } else {
                    continue;
                }
            }
            becoming_feature = None;
            let level = if line.contains("[unstable(") {
                Status::Unstable
            } else if line.contains("[stable(") {
                Status::Stable
            } else {
                continue;
            };
            let feature_name = match find_attr_val(line, "feature") {
                Some(name) => name,
                None => err!("malformed stability attribute"),
            };
            let since = match find_attr_val(line, "since") {
                Some(name) => name,
                None if level == Status::Stable => {
                    err!("malformed stability attribute");
                }
                None => "None",
            };
            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());

            let feature = Feature {
                level,
                since: since.to_owned(),
                has_gate_test: false,
                tracking_issue,
            };
            if line.contains("]") {
                mf(Ok((feature_name, feature)), file, i + 1);
            } else {
                becoming_feature = Some((feature_name.to_owned(), feature));
            }
        }
    });
}
<|endoftext|>


<commit_before>//! Tidy check to ensure that unstable features are all in order.
//!
//! This check will ensure properties like:
//!
//! * All stability attributes look reasonably well formed.
//! * The set of library features is disjoint from the set of language features.
//! * Library features have at most one stability level.
//! * Library features have at most one `since` value.
//! * All unstable lang features have tests to ensure they are actually unstable.
//! * Language features in a group are sorted by `since` value.

use std::collections::HashMap;
use std::fmt;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::Path;

use regex::{Regex, escape};

mod version;
use self::version::Version;

const FEATURE_GROUP_START_PREFIX: &str = "// feature group start:";
const FEATURE_GROUP_END_PREFIX: &str = "// feature group end";

#[derive(Debug, PartialEq, Clone)]
pub enum Status {
    Stable,
    Removed,
    Unstable,
}

impl fmt::Display for Status {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let as_str = match *self {
            Status::Stable => "stable",
            Status::Unstable => "unstable",
            Status::Removed => "removed",
        };
        fmt::Display::fmt(as_str, f)
    }
}

#[derive(Debug, Clone)]
pub struct Feature {
    pub level: Status,
    pub since: Option<Version>,
    pub has_gate_test: bool,
    pub tracking_issue: Option<u32>,
    pub group: Option<String>,
}

pub type Features = HashMap<String, Feature>;

pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
    let mut features = collect_lang_features(path, bad);
    assert!(!features.is_empty());

    let lib_features = get_and_check_lib_features(path, bad, &features);
    assert!(!lib_features.is_empty());

    let mut contents = String::new();

    super::walk_many(&[&path.join("test/ui"),
                       &path.join("test/ui-fulldeps"),
                       &path.join("test/compile-fail")],
                     &mut |path| super::filter_dirs(path),
                     &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        let filen_underscore = filename.replace('-',"_").replace(".rs","");
        let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        for (i, line) in contents.lines().enumerate() {
            let mut err = |msg: &str| {
                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
            };

            let gate_test_str = "gate-test-";

            let feature_name = match line.find(gate_test_str) {
                Some(i) => {
                    line[i+gate_test_str.len()..].splitn(2, ' ').next().unwrap()
                },
                None => continue,
            };
            match features.get_mut(feature_name) {
                Some(f) => {
                    if filename_is_gate_test {
                        err(&format!("The file is already marked as gate test \
                                      through its name, no need for a \
                                      'gate-test-{}' comment",
                                     feature_name));
                    }
                    f.has_gate_test = true;
                }
                None => {
                    err(&format!("gate-test test found referencing a nonexistent feature '{}'",
                                 feature_name));
                }
            }
        }
    });

    // Only check the number of lang features.
    // Obligatory testing for library features is dumb.
    let gate_untested = features.iter()
                                .filter(|&(_, f)| f.level == Status::Unstable)
                                .filter(|&(_, f)| !f.has_gate_test)
                                .collect::<Vec<_>>();

    for &(name, _) in gate_untested.iter() {
        println!("Expected a gate test for the feature '{}'.", name);
        println!("Hint: create a failing test file named 'feature-gate-{}.rs'\
                \n      in the 'ui' test suite, with its failures due to\
                \n      missing usage of #![feature({})].", name, name);
        println!("Hint: If you already have such a test and don't want to rename it,\
                \n      you can also add a // gate-test-{} line to the test file.",
                 name);
    }

    if !gate_untested.is_empty() {
        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
    }

    if *bad {
        return;
    }
    if quiet {
        println!("* {} features", features.len());
        return;
    }

    let mut lines = Vec::new();
    for (name, feature) in features.iter() {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lang",
                           feature.level,
                           feature.since.as_ref().map_or("None".to_owned(),
                                                         |since| since.to_string())));
    }
    for (name, feature) in lib_features {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lib",
                           feature.level,
                           feature.since.as_ref().map_or("None".to_owned(),
                                                         |since| since.to_string())));
    }

    lines.sort();
    for line in lines {
        println!("* {}", line);
    }
}

fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
    let r = Regex::new(&format!(r#"{} *= *"([^"]*)""#, escape(attr)))
        .expect("malformed regex for find_attr_val");
    r.captures(line)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
}

#[test]
fn test_find_attr_val() {
    let s = r#"#[unstable(feature = "checked_duration_since", issue = "58402")]"#;
    assert_eq!(find_attr_val(s, "feature"), Some("checked_duration_since"));
    assert_eq!(find_attr_val(s, "issue"), Some("58402"));
    assert_eq!(find_attr_val(s, "since"), None);
}

fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
    if filen_underscore.starts_with("feature_gate") {
        for (n, f) in features.iter_mut() {
            if filen_underscore == format!("feature_gate_{}", n) {
                f.has_gate_test = true;
                return true;
            }
        }
    }
    return false;
}

pub fn collect_lang_features(base_src_path: &Path, bad: &mut bool) -> Features {
    let contents = t!(fs::read_to_string(base_src_path.join("libsyntax/feature_gate.rs")));

    // We allow rustc-internal features to omit a tracking issue.
    // To make tidy accept omitting a tracking issue, group the list of features
    // without one inside `// no tracking issue START` and `// no tracking issue END`.
    let mut next_feature_omits_tracking_issue = false;

    let mut next_feature_group = None;
    let mut prev_since = None;

    contents.lines().zip(1..)
        .filter_map(|(line, line_number)| {
            let line = line.trim();

            // Within START and END, the tracking issue can be omitted.
            match line {
                "// no tracking issue START" => {
                    next_feature_omits_tracking_issue = true;
                    return None;
                }
                "// no tracking issue END" => {
                    next_feature_omits_tracking_issue = false;
                    return None;
                }
                _ => {}
            }

            if line.starts_with(FEATURE_GROUP_START_PREFIX) {
                let group = line.trim_start_matches(FEATURE_GROUP_START_PREFIX).trim();
                next_feature_group = Some(group.to_owned());
                prev_since = None;
                return None;
            } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {
                next_feature_group = None;
                prev_since = None;
                return None;
            }

            let mut parts = line.split(',');
            let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
                Some("active") => Status::Unstable,
                Some("removed") => Status::Removed,
                Some("accepted") => Status::Stable,
                _ => return None,
            };
            let name = parts.next().unwrap().trim();

            let since_str = parts.next().unwrap().trim().trim_matches('"');
            let since = match since_str.parse() {
                Ok(since) => Some(since),
                Err(err) => {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: failed to parse since: {} ({})",
                        line_number,
                        since_str,
                        err,
                    );
                    None
                }
            };
            if next_feature_group.is_some() {
                if prev_since > since {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: feature {} is not sorted by since",
                        line_number,
                        name,
                    );
                }
                prev_since = since.clone();
            }

            let issue_str = parts.next().unwrap().trim();
            let tracking_issue = if issue_str.starts_with("None") {
                if level == Status::Unstable && !next_feature_omits_tracking_issue {
                    *bad = true;
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: no tracking issue for feature {}",
                        line_number,
                        name,
                    );
                }
                None
            } else {
                let s = issue_str.split('(').nth(1).unwrap().split(')').nth(0).unwrap();
                Some(s.parse().unwrap())
            };
            Some((name.to_owned(),
                Feature {
                    level,
                    since,
                    has_gate_test: false,
                    tracking_issue,
                    group: next_feature_group.clone(),
                }))
        })
        .collect()
}

pub fn collect_lib_features(base_src_path: &Path) -> Features {
    let mut lib_features = Features::new();

    // This library feature is defined in the `compiler_builtins` crate, which
    // has been moved out-of-tree. Now it can no longer be auto-discovered by
    // `tidy`, because we need to filter out its (submodule) directory. Manually
    // add it to the set of known library features so we can still generate docs.
    lib_features.insert("compiler_builtins_lib".to_owned(), Feature {
        level: Status::Unstable,
        since: None,
        has_gate_test: false,
        tracking_issue: None,
        group: None,
    });

    map_lib_features(base_src_path,
                     &mut |res, _, _| {
        if let Ok((name, feature)) = res {
            if lib_features.contains_key(name) {
                return;
            }
            lib_features.insert(name.to_owned(), feature);
        }
    });
   lib_features
}

fn get_and_check_lib_features(base_src_path: &Path,
                              bad: &mut bool,
                              lang_features: &Features) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, file, line| {
            match res {
                Ok((name, f)) => {
                    let mut check_features = |f: &Feature, list: &Features, display: &str| {
                        if let Some(ref s) = list.get(name) {
                            if f.tracking_issue != s.tracking_issue {
                                tidy_error!(bad,
                                            "{}:{}: mismatches the `issue` in {}",
                                            file.display(),
                                            line,
                                            display);
                            }
                        }
                    };
                    check_features(&f, &lang_features, "corresponding lang feature");
                    check_features(&f, &lib_features, "previous");
                    lib_features.insert(name.to_owned(), f);
                },
                Err(msg) => {
                    tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                },
            }

    });
    lib_features
}

fn map_lib_features(base_src_path: &Path,
                    mf: &mut dyn FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
    let mut contents = String::new();
    super::walk(base_src_path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        let mut becoming_feature: Option<(String, Feature)> = None;
        for (i, line) in contents.lines().enumerate() {
            macro_rules! err {
                ($msg:expr) => {{
                    mf(Err($msg), file, i + 1);
                    continue;
                }};
            };
            if let Some((ref name, ref mut f)) = becoming_feature {
                if f.tracking_issue.is_none() {
                    f.tracking_issue = find_attr_val(line, "issue")
                    .map(|s| s.parse().unwrap());
                }
                if line.ends_with(']') {
                    mf(Ok((name, f.clone())), file, i + 1);
                } else if !line.ends_with(',') && !line.ends_with('\\') {
                    // We need to bail here because we might have missed the
                    // end of a stability attribute above because the ']'
                    // might not have been at the end of the line.
                    // We could then get into the very unfortunate situation that
                    // we continue parsing the file assuming the current stability
                    // attribute has not ended, and ignoring possible feature
                    // attributes in the process.
                    err!("malformed stability attribute");
                } else {
                    continue;
                }
            }
            becoming_feature = None;
            if line.contains("rustc_const_unstable(") {
                // `const fn` features are handled specially.
                let feature_name = match find_attr_val(line, "feature") {
                    Some(name) => name,
                    None => err!("malformed stability attribute"),
                };
                let feature = Feature {
                    level: Status::Unstable,
                    since: None,
                    has_gate_test: false,
                    // FIXME(#57563): #57563 is now used as a common tracking issue,
                    // although we would like to have specific tracking issues for each
                    // `rustc_const_unstable` in the future.
                    tracking_issue: Some(57563),
                    group: None,
                };
                mf(Ok((feature_name, feature)), file, i + 1);
                continue;
            }
            let level = if line.contains("[unstable(") {
                Status::Unstable
            } else if line.contains("[stable(") {
                Status::Stable
            } else {
                continue;
            };
            let feature_name = match find_attr_val(line, "feature") {
                Some(name) => name,
                None => err!("malformed stability attribute"),
            };
            let since = match find_attr_val(line, "since").map(|x| x.parse()) {
                Some(Ok(since)) => Some(since),
                Some(Err(_err)) => {
                    err!("malformed since attribute");
                },
                None if level == Status::Stable => {
                    err!("malformed stability attribute");
                }
                None => None,
            };
            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());

            let feature = Feature {
                level,
                since,
                has_gate_test: false,
                tracking_issue,
                group: None,
            };
            if line.contains(']') {
                mf(Ok((feature_name, feature)), file, i + 1);
            } else {
                becoming_feature = Some((feature_name.to_owned(), feature));
            }
        }
    });
}
<commit_msg>Address review comments<commit_after>//! Tidy check to ensure that unstable features are all in order.
//!
//! This check will ensure properties like:
//!
//! * All stability attributes look reasonably well formed.
//! * The set of library features is disjoint from the set of language features.
//! * Library features have at most one stability level.
//! * Library features have at most one `since` value.
//! * All unstable lang features have tests to ensure they are actually unstable.
//! * Language features in a group are sorted by `since` value.

use std::collections::HashMap;
use std::fmt;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::Path;

use regex::{Regex, escape};

mod version;
use self::version::Version;

const FEATURE_GROUP_START_PREFIX: &str = "// feature-group-start:";
const FEATURE_GROUP_END_PREFIX: &str = "// feature-group-end";

#[derive(Debug, PartialEq, Clone)]
pub enum Status {
    Stable,
    Removed,
    Unstable,
}

impl fmt::Display for Status {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let as_str = match *self {
            Status::Stable => "stable",
            Status::Unstable => "unstable",
            Status::Removed => "removed",
        };
        fmt::Display::fmt(as_str, f)
    }
}

#[derive(Debug, Clone)]
pub struct Feature {
    pub level: Status,
    pub since: Option<Version>,
    pub has_gate_test: bool,
    pub tracking_issue: Option<u32>,
}

pub type Features = HashMap<String, Feature>;

pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
    let mut features = collect_lang_features(path, bad);
    assert!(!features.is_empty());

    let lib_features = get_and_check_lib_features(path, bad, &features);
    assert!(!lib_features.is_empty());

    let mut contents = String::new();

    super::walk_many(&[&path.join("test/ui"),
                       &path.join("test/ui-fulldeps"),
                       &path.join("test/compile-fail")],
                     &mut |path| super::filter_dirs(path),
                     &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        let filen_underscore = filename.replace('-',"_").replace(".rs","");
        let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        for (i, line) in contents.lines().enumerate() {
            let mut err = |msg: &str| {
                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
            };

            let gate_test_str = "gate-test-";

            let feature_name = match line.find(gate_test_str) {
                Some(i) => {
                    line[i+gate_test_str.len()..].splitn(2, ' ').next().unwrap()
                },
                None => continue,
            };
            match features.get_mut(feature_name) {
                Some(f) => {
                    if filename_is_gate_test {
                        err(&format!("The file is already marked as gate test \
                                      through its name, no need for a \
                                      'gate-test-{}' comment",
                                     feature_name));
                    }
                    f.has_gate_test = true;
                }
                None => {
                    err(&format!("gate-test test found referencing a nonexistent feature '{}'",
                                 feature_name));
                }
            }
        }
    });

    // Only check the number of lang features.
    // Obligatory testing for library features is dumb.
    let gate_untested = features.iter()
                                .filter(|&(_, f)| f.level == Status::Unstable)
                                .filter(|&(_, f)| !f.has_gate_test)
                                .collect::<Vec<_>>();

    for &(name, _) in gate_untested.iter() {
        println!("Expected a gate test for the feature '{}'.", name);
        println!("Hint: create a failing test file named 'feature-gate-{}.rs'\
                \n      in the 'ui' test suite, with its failures due to\
                \n      missing usage of #![feature({})].", name, name);
        println!("Hint: If you already have such a test and don't want to rename it,\
                \n      you can also add a // gate-test-{} line to the test file.",
                 name);
    }

    if !gate_untested.is_empty() {
        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
    }

    if *bad {
        return;
    }
    if quiet {
        println!("* {} features", features.len());
        return;
    }

    let mut lines = Vec::new();
    lines.extend(format_features(&features, "lang"));
    lines.extend(format_features(&lib_features, "lib"));

    lines.sort();
    for line in lines {
        println!("* {}", line);
    }
}

fn format_features<'a>(features: &'a Features, family: &'a str) -> impl Iterator<Item=String> + 'a {
    features.iter().map(move |(name, feature)| {
        format!("{:<32} {:<8} {:<12} {:<8}",
                name,
                family,
                feature.level,
                feature.since.as_ref().map_or("None".to_owned(),
                                              |since| since.to_string()))
    })
}

fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
    let r = Regex::new(&format!(r#"{}\s*=\s*"([^"]*)""#, escape(attr)))
        .expect("malformed regex for find_attr_val");
    r.captures(line)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
}

#[test]
fn test_find_attr_val() {
    let s = r#"#[unstable(feature = "checked_duration_since", issue = "58402")]"#;
    assert_eq!(find_attr_val(s, "feature"), Some("checked_duration_since"));
    assert_eq!(find_attr_val(s, "issue"), Some("58402"));
    assert_eq!(find_attr_val(s, "since"), None);
}

fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
    if filen_underscore.starts_with("feature_gate") {
        for (n, f) in features.iter_mut() {
            if filen_underscore == format!("feature_gate_{}", n) {
                f.has_gate_test = true;
                return true;
            }
        }
    }
    return false;
}

pub fn collect_lang_features(base_src_path: &Path, bad: &mut bool) -> Features {
    let contents = t!(fs::read_to_string(base_src_path.join("libsyntax/feature_gate.rs")));

    // We allow rustc-internal features to omit a tracking issue.
    // To make tidy accept omitting a tracking issue, group the list of features
    // without one inside `// no tracking issue START` and `// no tracking issue END`.
    let mut next_feature_omits_tracking_issue = false;

    let mut next_feature_group = None;
    let mut prev_since = None;

    contents.lines().zip(1..)
        .filter_map(|(line, line_number)| {
            let line = line.trim();

            // Within START and END, the tracking issue can be omitted.
            match line {
                "// no tracking issue START" => {
                    next_feature_omits_tracking_issue = true;
                    return None;
                }
                "// no tracking issue END" => {
                    next_feature_omits_tracking_issue = false;
                    return None;
                }
                _ => {}
            }

            if line.starts_with(FEATURE_GROUP_START_PREFIX) {
                if next_feature_group.is_some() {
                    tidy_error!(
                        bad,
                        // ignore-tidy-linelength
                        "libsyntax/feature_gate.rs:{}: new feature group is started without ending the previous one",
                        line_number,
                    );
                }

                let group = line.trim_start_matches(FEATURE_GROUP_START_PREFIX).trim();
                next_feature_group = Some(group.to_owned());
                prev_since = None;
                return None;
            } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {
                next_feature_group = None;
                prev_since = None;
                return None;
            }

            let mut parts = line.split(',');
            let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
                Some("active") => Status::Unstable,
                Some("removed") => Status::Removed,
                Some("accepted") => Status::Stable,
                _ => return None,
            };
            let name = parts.next().unwrap().trim();

            let since_str = parts.next().unwrap().trim().trim_matches('"');
            let since = match since_str.parse() {
                Ok(since) => Some(since),
                Err(err) => {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: failed to parse since: {} ({})",
                        line_number,
                        since_str,
                        err,
                    );
                    None
                }
            };
            if next_feature_group.is_some() {
                if prev_since > since {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: feature {} is not sorted by since",
                        line_number,
                        name,
                    );
                }
                prev_since = since.clone();
            }

            let issue_str = parts.next().unwrap().trim();
            let tracking_issue = if issue_str.starts_with("None") {
                if level == Status::Unstable && !next_feature_omits_tracking_issue {
                    *bad = true;
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: no tracking issue for feature {}",
                        line_number,
                        name,
                    );
                }
                None
            } else {
                let s = issue_str.split('(').nth(1).unwrap().split(')').nth(0).unwrap();
                Some(s.parse().unwrap())
            };
            Some((name.to_owned(),
                Feature {
                    level,
                    since,
                    has_gate_test: false,
                    tracking_issue,
                }))
        })
        .collect()
}

pub fn collect_lib_features(base_src_path: &Path) -> Features {
    let mut lib_features = Features::new();

    // This library feature is defined in the `compiler_builtins` crate, which
    // has been moved out-of-tree. Now it can no longer be auto-discovered by
    // `tidy`, because we need to filter out its (submodule) directory. Manually
    // add it to the set of known library features so we can still generate docs.
    lib_features.insert("compiler_builtins_lib".to_owned(), Feature {
        level: Status::Unstable,
        since: None,
        has_gate_test: false,
        tracking_issue: None,
    });

    map_lib_features(base_src_path,
                     &mut |res, _, _| {
        if let Ok((name, feature)) = res {
            if lib_features.contains_key(name) {
                return;
            }
            lib_features.insert(name.to_owned(), feature);
        }
    });
   lib_features
}

fn get_and_check_lib_features(base_src_path: &Path,
                              bad: &mut bool,
                              lang_features: &Features) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, file, line| {
            match res {
                Ok((name, f)) => {
                    let mut check_features = |f: &Feature, list: &Features, display: &str| {
                        if let Some(ref s) = list.get(name) {
                            if f.tracking_issue != s.tracking_issue {
                                tidy_error!(bad,
                                            "{}:{}: mismatches the `issue` in {}",
                                            file.display(),
                                            line,
                                            display);
                            }
                        }
                    };
                    check_features(&f, &lang_features, "corresponding lang feature");
                    check_features(&f, &lib_features, "previous");
                    lib_features.insert(name.to_owned(), f);
                },
                Err(msg) => {
                    tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                },
            }

    });
    lib_features
}

fn map_lib_features(base_src_path: &Path,
                    mf: &mut dyn FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
    let mut contents = String::new();
    super::walk(base_src_path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        let mut becoming_feature: Option<(String, Feature)> = None;
        for (i, line) in contents.lines().enumerate() {
            macro_rules! err {
                ($msg:expr) => {{
                    mf(Err($msg), file, i + 1);
                    continue;
                }};
            };
            if let Some((ref name, ref mut f)) = becoming_feature {
                if f.tracking_issue.is_none() {
                    f.tracking_issue = find_attr_val(line, "issue")
                    .map(|s| s.parse().unwrap());
                }
                if line.ends_with(']') {
                    mf(Ok((name, f.clone())), file, i + 1);
                } else if !line.ends_with(',') && !line.ends_with('\\') {
                    // We need to bail here because we might have missed the
                    // end of a stability attribute above because the ']'
                    // might not have been at the end of the line.
                    // We could then get into the very unfortunate situation that
                    // we continue parsing the file assuming the current stability
                    // attribute has not ended, and ignoring possible feature
                    // attributes in the process.
                    err!("malformed stability attribute");
                } else {
                    continue;
                }
            }
            becoming_feature = None;
            if line.contains("rustc_const_unstable(") {
                // `const fn` features are handled specially.
                let feature_name = match find_attr_val(line, "feature") {
                    Some(name) => name,
                    None => err!("malformed stability attribute: missing `feature` key"),
                };
                let feature = Feature {
                    level: Status::Unstable,
                    since: None,
                    has_gate_test: false,
                    // FIXME(#57563): #57563 is now used as a common tracking issue,
                    // although we would like to have specific tracking issues for each
                    // `rustc_const_unstable` in the future.
                    tracking_issue: Some(57563),
                };
                mf(Ok((feature_name, feature)), file, i + 1);
                continue;
            }
            let level = if line.contains("[unstable(") {
                Status::Unstable
            } else if line.contains("[stable(") {
                Status::Stable
            } else {
                continue;
            };
            let feature_name = match find_attr_val(line, "feature") {
                Some(name) => name,
                None => err!("malformed stability attribute: missing `feature` key"),
            };
            let since = match find_attr_val(line, "since").map(|x| x.parse()) {
                Some(Ok(since)) => Some(since),
                Some(Err(_err)) => {
                    err!("malformed stability attribute: can't parse `since` key");
                },
                None if level == Status::Stable => {
                    err!("malformed stability attribute: missing the `since` key");
                }
                None => None,
            };
            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());

            let feature = Feature {
                level,
                since,
                has_gate_test: false,
                tracking_issue,
            };
            if line.contains(']') {
                mf(Ok((feature_name, feature)), file, i + 1);
            } else {
                becoming_feature = Some((feature_name.to_owned(), feature));
            }
        }
    });
}
<|endoftext|>