<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Counting and non-counting Bloom filters tuned for use as ancestor filters
//! for selector matching.

use fnv::FnvHasher;
use std::fmt::{self, Debug};
use std::hash::{Hash, Hasher};

// The top 8 bits of the 32-bit hash value are not used by the bloom filter.
// Consumers may rely on this to pack hashes more efficiently.
pub const BLOOM_HASH_MASK: u32 = 0x00ffffff;
const KEY_SIZE: usize = 12;

const ARRAY_SIZE: usize = 1 << KEY_SIZE;
const KEY_MASK: u32 = (1 << KEY_SIZE) - 1;

/// A counting Bloom filter with 8-bit counters.
pub type BloomFilter = CountingBloomFilter<BloomStorageU8>;

/// A non-counting Bloom filter.
///
/// Effectively a counting Bloom filter with 1-bit counters.
pub type NonCountingBloomFilter = CountingBloomFilter<BloomStorageBool>;

/// A counting Bloom filter with parameterized storage to handle
/// counters of different sizes.  For now we assume that having two hash
/// functions is enough, but we may revisit that decision later.
///
/// The filter uses an array with 2**KeySize entries.
///
/// Assuming a well-distributed hash function, a Bloom filter with
/// array size M containing N elements and
/// using k hash function has expected false positive rate exactly
///
/// $  (1 - (1 - 1/M)^{kN})^k  $
///
/// because each array slot has a
///
/// $  (1 - 1/M)^{kN}  $
///
/// chance of being 0, and the expected false positive rate is the
/// probability that all of the k hash functions will hit a nonzero
/// slot.
///
/// For reasonable assumptions (M large, kN large, which should both
/// hold if we're worried about false positives) about M and kN this
/// becomes approximately
///
/// $$  (1 - \exp(-kN/M))^k   $$
///
/// For our special case of k == 2, that's $(1 - \exp(-2N/M))^2$,
/// or in other words
///
/// $$    N/M = -0.5 * \ln(1 - \sqrt(r))   $$
///
/// where r is the false positive rate.  This can be used to compute
/// the desired KeySize for a given load N and false positive rate r.
///
/// If N/M is assumed small, then the false positive rate can
/// further be approximated as 4*N^2/M^2.  So increasing KeySize by
/// 1, which doubles M, reduces the false positive rate by about a
/// factor of 4, and a false positive rate of 1% corresponds to
/// about M/N == 20.
///
/// What this means in practice is that for a few hundred keys using a
/// KeySize of 12 gives false positive rates on the order of 0.25-4%.
///
/// Similarly, using a KeySize of 10 would lead to a 4% false
/// positive rate for N == 100 and to quite bad false positive
/// rates for larger N.
#[derive(Clone)]
pub struct CountingBloomFilter<S>
where
    S: BloomStorage,
{
    storage: S,
}

impl<S> CountingBloomFilter<S>
where
    S: BloomStorage,
{
    /// Creates a new bloom filter.
    #[inline]
    pub fn new() -> Self {
        CountingBloomFilter {
            storage: Default::default(),
        }
    }

    #[inline]
    pub fn clear(&mut self) {
        self.storage = Default::default();
    }

    // Slow linear accessor to make sure the bloom filter is zeroed. This should
    // never be used in release builds.
    #[cfg(debug_assertions)]
    pub fn is_zeroed(&self) -> bool {
        self.storage.is_zeroed()
    }

    #[cfg(not(debug_assertions))]
    pub fn is_zeroed(&self) -> bool {
        unreachable!()
    }

    #[inline]
    pub fn insert_hash(&mut self, hash: u32) {
        self.storage.adjust_first_slot(hash, true);
        self.storage.adjust_second_slot(hash, true);
    }

    /// Inserts an item into the bloom filter.
    #[inline]
    pub fn insert<T: Hash>(&mut self, elem: &T) {
        self.insert_hash(hash(elem))
    }

    #[inline]
    pub fn remove_hash(&mut self, hash: u32) {
        self.storage.adjust_first_slot(hash, false);
        self.storage.adjust_second_slot(hash, false);
    }

    /// Removes an item from the bloom filter.
    #[inline]
    pub fn remove<T: Hash>(&mut self, elem: &T) {
        self.remove_hash(hash(elem))
    }

    #[inline]
    pub fn might_contain_hash(&self, hash: u32) -> bool {
        !self.storage.first_slot_is_empty(hash) && !self.storage.second_slot_is_empty(hash)
    }

    /// Check whether the filter might contain an item.  This can
    /// sometimes return true even if the item is not in the filter,
    /// but will never return false for items that are actually in the
    /// filter.
    #[inline]
    pub fn might_contain<T: Hash>(&self, elem: &T) -> bool {
        self.might_contain_hash(hash(elem))
    }
}

impl<S> Debug for CountingBloomFilter<S>
where
    S: BloomStorage,
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let mut slots_used = 0;
        for i in 0..ARRAY_SIZE {
            if !self.storage.slot_is_empty(i) {
                slots_used += 1;
            }
        }
        write!(f, "BloomFilter({}/{})", slots_used, ARRAY_SIZE)
    }
}

pub trait BloomStorage: Clone + Default {
    fn slot_is_empty(&self, index: usize) -> bool;
    fn adjust_slot(&mut self, index: usize, increment: bool);
    fn is_zeroed(&self) -> bool;

    #[inline]
    fn first_slot_is_empty(&self, hash: u32) -> bool {
        self.slot_is_empty(Self::first_slot_index(hash))
    }

    #[inline]
    fn second_slot_is_empty(&self, hash: u32) -> bool {
        self.slot_is_empty(Self::second_slot_index(hash))
    }

    #[inline]
    fn adjust_first_slot(&mut self, hash: u32, increment: bool) {
        self.adjust_slot(Self::first_slot_index(hash), increment)
    }

    #[inline]
    fn adjust_second_slot(&mut self, hash: u32, increment: bool) {
        self.adjust_slot(Self::second_slot_index(hash), increment)
    }

    #[inline]
    fn first_slot_index(hash: u32) -> usize {
        hash1(hash) as usize
    }

    #[inline]
    fn second_slot_index(hash: u32) -> usize {
        hash2(hash) as usize
    }
}

/// Storage class for a CountingBloomFilter that has 8-bit counters.
pub struct BloomStorageU8 {
    counters: [u8; ARRAY_SIZE],
}

impl BloomStorage for BloomStorageU8 {
    #[inline]
    fn adjust_slot(&mut self, index: usize, increment: bool) {
        let slot = &mut self.counters[index];
        if *slot != 0xff {
            // full
            if increment {
                *slot += 1;
            } else {
                *slot -= 1;
            }
        }
    }

    #[inline]
    fn slot_is_empty(&self, index: usize) -> bool {
        self.counters[index] == 0
    }

    #[inline]
    fn is_zeroed(&self) -> bool {
        self.counters.iter().all(|x| *x == 0)
    }
}

impl Default for BloomStorageU8 {
    fn default() -> Self {
        BloomStorageU8 {
            counters: [0; ARRAY_SIZE],
        }
    }
}

impl Clone for BloomStorageU8 {
    fn clone(&self) -> Self {
        BloomStorageU8 {
            counters: self.counters,
        }
    }
}

/// Storage class for a CountingBloomFilter that has 1-bit counters.
pub struct BloomStorageBool {
    counters: [u8; ARRAY_SIZE / 8],
}

impl BloomStorage for BloomStorageBool {
    #[inline]
    fn adjust_slot(&mut self, index: usize, increment: bool) {
        let bit = 1 << (index % 8);
        let byte = &mut self.counters[index / 8];

        // Since we have only one bit for storage, decrementing it
        // should never do anything.  Assert against an accidental
        // decrementing of a bit that was never set.
        assert!(
            increment || (*byte & bit) != 0,
            "should not decrement if slot is already false"
        );

        if increment {
            *byte |= bit;
        }
    }

    #[inline]
    fn slot_is_empty(&self, index: usize) -> bool {
        let bit = 1 << (index % 8);
        (self.counters[index / 8] & bit) == 0
    }

    #[inline]
    fn is_zeroed(&self) -> bool {
        self.counters.iter().all(|x| *x == 0)
    }
}

impl Default for BloomStorageBool {
    fn default() -> Self {
        BloomStorageBool {
            counters: [0; ARRAY_SIZE / 8],
        }
    }
}

impl Clone for BloomStorageBool {
    fn clone(&self) -> Self {
        BloomStorageBool {
            counters: self.counters,
        }
    }
}

fn hash<T: Hash>(elem: &T) -> u32 {
    let mut hasher = FnvHasher::default();
    elem.hash(&mut hasher);
    let hash: u64 = hasher.finish();
    (hash >> 32) as u32 ^ (hash as u32)
}

#[inline]
fn hash1(hash: u32) -> u32 {
    hash & KEY_MASK
}

#[inline]
fn hash2(hash: u32) -> u32 {
    (hash >> KEY_SIZE) & KEY_MASK
}

#[test]
fn create_and_insert_some_stuff() {
    use std::mem::transmute;

    let mut bf = BloomFilter::new();

    // Statically assert that ARRAY_SIZE is a multiple of 8, which
    // BloomStorageBool relies on.
    unsafe {
        transmute::<[u8; ARRAY_SIZE % 8], [u8; 0]>([]);
    }

    for i in 0_usize..1000 {
        bf.insert(&i);
    }

    for i in 0_usize..1000 {
        assert!(bf.might_contain(&i));
    }

    let false_positives = (1001_usize..2000).filter(|i| bf.might_contain(i)).count();

    assert!(false_positives < 150, "{} is not < 150", false_positives); // 15%.

    for i in 0_usize..100 {
        bf.remove(&i);
    }

    for i in 100_usize..1000 {
        assert!(bf.might_contain(&i));
    }

    let false_positives = (0_usize..100).filter(|i| bf.might_contain(i)).count();

    assert!(false_positives < 20, "{} is not < 20", false_positives); // 20%.

    bf.clear();

    for i in 0_usize..2000 {
        assert!(!bf.might_contain(&i));
    }
}

#[cfg(feature = "bench")]
#[cfg(test)]
mod bench {
    extern crate test;
    use super::BloomFilter;

    #[derive(Default)]
    struct HashGenerator(u32);

    impl HashGenerator {
        fn next(&mut self) -> u32 {
            // Each hash is split into two twelve-bit segments, which are used
            // as an index into an array. We increment each by 64 so that we
            // hit the next cache line, and then another 1 so that our wrapping
            // behavior leads us to different entries each time.
            //
            // Trying to simulate cold caches is rather difficult with the cargo
            // benchmarking setup, so it may all be moot depending on the number
            // of iterations that end up being run. But we might as well.
            self.0 += (65) + (65 << super::KEY_SIZE);
            self.0
        }
    }

    #[bench]
    fn create_insert_1000_remove_100_lookup_100(b: &mut test::Bencher) {
        b.iter(|| {
            let mut gen1 = HashGenerator::default();
            let mut gen2 = HashGenerator::default();
            let mut bf = BloomFilter::new();
            for _ in 0_usize..1000 {
                bf.insert_hash(gen1.next());
            }
            for _ in 0_usize..100 {
                bf.remove_hash(gen2.next());
            }
            for _ in 100_usize..200 {
                test::black_box(bf.might_contain_hash(gen2.next()));
            }
        });
    }

    #[bench]
    fn might_contain_10(b: &mut test::Bencher) {
        let bf = BloomFilter::new();
        let mut gen = HashGenerator::default();
        b.iter(|| {
            for _ in 0..10 {
                test::black_box(bf.might_contain_hash(gen.next()));
            }
        });
    }

    #[bench]
    fn clear(b: &mut test::Bencher) {
        let mut bf = Box::new(BloomFilter::new());
        b.iter(|| test::black_box(&mut bf).clear());
    }

    #[bench]
    fn insert_10(b: &mut test::Bencher) {
        let mut bf = BloomFilter::new();
        let mut gen = HashGenerator::default();
        b.iter(|| {
            for _ in 0..10 {
                test::black_box(bf.insert_hash(gen.next()));
            }
        });
    }

    #[bench]
    fn remove_10(b: &mut test::Bencher) {
        let mut bf = BloomFilter::new();
        let mut gen = HashGenerator::default();
        // Note: this will underflow, and that's ok.
        b.iter(|| {
            for _ in 0..10 {
                bf.remove_hash(gen.next())
            }
        });
    }
}
<commit_msg>style: Allow 16% false positives in test bloom::create_and_insert_some_stuff.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Counting and non-counting Bloom filters tuned for use as ancestor filters
//! for selector matching.

use fnv::FnvHasher;
use std::fmt::{self, Debug};
use std::hash::{Hash, Hasher};

// The top 8 bits of the 32-bit hash value are not used by the bloom filter.
// Consumers may rely on this to pack hashes more efficiently.
pub const BLOOM_HASH_MASK: u32 = 0x00ffffff;
const KEY_SIZE: usize = 12;

const ARRAY_SIZE: usize = 1 << KEY_SIZE;
const KEY_MASK: u32 = (1 << KEY_SIZE) - 1;

/// A counting Bloom filter with 8-bit counters.
pub type BloomFilter = CountingBloomFilter<BloomStorageU8>;

/// A non-counting Bloom filter.
///
/// Effectively a counting Bloom filter with 1-bit counters.
pub type NonCountingBloomFilter = CountingBloomFilter<BloomStorageBool>;

/// A counting Bloom filter with parameterized storage to handle
/// counters of different sizes.  For now we assume that having two hash
/// functions is enough, but we may revisit that decision later.
///
/// The filter uses an array with 2**KeySize entries.
///
/// Assuming a well-distributed hash function, a Bloom filter with
/// array size M containing N elements and
/// using k hash function has expected false positive rate exactly
///
/// $  (1 - (1 - 1/M)^{kN})^k  $
///
/// because each array slot has a
///
/// $  (1 - 1/M)^{kN}  $
///
/// chance of being 0, and the expected false positive rate is the
/// probability that all of the k hash functions will hit a nonzero
/// slot.
///
/// For reasonable assumptions (M large, kN large, which should both
/// hold if we're worried about false positives) about M and kN this
/// becomes approximately
///
/// $$  (1 - \exp(-kN/M))^k   $$
///
/// For our special case of k == 2, that's $(1 - \exp(-2N/M))^2$,
/// or in other words
///
/// $$    N/M = -0.5 * \ln(1 - \sqrt(r))   $$
///
/// where r is the false positive rate.  This can be used to compute
/// the desired KeySize for a given load N and false positive rate r.
///
/// If N/M is assumed small, then the false positive rate can
/// further be approximated as 4*N^2/M^2.  So increasing KeySize by
/// 1, which doubles M, reduces the false positive rate by about a
/// factor of 4, and a false positive rate of 1% corresponds to
/// about M/N == 20.
///
/// What this means in practice is that for a few hundred keys using a
/// KeySize of 12 gives false positive rates on the order of 0.25-4%.
///
/// Similarly, using a KeySize of 10 would lead to a 4% false
/// positive rate for N == 100 and to quite bad false positive
/// rates for larger N.
#[derive(Clone)]
pub struct CountingBloomFilter<S>
where
    S: BloomStorage,
{
    storage: S,
}

impl<S> CountingBloomFilter<S>
where
    S: BloomStorage,
{
    /// Creates a new bloom filter.
    #[inline]
    pub fn new() -> Self {
        CountingBloomFilter {
            storage: Default::default(),
        }
    }

    #[inline]
    pub fn clear(&mut self) {
        self.storage = Default::default();
    }

    // Slow linear accessor to make sure the bloom filter is zeroed. This should
    // never be used in release builds.
    #[cfg(debug_assertions)]
    pub fn is_zeroed(&self) -> bool {
        self.storage.is_zeroed()
    }

    #[cfg(not(debug_assertions))]
    pub fn is_zeroed(&self) -> bool {
        unreachable!()
    }

    #[inline]
    pub fn insert_hash(&mut self, hash: u32) {
        self.storage.adjust_first_slot(hash, true);
        self.storage.adjust_second_slot(hash, true);
    }

    /// Inserts an item into the bloom filter.
    #[inline]
    pub fn insert<T: Hash>(&mut self, elem: &T) {
        self.insert_hash(hash(elem))
    }

    #[inline]
    pub fn remove_hash(&mut self, hash: u32) {
        self.storage.adjust_first_slot(hash, false);
        self.storage.adjust_second_slot(hash, false);
    }

    /// Removes an item from the bloom filter.
    #[inline]
    pub fn remove<T: Hash>(&mut self, elem: &T) {
        self.remove_hash(hash(elem))
    }

    #[inline]
    pub fn might_contain_hash(&self, hash: u32) -> bool {
        !self.storage.first_slot_is_empty(hash) && !self.storage.second_slot_is_empty(hash)
    }

    /// Check whether the filter might contain an item.  This can
    /// sometimes return true even if the item is not in the filter,
    /// but will never return false for items that are actually in the
    /// filter.
    #[inline]
    pub fn might_contain<T: Hash>(&self, elem: &T) -> bool {
        self.might_contain_hash(hash(elem))
    }
}

impl<S> Debug for CountingBloomFilter<S>
where
    S: BloomStorage,
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let mut slots_used = 0;
        for i in 0..ARRAY_SIZE {
            if !self.storage.slot_is_empty(i) {
                slots_used += 1;
            }
        }
        write!(f, "BloomFilter({}/{})", slots_used, ARRAY_SIZE)
    }
}

pub trait BloomStorage: Clone + Default {
    fn slot_is_empty(&self, index: usize) -> bool;
    fn adjust_slot(&mut self, index: usize, increment: bool);
    fn is_zeroed(&self) -> bool;

    #[inline]
    fn first_slot_is_empty(&self, hash: u32) -> bool {
        self.slot_is_empty(Self::first_slot_index(hash))
    }

    #[inline]
    fn second_slot_is_empty(&self, hash: u32) -> bool {
        self.slot_is_empty(Self::second_slot_index(hash))
    }

    #[inline]
    fn adjust_first_slot(&mut self, hash: u32, increment: bool) {
        self.adjust_slot(Self::first_slot_index(hash), increment)
    }

    #[inline]
    fn adjust_second_slot(&mut self, hash: u32, increment: bool) {
        self.adjust_slot(Self::second_slot_index(hash), increment)
    }

    #[inline]
    fn first_slot_index(hash: u32) -> usize {
        hash1(hash) as usize
    }

    #[inline]
    fn second_slot_index(hash: u32) -> usize {
        hash2(hash) as usize
    }
}

/// Storage class for a CountingBloomFilter that has 8-bit counters.
pub struct BloomStorageU8 {
    counters: [u8; ARRAY_SIZE],
}

impl BloomStorage for BloomStorageU8 {
    #[inline]
    fn adjust_slot(&mut self, index: usize, increment: bool) {
        let slot = &mut self.counters[index];
        if *slot != 0xff {
            // full
            if increment {
                *slot += 1;
            } else {
                *slot -= 1;
            }
        }
    }

    #[inline]
    fn slot_is_empty(&self, index: usize) -> bool {
        self.counters[index] == 0
    }

    #[inline]
    fn is_zeroed(&self) -> bool {
        self.counters.iter().all(|x| *x == 0)
    }
}

impl Default for BloomStorageU8 {
    fn default() -> Self {
        BloomStorageU8 {
            counters: [0; ARRAY_SIZE],
        }
    }
}

impl Clone for BloomStorageU8 {
    fn clone(&self) -> Self {
        BloomStorageU8 {
            counters: self.counters,
        }
    }
}

/// Storage class for a CountingBloomFilter that has 1-bit counters.
pub struct BloomStorageBool {
    counters: [u8; ARRAY_SIZE / 8],
}

impl BloomStorage for BloomStorageBool {
    #[inline]
    fn adjust_slot(&mut self, index: usize, increment: bool) {
        let bit = 1 << (index % 8);
        let byte = &mut self.counters[index / 8];

        // Since we have only one bit for storage, decrementing it
        // should never do anything.  Assert against an accidental
        // decrementing of a bit that was never set.
        assert!(
            increment || (*byte & bit) != 0,
            "should not decrement if slot is already false"
        );

        if increment {
            *byte |= bit;
        }
    }

    #[inline]
    fn slot_is_empty(&self, index: usize) -> bool {
        let bit = 1 << (index % 8);
        (self.counters[index / 8] & bit) == 0
    }

    #[inline]
    fn is_zeroed(&self) -> bool {
        self.counters.iter().all(|x| *x == 0)
    }
}

impl Default for BloomStorageBool {
    fn default() -> Self {
        BloomStorageBool {
            counters: [0; ARRAY_SIZE / 8],
        }
    }
}

impl Clone for BloomStorageBool {
    fn clone(&self) -> Self {
        BloomStorageBool {
            counters: self.counters,
        }
    }
}

fn hash<T: Hash>(elem: &T) -> u32 {
    let mut hasher = FnvHasher::default();
    elem.hash(&mut hasher);
    let hash: u64 = hasher.finish();
    (hash >> 32) as u32 ^ (hash as u32)
}

#[inline]
fn hash1(hash: u32) -> u32 {
    hash & KEY_MASK
}

#[inline]
fn hash2(hash: u32) -> u32 {
    (hash >> KEY_SIZE) & KEY_MASK
}

#[test]
fn create_and_insert_some_stuff() {
    use std::mem::transmute;

    let mut bf = BloomFilter::new();

    // Statically assert that ARRAY_SIZE is a multiple of 8, which
    // BloomStorageBool relies on.
    unsafe {
        transmute::<[u8; ARRAY_SIZE % 8], [u8; 0]>([]);
    }

    for i in 0_usize..1000 {
        bf.insert(&i);
    }

    for i in 0_usize..1000 {
        assert!(bf.might_contain(&i));
    }

    let false_positives = (1001_usize..2000).filter(|i| bf.might_contain(i)).count();

    assert!(false_positives < 160, "{} is not < 160", false_positives); // 16%.

    for i in 0_usize..100 {
        bf.remove(&i);
    }

    for i in 100_usize..1000 {
        assert!(bf.might_contain(&i));
    }

    let false_positives = (0_usize..100).filter(|i| bf.might_contain(i)).count();

    assert!(false_positives < 20, "{} is not < 20", false_positives); // 20%.

    bf.clear();

    for i in 0_usize..2000 {
        assert!(!bf.might_contain(&i));
    }
}

#[cfg(feature = "bench")]
#[cfg(test)]
mod bench {
    extern crate test;
    use super::BloomFilter;

    #[derive(Default)]
    struct HashGenerator(u32);

    impl HashGenerator {
        fn next(&mut self) -> u32 {
            // Each hash is split into two twelve-bit segments, which are used
            // as an index into an array. We increment each by 64 so that we
            // hit the next cache line, and then another 1 so that our wrapping
            // behavior leads us to different entries each time.
            //
            // Trying to simulate cold caches is rather difficult with the cargo
            // benchmarking setup, so it may all be moot depending on the number
            // of iterations that end up being run. But we might as well.
            self.0 += (65) + (65 << super::KEY_SIZE);
            self.0
        }
    }

    #[bench]
    fn create_insert_1000_remove_100_lookup_100(b: &mut test::Bencher) {
        b.iter(|| {
            let mut gen1 = HashGenerator::default();
            let mut gen2 = HashGenerator::default();
            let mut bf = BloomFilter::new();
            for _ in 0_usize..1000 {
                bf.insert_hash(gen1.next());
            }
            for _ in 0_usize..100 {
                bf.remove_hash(gen2.next());
            }
            for _ in 100_usize..200 {
                test::black_box(bf.might_contain_hash(gen2.next()));
            }
        });
    }

    #[bench]
    fn might_contain_10(b: &mut test::Bencher) {
        let bf = BloomFilter::new();
        let mut gen = HashGenerator::default();
        b.iter(|| {
            for _ in 0..10 {
                test::black_box(bf.might_contain_hash(gen.next()));
            }
        });
    }

    #[bench]
    fn clear(b: &mut test::Bencher) {
        let mut bf = Box::new(BloomFilter::new());
        b.iter(|| test::black_box(&mut bf).clear());
    }

    #[bench]
    fn insert_10(b: &mut test::Bencher) {
        let mut bf = BloomFilter::new();
        let mut gen = HashGenerator::default();
        b.iter(|| {
            for _ in 0..10 {
                test::black_box(bf.insert_hash(gen.next()));
            }
        });
    }

    #[bench]
    fn remove_10(b: &mut test::Bencher) {
        let mut bf = BloomFilter::new();
        let mut gen = HashGenerator::default();
        // Note: this will underflow, and that's ok.
        b.iter(|| {
            for _ in 0..10 {
                bf.remove_hash(gen.next())
            }
        });
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{DEFAULT_BUILD_DIR, DEFAULT_PACKAGE_DIR, DEFAULT_STORAGE_DIR};
use anyhow::anyhow;
use move_coverage::coverage_map::{CoverageMap, ExecCoverageMapWithModules};
use move_lang::{extension_equals, path_to_string, test_utils::*, MOVE_COMPILED_EXTENSION};
use std::{
    collections::{BTreeMap, HashMap, HashSet},
    env,
    fs::{self, File},
    io::{self, BufRead, Write},
    path::{Path, PathBuf},
    process::Command,
};
use vm::file_format::CompiledModule;

/// Basic datatest testing framework for the CLI. The `run_one` entrypoint expects
/// an `args.txt` file with arguments that the `move` binary understands (one set
/// of arguments per line). The testing framework runs the commands, compares the
/// result to the expected output, and runs `move clean` to discard resources,
/// modules, and event data created by running the test.

const EXP_EXT: &str = "exp";

/// If this env var is set, `move clean` will not be run after each test.
/// this is useful if you want to look at the `storage` or `move_events`
/// produced by a test. However, you'll have to manually run `move clean`
/// before re-running the test.
const NO_MOVE_CLEAN: &str = "NO_MOVE_CLEAN";
/// If either of these env vars is set, the test harness overwrites the
/// old .exp files with the output instead of checking them against the
/// output.
const UPDATE_BASELINE: &str = "UPDATE_BASELINE";
const UB: &str = "UB";

/// Name of the environment variable we need to set in order to get tracing
/// enabled in the move VM.
const MOVE_VM_TRACING_ENV_VAR_NAME: &str = "MOVE_VM_TRACE";

/// The default file name (inside the build output dir) for the runtime to
/// dump the execution trace to. The trace will be used by the coverage tool
/// if --track-cov is set. If --track-cov is not set, then no trace file will
/// be produced.
const DEFAULT_TRACE_FILE: &str = "trace";

fn format_diff(expected: String, actual: String) -> String {
    use difference::*;

    let changeset = Changeset::new(&expected, &actual, "\n");

    let mut ret = String::new();

    for seq in changeset.diffs {
        match &seq {
            Difference::Same(x) => {
                ret.push_str(x);
                ret.push_str("\n");
            }
            Difference::Add(x) => {
                ret.push_str("\x1B[92m");
                ret.push_str(x);
                ret.push_str("\x1B[0m");
                ret.push_str("\n");
            }
            Difference::Rem(x) => {
                ret.push_str("\x1B[91m");
                ret.push_str(x);
                ret.push_str("\x1B[0m");
                ret.push_str("\n");
            }
        }
    }
    ret
}

fn collect_coverage(
    trace_file: &Path,
    build_dir: &Path,
    storage_dir: &Path,
) -> anyhow::Result<ExecCoverageMapWithModules> {
    fn find_compiled_move_filenames(path: &Path) -> anyhow::Result<Vec<String>> {
        if path.exists() {
            move_lang::find_filenames(&[path_to_string(path)?], |fpath| {
                extension_equals(fpath, MOVE_COMPILED_EXTENSION)
            })
        } else {
            Ok(vec![])
        }
    }

    // collect modules compiled for packages (to be filtered out)
    let pkg_modules: HashSet<_> =
        find_compiled_move_filenames(&build_dir.join(DEFAULT_PACKAGE_DIR))?
            .into_iter()
            .map(|entry| PathBuf::from(entry).file_name().unwrap().to_owned())
            .collect();

    // collect modules published minus modules compiled for packages
    let src_module_files = move_lang::find_filenames(&[path_to_string(storage_dir)?], |fpath| {
        extension_equals(fpath, MOVE_COMPILED_EXTENSION)
            && !pkg_modules.contains(fpath.file_name().unwrap())
    })?;
    let src_modules = src_module_files
        .iter()
        .map(|entry| {
            let bytecode_bytes = fs::read(entry)?;
            let compiled_module = CompiledModule::deserialize(&bytecode_bytes)
                .map_err(|e| anyhow!("Failure deserializing module {:?}: {:?}", entry, e))?;

            // use absolute path to the compiled module file
            let module_absolute_path = path_to_string(&PathBuf::from(entry).canonicalize()?)?;
            Ok((module_absolute_path, compiled_module))
        })
        .collect::<anyhow::Result<HashMap<_, _>>>()?;

    // build the filter
    let mut filter = BTreeMap::new();
    for (entry, module) in src_modules.into_iter() {
        let module_id = module.self_id();
        filter
            .entry(*module_id.address())
            .or_insert_with(BTreeMap::new)
            .insert(module_id.name().to_owned(), (entry, module));
    }

    // collect filtered trace
    let coverage_map = CoverageMap::from_trace_file(trace_file)
        .to_unified_exec_map()
        .into_coverage_map_with_modules(filter);

    Ok(coverage_map)
}

/// Run the `args_path` batch file with`cli_binary`
pub fn run_one(
    args_path: &Path,
    cli_binary: &str,
    track_cov: bool,
) -> anyhow::Result<Option<ExecCoverageMapWithModules>> {
    let args_file = io::BufReader::new(File::open(args_path)?).lines();
    // path where we will run the binary
    let exe_dir = args_path.parent().unwrap();
    let cli_binary_path = Path::new(cli_binary).canonicalize()?;
    let storage_dir = Path::new(exe_dir).join(DEFAULT_STORAGE_DIR);
    let build_output = Path::new(exe_dir).join(DEFAULT_BUILD_DIR);
    if storage_dir.exists() || build_output.exists() {
        // need to clean before testing
        Command::new(cli_binary_path.clone())
            .current_dir(exe_dir)
            .arg("clean")
            .output()?;
    }
    let mut output = "".to_string();

    // for tracing file path: always use the absolute path so we do not need to worry about where
    // the VM is executed.
    let trace_file = env::current_dir()?
        .join(&build_output)
        .join(DEFAULT_TRACE_FILE);

    for args_line in args_file {
        let args_line = args_line?;
        if args_line.starts_with('#') {
            // allow comments in args.txt
            continue;
        }
        let args_iter: Vec<&str> = args_line.split_whitespace().collect();
        if args_iter.is_empty() {
            // allow blank lines in args.txt
            continue;
        }

        // enable tracing in the VM by setting the env var.
        if track_cov {
            env::set_var(MOVE_VM_TRACING_ENV_VAR_NAME, trace_file.as_os_str());
        } else if env::var_os(MOVE_VM_TRACING_ENV_VAR_NAME).is_some() {
            // this check prevents cascading the coverage tracking flag.
            // in particular, if
            //   1. we run with move-cli test <path-to-args-A.txt> --track-cov, and
            //   2. in this <args-A.txt>, there is another command: test <args-B.txt>
            // then, when running <args-B.txt>, coverage will not be tracked nor printed
            env::remove_var(MOVE_VM_TRACING_ENV_VAR_NAME);
        }

        let cmd_output = Command::new(cli_binary_path.clone())
            .current_dir(exe_dir)
            .args(args_iter)
            .output()?;
        output += &format!("Command `{}`:\n", args_line);
        output += std::str::from_utf8(&cmd_output.stdout)?;
        output += std::str::from_utf8(&cmd_output.stderr)?;
    }

    // collect coverage information
    let cov_info = if track_cov && trace_file.exists() {
        if !trace_file.exists() {
            eprintln!(
                "Trace file {:?} not found: coverage is only available with at least one `run` \
                command in the args.txt (after a `clean`, if there is one)",
                trace_file
            );
            None
        } else {
            Some(collect_coverage(&trace_file, &build_output, &storage_dir)?)
        }
    } else {
        None
    };

    // post-test cleanup and cleanup checks
    // check that the test command didn't create a src dir

    let run_move_clean = !read_bool_var(NO_MOVE_CLEAN);
    if run_move_clean {
        // run `move clean` to ensure that temporary state is cleaned up
        Command::new(cli_binary_path)
            .current_dir(exe_dir)
            .arg("clean")
            .output()?;
        // check that storage was deleted
        assert!(
            !storage_dir.exists(),
            "`move clean` failed to eliminate {} directory",
            DEFAULT_STORAGE_DIR
        );
        assert!(
            !storage_dir.exists(),
            "`move clean` failed to eliminate {} directory",
            DEFAULT_BUILD_DIR
        );
    }

    let update_baseline = read_bool_var(UPDATE_BASELINE) || read_bool_var(UB);
    let exp_path = args_path.with_extension(EXP_EXT);
    if update_baseline {
        fs::write(exp_path, &output)?;
        return Ok(cov_info);
    }
    // compare output and exp_file
    let expected_output = fs::read_to_string(exp_path).unwrap_or_else(|_| "".to_string());
    if expected_output != output {
        anyhow::bail!(
            "Expected output differs from actual output:\n{}",
            format_diff(expected_output, output)
        )
    } else {
        Ok(cov_info)
    }
}

pub fn run_all(args_path: &str, cli_binary: &str, track_cov: bool) -> anyhow::Result<()> {
    let mut test_total = 0;
    let mut test_passed = 0;
    let mut cov_info = ExecCoverageMapWithModules::empty();

    // find `args.txt` and iterate over them
    for entry in move_lang::find_filenames(&[args_path.to_owned()], |fpath| {
        fpath.file_name().expect("unexpected file entry path") == "args.txt"
    })? {
        match run_one(Path::new(&entry), cli_binary, track_cov) {
            Ok(cov_opt) => {
                test_passed += 1;
                if let Some(cov) = cov_opt {
                    cov_info.merge(cov);
                }
            }
            Err(ex) => eprintln!("Test {} failed with error: {}", entry, ex),
        }
        test_total += 1;
    }
    println!("{} / {} test(s) passed.", test_passed, test_total);

    // if any test fails, bail
    let test_failed = test_total - test_passed;
    if test_failed != 0 {
        anyhow::bail!("{} / {} test(s) failed.", test_failed, test_total)
    }

    // show coverage information if requested
    if track_cov {
        let mut summary_writer: Box<dyn Write> = Box::new(io::stdout());
        for (_, module_summary) in cov_info.into_module_summaries() {
            module_summary.summarize_human(&mut summary_writer, true)?;
        }
    }

    Ok(())
}
<commit_msg>[move-cli] use checked add/sub for test stat counting<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{DEFAULT_BUILD_DIR, DEFAULT_PACKAGE_DIR, DEFAULT_STORAGE_DIR};
use anyhow::anyhow;
use move_coverage::coverage_map::{CoverageMap, ExecCoverageMapWithModules};
use move_lang::{extension_equals, path_to_string, test_utils::*, MOVE_COMPILED_EXTENSION};
use std::{
    collections::{BTreeMap, HashMap, HashSet},
    env,
    fs::{self, File},
    io::{self, BufRead, Write},
    path::{Path, PathBuf},
    process::Command,
};
use vm::file_format::CompiledModule;

/// Basic datatest testing framework for the CLI. The `run_one` entrypoint expects
/// an `args.txt` file with arguments that the `move` binary understands (one set
/// of arguments per line). The testing framework runs the commands, compares the
/// result to the expected output, and runs `move clean` to discard resources,
/// modules, and event data created by running the test.

const EXP_EXT: &str = "exp";

/// If this env var is set, `move clean` will not be run after each test.
/// this is useful if you want to look at the `storage` or `move_events`
/// produced by a test. However, you'll have to manually run `move clean`
/// before re-running the test.
const NO_MOVE_CLEAN: &str = "NO_MOVE_CLEAN";
/// If either of these env vars is set, the test harness overwrites the
/// old .exp files with the output instead of checking them against the
/// output.
const UPDATE_BASELINE: &str = "UPDATE_BASELINE";
const UB: &str = "UB";

/// Name of the environment variable we need to set in order to get tracing
/// enabled in the move VM.
const MOVE_VM_TRACING_ENV_VAR_NAME: &str = "MOVE_VM_TRACE";

/// The default file name (inside the build output dir) for the runtime to
/// dump the execution trace to. The trace will be used by the coverage tool
/// if --track-cov is set. If --track-cov is not set, then no trace file will
/// be produced.
const DEFAULT_TRACE_FILE: &str = "trace";

fn format_diff(expected: String, actual: String) -> String {
    use difference::*;

    let changeset = Changeset::new(&expected, &actual, "\n");

    let mut ret = String::new();

    for seq in changeset.diffs {
        match &seq {
            Difference::Same(x) => {
                ret.push_str(x);
                ret.push_str("\n");
            }
            Difference::Add(x) => {
                ret.push_str("\x1B[92m");
                ret.push_str(x);
                ret.push_str("\x1B[0m");
                ret.push_str("\n");
            }
            Difference::Rem(x) => {
                ret.push_str("\x1B[91m");
                ret.push_str(x);
                ret.push_str("\x1B[0m");
                ret.push_str("\n");
            }
        }
    }
    ret
}

fn collect_coverage(
    trace_file: &Path,
    build_dir: &Path,
    storage_dir: &Path,
) -> anyhow::Result<ExecCoverageMapWithModules> {
    fn find_compiled_move_filenames(path: &Path) -> anyhow::Result<Vec<String>> {
        if path.exists() {
            move_lang::find_filenames(&[path_to_string(path)?], |fpath| {
                extension_equals(fpath, MOVE_COMPILED_EXTENSION)
            })
        } else {
            Ok(vec![])
        }
    }

    // collect modules compiled for packages (to be filtered out)
    let pkg_modules: HashSet<_> =
        find_compiled_move_filenames(&build_dir.join(DEFAULT_PACKAGE_DIR))?
            .into_iter()
            .map(|entry| PathBuf::from(entry).file_name().unwrap().to_owned())
            .collect();

    // collect modules published minus modules compiled for packages
    let src_module_files = move_lang::find_filenames(&[path_to_string(storage_dir)?], |fpath| {
        extension_equals(fpath, MOVE_COMPILED_EXTENSION)
            && !pkg_modules.contains(fpath.file_name().unwrap())
    })?;
    let src_modules = src_module_files
        .iter()
        .map(|entry| {
            let bytecode_bytes = fs::read(entry)?;
            let compiled_module = CompiledModule::deserialize(&bytecode_bytes)
                .map_err(|e| anyhow!("Failure deserializing module {:?}: {:?}", entry, e))?;

            // use absolute path to the compiled module file
            let module_absolute_path = path_to_string(&PathBuf::from(entry).canonicalize()?)?;
            Ok((module_absolute_path, compiled_module))
        })
        .collect::<anyhow::Result<HashMap<_, _>>>()?;

    // build the filter
    let mut filter = BTreeMap::new();
    for (entry, module) in src_modules.into_iter() {
        let module_id = module.self_id();
        filter
            .entry(*module_id.address())
            .or_insert_with(BTreeMap::new)
            .insert(module_id.name().to_owned(), (entry, module));
    }

    // collect filtered trace
    let coverage_map = CoverageMap::from_trace_file(trace_file)
        .to_unified_exec_map()
        .into_coverage_map_with_modules(filter);

    Ok(coverage_map)
}

/// Run the `args_path` batch file with`cli_binary`
pub fn run_one(
    args_path: &Path,
    cli_binary: &str,
    track_cov: bool,
) -> anyhow::Result<Option<ExecCoverageMapWithModules>> {
    let args_file = io::BufReader::new(File::open(args_path)?).lines();
    // path where we will run the binary
    let exe_dir = args_path.parent().unwrap();
    let cli_binary_path = Path::new(cli_binary).canonicalize()?;
    let storage_dir = Path::new(exe_dir).join(DEFAULT_STORAGE_DIR);
    let build_output = Path::new(exe_dir).join(DEFAULT_BUILD_DIR);
    if storage_dir.exists() || build_output.exists() {
        // need to clean before testing
        Command::new(cli_binary_path.clone())
            .current_dir(exe_dir)
            .arg("clean")
            .output()?;
    }
    let mut output = "".to_string();

    // for tracing file path: always use the absolute path so we do not need to worry about where
    // the VM is executed.
    let trace_file = env::current_dir()?
        .join(&build_output)
        .join(DEFAULT_TRACE_FILE);

    for args_line in args_file {
        let args_line = args_line?;
        if args_line.starts_with('#') {
            // allow comments in args.txt
            continue;
        }
        let args_iter: Vec<&str> = args_line.split_whitespace().collect();
        if args_iter.is_empty() {
            // allow blank lines in args.txt
            continue;
        }

        // enable tracing in the VM by setting the env var.
        if track_cov {
            env::set_var(MOVE_VM_TRACING_ENV_VAR_NAME, trace_file.as_os_str());
        } else if env::var_os(MOVE_VM_TRACING_ENV_VAR_NAME).is_some() {
            // this check prevents cascading the coverage tracking flag.
            // in particular, if
            //   1. we run with move-cli test <path-to-args-A.txt> --track-cov, and
            //   2. in this <args-A.txt>, there is another command: test <args-B.txt>
            // then, when running <args-B.txt>, coverage will not be tracked nor printed
            env::remove_var(MOVE_VM_TRACING_ENV_VAR_NAME);
        }

        let cmd_output = Command::new(cli_binary_path.clone())
            .current_dir(exe_dir)
            .args(args_iter)
            .output()?;
        output += &format!("Command `{}`:\n", args_line);
        output += std::str::from_utf8(&cmd_output.stdout)?;
        output += std::str::from_utf8(&cmd_output.stderr)?;
    }

    // collect coverage information
    let cov_info = if track_cov && trace_file.exists() {
        if !trace_file.exists() {
            eprintln!(
                "Trace file {:?} not found: coverage is only available with at least one `run` \
                command in the args.txt (after a `clean`, if there is one)",
                trace_file
            );
            None
        } else {
            Some(collect_coverage(&trace_file, &build_output, &storage_dir)?)
        }
    } else {
        None
    };

    // post-test cleanup and cleanup checks
    // check that the test command didn't create a src dir

    let run_move_clean = !read_bool_var(NO_MOVE_CLEAN);
    if run_move_clean {
        // run `move clean` to ensure that temporary state is cleaned up
        Command::new(cli_binary_path)
            .current_dir(exe_dir)
            .arg("clean")
            .output()?;
        // check that storage was deleted
        assert!(
            !storage_dir.exists(),
            "`move clean` failed to eliminate {} directory",
            DEFAULT_STORAGE_DIR
        );
        assert!(
            !storage_dir.exists(),
            "`move clean` failed to eliminate {} directory",
            DEFAULT_BUILD_DIR
        );
    }

    let update_baseline = read_bool_var(UPDATE_BASELINE) || read_bool_var(UB);
    let exp_path = args_path.with_extension(EXP_EXT);
    if update_baseline {
        fs::write(exp_path, &output)?;
        return Ok(cov_info);
    }
    // compare output and exp_file
    let expected_output = fs::read_to_string(exp_path).unwrap_or_else(|_| "".to_string());
    if expected_output != output {
        anyhow::bail!(
            "Expected output differs from actual output:\n{}",
            format_diff(expected_output, output)
        )
    } else {
        Ok(cov_info)
    }
}

pub fn run_all(args_path: &str, cli_binary: &str, track_cov: bool) -> anyhow::Result<()> {
    let mut test_total: u64 = 0;
    let mut test_passed: u64 = 0;
    let mut cov_info = ExecCoverageMapWithModules::empty();

    // find `args.txt` and iterate over them
    for entry in move_lang::find_filenames(&[args_path.to_owned()], |fpath| {
        fpath.file_name().expect("unexpected file entry path") == "args.txt"
    })? {
        match run_one(Path::new(&entry), cli_binary, track_cov) {
            Ok(cov_opt) => {
                test_passed = test_passed.checked_add(1).unwrap();
                if let Some(cov) = cov_opt {
                    cov_info.merge(cov);
                }
            }
            Err(ex) => eprintln!("Test {} failed with error: {}", entry, ex),
        }
        test_total = test_total.checked_add(1).unwrap();
    }
    println!("{} / {} test(s) passed.", test_passed, test_total);

    // if any test fails, bail
    let test_failed = test_total.checked_sub(test_passed).unwrap();
    if test_failed != 0 {
        anyhow::bail!("{} / {} test(s) failed.", test_failed, test_total)
    }

    // show coverage information if requested
    if track_cov {
        let mut summary_writer: Box<dyn Write> = Box::new(io::stdout());
        for (_, module_summary) in cov_info.into_module_summaries() {
            module_summary.summarize_human(&mut summary_writer, true)?;
        }
    }

    Ok(())
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::{self, Read, BufRead, BufReader};

enum DataType {
    Real,
    Complex,
    Integer,
}

pub struct MatrixMarketReader {
    rows: usize,
    cols: usize,
    data_type: DataType,
}

fn filename_to_string(s: &str) -> io::Result<String> {
    let mut file = File::open(s).expect("file not found");
    let mut s = String::new();
    file.read_to_string(&mut s)?;
    Ok(s)
}

fn words_by_line<'a>(s: &'a str) -> Vec<Vec<&'a str>> {
    s.lines().map(|line| { line.split_whitespace().collect() }).collect()
}

impl MatrixMarketReader {
    pub fn new(fname: &str) -> Result<MatrixMarketReader, String> {
        //let file = File::open(fname).expect("file not found!");
        //let mut f = BufReader::new(file);
        let whole_file = filename_to_string(fname).unwrap();
        let wbyl = &words_by_line(&whole_file);
        let mm_header = &wbyl[0];
        let (banner, mtx, coord, dtype, storage) = (mm_header[0],
                                                    mm_header[1],
                                                    mm_header[2],
                                                    mm_header[3],
                                                    mm_header[4]);
        //if banner.ne("%%MatrixMarket") { panic!("no banner"); }
        if banner.ne("%%MatrixMarket") { return Err(String::from("no banner")); }
        if mtx.ne("matrix") { return Err(String::from("not a matrix")); }
        let is_symmetric = match storage {
                             "symmetric" => true,
                             "general"   => false,
                              _          => return Err(String::from("unsupported storage type"))};
        let is_sparse = match coord {
                          "coordinate"   => true,
                          "array"        => false,
                           _             => return Err(String::from("unsupported coordinate type"))};
        let data_type = match dtype {
                           "real"        => DataType::Real,
                           "complex"     => DataType::Complex,
                           "integer"     => DataType::Integer,
                           _             => return Err(String::from("unsupported data type"))};

        //let mut (rows, cols, nnz) = (0, 0, 0)
        let mut rows = 0;
        let mut cols = 0;
        let mut nnz  = 0;
        for words in wbyl {
            if words[0].starts_with('%') { // skip a comment that starts from %
                println!("words: {:?}", words);
            } else {
                // read triples, the 1st one contains (rows, cols, nnz) the others - (i, j, val)
                //let parse_word = |i: usize| words[i].parse().unwrap();
                macro_rules! parse_word {
                    ($t:ty, $i:expr) => {
                        words[$i].parse::<$t>().unwrap()
                    };
                    ($i:expr) => { // `$t` defaulted to `i32`
                        parse_word!(i32, $i)
                    };
                }
                if rows == 0 {
                    rows = parse_word!(0);
                    cols = parse_word!(1);
                    nnz  = parse_word!(2);
                    println!("rows: {} cols: {} nnz: {}", rows, cols, nnz);
                } else {
                    let x: i32 = parse_word!(0);
                    let y: i32 = parse_word!(1);
                    let v: f64 = parse_word!(f64, 2);
                    println!("x: {} y: {} v: {:10.4e}", x, y, v);
                }
            }
            //println!("words: {:?}", words);
        }

        println!("wbyl[0]: {:?}", mm_header);
        Ok(MatrixMarketReader { rows: 0, cols: 0, data_type: DataType::Real, })
    }
}

fn use_it(fname: &str) {
    let mm = MatrixMarketReader::new("xxx");
}

<commit_msg>fill `MatrixMarketReader` `rows` etc.<commit_after>use std::fs::File;
use std::io::{self, Read, BufRead, BufReader};

enum DataType {
    Real,
    Complex,
    Integer,
}

pub struct MatrixMarketReader {
    rows: usize,
    cols: usize,
    data_type: DataType,
}

fn filename_to_string(s: &str) -> io::Result<String> {
    let mut file = File::open(s).expect("file not found");
    let mut s = String::new();
    file.read_to_string(&mut s)?;
    Ok(s)
}

fn words_by_line<'a>(s: &'a str) -> Vec<Vec<&'a str>> {
    s.lines().map(|line| { line.split_whitespace().collect() }).collect()
}

impl MatrixMarketReader {
    pub fn new(fname: &str) -> Result<MatrixMarketReader, String> {
        //let file = File::open(fname).expect("file not found!");
        //let mut f = BufReader::new(file);
        let whole_file = filename_to_string(fname).unwrap();
        let wbyl = &words_by_line(&whole_file);
        let mm_header = &wbyl[0];
        let (banner, mtx, coord, dtype, storage) = (mm_header[0],
                                                    mm_header[1],
                                                    mm_header[2],
                                                    mm_header[3],
                                                    mm_header[4]);
        //if banner.ne("%%MatrixMarket") { panic!("no banner"); }
        if banner.ne("%%MatrixMarket") { return Err(String::from("no banner")); }
        if mtx.ne("matrix") { return Err(String::from("not a matrix")); }
        let is_symmetric = match storage {
                             "symmetric" => true,
                             "general"   => false,
                              _          => return Err(String::from("unsupported storage type"))};
        let is_sparse = match coord {
                          "coordinate"   => true,
                          "array"        => false,
                           _             => return Err(String::from("unsupported coordinate type"))};
        let data_type = match dtype {
                           "real"        => DataType::Real,
                           "complex"     => DataType::Complex,
                           "integer"     => DataType::Integer,
                           _             => return Err(String::from("unsupported data type"))};

        //let mut (rows, cols, nnz) = (0, 0, 0)
        let mut rows = 0;
        let mut cols = 0;
        let mut nnz  = 0;
        for words in wbyl {
            if words[0].starts_with('%') { // skip a comment that starts from %
                println!("words: {:?}", words);
            } else {
                // read triples, the 1st one contains (rows, cols, nnz) the others - (i, j, val)
                //let parse_word = |i: usize| words[i].parse().unwrap();
                macro_rules! parse_word {
                    ($t:ty, $i:expr) => {
                        words[$i].parse::<$t>().unwrap()
                    };
                    ($i:expr) => { // `$t` defaulted to `i32`
                        parse_word!(i32, $i)
                    };
                }
                if rows == 0 {
                    rows = parse_word!(0);
                    cols = parse_word!(1);
                    nnz  = parse_word!(2);
                    println!("rows: {} cols: {} nnz: {}", rows, cols, nnz);
                } else {
                    let x: i32 = parse_word!(0);
                    let y: i32 = parse_word!(1);
                    let v: f64 = parse_word!(f64, 2);
                    println!("x: {} y: {} v: {:10.4e}", x, y, v);
                }
            }
            //println!("words: {:?}", words);
        }

        println!("wbyl[0]: {:?}", mm_header);
        Ok(MatrixMarketReader { rows: rows as usize, cols: cols as usize, data_type: data_type, })
    }
}

fn use_it(fname: &str) {
    let mm = MatrixMarketReader::new("xxx");
}

<|endoftext|>


<commit_before>//! Some benchmarks stress-testing various ways to build the standard
//! `HashMap` data structures from the standard library.

mod util {
    use rayon::prelude::*;
    use std::collections::{LinkedList, HashMap};
    use std::hash::Hash;
    use std::sync::Mutex;

    /// Do whatever `collect` does by default.
    pub fn collect<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        pi.collect()
    }

    /// Use a system mutex.
    pub fn mutex<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let mutex = Mutex::new(HashMap::new());
        pi.for_each(|(k, v)| {
            let mut guard = mutex.lock().unwrap();
            guard.insert(k, v);
        });
        mutex.into_inner().unwrap()
    }

    /// Use a linked list intermediary.
    pub fn linked_list<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let list: LinkedList<(_, _)> = pi.collect();
        list.into_iter().collect()
    }

    /// Use a linked list of vectors intermediary.
    pub fn linked_list_vec<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let list: LinkedList<Vec<(_, _)>> = pi
            .fold(|| Vec::new(),
                  |mut vec, elem| { vec.push(elem); vec })
            .collect();
        list.into_iter().flat_map(Vec::into_iter).collect()
    }

    /// Fold into hashmaps and then reduce them together.
    pub fn fold<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        pi.fold(|| HashMap::new(),
                |mut map, (k, v)| {
                    map.insert(k, v);
                    map
                })
          .reduce(|| HashMap::new(),
                  |mut map1, mut map2| {
                      if map1.len() > map2.len() {
                          map1.extend(map2);
                          map1
                      } else {
                          map2.extend(map1);
                          map2
                      }
                  })
    }
}

macro_rules! make_bench {
    ($generate:ident, $check:ident) => {
        #[bench]
        fn with_collect(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::collect($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_mutex(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::mutex($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_linked_list(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::linked_list($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_linked_list_vec(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::linked_list_vec($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_fold(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::fold($generate())));
            $check(&map.unwrap());
        }
    }
}

/// Tests a big map mapping `i -> i` forall i in 0 to N. This map is
/// interesting because it has no conflicts, so each parallel
/// iteration adds a distinct entry into the map.
mod i_to_i {
    use rayon::prelude::*;
    use std::collections::HashMap;

    const N: u32 = 256 * 1024;

    fn generate() -> impl ParallelIterator<Item=(u32, u32)> {
        (0_u32..N)
            .into_par_iter()
            .map(|i| (i, i))
    }

    fn check(hm: &HashMap<u32, u32>) {
        assert_eq!(hm.len(), N as usize);
        for i in 0..N {
            assert_eq!(hm[&i], i);
        }
    }

    make_bench!(generate, check);
}

/// Tests a big map mapping `i % 10 -> i` forall i in 0 to N. This map
/// is interesting because it has lots of conflicts, so parallel
/// iterations sometimes overwrite entries.
mod i_mod_10_to_i {
    use rayon::prelude::*;
    use std::collections::HashMap;

    const N: u32 = 256 * 1024;

    fn generate() -> impl ParallelIterator<Item=(u32, u32)> {
        (0_u32..N)
            .into_par_iter()
            .map(|i| (i % 10, i))
    }

    fn check(hm: &HashMap<u32, u32>) {
        assert_eq!(hm.len(), 10);
        for (&k, &v) in hm {
            assert_eq!(k, v % 10);
        }
    }

    make_bench!(generate, check);
}
<commit_msg>Add more map_collect benchmark alternatives<commit_after>//! Some benchmarks stress-testing various ways to build the standard
//! `HashMap` data structures from the standard library.

mod util {
    use rayon::prelude::*;
    use std::collections::{LinkedList, HashMap};
    use std::hash::Hash;
    use std::iter::FromIterator;
    use std::sync::Mutex;

    /// Do whatever `collect` does by default.
    pub fn collect<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        pi.collect()
    }

    /// Use a system mutex.
    pub fn mutex<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let mutex = Mutex::new(HashMap::new());
        pi.for_each(|(k, v)| {
            let mut guard = mutex.lock().unwrap();
            guard.insert(k, v);
        });
        mutex.into_inner().unwrap()
    }

    /// Use a system mutex over a folded vec.
    pub fn mutex_vec<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let mutex = Mutex::new(HashMap::new());
        pi.fold(|| Vec::new(),
                |mut vec, elem| { vec.push(elem); vec })
          .for_each(|vec| {
              let mut guard = mutex.lock().unwrap();
              guard.extend(vec);
          });
        mutex.into_inner().unwrap()
    }

    /// Use a linked list intermediary.
    pub fn linked_list<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let list: LinkedList<(_, _)> = pi.collect();
        list.into_iter().collect()
    }

    /// Use a linked list of vectors intermediary.
    pub fn linked_list_vec<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let list: LinkedList<Vec<(_, _)>> = pi
            .fold(|| Vec::new(),
                  |mut vec, elem| { vec.push(elem); vec })
            .collect();
        list.into_iter()
            .fold(HashMap::new(),
                  |mut map, vec| { map.extend(vec); map })
    }

    /// Use a linked list of vectors intermediary, with a size hint.
    pub fn linked_list_vec_sized<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        let list: LinkedList<Vec<(_, _)>> = pi
            .fold(|| Vec::new(),
                  |mut vec, elem| { vec.push(elem); vec })
            .collect();

        let len = list.iter().map(Vec::len).sum();
        list.into_iter()
            .fold(HashMap::with_capacity(len),
                  |mut map, vec| { map.extend(vec); map })
    }

    /// Fold into hashmaps and then reduce them together.
    pub fn fold<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        pi.fold(|| HashMap::new(),
                |mut map, (k, v)| {
                    map.insert(k, v);
                    map
                })
          .reduce(|| HashMap::new(),
                  |mut map1, mut map2| {
                      if map1.len() > map2.len() {
                          map1.extend(map2);
                          map1
                      } else {
                          map2.extend(map1);
                          map2
                      }
                  })
    }

    /// Fold into vecs and then reduce them together as hashmaps.
    pub fn fold_vec<K, V, PI>(pi: PI) -> HashMap<K, V>
        where K: Send + Hash + Eq,
              V: Send,
              PI: ParallelIterator<Item = (K, V)> + Send
    {
        pi.fold(|| Vec::new(),
                |mut vec, elem| { vec.push(elem); vec })
          .map(HashMap::from_iter)
          .reduce(|| HashMap::new(),
                  |mut map1, map2| {
                      map1.extend(map2);
                      map1
                  })
    }
}

macro_rules! make_bench {
    ($generate:ident, $check:ident) => {
        #[bench]
        fn with_collect(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::collect($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_mutex(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::mutex($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_mutex_vec(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::mutex_vec($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_linked_list(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::linked_list($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_linked_list_vec(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::linked_list_vec($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_linked_list_vec_sized(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::linked_list_vec_sized($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_fold(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::fold($generate())));
            $check(&map.unwrap());
        }

        #[bench]
        fn with_fold_vec(b: &mut ::test::Bencher) {
            use map_collect::util;
            let mut map = None;
            b.iter(|| map = Some(util::fold_vec($generate())));
            $check(&map.unwrap());
        }
    }
}

/// Tests a big map mapping `i -> i` forall i in 0 to N. This map is
/// interesting because it has no conflicts, so each parallel
/// iteration adds a distinct entry into the map.
mod i_to_i {
    use rayon::prelude::*;
    use std::collections::HashMap;

    const N: u32 = 256 * 1024;

    fn generate() -> impl ParallelIterator<Item=(u32, u32)> {
        (0_u32..N)
            .into_par_iter()
            .map(|i| (i, i))
    }

    fn check(hm: &HashMap<u32, u32>) {
        assert_eq!(hm.len(), N as usize);
        for i in 0..N {
            assert_eq!(hm[&i], i);
        }
    }

    make_bench!(generate, check);
}

/// Tests a big map mapping `i % 10 -> i` forall i in 0 to N. This map
/// is interesting because it has lots of conflicts, so parallel
/// iterations sometimes overwrite entries.
mod i_mod_10_to_i {
    use rayon::prelude::*;
    use std::collections::HashMap;

    const N: u32 = 256 * 1024;

    fn generate() -> impl ParallelIterator<Item=(u32, u32)> {
        (0_u32..N)
            .into_par_iter()
            .map(|i| (i % 10, i))
    }

    fn check(hm: &HashMap<u32, u32>) {
        assert_eq!(hm.len(), 10);
        for (&k, &v) in hm {
            assert_eq!(k, v % 10);
        }
    }

    make_bench!(generate, check);
}
<|endoftext|>


<commit_before>//! An iterator over the type substructure.
//! WARNING: this does not keep track of the region depth.

use crate::ty;
use crate::ty::subst::{GenericArg, GenericArgKind};
use smallvec::{self, SmallVec};

// The TypeWalker's stack is hot enough that it's worth going to some effort to
// avoid heap allocations.
type TypeWalkerStack<'tcx> = SmallVec<[GenericArg<'tcx>; 8]>;

pub struct TypeWalker<'tcx> {
    stack: TypeWalkerStack<'tcx>,
    last_subtree: usize,
}

impl<'tcx> TypeWalker<'tcx> {
    pub fn new(root: GenericArg<'tcx>) -> TypeWalker<'tcx> {
        TypeWalker { stack: smallvec![root], last_subtree: 1 }
    }

    /// Skips the subtree corresponding to the last type
    /// returned by `next()`.
    ///
    /// Example: Imagine you are walking `Foo<Bar<i32>, usize>`.
    ///
    /// ```
    /// let mut iter: TypeWalker = ...;
    /// iter.next(); // yields Foo
    /// iter.next(); // yields Bar<i32>
    /// iter.skip_current_subtree(); // skips i32
    /// iter.next(); // yields usize
    /// ```
    pub fn skip_current_subtree(&mut self) {
        self.stack.truncate(self.last_subtree);
    }
}

impl<'tcx> Iterator for TypeWalker<'tcx> {
    type Item = GenericArg<'tcx>;

    fn next(&mut self) -> Option<GenericArg<'tcx>> {
        debug!("next(): stack={:?}", self.stack);
        let next = self.stack.pop()?;
        self.last_subtree = self.stack.len();
        push_inner(&mut self.stack, next);
        debug!("next: stack={:?}", self.stack);
        Some(next)
    }
}

impl GenericArg<'tcx> {
    /// Iterator that walks `self` and any types reachable from
    /// `self`, in depth-first order. Note that just walks the types
    /// that appear in `self`, it does not descend into the fields of
    /// structs or variants. For example:
    ///
    /// ```notrust
    /// isize => { isize }
    /// Foo<Bar<isize>> => { Foo<Bar<isize>>, Bar<isize>, isize }
    /// [isize] => { [isize], isize }
    /// ```
    pub fn walk(self) -> TypeWalker<'tcx> {
        TypeWalker::new(self)
    }

    /// Iterator that walks the immediate children of `self`. Hence
    /// `Foo<Bar<i32>, u32>` yields the sequence `[Bar<i32>, u32]`
    /// (but not `i32`, like `walk`).
    pub fn walk_shallow(self) -> impl Iterator<Item = GenericArg<'tcx>> {
        let mut stack = SmallVec::new();
        push_inner(&mut stack, self);
        stack.into_iter()
    }
}

impl<'tcx> super::TyS<'tcx> {
    /// Iterator that walks `self` and any types reachable from
    /// `self`, in depth-first order. Note that just walks the types
    /// that appear in `self`, it does not descend into the fields of
    /// structs or variants. For example:
    ///
    /// ```notrust
    /// isize => { isize }
    /// Foo<Bar<isize>> => { Foo<Bar<isize>>, Bar<isize>, isize }
    /// [isize] => { [isize], isize }
    /// ```
    pub fn walk(&'tcx self) -> TypeWalker<'tcx> {
        TypeWalker::new(self.into())
    }
}

// We push `GenericArg`s on the stack in reverse order so as to
// maintain a pre-order traversal. As of the time of this
// writing, the fact that the traversal is pre-order is not
// known to be significant to any code, but it seems like the
// natural order one would expect (basically, the order of the
// types as they are written).
fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>) {
    match parent.unpack() {
        GenericArgKind::Type(parent_ty) => match parent_ty.kind {
            ty::Bool
            | ty::Char
            | ty::Int(_)
            | ty::Uint(_)
            | ty::Float(_)
            | ty::Str
            | ty::Infer(_)
            | ty::Param(_)
            | ty::Never
            | ty::Error(_)
            | ty::Placeholder(..)
            | ty::Bound(..)
            | ty::Foreign(..) => {}

            ty::Array(ty, len) => {
                stack.push(len.into());
                stack.push(ty.into());
            }
            ty::Slice(ty) => {
                stack.push(ty.into());
            }
            ty::RawPtr(mt) => {
                stack.push(mt.ty.into());
            }
            ty::Ref(lt, ty, _) => {
                stack.push(ty.into());
                stack.push(lt.into());
            }
            ty::Projection(data) => {
                stack.extend(data.substs.iter().rev());
            }
            ty::Dynamic(obj, lt) => {
                stack.push(lt.into());
                stack.extend(obj.iter().rev().flat_map(|predicate| {
                    let (substs, opt_ty) = match predicate.skip_binder() {
                        ty::ExistentialPredicate::Trait(tr) => (tr.substs, None),
                        ty::ExistentialPredicate::Projection(p) => (p.substs, Some(p.ty)),
                        ty::ExistentialPredicate::AutoTrait(_) =>
                        // Empty iterator
                        {
                            (ty::InternalSubsts::empty(), None)
                        }
                    };

                    substs.iter().rev().chain(opt_ty.map(|ty| ty.into()))
                }));
            }
            ty::Adt(_, substs)
            | ty::Opaque(_, substs)
            | ty::Closure(_, substs)
            | ty::Generator(_, substs, _)
            | ty::Tuple(substs)
            | ty::FnDef(_, substs) => {
                stack.extend(substs.iter().rev());
            }
            ty::GeneratorWitness(ts) => {
                stack.extend(ts.skip_binder().iter().rev().map(|ty| ty.into()));
            }
            ty::FnPtr(sig) => {
                stack.push(sig.skip_binder().output().into());
                stack.extend(sig.skip_binder().inputs().iter().copied().rev().map(|ty| ty.into()));
            }
        },
        GenericArgKind::Lifetime(_) => {}
        GenericArgKind::Const(parent_ct) => {
            stack.push(parent_ct.ty.into());
            match parent_ct.val {
                ty::ConstKind::Infer(_)
                | ty::ConstKind::Param(_)
                | ty::ConstKind::Placeholder(_)
                | ty::ConstKind::Bound(..)
                | ty::ConstKind::Value(_)
                | ty::ConstKind::Error(_) => {}

                ty::ConstKind::Unevaluated(_, substs, _) => {
                    stack.extend(substs.iter().rev());
                }
            }
        }
    }
}
<commit_msg>Remove notrust in rustc_middle<commit_after>//! An iterator over the type substructure.
//! WARNING: this does not keep track of the region depth.

use crate::ty;
use crate::ty::subst::{GenericArg, GenericArgKind};
use smallvec::{self, SmallVec};

// The TypeWalker's stack is hot enough that it's worth going to some effort to
// avoid heap allocations.
type TypeWalkerStack<'tcx> = SmallVec<[GenericArg<'tcx>; 8]>;

pub struct TypeWalker<'tcx> {
    stack: TypeWalkerStack<'tcx>,
    last_subtree: usize,
}

impl<'tcx> TypeWalker<'tcx> {
    pub fn new(root: GenericArg<'tcx>) -> TypeWalker<'tcx> {
        TypeWalker { stack: smallvec![root], last_subtree: 1 }
    }

    /// Skips the subtree corresponding to the last type
    /// returned by `next()`.
    ///
    /// Example: Imagine you are walking `Foo<Bar<i32>, usize>`.
    ///
    /// ```
    /// let mut iter: TypeWalker = ...;
    /// iter.next(); // yields Foo
    /// iter.next(); // yields Bar<i32>
    /// iter.skip_current_subtree(); // skips i32
    /// iter.next(); // yields usize
    /// ```
    pub fn skip_current_subtree(&mut self) {
        self.stack.truncate(self.last_subtree);
    }
}

impl<'tcx> Iterator for TypeWalker<'tcx> {
    type Item = GenericArg<'tcx>;

    fn next(&mut self) -> Option<GenericArg<'tcx>> {
        debug!("next(): stack={:?}", self.stack);
        let next = self.stack.pop()?;
        self.last_subtree = self.stack.len();
        push_inner(&mut self.stack, next);
        debug!("next: stack={:?}", self.stack);
        Some(next)
    }
}

impl GenericArg<'tcx> {
    /// Iterator that walks `self` and any types reachable from
    /// `self`, in depth-first order. Note that just walks the types
    /// that appear in `self`, it does not descend into the fields of
    /// structs or variants. For example:
    ///
    /// ```text
    /// isize => { isize }
    /// Foo<Bar<isize>> => { Foo<Bar<isize>>, Bar<isize>, isize }
    /// [isize] => { [isize], isize }
    /// ```
    pub fn walk(self) -> TypeWalker<'tcx> {
        TypeWalker::new(self)
    }

    /// Iterator that walks the immediate children of `self`. Hence
    /// `Foo<Bar<i32>, u32>` yields the sequence `[Bar<i32>, u32]`
    /// (but not `i32`, like `walk`).
    pub fn walk_shallow(self) -> impl Iterator<Item = GenericArg<'tcx>> {
        let mut stack = SmallVec::new();
        push_inner(&mut stack, self);
        stack.into_iter()
    }
}

impl<'tcx> super::TyS<'tcx> {
    /// Iterator that walks `self` and any types reachable from
    /// `self`, in depth-first order. Note that just walks the types
    /// that appear in `self`, it does not descend into the fields of
    /// structs or variants. For example:
    ///
    /// ```text
    /// isize => { isize }
    /// Foo<Bar<isize>> => { Foo<Bar<isize>>, Bar<isize>, isize }
    /// [isize] => { [isize], isize }
    /// ```
    pub fn walk(&'tcx self) -> TypeWalker<'tcx> {
        TypeWalker::new(self.into())
    }
}

// We push `GenericArg`s on the stack in reverse order so as to
// maintain a pre-order traversal. As of the time of this
// writing, the fact that the traversal is pre-order is not
// known to be significant to any code, but it seems like the
// natural order one would expect (basically, the order of the
// types as they are written).
fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>) {
    match parent.unpack() {
        GenericArgKind::Type(parent_ty) => match parent_ty.kind {
            ty::Bool
            | ty::Char
            | ty::Int(_)
            | ty::Uint(_)
            | ty::Float(_)
            | ty::Str
            | ty::Infer(_)
            | ty::Param(_)
            | ty::Never
            | ty::Error(_)
            | ty::Placeholder(..)
            | ty::Bound(..)
            | ty::Foreign(..) => {}

            ty::Array(ty, len) => {
                stack.push(len.into());
                stack.push(ty.into());
            }
            ty::Slice(ty) => {
                stack.push(ty.into());
            }
            ty::RawPtr(mt) => {
                stack.push(mt.ty.into());
            }
            ty::Ref(lt, ty, _) => {
                stack.push(ty.into());
                stack.push(lt.into());
            }
            ty::Projection(data) => {
                stack.extend(data.substs.iter().rev());
            }
            ty::Dynamic(obj, lt) => {
                stack.push(lt.into());
                stack.extend(obj.iter().rev().flat_map(|predicate| {
                    let (substs, opt_ty) = match predicate.skip_binder() {
                        ty::ExistentialPredicate::Trait(tr) => (tr.substs, None),
                        ty::ExistentialPredicate::Projection(p) => (p.substs, Some(p.ty)),
                        ty::ExistentialPredicate::AutoTrait(_) =>
                        // Empty iterator
                        {
                            (ty::InternalSubsts::empty(), None)
                        }
                    };

                    substs.iter().rev().chain(opt_ty.map(|ty| ty.into()))
                }));
            }
            ty::Adt(_, substs)
            | ty::Opaque(_, substs)
            | ty::Closure(_, substs)
            | ty::Generator(_, substs, _)
            | ty::Tuple(substs)
            | ty::FnDef(_, substs) => {
                stack.extend(substs.iter().rev());
            }
            ty::GeneratorWitness(ts) => {
                stack.extend(ts.skip_binder().iter().rev().map(|ty| ty.into()));
            }
            ty::FnPtr(sig) => {
                stack.push(sig.skip_binder().output().into());
                stack.extend(sig.skip_binder().inputs().iter().copied().rev().map(|ty| ty.into()));
            }
        },
        GenericArgKind::Lifetime(_) => {}
        GenericArgKind::Const(parent_ct) => {
            stack.push(parent_ct.ty.into());
            match parent_ct.val {
                ty::ConstKind::Infer(_)
                | ty::ConstKind::Param(_)
                | ty::ConstKind::Placeholder(_)
                | ty::ConstKind::Bound(..)
                | ty::ConstKind::Value(_)
                | ty::ConstKind::Error(_) => {}

                ty::ConstKind::Unevaluated(_, substs, _) => {
                    stack.extend(substs.iter().rev());
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use crate::command_prelude::*;

use cargo::ops::{self, CompileFilter, FilterRule, LibRule};

pub fn cli() -> App {
    subcommand("fix")
        .about("Automatically fix lint warnings reported by rustc")
        .arg(opt("quiet", "No output printed to stdout").short("q"))
        .arg_package_spec(
            "Package(s) to fix",
            "Fix all packages in the workspace",
            "Exclude packages from the fixes",
        )
        .arg_jobs()
        .arg_targets_all(
            "Fix only this package's library",
            "Fix only the specified binary",
            "Fix all binaries",
            "Fix only the specified example",
            "Fix all examples",
            "Fix only the specified test target",
            "Fix all tests",
            "Fix only the specified bench target",
            "Fix all benches",
            "Fix all targets (default)",
        )
        .arg_release("Fix artifacts in release mode, with optimizations")
        .arg(opt("profile", "Profile to build the selected target for").value_name("PROFILE"))
        .arg_features()
        .arg_target_triple("Fix for the target triple")
        .arg_target_dir()
        .arg_manifest_path()
        .arg_message_format()
        .arg(
            Arg::with_name("broken-code")
                .long("broken-code")
                .help("Fix code even if it already has compiler errors"),
        )
        .arg(
            Arg::with_name("edition")
                .long("edition")
                .help("Fix in preparation for the next edition"),
        )
        .arg(
            // This is a deprecated argument, we'll want to phase it out
            // eventually.
            Arg::with_name("prepare-for")
                .long("prepare-for")
                .help("Fix warnings in preparation of an edition upgrade")
                .takes_value(true)
                .possible_values(&["2018"])
                .conflicts_with("edition")
                .hidden(true),
        )
        .arg(
            Arg::with_name("idioms")
                .long("edition-idioms")
                .help("Fix warnings to migrate to the idioms of an edition"),
        )
        .arg(
            Arg::with_name("allow-no-vcs")
                .long("allow-no-vcs")
                .help("Fix code even if a VCS was not detected"),
        )
        .arg(
            Arg::with_name("allow-dirty")
                .long("allow-dirty")
                .help("Fix code even if the working directory is dirty"),
        )
        .arg(
            Arg::with_name("allow-staged")
                .long("allow-staged")
                .help("Fix code even if the working directory has staged changes"),
        )
        .arg(
            Arg::with_name("clippy")
                .long("clippy")
                .help("Get fix suggestions from clippy instead of check")
                .hidden(true),
        )
        .arg(
            Arg::with_name("clippy-arg")
                .long("clippy-arg")
                .help("Args to pass through to clippy, implies --clippy")
                .hidden(true)
                .multiple(true)
                .number_of_values(1),
        )
        .after_help(
            "\
This Cargo subcommand will automatically take rustc's suggestions from
diagnostics like warnings and apply them to your source code. This is intended
to help automate tasks that rustc itself already knows how to tell you to fix!
The `cargo fix` subcommand is also being developed for the Rust 2018 edition
to provide code the ability to easily opt-in to the new edition without having
to worry about any breakage.

Executing `cargo fix` will under the hood execute `cargo check`. Any warnings
applicable to your crate will be automatically fixed (if possible) and all
remaining warnings will be displayed when the check process is finished. For
example if you'd like to prepare for the 2018 edition, you can do so by
executing:

    cargo fix --edition

which behaves the same as `cargo check --all-targets`. Similarly if you'd like
to fix code for different platforms you can do:

    cargo fix --edition --target x86_64-pc-windows-gnu

or if your crate has optional features:

    cargo fix --edition --no-default-features --features foo

If you encounter any problems with `cargo fix` or otherwise have any questions
or feature requests please don't hesitate to file an issue at
https://github.com/rust-lang/cargo
",
        )
}

pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
    let ws = args.workspace(config)?;
    let test = match args.value_of("profile") {
        Some("test") => true,
        None => false,
        Some(profile) => {
            let err = failure::format_err!(
                "unknown profile: `{}`, only `test` is \
                 currently supported",
                profile
            );
            return Err(CliError::new(err, 101));
        }
    };
    let mode = CompileMode::Check { test };

    // Unlike other commands default `cargo fix` to all targets to fix as much
    // code as we can.
    let mut opts = args.compile_options(config, mode, Some(&ws))?;

    let clippy_args = args.values_of_lossy("clippy-args");
    let use_clippy = args.is_present("clippy") || clippy_args.is_some();

    if use_clippy && !config.cli_unstable().unstable_options {
        return Err(failure::format_err!(
            "`cargo fix --clippy` is unstable, pass `-Z unstable-options` to enable it"
        )
        .into());
    }

    if let CompileFilter::Default { .. } = opts.filter {
        opts.filter = CompileFilter::Only {
            all_targets: true,
            lib: LibRule::Default,
            bins: FilterRule::All,
            examples: FilterRule::All,
            benches: FilterRule::All,
            tests: FilterRule::All,
        }
    }

    ops::fix(
        &ws,
        &mut ops::FixOptions {
            edition: args.is_present("edition"),
            prepare_for: args.value_of("prepare-for"),
            idioms: args.is_present("idioms"),
            compile_opts: opts,
            allow_dirty: args.is_present("allow-dirty"),
            allow_no_vcs: args.is_present("allow-no-vcs"),
            allow_staged: args.is_present("allow-staged"),
            broken_code: args.is_present("broken-code"),
            use_clippy,
            clippy_args,
        },
    )?;
    Ok(())
}
<commit_msg>Update src/bin/cargo/commands/fix.rs<commit_after>use crate::command_prelude::*;

use cargo::ops::{self, CompileFilter, FilterRule, LibRule};

pub fn cli() -> App {
    subcommand("fix")
        .about("Automatically fix lint warnings reported by rustc")
        .arg(opt("quiet", "No output printed to stdout").short("q"))
        .arg_package_spec(
            "Package(s) to fix",
            "Fix all packages in the workspace",
            "Exclude packages from the fixes",
        )
        .arg_jobs()
        .arg_targets_all(
            "Fix only this package's library",
            "Fix only the specified binary",
            "Fix all binaries",
            "Fix only the specified example",
            "Fix all examples",
            "Fix only the specified test target",
            "Fix all tests",
            "Fix only the specified bench target",
            "Fix all benches",
            "Fix all targets (default)",
        )
        .arg_release("Fix artifacts in release mode, with optimizations")
        .arg(opt("profile", "Profile to build the selected target for").value_name("PROFILE"))
        .arg_features()
        .arg_target_triple("Fix for the target triple")
        .arg_target_dir()
        .arg_manifest_path()
        .arg_message_format()
        .arg(
            Arg::with_name("broken-code")
                .long("broken-code")
                .help("Fix code even if it already has compiler errors"),
        )
        .arg(
            Arg::with_name("edition")
                .long("edition")
                .help("Fix in preparation for the next edition"),
        )
        .arg(
            // This is a deprecated argument, we'll want to phase it out
            // eventually.
            Arg::with_name("prepare-for")
                .long("prepare-for")
                .help("Fix warnings in preparation of an edition upgrade")
                .takes_value(true)
                .possible_values(&["2018"])
                .conflicts_with("edition")
                .hidden(true),
        )
        .arg(
            Arg::with_name("idioms")
                .long("edition-idioms")
                .help("Fix warnings to migrate to the idioms of an edition"),
        )
        .arg(
            Arg::with_name("allow-no-vcs")
                .long("allow-no-vcs")
                .help("Fix code even if a VCS was not detected"),
        )
        .arg(
            Arg::with_name("allow-dirty")
                .long("allow-dirty")
                .help("Fix code even if the working directory is dirty"),
        )
        .arg(
            Arg::with_name("allow-staged")
                .long("allow-staged")
                .help("Fix code even if the working directory has staged changes"),
        )
        .arg(
            Arg::with_name("clippy")
                .long("clippy")
                .help("Get fix suggestions from clippy instead of rustc")
                .hidden(true),
        )
        .arg(
            Arg::with_name("clippy-arg")
                .long("clippy-arg")
                .help("Args to pass through to clippy, implies --clippy")
                .hidden(true)
                .multiple(true)
                .number_of_values(1),
        )
        .after_help(
            "\
This Cargo subcommand will automatically take rustc's suggestions from
diagnostics like warnings and apply them to your source code. This is intended
to help automate tasks that rustc itself already knows how to tell you to fix!
The `cargo fix` subcommand is also being developed for the Rust 2018 edition
to provide code the ability to easily opt-in to the new edition without having
to worry about any breakage.

Executing `cargo fix` will under the hood execute `cargo check`. Any warnings
applicable to your crate will be automatically fixed (if possible) and all
remaining warnings will be displayed when the check process is finished. For
example if you'd like to prepare for the 2018 edition, you can do so by
executing:

    cargo fix --edition

which behaves the same as `cargo check --all-targets`. Similarly if you'd like
to fix code for different platforms you can do:

    cargo fix --edition --target x86_64-pc-windows-gnu

or if your crate has optional features:

    cargo fix --edition --no-default-features --features foo

If you encounter any problems with `cargo fix` or otherwise have any questions
or feature requests please don't hesitate to file an issue at
https://github.com/rust-lang/cargo
",
        )
}

pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
    let ws = args.workspace(config)?;
    let test = match args.value_of("profile") {
        Some("test") => true,
        None => false,
        Some(profile) => {
            let err = failure::format_err!(
                "unknown profile: `{}`, only `test` is \
                 currently supported",
                profile
            );
            return Err(CliError::new(err, 101));
        }
    };
    let mode = CompileMode::Check { test };

    // Unlike other commands default `cargo fix` to all targets to fix as much
    // code as we can.
    let mut opts = args.compile_options(config, mode, Some(&ws))?;

    let clippy_args = args.values_of_lossy("clippy-args");
    let use_clippy = args.is_present("clippy") || clippy_args.is_some();

    if use_clippy && !config.cli_unstable().unstable_options {
        return Err(failure::format_err!(
            "`cargo fix --clippy` is unstable, pass `-Z unstable-options` to enable it"
        )
        .into());
    }

    if let CompileFilter::Default { .. } = opts.filter {
        opts.filter = CompileFilter::Only {
            all_targets: true,
            lib: LibRule::Default,
            bins: FilterRule::All,
            examples: FilterRule::All,
            benches: FilterRule::All,
            tests: FilterRule::All,
        }
    }

    ops::fix(
        &ws,
        &mut ops::FixOptions {
            edition: args.is_present("edition"),
            prepare_for: args.value_of("prepare-for"),
            idioms: args.is_present("idioms"),
            compile_opts: opts,
            allow_dirty: args.is_present("allow-dirty"),
            allow_no_vcs: args.is_present("allow-no-vcs"),
            allow_staged: args.is_present("allow-staged"),
            broken_code: args.is_present("broken-code"),
            use_clippy,
            clippy_args,
        },
    )?;
    Ok(())
}
<|endoftext|>