<commit_before>use std::str;
use std::collections::HashMap;

use rocksdb::{Writable, IteratorMode, Direction, WriteBatch};
use byteorder::{ByteOrder, BigEndian, WriteBytesExt};

use RocksDBIndexStore;
use errors::{RocksDBReadError, RocksDBWriteError};
use document_index::DocRef;
use key_builder::KeyBuilder;
use search::doc_id_set::DocIdSet;

#[derive(Debug)]
pub enum SegmentMergeError {
    TooManyDocs,
    RocksDBReadError(RocksDBReadError),
    RocksDBWriteError(RocksDBWriteError),
}

impl From<RocksDBReadError> for SegmentMergeError {
    fn from(e: RocksDBReadError) -> SegmentMergeError {
        SegmentMergeError::RocksDBReadError(e)
    }
}

impl From<RocksDBWriteError> for SegmentMergeError {
    fn from(e: RocksDBWriteError) -> SegmentMergeError {
        SegmentMergeError::RocksDBWriteError(e)
    }
}

impl From<SegmentMergeError> for String {
    fn from(e: SegmentMergeError) -> String {
        format!("{:?}", e).to_string()
    }
}

impl RocksDBIndexStore {
    fn merge_segment_data(&self, source_segments: &Vec<u32>, dest_segment: u32, doc_ref_mapping: &HashMap<DocRef, u16>) -> Result<(), SegmentMergeError> {
        // Merge the term directories
        // The term directory keys are ordered to be most convenient for retrieving all the segments
        // of for a term/field combination in one go (field/term/segment). So we don't end up pulling
        // in a lot of unwanted data, we firstly iterate the keys, it they one of the source segments
        // looking for then we load them and append them to our new segment.

        /// Converts term directory key strings "d1/2/3" into tuples of 3 i32s (1, 2, 3)
        fn parse_term_directory_key(key: &[u8]) -> (u32, u32, u32) {
            let mut nums_iter = key[1..].split(|b| *b == b'/').map(|s| str::from_utf8(s).unwrap().parse::<u32>().unwrap());
            (nums_iter.next().unwrap(), nums_iter.next().unwrap(), nums_iter.next().unwrap())
        }

        let mut current_td_key: Option<(u32, u32)> = None;
        let mut current_td = Vec::new();

        for k in self.db.keys_iterator(IteratorMode::From(b"d", Direction::Forward)) {
            if k[0] != b'd' {
                // No more term directories to merge
                break;
            }

            let (field, term, segment) = parse_term_directory_key(&k);

            if !source_segments.contains(&segment) {
                continue;
            }

            if current_td_key != Some((field, term)) {
                // Finished current term directory. Write it to the DB and start the next one
                if let Some((field, term)) = current_td_key {
                    let kb = KeyBuilder::segment_dir_list(dest_segment, field, term);
                    if let Err(e) = self.db.put(&kb.key(), &current_td) {
                        return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
                    }
                    current_td.clear();
                }

                current_td_key = Some((field, term));
            }

            // Merge term directory into the new one (and remap the doc ids)
            match self.db.get(&k) {
                Ok(Some(docid_set)) => {
                    for doc_id in DocIdSet::FromRDB(docid_set).iter() {
                        let doc_ref = DocRef::from_segment_ord(segment, doc_id);
                        let new_doc_id = doc_ref_mapping.get(&doc_ref).unwrap();
                        current_td.write_u16::<BigEndian>(*new_doc_id).unwrap();
                    }
                }
                Ok(None) => {},  // FIXME
                Err(e) => return Err(RocksDBReadError::new(k.to_vec(), e).into())
            }
        }

        // All done, write the last term directory
        if let Some((field, term)) = current_td_key {
            let kb = KeyBuilder::segment_dir_list(dest_segment, field, term);
            if let Err(e) = self.db.put(&kb.key(), &current_td) {
                return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
            }
            current_td.clear();
        }

        // Merge the stored values
        // All stored value keys start with the segment id. So we need to:
        // - Iterate all stored value keys that are prefixed by one of the stored segment ids
        // - Remap their doc ids to the one in the new segment
        // - Write the value back with the new segment/doc ids in the key

        /// Converts stored value key strings "v1/2/3/v" into tuples of 3 i32s and a Vec<u8> (1, 2, 3, vec![b'v', b'a', b'l'])
        fn parse_stored_value_key(key: &[u8]) -> (u32, u32, u32, Vec<u8>) {
            let mut parts_iter = key[1..].split(|b| *b == b'/');
            let segment = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let doc_id = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let field_ord = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let value_type = parts_iter.next().unwrap().to_vec();

            (segment, doc_id, field_ord, value_type)
        }

        for source_segment in source_segments.iter() {
            let kb = KeyBuilder::segment_stored_values_prefix(*source_segment);
            for (k, v) in self.db.iterator(IteratorMode::From(&kb.key(), Direction::Forward)) {
                if k[0] != b'v' {
                    // No more stored values to move
                    break;
                }

                let (segment, doc_id, field, value_type) = parse_stored_value_key(&k);

                if segment != *source_segment {
                    // Segment finished
                    break;
                }

                // Remap doc id
                let doc_ref = DocRef::from_segment_ord(segment, doc_id as u16);
                let new_doc_id = doc_ref_mapping.get(&doc_ref).unwrap();

                // Write value into new segment
                let kb = KeyBuilder::stored_field_value(dest_segment, *new_doc_id, field, &value_type);
                if let Err(e) = self.db.put(&kb.key(), &v) {
                    return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
                }
            }
        }

        // Merge the statistics
        // Like stored values, these start with segment ids. But instead of just rewriting the
        // key, we need to sum up all the statistics across the segments being merged.

        let mut statistics = HashMap::new();

        /// Converts statistic key strings "s1/total_docs" into tuples of 1 i32 and a Vec<u8> (1, ['t', 'o', 't', ...])
        fn parse_statistic_key(key: &[u8]) -> (u32, Vec<u8>) {
            let mut parts_iter = key[1..].split(|b| *b == b'/');
            let segment = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let statistic_name = parts_iter.next().unwrap().to_vec();

            (segment, statistic_name)
        }

        // Fetch and merge statistics
        for k in self.db.keys_iterator(IteratorMode::From(b"s", Direction::Forward)) {
            if k[0] != b's' {
                // No more statistics to merge
                break;
            }

            let (segment, statistic_name) = parse_statistic_key(&k);

            if !source_segments.contains(&segment) {
                continue;
            }

            match self.db.get(&k) {
                Ok(Some(val_bytes)) => {
                    let value = BigEndian::read_i64(&val_bytes);

                    let mut stat = statistics.entry(statistic_name).or_insert(0);
                    *stat += value;
                }
                Ok(None) => {},  // FIXME
                Err(e) => return Err(RocksDBReadError::new(k.to_vec(), e).into())
            }
        }

        // Write merged statistics to new segment
        for (stat_name, stat_value) in statistics {
            let kb = KeyBuilder::segment_stat(dest_segment, &stat_name);
            let mut val_bytes = [0; 8];
            BigEndian::write_i64(&mut val_bytes, stat_value);
            if let Err(e) = self.db.put(&kb.key(), &val_bytes) {
                return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
            }
        }

        // Note: Don't merge the deletion lists
        // Deletion lists can change at any time so we must lock the "document index"
        // before merging them so they can't be altered during merge. we cannot lock
        // this until the commit phase though.

        Ok(())
    }

    fn commit_segment_merge(&self, source_segments: &Vec<u32>, dest_segment: u32, doc_ref_mapping: &HashMap<DocRef, u16>) -> Result<(), SegmentMergeError> {
        let write_batch = WriteBatch::default();

        // Activate new segment
        let kb = KeyBuilder::segment_active(dest_segment);
        if let Err(e) = write_batch.put(&kb.key(), b"") {
            return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
        }

        // Deactivate old segments
        for source_segment in source_segments.iter() {
            // Activate new segment
            let kb = KeyBuilder::segment_active(*source_segment);
            if let Err(e) = write_batch.delete(&kb.key()) {
                return Err(RocksDBWriteError::new_delete(kb.key().to_vec(), e).into());
            }
        }

        // Update document index and commit
        // This will write the write batch
        try!(self.document_index.commit_segment_merge(&self.db, write_batch, source_segments, dest_segment, doc_ref_mapping));

        Ok(())
    }

    pub fn merge_segments(&self, source_segments: Vec<u32>) -> Result<u32, SegmentMergeError> {
        let dest_segment = try!(self.segments.new_segment(&self.db));

        // Generate a mapping between the ids of the documents in the old segments to the new one
        // This packs the id spaces of the old segments together:
        // For example, say we have to merge 3 segments with 100 documents each:
        //  - The first segment's ids will be the same as before
        //  - The second segment's ids will be remapped to 100 - 199
        //  - The third segment's ids will be remapped to 200 - 299

        let mut doc_ref_mapping: HashMap<DocRef, u16> = HashMap::new();
        let mut current_ord: u32 = 0;

        for source_segment in source_segments.iter() {
            let kb = KeyBuilder::segment_stat(*source_segment, b"total_docs");
            let total_docs = match self.db.get(&kb.key()) {
                Ok(Some(total_docs_bytes)) => {
                    BigEndian::read_i64(&total_docs_bytes)
                }
                Ok(None) => continue,
                Err(e) => return Err(RocksDBReadError::new(kb.key().to_vec(), e).into())
            };

            for source_ord in 0..total_docs {
                if current_ord >= 65536 {
                    return Err(SegmentMergeError::TooManyDocs);
                }

                let from = DocRef::from_segment_ord(*source_segment, source_ord as u16);
                doc_ref_mapping.insert(from, current_ord as u16);
                current_ord += 1;
            }
        }

        // Merge segment data
        // Most of the heavy lifting happens here. This merges all the immutable parts of
        // the segment (which is everything but the deletion list). It does not activate the
        // segment.
        // This means that nothing bad will happen if it crashes half way through -- the
        // worst that could happen is we're left with a partially-written segment that we
        // have to clean up.
        try!(self.merge_segment_data(&source_segments, dest_segment, &doc_ref_mapping));

        // Commit the merge
        // This activates the new segment and updates the document index. Effectively committing
        // the merge.
        // Throughout this stage we need an exclusive lock to the document index. This is to
        // prevent documents in the source segments being deleted/updated so we don't accidentally
        // undelete them (this will block until the merge is complete so they delete/update from
        // the new segment).
        try!(self.commit_segment_merge(&source_segments, dest_segment, &doc_ref_mapping));

        Ok(dest_segment)
    }
}
<commit_msg>Use prefix seek when merging statistics<commit_after>use std::str;
use std::collections::HashMap;

use rocksdb::{Writable, IteratorMode, Direction, WriteBatch};
use byteorder::{ByteOrder, BigEndian, WriteBytesExt};

use RocksDBIndexStore;
use errors::{RocksDBReadError, RocksDBWriteError};
use document_index::DocRef;
use key_builder::KeyBuilder;
use search::doc_id_set::DocIdSet;

#[derive(Debug)]
pub enum SegmentMergeError {
    TooManyDocs,
    RocksDBReadError(RocksDBReadError),
    RocksDBWriteError(RocksDBWriteError),
}

impl From<RocksDBReadError> for SegmentMergeError {
    fn from(e: RocksDBReadError) -> SegmentMergeError {
        SegmentMergeError::RocksDBReadError(e)
    }
}

impl From<RocksDBWriteError> for SegmentMergeError {
    fn from(e: RocksDBWriteError) -> SegmentMergeError {
        SegmentMergeError::RocksDBWriteError(e)
    }
}

impl From<SegmentMergeError> for String {
    fn from(e: SegmentMergeError) -> String {
        format!("{:?}", e).to_string()
    }
}

impl RocksDBIndexStore {
    fn merge_segment_data(&self, source_segments: &Vec<u32>, dest_segment: u32, doc_ref_mapping: &HashMap<DocRef, u16>) -> Result<(), SegmentMergeError> {
        // Merge the term directories
        // The term directory keys are ordered to be most convenient for retrieving all the segments
        // of for a term/field combination in one go (field/term/segment). So we don't end up pulling
        // in a lot of unwanted data, we firstly iterate the keys, it they one of the source segments
        // looking for then we load them and append them to our new segment.

        /// Converts term directory key strings "d1/2/3" into tuples of 3 i32s (1, 2, 3)
        fn parse_term_directory_key(key: &[u8]) -> (u32, u32, u32) {
            let mut nums_iter = key[1..].split(|b| *b == b'/').map(|s| str::from_utf8(s).unwrap().parse::<u32>().unwrap());
            (nums_iter.next().unwrap(), nums_iter.next().unwrap(), nums_iter.next().unwrap())
        }

        let mut current_td_key: Option<(u32, u32)> = None;
        let mut current_td = Vec::new();

        for k in self.db.keys_iterator(IteratorMode::From(b"d", Direction::Forward)) {
            if k[0] != b'd' {
                // No more term directories to merge
                break;
            }

            let (field, term, segment) = parse_term_directory_key(&k);

            if !source_segments.contains(&segment) {
                continue;
            }

            if current_td_key != Some((field, term)) {
                // Finished current term directory. Write it to the DB and start the next one
                if let Some((field, term)) = current_td_key {
                    let kb = KeyBuilder::segment_dir_list(dest_segment, field, term);
                    if let Err(e) = self.db.put(&kb.key(), &current_td) {
                        return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
                    }
                    current_td.clear();
                }

                current_td_key = Some((field, term));
            }

            // Merge term directory into the new one (and remap the doc ids)
            match self.db.get(&k) {
                Ok(Some(docid_set)) => {
                    for doc_id in DocIdSet::FromRDB(docid_set).iter() {
                        let doc_ref = DocRef::from_segment_ord(segment, doc_id);
                        let new_doc_id = doc_ref_mapping.get(&doc_ref).unwrap();
                        current_td.write_u16::<BigEndian>(*new_doc_id).unwrap();
                    }
                }
                Ok(None) => {},  // FIXME
                Err(e) => return Err(RocksDBReadError::new(k.to_vec(), e).into())
            }
        }

        // All done, write the last term directory
        if let Some((field, term)) = current_td_key {
            let kb = KeyBuilder::segment_dir_list(dest_segment, field, term);
            if let Err(e) = self.db.put(&kb.key(), &current_td) {
                return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
            }
            current_td.clear();
        }

        // Merge the stored values
        // All stored value keys start with the segment id. So we need to:
        // - Iterate all stored value keys that are prefixed by one of the stored segment ids
        // - Remap their doc ids to the one in the new segment
        // - Write the value back with the new segment/doc ids in the key

        /// Converts stored value key strings "v1/2/3/v" into tuples of 3 i32s and a Vec<u8> (1, 2, 3, vec![b'v', b'a', b'l'])
        fn parse_stored_value_key(key: &[u8]) -> (u32, u32, u32, Vec<u8>) {
            let mut parts_iter = key[1..].split(|b| *b == b'/');
            let segment = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let doc_id = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let field_ord = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let value_type = parts_iter.next().unwrap().to_vec();

            (segment, doc_id, field_ord, value_type)
        }

        for source_segment in source_segments.iter() {
            let kb = KeyBuilder::segment_stored_values_prefix(*source_segment);
            for (k, v) in self.db.iterator(IteratorMode::From(&kb.key(), Direction::Forward)) {
                if k[0] != b'v' {
                    // No more stored values to move
                    break;
                }

                let (segment, doc_id, field, value_type) = parse_stored_value_key(&k);

                if segment != *source_segment {
                    // Segment finished
                    break;
                }

                // Remap doc id
                let doc_ref = DocRef::from_segment_ord(segment, doc_id as u16);
                let new_doc_id = doc_ref_mapping.get(&doc_ref).unwrap();

                // Write value into new segment
                let kb = KeyBuilder::stored_field_value(dest_segment, *new_doc_id, field, &value_type);
                if let Err(e) = self.db.put(&kb.key(), &v) {
                    return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
                }
            }
        }

        // Merge the statistics
        // Like stored values, these start with segment ids. But instead of just rewriting the
        // key, we need to sum up all the statistics across the segments being merged.

        let mut statistics = HashMap::new();

        /// Converts statistic key strings "s1/total_docs" into tuples of 1 i32 and a Vec<u8> (1, ['t', 'o', 't', ...])
        fn parse_statistic_key(key: &[u8]) -> (u32, Vec<u8>) {
            let mut parts_iter = key[1..].split(|b| *b == b'/');
            let segment = str::from_utf8(parts_iter.next().unwrap()).unwrap().parse::<u32>().unwrap();
            let statistic_name = parts_iter.next().unwrap().to_vec();

            (segment, statistic_name)
        }

        // Fetch and merge statistics
        for source_segment in source_segments.iter() {
            let kb = KeyBuilder::segment_stat_prefix(*source_segment);
            for (k, v) in self.db.iterator(IteratorMode::From(&kb.key(), Direction::Forward)) {
                if k[0] != b's' {
                    // No more statistics to merge
                    break;
                }

                let (segment, statistic_name) = parse_statistic_key(&k);

                if segment != *source_segment {
                    // Segment finished
                    break;
                }

                let mut stat = statistics.entry(statistic_name).or_insert(0);
                *stat += BigEndian::read_i64(&v);
            }
        }

        // Write merged statistics to new segment
        for (stat_name, stat_value) in statistics {
            let kb = KeyBuilder::segment_stat(dest_segment, &stat_name);
            let mut val_bytes = [0; 8];
            BigEndian::write_i64(&mut val_bytes, stat_value);
            if let Err(e) = self.db.put(&kb.key(), &val_bytes) {
                return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
            }
        }

        // Note: Don't merge the deletion lists
        // Deletion lists can change at any time so we must lock the "document index"
        // before merging them so they can't be altered during merge. we cannot lock
        // this until the commit phase though.

        Ok(())
    }

    fn commit_segment_merge(&self, source_segments: &Vec<u32>, dest_segment: u32, doc_ref_mapping: &HashMap<DocRef, u16>) -> Result<(), SegmentMergeError> {
        let write_batch = WriteBatch::default();

        // Activate new segment
        let kb = KeyBuilder::segment_active(dest_segment);
        if let Err(e) = write_batch.put(&kb.key(), b"") {
            return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
        }

        // Deactivate old segments
        for source_segment in source_segments.iter() {
            // Activate new segment
            let kb = KeyBuilder::segment_active(*source_segment);
            if let Err(e) = write_batch.delete(&kb.key()) {
                return Err(RocksDBWriteError::new_delete(kb.key().to_vec(), e).into());
            }
        }

        // Update document index and commit
        // This will write the write batch
        try!(self.document_index.commit_segment_merge(&self.db, write_batch, source_segments, dest_segment, doc_ref_mapping));

        Ok(())
    }

    pub fn merge_segments(&self, source_segments: Vec<u32>) -> Result<u32, SegmentMergeError> {
        let dest_segment = try!(self.segments.new_segment(&self.db));

        // Generate a mapping between the ids of the documents in the old segments to the new one
        // This packs the id spaces of the old segments together:
        // For example, say we have to merge 3 segments with 100 documents each:
        //  - The first segment's ids will be the same as before
        //  - The second segment's ids will be remapped to 100 - 199
        //  - The third segment's ids will be remapped to 200 - 299

        let mut doc_ref_mapping: HashMap<DocRef, u16> = HashMap::new();
        let mut current_ord: u32 = 0;

        for source_segment in source_segments.iter() {
            let kb = KeyBuilder::segment_stat(*source_segment, b"total_docs");
            let total_docs = match self.db.get(&kb.key()) {
                Ok(Some(total_docs_bytes)) => {
                    BigEndian::read_i64(&total_docs_bytes)
                }
                Ok(None) => continue,
                Err(e) => return Err(RocksDBReadError::new(kb.key().to_vec(), e).into())
            };

            for source_ord in 0..total_docs {
                if current_ord >= 65536 {
                    return Err(SegmentMergeError::TooManyDocs);
                }

                let from = DocRef::from_segment_ord(*source_segment, source_ord as u16);
                doc_ref_mapping.insert(from, current_ord as u16);
                current_ord += 1;
            }
        }

        // Merge segment data
        // Most of the heavy lifting happens here. This merges all the immutable parts of
        // the segment (which is everything but the deletion list). It does not activate the
        // segment.
        // This means that nothing bad will happen if it crashes half way through -- the
        // worst that could happen is we're left with a partially-written segment that we
        // have to clean up.
        try!(self.merge_segment_data(&source_segments, dest_segment, &doc_ref_mapping));

        // Commit the merge
        // This activates the new segment and updates the document index. Effectively committing
        // the merge.
        // Throughout this stage we need an exclusive lock to the document index. This is to
        // prevent documents in the source segments being deleted/updated so we don't accidentally
        // undelete them (this will block until the merge is complete so they delete/update from
        // the new segment).
        try!(self.commit_segment_merge(&source_segments, dest_segment, &doc_ref_mapping));

        Ok(dest_segment)
    }
}
<|endoftext|>


<commit_before>extern crate e2d2;
use e2d2::state::*;
use std::str;
use std::slice;

/// Check rounding up to number of pages.
#[test]
fn round_pages_test() {
    assert!(ReorderedBuffer::round_to_pages(1) == 4096, "Rounding up 1 byte did not result in PAGE_SIZE");
    assert!(ReorderedBuffer::round_to_pages(0) == 0, "Rounding up 0 bytes did not result in 0");
    assert!(ReorderedBuffer::round_to_pages(8) == 4096, "Rounding up failure, expected 4096, got {}",
            ReorderedBuffer::round_to_pages(8));
    assert!(ReorderedBuffer::round_to_pages(512) == 4096, "Rounding up failure, expected 4096, got {}",
            ReorderedBuffer::round_to_pages(512));
    assert!(ReorderedBuffer::round_to_pages(4096) == 4096, "Rounding up exactly 1 page failed, expected 4096, got {}",
                    ReorderedBuffer::round_to_pages(4096));
    assert!(ReorderedBuffer::round_to_pages(4097) == 8192, "Rounding up > 1 page failed, expected 8192, got {}",
                    ReorderedBuffer::round_to_pages(4097));
}

/// Test rounding up to power of 2.
#[test]
fn round_to_power_of_2_test() {
    assert!(ReorderedBuffer::round_to_power_of_2(0) == 0, "Rounding to power of 2 failed, expected 0");
    assert!(ReorderedBuffer::round_to_power_of_2(1) == 1, "Rounding to power of 2 failed, expected 1");
    assert!(ReorderedBuffer::round_to_power_of_2(2) == 2, "Rounding to power of 2 failed, expected 2");
    assert!(ReorderedBuffer::round_to_power_of_2(3) == 4, "Rounding to power of 2 failed, expected 4");
    assert!(ReorderedBuffer::round_to_power_of_2(4) == 4, "Rounding to power of 2 failed, expected 4");
    assert!(ReorderedBuffer::round_to_power_of_2(5) == 8, "Rounding to power of 2 failed, expected 8");
}

/// Check that creation proceeds without a hitch.
#[test]
fn creation_test() {
    for i in 128..131072 {
        let ro = ReorderedBuffer::new(i);
        drop(ro);
    }
}

/// Check that in order insertion works correctly.
#[test]
fn in_order_insertion() {
    let mut ro = ReorderedBuffer::new(65536);
    let data0 = "food";
    let base_seq = 1232;
    if let InsertionResult::Inserted{ written, available } = ro.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len(), "When writing with seq, not all data was written expected {} got {}",
                written, data0.len());
        assert!(available == data0.len(), "When writing in-order, not all data is available. Expected {} got {}",
                available, data0.len());
    } else {
        panic!("Seq failed");
    }

    let data1 = ": hamburger";
    if let InsertionResult::Inserted{ written, available } = ro.add_data(base_seq + data0.len(), data1.as_bytes()) {
        assert!(written == data1.len());
        assert!(available == data0.len() + data1.len());
    } else {
        panic!("Writing data1 failed");
    }

    let read_buf_len = data0.len() + data1.len() + 1;
    let mut read_buffer: Vec<_> = (0..read_buf_len).map(|_| 0).collect();
    let read = ro.read_data(&mut read_buffer[..]);
    assert!(read == data0.len() + data1.len(), "Read less than expected, read: {}, expected: {}",
            read, data0.len() + data1.len());
    let read_str = str::from_utf8(&read_buffer[..read]).unwrap();
    assert!(read_str == format!("{}{}", data0, data1),
            "Read does not match expected, read: {}, expected: {}", read_str, format!("{}{}", data0, data1));
}

/// Check that out of order insertion works correctly.
#[test]
fn out_of_order_insertion() {
    let mut ro = ReorderedBuffer::new(65536);
    let data0 = "food";
    let base_seq = 1232;
    if let InsertionResult::Inserted{ written, available } = ro.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
        assert!(available == data0.len());
    } else {
        panic!("Seq failed");
    }

    let data1 = ": hamburger";
    let data2 = " american";
    if let InsertionResult::Inserted{ written, available } = ro.add_data(base_seq + data0.len() + data1.len(),
                                                                         data2.as_bytes()) {
        assert!(written == data2.len());
        assert!(available == data0.len());
    } else {
        panic!("Writing data2 failed");
    }

    if let InsertionResult::Inserted{ written, available } = ro.add_data(base_seq + data0.len(), data1.as_bytes()) {
        assert!(written == data1.len());
        assert!(available == data0.len() + data1.len() + data2.len());
    } else {
        panic!("Writing data1 failed");
    }
}

/// Check that things OOM correctly when out of memory.
#[test]
fn check_oom() {
    let mut r0 = ReorderedBuffer::new(4096);
    let base_seq = 32;
    let data0 = "food";

    let iters = (4096 / data0.len()) - 1;
    if let InsertionResult::Inserted { written, .. } = r0.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
    } else {
        panic!("Could not write");
    }

    for i in 1..iters {
        if let InsertionResult::Inserted{ written, .. } = r0.add_data(base_seq + (i * data0.len()),
                                                                             data0.as_bytes()) {
            assert!(written == data0.len());
        } else {
            panic!("Could not write");
        }
    }

    if let InsertionResult::OutOfMemory{ written, available } = r0.add_data(base_seq + (iters * data0.len()),
                                                                            data0.as_bytes()) {
        assert!(written != data0.len());
        assert!(available == 4096 - 1);
    } else {
        panic!("No OOM?");
    }
}

/// Check that reseting `ReorderedBuffer` works as expected.
#[test]
fn check_reset() {
    let mut r0 = ReorderedBuffer::new(4096);
    let base_seq = 155;
    let data0 = "food";

    let iters = (4096 / data0.len()) - 1;
    if let InsertionResult::Inserted { written, .. } = r0.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
    } else {
        panic!("Could not write");
    }

    for i in 1..iters {
        if let InsertionResult::Inserted{ written, .. } = r0.add_data(base_seq + (i * data0.len()),
                                                                             data0.as_bytes()) {
            assert!(written == data0.len());
        } else {
            panic!("Could not write");
        }
    }

    if let InsertionResult::OutOfMemory{ written, available } = r0.add_data(base_seq + (iters * data0.len()),
                                                                            data0.as_bytes()) {
        assert!(written != data0.len());
        assert!(available == 4096 - 1);
    } else {
        panic!("No OOM?");
    }

    r0.reset();
    let base_seq = 72;

    if let InsertionResult::Inserted { written, .. } = r0.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
    } else {
        panic!("Could not write");
    }

    for i in 1..iters {
        if let InsertionResult::Inserted{ written, .. } = r0.add_data(base_seq + (i * data0.len()),
                                                                             data0.as_bytes()) {
            assert!(written == data0.len());
        } else {
            panic!("Could not write");
        }
    }

    if let InsertionResult::OutOfMemory{ written, available } = r0.add_data(base_seq + (iters * data0.len()),
                                                                            data0.as_bytes()) {
        assert!(written != data0.len());
        assert!(available == 4096 - 1);
    } else {
        panic!("No OOM?");
    }
}
<commit_msg>Test that unordered data is read correctly<commit_after>extern crate e2d2;
use e2d2::state::*;
use std::str;
use std::slice;

/// Check rounding up to number of pages.
#[test]
fn round_pages_test() {
    assert!(ReorderedBuffer::round_to_pages(1) == 4096, "Rounding up 1 byte did not result in PAGE_SIZE");
    assert!(ReorderedBuffer::round_to_pages(0) == 0, "Rounding up 0 bytes did not result in 0");
    assert!(ReorderedBuffer::round_to_pages(8) == 4096, "Rounding up failure, expected 4096, got {}",
            ReorderedBuffer::round_to_pages(8));
    assert!(ReorderedBuffer::round_to_pages(512) == 4096, "Rounding up failure, expected 4096, got {}",
            ReorderedBuffer::round_to_pages(512));
    assert!(ReorderedBuffer::round_to_pages(4096) == 4096, "Rounding up exactly 1 page failed, expected 4096, got {}",
                    ReorderedBuffer::round_to_pages(4096));
    assert!(ReorderedBuffer::round_to_pages(4097) == 8192, "Rounding up > 1 page failed, expected 8192, got {}",
                    ReorderedBuffer::round_to_pages(4097));
}

/// Test rounding up to power of 2.
#[test]
fn round_to_power_of_2_test() {
    assert!(ReorderedBuffer::round_to_power_of_2(0) == 0, "Rounding to power of 2 failed, expected 0");
    assert!(ReorderedBuffer::round_to_power_of_2(1) == 1, "Rounding to power of 2 failed, expected 1");
    assert!(ReorderedBuffer::round_to_power_of_2(2) == 2, "Rounding to power of 2 failed, expected 2");
    assert!(ReorderedBuffer::round_to_power_of_2(3) == 4, "Rounding to power of 2 failed, expected 4");
    assert!(ReorderedBuffer::round_to_power_of_2(4) == 4, "Rounding to power of 2 failed, expected 4");
    assert!(ReorderedBuffer::round_to_power_of_2(5) == 8, "Rounding to power of 2 failed, expected 8");
}

/// Check that creation proceeds without a hitch.
#[test]
fn creation_test() {
    for i in 128..131072 {
        let ro = ReorderedBuffer::new(i);
        drop(ro);
    }
}

/// Check that in order insertion works correctly.
#[test]
fn in_order_insertion() {
    let mut ro = ReorderedBuffer::new(65536);
    let data0 = "food";
    let base_seq = 1232;
    if let InsertionResult::Inserted{ written, available } = ro.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len(), "When writing with seq, not all data was written expected {} got {}",
                written, data0.len());
        assert!(available == data0.len(), "When writing in-order, not all data is available. Expected {} got {}",
                available, data0.len());
    } else {
        panic!("Seq failed");
    }

    let data1 = ": hamburger";
    if let InsertionResult::Inserted{ written, available } = ro.add_data(base_seq + data0.len(), data1.as_bytes()) {
        assert!(written == data1.len());
        assert!(available == data0.len() + data1.len());
    } else {
        panic!("Writing data1 failed");
    }

    let read_buf_len = data0.len() + data1.len() + 1;
    let mut read_buffer: Vec<_> = (0..read_buf_len).map(|_| 0).collect();
    let read = ro.read_data(&mut read_buffer[..]);
    assert!(read == data0.len() + data1.len(), "Read less than expected, read: {}, expected: {}",
            read, data0.len() + data1.len());
    let read_str = str::from_utf8(&read_buffer[..read]).unwrap();
    assert!(read_str == format!("{}{}", data0, data1),
            "Read does not match expected, read: {}, expected: {}", read_str, format!("{}{}", data0, data1));
}

/// Check that out of order insertion works correctly.
#[test]
fn out_of_order_insertion() {
    let mut ro = ReorderedBuffer::new(65536);
    let data0 = "food";
    let base_seq = 1232;
    if let InsertionResult::Inserted{ written, available } = ro.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
        assert!(available == data0.len());
    } else {
        panic!("Seq failed");
    }

    let data1 = ": hamburger";
    let data2 = " american";
    if let InsertionResult::Inserted{ written, available } = ro.add_data(base_seq + data0.len() + data1.len(),
                                                                         data2.as_bytes()) {
        assert!(written == data2.len());
        assert!(available == data0.len());
    } else {
        panic!("Writing data2 failed");
    }

    if let InsertionResult::Inserted{ written, available } = ro.add_data(base_seq + data0.len(), data1.as_bytes()) {
        assert!(written == data1.len());
        assert!(available == data0.len() + data1.len() + data2.len());
    } else {
        panic!("Writing data1 failed");
    }

    let read_buf_len = ro.available();
    let mut read_buffer: Vec<_> = (0..read_buf_len).map(|_| 0).collect();
    let read = ro.read_data(&mut read_buffer[..]);
    assert!(read == read_buf_len, "Read less than what is available");
    assert!(ro.available() == 0, "Read everything but data is still available");
    let read = str::from_utf8(&read_buffer[..read]).unwrap();
    assert!(read == format!("{}{}{}", data0, data1, data2),
            "Read does not match expected, read: {}, expected: {}",
            read, format!("{}{}{}", data0, data1, data2));
}

/// Check that things OOM correctly when out of memory.
#[test]
fn check_oom() {
    let mut r0 = ReorderedBuffer::new(4096);
    let base_seq = 32;
    let data0 = "food";

    let iters = (4096 / data0.len()) - 1;
    if let InsertionResult::Inserted { written, .. } = r0.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
    } else {
        panic!("Could not write");
    }

    for i in 1..iters {
        if let InsertionResult::Inserted{ written, .. } = r0.add_data(base_seq + (i * data0.len()),
                                                                             data0.as_bytes()) {
            assert!(written == data0.len());
        } else {
            panic!("Could not write");
        }
    }

    if let InsertionResult::OutOfMemory{ written, available } = r0.add_data(base_seq + (iters * data0.len()),
                                                                            data0.as_bytes()) {
        assert!(written != data0.len());
        assert!(available == 4096 - 1);
    } else {
        panic!("No OOM?");
    }
}

/// Check that reseting `ReorderedBuffer` works as expected.
#[test]
fn check_reset() {
    let mut r0 = ReorderedBuffer::new(4096);
    let base_seq = 155;
    let data0 = "food";

    let iters = (4096 / data0.len()) - 1;
    if let InsertionResult::Inserted { written, .. } = r0.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
    } else {
        panic!("Could not write");
    }

    for i in 1..iters {
        if let InsertionResult::Inserted{ written, .. } = r0.add_data(base_seq + (i * data0.len()),
                                                                             data0.as_bytes()) {
            assert!(written == data0.len());
        } else {
            panic!("Could not write");
        }
    }

    if let InsertionResult::OutOfMemory{ written, available } = r0.add_data(base_seq + (iters * data0.len()),
                                                                            data0.as_bytes()) {
        assert!(written != data0.len());
        assert!(available == 4096 - 1);
    } else {
        panic!("No OOM?");
    }

    r0.reset();
    let base_seq = 72;

    if let InsertionResult::Inserted { written, .. } = r0.seq(base_seq, data0.as_bytes()) {
        assert!(written == data0.len());
    } else {
        panic!("Could not write");
    }

    for i in 1..iters {
        if let InsertionResult::Inserted{ written, .. } = r0.add_data(base_seq + (i * data0.len()),
                                                                             data0.as_bytes()) {
            assert!(written == data0.len());
        } else {
            panic!("Could not write");
        }
    }

    if let InsertionResult::OutOfMemory{ written, available } = r0.add_data(base_seq + (iters * data0.len()),
                                                                            data0.as_bytes()) {
        assert!(written != data0.len());
        assert!(available == 4096 - 1);
    } else {
        panic!("No OOM?");
    }
}
<|endoftext|>


<commit_before>use std::path::MAIN_SEPARATOR as SEP;
use support::{basic_bin_manifest, execs, project, ProjectBuilder};
use hamcrest::{assert_that};

fn setup() {
}

fn verbose_output_for_lib(p: &ProjectBuilder) -> String {
    format!("\
[COMPILING] {name} v{version} ({url})
[RUNNING] `rustc src{sep}lib.rs --crate-name {name} --crate-type lib -g \
        --out-dir {dir}{sep}target{sep}debug \
        --emit=dep-info,link \
        -L dependency={dir}{sep}target{sep}debug \
        -L dependency={dir}{sep}target{sep}debug{sep}deps`
", sep = SEP,
            dir = p.root().display(), url = p.url(),
            name = "foo", version = "0.0.1")
}

test!(build_lib_only {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]

            name = "foo"
            version = "0.0.1"
            authors = ["wycats@example.com"]
        "#)
        .file("src/main.rs", r#"
            fn main() {}
        "#)
        .file("src/lib.rs", r#" "#);

    assert_that(p.cargo_process("build").arg("--lib").arg("-v"),
                execs()
                .with_status(0)
                .with_stdout(verbose_output_for_lib(&p)));
});

test!(build_with_no_lib {
    let p = project("foo")
        .file("Cargo.toml", &basic_bin_manifest("foo"))
        .file("src/main.rs", r#"
            fn main() {}
        "#);

    assert_that(p.cargo_process("build").arg("--lib"),
                execs().with_status(101)
                       .with_stderr("[ERROR] no library targets found"));
});

test!(build_with_relative_cargo_home_path {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]

            name = "foo"
            version = "0.0.1"
            authors = ["wycats@example.com"]

            [dependencies]

            "test-dependency" = { path = "src/test_dependency" }
        "#)
        .file("src/main.rs", r#"
            fn main() {}
        "#)
        .file("src/test_dependency/src/lib.rs", r#" "#)
        .file("src/test_dependency/Cargo.toml", r#"
            [package]

            name = "test-dependency"
            version = "0.0.1"
            authors = ["wycats@example.com"]
        "#);

    assert_that(p.cargo_process("build").env("CARGO_HOME", "./cargo_home/"),
                execs()
                .with_status(0));
});
<commit_msg>fix test_cargo_build_lib<commit_after>use std::path::MAIN_SEPARATOR as SEP;
use support::{basic_bin_manifest, execs, project, ProjectBuilder};
use hamcrest::{assert_that};

fn setup() {
}

fn verbose_output_for_lib(p: &ProjectBuilder) -> String {
    format!("\
[COMPILING] {name} v{version} ({url})
[RUNNING] `rustc src{sep}lib.rs --crate-name {name} --crate-type lib -g \
        --out-dir {dir}{sep}target{sep}debug \
        --emit=dep-info,link \
        -L dependency={dir}{sep}target{sep}debug \
        -L dependency={dir}{sep}target{sep}debug{sep}deps`
", sep = SEP,
            dir = p.root().display(), url = p.url(),
            name = "foo", version = "0.0.1")
}

test!(build_lib_only {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]

            name = "foo"
            version = "0.0.1"
            authors = ["wycats@example.com"]
        "#)
        .file("src/main.rs", r#"
            fn main() {}
        "#)
        .file("src/lib.rs", r#" "#);

    assert_that(p.cargo_process("build").arg("--lib").arg("-v"),
                execs()
                .with_status(0)
                .with_stderr(verbose_output_for_lib(&p)));
});

test!(build_with_no_lib {
    let p = project("foo")
        .file("Cargo.toml", &basic_bin_manifest("foo"))
        .file("src/main.rs", r#"
            fn main() {}
        "#);

    assert_that(p.cargo_process("build").arg("--lib"),
                execs().with_status(101)
                       .with_stderr("[ERROR] no library targets found"));
});

test!(build_with_relative_cargo_home_path {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]

            name = "foo"
            version = "0.0.1"
            authors = ["wycats@example.com"]

            [dependencies]

            "test-dependency" = { path = "src/test_dependency" }
        "#)
        .file("src/main.rs", r#"
            fn main() {}
        "#)
        .file("src/test_dependency/src/lib.rs", r#" "#)
        .file("src/test_dependency/Cargo.toml", r#"
            [package]

            name = "test-dependency"
            version = "0.0.1"
            authors = ["wycats@example.com"]
        "#);

    assert_that(p.cargo_process("build").env("CARGO_HOME", "./cargo_home/"),
                execs()
                .with_status(0));
});
<|endoftext|>


<commit_before>#![feature(test)]

extern crate test;
extern crate ratelimit_meter;

use ratelimit_meter::{Allower, GCRA, Threadsafe, Limiter, Decider};
use std::time::{Instant, Duration};

#[bench]
fn bench_gcra(b: &mut test::Bencher) {
    let mut gcra = Limiter::new().capacity(50).weight(1).build::<GCRA>().unwrap();
    let now = Instant::now();
    let ms = Duration::from_millis(20);
    let mut i = 0;
    b.iter(|| {
        i += 1;
        gcra.test_and_update(now + (ms * i))
    });
}

#[bench]
fn bench_allower(b: &mut test::Bencher) {
    let mut allower = Limiter::new().capacity(50).weight(1).build::<Allower>().unwrap();
    b.iter(|| allower.check());
}

#[bench]
fn bench_threadsafe_gcra(b: &mut test::Bencher) {
    let mut gcra = Limiter::new().capacity(50).weight(1).build::<Threadsafe<GCRA>>().unwrap();
    let now = Instant::now();
    let ms = Duration::from_millis(20);
    let mut i = 0;
    b.iter(|| {
        i += 1;
        gcra.test_and_update(now + (ms * i))
    });
}

#[bench]
fn bench_threadsafe_allower(b: &mut test::Bencher) {
    let mut allower = Limiter::new().capacity(50).weight(1).build::<Threadsafe<Allower>>().unwrap();
    b.iter(|| allower.check());
}
<commit_msg>Update benchmarks to the new interface, too<commit_after>#![feature(test)]

extern crate test;
extern crate ratelimit_meter;

use ratelimit_meter::{Allower, GCRA, Threadsafe, Limiter, Decider};
use std::time::{Instant, Duration};

#[bench]
fn bench_gcra(b: &mut test::Bencher) {
    let mut gcra = Limiter::new().capacity(50).weight(1).build::<GCRA>().unwrap();
    let now = Instant::now();
    let ms = Duration::from_millis(20);
    let mut i = 0;
    b.iter(|| {
        i += 1;
        gcra.test_and_update(now + (ms * i)).unwrap();
    });
}

#[bench]
fn bench_allower(b: &mut test::Bencher) {
    let mut allower = Limiter::new().capacity(50).weight(1).build::<Allower>().unwrap();
    b.iter(|| allower.check().unwrap());
}

#[bench]
fn bench_threadsafe_gcra(b: &mut test::Bencher) {
    let mut gcra = Limiter::new().capacity(50).weight(1).build::<Threadsafe<GCRA>>().unwrap();
    let now = Instant::now();
    let ms = Duration::from_millis(20);
    let mut i = 0;
    b.iter(|| {
        i += 1;
        gcra.test_and_update(now + (ms * i)).unwrap();
    });
}

#[bench]
fn bench_threadsafe_allower(b: &mut test::Bencher) {
    let mut allower = Limiter::new().capacity(50).weight(1).build::<Threadsafe<Allower>>().unwrap();
    b.iter(|| allower.check());
}
<|endoftext|>


<commit_before>#![feature(slicing_syntax)]

extern crate test;
extern crate time;

use std::collections::Bitv;
use std::io::stdio;
use std::iter;
use std::os;

const LIMIT: uint = 2_000_000;
const SIZE: uint = (LIMIT - 1) / 2;

fn solution() -> uint {
    let mut sieve = Bitv::with_capacity(SIZE, false);
    let mut sum = 2;

    for i in range(0, SIZE) {
        if !sieve.get(i) {
            let p = 2 * i + 3;

            sum += p;

            for j in iter::range_step(p * p, LIMIT, 2 * p) {
                let j = (j - 3) / 2;

                sieve.set(j, true);
            }
        }
    }

    sum
}

fn main() {
    match os::args()[] {
        [_, ref flag] if flag[] == "-a" => return println!("{}", solution()),
        _ => {},
    }

    for line in stdio::stdin().lock().lines() {
        let iters: u64 = from_str(line.unwrap()[].trim()).unwrap();

        let start = time::precise_time_ns();
        for _ in range(0, iters) {
            test::black_box(solution());
        }
        let end = time::precise_time_ns();

        println!("{}", end - start);
    }
}
<commit_msg>fix(010.rs): adapt to changes in the `Bitv` API<commit_after>#![feature(slicing_syntax)]

extern crate test;
extern crate time;

use std::collections::Bitv;
use std::io::stdio;
use std::iter;
use std::os;

const LIMIT: uint = 2_000_000;
const SIZE: uint = (LIMIT - 1) / 2;

fn solution() -> uint {
    let mut sieve = Bitv::from_elem(SIZE, false);
    let mut sum = 2;

    for i in range(0, SIZE) {
        if !sieve[i] {
            let p = 2 * i + 3;

            sum += p;

            for j in iter::range_step(p * p, LIMIT, 2 * p) {
                let j = (j - 3) / 2;

                sieve.set(j, true);
            }
        }
    }

    sum
}

fn main() {
    match os::args()[] {
        [_, ref flag] if flag[] == "-a" => return println!("{}", solution()),
        _ => {},
    }

    for line in stdio::stdin().lock().lines() {
        let iters: u64 = from_str(line.unwrap()[].trim()).unwrap();

        let start = time::precise_time_ns();
        for _ in range(0, iters) {
            test::black_box(solution());
        }
        let end = time::precise_time_ns();

        println!("{}", end - start);
    }
}
<|endoftext|>


<commit_before>use clap::{crate_version};

use std::env;
use std::path::{Path, PathBuf};

use clap::{App, ArgMatches, SubCommand, AppSettings};

use mdbook::MDBook;
use mdbook::errors::{Result as Result3};

#[cfg(feature = "linkcheck")]
use mdbook::renderer::RenderContext;
#[cfg(feature = "linkcheck")]
use mdbook_linkcheck::{self, errors::BrokenLinks};
use failure::Error;

fn main() {
    let d_message = "-d, --dest-dir=[dest-dir]
'The output directory for your book{n}(Defaults to ./book when omitted)'";
    let dir_message = "[dir]
'A directory for your book{n}(Defaults to Current Directory when omitted)'";

    let matches = App::new("rustbook")
                    .about("Build a book with mdBook")
                    .author("Steve Klabnik <steve@steveklabnik.com>")
                    .version(&*format!("v{}", crate_version!()))
                    .setting(AppSettings::SubcommandRequired)
                    .subcommand(SubCommand::with_name("build")
                        .about("Build the book from the markdown files")
                        .arg_from_usage(d_message)
                        .arg_from_usage(dir_message))
                    .subcommand(SubCommand::with_name("linkcheck")
                        .about("Run linkcheck with mdBook 3")
                        .arg_from_usage(dir_message))
                    .get_matches();

    // Check which subcomamnd the user ran...
    match matches.subcommand() {
        ("build", Some(sub_matches)) => {
            if let Err(e) = build(sub_matches) {
                eprintln!("Error: {}", e);

                for cause in e.iter().skip(1) {
                    eprintln!("\tCaused By: {}", cause);
                }

                ::std::process::exit(101);
            }
        },
        ("linkcheck", Some(sub_matches)) => {
            if let Err(err) = linkcheck(sub_matches) {
                eprintln!("Error: {}", err);

                #[cfg(feature = "linkcheck")]
                {
                    if let Ok(broken_links) = err.downcast::<BrokenLinks>() {
                        for cause in broken_links.links().iter() {
                            eprintln!("\tCaused By: {}", cause);
                        }
                    }
                }

                ::std::process::exit(101);
            }
        },
        (_, _) => unreachable!(),
    };
}

#[cfg(feature = "linkcheck")]
pub fn linkcheck(args: &ArgMatches<'_>) -> Result<(), Error> {
    let book_dir = get_book_dir(args);
    let book = MDBook::load(&book_dir).unwrap();
    let cfg = book.config;
    let render_ctx = RenderContext::new(&book_dir, book.book, cfg, &book_dir);

    mdbook_linkcheck::check_links(&render_ctx)
}

#[cfg(not(feature = "linkcheck"))]
pub fn linkcheck(_args: &ArgMatches<'_>) -> Result<(), Error> {
    println!("mdbook-linkcheck is disabled.");
    Ok(())
}

// Build command implementation
pub fn build(args: &ArgMatches<'_>) -> Result3<()> {
    let book_dir = get_book_dir(args);
    let mut book = MDBook::load(&book_dir)?;

    // Set this to allow us to catch bugs in advance.
    book.config.build.create_missing = false;

    if let Some(dest_dir) = args.value_of("dest-dir") {
        book.config.build.build_dir = PathBuf::from(dest_dir);
    }

    book.build()?;

    Ok(())
}

fn get_book_dir(args: &ArgMatches<'_>) -> PathBuf {
    if let Some(dir) = args.value_of("dir") {
        // Check if path is relative from current dir, or absolute...
        let p = Path::new(dir);
        if p.is_relative() {
            env::current_dir().unwrap().join(dir)
        } else {
            p.to_path_buf()
        }
    } else {
        env::current_dir().unwrap()
    }
}
<commit_msg>filter linkcheck spurious failure<commit_after>use clap::crate_version;

use std::env;
use std::path::{Path, PathBuf};

use clap::{App, AppSettings, ArgMatches, SubCommand};

use mdbook::errors::Result as Result3;
use mdbook::MDBook;

use failure::Error;
#[cfg(feature = "linkcheck")]
use mdbook::renderer::RenderContext;
#[cfg(feature = "linkcheck")]
use mdbook_linkcheck::{self, errors::BrokenLinks};

fn main() {
    let d_message = "-d, --dest-dir=[dest-dir]
'The output directory for your book{n}(Defaults to ./book when omitted)'";
    let dir_message = "[dir]
'A directory for your book{n}(Defaults to Current Directory when omitted)'";

    let matches = App::new("rustbook")
        .about("Build a book with mdBook")
        .author("Steve Klabnik <steve@steveklabnik.com>")
        .version(&*format!("v{}", crate_version!()))
        .setting(AppSettings::SubcommandRequired)
        .subcommand(
            SubCommand::with_name("build")
                .about("Build the book from the markdown files")
                .arg_from_usage(d_message)
                .arg_from_usage(dir_message),
        )
        .subcommand(
            SubCommand::with_name("linkcheck")
                .about("Run linkcheck with mdBook 3")
                .arg_from_usage(dir_message),
        )
        .get_matches();

    // Check which subcomamnd the user ran...
    match matches.subcommand() {
        ("build", Some(sub_matches)) => {
            if let Err(e) = build(sub_matches) {
                eprintln!("Error: {}", e);

                for cause in e.iter().skip(1) {
                    eprintln!("\tCaused By: {}", cause);
                }

                ::std::process::exit(101);
            }
        }
        ("linkcheck", Some(sub_matches)) => {
            if let Err(err) = linkcheck(sub_matches) {
                eprintln!("Error: {}", err);

                // HACK: ignore timeouts
                #[allow(unused_mut)]
                let mut actually_broken = false;

                #[cfg(feature = "linkcheck")]
                {
                    if let Ok(broken_links) = err.downcast::<BrokenLinks>() {
                        for cause in broken_links.links().iter() {
                            eprintln!("\tCaused By: {}", cause);

                            if cause.contains("timed out") {
                                actually_broken = true;
                            }
                        }
                    }
                }

                if actually_broken {
                    std::process::exit(101);
                } else {
                    std::process::exit(0);
                }
            }
        }
        (_, _) => unreachable!(),
    };
}

#[cfg(feature = "linkcheck")]
pub fn linkcheck(args: &ArgMatches<'_>) -> Result<(), Error> {
    let book_dir = get_book_dir(args);
    let book = MDBook::load(&book_dir).unwrap();
    let cfg = book.config;
    let render_ctx = RenderContext::new(&book_dir, book.book, cfg, &book_dir);

    mdbook_linkcheck::check_links(&render_ctx)
}

#[cfg(not(feature = "linkcheck"))]
pub fn linkcheck(_args: &ArgMatches<'_>) -> Result<(), Error> {
    println!("mdbook-linkcheck is disabled.");
    Ok(())
}

// Build command implementation
pub fn build(args: &ArgMatches<'_>) -> Result3<()> {
    let book_dir = get_book_dir(args);
    let mut book = MDBook::load(&book_dir)?;

    // Set this to allow us to catch bugs in advance.
    book.config.build.create_missing = false;

    if let Some(dest_dir) = args.value_of("dest-dir") {
        book.config.build.build_dir = PathBuf::from(dest_dir);
    }

    book.build()?;

    Ok(())
}

fn get_book_dir(args: &ArgMatches<'_>) -> PathBuf {
    if let Some(dir) = args.value_of("dir") {
        // Check if path is relative from current dir, or absolute...
        let p = Path::new(dir);
        if p.is_relative() {
            env::current_dir().unwrap().join(dir)
        } else {
            p.to_path_buf()
        }
    } else {
        env::current_dir().unwrap()
    }
}
<|endoftext|>


<commit_before>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

pub mod create {
    use storeid::StoreIdIterator;
    use store::FileLockEntry;
    use store::Store;
    use error::Result;

    pub struct StoreCreateIterator<'a>(StoreIdIterator, &'a Store);

    impl<'a> Iterator for StoreCreateIterator<'a> {
        type Item = Result<FileLockEntry<'a>>;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next().map(|id| self.1.create(id))
        }
    }

    pub trait StoreIdCreateIteratorExtension<'a> {
        fn into_create_iter(self, store: &'a Store) -> StoreCreateIterator<'a>;
    }

    impl<'a> StoreIdCreateIteratorExtension<'a> for StoreIdIterator {
        fn into_create_iter(self, store: &'a Store) -> StoreCreateIterator<'a> {
            StoreCreateIterator(self, store)
        }
    }
}

pub mod delete {
    use storeid::StoreIdIterator;
    use store::Store;
    use error::Result;

    pub struct StoreDeleteIterator<'a>(StoreIdIterator, &'a Store);

    impl<'a> Iterator for StoreDeleteIterator<'a> {
        type Item = Result<()>;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next().map(|id| self.1.delete(id))
        }
    }

    pub trait StoreIdDeleteIteratorExtension<'a> {
        fn into_delete_iter(self, store: &'a Store) -> StoreDeleteIterator<'a>;
    }

    impl<'a> StoreIdDeleteIteratorExtension<'a> for StoreIdIterator {
        fn into_delete_iter(self, store: &'a Store) -> StoreDeleteIterator<'a> {
            StoreDeleteIterator(self, store)
        }
    }
}

pub mod get {
    use storeid::StoreIdIterator;
    use store::FileLockEntry;
    use store::Store;
    use error::Result;

    pub struct StoreGetIterator<'a>(StoreIdIterator, &'a Store);

    impl<'a> Iterator for StoreGetIterator<'a> {
        type Item = Result<Option<FileLockEntry<'a>>>;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next().map(|id| self.1.get(id))
        }
    }

    pub trait StoreIdGetIteratorExtension<'a> {
        fn into_get_iter(self, store: &'a Store) -> StoreGetIterator<'a>;
    }

    impl<'a> StoreIdGetIteratorExtension<'a> for StoreIdIterator {
        fn into_get_iter(self, store: &'a Store) -> StoreGetIterator<'a> {
            StoreGetIterator(self, store)
        }
    }
}

pub mod retrieve {
    use storeid::StoreIdIterator;
    use store::FileLockEntry;
    use store::Store;
    use error::Result;

    pub struct StoreRetrieveIterator<'a>(StoreIdIterator, &'a Store);

    impl<'a> Iterator for StoreRetrieveIterator<'a> {
        type Item = Result<FileLockEntry<'a>>;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next().map(|id| self.1.retrieve(id))
        }
    }

    pub trait StoreIdRetrieveIteratorExtension<'a> {
        fn into_retrieve_iter(self, store: &'a Store) -> StoreRetrieveIterator<'a>;
    }

    impl<'a> StoreIdRetrieveIteratorExtension<'a> for StoreIdIterator {
        fn into_retrieve_iter(self, store: &'a Store) -> StoreRetrieveIterator<'a> {
            StoreRetrieveIterator(self, store)
        }
    }
}

<commit_msg>Minify iterator helper implementation<commit_after>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

macro_rules! mk_iterator_mod {
    {
        modname   = $modname:ident,
        itername  = $itername:ident,
        iteryield = $yield:ty,
        extname   = $extname:ident,
        extfnname = $extfnname:ident,
        fun       = $fun:expr
    } => {
        pub mod $modname {
            use storeid::StoreIdIterator;
            use storeid::StoreId;
            #[allow(unused_imports)]
            use store::FileLockEntry;
            use store::Store;
            use error::Result;

            pub struct $itername<'a>(StoreIdIterator, &'a Store);

            impl<'a> Iterator for $itername<'a> {
                type Item = Result<$yield>;

                fn next(&mut self) -> Option<Self::Item> {
                    self.0.next().map(|id| $fun(id, self.1))
                }
            }

            pub trait $extname<'a> {
                fn $extfnname(self, store: &'a Store) -> $itername<'a>;
            }

            impl<'a> $extname<'a> for StoreIdIterator {
                fn $extfnname(self, store: &'a Store) -> $itername<'a> {
                    $itername(self, store)
                }
            }
        }
    }
}

mk_iterator_mod! {
    modname   = create,
    itername  = StoreCreateIterator,
    iteryield = FileLockEntry<'a>,
    extname   = StoreIdCreateIteratorExtension,
    extfnname = into_create_iter,
    fun       = |id: StoreId, store: &'a Store| store.create(id)
}

mk_iterator_mod! {
    modname   = delete,
    itername  = StoreDeleteIterator,
    iteryield = (),
    extname   = StoreIdDeleteIteratorExtension,
    extfnname = into_delete_iter,
    fun       = |id: StoreId, store: &'a Store| store.delete(id)
}

mk_iterator_mod! {
    modname   = get,
    itername  = StoreGetIterator,
    iteryield = Option<FileLockEntry<'a>>,
    extname   = StoreIdGetIteratorExtension,
    extfnname = into_get_iter,
    fun       = |id: StoreId, store: &'a Store| store.get(id)
}

mk_iterator_mod! {
    modname   = retrieve,
    itername  = StoreRetrieveIterator,
    iteryield = FileLockEntry<'a>,
    extname   = StoreIdRetrieveIteratorExtension,
    extfnname = into_retrieve_iter,
    fun       = |id: StoreId, store: &'a Store| store.retrieve(id)
}

<|endoftext|>


<commit_before>#![feature(test)] // compiletest_rs requires this attribute

use compiletest_rs as compiletest;
use compiletest_rs::common::Mode as TestMode;

use std::env::{self, set_var};
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};

mod cargo;

fn host_lib() -> PathBuf {
    option_env!("HOST_LIBS").map_or(cargo::CARGO_TARGET_DIR.join(env!("PROFILE")), PathBuf::from)
}

fn clippy_driver_path() -> PathBuf {
    option_env!("CLIPPY_DRIVER_PATH").map_or(cargo::TARGET_LIB.join("clippy-driver"), PathBuf::from)
}

// When we'll want to use `extern crate ..` for a dependency that is used
// both by the crate and the compiler itself, we can't simply pass -L flags
// as we'll get a duplicate matching versions. Instead, disambiguate with
// `--extern dep=path`.
// See https://github.com/rust-lang/rust-clippy/issues/4015.
//
// FIXME: We cannot use `cargo build --message-format=json` to resolve to dependency files.
//        Because it would force-rebuild if the options passed to `build` command is not the same
//        as what we manually pass to `cargo` invocation
fn third_party_crates() -> String {
    use std::collections::HashMap;
    static CRATES: &[&str] = &["serde", "serde_derive", "regex", "clippy_lints", "syn", "quote"];
    let dep_dir = cargo::TARGET_LIB.join("deps");
    let mut crates: HashMap<&str, PathBuf> = HashMap::with_capacity(CRATES.len());
    for entry in fs::read_dir(dep_dir).unwrap() {
        let path = match entry {
            Ok(entry) => entry.path(),
            Err(_) => continue,
        };
        if let Some(name) = path.file_name().and_then(OsStr::to_str) {
            for dep in CRATES {
                if name.starts_with(&format!("lib{}-", dep)) && name.ends_with(".rlib") {
                    if let Some(old) = crates.insert(dep, path.clone()) {
                        panic!("Found multiple rlibs for crate `{}`: `{:?}` and `{:?}", dep, old, path);
                    }
                    break;
                }
            }
        }
    }

    let v: Vec<_> = crates
        .into_iter()
        .map(|(dep, path)| format!("--extern {}={}", dep, path.display()))
        .collect();
    v.join(" ")
}

fn default_config() -> compiletest::Config {
    let mut config = compiletest::Config::default();

    if let Ok(name) = env::var("TESTNAME") {
        config.filter = Some(name);
    }

    if let Some(path) = option_env!("RUSTC_LIB_PATH") {
        let path = PathBuf::from(path);
        config.run_lib_path = path.clone();
        config.compile_lib_path = path;
    }

    config.target_rustcflags = Some(format!(
        "-L {0} -L {1} -Dwarnings -Zui-testing {2}",
        host_lib().join("deps").display(),
        cargo::TARGET_LIB.join("deps").display(),
        third_party_crates(),
    ));

    config.build_base = if cargo::is_rustc_test_suite() {
        // This make the stderr files go to clippy OUT_DIR on rustc repo build dir
        let mut path = PathBuf::from(env!("OUT_DIR"));
        path.push("test_build_base");
        path
    } else {
        host_lib().join("test_build_base")
    };
    config.rustc_path = clippy_driver_path();
    config
}

fn run_mode(cfg: &mut compiletest::Config) {
    cfg.mode = TestMode::Ui;
    cfg.src_base = Path::new("tests").join("ui");
    compiletest::run_tests(&cfg);
}

fn run_ui_toml(config: &mut compiletest::Config) {
    fn run_tests(config: &compiletest::Config, mut tests: Vec<tester::TestDescAndFn>) -> Result<bool, io::Error> {
        let mut result = true;
        let opts = compiletest::test_opts(config);
        for dir in fs::read_dir(&config.src_base)? {
            let dir = dir?;
            if !dir.file_type()?.is_dir() {
                continue;
            }
            let dir_path = dir.path();
            set_var("CARGO_MANIFEST_DIR", &dir_path);
            for file in fs::read_dir(&dir_path)? {
                let file = file?;
                let file_path = file.path();
                if file.file_type()?.is_dir() {
                    continue;
                }
                if file_path.extension() != Some(OsStr::new("rs")) {
                    continue;
                }
                let paths = compiletest::common::TestPaths {
                    file: file_path,
                    base: config.src_base.clone(),
                    relative_dir: dir_path.file_name().unwrap().into(),
                };
                let test_name = compiletest::make_test_name(&config, &paths);
                let index = tests
                    .iter()
                    .position(|test| test.desc.name == test_name)
                    .expect("The test should be in there");
                result &= tester::run_tests_console(&opts, vec![tests.swap_remove(index)])?;
            }
        }
        Ok(result)
    }

    config.mode = TestMode::Ui;
    config.src_base = Path::new("tests").join("ui-toml").canonicalize().unwrap();

    let tests = compiletest::make_tests(&config);

    let res = run_tests(&config, tests);
    match res {
        Ok(true) => {},
        Ok(false) => panic!("Some tests failed"),
        Err(e) => {
            panic!("I/O failure during tests: {:?}", e);
        },
    }
}

fn run_ui_cargo(config: &mut compiletest::Config) {
    fn run_tests(
        config: &compiletest::Config,
        filter: &Option<String>,
        mut tests: Vec<tester::TestDescAndFn>,
    ) -> Result<bool, io::Error> {
        let mut result = true;
        let opts = compiletest::test_opts(config);

        for dir in fs::read_dir(&config.src_base)? {
            let dir = dir?;
            if !dir.file_type()?.is_dir() {
                continue;
            }

            // Use the filter if provided
            let dir_path = dir.path();
            match &filter {
                Some(name) if !dir_path.ends_with(name) => continue,
                _ => {},
            }

            for case in fs::read_dir(&dir_path)? {
                let case = case?;
                if !case.file_type()?.is_dir() {
                    continue;
                }

                let src_path = case.path().join("src");

                // When switching between branches, if the previous branch had a test
                // that the current branch does not have, the directory is not removed
                // because an ignored Cargo.lock file exists.
                if !src_path.exists() {
                    continue;
                }

                env::set_current_dir(&src_path)?;
                for file in fs::read_dir(&src_path)? {
                    let file = file?;
                    if file.file_type()?.is_dir() {
                        continue;
                    }

                    // Search for the main file to avoid running a test for each file in the project
                    let file_path = file.path();
                    match file_path.file_name().and_then(OsStr::to_str) {
                        Some("main.rs") => {},
                        _ => continue,
                    }

                    let paths = compiletest::common::TestPaths {
                        file: file_path,
                        base: config.src_base.clone(),
                        relative_dir: src_path.strip_prefix(&config.src_base).unwrap().into(),
                    };
                    let test_name = compiletest::make_test_name(&config, &paths);
                    let index = tests
                        .iter()
                        .position(|test| test.desc.name == test_name)
                        .expect("The test should be in there");
                    result &= tester::run_tests_console(&opts, vec![tests.swap_remove(index)])?;
                }
            }
        }
        Ok(result)
    }

    config.mode = TestMode::Ui;
    config.src_base = Path::new("tests").join("ui-cargo").canonicalize().unwrap();

    let tests = compiletest::make_tests(&config);

    let current_dir = env::current_dir().unwrap();
    let filter = env::var("TESTNAME").ok();
    let res = run_tests(&config, &filter, tests);
    env::set_current_dir(current_dir).unwrap();

    match res {
        Ok(true) => {},
        Ok(false) => panic!("Some tests failed"),
        Err(e) => {
            panic!("I/O failure during tests: {:?}", e);
        },
    }
}

fn prepare_env() {
    set_var("CLIPPY_DISABLE_DOCS_LINKS", "true");
    set_var("CLIPPY_TESTS", "true");
    //set_var("RUST_BACKTRACE", "0");
}

#[test]
fn compile_test() {
    prepare_env();
    let mut config = default_config();
    run_mode(&mut config);
    run_ui_toml(&mut config);
    run_ui_cargo(&mut config);
}
<commit_msg>Auto merge of #5864 - rust-lang:ci_debug, r=Manishearth<commit_after>#![feature(test)] // compiletest_rs requires this attribute

use compiletest_rs as compiletest;
use compiletest_rs::common::Mode as TestMode;

use std::env::{self, set_var, var};
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};

mod cargo;

fn host_lib() -> PathBuf {
    option_env!("HOST_LIBS").map_or(cargo::CARGO_TARGET_DIR.join(env!("PROFILE")), PathBuf::from)
}

fn clippy_driver_path() -> PathBuf {
    option_env!("CLIPPY_DRIVER_PATH").map_or(cargo::TARGET_LIB.join("clippy-driver"), PathBuf::from)
}

// When we'll want to use `extern crate ..` for a dependency that is used
// both by the crate and the compiler itself, we can't simply pass -L flags
// as we'll get a duplicate matching versions. Instead, disambiguate with
// `--extern dep=path`.
// See https://github.com/rust-lang/rust-clippy/issues/4015.
//
// FIXME: We cannot use `cargo build --message-format=json` to resolve to dependency files.
//        Because it would force-rebuild if the options passed to `build` command is not the same
//        as what we manually pass to `cargo` invocation
fn third_party_crates() -> String {
    use std::collections::HashMap;
    static CRATES: &[&str] = &["serde", "serde_derive", "regex", "clippy_lints", "syn", "quote"];
    let dep_dir = cargo::TARGET_LIB.join("deps");
    let mut crates: HashMap<&str, PathBuf> = HashMap::with_capacity(CRATES.len());
    for entry in fs::read_dir(dep_dir).unwrap() {
        let path = match entry {
            Ok(entry) => entry.path(),
            Err(_) => continue,
        };
        if let Some(name) = path.file_name().and_then(OsStr::to_str) {
            for dep in CRATES {
                if name.starts_with(&format!("lib{}-", dep)) && name.ends_with(".rlib") {
                    if let Some(old) = crates.insert(dep, path.clone()) {
                        panic!("Found multiple rlibs for crate `{}`: `{:?}` and `{:?}", dep, old, path);
                    }
                    break;
                }
            }
        }
    }

    let v: Vec<_> = crates
        .into_iter()
        .map(|(dep, path)| format!("--extern {}={}", dep, path.display()))
        .collect();
    v.join(" ")
}

fn default_config() -> compiletest::Config {
    let mut config = compiletest::Config::default();

    if let Ok(name) = env::var("TESTNAME") {
        config.filter = Some(name);
    }

    if let Some(path) = option_env!("RUSTC_LIB_PATH") {
        let path = PathBuf::from(path);
        config.run_lib_path = path.clone();
        config.compile_lib_path = path;
    }

    config.target_rustcflags = Some(format!(
        "-L {0} -L {1} -Dwarnings -Zui-testing {2}",
        host_lib().join("deps").display(),
        cargo::TARGET_LIB.join("deps").display(),
        third_party_crates(),
    ));

    config.build_base = if cargo::is_rustc_test_suite() {
        // This make the stderr files go to clippy OUT_DIR on rustc repo build dir
        let mut path = PathBuf::from(env!("OUT_DIR"));
        path.push("test_build_base");
        path
    } else {
        host_lib().join("test_build_base")
    };
    config.rustc_path = clippy_driver_path();
    config
}

fn run_mode(cfg: &mut compiletest::Config) {
    cfg.mode = TestMode::Ui;
    cfg.src_base = Path::new("tests").join("ui");
    compiletest::run_tests(&cfg);
}

fn run_ui_toml(config: &mut compiletest::Config) {
    fn run_tests(config: &compiletest::Config, mut tests: Vec<tester::TestDescAndFn>) -> Result<bool, io::Error> {
        let mut result = true;
        let opts = compiletest::test_opts(config);
        for dir in fs::read_dir(&config.src_base)? {
            let dir = dir?;
            if !dir.file_type()?.is_dir() {
                continue;
            }
            let dir_path = dir.path();
            set_var("CARGO_MANIFEST_DIR", &dir_path);
            for file in fs::read_dir(&dir_path)? {
                let file = file?;
                let file_path = file.path();
                if file.file_type()?.is_dir() {
                    continue;
                }
                if file_path.extension() != Some(OsStr::new("rs")) {
                    continue;
                }
                let paths = compiletest::common::TestPaths {
                    file: file_path,
                    base: config.src_base.clone(),
                    relative_dir: dir_path.file_name().unwrap().into(),
                };
                let test_name = compiletest::make_test_name(&config, &paths);
                let index = tests
                    .iter()
                    .position(|test| test.desc.name == test_name)
                    .expect("The test should be in there");
                result &= tester::run_tests_console(&opts, vec![tests.swap_remove(index)])?;
            }
        }
        Ok(result)
    }

    config.mode = TestMode::Ui;
    config.src_base = Path::new("tests").join("ui-toml").canonicalize().unwrap();

    let tests = compiletest::make_tests(&config);

    let manifest_dir = var("CARGO_MANIFEST_DIR").unwrap_or_default();
    let res = run_tests(&config, tests);
    set_var("CARGO_MANIFEST_DIR", &manifest_dir);
    match res {
        Ok(true) => {},
        Ok(false) => panic!("Some tests failed"),
        Err(e) => {
            panic!("I/O failure during tests: {:?}", e);
        },
    }
}

fn run_ui_cargo(config: &mut compiletest::Config) {
    fn run_tests(
        config: &compiletest::Config,
        filter: &Option<String>,
        mut tests: Vec<tester::TestDescAndFn>,
    ) -> Result<bool, io::Error> {
        let mut result = true;
        let opts = compiletest::test_opts(config);

        for dir in fs::read_dir(&config.src_base)? {
            let dir = dir?;
            if !dir.file_type()?.is_dir() {
                continue;
            }

            // Use the filter if provided
            let dir_path = dir.path();
            match &filter {
                Some(name) if !dir_path.ends_with(name) => continue,
                _ => {},
            }

            for case in fs::read_dir(&dir_path)? {
                let case = case?;
                if !case.file_type()?.is_dir() {
                    continue;
                }

                let src_path = case.path().join("src");

                // When switching between branches, if the previous branch had a test
                // that the current branch does not have, the directory is not removed
                // because an ignored Cargo.lock file exists.
                if !src_path.exists() {
                    continue;
                }

                env::set_current_dir(&src_path)?;
                for file in fs::read_dir(&src_path)? {
                    let file = file?;
                    if file.file_type()?.is_dir() {
                        continue;
                    }

                    // Search for the main file to avoid running a test for each file in the project
                    let file_path = file.path();
                    match file_path.file_name().and_then(OsStr::to_str) {
                        Some("main.rs") => {},
                        _ => continue,
                    }

                    let paths = compiletest::common::TestPaths {
                        file: file_path,
                        base: config.src_base.clone(),
                        relative_dir: src_path.strip_prefix(&config.src_base).unwrap().into(),
                    };
                    let test_name = compiletest::make_test_name(&config, &paths);
                    let index = tests
                        .iter()
                        .position(|test| test.desc.name == test_name)
                        .expect("The test should be in there");
                    result &= tester::run_tests_console(&opts, vec![tests.swap_remove(index)])?;
                }
            }
        }
        Ok(result)
    }

    config.mode = TestMode::Ui;
    config.src_base = Path::new("tests").join("ui-cargo").canonicalize().unwrap();

    let tests = compiletest::make_tests(&config);

    let current_dir = env::current_dir().unwrap();
    let filter = env::var("TESTNAME").ok();
    let res = run_tests(&config, &filter, tests);
    env::set_current_dir(current_dir).unwrap();

    match res {
        Ok(true) => {},
        Ok(false) => panic!("Some tests failed"),
        Err(e) => {
            panic!("I/O failure during tests: {:?}", e);
        },
    }
}

fn prepare_env() {
    set_var("CLIPPY_DISABLE_DOCS_LINKS", "true");
    set_var("CLIPPY_TESTS", "true");
    //set_var("RUST_BACKTRACE", "0");
}

#[test]
fn compile_test() {
    prepare_env();
    let mut config = default_config();
    run_mode(&mut config);
    run_ui_toml(&mut config);
    run_ui_cargo(&mut config);
}
<|endoftext|>