<commit_before>use schema::Schema;
use schema::Document;
use indexer::SegmentSerializer;
use core::Index;
use core::SerializableSegment;
use core::Segment;
use std::thread::JoinHandle;
use indexer::SegmentWriter;
use std::clone::Clone;
use std::io;
use std::path::Path;
use std::thread;
use std::collections::HashSet;
use indexer::merger::IndexMerger;
use core::SegmentId;
use datastruct::stacker::Heap;
use std::mem::swap;
use chan;
use directory::WritePtr;

use Result;
use Error;

// Size of the margin for the heap. A segment is closed when the remaining memory
// in the heap goes below MARGIN_IN_BYTES.
pub const MARGIN_IN_BYTES: u32 = 10_000_000u32;

// We impose the memory per thread to be at least 30 MB.
pub const HEAP_SIZE_LIMIT: u32 = MARGIN_IN_BYTES * 3u32;

// Add document will block if the number of docs waiting in the queue to be indexed reaches PIPELINE_MAX_SIZE_IN_DOCS
const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;

pub const LOCKFILE_NAME: &'static str = ".tantivy-indexer.lock";

type DocumentSender = chan::Sender<Document>;
type DocumentReceiver = chan::Receiver<Document>;

type NewSegmentSender = chan::Sender<Result<(SegmentId, usize)>>;
type NewSegmentReceiver = chan::Receiver<Result<(SegmentId, usize)>>;

/// `IndexWriter` is the user entry-point to add document to an index.
///
/// It manages a small number of indexing thread, as well as a shared
/// indexing queue.
/// Each indexing thread builds its own independant `Segment`, via
/// a `SegmentWriter` object.
pub struct IndexWriter {
	index: Index,
	heap_size_in_bytes_per_thread: usize,
	workers_join_handle: Vec<JoinHandle<()>>,
	segment_ready_sender: NewSegmentSender,
	segment_ready_receiver: NewSegmentReceiver,
	document_receiver: DocumentReceiver,
	document_sender: DocumentSender,
	num_threads: usize,
	  docstamp: u64,
}

fn index_documents(heap: &mut Heap,
				   segment: Segment,
				   schema: &Schema,
				   document_iterator: &mut Iterator<Item=Document>) -> Result<usize> {
	heap.clear();
	let mut segment_writer = try!(SegmentWriter::for_segment(heap, segment, &schema));
	for doc in document_iterator {
		try!(segment_writer.add_document(&doc, &schema));
		if segment_writer.is_buffer_full() {
			info!("Buffer limit reached, flushing segment with maxdoc={}.", segment_writer.max_doc());
			break;
		}
	}
	let num_docs = segment_writer.max_doc() as usize;
	try!(segment_writer.finalize());
	Ok(num_docs)
}

impl Drop for IndexWriter {
    fn drop(&mut self) {
        let lockfile_path = Path::new(LOCKFILE_NAME);
        match self.index.directory_mut().delete(lockfile_path) {
            Ok(_) => (),
            Err(_) => ()
        }
    }
}
impl IndexWriter {

	/// Spawns a new worker thread for indexing.
	/// The thread consumes documents from the pipeline.
	///
	fn add_indexing_worker(&mut self,) -> Result<()> {
		let index = self.index.clone();
		let schema = self.index.schema();
		let segment_ready_sender_clone = self.segment_ready_sender.clone();
		let document_receiver_clone = self.document_receiver.clone();

		let mut heap = Heap::with_capacity(self.heap_size_in_bytes_per_thread);
		let join_handle: JoinHandle<()> = thread::spawn(move || {
			loop {
				let segment = index.new_segment();
				let segment_id = segment.id();
				let mut document_iterator = document_receiver_clone
					.clone()
					.into_iter()
					.peekable();
				// the peeking here is to avoid
				// creating a new segment's files
				// if no document are available.
				if document_iterator.peek().is_some() {
					let index_result = index_documents(&mut heap, segment, &schema, &mut document_iterator)
						.map(|num_docs| (segment_id, num_docs));
					segment_ready_sender_clone.send(index_result);
				}
				else {
					return;
				}
			}
		});
		self.workers_join_handle.push(join_handle);
		Ok(())
	}

	/// Open a new index writer
	///
	/// num_threads tells the number of indexing worker that
	/// should work at the same time.
	pub fn open(index: &Index,
				num_threads: usize,
				heap_size_in_bytes_per_thread: usize) -> Result<IndexWriter> {
		if heap_size_in_bytes_per_thread <= HEAP_SIZE_LIMIT as usize {
			panic!(format!("The heap size per thread needs to be at least {}.", HEAP_SIZE_LIMIT));
		}

      let mut cloned_index = index.clone();
      let lockfile_path = Path::new(LOCKFILE_NAME);

      try!(cloned_index.directory_mut().open_write(lockfile_path));
		let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) = chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
		let (segment_ready_sender, segment_ready_receiver): (NewSegmentSender, NewSegmentReceiver) = chan::async();
		let mut index_writer = IndexWriter {
			heap_size_in_bytes_per_thread: heap_size_in_bytes_per_thread,
			index: cloned_index,
			segment_ready_receiver: segment_ready_receiver,
			segment_ready_sender: segment_ready_sender,
			document_receiver: document_receiver,
			document_sender: document_sender,
			workers_join_handle: Vec::new(),
			num_threads: num_threads,
			docstamp: try!(index.docstamp()),
		};
		try!(index_writer.start_workers());
		Ok(index_writer)
	}

	fn start_workers(&mut self,) -> Result<()> {
		for _ in 0 .. self.num_threads {
			try!(self.add_indexing_worker());
		}
		Ok(())
	}

	/// Merges a given list of segments
	pub fn merge(&mut self, segments: &[Segment]) -> Result<()> {
		let schema = self.index.schema();
		let merger = try!(IndexMerger::open(schema, segments));
		let mut merged_segment = self.index.new_segment();
		let segment_serializer = try!(SegmentSerializer::for_segment(&mut merged_segment));
		try!(merger.write(segment_serializer));
		let merged_segment_ids: HashSet<SegmentId> = segments.iter().map(|segment| segment.id()).collect();
		try!(self.index.publish_merge_segment(merged_segment_ids, merged_segment.id()));
		Ok(())
	}

	/// Closes the current document channel send.
	/// and replace all the channels by new ones.
	///
	/// The current workers will keep on indexing
	/// the pending document and stop
	/// when no documents are remaining.
	///
	/// Returns the former segment_ready channel.
	fn recreate_channels(&mut self,) -> (DocumentReceiver, chan::Receiver<Result<(SegmentId, usize)>>) {
		let (mut document_sender, mut document_receiver): (DocumentSender, DocumentReceiver) = chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
		let (mut segment_ready_sender, mut segment_ready_receiver): (NewSegmentSender, NewSegmentReceiver) = chan::async();
		swap(&mut self.document_sender, &mut document_sender);
		swap(&mut self.document_receiver, &mut document_receiver);
		swap(&mut self.segment_ready_sender, &mut segment_ready_sender);
		swap(&mut self.segment_ready_receiver, &mut segment_ready_receiver);
		(document_receiver, segment_ready_receiver)
	}

	/// Rollback to the last commit
	///
	/// This cancels all of the update that
	/// happened before after the last commit.
	/// After calling rollback, the index is in the same
	/// state as it was after the last commit.
	///
	/// The docstamp at the last commit is returned.
	pub fn rollback(&mut self,) -> Result<u64> {

		// we cannot drop segment ready receiver yet
		// as it would block the workers.
		let (document_receiver, mut _segment_ready_receiver) = self.recreate_channels();

		// consumes the document receiver pipeline
		// worker don't need to index the pending documents.
		for _ in document_receiver {};

		let mut former_workers_join_handle = Vec::new();
		swap(&mut former_workers_join_handle, &mut self.workers_join_handle);

		// wait for all the worker to finish their work
		// (it should be fast since we consumed all pending documents)
		for worker_handle in former_workers_join_handle {
			try!(worker_handle
				.join()
				.map_err(|e| Error::ErrorInThread(format!("{:?}", e)))
			);
			// add a new worker for the next generation.
			try!(self.add_indexing_worker());
		}

		// reset the docstamp to what it was before
		self.docstamp = try!(self.index.docstamp());
		Ok(self.docstamp)
	}

	/// Commits all of the pending changes
	///
	/// A call to commit blocks.
	/// After it returns, all of the document that
	/// were added since the last commit are published
	/// and persisted.
	///
	/// In case of a crash or an hardware failure (as
	/// long as the hard disk is spared), it will be possible
	/// to resume indexing from this point.
	///
	/// Commit returns the `docstamp` of the last document
	/// that made it in the commit.
	///
	pub fn commit(&mut self,) -> Result<u64> {

		let (document_receiver, segment_ready_receiver) = self.recreate_channels();
		drop(document_receiver);

		// Docstamp of the last document in this commit.
		let commit_docstamp = self.docstamp;

		let mut former_workers_join_handle = Vec::new();
		swap(&mut former_workers_join_handle, &mut self.workers_join_handle);

		for worker_handle in former_workers_join_handle {
			try!(worker_handle
				.join()
				.map_err(|e| Error::ErrorInThread(format!("{:?}", e)))
			);
			// add a new worker for the next generation.
			try!(self.add_indexing_worker());
		}

		let segment_ids_and_size: Vec<(SegmentId, usize)> = try!(
			segment_ready_receiver
				.into_iter()
				.collect()
		);

		let segment_ids: Vec<SegmentId> = segment_ids_and_size
			.iter()
			.map(|&(segment_id, _num_docs)| segment_id)
			.collect();

		try!(self.index.publish_segments(&segment_ids, commit_docstamp));

		Ok(commit_docstamp)
	}

	/// Adds a document.
	///
	/// If the indexing pipeline is full, this call may block.
	///
	/// The docstamp is an increasing `u64` that can
	/// be used by the client to align commits with its own
	/// document queue.
	///
	/// Currently it represents the number of documents that
	/// have been added since the creation of the index.
	pub fn add_document(&mut self, doc: Document) -> io::Result<u64> {
		self.document_sender.send(doc);
		self.docstamp += 1;
		Ok(self.docstamp)
	}

}

#[cfg(test)]
mod tests {

	use schema::{self, Document};
	use Index;
	use Term;

    #[test]
    #[should_panic]
    fn test_lockfile_stops_duplicates() {

		    let mut schema_builder = schema::SchemaBuilder::default();
		    let text_field = schema_builder.add_text_field("text", schema::TEXT);
		    let index = Index::create_in_ram(schema_builder.build());
			  let index_writer = index.writer(40_000_000).unwrap();
        let index_writer_two = index.writer(40_000_000).unwrap();
    }

    #[test]
    fn test_lockfile_released_on_drop() {
		    let mut schema_builder = schema::SchemaBuilder::default();
		    let text_field = schema_builder.add_text_field("text", schema::TEXT);
		    let index = Index::create_in_ram(schema_builder.build());
			  {
            let index_writer = index.writer(40_000_000).unwrap();
        }

        let index_writer_two = index.writer(40_000_000).unwrap();
    }
	#[test]
	fn test_commit_and_rollback() {
		let mut schema_builder = schema::SchemaBuilder::default();
		let text_field = schema_builder.add_text_field("text", schema::TEXT);
		let index = Index::create_in_ram(schema_builder.build());

		let num_docs_containing = |s: &str| {
			let searcher = index.searcher();
			let term_a = Term::from_field_text(text_field, s);
			searcher.doc_freq(&term_a)
		};

		{
			// writing the segment
			let mut index_writer = index.writer_with_num_threads(3, 40_000_000).unwrap();
			{
				let mut doc = Document::default();
				doc.add_text(text_field, "a");
				index_writer.add_document(doc).unwrap();
			}
			assert_eq!(index_writer.rollback().unwrap(), 0u64);
			assert_eq!(num_docs_containing("a"), 0);

			{
				let mut doc = Document::default();
				doc.add_text(text_field, "b");
				index_writer.add_document(doc).unwrap();
			}
			{
				let mut doc = Document::default();
				doc.add_text(text_field, "c");
				index_writer.add_document(doc).unwrap();
			}
			assert_eq!(index_writer.commit().unwrap(), 2u64);

			assert_eq!(num_docs_containing("a"), 0);
			assert_eq!(num_docs_containing("b"), 1);
			assert_eq!(num_docs_containing("c"), 1);
		}
		index.searcher();
	}

}
<commit_msg>check for specific error rather than any panic<commit_after>use schema::Schema;
use schema::Document;
use indexer::SegmentSerializer;
use core::Index;
use core::SerializableSegment;
use core::Segment;
use std::thread::JoinHandle;
use indexer::SegmentWriter;
use std::clone::Clone;
use std::io;
use std::path::Path;
use std::thread;
use std::collections::HashSet;
use indexer::merger::IndexMerger;
use core::SegmentId;
use datastruct::stacker::Heap;
use std::mem::swap;
use chan;
use directory::WritePtr;

use Result;
use Error;

// Size of the margin for the heap. A segment is closed when the remaining memory
// in the heap goes below MARGIN_IN_BYTES.
pub const MARGIN_IN_BYTES: u32 = 10_000_000u32;

// We impose the memory per thread to be at least 30 MB.
pub const HEAP_SIZE_LIMIT: u32 = MARGIN_IN_BYTES * 3u32;

// Add document will block if the number of docs waiting in the queue to be indexed reaches PIPELINE_MAX_SIZE_IN_DOCS
const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;

pub const LOCKFILE_NAME: &'static str = ".tantivy-indexer.lock";

type DocumentSender = chan::Sender<Document>;
type DocumentReceiver = chan::Receiver<Document>;

type NewSegmentSender = chan::Sender<Result<(SegmentId, usize)>>;
type NewSegmentReceiver = chan::Receiver<Result<(SegmentId, usize)>>;

/// `IndexWriter` is the user entry-point to add document to an index.
///
/// It manages a small number of indexing thread, as well as a shared
/// indexing queue.
/// Each indexing thread builds its own independant `Segment`, via
/// a `SegmentWriter` object.
pub struct IndexWriter {
	index: Index,
	heap_size_in_bytes_per_thread: usize,
	workers_join_handle: Vec<JoinHandle<()>>,
	segment_ready_sender: NewSegmentSender,
	segment_ready_receiver: NewSegmentReceiver,
	document_receiver: DocumentReceiver,
	document_sender: DocumentSender,
	num_threads: usize,
	  docstamp: u64,
}

fn index_documents(heap: &mut Heap,
				   segment: Segment,
				   schema: &Schema,
				   document_iterator: &mut Iterator<Item=Document>) -> Result<usize> {
	heap.clear();
	let mut segment_writer = try!(SegmentWriter::for_segment(heap, segment, &schema));
	for doc in document_iterator {
		try!(segment_writer.add_document(&doc, &schema));
		if segment_writer.is_buffer_full() {
			info!("Buffer limit reached, flushing segment with maxdoc={}.", segment_writer.max_doc());
			break;
		}
	}
	let num_docs = segment_writer.max_doc() as usize;
	try!(segment_writer.finalize());
	Ok(num_docs)
}

impl Drop for IndexWriter {
    fn drop(&mut self) {
        let lockfile_path = Path::new(LOCKFILE_NAME);
        match self.index.directory_mut().delete(lockfile_path) {
            Ok(_) => (),
            Err(_) => ()
        }
    }
}
impl IndexWriter {

	/// Spawns a new worker thread for indexing.
	/// The thread consumes documents from the pipeline.
	///
	fn add_indexing_worker(&mut self,) -> Result<()> {
		let index = self.index.clone();
		let schema = self.index.schema();
		let segment_ready_sender_clone = self.segment_ready_sender.clone();
		let document_receiver_clone = self.document_receiver.clone();

		let mut heap = Heap::with_capacity(self.heap_size_in_bytes_per_thread);
		let join_handle: JoinHandle<()> = thread::spawn(move || {
			loop {
				let segment = index.new_segment();
				let segment_id = segment.id();
				let mut document_iterator = document_receiver_clone
					.clone()
					.into_iter()
					.peekable();
				// the peeking here is to avoid
				// creating a new segment's files
				// if no document are available.
				if document_iterator.peek().is_some() {
					let index_result = index_documents(&mut heap, segment, &schema, &mut document_iterator)
						.map(|num_docs| (segment_id, num_docs));
					segment_ready_sender_clone.send(index_result);
				}
				else {
					return;
				}
			}
		});
		self.workers_join_handle.push(join_handle);
		Ok(())
	}

	/// Open a new index writer
	///
	/// num_threads tells the number of indexing worker that
	/// should work at the same time.
	pub fn open(index: &Index,
				num_threads: usize,
				heap_size_in_bytes_per_thread: usize) -> Result<IndexWriter> {
		if heap_size_in_bytes_per_thread <= HEAP_SIZE_LIMIT as usize {
			panic!(format!("The heap size per thread needs to be at least {}.", HEAP_SIZE_LIMIT));
		}

      let mut cloned_index = index.clone();
      let lockfile_path = Path::new(LOCKFILE_NAME);

      try!(cloned_index.directory_mut().open_write(lockfile_path));
		let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) = chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
		let (segment_ready_sender, segment_ready_receiver): (NewSegmentSender, NewSegmentReceiver) = chan::async();
		let mut index_writer = IndexWriter {
			heap_size_in_bytes_per_thread: heap_size_in_bytes_per_thread,
			index: cloned_index,
			segment_ready_receiver: segment_ready_receiver,
			segment_ready_sender: segment_ready_sender,
			document_receiver: document_receiver,
			document_sender: document_sender,
			workers_join_handle: Vec::new(),
			num_threads: num_threads,
			docstamp: try!(index.docstamp()),
		};
		try!(index_writer.start_workers());
		Ok(index_writer)
	}

	fn start_workers(&mut self,) -> Result<()> {
		for _ in 0 .. self.num_threads {
			try!(self.add_indexing_worker());
		}
		Ok(())
	}

	/// Merges a given list of segments
	pub fn merge(&mut self, segments: &[Segment]) -> Result<()> {
		let schema = self.index.schema();
		let merger = try!(IndexMerger::open(schema, segments));
		let mut merged_segment = self.index.new_segment();
		let segment_serializer = try!(SegmentSerializer::for_segment(&mut merged_segment));
		try!(merger.write(segment_serializer));
		let merged_segment_ids: HashSet<SegmentId> = segments.iter().map(|segment| segment.id()).collect();
		try!(self.index.publish_merge_segment(merged_segment_ids, merged_segment.id()));
		Ok(())
	}

	/// Closes the current document channel send.
	/// and replace all the channels by new ones.
	///
	/// The current workers will keep on indexing
	/// the pending document and stop
	/// when no documents are remaining.
	///
	/// Returns the former segment_ready channel.
	fn recreate_channels(&mut self,) -> (DocumentReceiver, chan::Receiver<Result<(SegmentId, usize)>>) {
		let (mut document_sender, mut document_receiver): (DocumentSender, DocumentReceiver) = chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
		let (mut segment_ready_sender, mut segment_ready_receiver): (NewSegmentSender, NewSegmentReceiver) = chan::async();
		swap(&mut self.document_sender, &mut document_sender);
		swap(&mut self.document_receiver, &mut document_receiver);
		swap(&mut self.segment_ready_sender, &mut segment_ready_sender);
		swap(&mut self.segment_ready_receiver, &mut segment_ready_receiver);
		(document_receiver, segment_ready_receiver)
	}

	/// Rollback to the last commit
	///
	/// This cancels all of the update that
	/// happened before after the last commit.
	/// After calling rollback, the index is in the same
	/// state as it was after the last commit.
	///
	/// The docstamp at the last commit is returned.
	pub fn rollback(&mut self,) -> Result<u64> {

		// we cannot drop segment ready receiver yet
		// as it would block the workers.
		let (document_receiver, mut _segment_ready_receiver) = self.recreate_channels();

		// consumes the document receiver pipeline
		// worker don't need to index the pending documents.
		for _ in document_receiver {};

		let mut former_workers_join_handle = Vec::new();
		swap(&mut former_workers_join_handle, &mut self.workers_join_handle);

		// wait for all the worker to finish their work
		// (it should be fast since we consumed all pending documents)
		for worker_handle in former_workers_join_handle {
			try!(worker_handle
				.join()
				.map_err(|e| Error::ErrorInThread(format!("{:?}", e)))
			);
			// add a new worker for the next generation.
			try!(self.add_indexing_worker());
		}

		// reset the docstamp to what it was before
		self.docstamp = try!(self.index.docstamp());
		Ok(self.docstamp)
	}

	/// Commits all of the pending changes
	///
	/// A call to commit blocks.
	/// After it returns, all of the document that
	/// were added since the last commit are published
	/// and persisted.
	///
	/// In case of a crash or an hardware failure (as
	/// long as the hard disk is spared), it will be possible
	/// to resume indexing from this point.
	///
	/// Commit returns the `docstamp` of the last document
	/// that made it in the commit.
	///
	pub fn commit(&mut self,) -> Result<u64> {

		let (document_receiver, segment_ready_receiver) = self.recreate_channels();
		drop(document_receiver);

		// Docstamp of the last document in this commit.
		let commit_docstamp = self.docstamp;

		let mut former_workers_join_handle = Vec::new();
		swap(&mut former_workers_join_handle, &mut self.workers_join_handle);

		for worker_handle in former_workers_join_handle {
			try!(worker_handle
				.join()
				.map_err(|e| Error::ErrorInThread(format!("{:?}", e)))
			);
			// add a new worker for the next generation.
			try!(self.add_indexing_worker());
		}

		let segment_ids_and_size: Vec<(SegmentId, usize)> = try!(
			segment_ready_receiver
				.into_iter()
				.collect()
		);

		let segment_ids: Vec<SegmentId> = segment_ids_and_size
			.iter()
			.map(|&(segment_id, _num_docs)| segment_id)
			.collect();

		try!(self.index.publish_segments(&segment_ids, commit_docstamp));

		Ok(commit_docstamp)
	}

	/// Adds a document.
	///
	/// If the indexing pipeline is full, this call may block.
	///
	/// The docstamp is an increasing `u64` that can
	/// be used by the client to align commits with its own
	/// document queue.
	///
	/// Currently it represents the number of documents that
	/// have been added since the creation of the index.
	pub fn add_document(&mut self, doc: Document) -> io::Result<u64> {
		self.document_sender.send(doc);
		self.docstamp += 1;
		Ok(self.docstamp)
	}

}

#[cfg(test)]
mod tests {

	use schema::{self, Document};
	use Index;
	use Term;
    use Error;
    use directory::error::OpenWriteError;

    #[test]
    fn test_lockfile_stops_duplicates() {

		    let mut schema_builder = schema::SchemaBuilder::default();
		    let text_field = schema_builder.add_text_field("text", schema::TEXT);
		    let index = Index::create_in_ram(schema_builder.build());
			  let index_writer = index.writer(40_000_000).unwrap();
        match index.writer(40_000_000) {
            Err(Error::FileAlreadyExists(_)) => {},
            _ => panic!("Expected FileAlreadyExists error")
        }
    }

    #[test]
    fn test_lockfile_released_on_drop() {
		    let mut schema_builder = schema::SchemaBuilder::default();
		    let text_field = schema_builder.add_text_field("text", schema::TEXT);
		    let index = Index::create_in_ram(schema_builder.build());
			  {
            let index_writer = index.writer(40_000_000).unwrap();
        }

        let index_writer_two = index.writer(40_000_000).unwrap();
    }
	#[test]
	fn test_commit_and_rollback() {
		let mut schema_builder = schema::SchemaBuilder::default();
		let text_field = schema_builder.add_text_field("text", schema::TEXT);
		let index = Index::create_in_ram(schema_builder.build());

		let num_docs_containing = |s: &str| {
			let searcher = index.searcher();
			let term_a = Term::from_field_text(text_field, s);
			searcher.doc_freq(&term_a)
		};

		{
			// writing the segment
			let mut index_writer = index.writer_with_num_threads(3, 40_000_000).unwrap();
			{
				let mut doc = Document::default();
				doc.add_text(text_field, "a");
				index_writer.add_document(doc).unwrap();
			}
			assert_eq!(index_writer.rollback().unwrap(), 0u64);
			assert_eq!(num_docs_containing("a"), 0);

			{
				let mut doc = Document::default();
				doc.add_text(text_field, "b");
				index_writer.add_document(doc).unwrap();
			}
			{
				let mut doc = Document::default();
				doc.add_text(text_field, "c");
				index_writer.add_document(doc).unwrap();
			}
			assert_eq!(index_writer.commit().unwrap(), 2u64);

			assert_eq!(num_docs_containing("a"), 0);
			assert_eq!(num_docs_containing("b"), 1);
			assert_eq!(num_docs_containing("c"), 1);
		}
		index.searcher();
	}

}
<|endoftext|>


<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use attr;
use ast;
use codemap::respan;
use parse::common::SeqSep;
use parse::PResult;
use parse::token::{self, Nonterminal};
use parse::parser::{Parser, TokenType, PathStyle};
use tokenstream::TokenStream;

#[derive(PartialEq, Eq, Debug)]
enum InnerAttributeParsePolicy<'a> {
    Permitted,
    NotPermitted { reason: &'a str },
}

const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &'static str = "an inner attribute is not \
                                                             permitted in this context";

impl<'a> Parser<'a> {
    /// Parse attributes that appear before an item
    pub fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
        let mut attrs: Vec<ast::Attribute> = Vec::new();
        let mut just_parsed_doc_comment = false;
        loop {
            debug!("parse_outer_attributes: self.token={:?}", self.token);
            match self.token {
                token::Pound => {
                    let inner_error_reason = if just_parsed_doc_comment {
                        "an inner attribute is not permitted following an outer doc comment"
                    } else if !attrs.is_empty() {
                        "an inner attribute is not permitted following an outer attribute"
                    } else {
                        DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG
                    };
                    let inner_parse_policy =
                        InnerAttributeParsePolicy::NotPermitted { reason: inner_error_reason };
                    attrs.push(self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?);
                    just_parsed_doc_comment = false;
                }
                token::DocComment(s) => {
                    let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(), s, self.span);
                    if attr.style != ast::AttrStyle::Outer {
                        let mut err = self.fatal("expected outer doc comment");
                        err.note("inner doc comments like this (starting with \
                                  `//!` or `/*!`) can only appear before items");
                        return Err(err);
                    }
                    attrs.push(attr);
                    self.bump();
                    just_parsed_doc_comment = true;
                }
                _ => break,
            }
        }
        Ok(attrs)
    }

    /// Matches `attribute = # ! [ meta_item ]`
    ///
    /// If permit_inner is true, then a leading `!` indicates an inner
    /// attribute
    pub fn parse_attribute(&mut self, permit_inner: bool) -> PResult<'a, ast::Attribute> {
        debug!("parse_attribute: permit_inner={:?} self.token={:?}",
               permit_inner,
               self.token);
        let inner_parse_policy = if permit_inner {
            InnerAttributeParsePolicy::Permitted
        } else {
            InnerAttributeParsePolicy::NotPermitted
                { reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG }
        };
        self.parse_attribute_with_inner_parse_policy(inner_parse_policy)
    }

    /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy`
    /// that prescribes how to handle inner attributes.
    fn parse_attribute_with_inner_parse_policy(&mut self,
                                               inner_parse_policy: InnerAttributeParsePolicy)
                                               -> PResult<'a, ast::Attribute> {
        debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
               inner_parse_policy,
               self.token);
        let (span, path, tokens, mut style) = match self.token {
            token::Pound => {
                let lo = self.span;
                self.bump();

                if inner_parse_policy == InnerAttributeParsePolicy::Permitted {
                    self.expected_tokens.push(TokenType::Token(token::Not));
                }
                let style = if self.token == token::Not {
                    self.bump();
                    if let InnerAttributeParsePolicy::NotPermitted { reason } = inner_parse_policy
                    {
                        let span = self.span;
                        self.diagnostic()
                            .struct_span_err(span, reason)
                            .note("inner attributes and doc comments, like `#![no_std]` or \
                                   `//! My crate`, annotate the item enclosing them, and are \
                                   usually found at the beginning of source files. Outer \
                                   attributes and doc comments, like `#[test]` and
                                   `/// My function`, annotate the item following them.")
                            .emit()
                    }
                    ast::AttrStyle::Inner
                } else {
                    ast::AttrStyle::Outer
                };

                self.expect(&token::OpenDelim(token::Bracket))?;
                let (path, tokens) = self.parse_path_and_tokens()?;
                self.expect(&token::CloseDelim(token::Bracket))?;
                let hi = self.prev_span;

                (lo.to(hi), path, tokens, style)
            }
            _ => {
                let token_str = self.this_token_to_string();
                return Err(self.fatal(&format!("expected `#`, found `{}`", token_str)));
            }
        };

        if inner_parse_policy == InnerAttributeParsePolicy::Permitted &&
           self.token == token::Semi {
            self.bump();
            self.span_warn(span,
                           "this inner attribute syntax is deprecated. The new syntax is \
                            `#![foo]`, with a bang and no semicolon");
            style = ast::AttrStyle::Inner;
        }

        Ok(ast::Attribute {
            id: attr::mk_attr_id(),
            style: style,
            path: path,
            tokens: tokens,
            is_sugared_doc: false,
            span: span,
        })
    }

    pub fn parse_path_and_tokens(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
        let meta = match self.token {
            token::Interpolated(ref nt) => match nt.0 {
                Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
                _ => None,
            },
            _ => None,
        };
        Ok(if let Some(meta) = meta {
            self.bump();
            (ast::Path::from_ident(meta.span, ast::Ident::with_empty_ctxt(meta.name)),
             meta.node.tokens(meta.span))
        } else {
            (self.parse_path(PathStyle::Mod)?, self.parse_tokens())
        })
    }

    /// Parse attributes that appear after the opening of an item. These should
    /// be preceded by an exclamation mark, but we accept and warn about one
    /// terminated by a semicolon.

    /// matches inner_attrs*
    pub fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
        let mut attrs: Vec<ast::Attribute> = vec![];
        loop {
            match self.token {
                token::Pound => {
                    // Don't even try to parse if it's not an inner attribute.
                    if !self.look_ahead(1, |t| t == &token::Not) {
                        break;
                    }

                    let attr = self.parse_attribute(true)?;
                    assert_eq!(attr.style, ast::AttrStyle::Inner);
                    attrs.push(attr);
                }
                token::DocComment(s) => {
                    // we need to get the position of this token before we bump.
                    let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(), s, self.span);
                    if attr.style == ast::AttrStyle::Inner {
                        attrs.push(attr);
                        self.bump();
                    } else {
                        break;
                    }
                }
                _ => break,
            }
        }
        Ok(attrs)
    }

    fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
        let lit = self.parse_lit()?;
        debug!("Checking if {:?} is unusuffixed.", lit);

        if !lit.node.is_unsuffixed() {
            let msg = "suffixed literals are not allowed in attributes";
            self.diagnostic().struct_span_err(lit.span, msg)
                             .help("instead of using a suffixed literal \
                                    (1u8, 1.0f32, etc.), use an unsuffixed version \
                                    (1, 1.0, etc.).")
                             .emit()
        }

        Ok(lit)
    }

    /// Per RFC#1559, matches the following grammar:
    ///
    /// meta_item : IDENT ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ;
    /// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
    pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
        let nt_meta = match self.token {
            token::Interpolated(ref nt) => match nt.0 {
                token::NtMeta(ref e) => Some(e.clone()),
                _ => None,
            },
            _ => None,
        };

        if let Some(meta) = nt_meta {
            self.bump();
            return Ok(meta);
        }

        let lo = self.span;
        let ident = self.parse_ident()?;
        let node = self.parse_meta_item_kind()?;
        Ok(ast::MetaItem { name: ident.name, node: node, span: lo.to(self.prev_span) })
    }

    pub fn parse_meta_item_kind(&mut self) -> PResult<'a, ast::MetaItemKind> {
        Ok(if self.eat(&token::Eq) {
            ast::MetaItemKind::NameValue(self.parse_unsuffixed_lit()?)
        } else if self.token == token::OpenDelim(token::Paren) {
            ast::MetaItemKind::List(self.parse_meta_seq()?)
        } else {
            self.eat(&token::OpenDelim(token::Paren));
            ast::MetaItemKind::Word
        })
    }

    /// matches meta_item_inner : (meta_item | UNSUFFIXED_LIT) ;
    fn parse_meta_item_inner(&mut self) -> PResult<'a, ast::NestedMetaItem> {
        let lo = self.span;

        match self.parse_unsuffixed_lit() {
            Ok(lit) => {
                return Ok(respan(lo.to(self.prev_span), ast::NestedMetaItemKind::Literal(lit)))
            }
            Err(ref mut err) => self.diagnostic().cancel(err)
        }

        match self.parse_meta_item() {
            Ok(mi) => {
                return Ok(respan(lo.to(self.prev_span), ast::NestedMetaItemKind::MetaItem(mi)))
            }
            Err(ref mut err) => self.diagnostic().cancel(err)
        }

        let found = self.this_token_to_string();
        let msg = format!("expected unsuffixed literal or identifier, found {}", found);
        Err(self.diagnostic().struct_span_err(lo, &msg))
    }

    /// matches meta_seq = ( COMMASEP(meta_item_inner) )
    fn parse_meta_seq(&mut self) -> PResult<'a, Vec<ast::NestedMetaItem>> {
        self.parse_unspanned_seq(&token::OpenDelim(token::Paren),
                                 &token::CloseDelim(token::Paren),
                                 SeqSep::trailing_allowed(token::Comma),
                                 |p: &mut Parser<'a>| p.parse_meta_item_inner())
    }
}
<commit_msg>Eat open paren when parsing list<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use attr;
use ast;
use codemap::respan;
use parse::common::SeqSep;
use parse::PResult;
use parse::token::{self, Nonterminal};
use parse::parser::{Parser, TokenType, PathStyle};
use tokenstream::TokenStream;

#[derive(PartialEq, Eq, Debug)]
enum InnerAttributeParsePolicy<'a> {
    Permitted,
    NotPermitted { reason: &'a str },
}

const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &'static str = "an inner attribute is not \
                                                             permitted in this context";

impl<'a> Parser<'a> {
    /// Parse attributes that appear before an item
    pub fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
        let mut attrs: Vec<ast::Attribute> = Vec::new();
        let mut just_parsed_doc_comment = false;
        loop {
            debug!("parse_outer_attributes: self.token={:?}", self.token);
            match self.token {
                token::Pound => {
                    let inner_error_reason = if just_parsed_doc_comment {
                        "an inner attribute is not permitted following an outer doc comment"
                    } else if !attrs.is_empty() {
                        "an inner attribute is not permitted following an outer attribute"
                    } else {
                        DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG
                    };
                    let inner_parse_policy =
                        InnerAttributeParsePolicy::NotPermitted { reason: inner_error_reason };
                    attrs.push(self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?);
                    just_parsed_doc_comment = false;
                }
                token::DocComment(s) => {
                    let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(), s, self.span);
                    if attr.style != ast::AttrStyle::Outer {
                        let mut err = self.fatal("expected outer doc comment");
                        err.note("inner doc comments like this (starting with \
                                  `//!` or `/*!`) can only appear before items");
                        return Err(err);
                    }
                    attrs.push(attr);
                    self.bump();
                    just_parsed_doc_comment = true;
                }
                _ => break,
            }
        }
        Ok(attrs)
    }

    /// Matches `attribute = # ! [ meta_item ]`
    ///
    /// If permit_inner is true, then a leading `!` indicates an inner
    /// attribute
    pub fn parse_attribute(&mut self, permit_inner: bool) -> PResult<'a, ast::Attribute> {
        debug!("parse_attribute: permit_inner={:?} self.token={:?}",
               permit_inner,
               self.token);
        let inner_parse_policy = if permit_inner {
            InnerAttributeParsePolicy::Permitted
        } else {
            InnerAttributeParsePolicy::NotPermitted
                { reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG }
        };
        self.parse_attribute_with_inner_parse_policy(inner_parse_policy)
    }

    /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy`
    /// that prescribes how to handle inner attributes.
    fn parse_attribute_with_inner_parse_policy(&mut self,
                                               inner_parse_policy: InnerAttributeParsePolicy)
                                               -> PResult<'a, ast::Attribute> {
        debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
               inner_parse_policy,
               self.token);
        let (span, path, tokens, mut style) = match self.token {
            token::Pound => {
                let lo = self.span;
                self.bump();

                if inner_parse_policy == InnerAttributeParsePolicy::Permitted {
                    self.expected_tokens.push(TokenType::Token(token::Not));
                }
                let style = if self.token == token::Not {
                    self.bump();
                    if let InnerAttributeParsePolicy::NotPermitted { reason } = inner_parse_policy
                    {
                        let span = self.span;
                        self.diagnostic()
                            .struct_span_err(span, reason)
                            .note("inner attributes and doc comments, like `#![no_std]` or \
                                   `//! My crate`, annotate the item enclosing them, and are \
                                   usually found at the beginning of source files. Outer \
                                   attributes and doc comments, like `#[test]` and
                                   `/// My function`, annotate the item following them.")
                            .emit()
                    }
                    ast::AttrStyle::Inner
                } else {
                    ast::AttrStyle::Outer
                };

                self.expect(&token::OpenDelim(token::Bracket))?;
                let (path, tokens) = self.parse_path_and_tokens()?;
                self.expect(&token::CloseDelim(token::Bracket))?;
                let hi = self.prev_span;

                (lo.to(hi), path, tokens, style)
            }
            _ => {
                let token_str = self.this_token_to_string();
                return Err(self.fatal(&format!("expected `#`, found `{}`", token_str)));
            }
        };

        if inner_parse_policy == InnerAttributeParsePolicy::Permitted &&
           self.token == token::Semi {
            self.bump();
            self.span_warn(span,
                           "this inner attribute syntax is deprecated. The new syntax is \
                            `#![foo]`, with a bang and no semicolon");
            style = ast::AttrStyle::Inner;
        }

        Ok(ast::Attribute {
            id: attr::mk_attr_id(),
            style: style,
            path: path,
            tokens: tokens,
            is_sugared_doc: false,
            span: span,
        })
    }

    pub fn parse_path_and_tokens(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
        let meta = match self.token {
            token::Interpolated(ref nt) => match nt.0 {
                Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
                _ => None,
            },
            _ => None,
        };
        Ok(if let Some(meta) = meta {
            self.bump();
            (ast::Path::from_ident(meta.span, ast::Ident::with_empty_ctxt(meta.name)),
             meta.node.tokens(meta.span))
        } else {
            (self.parse_path(PathStyle::Mod)?, self.parse_tokens())
        })
    }

    /// Parse attributes that appear after the opening of an item. These should
    /// be preceded by an exclamation mark, but we accept and warn about one
    /// terminated by a semicolon.

    /// matches inner_attrs*
    pub fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
        let mut attrs: Vec<ast::Attribute> = vec![];
        loop {
            match self.token {
                token::Pound => {
                    // Don't even try to parse if it's not an inner attribute.
                    if !self.look_ahead(1, |t| t == &token::Not) {
                        break;
                    }

                    let attr = self.parse_attribute(true)?;
                    assert_eq!(attr.style, ast::AttrStyle::Inner);
                    attrs.push(attr);
                }
                token::DocComment(s) => {
                    // we need to get the position of this token before we bump.
                    let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(), s, self.span);
                    if attr.style == ast::AttrStyle::Inner {
                        attrs.push(attr);
                        self.bump();
                    } else {
                        break;
                    }
                }
                _ => break,
            }
        }
        Ok(attrs)
    }

    fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
        let lit = self.parse_lit()?;
        debug!("Checking if {:?} is unusuffixed.", lit);

        if !lit.node.is_unsuffixed() {
            let msg = "suffixed literals are not allowed in attributes";
            self.diagnostic().struct_span_err(lit.span, msg)
                             .help("instead of using a suffixed literal \
                                    (1u8, 1.0f32, etc.), use an unsuffixed version \
                                    (1, 1.0, etc.).")
                             .emit()
        }

        Ok(lit)
    }

    /// Per RFC#1559, matches the following grammar:
    ///
    /// meta_item : IDENT ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ;
    /// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
    pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
        let nt_meta = match self.token {
            token::Interpolated(ref nt) => match nt.0 {
                token::NtMeta(ref e) => Some(e.clone()),
                _ => None,
            },
            _ => None,
        };

        if let Some(meta) = nt_meta {
            self.bump();
            return Ok(meta);
        }

        let lo = self.span;
        let ident = self.parse_ident()?;
        let node = self.parse_meta_item_kind()?;
        Ok(ast::MetaItem { name: ident.name, node: node, span: lo.to(self.prev_span) })
    }

    pub fn parse_meta_item_kind(&mut self) -> PResult<'a, ast::MetaItemKind> {
        Ok(if self.eat(&token::Eq) {
            ast::MetaItemKind::NameValue(self.parse_unsuffixed_lit()?)
        } else if self.eat(&token::OpenDelim(token::Paren)) {
            ast::MetaItemKind::List(self.parse_meta_seq()?)
        } else {
            ast::MetaItemKind::Word
        })
    }

    /// matches meta_item_inner : (meta_item | UNSUFFIXED_LIT) ;
    fn parse_meta_item_inner(&mut self) -> PResult<'a, ast::NestedMetaItem> {
        let lo = self.span;

        match self.parse_unsuffixed_lit() {
            Ok(lit) => {
                return Ok(respan(lo.to(self.prev_span), ast::NestedMetaItemKind::Literal(lit)))
            }
            Err(ref mut err) => self.diagnostic().cancel(err)
        }

        match self.parse_meta_item() {
            Ok(mi) => {
                return Ok(respan(lo.to(self.prev_span), ast::NestedMetaItemKind::MetaItem(mi)))
            }
            Err(ref mut err) => self.diagnostic().cancel(err)
        }

        let found = self.this_token_to_string();
        let msg = format!("expected unsuffixed literal or identifier, found {}", found);
        Err(self.diagnostic().struct_span_err(lo, &msg))
    }

    /// matches meta_seq = ( COMMASEP(meta_item_inner) )
    fn parse_meta_seq(&mut self) -> PResult<'a, Vec<ast::NestedMetaItem>> {
        self.parse_seq_to_end(&token::CloseDelim(token::Paren),
                              SeqSep::trailing_allowed(token::Comma),
                              |p: &mut Parser<'a>| p.parse_meta_item_inner())
    }
}
<|endoftext|>


<commit_before>// $ cargo bench --features full --bench rust
//
// Syn only, useful for profiling:
// $ RUSTFLAGS='--cfg syn_only' cargo build --release --features full --bench rust

#![cfg_attr(not(syn_only), feature(rustc_private))]
#![recursion_limit = "1024"]

#[macro_use]
#[path = "../tests/macros/mod.rs"]
mod macros;

#[path = "../tests/common/mod.rs"]
mod common;
#[path = "../tests/repo/mod.rs"]
mod repo;

use std::fs;
use std::time::{Duration, Instant};

#[cfg(not(syn_only))]
mod tokenstream_parse {
    use proc_macro2::TokenStream;
    use std::str::FromStr;

    pub fn bench(content: &str) -> Result<(), ()> {
        TokenStream::from_str(content).map(drop).map_err(drop)
    }
}

mod syn_parse {
    pub fn bench(content: &str) -> Result<(), ()> {
        syn::parse_file(content).map(drop).map_err(drop)
    }
}

#[cfg(not(syn_only))]
mod libsyntax_parse {
    extern crate rustc_ast;
    extern crate rustc_data_structures;
    extern crate rustc_errors;
    extern crate rustc_parse;
    extern crate rustc_session;
    extern crate rustc_span;

    use rustc_data_structures::sync::Lrc;
    use rustc_errors::{emitter::Emitter, Diagnostic, Handler};
    use rustc_session::parse::ParseSess;
    use rustc_span::source_map::{FilePathMapping, SourceMap};
    use rustc_span::{edition::Edition, FileName};

    pub fn bench(content: &str) -> Result<(), ()> {
        struct SilentEmitter;

        impl Emitter for SilentEmitter {
            fn emit_diagnostic(&mut self, _diag: &Diagnostic) {}
            fn source_map(&self) -> Option<&Lrc<SourceMap>> {
                None
            }
        }

        rustc_ast::with_globals(Edition::Edition2018, || {
            let cm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
            let emitter = Box::new(SilentEmitter);
            let handler = Handler::with_emitter(false, None, emitter);
            let sess = ParseSess::with_span_handler(handler, cm);
            if let Err(mut diagnostic) = rustc_parse::parse_crate_from_source_str(
                FileName::Custom("bench".to_owned()),
                content.to_owned(),
                &sess,
            ) {
                diagnostic.cancel();
                return Err(());
            };
            Ok(())
        })
    }
}

#[cfg(not(syn_only))]
mod read_from_disk {
    pub fn bench(content: &str) -> Result<(), ()> {
        let _ = content;
        Ok(())
    }
}

fn exec(mut codepath: impl FnMut(&str) -> Result<(), ()>) -> Duration {
    let begin = Instant::now();
    let mut success = 0;
    let mut total = 0;

    walkdir::WalkDir::new("tests/rust/src")
        .into_iter()
        .filter_entry(repo::base_dir_filter)
        .for_each(|entry| {
            let entry = entry.unwrap();
            let path = entry.path();
            if path.is_dir() {
                return;
            }
            let content = fs::read_to_string(path).unwrap();
            let ok = codepath(&content).is_ok();
            success += ok as usize;
            total += 1;
            if !ok {
                eprintln!("FAIL {}", path.display());
            }
        });

    assert_eq!(success, total);
    begin.elapsed()
}

fn main() {
    repo::clone_rust();

    macro_rules! testcases {
        ($($(#[$cfg:meta])* $name:path,)*) => {
            vec![
                $(
                    $(#[$cfg])*
                    (stringify!($name), $name as fn(&str) -> Result<(), ()>),
                )*
            ]
        };
    }

    #[cfg(not(syn_only))]
    {
        let mut lines = 0;
        let mut files = 0;
        exec(|content| {
            lines += content.lines().count();
            files += 1;
            Ok(())
        });
        eprintln!("\n{} lines in {} files", lines, files);
    }

    for (name, f) in testcases!(
        #[cfg(not(syn_only))]
        read_from_disk::bench,
        #[cfg(not(syn_only))]
        tokenstream_parse::bench,
        syn_parse::bench,
        #[cfg(not(syn_only))]
        libsyntax_parse::bench,
    ) {
        eprint!("{:20}", format!("{}:", name));
        let elapsed = exec(f);
        eprintln!(
            "elapsed={}.{:03}s",
            elapsed.as_secs(),
            elapsed.subsec_millis(),
        );
    }
    eprintln!();
}
<commit_msg>Reduce noise in benchmark output<commit_after>// $ cargo bench --features full --bench rust
//
// Syn only, useful for profiling:
// $ RUSTFLAGS='--cfg syn_only' cargo build --release --features full --bench rust

#![cfg_attr(not(syn_only), feature(rustc_private))]
#![recursion_limit = "1024"]

#[macro_use]
#[path = "../tests/macros/mod.rs"]
mod macros;

#[path = "../tests/common/mod.rs"]
mod common;
#[path = "../tests/repo/mod.rs"]
mod repo;

use std::fs;
use std::time::{Duration, Instant};

#[cfg(not(syn_only))]
mod tokenstream_parse {
    use proc_macro2::TokenStream;
    use std::str::FromStr;

    pub fn bench(content: &str) -> Result<(), ()> {
        TokenStream::from_str(content).map(drop).map_err(drop)
    }
}

mod syn_parse {
    pub fn bench(content: &str) -> Result<(), ()> {
        syn::parse_file(content).map(drop).map_err(drop)
    }
}

#[cfg(not(syn_only))]
mod libsyntax_parse {
    extern crate rustc_ast;
    extern crate rustc_data_structures;
    extern crate rustc_errors;
    extern crate rustc_parse;
    extern crate rustc_session;
    extern crate rustc_span;

    use rustc_data_structures::sync::Lrc;
    use rustc_errors::{emitter::Emitter, Diagnostic, Handler};
    use rustc_session::parse::ParseSess;
    use rustc_span::source_map::{FilePathMapping, SourceMap};
    use rustc_span::{edition::Edition, FileName};

    pub fn bench(content: &str) -> Result<(), ()> {
        struct SilentEmitter;

        impl Emitter for SilentEmitter {
            fn emit_diagnostic(&mut self, _diag: &Diagnostic) {}
            fn source_map(&self) -> Option<&Lrc<SourceMap>> {
                None
            }
        }

        rustc_ast::with_globals(Edition::Edition2018, || {
            let cm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
            let emitter = Box::new(SilentEmitter);
            let handler = Handler::with_emitter(false, None, emitter);
            let sess = ParseSess::with_span_handler(handler, cm);
            if let Err(mut diagnostic) = rustc_parse::parse_crate_from_source_str(
                FileName::Custom("bench".to_owned()),
                content.to_owned(),
                &sess,
            ) {
                diagnostic.cancel();
                return Err(());
            };
            Ok(())
        })
    }
}

#[cfg(not(syn_only))]
mod read_from_disk {
    pub fn bench(content: &str) -> Result<(), ()> {
        let _ = content;
        Ok(())
    }
}

fn exec(mut codepath: impl FnMut(&str) -> Result<(), ()>) -> Duration {
    let begin = Instant::now();
    let mut success = 0;
    let mut total = 0;

    walkdir::WalkDir::new("tests/rust/src")
        .into_iter()
        .filter_entry(repo::base_dir_filter)
        .for_each(|entry| {
            let entry = entry.unwrap();
            let path = entry.path();
            if path.is_dir() {
                return;
            }
            let content = fs::read_to_string(path).unwrap();
            let ok = codepath(&content).is_ok();
            success += ok as usize;
            total += 1;
            if !ok {
                eprintln!("FAIL {}", path.display());
            }
        });

    assert_eq!(success, total);
    begin.elapsed()
}

fn main() {
    repo::clone_rust();

    macro_rules! testcases {
        ($($(#[$cfg:meta])* $name:ident,)*) => {
            vec![
                $(
                    $(#[$cfg])*
                    (stringify!($name), $name::bench as fn(&str) -> Result<(), ()>),
                )*
            ]
        };
    }

    #[cfg(not(syn_only))]
    {
        let mut lines = 0;
        let mut files = 0;
        exec(|content| {
            lines += content.lines().count();
            files += 1;
            Ok(())
        });
        eprintln!("\n{} lines in {} files", lines, files);
    }

    for (name, f) in testcases!(
        #[cfg(not(syn_only))]
        read_from_disk,
        #[cfg(not(syn_only))]
        tokenstream_parse,
        syn_parse,
        #[cfg(not(syn_only))]
        libsyntax_parse,
    ) {
        eprint!("{:20}", format!("{}:", name));
        let elapsed = exec(f);
        eprintln!(
            "elapsed={}.{:03}s",
            elapsed.as_secs(),
            elapsed.subsec_millis(),
        );
    }
    eprintln!();
}
<|endoftext|>


<commit_before>use ra_syntax::{
    ast::{self, NameOwner, VisibilityOwner},
    AstNode,
    SyntaxKind::{
        ATTR, COMMENT, CONST_DEF, ENUM_DEF, FN_DEF, MODULE, STRUCT_DEF, TRAIT_DEF, VISIBILITY,
        WHITESPACE,
    },
    SyntaxNode, TextSize, T,
};
use test_utils::mark;

use crate::{AssistContext, AssistId, Assists};

// Assist: change_visibility
//
// Adds or changes existing visibility specifier.
//
// ```
// <|>fn frobnicate() {}
// ```
// ->
// ```
// pub(crate) fn frobnicate() {}
// ```
pub(crate) fn change_visibility(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
    if let Some(vis) = ctx.find_node_at_offset::<ast::Visibility>() {
        return change_vis(acc, vis);
    }
    add_vis(acc, ctx)
}

fn add_vis(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
    let item_keyword = ctx.token_at_offset().find(|leaf| match leaf.kind() {
        T![const] | T![fn] | T![mod] | T![struct] | T![enum] | T![trait] => true,
        _ => false,
    });

    let (offset, target) = if let Some(keyword) = item_keyword {
        let parent = keyword.parent();
        let def_kws = vec![CONST_DEF, FN_DEF, MODULE, STRUCT_DEF, ENUM_DEF, TRAIT_DEF];
        // Parent is not a definition, can't add visibility
        if !def_kws.iter().any(|&def_kw| def_kw == parent.kind()) {
            return None;
        }
        // Already have visibility, do nothing
        if parent.children().any(|child| child.kind() == VISIBILITY) {
            return None;
        }
        (vis_offset(&parent), keyword.text_range())
    } else if let Some(field_name) = ctx.find_node_at_offset::<ast::Name>() {
        let field = field_name.syntax().ancestors().find_map(ast::RecordFieldDef::cast)?;
        if field.name()? != field_name {
            mark::hit!(change_visibility_field_false_positive);
            return None;
        }
        if field.visibility().is_some() {
            return None;
        }
        (vis_offset(field.syntax()), field_name.syntax().text_range())
    } else if let Some(field) = ctx.find_node_at_offset::<ast::TupleFieldDef>() {
        if field.visibility().is_some() {
            return None;
        }
        (vis_offset(field.syntax()), field.syntax().text_range())
    } else {
        return None;
    };

    acc.add(AssistId("change_visibility"), "Change visibility to pub(crate)", target, |edit| {
        edit.insert(offset, "pub(crate) ");
        edit.set_cursor(offset);
    })
}

fn vis_offset(node: &SyntaxNode) -> TextSize {
    node.children_with_tokens()
        .skip_while(|it| match it.kind() {
            WHITESPACE | COMMENT | ATTR => true,
            _ => false,
        })
        .next()
        .map(|it| it.text_range().start())
        .unwrap_or_else(|| node.text_range().start())
}

fn change_vis(acc: &mut Assists, vis: ast::Visibility) -> Option<()> {
    if vis.syntax().text() == "pub" {
        let target = vis.syntax().text_range();
        return acc.add(
            AssistId("change_visibility"),
            "Change Visibility to pub(crate)",
            target,
            |edit| {
                edit.replace(vis.syntax().text_range(), "pub(crate)");
                edit.set_cursor(vis.syntax().text_range().start())
            },
        );
    }
    if vis.syntax().text() == "pub(crate)" {
        let target = vis.syntax().text_range();
        return acc.add(
            AssistId("change_visibility"),
            "Change visibility to pub",
            target,
            |edit| {
                edit.replace(vis.syntax().text_range(), "pub");
                edit.set_cursor(vis.syntax().text_range().start());
            },
        );
    }
    None
}

#[cfg(test)]
mod tests {
    use test_utils::mark;

    use crate::tests::{check_assist, check_assist_not_applicable, check_assist_target};

    use super::*;

    #[test]
    fn change_visibility_adds_pub_crate_to_items() {
        check_assist(change_visibility, "<|>fn foo() {}", "<|>pub(crate) fn foo() {}");
        check_assist(change_visibility, "f<|>n foo() {}", "<|>pub(crate) fn foo() {}");
        check_assist(change_visibility, "<|>struct Foo {}", "<|>pub(crate) struct Foo {}");
        check_assist(change_visibility, "<|>mod foo {}", "<|>pub(crate) mod foo {}");
        check_assist(change_visibility, "<|>trait Foo {}", "<|>pub(crate) trait Foo {}");
        check_assist(change_visibility, "m<|>od {}", "<|>pub(crate) mod {}");
        check_assist(
            change_visibility,
            "unsafe f<|>n foo() {}",
            "<|>pub(crate) unsafe fn foo() {}",
        );
    }

    #[test]
    fn change_visibility_works_with_struct_fields() {
        check_assist(
            change_visibility,
            r"struct S { <|>field: u32 }",
            r"struct S { <|>pub(crate) field: u32 }",
        );
        check_assist(change_visibility, r"struct S ( <|>u32 )", r"struct S ( <|>pub(crate) u32 )");
    }

    #[test]
    fn change_visibility_field_false_positive() {
        mark::check!(change_visibility_field_false_positive);
        check_assist_not_applicable(
            change_visibility,
            r"struct S { field: [(); { let <|>x = ();}] }",
        )
    }

    #[test]
    fn change_visibility_pub_to_pub_crate() {
        check_assist(change_visibility, "<|>pub fn foo() {}", "<|>pub(crate) fn foo() {}")
    }

    #[test]
    fn change_visibility_pub_crate_to_pub() {
        check_assist(change_visibility, "<|>pub(crate) fn foo() {}", "<|>pub fn foo() {}")
    }

    #[test]
    fn change_visibility_const() {
        check_assist(change_visibility, "<|>const FOO = 3u8;", "<|>pub(crate) const FOO = 3u8;");
    }

    #[test]
    fn change_visibility_handles_comment_attrs() {
        check_assist(
            change_visibility,
            r"
            /// docs

            // comments

            #[derive(Debug)]
            <|>struct Foo;
            ",
            r"
            /// docs

            // comments

            #[derive(Debug)]
            <|>pub(crate) struct Foo;
            ",
        )
    }

    #[test]
    fn not_applicable_for_enum_variants() {
        check_assist_not_applicable(
            change_visibility,
            r"mod foo { pub enum Foo {Foo1} }
              fn main() { foo::Foo::Foo1<|> } ",
        );
    }

    #[test]
    fn change_visibility_target() {
        check_assist_target(change_visibility, "<|>fn foo() {}", "fn");
        check_assist_target(change_visibility, "pub(crate)<|> fn foo() {}", "pub(crate)");
        check_assist_target(change_visibility, "struct S { <|>field: u32 }", "field");
    }
}
<commit_msg>Don't set cursor in change_visibility<commit_after>use ra_syntax::{
    ast::{self, NameOwner, VisibilityOwner},
    AstNode,
    SyntaxKind::{
        ATTR, COMMENT, CONST_DEF, ENUM_DEF, FN_DEF, MODULE, STRUCT_DEF, TRAIT_DEF, VISIBILITY,
        WHITESPACE,
    },
    SyntaxNode, TextSize, T,
};
use test_utils::mark;

use crate::{AssistContext, AssistId, Assists};

// Assist: change_visibility
//
// Adds or changes existing visibility specifier.
//
// ```
// <|>fn frobnicate() {}
// ```
// ->
// ```
// pub(crate) fn frobnicate() {}
// ```
pub(crate) fn change_visibility(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
    if let Some(vis) = ctx.find_node_at_offset::<ast::Visibility>() {
        return change_vis(acc, vis);
    }
    add_vis(acc, ctx)
}

fn add_vis(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
    let item_keyword = ctx.token_at_offset().find(|leaf| match leaf.kind() {
        T![const] | T![fn] | T![mod] | T![struct] | T![enum] | T![trait] => true,
        _ => false,
    });

    let (offset, target) = if let Some(keyword) = item_keyword {
        let parent = keyword.parent();
        let def_kws = vec![CONST_DEF, FN_DEF, MODULE, STRUCT_DEF, ENUM_DEF, TRAIT_DEF];
        // Parent is not a definition, can't add visibility
        if !def_kws.iter().any(|&def_kw| def_kw == parent.kind()) {
            return None;
        }
        // Already have visibility, do nothing
        if parent.children().any(|child| child.kind() == VISIBILITY) {
            return None;
        }
        (vis_offset(&parent), keyword.text_range())
    } else if let Some(field_name) = ctx.find_node_at_offset::<ast::Name>() {
        let field = field_name.syntax().ancestors().find_map(ast::RecordFieldDef::cast)?;
        if field.name()? != field_name {
            mark::hit!(change_visibility_field_false_positive);
            return None;
        }
        if field.visibility().is_some() {
            return None;
        }
        (vis_offset(field.syntax()), field_name.syntax().text_range())
    } else if let Some(field) = ctx.find_node_at_offset::<ast::TupleFieldDef>() {
        if field.visibility().is_some() {
            return None;
        }
        (vis_offset(field.syntax()), field.syntax().text_range())
    } else {
        return None;
    };

    acc.add(AssistId("change_visibility"), "Change visibility to pub(crate)", target, |edit| {
        edit.insert(offset, "pub(crate) ");
    })
}

fn vis_offset(node: &SyntaxNode) -> TextSize {
    node.children_with_tokens()
        .skip_while(|it| match it.kind() {
            WHITESPACE | COMMENT | ATTR => true,
            _ => false,
        })
        .next()
        .map(|it| it.text_range().start())
        .unwrap_or_else(|| node.text_range().start())
}

fn change_vis(acc: &mut Assists, vis: ast::Visibility) -> Option<()> {
    if vis.syntax().text() == "pub" {
        let target = vis.syntax().text_range();
        return acc.add(
            AssistId("change_visibility"),
            "Change Visibility to pub(crate)",
            target,
            |edit| {
                edit.replace(vis.syntax().text_range(), "pub(crate)");
            },
        );
    }
    if vis.syntax().text() == "pub(crate)" {
        let target = vis.syntax().text_range();
        return acc.add(
            AssistId("change_visibility"),
            "Change visibility to pub",
            target,
            |edit| {
                edit.replace(vis.syntax().text_range(), "pub");
            },
        );
    }
    None
}

#[cfg(test)]
mod tests {
    use test_utils::mark;

    use crate::tests::{check_assist, check_assist_not_applicable, check_assist_target};

    use super::*;

    #[test]
    fn change_visibility_adds_pub_crate_to_items() {
        check_assist(change_visibility, "<|>fn foo() {}", "<|>pub(crate) fn foo() {}");
        check_assist(change_visibility, "f<|>n foo() {}", "pub(crate) f<|>n foo() {}");
        check_assist(change_visibility, "<|>struct Foo {}", "<|>pub(crate) struct Foo {}");
        check_assist(change_visibility, "<|>mod foo {}", "<|>pub(crate) mod foo {}");
        check_assist(change_visibility, "<|>trait Foo {}", "<|>pub(crate) trait Foo {}");
        check_assist(change_visibility, "m<|>od {}", "pub(crate) m<|>od {}");
        check_assist(
            change_visibility,
            "unsafe f<|>n foo() {}",
            "pub(crate) unsafe f<|>n foo() {}",
        );
    }

    #[test]
    fn change_visibility_works_with_struct_fields() {
        check_assist(
            change_visibility,
            r"struct S { <|>field: u32 }",
            r"struct S { <|>pub(crate) field: u32 }",
        );
        check_assist(change_visibility, r"struct S ( <|>u32 )", r"struct S ( <|>pub(crate) u32 )");
    }

    #[test]
    fn change_visibility_field_false_positive() {
        mark::check!(change_visibility_field_false_positive);
        check_assist_not_applicable(
            change_visibility,
            r"struct S { field: [(); { let <|>x = ();}] }",
        )
    }

    #[test]
    fn change_visibility_pub_to_pub_crate() {
        check_assist(change_visibility, "<|>pub fn foo() {}", "<|>pub(crate) fn foo() {}")
    }

    #[test]
    fn change_visibility_pub_crate_to_pub() {
        check_assist(change_visibility, "<|>pub(crate) fn foo() {}", "<|>pub fn foo() {}")
    }

    #[test]
    fn change_visibility_const() {
        check_assist(change_visibility, "<|>const FOO = 3u8;", "<|>pub(crate) const FOO = 3u8;");
    }

    #[test]
    fn change_visibility_handles_comment_attrs() {
        check_assist(
            change_visibility,
            r"
            /// docs

            // comments

            #[derive(Debug)]
            <|>struct Foo;
            ",
            r"
            /// docs

            // comments

            #[derive(Debug)]
            <|>pub(crate) struct Foo;
            ",
        )
    }

    #[test]
    fn not_applicable_for_enum_variants() {
        check_assist_not_applicable(
            change_visibility,
            r"mod foo { pub enum Foo {Foo1} }
              fn main() { foo::Foo::Foo1<|> } ",
        );
    }

    #[test]
    fn change_visibility_target() {
        check_assist_target(change_visibility, "<|>fn foo() {}", "fn");
        check_assist_target(change_visibility, "pub(crate)<|> fn foo() {}", "pub(crate)");
        check_assist_target(change_visibility, "struct S { <|>field: u32 }", "field");
    }
}
<|endoftext|>


<commit_before>#![feature(custom_test_frameworks)]
// Custom test runner, to avoid libtest being wrapped around compiletest which wraps libtest.
#![test_runner(test_runner)]

use std::path::PathBuf;
use std::env;

use compiletest_rs as compiletest;
use colored::*;

fn miri_path() -> PathBuf {
    if rustc_test_suite().is_some() {
        PathBuf::from(option_env!("MIRI_PATH").unwrap())
    } else {
        PathBuf::from(concat!("target/", env!("PROFILE"), "/miri"))
    }
}

fn rustc_test_suite() -> Option<PathBuf> {
    option_env!("RUSTC_TEST_SUITE").map(PathBuf::from)
}

fn rustc_lib_path() -> PathBuf {
    option_env!("RUSTC_LIB_PATH").unwrap().into()
}

fn run_tests(mode: &str, path: &str, target: &str, mut flags: Vec<String>) {
    // Some flags we always want.
    flags.push("-Dwarnings -Dunused".to_owned()); // overwrite the -Aunused in compiletest-rs
    flags.push("--edition 2018".to_owned());
    if let Ok(sysroot) = std::env::var("MIRI_SYSROOT") {
        flags.push(format!("--sysroot {}", sysroot));
    }

    // The rest of the configuration.
    let mut config = compiletest::Config::default().tempdir();
    config.mode = mode.parse().expect("Invalid mode");
    config.rustc_path = miri_path();
    if rustc_test_suite().is_some() {
        config.run_lib_path = rustc_lib_path();
        config.compile_lib_path = rustc_lib_path();
    }
    config.filter = env::args().nth(1);
    config.host = get_host();
    config.src_base = PathBuf::from(path);
    config.target = target.to_owned();
    config.target_rustcflags = Some(flags.join(" "));
    compiletest::run_tests(&config);
}

fn compile_fail(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running compile-fail tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        // Optimizing too aggressivley makes UB detection harder, but test at least
        // the default value.
        // FIXME: Opt level 3 ICEs during stack trace generation.
        flags.push("-Zmir-opt-level=1".to_owned());
    }

    run_tests("compile-fail", path, target, flags);
}

fn miri_pass(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running run-pass tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        flags.push("-Zmir-opt-level=3".to_owned());
    }

    run_tests("ui", path, target, flags);
}

fn get_host() -> String {
    let rustc = rustc_test_suite().unwrap_or(PathBuf::from("rustc"));
    let rustc_version = std::process::Command::new(rustc)
        .arg("-vV")
        .output()
        .expect("rustc not found for -vV")
        .stdout;
    let rustc_version = std::str::from_utf8(&rustc_version).expect("rustc -vV is not utf8");
    let version_meta = rustc_version::version_meta_for(&rustc_version)
        .expect("failed to parse rustc version info");
    version_meta.host
}

fn get_target() -> String {
    std::env::var("MIRI_TEST_TARGET").unwrap_or_else(|_| get_host())
}

fn run_pass_miri(opt: bool) {
    miri_pass("tests/run-pass", &get_target(), opt);
}

fn compile_fail_miri(opt: bool) {
    compile_fail("tests/compile-fail", &get_target(), opt);
    if rustc_test_suite().is_none() {
        // FIXME: Some tests disabled in rustc test suite because
        // they run with a debug-assertion libstd which changes the errors.
        // We should build our own libstd for testing, see
        // <https://github.com/rust-lang/rust/issues/61833>.
        compile_fail("tests/compile-fail-norustc", &get_target(), opt);
    }
}

fn test_runner(_tests: &[&()]) {
    run_pass_miri(false);
    run_pass_miri(true);

    compile_fail_miri(false);
    compile_fail_miri(true);
}
<commit_msg>test suite: be fine with warnings when running on rustc CI<commit_after>#![feature(custom_test_frameworks)]
// Custom test runner, to avoid libtest being wrapped around compiletest which wraps libtest.
#![test_runner(test_runner)]

use std::path::PathBuf;
use std::env;

use compiletest_rs as compiletest;
use colored::*;

fn miri_path() -> PathBuf {
    if rustc_test_suite().is_some() {
        PathBuf::from(option_env!("MIRI_PATH").unwrap())
    } else {
        PathBuf::from(concat!("target/", env!("PROFILE"), "/miri"))
    }
}

fn rustc_test_suite() -> Option<PathBuf> {
    option_env!("RUSTC_TEST_SUITE").map(PathBuf::from)
}

fn rustc_lib_path() -> PathBuf {
    option_env!("RUSTC_LIB_PATH").unwrap().into()
}

fn run_tests(mode: &str, path: &str, target: &str, mut flags: Vec<String>) {
    // Some flags we always want.
    if rustc_test_suite().is_none() {
      // Only `-Dwarnings` on the Miri side to make the rustc toolstate management less painful.
      // (We often get warnings when e.g. a feature gets stabilized or some lint gets added/improved.)
      flags.push("-Dwarnings -Dunused".to_owned()); // overwrite the -Aunused in compiletest-rs
    }
    flags.push("--edition 2018".to_owned());
    if let Ok(sysroot) = std::env::var("MIRI_SYSROOT") {
        flags.push(format!("--sysroot {}", sysroot));
    }

    // The rest of the configuration.
    let mut config = compiletest::Config::default().tempdir();
    config.mode = mode.parse().expect("Invalid mode");
    config.rustc_path = miri_path();
    if rustc_test_suite().is_some() {
        config.run_lib_path = rustc_lib_path();
        config.compile_lib_path = rustc_lib_path();
    }
    config.filter = env::args().nth(1);
    config.host = get_host();
    config.src_base = PathBuf::from(path);
    config.target = target.to_owned();
    config.target_rustcflags = Some(flags.join(" "));
    compiletest::run_tests(&config);
}

fn compile_fail(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running compile-fail tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        // Optimizing too aggressivley makes UB detection harder, but test at least
        // the default value.
        // FIXME: Opt level 3 ICEs during stack trace generation.
        flags.push("-Zmir-opt-level=1".to_owned());
    }

    run_tests("compile-fail", path, target, flags);
}

fn miri_pass(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running run-pass tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        flags.push("-Zmir-opt-level=3".to_owned());
    }

    run_tests("ui", path, target, flags);
}

fn get_host() -> String {
    let rustc = rustc_test_suite().unwrap_or(PathBuf::from("rustc"));
    let rustc_version = std::process::Command::new(rustc)
        .arg("-vV")
        .output()
        .expect("rustc not found for -vV")
        .stdout;
    let rustc_version = std::str::from_utf8(&rustc_version).expect("rustc -vV is not utf8");
    let version_meta = rustc_version::version_meta_for(&rustc_version)
        .expect("failed to parse rustc version info");
    version_meta.host
}

fn get_target() -> String {
    std::env::var("MIRI_TEST_TARGET").unwrap_or_else(|_| get_host())
}

fn run_pass_miri(opt: bool) {
    miri_pass("tests/run-pass", &get_target(), opt);
}

fn compile_fail_miri(opt: bool) {
    compile_fail("tests/compile-fail", &get_target(), opt);
    if rustc_test_suite().is_none() {
        // FIXME: Some tests disabled in rustc test suite because
        // they run with a debug-assertion libstd which changes the errors.
        // We should build our own libstd for testing, see
        // <https://github.com/rust-lang/rust/issues/61833>.
        compile_fail("tests/compile-fail-norustc", &get_target(), opt);
    }
}

fn test_runner(_tests: &[&()]) {
    run_pass_miri(false);
    run_pass_miri(true);

    compile_fail_miri(false);
    compile_fail_miri(true);
}
<|endoftext|>


<commit_before>// Copyright 2020 The Tink-Rust Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

//! Provides an implementation of PRF using HKDF.

use tink::{proto::HashType, TinkError};

// We use a somewhat larger minimum key size than usual, because PRFs might be
// used by many users, in which case the security can degrade by a factor
// depending on the number of users. (Discussed for example in
// https://eprint.iacr.org/2012/159)
const MIN_HKDF_KEY_SIZE_IN_BYTES: usize = 32;

// `HkdfPrf` is a type that can be used to compute several HKDFs with the same key material.
#[derive(Clone)]
pub struct HkdfPrf {
    prk: HkdfPrfVariant,
}

#[derive(Clone)]
enum HkdfPrfVariant {
    Sha1(hkdf::Hkdf<sha1::Sha1>),
    Sha256(hkdf::Hkdf<sha2::Sha256>),
    Sha384(hkdf::Hkdf<sha2::Sha384>),
    Sha512(hkdf::Hkdf<sha2::Sha512>),
}

impl HkdfPrf {
    /// Create a new [`HkdfPrf`] object and initialize it with the correct key material.
    pub fn new(hash_alg: HashType, key: &[u8], salt: &[u8]) -> Result<HkdfPrf, TinkError> {
        let prk = match hash_alg {
            HashType::Sha1 => HkdfPrfVariant::Sha1(hkdf::Hkdf::<sha1::Sha1>::new(Some(salt), key)),
            HashType::Sha256 => {
                HkdfPrfVariant::Sha256(hkdf::Hkdf::<sha2::Sha256>::new(Some(salt), key))
            }
            HashType::Sha384 => {
                HkdfPrfVariant::Sha384(hkdf::Hkdf::<sha2::Sha384>::new(Some(salt), key))
            }
            HashType::Sha512 => {
                HkdfPrfVariant::Sha512(hkdf::Hkdf::<sha2::Sha512>::new(Some(salt), key))
            }
            h => return Err(format!("HkdfPrf: unsupported hash {:?}", h).into()),
        };
        Ok(HkdfPrf { prk })
    }
}

/// Validate parameters of HKDF constructor.
pub fn validate_hkdf_prf_params(
    hash: HashType,
    key_size: usize,
    _salt: &[u8],
) -> Result<(), TinkError> {
    // validate key size
    if key_size < MIN_HKDF_KEY_SIZE_IN_BYTES {
        Err("key too short".into())
    } else if tink::subtle::get_hash_func(hash).is_none() {
        Err("invalid hash function".into())
    } else if hash != HashType::Sha256 && hash != HashType::Sha512 {
        Err("Only SHA-256 and SHA-512 currently allowed for HKDF".into())
    } else {
        Ok(())
    }
}

impl tink::Prf for HkdfPrf {
    fn compute_prf(&self, data: &[u8], out_len: usize) -> Result<Vec<u8>, TinkError> {
        match &self.prk {
            HkdfPrfVariant::Sha1(prk) => compute_hkdf_with::<sha1::Sha1>(prk, data, out_len),
            HkdfPrfVariant::Sha256(prk) => compute_hkdf_with::<sha2::Sha256>(prk, data, out_len),
            HkdfPrfVariant::Sha384(prk) => compute_hkdf_with::<sha2::Sha384>(prk, data, out_len),
            HkdfPrfVariant::Sha512(prk) => compute_hkdf_with::<sha2::Sha512>(prk, data, out_len),
        }
    }
}

fn compute_hkdf_with<D>(
    prk: &hkdf::Hkdf<D>,
    data: &[u8],
    out_len: usize,
) -> Result<Vec<u8>, TinkError>
where
    D: digest::Update + digest::BlockInput + digest::FixedOutput + digest::Reset + Default + Clone,
{
    let mut okm = vec![0; out_len];
    prk.expand(data, &mut okm)
        .map_err(|_| TinkError::new("HkdfPrf: compute of hkdf failed"))?;

    Ok(okm)
}
<commit_msg>tink-prf: fix doc comment<commit_after>// Copyright 2020 The Tink-Rust Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

//! Provides an implementation of PRF using HKDF.

use tink::{proto::HashType, TinkError};

// We use a somewhat larger minimum key size than usual, because PRFs might be
// used by many users, in which case the security can degrade by a factor
// depending on the number of users. (Discussed for example in
// https://eprint.iacr.org/2012/159)
const MIN_HKDF_KEY_SIZE_IN_BYTES: usize = 32;

/// `HkdfPrf` is a type that can be used to compute several HKDFs with the same key material.
#[derive(Clone)]
pub struct HkdfPrf {
    prk: HkdfPrfVariant,
}

#[derive(Clone)]
enum HkdfPrfVariant {
    Sha1(hkdf::Hkdf<sha1::Sha1>),
    Sha256(hkdf::Hkdf<sha2::Sha256>),
    Sha384(hkdf::Hkdf<sha2::Sha384>),
    Sha512(hkdf::Hkdf<sha2::Sha512>),
}

impl HkdfPrf {
    /// Create a new [`HkdfPrf`] object and initialize it with the correct key material.
    pub fn new(hash_alg: HashType, key: &[u8], salt: &[u8]) -> Result<HkdfPrf, TinkError> {
        let prk = match hash_alg {
            HashType::Sha1 => HkdfPrfVariant::Sha1(hkdf::Hkdf::<sha1::Sha1>::new(Some(salt), key)),
            HashType::Sha256 => {
                HkdfPrfVariant::Sha256(hkdf::Hkdf::<sha2::Sha256>::new(Some(salt), key))
            }
            HashType::Sha384 => {
                HkdfPrfVariant::Sha384(hkdf::Hkdf::<sha2::Sha384>::new(Some(salt), key))
            }
            HashType::Sha512 => {
                HkdfPrfVariant::Sha512(hkdf::Hkdf::<sha2::Sha512>::new(Some(salt), key))
            }
            h => return Err(format!("HkdfPrf: unsupported hash {:?}", h).into()),
        };
        Ok(HkdfPrf { prk })
    }
}

/// Validate parameters of HKDF constructor.
pub fn validate_hkdf_prf_params(
    hash: HashType,
    key_size: usize,
    _salt: &[u8],
) -> Result<(), TinkError> {
    // validate key size
    if key_size < MIN_HKDF_KEY_SIZE_IN_BYTES {
        Err("key too short".into())
    } else if tink::subtle::get_hash_func(hash).is_none() {
        Err("invalid hash function".into())
    } else if hash != HashType::Sha256 && hash != HashType::Sha512 {
        Err("Only SHA-256 and SHA-512 currently allowed for HKDF".into())
    } else {
        Ok(())
    }
}

impl tink::Prf for HkdfPrf {
    fn compute_prf(&self, data: &[u8], out_len: usize) -> Result<Vec<u8>, TinkError> {
        match &self.prk {
            HkdfPrfVariant::Sha1(prk) => compute_hkdf_with::<sha1::Sha1>(prk, data, out_len),
            HkdfPrfVariant::Sha256(prk) => compute_hkdf_with::<sha2::Sha256>(prk, data, out_len),
            HkdfPrfVariant::Sha384(prk) => compute_hkdf_with::<sha2::Sha384>(prk, data, out_len),
            HkdfPrfVariant::Sha512(prk) => compute_hkdf_with::<sha2::Sha512>(prk, data, out_len),
        }
    }
}

fn compute_hkdf_with<D>(
    prk: &hkdf::Hkdf<D>,
    data: &[u8],
    out_len: usize,
) -> Result<Vec<u8>, TinkError>
where
    D: digest::Update + digest::BlockInput + digest::FixedOutput + digest::Reset + Default + Clone,
{
    let mut okm = vec![0; out_len];
    prk.expand(data, &mut okm)
        .map_err(|_| TinkError::new("HkdfPrf: compute of hkdf failed"))?;

    Ok(okm)
}
<|endoftext|>


<commit_before>macro_rules! replace(
    ($name:ident, $which:ident, $re:expr,
     $search:expr, $replace:expr, $result:expr) => (
        #[test]
        fn $name() {
            let re = regex!($re);
            assert_eq!(re.$which(text!($search), $replace), text!($result));
        }
    );
);

replace!(first, replace, r"\d", "age: 26", t!("Z"), "age: Z6");
replace!(plus, replace, r"\d+", "age: 26", t!("Z"), "age: Z");
replace!(all, replace_all, r"\d", "age: 26", t!("Z"), "age: ZZ");
replace!(groups, replace, r"(\S+)\s+(\S+)", "w1 w2", t!("$2 $1"), "w2 w1");
replace!(double_dollar, replace,
         r"(\S+)\s+(\S+)", "w1 w2", t!("$2 $$1"), "w2 $1");
// replace!(adjacent_index, replace,
         // r"([^aeiouy])ies$", "skies", t!("$1y"), "sky");
replace!(named, replace_all,
         r"(?P<first>\S+)\s+(?P<last>\S+)(?P<space>\s*)",
         "w1 w2 w3 w4", t!("$last $first$space"), "w2 w1 w4 w3");
replace!(trim, replace_all, "^[ \t]+|[ \t]+$", " \t  trim me\t   \t",
         t!(""), "trim me");
replace!(number_hypen, replace, r"(.)(.)", "ab", t!("$1-$2"), "a-b");
// replace!(number_underscore, replace, r"(.)(.)", "ab", t!("$1_$2"), "a_b");
replace!(simple_expand, replace_all, r"(\w) (\w)", "a b", t!("$2 $1"), "b a");
replace!(literal_dollar1, replace_all,
         r"(\w+) (\w+)", "a b", t!("$$1"), "$1");
replace!(literal_dollar2, replace_all,
         r"(\w+) (\w+)", "a b", t!("$2 $$c $1"), "b $c a");
replace!(no_expand1, replace,
         r"(\S+)\s+(\S+)", "w1 w2", no_expand!("$2 $1"), "$2 $1");
replace!(no_expand2, replace,
         r"(\S+)\s+(\S+)", "w1 w2", no_expand!("$$1"), "$$1");

// See https://github.com/rust-lang/regex/issues/314
replace!(match_at_start_replace_with_empty, replace_all, r"foo", "foobar", t!(""), "bar");

// See https://github.com/rust-lang/regex/issues/393
replace!(single_empty_match, replace, r"^", "bar", t!("foo"), "foobar");
<commit_msg>tests: added test for demoing capture names<commit_after>macro_rules! replace(
    ($name:ident, $which:ident, $re:expr,
     $search:expr, $replace:expr, $result:expr) => (
        #[test]
        fn $name() {
            let re = regex!($re);
            assert_eq!(re.$which(text!($search), $replace), text!($result));
        }
    );
);

replace!(first, replace, r"\d", "age: 26", t!("Z"), "age: Z6");
replace!(plus, replace, r"\d+", "age: 26", t!("Z"), "age: Z");
replace!(all, replace_all, r"\d", "age: 26", t!("Z"), "age: ZZ");
replace!(groups, replace, r"(\S+)\s+(\S+)", "w1 w2", t!("$2 $1"), "w2 w1");
replace!(double_dollar, replace,
         r"(\S+)\s+(\S+)", "w1 w2", t!("$2 $$1"), "w2 $1");
// replace!(adjacent_index, replace,
         // r"([^aeiouy])ies$", "skies", t!("$1y"), "sky");
replace!(named, replace_all,
         r"(?P<first>\S+)\s+(?P<last>\S+)(?P<space>\s*)",
         "w1 w2 w3 w4", t!("$last $first$space"), "w2 w1 w4 w3");
replace!(trim, replace_all, "^[ \t]+|[ \t]+$", " \t  trim me\t   \t",
         t!(""), "trim me");
replace!(number_hypen, replace, r"(.)(.)", "ab", t!("$1-$2"), "a-b");
// replace!(number_underscore, replace, r"(.)(.)", "ab", t!("$1_$2"), "a_b");
replace!(simple_expand, replace_all, r"(\w) (\w)", "a b", t!("$2 $1"), "b a");
replace!(literal_dollar1, replace_all,
         r"(\w+) (\w+)", "a b", t!("$$1"), "$1");
replace!(literal_dollar2, replace_all,
         r"(\w+) (\w+)", "a b", t!("$2 $$c $1"), "b $c a");
replace!(no_expand1, replace,
         r"(\S+)\s+(\S+)", "w1 w2", no_expand!("$2 $1"), "$2 $1");
replace!(no_expand2, replace,
         r"(\S+)\s+(\S+)", "w1 w2", no_expand!("$$1"), "$$1");

// See https://github.com/rust-lang/regex/issues/314
replace!(match_at_start_replace_with_empty, replace_all, r"foo", "foobar", t!(""), "bar");

// See https://github.com/rust-lang/regex/issues/393
replace!(single_empty_match, replace, r"^", "bar", t!("foo"), "foobar");

// See https://github.com/rust-lang/regex/issues/399
replace!(capture_longest_possible_name, replace_all, r"(.)", "b", t!("${1}a $1a"), "ba ");
<|endoftext|>