<commit_before>//! A type that can be treated as a difference.
//!
//! Differential dataflow most commonly tracks the counts associated with records in a multiset, but it
//! generalizes to tracking any map from the records to an Abelian group. The most common generalization
//! is when we maintain both a count and another accumulation, for example height. The differential
//! dataflow collections would then track for each record the total of counts and heights, which allows
//! us to track something like the average.

use std::ops::{AddAssign, Neg, Mul};
use std::iter::Iterator;

use ::Data;

pub use self::Abelian as Diff;

/// A type that can be treated as a difference.
///
/// The mathematical requirements are, I believe, an Abelian group, in that we require addition, inverses,
/// and almost certainly use commutativity somewhere (it isn't clear if it is a requirement, as it isn't
/// clear that there are semantics other than "we accumulate your differences"; I suspect we don't always
/// accumulate them in the right order, so commutativity is important until we conclude otherwise).
pub trait Monoid : for<'a> AddAssign<&'a Self> + ::std::marker::Sized + Data + Clone {
	/// Returns true if the element is the additive identity.
	///
	/// This is primarily used by differential dataflow to know when it is safe to delete an update.
	/// When a difference accumulates to zero, the difference has no effect on any accumulation and can
	/// be removed.
	#[inline]
	fn is_zero(&self) -> bool { self.eq(&Self::zero()) }
	/// The additive identity.
	///
	/// This method is primarily used by differential dataflow internals as part of consolidation, when
	/// one value is accumulated elsewhere and must be replaced by valid but harmless value.
	fn zero() -> Self;
}

/// A commutative monoid with negation.
///
/// This trait represents a commutative group, here a commutative monoid with
/// the additional support for subtraction and negation. An identity subtracted
/// from itself or added to its negation should be the zero element from the
/// underlying monoid.
pub trait Abelian : Monoid + Neg<Output=Self> { }
impl<T: Monoid + Neg<Output=Self>> Abelian for T { }

impl Monoid for isize {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i64 {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i32 {
	#[inline] fn zero() -> Self { 0 }
}

/// The difference defined by a pair of difference elements.
///
/// This type is essentially a "pair", though in Rust the tuple types do not derive the numeric
/// traits we require, and so we need to emulate the types ourselves. In the interest of ergonomics,
/// we may eventually replace the numeric traits with our own, so that we can implement them for
/// tuples and allow users to ignore details like these.
#[derive(Abomonation, Copy, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
pub struct DiffPair<R1, R2> {
	/// The first element in the pair.
	pub element1: R1,
	/// The second element in the pair.
	pub element2: R2,
}

impl<R1, R2> DiffPair<R1, R2> {
	/// Creates a new Diff pair from two elements.
	#[inline] pub fn new(elt1: R1, elt2: R2) -> Self {
		DiffPair {
			element1: elt1,
			element2: elt2,
		}
	}
}

impl<R1: Monoid, R2: Monoid> Monoid for DiffPair<R1, R2> {
	#[inline] fn zero() -> Self {
		DiffPair {
			element1: R1::zero(),
			element2: R2::zero(),
		}
	}
}

impl<'a, R1: AddAssign<&'a R1>, R2: AddAssign<&'a R2>> AddAssign<&'a DiffPair<R1, R2>> for DiffPair<R1, R2> {
	#[inline] fn add_assign(&mut self, rhs: &'a Self) {
		self.element1 += &rhs.element1;
		self.element2 += &rhs.element2;
	}
}

impl<R1: Neg, R2: Neg> Neg for DiffPair<R1, R2> {
	type Output = DiffPair<<R1 as Neg>::Output, <R2 as Neg>::Output>;
	#[inline] fn neg(self) -> Self::Output {
		DiffPair {
			element1: -self.element1,
			element2: -self.element2,
		}
	}
}

impl<T: Copy, R1: Mul<T>, R2: Mul<T>> Mul<T> for DiffPair<R1,R2> {
	type Output = DiffPair<<R1 as Mul<T>>::Output, <R2 as Mul<T>>::Output>;
	fn mul(self, other: T) -> Self::Output {
		DiffPair::new(
			self.element1 * other,
			self.element2 * other,
		)
	}
}

// // TODO: This currently causes rustc to trip a recursion limit, because who knows why.
// impl<R1: Diff, R2: Diff> Mul<DiffPair<R1,R2>> for isize
// where isize: Mul<R1>, isize: Mul<R2>, <isize as Mul<R1>>::Output: Diff, <isize as Mul<R2>>::Output: Diff {
// 	type Output = DiffPair<<isize as Mul<R1>>::Output, <isize as Mul<R2>>::Output>;
// 	fn mul(self, other: DiffPair<R1,R2>) -> Self::Output {
// 		DiffPair::new(
// 			self * other.element1,
// 			self * other.element2,
// 		)
// 	}
// }

/// A variable number of accumulable updates.
#[derive(Abomonation, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
pub struct DiffVector<R> {
	buffer: Vec<R>,
}

impl<R> DiffVector<R> {
    /// Create new DiffVector from Vec
    #[inline(always)]
    pub fn new(vec: Vec<R>) -> DiffVector<R> {
        DiffVector { buffer: vec }
    }
}

impl<R> IntoIterator for DiffVector<R> {
    type Item = R;
    type IntoIter = ::std::vec::IntoIter<R>;
    fn into_iter(self) -> Self::IntoIter {
        self.buffer.into_iter()
    }
}

impl<R> std::ops::Deref for DiffVector<R> {
	type Target = [R];
	fn deref(&self) -> &Self::Target {
		&self.buffer[..]
	}
}

impl<R> std::ops::DerefMut for DiffVector<R> {
	fn deref_mut(&mut self) -> &mut Self::Target {
		&mut self.buffer[..]
	}
}

impl<R: Monoid> Monoid for DiffVector<R> {
	#[inline] fn is_zero(&self) -> bool {
		self.buffer.iter().all(|x| x.is_zero())
	}
	#[inline] fn zero() -> Self {
		Self { buffer: Vec::new() }
	}
}

impl<'a, R: AddAssign<&'a R>+Clone> AddAssign<&'a DiffVector<R>> for DiffVector<R> {
	#[inline]
	fn add_assign(&mut self, rhs: &'a Self) {

		// Ensure sufficient length to receive addition.
		while self.buffer.len() < rhs.buffer.len() {
			let element = &rhs.buffer[self.buffer.len()];
			self.buffer.push(element.clone());
		}

		// As other is not longer, apply updates without tests.
		for (index, update) in rhs.buffer.iter().enumerate() {
			self.buffer[index] += update;
		}
	}
}

impl<R: Neg<Output=R>+Clone> Neg for DiffVector<R> {
	type Output = DiffVector<<R as Neg>::Output>;
	#[inline]
	fn neg(mut self) -> Self::Output {
		for update in self.buffer.iter_mut() {
			*update = -update.clone();
		}
		self
	}
}

impl<T: Copy, R: Mul<T>> Mul<T> for DiffVector<R> {
	type Output = DiffVector<<R as Mul<T>>::Output>;
	fn mul(self, other: T) -> Self::Output {
		let buffer =
		self.buffer
			.into_iter()
			.map(|x| x * other)
			.collect();

		DiffVector { buffer }
	}
}
<commit_msg>extend monoid types<commit_after>//! A type that can be treated as a difference.
//!
//! Differential dataflow most commonly tracks the counts associated with records in a multiset, but it
//! generalizes to tracking any map from the records to an Abelian group. The most common generalization
//! is when we maintain both a count and another accumulation, for example height. The differential
//! dataflow collections would then track for each record the total of counts and heights, which allows
//! us to track something like the average.

use std::ops::{AddAssign, Neg, Mul};
use std::iter::Iterator;

use ::Data;

pub use self::Abelian as Diff;

/// A type that can be treated as a difference.
///
/// The mathematical requirements are, I believe, an Abelian group, in that we require addition, inverses,
/// and almost certainly use commutativity somewhere (it isn't clear if it is a requirement, as it isn't
/// clear that there are semantics other than "we accumulate your differences"; I suspect we don't always
/// accumulate them in the right order, so commutativity is important until we conclude otherwise).
pub trait Monoid : for<'a> AddAssign<&'a Self> + ::std::marker::Sized + Data + Clone {
	/// Returns true if the element is the additive identity.
	///
	/// This is primarily used by differential dataflow to know when it is safe to delete an update.
	/// When a difference accumulates to zero, the difference has no effect on any accumulation and can
	/// be removed.
	#[inline]
	fn is_zero(&self) -> bool { self.eq(&Self::zero()) }
	/// The additive identity.
	///
	/// This method is primarily used by differential dataflow internals as part of consolidation, when
	/// one value is accumulated elsewhere and must be replaced by valid but harmless value.
	fn zero() -> Self;
}

/// A commutative monoid with negation.
///
/// This trait represents a commutative group, here a commutative monoid with
/// the additional support for subtraction and negation. An identity subtracted
/// from itself or added to its negation should be the zero element from the
/// underlying monoid.
pub trait Abelian : Monoid + Neg<Output=Self> { }
impl<T: Monoid + Neg<Output=Self>> Abelian for T { }

impl Monoid for isize {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i128 {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i64 {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i32 {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i16 {
	#[inline] fn zero() -> Self { 0 }
}

impl Monoid for i8 {
	#[inline] fn zero() -> Self { 0 }
}

/// The difference defined by a pair of difference elements.
///
/// This type is essentially a "pair", though in Rust the tuple types do not derive the numeric
/// traits we require, and so we need to emulate the types ourselves. In the interest of ergonomics,
/// we may eventually replace the numeric traits with our own, so that we can implement them for
/// tuples and allow users to ignore details like these.
#[derive(Abomonation, Copy, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
pub struct DiffPair<R1, R2> {
	/// The first element in the pair.
	pub element1: R1,
	/// The second element in the pair.
	pub element2: R2,
}

impl<R1, R2> DiffPair<R1, R2> {
	/// Creates a new Diff pair from two elements.
	#[inline] pub fn new(elt1: R1, elt2: R2) -> Self {
		DiffPair {
			element1: elt1,
			element2: elt2,
		}
	}
}

impl<R1: Monoid, R2: Monoid> Monoid for DiffPair<R1, R2> {
	#[inline] fn zero() -> Self {
		DiffPair {
			element1: R1::zero(),
			element2: R2::zero(),
		}
	}
}

impl<'a, R1: AddAssign<&'a R1>, R2: AddAssign<&'a R2>> AddAssign<&'a DiffPair<R1, R2>> for DiffPair<R1, R2> {
	#[inline] fn add_assign(&mut self, rhs: &'a Self) {
		self.element1 += &rhs.element1;
		self.element2 += &rhs.element2;
	}
}

impl<R1: Neg, R2: Neg> Neg for DiffPair<R1, R2> {
	type Output = DiffPair<<R1 as Neg>::Output, <R2 as Neg>::Output>;
	#[inline] fn neg(self) -> Self::Output {
		DiffPair {
			element1: -self.element1,
			element2: -self.element2,
		}
	}
}

impl<T: Copy, R1: Mul<T>, R2: Mul<T>> Mul<T> for DiffPair<R1,R2> {
	type Output = DiffPair<<R1 as Mul<T>>::Output, <R2 as Mul<T>>::Output>;
	fn mul(self, other: T) -> Self::Output {
		DiffPair::new(
			self.element1 * other,
			self.element2 * other,
		)
	}
}

// // TODO: This currently causes rustc to trip a recursion limit, because who knows why.
// impl<R1: Diff, R2: Diff> Mul<DiffPair<R1,R2>> for isize
// where isize: Mul<R1>, isize: Mul<R2>, <isize as Mul<R1>>::Output: Diff, <isize as Mul<R2>>::Output: Diff {
// 	type Output = DiffPair<<isize as Mul<R1>>::Output, <isize as Mul<R2>>::Output>;
// 	fn mul(self, other: DiffPair<R1,R2>) -> Self::Output {
// 		DiffPair::new(
// 			self * other.element1,
// 			self * other.element2,
// 		)
// 	}
// }

/// A variable number of accumulable updates.
#[derive(Abomonation, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
pub struct DiffVector<R> {
	buffer: Vec<R>,
}

impl<R> DiffVector<R> {
    /// Create new DiffVector from Vec
    #[inline(always)]
    pub fn new(vec: Vec<R>) -> DiffVector<R> {
        DiffVector { buffer: vec }
    }
}

impl<R> IntoIterator for DiffVector<R> {
    type Item = R;
    type IntoIter = ::std::vec::IntoIter<R>;
    fn into_iter(self) -> Self::IntoIter {
        self.buffer.into_iter()
    }
}

impl<R> std::ops::Deref for DiffVector<R> {
	type Target = [R];
	fn deref(&self) -> &Self::Target {
		&self.buffer[..]
	}
}

impl<R> std::ops::DerefMut for DiffVector<R> {
	fn deref_mut(&mut self) -> &mut Self::Target {
		&mut self.buffer[..]
	}
}

impl<R: Monoid> Monoid for DiffVector<R> {
	#[inline] fn is_zero(&self) -> bool {
		self.buffer.iter().all(|x| x.is_zero())
	}
	#[inline] fn zero() -> Self {
		Self { buffer: Vec::new() }
	}
}

impl<'a, R: AddAssign<&'a R>+Clone> AddAssign<&'a DiffVector<R>> for DiffVector<R> {
	#[inline]
	fn add_assign(&mut self, rhs: &'a Self) {

		// Ensure sufficient length to receive addition.
		while self.buffer.len() < rhs.buffer.len() {
			let element = &rhs.buffer[self.buffer.len()];
			self.buffer.push(element.clone());
		}

		// As other is not longer, apply updates without tests.
		for (index, update) in rhs.buffer.iter().enumerate() {
			self.buffer[index] += update;
		}
	}
}

impl<R: Neg<Output=R>+Clone> Neg for DiffVector<R> {
	type Output = DiffVector<<R as Neg>::Output>;
	#[inline]
	fn neg(mut self) -> Self::Output {
		for update in self.buffer.iter_mut() {
			*update = -update.clone();
		}
		self
	}
}

impl<T: Copy, R: Mul<T>> Mul<T> for DiffVector<R> {
	type Output = DiffVector<<R as Mul<T>>::Output>;
	fn mul(self, other: T) -> Self::Output {
		let buffer =
		self.buffer
			.into_iter()
			.map(|x| x * other)
			.collect();

		DiffVector { buffer }
	}
}
<|endoftext|>


<commit_before>/// Disk IO abstraction for rustup.
///
/// This exists to facilitate high performance extraction even though OS's are
/// imperfect beasts. For detailed design notes see the module source.
//
// When performing IO we have a choice:
// - perform some IO in this thread
// - dispatch some or all IO to another thread
// known tradeoffs:
// NFS: network latency incurred on create, chmod, close calls
// WSLv1: Defender latency incurred on close calls; mutex shared with create calls
// Windows: Defender latency incurred on close calls
// Unix: limited open file count
// Defender : CPU limited, so more service points than cores brings no gain.
// Some machines: IO limited, more service points than cores brings more efficient
// Hello world footprint ~350MB, so around 400MB to install is considered ok.
// IO utilisation.
// All systems: dispatching to a thread has some overhead.
// Basic idea then is a locally measured congestion control problem.
// Underlying system has two
// dimensions - how much work we have queued, and how much work we execute
// at once. Queued work is both memory footprint, and unless each executor
// is performing complex logic, potentially concurrent work.
// Single core machines - thread anyway, they probably don't have SSDs?
// How many service points? Blocking latency due to networks and disks
// is independent of CPU: more threads will garner more throughput up
// to actual resource service capability.
// so:
// a) measure time around each IO op from dispatch to completion.
// b) create more threads than CPUs - 2x for now (because threadpool
//    doesn't allow creating dynamically), with very shallow stacks
//    (say 1MB)
// c) keep adding work while the P95? P80? of completion stays the same
//    when pNN starts to increase either (i) we've saturated the system
//    or (ii) other work coming in has saturated the system or (iii) this
//    sort of work is a lot harder to complete. We use NN<100 to avoid
//    having jitter throttle us inappropriately. We use a high NN to
//    avoid making the system perform poorly for the user / other users
//    on shared components. Perhaps time-to-completion should be scaled by size.
// d) if we have a lot of (iii) we should respond to it the same as (i), so
//    lets reduce this to (i) and (ii). Being unable to tell the difference
//    between load we created and others, we have to throttle back when
//    the system saturates. Our most throttled position will be one service
//    worker: dispatch IO, extract the next text, wait for IO completion,
//    repeat.
// e) scaling up and down: TCP's lessons here are pretty good. So exponential
//    up - single thread and measure. two, 4 etc. When Pnn goes bad back off
//    for a time and then try again with linear increase (it could be case (ii)
//    - lots of room to experiment here; working with a time based approach is important
//    as that is the only way we can detect saturation: we are not facing
//    loss or errors in this model.
// f) data gathering: record (name, bytes, start, duration)
//    write to disk afterwards as a csv file?
pub mod immediate;
#[cfg(test)]
mod test;
pub mod threaded;

use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc::Receiver;
use std::time::{Duration, Instant};
use std::{fmt::Debug, fs::OpenOptions};

use anyhow::{Context, Result};

use crate::process;
use crate::utils::notifications::Notification;

/// Carries the implementation specific channel data into the executor.
#[derive(Debug)]
pub enum IncrementalFile {
    ImmediateReceiver,
    ThreadedReceiver(Receiver<Vec<u8>>),
}

// The basic idea is that in single threaded mode we get this pattern:
// package budget io-layer
// +<-claim->
// +-submit--------+ | write
// +-complete------+
// +<reclaim>
// .. loop ..
// In thread mode with lots of memory we want the following:
// +<-claim->
// +-submit--------+
// +<-claim->
// +-submit--------+
// .. loop .. | writes
// +-complete------+
// +<reclaim>
// +-complete------+
// +<reclaim>
// In thread mode with limited memory we want the following:
// +<-claim->
// +-submit--------+
// +<-claim->
// +-submit--------+
// .. loop up to budget .. | writes
// +-complete------+
// +<reclaim>
// +<-claim->
// +-submit--------+
// .. loop etc ..
//
// lastly we want pending IOs such as directory creation to be able to complete in the same way, so a chunk completion
// needs to be able to report back in the same fashion; folding it into the same enum will make the driver code easier to write.
//
// The implementation is done via a pair of MPSC channels. One to send data to write. In
// the immediate model, acknowledgements are sent after doing the write immediately. In the threaded model,
// acknowledgements are sent after the write completes in the thread pool handler. In the packages code the inner that
// handles iops and continues processing incremental mode files handles the connection between the acks and the budget.
// Error reporting is passed through the regular completion port, to avoid creating a new special case.

/// What kind of IO operation to perform
#[derive(Debug)]
pub enum Kind {
    Directory,
    File(Vec<u8>),
    IncrementalFile(IncrementalFile),
}

/// The details of the IO operation
#[derive(Debug)]
pub struct Item {
    /// The path to operate on
    pub full_path: PathBuf,
    /// The operation to perform
    pub kind: Kind,
    /// When the operation started
    pub start: Option<Instant>,
    /// Amount of time the operation took to finish
    pub finish: Option<Duration>,
    /// The length of the file, for files (for stats)
    pub size: Option<usize>,
    /// The result of the operation (could now be factored into CompletedIO...)
    pub result: io::Result<()>,
    /// The mode to apply
    pub mode: u32,
}

#[derive(Debug)]
pub enum CompletedIo {
    /// A submitted Item has completed
    Item(Item),
    /// An IncrementalFile has completed a single chunk
    Chunk(usize),
}

impl Item {
    pub fn make_dir(full_path: PathBuf, mode: u32) -> Self {
        Self {
            full_path,
            kind: Kind::Directory,
            start: None,
            finish: None,
            size: None,
            result: Ok(()),
            mode,
        }
    }

    pub fn write_file(full_path: PathBuf, content: Vec<u8>, mode: u32) -> Self {
        let len = content.len();
        Self {
            full_path,
            kind: Kind::File(content),
            start: None,
            finish: None,
            size: Some(len),
            result: Ok(()),
            mode,
        }
    }

    pub fn write_file_segmented<'a>(
        full_path: PathBuf,
        mode: u32,
        state: IncrementalFileState,
    ) -> Result<(Self, Box<dyn FnMut(Vec<u8>) -> bool + 'a>)> {
        let (chunk_submit, content_callback) = state.incremental_file_channel(&full_path, mode)?;
        let result = Self {
            full_path,
            kind: Kind::IncrementalFile(content_callback),
            start: None,
            finish: None,
            size: None,
            result: Ok(()),
            mode,
        };
        Ok((result, Box::new(chunk_submit)))
    }
}

// This could be a boxed trait object perhaps... but since we're looking at
// rewriting this all into an aio layer anyway, and not looking at plugging
// different backends in at this time, it can keep.
/// Implementation specific state for incremental file writes. This effectively
/// just allows the immediate codepath to get access to the Arc referenced state
/// without holding a lifetime reference to the executor, as the threaded code
/// path is all message passing.
pub enum IncrementalFileState {
    Threaded,
    Immediate(immediate::IncrementalFileState),
}

impl IncrementalFileState {
    /// Get a channel for submitting incremental file chunks to the executor
    fn incremental_file_channel(
        &self,
        path: &Path,
        mode: u32,
    ) -> Result<(Box<dyn FnMut(Vec<u8>) -> bool>, IncrementalFile)> {
        use std::sync::mpsc::channel;
        match *self {
            IncrementalFileState::Threaded => {
                let (tx, rx) = channel::<Vec<u8>>();
                let content_callback = IncrementalFile::ThreadedReceiver(rx);
                let chunk_submit = move |chunk: Vec<u8>| tx.send(chunk).is_ok();
                Ok((Box::new(chunk_submit), content_callback))
            }
            IncrementalFileState::Immediate(ref state) => {
                let content_callback = IncrementalFile::ImmediateReceiver;
                let mut writer = immediate::IncrementalFileWriter::new(path, mode, state.clone())?;
                let chunk_submit = move |chunk: Vec<u8>| writer.chunk_submit(chunk);
                Ok((Box::new(chunk_submit), content_callback))
            }
        }
    }
}

/// Trait object for performing IO. At this point the overhead
/// of trait invocation is not a bottleneck, but if it becomes
/// one we could consider an enum variant based approach instead.
pub trait Executor {
    /// Perform a single operation.
    /// During overload situations previously queued items may
    /// need to be completed before the item is accepted:
    /// consume the returned iterator.
    fn execute(&self, mut item: Item) -> Box<dyn Iterator<Item = CompletedIo> + '_> {
        item.start = Some(Instant::now());
        self.dispatch(item)
    }

    /// Actually dispatch a operation.
    /// This is called by the default execute() implementation and
    /// should not be called directly.
    fn dispatch(&self, item: Item) -> Box<dyn Iterator<Item = CompletedIo> + '_>;

    /// Wrap up any pending operations and iterate over them.
    /// All operations submitted before the join will have been
    /// returned either through ready/complete or join once join
    /// returns.
    fn join(&mut self) -> Box<dyn Iterator<Item = CompletedIo> + '_>;

    /// Iterate over completed items.
    fn completed(&self) -> Box<dyn Iterator<Item = CompletedIo> + '_>;

    /// Get any state needed for incremental file processing
    fn incremental_file_state(&self) -> IncrementalFileState;
}

/// Trivial single threaded IO to be used from executors.
/// (Crazy sophisticated ones can obviously ignore this)
pub fn perform<F: Fn(usize)>(item: &mut Item, chunk_complete_callback: F) {
    // directories: make them, TODO: register with the dir existence cache.
    // Files, write them.
    item.result = match &mut item.kind {
        Kind::Directory => create_dir(&item.full_path),
        Kind::File(ref contents) => write_file(&item.full_path, &contents, item.mode),
        Kind::IncrementalFile(incremental_file) => write_file_incremental(
            &item.full_path,
            incremental_file,
            item.mode,
            chunk_complete_callback,
        ),
    };
    item.finish = item
        .start
        .map(|s| Instant::now().saturating_duration_since(s));
}

#[allow(unused_variables)]
pub fn write_file<P: AsRef<Path>, C: AsRef<[u8]>>(
    path: P,
    contents: C,
    mode: u32,
) -> io::Result<()> {
    let mut opts = OpenOptions::new();
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        opts.mode(mode);
    }
    let path = path.as_ref();
    let path_display = format!("{}", path.display());
    let mut f = {
        trace_scoped!("creat", "name": path_display);
        opts.write(true).create(true).truncate(true).open(path)?
    };
    let contents = contents.as_ref();
    let len = contents.len();
    {
        trace_scoped!("write", "name": path_display, "len": len);
        f.write_all(contents)?;
    }
    {
        trace_scoped!("close", "name:": path_display);
        drop(f);
    }
    Ok(())
}

#[allow(unused_variables)]
pub fn write_file_incremental<P: AsRef<Path>, F: Fn(usize)>(
    path: P,
    content_callback: &mut IncrementalFile,
    mode: u32,
    chunk_complete_callback: F,
) -> io::Result<()> {
    let mut opts = OpenOptions::new();
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        opts.mode(mode);
    }
    let path = path.as_ref();
    let path_display = format!("{}", path.display());
    let mut f = {
        trace_scoped!("creat", "name": path_display);
        opts.write(true).create(true).truncate(true).open(path)?
    };
    if let IncrementalFile::ThreadedReceiver(recv) = content_callback {
        loop {
            // We unwrap here because the documented only reason for recv to fail is a close by the sender, which is reading
            // from the tar file: a failed read there will propogate the error in the main thread directly.
            let contents = recv.recv().unwrap();
            let len = contents.len();
            // Length 0 vector is used for clean EOF signalling.
            if len == 0 {
                break;
            }
            {
                trace_scoped!("write_segment", "name": path_display, "len": len);
                f.write_all(&contents)?;
                chunk_complete_callback(len);
            }
        }
    } else {
        unreachable!();
    }
    {
        trace_scoped!("close", "name:": path_display);
        drop(f);
    }
    Ok(())
}

pub fn create_dir<P: AsRef<Path>>(path: P) -> io::Result<()> {
    let path = path.as_ref();
    let path_display = format!("{}", path.display());
    trace_scoped!("create_dir", "name": path_display);
    std::fs::create_dir(path)
}

/// Get the executor for disk IO.
pub fn get_executor<'a>(
    notify_handler: Option<&'a dyn Fn(Notification<'_>)>,
) -> Result<Box<dyn Executor + 'a>> {
    // If this gets lots of use, consider exposing via the config file.
    let thread_count = match process().var("RUSTUP_IO_THREADS") {
        Err(_) => num_cpus::get(),
        Ok(n) => n
            .parse::<usize>()
            .context("invalid value in RUSTUP_IO_THREADS. Must be a natural number")?,
    };
    Ok(match thread_count {
        0 | 1 => Box::new(immediate::ImmediateUnpacker::new()),
        n => Box::new(threaded::Threaded::new(notify_handler, n)),
    })
}
<commit_msg>Drop written buffers before reclaiming the budget.<commit_after>/// Disk IO abstraction for rustup.
///
/// This exists to facilitate high performance extraction even though OS's are
/// imperfect beasts. For detailed design notes see the module source.
//
// When performing IO we have a choice:
// - perform some IO in this thread
// - dispatch some or all IO to another thread
// known tradeoffs:
// NFS: network latency incurred on create, chmod, close calls
// WSLv1: Defender latency incurred on close calls; mutex shared with create calls
// Windows: Defender latency incurred on close calls
// Unix: limited open file count
// Defender : CPU limited, so more service points than cores brings no gain.
// Some machines: IO limited, more service points than cores brings more efficient
// Hello world footprint ~350MB, so around 400MB to install is considered ok.
// IO utilisation.
// All systems: dispatching to a thread has some overhead.
// Basic idea then is a locally measured congestion control problem.
// Underlying system has two
// dimensions - how much work we have queued, and how much work we execute
// at once. Queued work is both memory footprint, and unless each executor
// is performing complex logic, potentially concurrent work.
// Single core machines - thread anyway, they probably don't have SSDs?
// How many service points? Blocking latency due to networks and disks
// is independent of CPU: more threads will garner more throughput up
// to actual resource service capability.
// so:
// a) measure time around each IO op from dispatch to completion.
// b) create more threads than CPUs - 2x for now (because threadpool
//    doesn't allow creating dynamically), with very shallow stacks
//    (say 1MB)
// c) keep adding work while the P95? P80? of completion stays the same
//    when pNN starts to increase either (i) we've saturated the system
//    or (ii) other work coming in has saturated the system or (iii) this
//    sort of work is a lot harder to complete. We use NN<100 to avoid
//    having jitter throttle us inappropriately. We use a high NN to
//    avoid making the system perform poorly for the user / other users
//    on shared components. Perhaps time-to-completion should be scaled by size.
// d) if we have a lot of (iii) we should respond to it the same as (i), so
//    lets reduce this to (i) and (ii). Being unable to tell the difference
//    between load we created and others, we have to throttle back when
//    the system saturates. Our most throttled position will be one service
//    worker: dispatch IO, extract the next text, wait for IO completion,
//    repeat.
// e) scaling up and down: TCP's lessons here are pretty good. So exponential
//    up - single thread and measure. two, 4 etc. When Pnn goes bad back off
//    for a time and then try again with linear increase (it could be case (ii)
//    - lots of room to experiment here; working with a time based approach is important
//    as that is the only way we can detect saturation: we are not facing
//    loss or errors in this model.
// f) data gathering: record (name, bytes, start, duration)
//    write to disk afterwards as a csv file?
pub mod immediate;
#[cfg(test)]
mod test;
pub mod threaded;

use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc::Receiver;
use std::time::{Duration, Instant};
use std::{fmt::Debug, fs::OpenOptions};

use anyhow::{Context, Result};

use crate::process;
use crate::utils::notifications::Notification;

/// Carries the implementation specific channel data into the executor.
#[derive(Debug)]
pub enum IncrementalFile {
    ImmediateReceiver,
    ThreadedReceiver(Receiver<Vec<u8>>),
}

// The basic idea is that in single threaded mode we get this pattern:
// package budget io-layer
// +<-claim->
// +-submit--------+ | write
// +-complete------+
// +<reclaim>
// .. loop ..
// In thread mode with lots of memory we want the following:
// +<-claim->
// +-submit--------+
// +<-claim->
// +-submit--------+
// .. loop .. | writes
// +-complete------+
// +<reclaim>
// +-complete------+
// +<reclaim>
// In thread mode with limited memory we want the following:
// +<-claim->
// +-submit--------+
// +<-claim->
// +-submit--------+
// .. loop up to budget .. | writes
// +-complete------+
// +<reclaim>
// +<-claim->
// +-submit--------+
// .. loop etc ..
//
// lastly we want pending IOs such as directory creation to be able to complete in the same way, so a chunk completion
// needs to be able to report back in the same fashion; folding it into the same enum will make the driver code easier to write.
//
// The implementation is done via a pair of MPSC channels. One to send data to write. In
// the immediate model, acknowledgements are sent after doing the write immediately. In the threaded model,
// acknowledgements are sent after the write completes in the thread pool handler. In the packages code the inner that
// handles iops and continues processing incremental mode files handles the connection between the acks and the budget.
// Error reporting is passed through the regular completion port, to avoid creating a new special case.

/// What kind of IO operation to perform
#[derive(Debug)]
pub enum Kind {
    Directory,
    File(Vec<u8>),
    IncrementalFile(IncrementalFile),
}

/// The details of the IO operation
#[derive(Debug)]
pub struct Item {
    /// The path to operate on
    pub full_path: PathBuf,
    /// The operation to perform
    pub kind: Kind,
    /// When the operation started
    pub start: Option<Instant>,
    /// Amount of time the operation took to finish
    pub finish: Option<Duration>,
    /// The length of the file, for files (for stats)
    pub size: Option<usize>,
    /// The result of the operation (could now be factored into CompletedIO...)
    pub result: io::Result<()>,
    /// The mode to apply
    pub mode: u32,
}

#[derive(Debug)]
pub enum CompletedIo {
    /// A submitted Item has completed
    Item(Item),
    /// An IncrementalFile has completed a single chunk
    Chunk(usize),
}

impl Item {
    pub fn make_dir(full_path: PathBuf, mode: u32) -> Self {
        Self {
            full_path,
            kind: Kind::Directory,
            start: None,
            finish: None,
            size: None,
            result: Ok(()),
            mode,
        }
    }

    pub fn write_file(full_path: PathBuf, content: Vec<u8>, mode: u32) -> Self {
        let len = content.len();
        Self {
            full_path,
            kind: Kind::File(content),
            start: None,
            finish: None,
            size: Some(len),
            result: Ok(()),
            mode,
        }
    }

    pub fn write_file_segmented<'a>(
        full_path: PathBuf,
        mode: u32,
        state: IncrementalFileState,
    ) -> Result<(Self, Box<dyn FnMut(Vec<u8>) -> bool + 'a>)> {
        let (chunk_submit, content_callback) = state.incremental_file_channel(&full_path, mode)?;
        let result = Self {
            full_path,
            kind: Kind::IncrementalFile(content_callback),
            start: None,
            finish: None,
            size: None,
            result: Ok(()),
            mode,
        };
        Ok((result, Box::new(chunk_submit)))
    }
}

// This could be a boxed trait object perhaps... but since we're looking at
// rewriting this all into an aio layer anyway, and not looking at plugging
// different backends in at this time, it can keep.
/// Implementation specific state for incremental file writes. This effectively
/// just allows the immediate codepath to get access to the Arc referenced state
/// without holding a lifetime reference to the executor, as the threaded code
/// path is all message passing.
pub enum IncrementalFileState {
    Threaded,
    Immediate(immediate::IncrementalFileState),
}

impl IncrementalFileState {
    /// Get a channel for submitting incremental file chunks to the executor
    fn incremental_file_channel(
        &self,
        path: &Path,
        mode: u32,
    ) -> Result<(Box<dyn FnMut(Vec<u8>) -> bool>, IncrementalFile)> {
        use std::sync::mpsc::channel;
        match *self {
            IncrementalFileState::Threaded => {
                let (tx, rx) = channel::<Vec<u8>>();
                let content_callback = IncrementalFile::ThreadedReceiver(rx);
                let chunk_submit = move |chunk: Vec<u8>| tx.send(chunk).is_ok();
                Ok((Box::new(chunk_submit), content_callback))
            }
            IncrementalFileState::Immediate(ref state) => {
                let content_callback = IncrementalFile::ImmediateReceiver;
                let mut writer = immediate::IncrementalFileWriter::new(path, mode, state.clone())?;
                let chunk_submit = move |chunk: Vec<u8>| writer.chunk_submit(chunk);
                Ok((Box::new(chunk_submit), content_callback))
            }
        }
    }
}

/// Trait object for performing IO. At this point the overhead
/// of trait invocation is not a bottleneck, but if it becomes
/// one we could consider an enum variant based approach instead.
pub trait Executor {
    /// Perform a single operation.
    /// During overload situations previously queued items may
    /// need to be completed before the item is accepted:
    /// consume the returned iterator.
    fn execute(&self, mut item: Item) -> Box<dyn Iterator<Item = CompletedIo> + '_> {
        item.start = Some(Instant::now());
        self.dispatch(item)
    }

    /// Actually dispatch a operation.
    /// This is called by the default execute() implementation and
    /// should not be called directly.
    fn dispatch(&self, item: Item) -> Box<dyn Iterator<Item = CompletedIo> + '_>;

    /// Wrap up any pending operations and iterate over them.
    /// All operations submitted before the join will have been
    /// returned either through ready/complete or join once join
    /// returns.
    fn join(&mut self) -> Box<dyn Iterator<Item = CompletedIo> + '_>;

    /// Iterate over completed items.
    fn completed(&self) -> Box<dyn Iterator<Item = CompletedIo> + '_>;

    /// Get any state needed for incremental file processing
    fn incremental_file_state(&self) -> IncrementalFileState;
}

/// Trivial single threaded IO to be used from executors.
/// (Crazy sophisticated ones can obviously ignore this)
pub fn perform<F: Fn(usize)>(item: &mut Item, chunk_complete_callback: F) {
    // directories: make them, TODO: register with the dir existence cache.
    // Files, write them.
    item.result = match &mut item.kind {
        Kind::Directory => create_dir(&item.full_path),
        Kind::File(ref contents) => write_file(&item.full_path, &contents, item.mode),
        Kind::IncrementalFile(incremental_file) => write_file_incremental(
            &item.full_path,
            incremental_file,
            item.mode,
            chunk_complete_callback,
        ),
    };
    item.finish = item
        .start
        .map(|s| Instant::now().saturating_duration_since(s));
}

#[allow(unused_variables)]
pub fn write_file<P: AsRef<Path>, C: AsRef<[u8]>>(
    path: P,
    contents: C,
    mode: u32,
) -> io::Result<()> {
    let mut opts = OpenOptions::new();
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        opts.mode(mode);
    }
    let path = path.as_ref();
    let path_display = format!("{}", path.display());
    let mut f = {
        trace_scoped!("creat", "name": path_display);
        opts.write(true).create(true).truncate(true).open(path)?
    };
    let contents = contents.as_ref();
    let len = contents.len();
    {
        trace_scoped!("write", "name": path_display, "len": len);
        f.write_all(contents)?;
    }
    {
        trace_scoped!("close", "name:": path_display);
        drop(f);
    }
    Ok(())
}

#[allow(unused_variables)]
pub fn write_file_incremental<P: AsRef<Path>, F: Fn(usize)>(
    path: P,
    content_callback: &mut IncrementalFile,
    mode: u32,
    chunk_complete_callback: F,
) -> io::Result<()> {
    let mut opts = OpenOptions::new();
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        opts.mode(mode);
    }
    let path = path.as_ref();
    let path_display = format!("{}", path.display());
    let mut f = {
        trace_scoped!("creat", "name": path_display);
        opts.write(true).create(true).truncate(true).open(path)?
    };
    if let IncrementalFile::ThreadedReceiver(recv) = content_callback {
        loop {
            // We unwrap here because the documented only reason for recv to fail is a close by the sender, which is reading
            // from the tar file: a failed read there will propogate the error in the main thread directly.
            let contents = recv.recv().unwrap();
            let len = contents.len();
            // Length 0 vector is used for clean EOF signalling.
            if len == 0 {
                break;
            }
            {
                trace_scoped!("write_segment", "name": path_display, "len": len);
                f.write_all(&contents)?;
                drop(contents);
                chunk_complete_callback(len);
            }
        }
    } else {
        unreachable!();
    }
    {
        trace_scoped!("close", "name:": path_display);
        drop(f);
    }
    Ok(())
}

pub fn create_dir<P: AsRef<Path>>(path: P) -> io::Result<()> {
    let path = path.as_ref();
    let path_display = format!("{}", path.display());
    trace_scoped!("create_dir", "name": path_display);
    std::fs::create_dir(path)
}

/// Get the executor for disk IO.
pub fn get_executor<'a>(
    notify_handler: Option<&'a dyn Fn(Notification<'_>)>,
) -> Result<Box<dyn Executor + 'a>> {
    // If this gets lots of use, consider exposing via the config file.
    let thread_count = match process().var("RUSTUP_IO_THREADS") {
        Err(_) => num_cpus::get(),
        Ok(n) => n
            .parse::<usize>()
            .context("invalid value in RUSTUP_IO_THREADS. Must be a natural number")?,
    };
    Ok(match thread_count {
        0 | 1 => Box::new(immediate::ImmediateUnpacker::new()),
        n => Box::new(threaded::Threaded::new(notify_handler, n)),
    })
}
<|endoftext|>


<commit_before>//! Graphics draw state.

/// Graphics draw state used for blending, clipping and stencil rendering.
#[derive(Copy, Clone, PartialEq, Debug, PartialOrd)]
pub struct DrawState {
    /// Scissor mask to use. If set, no pixel outside of this
    /// rectangle (in screen space) will be written to as a result of rendering.
    pub scissor: Option<[u32; 4]>,
    /// Stencil test to use. If None, no stencil testing is done.
    pub stencil: Option<Stencil>,
    /// Blend function to use. If None, blending is disabled.
    pub blend: Option<Blend>,
}

impl Default for DrawState {
    fn default() -> Self {
        DrawState::new_alpha()
    }
}

impl DrawState {
    /// Uses alpha blending.
    pub fn new_alpha() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: None,
            scissor: None,
        }
    }

    /// Draws to stencil buffer with value 255.
    /// This can be used for clipping.
    pub fn new_clip() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Clip(255)),
            scissor: None,
        }
    }

    /// Tests against stencil buffer with value 255.
    /// Draws inside the shape defined by stencil buffer.
    pub fn new_inside() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Inside(255)),
            scissor: None,
        }
    }

    /// Tests against stencil buffer with value 255.
    /// Draws outside the shape defined by stencil buffer.
    pub fn new_outside() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Outside(255)),
            scissor: None,
        }
    }

    /// Sets blending.
    pub fn blend(mut self, blend: Blend) -> DrawState {
        self.blend = Some(blend);
        self
    }

    /// Sets scissor `[x, y, w, h]`.
    pub fn scissor(mut self, scissor: [u32; 4]) -> DrawState {
        self.scissor = Some(scissor);
        self
    }
}

/// The blend setting to use when drawing.
///
/// Using presets since some backends need one pipeline state object instance
/// per blending technique.
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub enum Blend {
    /// Alpha blending (allows semi-transparent pixels).
    ///
    /// ```not_rust
    /// new_dest_color = src_color * src_alpha + dest_color * (1 - src_alpha)
    /// new_dest_alpha = src_alpha + dest_alpha
    /// ```
    Alpha,
    /// Additive blending.
    ///
    /// ```not_rust
    /// new_dest_color = src_color + dest_color
    /// new_dest_alpha = src_alpha + dest_alpha
    /// ```
    Add,
    /// Additive blending with alpha channel.
    ///
    /// ```not_rust
    /// new_dest_color = src_color * src_alpha + dest_color
    /// new_dest_alpha = dest_alpha
    /// ```
    Lighter,
    /// Multiply color components.
    ///
    /// ```not_rust
    /// new_dest_color = src_color * dest_color
    /// new_dest_alpha = src_alpha * dest_alpha
    /// ```
    Multiply,
    /// Invert colors when rendering a white shape.
    ///
    /// ```not_rust
    /// new_dest_color = ref_color - src_color
    /// new_dest_alpha = dest_alpha
    /// ```
    ///
    /// When combining two fragments, subtract the destination color from a constant color
    /// using the source color as weight. Has an invert effect with the constant color
    /// as base and source color controlling displacement from the base color.
    /// A white source color and a white value results in plain invert.
    /// The output alpha is same as destination alpha.
    Invert,
}

/// Stencil buffer settings.
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub enum Stencil {
    /// Draw to stencil buffer.
    Clip(u8),
    /// Draw pixels that have stencil value.
    Inside(u8),
    /// Draw pixels that does not have stencil value.
    Outside(u8),
}
<commit_msg>Added `Stencil::Increment`<commit_after>//! Graphics draw state.

/// Graphics draw state used for blending, clipping and stencil rendering.
#[derive(Copy, Clone, PartialEq, Debug, PartialOrd)]
pub struct DrawState {
    /// Scissor mask to use. If set, no pixel outside of this
    /// rectangle (in screen space) will be written to as a result of rendering.
    pub scissor: Option<[u32; 4]>,
    /// Stencil test to use. If None, no stencil testing is done.
    pub stencil: Option<Stencil>,
    /// Blend function to use. If None, blending is disabled.
    pub blend: Option<Blend>,
}

impl Default for DrawState {
    fn default() -> Self {
        DrawState::new_alpha()
    }
}

impl DrawState {
    /// Uses alpha blending.
    pub fn new_alpha() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: None,
            scissor: None,
        }
    }

    /// Draws to stencil buffer with value 255.
    /// This can be used for clipping.
    ///
    /// For nested clipping, clear the stencil buffer and use `DrawState::new_increment`.
    pub fn new_clip() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Clip(255)),
            scissor: None,
        }
    }

    /// Increases stencil buffer.
    /// This can be used for nested clipping.
    pub fn new_increment() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Increment),
            scissor: None,
        }
    }

    /// Tests against stencil buffer with value 255.
    /// Draws inside the shape defined by stencil buffer.
    pub fn new_inside() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Inside(255)),
            scissor: None,
        }
    }

    /// Tests against stencil buffer with value 255.
    /// Draws outside the shape defined by stencil buffer.
    pub fn new_outside() -> DrawState {
        DrawState {
            blend: Some(Blend::Alpha),
            stencil: Some(Stencil::Outside(255)),
            scissor: None,
        }
    }

    /// Sets blending.
    pub fn blend(mut self, blend: Blend) -> DrawState {
        self.blend = Some(blend);
        self
    }

    /// Sets scissor `[x, y, w, h]`.
    pub fn scissor(mut self, scissor: [u32; 4]) -> DrawState {
        self.scissor = Some(scissor);
        self
    }
}

/// The blend setting to use when drawing.
///
/// Using presets since some backends need one pipeline state object instance
/// per blending technique.
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub enum Blend {
    /// Alpha blending (allows semi-transparent pixels).
    ///
    /// ```not_rust
    /// new_dest_color = src_color * src_alpha + dest_color * (1 - src_alpha)
    /// new_dest_alpha = src_alpha + dest_alpha
    /// ```
    Alpha,
    /// Additive blending.
    ///
    /// ```not_rust
    /// new_dest_color = src_color + dest_color
    /// new_dest_alpha = src_alpha + dest_alpha
    /// ```
    Add,
    /// Additive blending with alpha channel.
    ///
    /// ```not_rust
    /// new_dest_color = src_color * src_alpha + dest_color
    /// new_dest_alpha = dest_alpha
    /// ```
    Lighter,
    /// Multiply color components.
    ///
    /// ```not_rust
    /// new_dest_color = src_color * dest_color
    /// new_dest_alpha = src_alpha * dest_alpha
    /// ```
    Multiply,
    /// Invert colors when rendering a white shape.
    ///
    /// ```not_rust
    /// new_dest_color = ref_color - src_color
    /// new_dest_alpha = dest_alpha
    /// ```
    ///
    /// When combining two fragments, subtract the destination color from a constant color
    /// using the source color as weight. Has an invert effect with the constant color
    /// as base and source color controlling displacement from the base color.
    /// A white source color and a white value results in plain invert.
    /// The output alpha is same as destination alpha.
    Invert,
}

/// Stencil buffer settings.
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub enum Stencil {
    /// Draw to stencil buffer.
    Clip(u8),
    /// Draw pixels that have stencil value.
    Inside(u8),
    /// Draw pixels that does not have stencil value.
    Outside(u8),
    /// Increment stencil buffer.
    Increment,
}
<|endoftext|>


<commit_before>use engine::state::{State, StateMachine};
use engine::timing::{Duration, SteadyTime, Stopwatch};

/// Times a function and returns the elapsed time.
macro_rules! benchmark {
    ($function:expr) => {{
        let mut timer = Stopwatch::new();
        timer.restart();
        $function;
        timer.stop();
        timer.elapsed()
    }}
}

/// User-friendly facade for building games. Manages main loop.
pub struct Application {
    pub states: StateMachine,
    last_fixed_update: SteadyTime,
    fixed_step: Duration,
    delta_time: Duration,
}

impl Application {
    pub fn new<T: 'static>(initial_state: T) -> Application where T: State {
        Application {
            states: StateMachine::new(initial_state),
            last_fixed_update: SteadyTime::now(),
            fixed_step: Duration::microseconds(16666),
            delta_time: Duration::zero(),
        }
    }

    /// Starts the application and manages the game loop.
    pub fn run(&mut self) {
        self.initialize();

        loop {
            self.delta_time = benchmark!(self.advance_frame());
        }

        self.shutdown();
    }

    /// Sets up the application.
    fn initialize(&mut self) {
        self.states.start();
    }

    /// Advances the game world by one tick.
    fn advance_frame(&mut self) {
        //self.states.handle_events(&self.event_queue.poll());

        while SteadyTime::now() - self.last_fixed_update > self.fixed_step {
            self.states.fixed_update(self.fixed_step);
            //self.systems.fixed_iterate(self.fixed_step);
            self.last_fixed_update = self.last_fixed_update + self.fixed_step;
        }

        self.states.update(self.delta_time);
        //self.systems.iterate(self.delta_time);
    }

    /// Cleans up after the quit signal is received.
    fn shutdown(&mut self) {
        self.states.stop()
    }
}

<commit_msg>state stack doesn't need to be public<commit_after>use engine::state::{State, StateMachine};
use engine::timing::{Duration, SteadyTime, Stopwatch};

/// Times a function and returns the elapsed time.
macro_rules! benchmark {
    ($function:expr) => {{
        let mut timer = Stopwatch::new();
        timer.restart();
        $function;
        timer.stop();
        timer.elapsed()
    }}
}

/// User-friendly facade for building games. Manages main loop.
pub struct Application {
    states: StateMachine,
    last_fixed_update: SteadyTime,
    fixed_step: Duration,
    delta_time: Duration,
}

impl Application {
    pub fn new<T: 'static>(initial_state: T) -> Application where T: State {
        Application {
            states: StateMachine::new(initial_state),
            last_fixed_update: SteadyTime::now(),
            fixed_step: Duration::microseconds(16666),
            delta_time: Duration::zero(),
        }
    }

    /// Starts the application and manages the game loop.
    pub fn run(&mut self) {
        self.initialize();

        loop {
            self.delta_time = benchmark!(self.advance_frame());
        }

        self.shutdown();
    }

    /// Sets up the application.
    fn initialize(&mut self) {
        self.states.start();
    }

    /// Advances the game world by one tick.
    fn advance_frame(&mut self) {
        //self.states.handle_events(&self.event_queue.poll());

        while SteadyTime::now() - self.last_fixed_update > self.fixed_step {
            self.states.fixed_update(self.fixed_step);
            //self.systems.fixed_iterate(self.fixed_step);
            self.last_fixed_update = self.last_fixed_update + self.fixed_step;
        }

        self.states.update(self.delta_time);
        //self.systems.iterate(self.delta_time);
    }

    /// Cleans up after the quit signal is received.
    fn shutdown(&mut self) {
        self.states.stop()
    }
}

<|endoftext|>


<commit_before>#[crate_id = "euler"];
#[crate_type = "bin"];

extern mod extra;
extern mod common;

use std::os;
use std::iter;
use problem::{Problem, Solution};

mod problem;
mod problem_list;

fn solve_all<T: Iterator<&'static Problem<'static>>>(mut it: T) -> bool {
    let mut total_time  = 0;
    let mut solve_cnt   = 0;
    let mut all_correct = true;

    for p in it {
        let sol = p.solve();
        sol.print(true);

        total_time += sol.time();
        solve_cnt  += 1;
        all_correct &= sol.is_correct();
    }

    if solve_cnt > 1 {
        let avg_time = total_time / solve_cnt;
        Solution::new(~"AVG",   all_correct, ~"", avg_time).print(true);
        Solution::new(~"TOTAL", all_correct, ~"", total_time).print(false);
    }

    all_correct
}

fn parse_range(s: &str) -> Option<(uint, uint)> {
    if !s.contains_char('-') {
        from_str::<uint>(s).map(|n| (n, n))
    } else {
        let mut ss = s.splitn('-', 1);
        let opt_a = ss.next().and_then(|sa| from_str(sa));
        let opt_b = ss.next().and_then(|sb| from_str(sb));
        opt_a.and_then(|a| opt_b.map(|b| (a, b)))
    }
}

fn main() {
    let mut args = os::args();
    args.shift();

    let all_correct = if args.is_empty() {
        let it = problem_list::PROBLEMS.iter().map(|x| *x);
        solve_all(it)
    } else {
        let it = args.iter()
            .filter_map(|s| parse_range(*s))
            .flat_map(|(a, b)| iter::range_inclusive(a, b))
            .filter_map(|id| problem_list::PROBLEMS.bsearch(|p| p.id.cmp(&id)))
            .map(|i| problem_list::PROBLEMS[i]);
        solve_all(it)
    };

    if !all_correct { os::set_exit_status(1) }
}
<commit_msg>euler: Add ProblemIterator<commit_after>#[crate_id = "euler"];
#[crate_type = "bin"];

extern mod extra;
extern mod common;

use std::os;
use std::iter;
use problem::{Problem, Solution};

mod problem;
mod problem_list;

trait ProblemIterator {
    fn solve_all(&mut self) -> bool;
}

impl<T: Iterator<&'static Problem<'static>>> ProblemIterator for T {
    fn solve_all(&mut self) -> bool {
        let mut total_time  = 0;
        let mut solve_cnt   = 0;
        let mut all_correct = true;

        for p in &mut *self {
            let sol = p.solve();
            sol.print(true);

            total_time += sol.time();
            solve_cnt  += 1;
            all_correct &= sol.is_correct();
        }

        if solve_cnt > 1 {
            let avg_time = total_time / solve_cnt;
            Solution::new(~"AVG",   all_correct, ~"", avg_time).print(true);
            Solution::new(~"TOTAL", all_correct, ~"", total_time).print(false);
        }

        all_correct
    }
}

fn parse_range(s: &str) -> Option<(uint, uint)> {
    if !s.contains_char('-') {
        from_str::<uint>(s).map(|n| (n, n))
    } else {
        let mut ss = s.splitn('-', 1);
        let opt_a = ss.next().and_then(|sa| from_str(sa));
        let opt_b = ss.next().and_then(|sb| from_str(sb));
        opt_a.and_then(|a| opt_b.map(|b| (a, b)))
    }
}

fn main() {
    let mut args = os::args();
    args.shift();

    let all_correct = if args.is_empty() {
        problem_list::PROBLEMS.iter()
            .map(|x| *x)
            .solve_all()
    } else {
        args.iter()
            .filter_map(|s| parse_range(*s))
            .flat_map(|(a, b)| iter::range_inclusive(a, b))
            .filter_map(|id| problem_list::PROBLEMS.bsearch(|p| p.id.cmp(&id)))
            .map(|i| problem_list::PROBLEMS[i])
            .solve_all()
    };

    if !all_correct { os::set_exit_status(1) }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate quote;
extern crate reqwest;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate error_chain;
#[macro_use]
extern crate lazy_static;

use std::fs::File;
use serde_json::{ Value, Map };
use std::collections::HashSet;

mod util;

use util::*;

mod errors {
    error_chain! {
        types {
            Error, ErrorKind, ResultExt, Result;
        }

        links {
        }

        foreign_links {
            ReqwestError(::reqwest::Error);
            JsonError(::serde_json::Error);
            IoError(::std::io::Error);
        }

        errors {
            MissingSource {
                description("No source for sample specified")
            }
            ExistingType(t: String) {
                display("No code generated, JSON matches existing type {}", t)
            }
        }
    }
}

pub use errors::*;

const RUST_KEYWORDS_ARR: &'static [&'static str] = &["abstract", "alignof", "as", "become", "box",
    "break", "const", "continue", "crate", "do", "else", "enum", "extern", "false", "final", "fn",
    "for", "if", "impl", "in", "let", "loop", "macro", "match", "mod", "move", "mut", "offsetof",
    "override", "priv", "proc", "pub", "pure", "ref", "return", "Self", "self", "sizeof", "static",
    "struct", "super", "trait", "true", "type", "typeof", "unsafe", "unsized", "use", "virtual",
    "where", "while", "yield"];

lazy_static! {
    static ref RUST_KEYWORDS: HashSet<&'static str> = {
        RUST_KEYWORDS_ARR.iter().cloned().collect()
    };
}

pub enum SampleSource<'a> {
    Url(&'a str),
    File(&'a str),
    Text(&'a str),
}

pub struct Options {
    use_serde: bool,
    extern_crate: bool,
    runnable_example: bool
}

impl Default for Options {
    fn default() -> Options {
        Options {
            use_serde: true,
            extern_crate: false,
            runnable_example: false
        }
    }
}

struct Ctxt {
    options: Options
}

macro_rules! some_if {
    ($cond:expr, $then:expr) => ({
        if $cond {
            Some($then)
        } else {
            None
        }
    })
}

pub fn codegen_from_sample(name: &str, source: SampleSource) -> Result<quote::Tokens> {
    let sample = get_and_parse_sample(source)?;
    let mut ctxt = Ctxt {
        options: Options::default()
    };
    let (type_name, type_def) = generate_type_from_value(&mut ctxt, name, &sample);

    let example = some_if!(ctxt.options.runnable_example, {
        ctxt.options.extern_crate = true;
        usage_example(&type_name)
    });

    let crates = some_if!(ctxt.options.extern_crate, quote! {
        #[macro_use]
        extern crate serde_derive;
        extern crate serde_json;
    });

    if type_def.is_none() && !ctxt.options.runnable_example {
        return Err(ErrorKind::ExistingType(String::from(type_name.as_str())).into());
    }

    Ok(quote! {
        #crates

        #type_def

        #example
    })
}

fn usage_example(type_id: &quote::Tokens) -> quote::Tokens {
    let var_id = quote::Ident::from(snake_case(type_id.as_str()));

    quote! {
        fn main() {
            let #var_id: #type_id = Default::default();

            let serialized = serde_json::to_string(&#var_id).unwrap();

            println!("serialized = {}", serialized);

            let deserialized: #type_id = serde_json::from_str(&serialized).unwrap();

            println!("deserialized = {:?}", deserialized);
        }
    }
}

fn generate_type_from_value(ctxt: &mut Ctxt, path: &str, value: &Value) -> (quote::Tokens, Option<quote::Tokens>) {
    match *value {
        Value::Null => (quote! { Option<::serde_json::Value> }, None),
        Value::Bool(_) => (quote! { bool }, None),
        Value::Number(ref n) => {
            if n.is_i64() {
                (quote! { i64 }, None)
            } else {
                (quote! { f64 }, None)
            }
        },
        Value::String(_) => (quote! { String }, None),
        Value::Array(ref values) => {
            generate_type_for_array(ctxt, path, values)
        },
        Value::Object(ref map) => {
            generate_struct_from_object(ctxt, path, map)
        }
    }
}

fn generate_type_for_array(ctxt: &mut Ctxt, path: &str, values: &Vec<Value>) -> (quote::Tokens, Option<quote::Tokens>) {
    let mut defs = Vec::new();
    let mut types = HashSet::new();

    for value in values.iter() {
        let (elemtype, elemtype_def) = generate_type_from_value(ctxt, path, value);
        types.insert(elemtype.into_string());
        if let Some(def) = elemtype_def {
            defs.push(def);
        }
    }

    if types.len() == 1 {
        let ident = quote::Ident::new(types.into_iter().next().unwrap());
        (quote! { Vec<#ident> }, defs.into_iter().next())
    } else {
        (quote! { Vec<::serde_json::Value> }, None)
    }
}

fn generate_struct_from_object(ctxt: &mut Ctxt, path: &str, map: &Map<String, Value>) -> (quote::Tokens, Option<quote::Tokens>) {
    let type_name = type_case(path);
    let ident = quote::Ident::new(&type_name as &str);
    let mut defs = Vec::new();

    let fields: Vec<quote::Tokens> = map.iter()
        .map(|(name, value)| {
            let mut field_name = snake_case(name);
            if RUST_KEYWORDS.contains::<str>(&field_name) {
                field_name.push_str("_field")
            }
            let rename = some_if!(&field_name != name,
                quote! { #[serde(rename = #name)] });

            let field_ident = quote::Ident::new(&field_name as &str);
            let (fieldtype, fieldtype_def) = generate_type_from_value(ctxt, name, value);
            if let Some(def) = fieldtype_def {
                defs.push(def);
            }
            quote! {
                #rename
                #field_ident: #fieldtype
            }
        })
        .collect();

    let derives = if ctxt.options.use_serde {
        quote! { #[derive(Default, Debug, Clone, Serialize, Deserialize)] }
    } else {
        quote! { #[derive(Default, Debug, Clone)] }
    };

    let code = quote! {
        #derives
        struct #ident {
            #(#fields),*
        }

        #(#defs)*
    };

    (quote! { #ident }, Some(code))
}

fn get_and_parse_sample(source: SampleSource) -> Result<Value> {
    let parse_result = match source {
        SampleSource::Url(url) => serde_json::de::from_reader(reqwest::get(url)?),
        SampleSource::File(path) => serde_json::de::from_reader(File::open(path)?),
        SampleSource::Text(text) => serde_json::from_str(text),
    };
    Ok(parse_result.chain_err(|| "Unable to parse JSON sample")?)
}
<commit_msg>shared: cleanup, use quote::{ Tokens, Ident }<commit_after>#[macro_use]
extern crate quote;
extern crate reqwest;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate error_chain;
#[macro_use]
extern crate lazy_static;

use std::fs::File;
use serde_json::{ Value, Map };
use std::collections::HashSet;
use quote::{ Tokens, Ident };

mod util;

use util::*;

mod errors {
    error_chain! {
        types {
            Error, ErrorKind, ResultExt, Result;
        }

        links {
        }

        foreign_links {
            ReqwestError(::reqwest::Error);
            JsonError(::serde_json::Error);
            IoError(::std::io::Error);
        }

        errors {
            MissingSource {
                description("No source for sample specified")
            }
            ExistingType(t: String) {
                display("No code generated, JSON matches existing type {}", t)
            }
        }
    }
}

pub use errors::*;

const RUST_KEYWORDS_ARR: &'static [&'static str] = &["abstract", "alignof", "as", "become", "box",
    "break", "const", "continue", "crate", "do", "else", "enum", "extern", "false", "final", "fn",
    "for", "if", "impl", "in", "let", "loop", "macro", "match", "mod", "move", "mut", "offsetof",
    "override", "priv", "proc", "pub", "pure", "ref", "return", "Self", "self", "sizeof", "static",
    "struct", "super", "trait", "true", "type", "typeof", "unsafe", "unsized", "use", "virtual",
    "where", "while", "yield"];

lazy_static! {
    static ref RUST_KEYWORDS: HashSet<&'static str> = {
        RUST_KEYWORDS_ARR.iter().cloned().collect()
    };
}

pub enum SampleSource<'a> {
    Url(&'a str),
    File(&'a str),
    Text(&'a str),
}

pub struct Options {
    use_serde: bool,
    extern_crate: bool,
    runnable_example: bool
}

impl Default for Options {
    fn default() -> Options {
        Options {
            use_serde: true,
            extern_crate: false,
            runnable_example: false
        }
    }
}

struct Ctxt {
    options: Options
}

macro_rules! some_if {
    ($cond:expr, $then:expr) => ({
        if $cond {
            Some($then)
        } else {
            None
        }
    })
}

pub fn codegen_from_sample(name: &str, source: SampleSource) -> Result<Tokens> {
    let sample = get_and_parse_sample(source)?;
    let mut ctxt = Ctxt {
        options: Options::default()
    };
    let (type_name, type_def) = generate_type_from_value(&mut ctxt, name, &sample);

    let example = some_if!(ctxt.options.runnable_example, {
        ctxt.options.extern_crate = true;
        usage_example(&type_name)
    });

    let crates = some_if!(ctxt.options.extern_crate, quote! {
        #[macro_use]
        extern crate serde_derive;
        extern crate serde_json;
    });

    if type_def.is_none() && !ctxt.options.runnable_example {
        return Err(ErrorKind::ExistingType(String::from(type_name.as_str())).into());
    }

    Ok(quote! {
        #crates

        #type_def

        #example
    })
}

fn usage_example(type_id: &Tokens) -> Tokens {
    let var_id = Ident::from(snake_case(type_id.as_str()));

    quote! {
        fn main() {
            let #var_id: #type_id = Default::default();

            let serialized = serde_json::to_string(&#var_id).unwrap();

            println!("serialized = {}", serialized);

            let deserialized: #type_id = serde_json::from_str(&serialized).unwrap();

            println!("deserialized = {:?}", deserialized);
        }
    }
}

fn generate_type_from_value(ctxt: &mut Ctxt, path: &str, value: &Value) -> (Tokens, Option<Tokens>) {
    match *value {
        Value::Null => (quote! { Option<::serde_json::Value> }, None),
        Value::Bool(_) => (quote! { bool }, None),
        Value::Number(ref n) => {
            if n.is_i64() {
                (quote! { i64 }, None)
            } else {
                (quote! { f64 }, None)
            }
        },
        Value::String(_) => (quote! { String }, None),
        Value::Array(ref values) => {
            generate_type_for_array(ctxt, path, values)
        },
        Value::Object(ref map) => {
            generate_struct_from_object(ctxt, path, map)
        }
    }
}

fn generate_type_for_array(ctxt: &mut Ctxt, path: &str, values: &Vec<Value>) -> (Tokens, Option<Tokens>) {
    let mut defs = Vec::new();
    let mut types = HashSet::new();

    for value in values.iter() {
        let (elemtype, elemtype_def) = generate_type_from_value(ctxt, path, value);
        types.insert(elemtype.into_string());
        if let Some(def) = elemtype_def {
            defs.push(def);
        }
    }

    if types.len() == 1 {
        let ident = Ident::new(types.into_iter().next().unwrap());
        (quote! { Vec<#ident> }, defs.into_iter().next())
    } else {
        (quote! { Vec<::serde_json::Value> }, None)
    }
}

fn generate_struct_from_object(ctxt: &mut Ctxt, path: &str, map: &Map<String, Value>) -> (Tokens, Option<Tokens>) {
    let type_name = type_case(path);
    let ident = Ident::from(type_name);
    let mut defs = Vec::new();

    let fields: Vec<Tokens> = map.iter()
        .map(|(name, value)| {
            let mut field_name = snake_case(name);
            if RUST_KEYWORDS.contains::<str>(&field_name) {
                field_name.push_str("_field")
            }
            let rename = some_if!(&field_name != name,
                quote! { #[serde(rename = #name)] });

            let field_ident = Ident::from(field_name);
            let (fieldtype, fieldtype_def) = generate_type_from_value(ctxt, name, value);
            if let Some(def) = fieldtype_def {
                defs.push(def);
            }
            quote! {
                #rename
                #field_ident: #fieldtype
            }
        })
        .collect();

    let derives = if ctxt.options.use_serde {
        quote! { #[derive(Default, Debug, Clone, Serialize, Deserialize)] }
    } else {
        quote! { #[derive(Default, Debug, Clone)] }
    };

    let code = quote! {
        #derives
        struct #ident {
            #(#fields),*
        }

        #(#defs)*
    };

    (quote! { #ident }, Some(code))
}

fn get_and_parse_sample(source: SampleSource) -> Result<Value> {
    let parse_result = match source {
        SampleSource::Url(url) => serde_json::de::from_reader(reqwest::get(url)?),
        SampleSource::File(path) => serde_json::de::from_reader(File::open(path)?),
        SampleSource::Text(text) => serde_json::from_str(text),
    };
    Ok(parse_result.chain_err(|| "Unable to parse JSON sample")?)
}
<|endoftext|>


<commit_before>/// The event system will have a trigger function that takes an
/// argument that decides if the event will be triggered this tick
/// or the next tick. The default should be next tick to prevent
/// inadvertent infinite loop bugs.
///
/// The event system can then schedule all events handlers that
/// do not mutate components to be run in parallel.
/// All handlers that do mutate the component list can be run
/// sequentially after that. Perhaps we can even schedule
/// those with non-overlapping mutable component lists to
/// run in parallel.

use std::collections::HashMap;
use std::any::Any;
use shared_mutex::{ SharedMutex };

pub mod example;

lazy_static! {
	pub static ref EventQueues: SharedMutex<HashMap<String, Box<Any+'static+Sync>>> = SharedMutex::new(HashMap::new());
}

pub fn event_queue_push<T>(event_name: &String, event: T) where T: Any+'static+Sync {
	let mut map = EventQueues.write().unwrap();
	let mut entry = map.entry(event_name.clone()).or_insert(Box::new(Vec::<T>::new()));
	let mut casted_entry = &mut **entry as &mut Any;
	let mut vec = casted_entry.downcast_mut::<Vec<T>>().unwrap();
	vec.push(event);
}

pub fn get_events<'a,T>(event_name: &String) -> &'a Vec<T> where T: Any+'static+Sync {
	let map = EventQueues.read().unwrap();
	let entry = map.get(event_name).unwrap();
	let casted_entry = entry as &Any;
	let vec = casted_entry.downcast_ref::<Vec<T>>().unwrap();
	return vec;
}

/// The event loop should trigger every n ms and execute any
/// events that are queued up. Every event has a function
/// that is called by the trigger! macro that puts the event
/// on the queue.
pub fn run_loop() {

}

/// Defines an event.
macro_rules! event {
	() => ()
}

/// Queues an event to be dispatched.
/// This means that the argument is put into the trigger queue for the
/// event and the event handlers will be invoked either at the next run loop.
macro_rules! trigger {
	//
	// Implementation idea: since we can't initialize events implicitly
	// in any way (not even through a rustc plugin atm) maybe we can
	// do it like lazy_static does it and initialize the event on the
	// first invocation of trigger.
	//
	// Initialisation that needs to happen:
	//   The trigger queue for this particular event needs to be initialized
	//   The run loop needs to be handed a reference to the trigger queue
	//
	() => ()
}

<commit_msg>woops disregard experiment<commit_after>/// The event system will have a trigger function that takes an
/// argument that decides if the event will be triggered this tick
/// or the next tick. The default should be next tick to prevent
/// inadvertent infinite loop bugs.
///
/// The event system can then schedule all events handlers that
/// do not mutate components to be run in parallel.
/// All handlers that do mutate the component list can be run
/// sequentially after that. Perhaps we can even schedule
/// those with non-overlapping mutable component lists to
/// run in parallel.

use std::collections::HashMap;
use std::any::Any;
use shared_mutex::{ SharedMutex };

pub mod example;

lazy_static! {
	pub static ref EventQueues: SharedMutex<HashMap<String, Box<Any+'static+Sync>>> = SharedMutex::new(HashMap::new());
}

pub fn event_queue_push<T>(event_name: &String, event: T) where T: Any+'static+Sync {
	let mut map = EventQueues.write().unwrap();
	let mut entry = map.entry(event_name.clone()).or_insert(Box::new(Vec::<T>::new()));
	let mut casted_entry = &mut **entry as &mut Any;
	let mut vec = casted_entry.downcast_mut::<Vec<T>>().unwrap();
	vec.push(event);
}

pub fn get_events<T>(event_name: &String) -> &Vec<T> where T: Any+'static+Sync {
	let map = EventQueues.read().unwrap();
	let entry = map.get(event_name).unwrap();
	let casted_entry = entry as &Any;
	let vec = casted_entry.downcast_ref::<Vec<T>>().unwrap();
	return vec;
}

/// The event loop should trigger every n ms and execute any
/// events that are queued up. Every event has a function
/// that is called by the trigger! macro that puts the event
/// on the queue.
pub fn run_loop() {

}

/// Defines an event.
macro_rules! event {
	() => ()
}

/// Queues an event to be dispatched.
/// This means that the argument is put into the trigger queue for the
/// event and the event handlers will be invoked either at the next run loop.
macro_rules! trigger {
	//
	// Implementation idea: since we can't initialize events implicitly
	// in any way (not even through a rustc plugin atm) maybe we can
	// do it like lazy_static does it and initialize the event on the
	// first invocation of trigger.
	//
	// Initialisation that needs to happen:
	//   The trigger queue for this particular event needs to be initialized
	//   The run loop needs to be handed a reference to the trigger queue
	//
	() => ()
}

<|endoftext|>


<commit_before>//! Endpoint for exposing crate download counts
//!
//! The enpoints for download a crate and exposing version specific
//! download counts are located in `krate::downloads`.

use std::cmp;

use crate::controllers::frontend_prelude::*;

use crate::models::{Crate, CrateVersions, Version, VersionDownload};
use crate::schema::version_downloads;
use crate::views::EncodableVersionDownload;

use crate::models::krate::to_char;

/// Handles the `GET /crates/:crate_id/downloads` route.
pub fn downloads(req: &mut dyn RequestExt) -> EndpointResult {
    use diesel::dsl::*;
    use diesel::sql_types::BigInt;

    let crate_name = &req.params()["crate_id"];
    let conn = req.db_read_only()?;
    let krate: Crate = Crate::by_name(crate_name).first(&*conn)?;

    let mut versions: Vec<Version> = krate.all_versions().load(&*conn)?;
    versions.sort_by_cached_key(|version| cmp::Reverse(semver::Version::parse(&version.num).ok()));
    let (latest_five, rest) = versions.split_at(cmp::min(5, versions.len()));

    let downloads = VersionDownload::belonging_to(latest_five)
        .filter(version_downloads::date.gt(date(now - 90.days())))
        .order(version_downloads::date.asc())
        .load(&*conn)?
        .into_iter()
        .map(VersionDownload::into)
        .collect::<Vec<EncodableVersionDownload>>();

    let sum_downloads = sql::<BigInt>("SUM(version_downloads.downloads)");
    let extra: Vec<ExtraDownload> = VersionDownload::belonging_to(rest)
        .select((
            to_char(version_downloads::date, "YYYY-MM-DD"),
            sum_downloads,
        ))
        .filter(version_downloads::date.gt(date(now - 90.days())))
        .group_by(version_downloads::date)
        .order(version_downloads::date.asc())
        .load(&*conn)?;

    #[derive(Serialize, Queryable)]
    struct ExtraDownload {
        date: String,
        downloads: i64,
    }

    Ok(req.json(&json!({
        "version_downloads": downloads,
        "meta": {
            "extra_downloads": extra,
        },
    })))
}
<commit_msg>Update documentation on krate::downloads controller<commit_after>//! Endpoint for exposing crate download counts
//!
//! The endpoint for downloading a crate and exposing version specific
//! download counts are located in `version::downloads`.

use std::cmp;

use crate::controllers::frontend_prelude::*;

use crate::models::{Crate, CrateVersions, Version, VersionDownload};
use crate::schema::version_downloads;
use crate::views::EncodableVersionDownload;

use crate::models::krate::to_char;

/// Handles the `GET /crates/:crate_id/downloads` route.
pub fn downloads(req: &mut dyn RequestExt) -> EndpointResult {
    use diesel::dsl::*;
    use diesel::sql_types::BigInt;

    let crate_name = &req.params()["crate_id"];
    let conn = req.db_read_only()?;
    let krate: Crate = Crate::by_name(crate_name).first(&*conn)?;

    let mut versions: Vec<Version> = krate.all_versions().load(&*conn)?;
    versions.sort_by_cached_key(|version| cmp::Reverse(semver::Version::parse(&version.num).ok()));
    let (latest_five, rest) = versions.split_at(cmp::min(5, versions.len()));

    let downloads = VersionDownload::belonging_to(latest_five)
        .filter(version_downloads::date.gt(date(now - 90.days())))
        .order(version_downloads::date.asc())
        .load(&*conn)?
        .into_iter()
        .map(VersionDownload::into)
        .collect::<Vec<EncodableVersionDownload>>();

    let sum_downloads = sql::<BigInt>("SUM(version_downloads.downloads)");
    let extra: Vec<ExtraDownload> = VersionDownload::belonging_to(rest)
        .select((
            to_char(version_downloads::date, "YYYY-MM-DD"),
            sum_downloads,
        ))
        .filter(version_downloads::date.gt(date(now - 90.days())))
        .group_by(version_downloads::date)
        .order(version_downloads::date.asc())
        .load(&*conn)?;

    #[derive(Serialize, Queryable)]
    struct ExtraDownload {
        date: String,
        downloads: i64,
    }

    Ok(req.json(&json!({
        "version_downloads": downloads,
        "meta": {
            "extra_downloads": extra,
        },
    })))
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Exposes the NonZero lang item which provides optimization hints.

use ops::Deref;

/// A wrapper type for raw pointers and integers that will never be
/// NULL or 0 that might allow certain optimizations.
#[lang="non_zero"]
#[deriving(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Show)]
#[experimental]
pub struct NonZero<T>(T);

impl<T> NonZero<T> {
    /// Create an instance of NonZero with the provided value.
    /// You must indeed ensure that the value is actually "non-zero".
    #[inline(always)]
    pub unsafe fn new(inner: T) -> NonZero<T> {
        NonZero(inner)
    }
}

impl<T> Deref<T> for NonZero<T> {
    #[inline]
    fn deref<'a>(&'a self) -> &'a T {
        let NonZero(ref inner) = *self;
        inner
    }
}
<commit_msg>libcore: Use Zeroable trait to try to limit what types may be used with NonZero.<commit_after>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Exposes the NonZero lang item which provides optimization hints.

use ops::Deref;

/// Unsafe trait to indicate what types are usable with the NonZero struct
pub unsafe trait Zeroable {}

unsafe impl<T> Zeroable for *const T {}
unsafe impl<T> Zeroable for *mut T {}
unsafe impl Zeroable for int {}
unsafe impl Zeroable for uint {}
unsafe impl Zeroable for i8 {}
unsafe impl Zeroable for u8 {}
unsafe impl Zeroable for i16 {}
unsafe impl Zeroable for u16 {}
unsafe impl Zeroable for i32 {}
unsafe impl Zeroable for u32 {}
unsafe impl Zeroable for i64 {}
unsafe impl Zeroable for u64 {}

/// A wrapper type for raw pointers and integers that will never be
/// NULL or 0 that might allow certain optimizations.
#[lang="non_zero"]
#[deriving(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Show)]
#[experimental]
pub struct NonZero<T: Zeroable>(T);

impl<T: Zeroable> NonZero<T> {
    /// Create an instance of NonZero with the provided value.
    /// You must indeed ensure that the value is actually "non-zero".
    #[inline(always)]
    pub unsafe fn new(inner: T) -> NonZero<T> {
        NonZero(inner)
    }
}

impl<T: Zeroable> Deref<T> for NonZero<T> {
    #[inline]
    fn deref<'a>(&'a self) -> &'a T {
        let NonZero(ref inner) = *self;
        inner
    }
}

#[cfg(test)]
mod test {
    use super::NonZero;

    #[test]
    fn test_create_nonzero_instance() {
        let _a = unsafe {
            NonZero::new(21)
        };
    }

    #[test]
    fn test_size_nonzero_in_option() {
        use mem::size_of;
        use option::Option;

        assert_eq!(size_of::<NonZero<u32>>(), size_of::<Option<NonZero<u32>>>());
    }

    #[test]
    fn test_match_on_nonzero_option() {
        use option::Some;

        let a = Some(unsafe {
            NonZero::new(42)
        });
        match a {
            Some(val) => assert_eq!(*val, 42),
            None => panic!("unexpected None while matching on Some(NonZero(_))")
        }

        match unsafe { NonZero::new(43) } {
            Some(val) => assert_eq!(*val, 43),
            None => panic!("unexpected None while matching on Some(NonZero(_))")
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! Utils for forming gzchunked streams.

mod read;
mod write;

pub use write::{Encoder, Compression};
pub use read::{Decoder};

pub struct Encode<I> {
    chunked: crate::chunked::Encode<I>,
}

impl<'a, I> Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    pub fn new(iter: I) -> Encode<I> {
        Encode {
            chunked: crate::chunked::encode(iter),
        }
    }

    fn pull(&mut self) -> std::io::Result<Option<Vec<u8>>> {
        // TODO: Customize compression.
        let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
        crate::io::copy_until(&mut self.chunked, &mut encoder, |_, encoder| {
            // TODO: Move the magic number to a constant.
            encoder.get_ref().len() >= 1024
        })?;

        let chunk = encoder.finish()?;
        if !chunk.is_empty() {
            Ok(Some(chunk))
        } else {
            Ok(None)
        }
    }
}

impl<'a, I> Iterator for Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    type Item = std::io::Result<Vec<u8>>;

    fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> {
        self.pull().transpose()
    }
}

#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn test_encode_and_decode_empty() {
        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();

        let encoded_block = encoder.next_chunk().unwrap();
        // This is because the block should at least contain gzip header.
        assert!(!encoded_block.is_empty());
        decoder.write(encoded_block.as_slice()).unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();

        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_all_in_one_block() {
        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        encoder.write(&[]).unwrap();
        encoder.write(&[1, 2, 3, 4, 5, 6, 7, 8]).unwrap();
        encoder.write(&[1]).unwrap();

        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();

        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), Some(Vec::new()));
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4, 5, 6, 7, 8]));
        assert_eq!(decoder.try_next_data(), Some(vec![1]));
        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_one_per_block() {
        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), None);

        encoder.write(&[]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(Vec::new()));
        assert_eq!(decoder.try_next_data(), None);

        encoder.write(&[1, 2, 3, 4, 5, 6, 7, 8]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4, 5, 6, 7, 8]));
        assert_eq!(decoder.try_next_data(), None);

        encoder.write(&[1]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1]));
        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_random() {
        use rand::{Rng as _, SeedableRng as _};
        let mut rng = rand::rngs::StdRng::seed_from_u64(20200509);

        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();
        let mut expected_data: Vec<Vec<u8>> = Vec::new();
        let mut decoded_data: Vec<Vec<u8>> = Vec::new();
        for _ in 0..256 {
            let size = rng.gen_range(128, 256);
            let data: Vec<u8> = (0..size).map(|_| rng.gen()).collect();
            encoder.write(data.as_slice()).unwrap();
            if let Some(block) = encoder.try_next_chunk().unwrap() {
                decoder.write(block.as_slice()).unwrap();
                while let Some(data) = decoder.try_next_data() {
                    decoded_data.push(data);
                }
            }
            expected_data.push(data);
        }
        let last_block = encoder.next_chunk().unwrap();
        decoder.write(last_block.as_slice()).unwrap();
        while let Some(data) = decoder.try_next_data() {
            decoded_data.push(data);
        }
        assert_eq!(decoded_data, expected_data);
    }

    #[test]
    fn test_encode_and_decode_max_compression() {
        let mut encoder = Encoder::new(Compression::new(9));
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_min_compression() {
        let mut encoder = Encoder::new(Compression::new(0));
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), None);
    }

    // Should panic because the underlying gzip only supports levels from 0 to 9.
    #[test]
    #[should_panic]
    fn test_huge_compression() {
        Encoder::new(Compression::new(100500));
    }
}
<commit_msg>Reverse a conditional for better readability.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! Utils for forming gzchunked streams.

mod read;
mod write;

pub use write::{Encoder, Compression};
pub use read::{Decoder};

pub struct Encode<I> {
    chunked: crate::chunked::Encode<I>,
}

impl<'a, I> Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    pub fn new(iter: I) -> Encode<I> {
        Encode {
            chunked: crate::chunked::encode(iter),
        }
    }

    fn pull(&mut self) -> std::io::Result<Option<Vec<u8>>> {
        // TODO: Customize compression.
        let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
        crate::io::copy_until(&mut self.chunked, &mut encoder, |_, encoder| {
            // TODO: Move the magic number to a constant.
            encoder.get_ref().len() >= 1024
        })?;

        let chunk = encoder.finish()?;
        if chunk.is_empty() {
            Ok(None)
        } else {
            Ok(Some(chunk))
        }
    }
}

impl<'a, I> Iterator for Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    type Item = std::io::Result<Vec<u8>>;

    fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> {
        self.pull().transpose()
    }
}

#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn test_encode_and_decode_empty() {
        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();

        let encoded_block = encoder.next_chunk().unwrap();
        // This is because the block should at least contain gzip header.
        assert!(!encoded_block.is_empty());
        decoder.write(encoded_block.as_slice()).unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();

        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_all_in_one_block() {
        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        encoder.write(&[]).unwrap();
        encoder.write(&[1, 2, 3, 4, 5, 6, 7, 8]).unwrap();
        encoder.write(&[1]).unwrap();

        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();

        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), Some(Vec::new()));
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4, 5, 6, 7, 8]));
        assert_eq!(decoder.try_next_data(), Some(vec![1]));
        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_one_per_block() {
        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), None);

        encoder.write(&[]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(Vec::new()));
        assert_eq!(decoder.try_next_data(), None);

        encoder.write(&[1, 2, 3, 4, 5, 6, 7, 8]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4, 5, 6, 7, 8]));
        assert_eq!(decoder.try_next_data(), None);

        encoder.write(&[1]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1]));
        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_random() {
        use rand::{Rng as _, SeedableRng as _};
        let mut rng = rand::rngs::StdRng::seed_from_u64(20200509);

        let mut encoder = Encoder::new(Compression::default());
        let mut decoder = Decoder::new();
        let mut expected_data: Vec<Vec<u8>> = Vec::new();
        let mut decoded_data: Vec<Vec<u8>> = Vec::new();
        for _ in 0..256 {
            let size = rng.gen_range(128, 256);
            let data: Vec<u8> = (0..size).map(|_| rng.gen()).collect();
            encoder.write(data.as_slice()).unwrap();
            if let Some(block) = encoder.try_next_chunk().unwrap() {
                decoder.write(block.as_slice()).unwrap();
                while let Some(data) = decoder.try_next_data() {
                    decoded_data.push(data);
                }
            }
            expected_data.push(data);
        }
        let last_block = encoder.next_chunk().unwrap();
        decoder.write(last_block.as_slice()).unwrap();
        while let Some(data) = decoder.try_next_data() {
            decoded_data.push(data);
        }
        assert_eq!(decoded_data, expected_data);
    }

    #[test]
    fn test_encode_and_decode_max_compression() {
        let mut encoder = Encoder::new(Compression::new(9));
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), None);
    }

    #[test]
    fn test_encode_and_decode_min_compression() {
        let mut encoder = Encoder::new(Compression::new(0));
        let mut decoder = Decoder::new();

        encoder.write(&[1, 2, 3, 4]).unwrap();
        let encoded_block = encoder.next_chunk().unwrap();
        decoder.write(encoded_block.as_slice()).unwrap();
        assert_eq!(decoder.try_next_data(), Some(vec![1, 2, 3, 4]));
        assert_eq!(decoder.try_next_data(), None);
    }

    // Should panic because the underlying gzip only supports levels from 0 to 9.
    #[test]
    #[should_panic]
    fn test_huge_compression() {
        Encoder::new(Compression::new(100500));
    }
}
<|endoftext|>