<commit_before>#![allow(dead_code)]

use core::fmt;
use core::cell::UnsafeCell;
use core::sync::atomic::AtomicUsize;
use core::sync::atomic::Ordering::{Acquire, Release};

use task::Waker;

/// A synchronization primitive for task wakeup.
///
/// Sometimes the task interested in a given event will change over time.
/// An `AtomicWaker` can coordinate concurrent notifications with the consumer
/// potentially "updating" the underlying task to wake up. This is useful in
/// scenarios where a computation completes in another thread and wants to
/// notify the consumer, but the consumer is in the process of being migrated to
/// a new logical task.
///
/// Consumers should call `register` before checking the result of a computation
/// and producers should call `notify` after producing the computation (this
/// differs from the usual `thread::park` pattern). It is also permitted for
/// `notify` to be called **before** `register`. This results in a no-op.
///
/// A single `AtomicWaker` may be reused for any number of calls to `register` or
/// `notify`.
///
/// `AtomicWaker` does not provide any memory ordering guarantees, as such the
/// user should use caution and use other synchronization primitives to guard
/// the result of the underlying computation.
pub struct AtomicWaker {
    state: AtomicUsize,
    waker: UnsafeCell<Option<Waker>>,
}

/// Initial state, the `AtomicWaker` is currently not being used.
///
/// The value `2` is picked specifically because it between the write lock &
/// read lock values. Since the read lock is represented by an incrementing
/// counter, this enables an atomic fetch_sub operation to be used for releasing
/// a lock.
const WAITING: usize = 2;

/// The `register` function has determined that the task is no longer current.
/// This implies that `AtomicWaker::register` is being called from a different
/// task than is represented by the currently stored task. The write lock is
/// obtained to update the task cell.
const LOCKED_WRITE: usize = 0;

/// At least one call to `notify` happened concurrently to `register` updating
/// the task cell. This state is detected when `register` exits the mutation
/// code and signals to `register` that it is responsible for notifying its own
/// task.
const LOCKED_WRITE_NOTIFIED: usize = 1;

/// The `notify` function has locked access to the task cell for notification.
///
/// The constant is left here mostly for documentation reasons.
#[allow(dead_code)]
const LOCKED_READ: usize = 3;

impl AtomicWaker {
    /// Create an `AtomicWaker` with no initial `Waker`
    pub fn new() -> AtomicWaker {
        // Make sure that task is Sync
        trait AssertSync: Sync {}
        impl AssertSync for Waker {}

        AtomicWaker {
            state: AtomicUsize::new(WAITING),
            waker: UnsafeCell::new(None),
        }
    }

    /// Registers the waker to be notified on calls to `notify`.
    ///
    /// The new task will take place of any previous tasks that were registered
    /// by previous calls to `register`. Any calls to `notify` that happen after
    /// a call to `register` (as defined by the memory ordering rules), will
    /// notify the `register` caller's task.
    ///
    /// It is safe to call `register` with multiple other threads concurrently
    /// calling `notify`. This will result in the `register` caller's current
    /// task being notified once.
    ///
    /// This function is safe to call concurrently, but this is generally a bad
    /// idea. Concurrent calls to `register` will attempt to register different
    /// tasks to be notified. One of the callers will win and have its task set,
    /// but there is no guarantee as to which caller will succeed.
    pub fn register(&self, waker: &Waker) {
        match self.state.compare_and_swap(WAITING, LOCKED_WRITE, Acquire) {
            WAITING => {
                unsafe {
                    // Locked acquired, update the waker cell
                    *self.waker.get() = Some(waker.clone());

                    // Release the lock. If the state transitioned to
                    // `LOCKED_NOTIFIED`, this means that an notify has been
                    // signaled, so notify the task.
                    if LOCKED_WRITE_NOTIFIED == self.state.swap(WAITING, Release) {
                        (*self.waker.get()).as_ref().unwrap().wake();
                    }
                }
            }
            LOCKED_WRITE | LOCKED_WRITE_NOTIFIED => {
                // A thread is concurrently calling `register`. This shouldn't
                // happen as it doesn't really make much sense, but it isn't
                // unsafe per se. Since two threads are concurrently trying to
                // update the task, it's undefined which one "wins" (no ordering
                // guarantees), so we can just do nothing.
            }
            state => {
                debug_assert!(state != LOCKED_WRITE, "unexpected state LOCKED_WRITE");
                debug_assert!(state != LOCKED_WRITE_NOTIFIED, "unexpected state LOCKED_WRITE_NOTIFIED");

                // Currently in a read locked state, this implies that `notify`
                // is currently being called on the old task handle. So, we call
                // notify on the new task handle
                waker.wake();
            }
        }
    }

    /// Calls `wake` on the last `Waker` passed to `register`.
    ///
    /// If `register` has not been called yet, then this does nothing.
    pub fn wake(&self) {
        let mut curr = WAITING;

        loop {
            if curr == LOCKED_WRITE {
                // Transition the state to LOCKED_NOTIFIED
                let actual = self.state.compare_and_swap(LOCKED_WRITE, LOCKED_WRITE_NOTIFIED, Release);

                if curr == actual {
                    // Success, return
                    return;
                }

                // update current state variable and try again
                curr = actual;

            } else if curr == LOCKED_WRITE_NOTIFIED {
                // Currently in `LOCKED_WRITE_NOTIFIED` state, nothing else to do.
                return;

            } else {
                // Currently in a LOCKED_READ state, so attempt to increment the
                // lock count.
                let actual = self.state.compare_and_swap(curr, curr + 1, Acquire);

                // Locked acquired
                if actual == curr {
                    // Notify the task
                    unsafe {
                        if let Some(ref waker) = *self.waker.get() {
                            waker.wake();
                        }
                    }

                    // Release the lock
                    self.state.fetch_sub(1, Release);

                    // Done
                    return;
                }

                // update current state variable and try again
                curr = actual;

            }
        }
    }
}

impl Default for AtomicWaker {
    fn default() -> Self {
        AtomicWaker::new()
    }
}

impl fmt::Debug for AtomicWaker {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        write!(fmt, "AtomicWaker")
    }
}

unsafe impl Send for AtomicWaker {}
unsafe impl Sync for AtomicWaker {}
<commit_msg>Update `AtomicWaker` docstrings to refer to `wake` instead of `notify`<commit_after>#![allow(dead_code)]

use core::fmt;
use core::cell::UnsafeCell;
use core::sync::atomic::AtomicUsize;
use core::sync::atomic::Ordering::{Acquire, Release};

use task::Waker;

/// A synchronization primitive for task wakeup.
///
/// Sometimes the task interested in a given event will change over time.
/// An `AtomicWaker` can coordinate concurrent notifications with the consumer
/// potentially "updating" the underlying task to wake up. This is useful in
/// scenarios where a computation completes in another thread and wants to
/// notify the consumer, but the consumer is in the process of being migrated to
/// a new logical task.
///
/// Consumers should call `register` before checking the result of a computation
/// and producers should call `wake` after producing the computation (this
/// differs from the usual `thread::park` pattern). It is also permitted for
/// `wake` to be called **before** `register`. This results in a no-op.
///
/// A single `AtomicWaker` may be reused for any number of calls to `register` or
/// `wake`.
///
/// `AtomicWaker` does not provide any memory ordering guarantees, as such the
/// user should use caution and use other synchronization primitives to guard
/// the result of the underlying computation.
pub struct AtomicWaker {
    state: AtomicUsize,
    waker: UnsafeCell<Option<Waker>>,
}

/// Initial state, the `AtomicWaker` is currently not being used.
///
/// The value `2` is picked specifically because it between the write lock &
/// read lock values. Since the read lock is represented by an incrementing
/// counter, this enables an atomic fetch_sub operation to be used for releasing
/// a lock.
const WAITING: usize = 2;

/// The `register` function has determined that the task is no longer current.
/// This implies that `AtomicWaker::register` is being called from a different
/// task than is represented by the currently stored task. The write lock is
/// obtained to update the task cell.
const LOCKED_WRITE: usize = 0;

/// At least one call to `wake` happened concurrently to `register` updating
/// the task cell. This state is detected when `register` exits the mutation
/// code and signals to `register` that it is responsible for notifying its own
/// task.
const LOCKED_WRITE_NOTIFIED: usize = 1;

/// The `wake` function has locked access to the task cell for notification.
///
/// The constant is left here mostly for documentation reasons.
#[allow(dead_code)]
const LOCKED_READ: usize = 3;

impl AtomicWaker {
    /// Create an `AtomicWaker` with no initial `Waker`
    pub fn new() -> AtomicWaker {
        // Make sure that task is Sync
        trait AssertSync: Sync {}
        impl AssertSync for Waker {}

        AtomicWaker {
            state: AtomicUsize::new(WAITING),
            waker: UnsafeCell::new(None),
        }
    }

    /// Registers the waker to be notified on calls to `wake`.
    ///
    /// The new task will take place of any previous tasks that were registered
    /// by previous calls to `register`. Any calls to `wake` that happen after
    /// a call to `register` (as defined by the memory ordering rules), will
    /// notify the `register` caller's task.
    ///
    /// It is safe to call `register` with multiple other threads concurrently
    /// calling `wake`. This will result in the `register` caller's current
    /// task being notified once.
    ///
    /// This function is safe to call concurrently, but this is generally a bad
    /// idea. Concurrent calls to `register` will attempt to register different
    /// tasks to be notified. One of the callers will win and have its task set,
    /// but there is no guarantee as to which caller will succeed.
    pub fn register(&self, waker: &Waker) {
        match self.state.compare_and_swap(WAITING, LOCKED_WRITE, Acquire) {
            WAITING => {
                unsafe {
                    // Locked acquired, update the waker cell
                    *self.waker.get() = Some(waker.clone());

                    // Release the lock. If the state transitioned to
                    // `LOCKED_NOTIFIED`, this means that an notify has been
                    // signaled, so notify the task.
                    if LOCKED_WRITE_NOTIFIED == self.state.swap(WAITING, Release) {
                        (*self.waker.get()).as_ref().unwrap().wake();
                    }
                }
            }
            LOCKED_WRITE | LOCKED_WRITE_NOTIFIED => {
                // A thread is concurrently calling `register`. This shouldn't
                // happen as it doesn't really make much sense, but it isn't
                // unsafe per se. Since two threads are concurrently trying to
                // update the task, it's undefined which one "wins" (no ordering
                // guarantees), so we can just do nothing.
            }
            state => {
                debug_assert!(state != LOCKED_WRITE, "unexpected state LOCKED_WRITE");
                debug_assert!(state != LOCKED_WRITE_NOTIFIED, "unexpected state LOCKED_WRITE_NOTIFIED");

                // Currently in a read locked state, this implies that `wake`
                // is currently being called on the old task handle. So, we call
                // `wake` on the new task handle
                waker.wake();
            }
        }
    }

    /// Calls `wake` on the last `Waker` passed to `register`.
    ///
    /// If `register` has not been called yet, then this does nothing.
    pub fn wake(&self) {
        let mut curr = WAITING;

        loop {
            if curr == LOCKED_WRITE {
                // Transition the state to LOCKED_NOTIFIED
                let actual = self.state.compare_and_swap(LOCKED_WRITE, LOCKED_WRITE_NOTIFIED, Release);

                if curr == actual {
                    // Success, return
                    return;
                }

                // update current state variable and try again
                curr = actual;

            } else if curr == LOCKED_WRITE_NOTIFIED {
                // Currently in `LOCKED_WRITE_NOTIFIED` state, nothing else to do.
                return;

            } else {
                // Currently in a LOCKED_READ state, so attempt to increment the
                // lock count.
                let actual = self.state.compare_and_swap(curr, curr + 1, Acquire);

                // Locked acquired
                if actual == curr {
                    // Notify the task
                    unsafe {
                        if let Some(ref waker) = *self.waker.get() {
                            waker.wake();
                        }
                    }

                    // Release the lock
                    self.state.fetch_sub(1, Release);

                    // Done
                    return;
                }

                // update current state variable and try again
                curr = actual;

            }
        }
    }
}

impl Default for AtomicWaker {
    fn default() -> Self {
        AtomicWaker::new()
    }
}

impl fmt::Debug for AtomicWaker {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        write!(fmt, "AtomicWaker")
    }
}

unsafe impl Send for AtomicWaker {}
unsafe impl Sync for AtomicWaker {}
<|endoftext|>


<commit_before>#![feature(core)]

type IdType = u32;
struct Id<T>(IdType, std::marker::PhantomData<T>);

pub struct QueueIter<'a, T: 'a> {
    objects: &'a [T],
    id_iter: std::slice::Iter<'a, Id<T>>,
}

impl<'a, T> Iterator for QueueIter<'a, T> {
    type Item = &'a T;

    fn next(&mut self) -> Option<&'a T> {
        self.id_iter.next().map(|&Id(i, _)|
            &self.objects[i as usize]
        )
    }
}

pub struct Queue<T> {
    pub objects: Vec<T>,
    indices: Vec<Id<T>>,
}

impl<T> Queue<T> {
    pub fn new() -> Queue<T> {
        Queue {
            objects: Vec::new(),
            indices: Vec::new(),
        }
    }

    fn is_ready(&self) -> bool {
        self.objects.len() == self.indices.len()
    }

    pub fn update(&mut self) {
        let ni = self.indices.len();
        if self.objects.len() > ni {
            self.indices.extend((ni.. self.objects.len()).map(|i|
                Id(i as IdType, std::marker::PhantomData)
            ));
        }else
        if self.objects.len() < ni {
            self.indices.retain(|&Id(i, _)| (i as usize) < ni);
        }
        debug_assert!(self.is_ready());
    }

    pub fn sort<F: Sized + Fn(&T, &T) -> std::cmp::Ordering>(&mut self, fun: F) {
        self.update();
        let objects = self.objects.as_slice();
        self.indices.sort_by(|&Id(a, _), &Id(b, _)|
            fun(&objects[a as usize], &objects[b as usize])
        );
    }

    pub fn iter<'a>(&'a self) -> QueueIter<'a, T> {
        assert!(self.is_ready());
        QueueIter {
            objects: self.objects.as_slice(),
            id_iter: self.indices.iter(),
        }
    }
}
<commit_msg>Commented the draw queue.<commit_after>#![feature(core)]
#![deny(missing_docs)]

//! Generic draw queue that keeps item ordering, supposedly minimizing
//! the sorting time per frame by exploiting temporal coherency.

type IdType = u32;
struct Id<T>(IdType, std::marker::PhantomData<T>);

/// Iterator over queue objects.
pub struct QueueIter<'a, T: 'a> {
    objects: &'a [T],
    id_iter: std::slice::Iter<'a, Id<T>>,
}

impl<'a, T> Iterator for QueueIter<'a, T> {
    type Item = &'a T;

    fn next(&mut self) -> Option<&'a T> {
        self.id_iter.next().map(|&Id(i, _)|
            &self.objects[i as usize]
        )
    }
}

/// Generic draw queue.
pub struct Queue<T> {
    /// Exposed objects list that can be modified directly with no harm.
    pub objects: Vec<T>,
    indices: Vec<Id<T>>,
}

impl<T> Queue<T> {
    /// Create an empty queue.
    pub fn new() -> Queue<T> {
        Queue {
            objects: Vec::new(),
            indices: Vec::new(),
        }
    }

    fn is_ready(&self) -> bool {
        self.objects.len() == self.indices.len()
    }

    /// Synchronize the indices with objects.
    pub fn update(&mut self) {
        let ni = self.indices.len();
        if self.objects.len() > ni {
            self.indices.extend((ni.. self.objects.len()).map(|i|
                Id(i as IdType, std::marker::PhantomData)
            ));
        }else
        if self.objects.len() < ni {
            self.indices.retain(|&Id(i, _)| (i as usize) < ni);
        }
        debug_assert!(self.is_ready());
    }

    /// Sort the draw queue.
    pub fn sort<F: Sized + Fn(&T, &T) -> std::cmp::Ordering>(&mut self, fun: F) {
        self.update();
        let objects = self.objects.as_slice();
        self.indices.sort_by(|&Id(a, _), &Id(b, _)|
            fun(&objects[a as usize], &objects[b as usize])
        );
    }

    /// Iterate over sorted objects.
    pub fn iter<'a>(&'a self) -> QueueIter<'a, T> {
        assert!(self.is_ready());
        QueueIter {
            objects: self.objects.as_slice(),
            id_iter: self.indices.iter(),
        }
    }
}
<|endoftext|>


<commit_before>#![allow(dead_code)]

use std::collections::BTreeSet;
use std::fmt::{self, Display};
use parse::NextPrev;

#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq)]
pub struct Range(pub char, pub char);

impl fmt::Display for Range {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "({}, {})", self.0, self.1)
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Set(BTreeSet<Range>);

impl fmt::Display for Set {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let Set(ref set) = *self;
        let len = BTreeSet::len(set);

        for (count, s) in set.iter().enumerate() {
            if count < len - 1 { try!(write!(f, "{}, ", s)) }
            else { return write!(f, "{}", s) }
        }

        Ok(())
    }
}

impl Set {
    pub fn new() -> Self { Set(BTreeSet::new()) }
    pub fn insert(&mut self, value: Range) {
        let mut ret = BTreeSet::new();
        // value is a complete subset of one of the other ranges.
        let mut subset = false;

        // Borrowing self blocks later operation. Add a new scope.
        {   let Set(ref set) = *self;

            let Range(mut min_val, mut max_val) = value;
            if min_val > max_val { panic!("First value cannot be greater than the second.") }

            // Loop over set adding old disjoint pieces and supersets back.
            // When partially overlapped, expand value to the union. At the
            // end, insert union after it has been fully expanded.
            for &Range(min, max) in &*set {
                // value overlaps at the beginning.
                if min_val < min && max_val >= min && max_val < max { max_val = max }
                // value overlaps at the end.
                else if min_val > min && min_val <= max && max_val > max { min_val = min }
                // value is entirely contained between min and max. Insert original
                // into new array because new is a subset.
                else if min_val >= min && max_val <= max {
                    ret.insert(Range(min, max));
                    subset = true;
                }
                // value is a superset to the current so don't add current.
                else if min_val < min && max_val > max {}
                // value is disjoint with current so add current.
                else { ret.insert(Range(min, max)); }
            }

            // Insert value only when it's not a subset.
            if !subset { ret.insert(Range(min_val, max_val)); }
        }

        *self = Set(ret);
    }
    pub fn remove(&mut self, value: Range) {
        let mut ret = BTreeSet::new();

        // Borrowing self blocks later modification. Make a new scope to contain it.
        {   let Set(ref set) = *self;

            let Range(min_val, max_val) = value;
            if min_val > max_val { panic!("First value cannot be greater than the second.") }

            // Loop over set inserting whatever doesn't intersect.
            for &Range(min, max) in &*set {
                // value overlaps at the beginning.
                if min_val <= min && max_val >= min && max_val < max { ret.insert(Range(max_val.next(), max)); }
                // value overlaps at the end.
                else if min_val > min && min_val <= max && max_val >= max { ret.insert(Range(min, min_val.prev())); }
                // value is entirely contained between min and max. Split set
                // into two pieces.
                else if min_val > min && max_val < max {
                    ret.insert(Range(min, min_val.prev()));
                    ret.insert(Range(max_val.next(), max));

                    // Current piece was a superset so value cannot be anywhere else.
                    break;
                // value is a superset to the current so don't add current.
                } else if min_val <= min && max_val >= max {}
                // value is disjoint with current so add current.
                else { ret.insert(Range(min, max)); }
            }
        }

        *self = Set(ret)
    }
    // 123 + 345 = 12345.
    pub fn union(&self, value: &Self) -> Self {
        let mut ret = self.clone();

        // Loop over the btreeset of Range(char, char).
        for &x in &value.0 { ret.insert(x) }

        ret
    }
    // Intersection of `A` & `B` is `A - (A - B)`: 123 & 345 = 3.
    pub fn intersection(&self, value: &Self) -> Self {
        let diff = self.difference(value);

        self.difference(&diff)
    }
    // 123 - 345 = 12.
    pub fn difference(&self, value: &Self) -> Self {
        let mut ret = self.clone();

        for &x in &value.0 { ret.remove(x) }

        ret
    }
    // `A` ^ `B` is `(A + B) - (A & B)`: 123 ^ 345 = 1245.
    pub fn symmetric_difference(&self, value: &Self) -> Self {
        let union = self.union(value);
        let intersection = self.intersection(value);

        union.difference(&intersection)
    }
}

#[cfg(test)]
mod test {
    use super::{Range, Set};

    fn generate(vec: Vec<(char, char)>) -> Set {
        let mut set = Set::new();

        for (a, b) in vec {
            set.insert(Range(a, b));
        }

        set
    }
    #[test]
    fn insert_partial_overlap() {
        let set1 = generate(vec![('3', '5'), ('4', '6')]);
        let set2 = generate(vec![('3', '5'), ('1', '4')]);

        let other1 = generate(vec![('3', '6')]);
        let other2 = generate(vec![('1', '5')]);

        assert_eq!(other1, set1);
        assert_eq!(other2, set2);
    }
    #[test]
    fn insert_subset() {
        let set1 = generate(vec![('1', '5'), ('2', '3')]); // Complete overlap.
        let set2 = generate(vec![('1', '5'), ('1', '3')]); // Left is exact.
        let set3 = generate(vec![('1', '5'), ('2', '5')]); // Right is exact.
        let set4 = generate(vec![('1', '5'), ('1', '5')]); // Both are exact.

        let other = generate(vec![('1', '5')]);

        assert_eq!(set1, other);
        assert_eq!(set2, other);
        assert_eq!(set3, other);
        assert_eq!(set4, other);
    }
    #[test]
    fn insert_superset() {
        let set   = generate(vec![('3', '5'), ('2', '6')]);
        let other = generate(vec![('2', '6')]);

        assert_eq!(other, set);
    }
    #[test]
    #[should_panic]
    fn panic_on_decreasing_order() {
        generate(vec![('5', '3')]);
    }
    #[test]
    fn set_union() {
        let mut set1 = generate(vec![('1', '4'), ('5', '6')]);
        let set2 =     generate(vec![('0', '5'), ('8', '9')]);

        set1.union(set2);

        let other = generate(vec![('0', '6'), ('8', '9')]);
        assert_eq!(set1, other);
    }
    #[test]
    fn remove_partial_overlap() {
        let mut set1 = generate(vec![('5', '9')]);
        set1.remove(Range('3', '6')); // beginnings different.

        let mut set2 = generate(vec![('5', '9')]);
        set2.remove(Range('5', '6')); // beginnings same.

        let mut set3 = generate(vec![('2', '6')]);
        set3.remove(Range('5', '9')); // end different.

        let mut set4 = generate(vec![('2', '6')]);
        set4.remove(Range('5', '6')); // end same.

        let other12 = generate(vec![('7', '9')]);
        let other34 = generate(vec![('2', '4')]);

        assert_eq!(set1, other12);
        assert_eq!(set2, other12);
        assert_eq!(set3, other34);
        assert_eq!(set4, other34);
    }
    #[test]
    fn remove_subset() {
        let mut set = generate(vec![('1', '9')]);
        set.remove(Range('3', '6'));

        let other = generate(vec![('1', '2'), ('7', '9')]);

        assert_eq!(set, other);
    }
    #[test]
    fn remove_superset() {
        let mut set = generate(vec![('5', '6')]);
        set.remove(Range('3', '8'));

        let other = generate(vec![]);
        assert_eq!(set, other);
    }
    #[test]
    fn remove_disjoint() {
        let mut set = generate(vec![('2', '3')]);
        set.remove(Range('6', '8'));

        let other = generate(vec![('2', '3')]);
        assert_eq!(set, other);
    }
    #[test]
    fn set_difference() {
        let mut set1 = generate(vec![('2', '7')]);
        let set2     = generate(vec![('0', '1'),   // disjoint left
                                     ('1', '2'),   // partial overlap left
                                     ('4', '5'),   // subset
                                     ('7', '8'),   // partial overlap right
                                     ('9', '9')]); // disjoint right
        let mut letters1 = generate(vec![('c', 'e')]);
        let     letters2 = generate(vec![('a', 'g')]); // superset

        set1.difference(set2);
        letters1.difference(letters2);

        let other_set     = generate(vec![('3', '3'), ('6', '6')]);
        let other_letters = generate(vec![]);

        assert_eq!(set1, other_set);
        assert_eq!(letters1, other_letters);
    }
    #[test]
    fn set_intersection_partial_overlap() {
        let mut set_left  = generate(vec![('2', '7')]);
        let partial_left  = generate(vec![('0', '3')]);

        let mut set_right = generate(vec![('2', '7')]);
        let partial_right = generate(vec![('6', '8')]);

        set_left.intersection(partial_left);
        set_right.intersection(partial_right);

        let other_left  = generate(vec![('2', '3')]);
        let other_right = generate(vec![('6', '7')]);

        assert_eq!(set_left, other_left);
        assert_eq!(set_right, other_right);
    }
    #[test]
    fn set_intersection_subset() {
        let mut set_subset = generate(vec![('2', '7')]);
        let subset         = generate(vec![('3', '6')]);

        let mut set_left = generate(vec![('2', '7')]);
        let exact_left   = generate(vec![('2', '6')]);

        let mut set_right = generate(vec![('2', '7')]);
        let exact_right   = generate(vec![('3', '7')]);

        let mut set_both = generate(vec![('2', '7')]);
        let exact_both   = generate(vec![('2', '7')]);

        set_subset.intersection(subset);
        set_left.intersection(exact_left);
        set_right.intersection(exact_right);
        set_both.intersection(exact_both);

        let other_subset = generate(vec![('3', '6')]);
        let other_left   = generate(vec![('2', '6')]);
        let other_right  = generate(vec![('3', '7')]);
        let other_both   = generate(vec![('2', '7')]);

        assert_eq!(set_subset, other_subset);
        assert_eq!(set_left, other_left);
        assert_eq!(set_right, other_right);
        assert_eq!(set_both, other_both);
    }
    #[test]
    fn set_intersection_superset() {
        let mut set  = generate(vec![('2', '7')]);
        let superset = generate(vec![('1', '8')]);

        set.intersection(superset);

        let other = generate(vec![('2', '7')]);
        assert_eq!(set, other);
    }
    #[test]
    fn set_intersection_disjoint() {
        let mut set_low  = generate(vec![('3', '4')]);
        let mut set_high = generate(vec![('3', '4')]);
        let low  = generate(vec![('1', '2')]);
        let high = generate(vec![('5', '6')]);

        set_low.intersection(low);
        set_high.intersection(high);

        let other_low  = generate(vec![]);
        let other_high = generate(vec![]);

        assert_eq!(set_low, other_low);
        assert_eq!(set_high, other_high);
    }
}
<commit_msg>Update tests to use borrows instead of by value<commit_after>#![allow(dead_code)]

use std::collections::BTreeSet;
use std::fmt::{self, Display};
use parse::NextPrev;

#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq)]
pub struct Range(pub char, pub char);

impl fmt::Display for Range {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "({}, {})", self.0, self.1)
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Set(BTreeSet<Range>);

impl fmt::Display for Set {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let Set(ref set) = *self;
        let len = BTreeSet::len(set);

        for (count, s) in set.iter().enumerate() {
            if count < len - 1 { try!(write!(f, "{}, ", s)) }
            else { return write!(f, "{}", s) }
        }

        Ok(())
    }
}

impl Set {
    pub fn new() -> Self { Set(BTreeSet::new()) }
    pub fn insert(&mut self, value: Range) {
        let mut ret = BTreeSet::new();
        // value is a complete subset of one of the other ranges.
        let mut subset = false;

        // Borrowing self blocks later operation. Add a new scope.
        {   let Set(ref set) = *self;

            let Range(mut min_val, mut max_val) = value;
            if min_val > max_val { panic!("First value cannot be greater than the second.") }

            // Loop over set adding old disjoint pieces and supersets back.
            // When partially overlapped, expand value to the union. At the
            // end, insert union after it has been fully expanded.
            for &Range(min, max) in &*set {
                // value overlaps at the beginning.
                if min_val < min && max_val >= min && max_val < max { max_val = max }
                // value overlaps at the end.
                else if min_val > min && min_val <= max && max_val > max { min_val = min }
                // value is entirely contained between min and max. Insert original
                // into new array because new is a subset.
                else if min_val >= min && max_val <= max {
                    ret.insert(Range(min, max));
                    subset = true;
                }
                // value is a superset to the current so don't add current.
                else if min_val < min && max_val > max {}
                // value is disjoint with current so add current.
                else { ret.insert(Range(min, max)); }
            }

            // Insert value only when it's not a subset.
            if !subset { ret.insert(Range(min_val, max_val)); }
        }

        *self = Set(ret);
    }
    pub fn remove(&mut self, value: Range) {
        let mut ret = BTreeSet::new();

        // Borrowing self blocks later modification. Make a new scope to contain it.
        {   let Set(ref set) = *self;

            let Range(min_val, max_val) = value;
            if min_val > max_val { panic!("First value cannot be greater than the second.") }

            // Loop over set inserting whatever doesn't intersect.
            for &Range(min, max) in &*set {
                // value overlaps at the beginning.
                if min_val <= min && max_val >= min && max_val < max { ret.insert(Range(max_val.next(), max)); }
                // value overlaps at the end.
                else if min_val > min && min_val <= max && max_val >= max { ret.insert(Range(min, min_val.prev())); }
                // value is entirely contained between min and max. Split set
                // into two pieces.
                else if min_val > min && max_val < max {
                    ret.insert(Range(min, min_val.prev()));
                    ret.insert(Range(max_val.next(), max));

                    // Current piece was a superset so value cannot be anywhere else.
                    break;
                // value is a superset to the current so don't add current.
                } else if min_val <= min && max_val >= max {}
                // value is disjoint with current so add current.
                else { ret.insert(Range(min, max)); }
            }
        }

        *self = Set(ret)
    }
    // 123 + 345 = 12345.
    pub fn union(&self, value: &Self) -> Self {
        let mut ret = self.clone();

        // Loop over the btreeset of Range(char, char).
        for &x in &value.0 { ret.insert(x) }

        ret
    }
    // Intersection of `A` & `B` is `A - (A - B)`: 123 & 345 = 3.
    pub fn intersection(&self, value: &Self) -> Self {
        let diff = self.difference(value);

        self.difference(&diff)
    }
    // 123 - 345 = 12.
    pub fn difference(&self, value: &Self) -> Self {
        let mut ret = self.clone();

        for &x in &value.0 { ret.remove(x) }

        ret
    }
    // `A` ^ `B` is `(A + B) - (A & B)`: 123 ^ 345 = 1245.
    pub fn symmetric_difference(&self, value: &Self) -> Self {
        let union = self.union(value);
        let intersection = self.intersection(value);

        union.difference(&intersection)
    }
}

#[cfg(test)]
mod test {
    use super::{Range, Set};

    fn generate(vec: Vec<(char, char)>) -> Set {
        let mut set = Set::new();

        for (a, b) in vec {
            set.insert(Range(a, b));
        }

        set
    }
    #[test]
    fn insert_partial_overlap() {
        let set1 = generate(vec![('3', '5'), ('4', '6')]);
        let set2 = generate(vec![('3', '5'), ('1', '4')]);

        let other1 = generate(vec![('3', '6')]);
        let other2 = generate(vec![('1', '5')]);

        assert_eq!(other1, set1);
        assert_eq!(other2, set2);
    }
    #[test]
    fn insert_subset() {
        let set1 = generate(vec![('1', '5'), ('2', '3')]); // Complete overlap.
        let set2 = generate(vec![('1', '5'), ('1', '3')]); // Left is exact.
        let set3 = generate(vec![('1', '5'), ('2', '5')]); // Right is exact.
        let set4 = generate(vec![('1', '5'), ('1', '5')]); // Both are exact.

        let other = generate(vec![('1', '5')]);

        assert_eq!(set1, other);
        assert_eq!(set2, other);
        assert_eq!(set3, other);
        assert_eq!(set4, other);
    }
    #[test]
    fn insert_superset() {
        let set   = generate(vec![('3', '5'), ('2', '6')]);
        let other = generate(vec![('2', '6')]);

        assert_eq!(other, set);
    }
    #[test]
    #[should_panic]
    fn panic_on_decreasing_order() {
        generate(vec![('5', '3')]);
    }
    #[test]
    fn set_union() {
        let set1 = generate(vec![('1', '4'), ('5', '6')]);
        let set2 = generate(vec![('0', '5'), ('8', '9')]);

        let union = set1.union(&set2);

        let other = generate(vec![('0', '6'), ('8', '9')]);
        assert_eq!(union, other);
    }
    #[test]
    fn remove_partial_overlap() {
        let mut set1 = generate(vec![('5', '9')]);
        set1.remove(Range('3', '6')); // beginnings different.

        let mut set2 = generate(vec![('5', '9')]);
        set2.remove(Range('5', '6')); // beginnings same.

        let mut set3 = generate(vec![('2', '6')]);
        set3.remove(Range('5', '9')); // end different.

        let mut set4 = generate(vec![('2', '6')]);
        set4.remove(Range('5', '6')); // end same.

        let other12 = generate(vec![('7', '9')]);
        let other34 = generate(vec![('2', '4')]);

        assert_eq!(set1, other12);
        assert_eq!(set2, other12);
        assert_eq!(set3, other34);
        assert_eq!(set4, other34);
    }
    #[test]
    fn remove_subset() {
        let mut set = generate(vec![('1', '9')]);
        set.remove(Range('3', '6'));

        let other = generate(vec![('1', '2'), ('7', '9')]);

        assert_eq!(set, other);
    }
    #[test]
    fn remove_superset() {
        let mut set = generate(vec![('5', '6')]);
        set.remove(Range('3', '8'));

        let other = generate(vec![]);
        assert_eq!(set, other);
    }
    #[test]
    fn remove_disjoint() {
        let mut set = generate(vec![('2', '3')]);
        set.remove(Range('6', '8'));

        let other = generate(vec![('2', '3')]);
        assert_eq!(set, other);
    }
    #[test]
    fn set_difference() {
        let set1 = generate(vec![('2', '7')]);
        let set2 = generate(vec![('0', '1'),   // disjoint left
                                 ('1', '2'),   // partial overlap left
                                 ('4', '5'),   // subset
                                 ('7', '8'),   // partial overlap right
                                 ('9', '9')]); // disjoint right
        let letters1 = generate(vec![('c', 'e')]);
        let letters2 = generate(vec![('a', 'g')]); // superset

        let difference_set     = set1.difference(&set2);
        let difference_letters = letters1.difference(&letters2);

        let other_set     = generate(vec![('3', '3'), ('6', '6')]);
        let other_letters = generate(vec![]);

        assert_eq!(difference_set, other_set);
        assert_eq!(difference_letters, other_letters);
    }
    #[test]
    fn set_intersection_partial_overlap() {
        let set   = generate(vec![('2', '7')]);
        let left  = generate(vec![('0', '3')]);
        let right = generate(vec![('6', '8')]);

        let intersection_left  = set.intersection(&left);
        let intersection_right = set.intersection(&right);

        let other_left  = generate(vec![('2', '3')]);
        let other_right = generate(vec![('6', '7')]);

        assert_eq!(intersection_left, other_left);
        assert_eq!(intersection_right, other_right);
    }
    #[test]
    fn set_intersection_subset() {
        let set = generate(vec![('2', '7')]);

        let subset      = generate(vec![('3', '6')]);
        let exact_left  = generate(vec![('2', '6')]);
        let exact_right = generate(vec![('3', '7')]);

        let intersection_subset = set.intersection(&subset);
        let intersection_left   = set.intersection(&exact_left);
        let intersection_right  = set.intersection(&exact_right);
        let intersection_both   = set.intersection(&set);

        assert_eq!(intersection_subset, subset);
        assert_eq!(intersection_left, exact_left);
        assert_eq!(intersection_right, exact_right);
        assert_eq!(intersection_both, set);
    }
    #[test]
    fn set_intersection_superset() {
        let set      = generate(vec![('2', '7')]);
        let superset = generate(vec![('1', '8')]);

        let intersection = set.intersection(&superset);

        let other = generate(vec![('2', '7')]);
        assert_eq!(intersection, other);
    }
    #[test]
    fn set_intersection_disjoint() {
        let set = generate(vec![('3', '4')]);

        let low  = generate(vec![('1', '2')]);
        let high = generate(vec![('5', '6')]);

        let intersection_low  = set.intersection(&low);
        let intersection_high = set.intersection(&high);

        let other_low  = generate(vec![]);
        let other_high = generate(vec![]);

        assert_eq!(intersection_low, other_low);
        assert_eq!(intersection_high, other_high);
    }
}
<|endoftext|>


<commit_before>use std::process::Command;

use clap::ArgMatches;
use rusoto::{ChainProvider, ProvideAwsCredentials};

use aws::credentials_provider;
use error::{KawsError, KawsResult};

pub struct Terraform<'a> {
    aws_credentials_provider: ChainProvider,
    cluster: &'a str,
    output: Option<&'a str>,
    terraform_args: Option<Vec<&'a str>>,
}

impl<'a> Terraform<'a> {
    pub fn new(matches: &'a ArgMatches) -> Terraform<'a> {
        Terraform {
            aws_credentials_provider: credentials_provider(
                matches.value_of("aws-credentials-path"),
                matches.value_of("aws-credentials-profile"),
            ),
            cluster: matches.value_of("cluster").expect("clap should have required cluster"),
            output: matches.value_of("output"),
            terraform_args: matches.values_of("terraform-args").map(|values| values.collect()),
        }
    }

    pub fn apply(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "apply",
            "-backup=-",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
            &format!("-var-file=clusters/{}/terraform.tfvars", self.cluster),
        ]);

        if self.terraform_args.is_some() {
            command.args(self.terraform_args.as_ref().unwrap());
        }

        command.arg("terraform/kaws").env(
            "AWS_ACCESS_KEY_ID",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_access_key_id(),
        ).env(
            "AWS_SECRET_ACCESS_KEY",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_secret_access_key(),
        );

        try!(command.status());

        Ok(None)
    }

    pub fn destroy(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "destroy",
            "-backup=-",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
            &format!("-var-file=clusters/{}/terraform.tfvars", self.cluster),
        ]);

        if self.terraform_args.is_some() {
            command.args(self.terraform_args.as_ref().unwrap());
        }

        command.arg("terraform/kaws").env(
            "AWS_ACCESS_KEY_ID",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_access_key_id(),
        ).env(
            "AWS_SECRET_ACCESS_KEY",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_secret_access_key(),
        );

        let exit_status = try!(command.status());

        if exit_status.success() {
            Ok(Some(format!(
                "Destroyed cluster \"{}\"! You should remove clusters/{} from Git.",
                self.cluster,
                self.cluster,
            )))
        } else {
            Err(KawsError::new(format!("Failed to destroy cluster!")))
        }
    }

    pub fn output(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "output",
            "-module=kaws",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
        ]);

        if let Some(output) = self.output {
            command.arg(output);
        }

        try!(command.status());

        Ok(None)
    }

    pub fn plan(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "plan",
            "-backup=-",
            "-module-depth=-1",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
            &format!("-var-file=clusters/{}/terraform.tfvars", self.cluster),
        ]);

        if self.terraform_args.is_some() {
            command.args(self.terraform_args.as_ref().unwrap());
        }

        command.arg("terraform/kaws").env(
            "AWS_ACCESS_KEY_ID",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_access_key_id(),
        ).env(
            "AWS_SECRET_ACCESS_KEY",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_secret_access_key(),
        );

        try!(command.status());

        Ok(None)
    }

    fn get(&self) -> KawsResult {
        let exit_status = try!(Command::new("terraform").args(&[
            "get",
            "terraform/kaws",
        ]).status());

        if exit_status.success() {
            Ok(None)
        } else {
            Err(KawsError::new(format!("Failed to download Terraform module!")))
        }
    }
}
<commit_msg>Ignore the output of `terraform get`<commit_after>use std::process::{Command, Stdio};

use clap::ArgMatches;
use rusoto::{ChainProvider, ProvideAwsCredentials};

use aws::credentials_provider;
use error::{KawsError, KawsResult};

pub struct Terraform<'a> {
    aws_credentials_provider: ChainProvider,
    cluster: &'a str,
    output: Option<&'a str>,
    terraform_args: Option<Vec<&'a str>>,
}

impl<'a> Terraform<'a> {
    pub fn new(matches: &'a ArgMatches) -> Terraform<'a> {
        Terraform {
            aws_credentials_provider: credentials_provider(
                matches.value_of("aws-credentials-path"),
                matches.value_of("aws-credentials-profile"),
            ),
            cluster: matches.value_of("cluster").expect("clap should have required cluster"),
            output: matches.value_of("output"),
            terraform_args: matches.values_of("terraform-args").map(|values| values.collect()),
        }
    }

    pub fn apply(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "apply",
            "-backup=-",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
            &format!("-var-file=clusters/{}/terraform.tfvars", self.cluster),
        ]);

        if self.terraform_args.is_some() {
            command.args(self.terraform_args.as_ref().unwrap());
        }

        command.arg("terraform/kaws").env(
            "AWS_ACCESS_KEY_ID",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_access_key_id(),
        ).env(
            "AWS_SECRET_ACCESS_KEY",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_secret_access_key(),
        );

        try!(command.status());

        Ok(None)
    }

    pub fn destroy(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "destroy",
            "-backup=-",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
            &format!("-var-file=clusters/{}/terraform.tfvars", self.cluster),
        ]);

        if self.terraform_args.is_some() {
            command.args(self.terraform_args.as_ref().unwrap());
        }

        command.arg("terraform/kaws").env(
            "AWS_ACCESS_KEY_ID",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_access_key_id(),
        ).env(
            "AWS_SECRET_ACCESS_KEY",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_secret_access_key(),
        );

        let exit_status = try!(command.status());

        if exit_status.success() {
            Ok(Some(format!(
                "Destroyed cluster \"{}\"! You should remove clusters/{} from Git.",
                self.cluster,
                self.cluster,
            )))
        } else {
            Err(KawsError::new(format!("Failed to destroy cluster!")))
        }
    }

    pub fn output(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "output",
            "-module=kaws",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
        ]);

        if let Some(output) = self.output {
            command.arg(output);
        }

        try!(command.status());

        Ok(None)
    }

    pub fn plan(&mut self) -> KawsResult {
        try!(self.get());

        let mut command = Command::new("terraform");

        command.args(&[
            "plan",
            "-backup=-",
            "-module-depth=-1",
            &format!("-state=clusters/{}/terraform.tfstate", self.cluster),
            &format!("-var-file=clusters/{}/terraform.tfvars", self.cluster),
        ]);

        if self.terraform_args.is_some() {
            command.args(self.terraform_args.as_ref().unwrap());
        }

        command.arg("terraform/kaws").env(
            "AWS_ACCESS_KEY_ID",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_access_key_id(),
        ).env(
            "AWS_SECRET_ACCESS_KEY",
            self.aws_credentials_provider.credentials().expect(
                "Failed to get AWS credentials"
            ).aws_secret_access_key(),
        );

        try!(command.status());

        Ok(None)
    }

    fn get(&self) -> KawsResult {
        let exit_status = try!(Command::new("terraform").args(&[
            "get",
            "terraform/kaws",
        ]).stdout(Stdio::null()).status());

        if exit_status.success() {
            Ok(None)
        } else {
            Err(KawsError::new(format!("Failed to download Terraform module!")))
        }
    }
}
<|endoftext|>


<commit_before>#![deny(warnings)]
#![feature(io, path_ext)]

extern crate cargo_registry;
extern crate conduit_middleware;
extern crate conduit_test;
extern crate rustc_serialize;
extern crate conduit;
extern crate curl;
extern crate git2;
extern crate time;
extern crate url;
extern crate semver;
extern crate bufstream;

use std::collections::HashMap;
use std::error::Error as StdError;
use std::process::Command;
use std::env;
use std::sync::{Once, ONCE_INIT, Arc};
use std::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering};
use rustc_serialize::json::{self, Json};

use conduit::{Request, Method};
use conduit_test::MockRequest;
use cargo_registry::app::App;
use cargo_registry::db::{self, RequestTransaction};
use cargo_registry::dependency::Kind;
use cargo_registry::{User, Crate, Version, Keyword, Dependency};
use cargo_registry::upload as u;

macro_rules! t {
    ($e:expr) => (
        match $e {
            Ok(e) => e,
            Err(m) => panic!("{} failed with: {}", stringify!($e), m),
        }
    )
}

macro_rules! t_resp { ($e:expr) => (t!($e)) }

macro_rules! ok_resp {
    ($e:expr) => ({
        let resp = t_resp!($e);
        if !::ok_resp(&resp) { panic!("bad response: {:?}", resp.status); }
        resp
    })
}

macro_rules! bad_resp {
    ($e:expr) => ({
        let mut resp = t_resp!($e);
        match ::bad_resp(&mut resp) {
            None => panic!("ok response: {:?}", resp.status),
            Some(b) => b,
        }
    })
}

#[derive(RustcDecodable, Debug)]
struct Error { detail: String }
#[derive(RustcDecodable)]
struct Bad { errors: Vec<Error> }

mod middleware;
mod keyword;
mod krate;
mod user;
mod record;
mod git;
mod version;
mod team;

fn app() -> (record::Bomb, Arc<App>, conduit_middleware::MiddlewareBuilder) {
    struct NoCommit;
    static INIT: Once = ONCE_INIT;
    git::init();

    let (proxy, bomb) = record::proxy();
    let config = cargo_registry::Config {
        s3_bucket: env::var("S3_BUCKET").unwrap_or(String::new()),
        s3_access_key: env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
        s3_secret_key: env::var("S3_SECRET_KEY").unwrap_or(String::new()),
        s3_region: env::var("S3_REGION").ok(),
        s3_proxy: Some(proxy),
        session_key: "test".to_string(),
        git_repo_checkout: git::checkout(),
        gh_client_id: env::var("GH_CLIENT_ID").unwrap_or(String::new()),
        gh_client_secret: env::var("GH_CLIENT_SECRET").unwrap_or(String::new()),
        db_url: env("TEST_DATABASE_URL"),
        env: cargo_registry::Env::Test,
        max_upload_size: 1000,
    };
    INIT.call_once(|| db_setup(&config.db_url));
    let app = App::new(&config);
    let app = Arc::new(app);
    let mut middleware = cargo_registry::middleware(app.clone());
    middleware.add(NoCommit);
    return (bomb, app, middleware);

    fn db_setup(db: &str) {
        let migrate = t!(env::current_exe()).parent().unwrap().join("migrate");
        assert!(t!(Command::new(&migrate).env("DATABASE_URL", db)
                           .status()).success());
    }

    impl conduit_middleware::Middleware for NoCommit {
        fn after(&self, req: &mut Request,
                 res: Result<conduit::Response, Box<StdError+Send>>)
                 -> Result<conduit::Response, Box<StdError+Send>> {
            req.extensions().find::<db::Transaction>()
               .expect("Transaction not present in request")
               .rollback();
            return res;
        }
    }
}

fn env(s: &str) -> String {
    match env::var(s).ok() {
        Some(s) => s,
        None => panic!("must have `{}` defined", s),
    }
}

fn req(app: Arc<App>, method: conduit::Method, path: &str) -> MockRequest {
    let mut req = MockRequest::new(method, path);
    req.mut_extensions().insert(db::Transaction::new(app));
    return req;
}

fn ok_resp(r: &conduit::Response) -> bool {
    r.status.0 == 200
}

fn bad_resp(r: &mut conduit::Response) -> Option<Bad> {
    let bad = json::<Bad>(r);
    if bad.errors.len() == 0 { return None }
    Some(bad)
}

fn json<T: rustc_serialize::Decodable>(r: &mut conduit::Response) -> T {
    let mut data = Vec::new();
    r.body.read_to_end(&mut data).unwrap();
    let s = std::str::from_utf8(&data).unwrap();
    let j = match Json::from_str(s) {
        Ok(t) => t,
        Err(e) => panic!("failed to decode: {:?}\n{}", e, s),
    };
    let j = fixup(j);
    let s = j.to_string();
    return match json::decode(&s) {
        Ok(t) => t,
        Err(e) => panic!("failed to decode: {:?}\n{}", e, s),
    };

    fn fixup(json: Json) -> Json {
        match json {
            Json::Object(object) => {
                Json::Object(object.into_iter().map(|(k, v)| {
                    let k = if k == "crate" {
                        "krate".to_string()
                    } else {
                        k
                    };
                    (k, fixup(v))
                }).collect())
            }
            Json::Array(list) => {
                Json::Array(list.into_iter().map(fixup).collect())
            }
            j => j,
        }
    }
}

static NEXT_ID: AtomicUsize = ATOMIC_USIZE_INIT;

fn user(login: &str) -> User {
    User {
        id: NEXT_ID.fetch_add(1, Ordering::SeqCst) as i32,
        gh_login: login.to_string(),
        email: None,
        name: None,
        avatar: None,
        gh_access_token: User::new_api_token(), // just randomize it
        api_token: User::new_api_token(),
    }
}

fn krate(name: &str) -> Crate {
    cargo_registry::krate::Crate {
        id: NEXT_ID.fetch_add(1, Ordering::SeqCst) as i32,
        name: name.to_string(),
        user_id: 100,
        updated_at: time::now().to_timespec(),
        created_at: time::now().to_timespec(),
        downloads: 10,
        max_version: semver::Version::parse("0.0.0").unwrap(),
        documentation: None,
        homepage: None,
        description: None,
        readme: None,
        keywords: Vec::new(),
        license: None,
        repository: None,
    }
}

fn mock_user(req: &mut Request, u: User) -> User {
    let u = User::find_or_insert(req.tx().unwrap(),
                                 &u.gh_login,
                                 u.email.as_ref().map(|s| &s[..]),
                                 u.name.as_ref().map(|s| &s[..]),
                                 u.avatar.as_ref().map(|s| &s[..]),
                                 &u.gh_access_token,
                                 &u.api_token).unwrap();
    req.mut_extensions().insert(u.clone());
    return u;
}

fn mock_crate(req: &mut Request, krate: Crate) -> (Crate, Version) {
    mock_crate_vers(req, krate, &semver::Version::parse("1.0.0").unwrap())
}

fn mock_crate_vers(req: &mut Request, krate: Crate, v: &semver::Version)
                   -> (Crate, Version) {
    let user = req.extensions().find::<User>().unwrap();
    let mut krate = Crate::find_or_insert(req.tx().unwrap(), &krate.name,
                                      user.id, &krate.description,
                                      &krate.homepage,
                                      &krate.documentation,
                                      &krate.readme,
                                      &krate.keywords,
                                      &krate.repository,
                                      &krate.license,
                                      &None).unwrap();
    Keyword::update_crate(req.tx().unwrap(), &krate,
                          &krate.keywords).unwrap();
    let v = krate.add_version(req.tx().unwrap(), v, &HashMap::new(), &[]);
    (krate, v.unwrap())
}

fn mock_dep(req: &mut Request, version: &Version, krate: &Crate,
            target: Option<&str>) -> Dependency {
    Dependency::insert(req.tx().unwrap(),
                       version.id,
                       krate.id,
                       &semver::VersionReq::parse(">= 0").unwrap(),
                       Kind::Normal,
                       false, true, &[],
                       &target.map(|s| s.to_string())).unwrap()
}

fn mock_keyword(req: &mut Request, name: &str) -> Keyword {
    Keyword::find_or_insert(req.tx().unwrap(), name).unwrap()
}

fn logout(req: &mut Request) {
    req.mut_extensions().pop::<User>();
}

fn new_req(app: Arc<App>, krate: &str, version: &str) -> MockRequest {
    new_req_full(app, ::krate(krate), version, Vec::new())
}

fn new_req_full(app: Arc<App>, krate: Crate, version: &str,
                deps: Vec<u::CrateDependency>) -> MockRequest {
    let mut req = ::req(app, Method::Put, "/api/v1/crates/new");
    req.with_body(&new_req_body(krate, version, deps));
    return req;
}

fn new_req_body(krate: Crate, version: &str, deps: Vec<u::CrateDependency>)
                -> Vec<u8> {
    let kws = krate.keywords.into_iter().map(u::Keyword).collect();
    new_crate_to_body(&u::NewCrate {
        name: u::CrateName(krate.name),
        vers: u::CrateVersion(semver::Version::parse(version).unwrap()),
        features: HashMap::new(),
        deps: deps,
        authors: vec!["foo".to_string()],
        description: Some("description".to_string()),
        homepage: krate.homepage,
        documentation: krate.documentation,
        readme: krate.readme,
        keywords: Some(u::KeywordList(kws)),
        license: Some("MIT".to_string()),
        license_file: None,
        repository: krate.repository,
    })
}

fn new_crate_to_body(new_crate: &u::NewCrate) -> Vec<u8> {
    let json = json::encode(&new_crate).unwrap();
    let mut body = Vec::new();
    body.extend([
        (json.len() >>  0) as u8,
        (json.len() >>  8) as u8,
        (json.len() >> 16) as u8,
        (json.len() >> 24) as u8,
    ].iter().cloned());
    body.extend(json.as_bytes().iter().cloned());
    body.extend([0, 0, 0, 0].iter().cloned());
    body
}
<commit_msg>PathExt stabilized<commit_after>#![deny(warnings)]
#![feature(io)]

extern crate cargo_registry;
extern crate conduit_middleware;
extern crate conduit_test;
extern crate rustc_serialize;
extern crate conduit;
extern crate curl;
extern crate git2;
extern crate time;
extern crate url;
extern crate semver;
extern crate bufstream;

use std::collections::HashMap;
use std::error::Error as StdError;
use std::process::Command;
use std::env;
use std::sync::{Once, ONCE_INIT, Arc};
use std::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering};
use rustc_serialize::json::{self, Json};

use conduit::{Request, Method};
use conduit_test::MockRequest;
use cargo_registry::app::App;
use cargo_registry::db::{self, RequestTransaction};
use cargo_registry::dependency::Kind;
use cargo_registry::{User, Crate, Version, Keyword, Dependency};
use cargo_registry::upload as u;

macro_rules! t {
    ($e:expr) => (
        match $e {
            Ok(e) => e,
            Err(m) => panic!("{} failed with: {}", stringify!($e), m),
        }
    )
}

macro_rules! t_resp { ($e:expr) => (t!($e)) }

macro_rules! ok_resp {
    ($e:expr) => ({
        let resp = t_resp!($e);
        if !::ok_resp(&resp) { panic!("bad response: {:?}", resp.status); }
        resp
    })
}

macro_rules! bad_resp {
    ($e:expr) => ({
        let mut resp = t_resp!($e);
        match ::bad_resp(&mut resp) {
            None => panic!("ok response: {:?}", resp.status),
            Some(b) => b,
        }
    })
}

#[derive(RustcDecodable, Debug)]
struct Error { detail: String }
#[derive(RustcDecodable)]
struct Bad { errors: Vec<Error> }

mod middleware;
mod keyword;
mod krate;
mod user;
mod record;
mod git;
mod version;
mod team;

fn app() -> (record::Bomb, Arc<App>, conduit_middleware::MiddlewareBuilder) {
    struct NoCommit;
    static INIT: Once = ONCE_INIT;
    git::init();

    let (proxy, bomb) = record::proxy();
    let config = cargo_registry::Config {
        s3_bucket: env::var("S3_BUCKET").unwrap_or(String::new()),
        s3_access_key: env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
        s3_secret_key: env::var("S3_SECRET_KEY").unwrap_or(String::new()),
        s3_region: env::var("S3_REGION").ok(),
        s3_proxy: Some(proxy),
        session_key: "test".to_string(),
        git_repo_checkout: git::checkout(),
        gh_client_id: env::var("GH_CLIENT_ID").unwrap_or(String::new()),
        gh_client_secret: env::var("GH_CLIENT_SECRET").unwrap_or(String::new()),
        db_url: env("TEST_DATABASE_URL"),
        env: cargo_registry::Env::Test,
        max_upload_size: 1000,
    };
    INIT.call_once(|| db_setup(&config.db_url));
    let app = App::new(&config);
    let app = Arc::new(app);
    let mut middleware = cargo_registry::middleware(app.clone());
    middleware.add(NoCommit);
    return (bomb, app, middleware);

    fn db_setup(db: &str) {
        let migrate = t!(env::current_exe()).parent().unwrap().join("migrate");
        assert!(t!(Command::new(&migrate).env("DATABASE_URL", db)
                           .status()).success());
    }

    impl conduit_middleware::Middleware for NoCommit {
        fn after(&self, req: &mut Request,
                 res: Result<conduit::Response, Box<StdError+Send>>)
                 -> Result<conduit::Response, Box<StdError+Send>> {
            req.extensions().find::<db::Transaction>()
               .expect("Transaction not present in request")
               .rollback();
            return res;
        }
    }
}

fn env(s: &str) -> String {
    match env::var(s).ok() {
        Some(s) => s,
        None => panic!("must have `{}` defined", s),
    }
}

fn req(app: Arc<App>, method: conduit::Method, path: &str) -> MockRequest {
    let mut req = MockRequest::new(method, path);
    req.mut_extensions().insert(db::Transaction::new(app));
    return req;
}

fn ok_resp(r: &conduit::Response) -> bool {
    r.status.0 == 200
}

fn bad_resp(r: &mut conduit::Response) -> Option<Bad> {
    let bad = json::<Bad>(r);
    if bad.errors.len() == 0 { return None }
    Some(bad)
}

fn json<T: rustc_serialize::Decodable>(r: &mut conduit::Response) -> T {
    let mut data = Vec::new();
    r.body.read_to_end(&mut data).unwrap();
    let s = std::str::from_utf8(&data).unwrap();
    let j = match Json::from_str(s) {
        Ok(t) => t,
        Err(e) => panic!("failed to decode: {:?}\n{}", e, s),
    };
    let j = fixup(j);
    let s = j.to_string();
    return match json::decode(&s) {
        Ok(t) => t,
        Err(e) => panic!("failed to decode: {:?}\n{}", e, s),
    };

    fn fixup(json: Json) -> Json {
        match json {
            Json::Object(object) => {
                Json::Object(object.into_iter().map(|(k, v)| {
                    let k = if k == "crate" {
                        "krate".to_string()
                    } else {
                        k
                    };
                    (k, fixup(v))
                }).collect())
            }
            Json::Array(list) => {
                Json::Array(list.into_iter().map(fixup).collect())
            }
            j => j,
        }
    }
}

static NEXT_ID: AtomicUsize = ATOMIC_USIZE_INIT;

fn user(login: &str) -> User {
    User {
        id: NEXT_ID.fetch_add(1, Ordering::SeqCst) as i32,
        gh_login: login.to_string(),
        email: None,
        name: None,
        avatar: None,
        gh_access_token: User::new_api_token(), // just randomize it
        api_token: User::new_api_token(),
    }
}

fn krate(name: &str) -> Crate {
    cargo_registry::krate::Crate {
        id: NEXT_ID.fetch_add(1, Ordering::SeqCst) as i32,
        name: name.to_string(),
        user_id: 100,
        updated_at: time::now().to_timespec(),
        created_at: time::now().to_timespec(),
        downloads: 10,
        max_version: semver::Version::parse("0.0.0").unwrap(),
        documentation: None,
        homepage: None,
        description: None,
        readme: None,
        keywords: Vec::new(),
        license: None,
        repository: None,
    }
}

fn mock_user(req: &mut Request, u: User) -> User {
    let u = User::find_or_insert(req.tx().unwrap(),
                                 &u.gh_login,
                                 u.email.as_ref().map(|s| &s[..]),
                                 u.name.as_ref().map(|s| &s[..]),
                                 u.avatar.as_ref().map(|s| &s[..]),
                                 &u.gh_access_token,
                                 &u.api_token).unwrap();
    req.mut_extensions().insert(u.clone());
    return u;
}

fn mock_crate(req: &mut Request, krate: Crate) -> (Crate, Version) {
    mock_crate_vers(req, krate, &semver::Version::parse("1.0.0").unwrap())
}

fn mock_crate_vers(req: &mut Request, krate: Crate, v: &semver::Version)
                   -> (Crate, Version) {
    let user = req.extensions().find::<User>().unwrap();
    let mut krate = Crate::find_or_insert(req.tx().unwrap(), &krate.name,
                                      user.id, &krate.description,
                                      &krate.homepage,
                                      &krate.documentation,
                                      &krate.readme,
                                      &krate.keywords,
                                      &krate.repository,
                                      &krate.license,
                                      &None).unwrap();
    Keyword::update_crate(req.tx().unwrap(), &krate,
                          &krate.keywords).unwrap();
    let v = krate.add_version(req.tx().unwrap(), v, &HashMap::new(), &[]);
    (krate, v.unwrap())
}

fn mock_dep(req: &mut Request, version: &Version, krate: &Crate,
            target: Option<&str>) -> Dependency {
    Dependency::insert(req.tx().unwrap(),
                       version.id,
                       krate.id,
                       &semver::VersionReq::parse(">= 0").unwrap(),
                       Kind::Normal,
                       false, true, &[],
                       &target.map(|s| s.to_string())).unwrap()
}

fn mock_keyword(req: &mut Request, name: &str) -> Keyword {
    Keyword::find_or_insert(req.tx().unwrap(), name).unwrap()
}

fn logout(req: &mut Request) {
    req.mut_extensions().pop::<User>();
}

fn new_req(app: Arc<App>, krate: &str, version: &str) -> MockRequest {
    new_req_full(app, ::krate(krate), version, Vec::new())
}

fn new_req_full(app: Arc<App>, krate: Crate, version: &str,
                deps: Vec<u::CrateDependency>) -> MockRequest {
    let mut req = ::req(app, Method::Put, "/api/v1/crates/new");
    req.with_body(&new_req_body(krate, version, deps));
    return req;
}

fn new_req_body(krate: Crate, version: &str, deps: Vec<u::CrateDependency>)
                -> Vec<u8> {
    let kws = krate.keywords.into_iter().map(u::Keyword).collect();
    new_crate_to_body(&u::NewCrate {
        name: u::CrateName(krate.name),
        vers: u::CrateVersion(semver::Version::parse(version).unwrap()),
        features: HashMap::new(),
        deps: deps,
        authors: vec!["foo".to_string()],
        description: Some("description".to_string()),
        homepage: krate.homepage,
        documentation: krate.documentation,
        readme: krate.readme,
        keywords: Some(u::KeywordList(kws)),
        license: Some("MIT".to_string()),
        license_file: None,
        repository: krate.repository,
    })
}

fn new_crate_to_body(new_crate: &u::NewCrate) -> Vec<u8> {
    let json = json::encode(&new_crate).unwrap();
    let mut body = Vec::new();
    body.extend([
        (json.len() >>  0) as u8,
        (json.len() >>  8) as u8,
        (json.len() >> 16) as u8,
        (json.len() >> 24) as u8,
    ].iter().cloned());
    body.extend(json.as_bytes().iter().cloned());
    body.extend([0, 0, 0, 0].iter().cloned());
    body
}
<|endoftext|>


<commit_before>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! Tokens also track their source location within the source
//! text. This allows the parser using the tokenizer to provide
//! better error highlighting and other functionality.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! let tokens = tokenize("word");
//! assert_eq!(tokens.len(), 1);
//!
//! let tokens = tokenize("show interface");
//! assert_eq!(tokens.len(), 3);
//!
//! let tokens = tokenize("echo -n \"a b c\"");
//! assert_eq!(tokens.len(), 5);
//!
//! let tokens = tokenize("ls My\\ Documents");
//! assert_eq!(tokens.len(), 3);
//! assert_eq!(tokens[2].text, "My\\ Documents");
//! ```

/// A position within a string.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the string.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the string.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a string.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the string. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' {
            self.special(offset);
        } else if c == '?' {
            self.special(offset);
        } else if c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Special => {
                    panic!("Special not yet implemented.");
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word => self.reduce(),
            State::Whitespace => self.reduce(),
            State::WordBackslash => panic!("Escaping backslash at end of input"),
            State::Doublequote => panic!("Unclosed double quote at end of input"),
            State::DoublequoteBackslash => panic!("Escaped doublequote at end of input"),
            State::Special => panic!("Special not yet implemented"),
        }
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Vec<Token> {
    let mut tokenizer = Tokenizer::new(text);
    tokenizer.tokenize();
    tokenizer.tokens
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        let ts = tokenize("");
        assert_eq!(ts.len(), 0);
    }

    #[test]
    fn single_word() {
        let ts = tokenize("a");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
    }

    #[test]
    fn multiple_words() {
        let ts = tokenize(" aa bb  ccc ");
        assert_eq!(ts.len(), 7);
        assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
        assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
        assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
        assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
        assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
        assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
        assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
    }

    #[test]
    fn quoted_text() {
        let ts = tokenize("a \"b c\"");
        assert_eq!(ts.len(), 3);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
        assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
        assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
    }

    #[test]
    fn escaped_whitespace_in_word() {
        let ts = tokenize("a\\ b");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
    }
}
<commit_msg>tokenizer: added TokenizeError enums and made tokenize() return a Result<commit_after>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! Tokens also track their source location within the source
//! text. This allows the parser using the tokenizer to provide
//! better error highlighting and other functionality.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! if let Ok(tokens) = tokenize("word") {
//!     assert_eq!(tokens.len(), 1);
//! }
//!
//! if let Ok(tokens) = tokenize("show interface") {
//!     assert_eq!(tokens.len(), 3);
//! }
//!
//! if let Ok(tokens) = tokenize("echo -n \"a b c\"") {
//!     assert_eq!(tokens.len(), 5);
//! }
//!
//! if let Ok(tokens) = tokenize("ls My\\ Documents") {
//!     assert_eq!(tokens.len(), 3);
//!     assert_eq!(tokens[2].text, "My\\ Documents");
//! }
//! ```

/// A position within a string.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the string.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the string.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a string.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// Errors
pub enum TokenizerError {
    /// Character not allowed here
    CharacterNotAllowedHere(usize),

    /// Special not yet implemented
    SpecialNotYetImplemented(usize),

    /// Escaping backslash at end of input
    EscapingBackslashAtEndOfInput,

    /// Unclosed double quote at end of input
    UnclosedDoubleQuoteAtEndOfInput,

    /// Escaped double quote at end of input
    EscapedDoubleQuoteAtEndOfInput,
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the string. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' {
            self.special(offset);
        } else if c == '?' {
            self.special(offset);
        } else if c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) -> Result<(), TokenizerError> {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        return Err(TokenizerError::CharacterNotAllowedHere(offset));
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        return Err(TokenizerError::CharacterNotAllowedHere(offset));
                    };
                }
                State::Special => {
                    return Err(TokenizerError::SpecialNotYetImplemented(offset));
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word => self.reduce(),
            State::Whitespace => self.reduce(),
            State::WordBackslash => return Err(TokenizerError::EscapingBackslashAtEndOfInput),
            State::Doublequote => return Err(TokenizerError::UnclosedDoubleQuoteAtEndOfInput),
            State::DoublequoteBackslash => return Err(TokenizerError::EscapedDoubleQuoteAtEndOfInput),
            State::Special => return Err(TokenizerError::SpecialNotYetImplemented(self.text.len() - 1)),
        }

        Ok(())
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Result<Vec<Token>, TokenizerError> {
    let mut tokenizer = Tokenizer::new(text);
    match tokenizer.tokenize() {
        Ok(_) => Ok(tokenizer.tokens),
        Err(error) => Err(error),
    }
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        match tokenize("") {
            Ok(ts) => assert_eq!(ts.len(), 0),
            _ => {},
        };
    }

    #[test]
    fn single_word() {
        match tokenize("a") {
            Ok(ts) => {
                assert_eq!(ts.len(), 1);
                assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
            },
            _ => {},
        };
    }

    #[test]
    fn multiple_words() {
        match tokenize(" aa bb  ccc ") {
            Ok(ts) => {
                assert_eq!(ts.len(), 7);
                assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
                assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
                assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
                assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
                assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
                assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
                assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
            },
            _ => {},
        };
    }

    #[test]
    fn quoted_text() {
        match tokenize("a \"b c\"") {
            Ok(ts) => {
                assert_eq!(ts.len(), 3);
                assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
                assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
                assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
            },
            _ => {},
        };
    }

    #[test]
    fn escaped_whitespace_in_word() {
        match tokenize("a\\ b") {
            Ok(ts) => {
                assert_eq!(ts.len(), 1);
                assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
            },
            _ => {},
        };
    }
}
<|endoftext|>