<commit_before>//! A double-ended priority queue implemented with an interval heap.

use std::slice;
use std::default::Default;

// An interval heap is a binary tree structure with the following properties:
//
// (1) Each node (except possibly the last leaf) contains two values
//     where the first one is less than or equal to the second one.
// (2) Each node represents a closed interval.
// (3) A child node's interval is completely contained in the parent node's
//     interval.
//
// This implies that the min and max elements are always in the root node.
//
// This interval heap implementation stores its nodes in a linear array
// using a Vec. Here's an example of the layout of a tree with 13 items
// (7 nodes) where the numbers represent the *offsets* in the array:
//
//          (0 1)
//         /     \
//    (2 3)       (4 5)
//    /   \       /    \
//  (6 7)(8 9)(10 11)(12 --)
//
// Even indices are used for the "left" element of a node while odd indices
// are used for the "right" element of a node. Note: the last node may not
// have a "right" element.

// FIXME: There may be a better algorithm for turning a vector into an
// interval heap. Right now, this takes O(n log n) time, I think.

fn is_root(x: uint) -> bool { x < 2 }

/// Set LSB to zero for the "left" item index of a node.
fn left(x: uint) -> uint { x & !1u }

/// Returns index of "left" item of parent node.
fn parent_left(x: uint) -> uint {
    debug_assert!(!is_root(x));
    left((x - 2) / 2)
}

/// The first `v.len() - 1` elements are considered a valid interval heap
/// and the last element is to be inserted.
fn inteval_heap_push<T: Ord>(v: &mut [T]) {
    debug_assert!(v.len() > 0);
    // Start with the last new/modified node and work our way to
    // the root if necessary...
    let mut node_max = v.len() - 1;
    let mut node_min = left(node_max);
    // The reason for using two variables instead of one is to
    // get around the special case of the last node only containing
    // one element (node_min == node_max).
    if v[node_min] > v[node_max] { v.swap(node_min, node_max); }
    while !is_root(node_min) {
        let par_min = parent_left(node_min);
        let par_max = par_min + 1;
        if v[node_min] < v[par_min] {
            v.swap(par_min, node_min);
        } else if v[par_max] < v[node_max] {
            v.swap(par_max, node_max);
        } else {
            return; // nothing to do anymore
        }
        debug_assert!(v[node_min] <= v[node_max]);
        node_min = par_min;
        node_max = par_max;
    }
}

/// The min element in the root node of an otherwise valid interval heap
/// has been been replaced with some other value without violating rule (1)
/// for the root node. This function restores the interval heap properties.
fn update_min<T: Ord>(v: &mut [T]) {
    // Starting at the root, we go down the tree...
    debug_assert!(v[0] <= v[1]);
    let mut left = 0;
    loop {
        let c1 = left * 2 + 2; // index of 1st child's left element
        let c2 = left * 2 + 4; // index of 2nd child's left element
        if v.len() <= c1 { return; } // No children. We're done.
        // Pick child with lowest min
        let ch = if v.len() <= c2 || v[c1] < v[c2] { c1 }
                 else { c2 };
        if v[ch] < v[left] {
            v.swap(ch, left);
            left = ch;
            let right = left + 1;
            if right < v.len() {
                if v[left] > v[right] { v.swap(left, right); }
            }
        } else {
            break;
        }
    }
}

/// The max element in the root node of an otherwise valid interval heap
/// has been been replaced with some other value without violating rule (1)
/// for the root node. This function restores the interval heap properties.
fn update_max<T: Ord>(v: &mut [T]) {
    debug_assert!(v[0] <= v[1]);
    // Starting at the root, we go down the tree...
    let mut right = 1;
    loop {
        let c1 = right * 2 + 1; // index of 1st child's right element
        let c2 = right * 2 + 3; // index of 2nd child's right element
        if v.len() <= c1 { return; } // No children. We're done.
        // Pick child with greatest max
        let ch = if v.len() <= c2 || v[c1] > v[c2] { c1 }
                 else { c2 };
        if v[ch] > v[right] {
            v.swap(ch, right);
            right = ch;
            let left = right - 1; // always exists
            if v[left] > v[right] { v.swap(left, right); }
        } else {
            break;
        }
    }
}

/// An `IntervalHeap` is an implementation of a double-ended priority queue.
/// As such, it supports the following operations: `push`, `get_min`,
/// `get_max`, `pop_min`, `pop_max` where insertion takes amortized O(log n)
/// time, removal takes O(log n) time and accessing minimum and maximum can
/// be done in constant time. Also, other convenient functions are provided
/// that handle conversion from and into vectors and allow iteration etc.
#[deriving(Clone)]
pub struct IntervalHeap<T> {
    data: Vec<T>
}

impl<T: Ord> Default for IntervalHeap<T> {
    /// Creates an empty `IntervalHeap`.
    #[inline]
    fn default() -> IntervalHeap<T> { IntervalHeap::new() }
}

/// `IntervalHeap` iterator.
pub type Items<'a, T> = slice::Items<'a, T>;

impl<T: Ord> IntervalHeap<T> {
    /// Creates an empty `IntervalHeap`.
    pub fn new() -> IntervalHeap<T> {
        IntervalHeap { data: Vec::new() }
    }

    /// Creates an empty `IntervalHeap` with a specific capacity. This
    /// preallocates enough memory for capacity elements, so that the
    /// `IntervalHeap` does not have to be reallocated until it contains at
    /// least that many values.
    pub fn with_capacity(c: uint) -> IntervalHeap<T> {
        IntervalHeap { data: Vec::with_capacity(c) }
    }

    /// Creates an `IntervalHeap` from a vector.
    pub fn from_vec(mut v: Vec<T>) -> IntervalHeap<T> {
        for to in range(2, v.len()) {
            inteval_heap_push(v.slice_to_mut(to));
        }
        IntervalHeap { data: v }
    }

    /// An iterator visiting all values in underlying vector,
    /// in arbitrary order.
    pub fn iter(&self) -> Items<T> {
        self.data.iter()
    }

    /// Returns a reference to the smallest item or None (if empty).
    pub fn get_min(&self) -> Option<&T> {
        match self.data.len() {
            0 => None,
            _ => Some(&self.data[0])
        }
    }

    /// Returns a reference to the greatest item or None (if empty).
    pub fn get_max(&self) -> Option<&T> {
        match self.data.len() {
            0 => None,
            1 => Some(&self.data[0]),
            _ => Some(&self.data[1])
        }
    }

    /// Returns references to the smallest and greatest item or None (if empty).
    pub fn get_min_max(&self) -> Option<(&T, &T)> {
        match self.data.len() {
            0 => None,
            1 => Some((&self.data[0],&self.data[0])),
            _ => Some((&self.data[0],&self.data[1]))
        }
    }

    /// Returns the number of items the interval heap could hold
    /// without reallocation.
    pub fn capacity(&self) -> uint {
        self.data.capacity()
    }

    /// Reserves the minimum capacity for exactly `additional` more elements
    /// to be inserted in the given `IntervalHeap`. Does nothing if the capacity
    /// is already sufficient.
    ///
    /// Note that the allocator may give the collection more space than it
    /// requests. Therefore capacity can not be relied upon to be precisely
    /// minimal. Prefer `reserve` if future insertions are expected.
    pub fn reserve_exact(&mut self, additional: uint) {
        self.data.reserve_exact(additional);
    }

    /// Reserves capacity for at least `additional` more elements to be inserted
    /// in the `IntervalHeap`. The collection may reserve more space to avoid
    /// frequent reallocations.
    pub fn reserve(&mut self, additional: uint) {
        self.data.reserve(additional);
    }

    /// Discards as much additional capacity as possible.
    pub fn shrink_to_fit(&mut self) {
        self.data.shrink_to_fit()
    }

    /// Removes the smallest item and returns it, or None if is empty.
    pub fn pop_min(&mut self) -> Option<T> {
        match self.data.len() {
            0 => None,
            1...2 => self.data.swap_remove(0),
            _ => {
                let res = self.data.swap_remove(0);
                update_min(self.data.as_mut_slice());
                res
            }
        }
    }

    /// Removes the greatest item and returns it, or None if is empty.
    pub fn pop_max(&mut self) -> Option<T> {
        match self.data.len() {
            0...2 => self.data.pop(),
            _ => {
                let res = self.data.swap_remove(1);
                update_max(self.data.as_mut_slice());
                res
            }
        }
    }

    /// Pushes an item onto the queue.
    pub fn push(&mut self, x: T) {
        self.data.push(x);
        inteval_heap_push(self.data.as_mut_slice());
    }

    /// Consumes the `IntervalHeap` and returns the underlying vector
    /// in arbitrary order.
    pub fn into_vec(self) -> Vec<T> { self.data }

    /// Consumes the `IntervalHeap` and returns a vector in sorted
    /// (ascending) order.
    pub fn into_sorted_vec(self) -> Vec<T> {
        let mut vec = self.data;
        for hsize in range(2, vec.len()).rev() {
            vec.swap(1, hsize);
            update_max(vec.slice_to_mut(hsize));
        }
        vec
    }

    /// Returns the number of items in the interval heap
    pub fn len(&self) -> uint {
        self.data.len()
    }

    /// Returns true if the queue contains no items.
    pub fn is_empty(&self) -> bool {
        self.data.is_empty()
    }

    /// Drops all items from the queue.
    pub fn clear(&mut self) {
        self.data.clear();
    }
}

impl<T: Ord> FromIterator<T> for IntervalHeap<T> {
    /// Creates an interval heap with all the items from an iterator
    fn from_iter<Iter: Iterator<T>>(mut iter: Iter) -> IntervalHeap<T> {
        let vec: Vec<T> = iter.collect();
        IntervalHeap::from_vec(vec)
    }
}

impl<T: Ord> Extend<T> for IntervalHeap<T> {
    /// Extends the interval heap by a new chunk of items given by
    /// an iterator.
    fn extend<Iter: Iterator<T>>(&mut self, mut iter: Iter) {
        let (lower, _) = iter.size_hint();
        self.reserve(lower);
        for elem in iter {
            self.push(elem);
        }
    }
}

#[cfg(test)]
pub fn as_slice<T>(x: &IntervalHeap<T>) -> &[T] {
    x.data.as_slice()
}

#[cfg(test)]
mod test {
    use std::rand::{ task_rng, Rng };
    use super::{ IntervalHeap, as_slice };

    fn is_interval_heap<T: Ord>(x: &[T]) -> bool {
        if x.len() < 2 { return true; }
        if x[1] < x[0] { return false; }
        let mut ofs = 2;
        while ofs < x.len() {
            let ofz = ofs + (ofs + 1 < x.len()) as uint;
            if x[ofz] < x[ofs] { return false; }
            let parent = (ofs / 2 - 1) & !1u;
            if x[ofs] < x[parent] { return false; }
            if x[parent+1] < x[ofz] { return false; }
            ofs += 2;
        }
        true
    }

    #[test]
    fn fuzz_push_into_sorted_vec() {
        let mut rng = task_rng();
        let mut tmp = Vec::with_capacity(100);
        for _ in range(0, 100u) {
            tmp.clear();
            let mut ih: IntervalHeap<u32> = IntervalHeap::from_vec(tmp);
            for _ in range(0, 100u) {
                ih.push(rng.next_u32());
                assert!(is_interval_heap(as_slice(&ih)));
            }
            tmp = ih.into_sorted_vec();
            for pair in tmp.windows(2) {
                assert!(pair[0] <= pair[1]);
            }
        }
    }

    #[test]
    fn fuzz_pop_min() {
        let mut rng = task_rng();
        let mut tmp = Vec::with_capacity(100);
        for _ in range(0, 100u) {
            tmp.clear();
            let mut ih: IntervalHeap<u32> = IntervalHeap::from_vec(tmp);
            for _ in range(0, 100u) {
                ih.push(rng.next_u32());
            }
            let mut tmpx: Option<u32> = None;
            loop {
                let tmpy = ih.pop_min();
                assert!(is_interval_heap(as_slice(&ih)));
                match (tmpx, tmpy) {
                    (_, None) => break,
                    (Some(x), Some(y)) => assert!(x <= y),
                    _ => ()
                }
                tmpx = tmpy;
            }
            tmp = ih.into_vec();
        }
    }

    #[test]
    fn fuzz_pop_max() {
        let mut rng = task_rng();
        let mut tmp = Vec::with_capacity(100);
        for _ in range(0, 100u) {
            tmp.clear();
            let mut ih: IntervalHeap<u32> = IntervalHeap::from_vec(tmp);
            for _ in range(0, 100u) {
                ih.push(rng.next_u32());
            }
            let mut tmpx: Option<u32> = None;
            loop {
                let tmpy = ih.pop_max();
                assert!(is_interval_heap(as_slice(&ih)));
                match (tmpx, tmpy) {
                    (_, None) => break,
                    (Some(x), Some(y)) => assert!(x >= y),
                    _ => ()
                }
                tmpx = tmpy;
            }
            tmp = ih.into_vec();
        }
    }

} // mod test

<commit_msg>Silence unused mut warning<commit_after>//! A double-ended priority queue implemented with an interval heap.

use std::slice;
use std::default::Default;

// An interval heap is a binary tree structure with the following properties:
//
// (1) Each node (except possibly the last leaf) contains two values
//     where the first one is less than or equal to the second one.
// (2) Each node represents a closed interval.
// (3) A child node's interval is completely contained in the parent node's
//     interval.
//
// This implies that the min and max elements are always in the root node.
//
// This interval heap implementation stores its nodes in a linear array
// using a Vec. Here's an example of the layout of a tree with 13 items
// (7 nodes) where the numbers represent the *offsets* in the array:
//
//          (0 1)
//         /     \
//    (2 3)       (4 5)
//    /   \       /    \
//  (6 7)(8 9)(10 11)(12 --)
//
// Even indices are used for the "left" element of a node while odd indices
// are used for the "right" element of a node. Note: the last node may not
// have a "right" element.

// FIXME: There may be a better algorithm for turning a vector into an
// interval heap. Right now, this takes O(n log n) time, I think.

fn is_root(x: uint) -> bool { x < 2 }

/// Set LSB to zero for the "left" item index of a node.
fn left(x: uint) -> uint { x & !1u }

/// Returns index of "left" item of parent node.
fn parent_left(x: uint) -> uint {
    debug_assert!(!is_root(x));
    left((x - 2) / 2)
}

/// The first `v.len() - 1` elements are considered a valid interval heap
/// and the last element is to be inserted.
fn inteval_heap_push<T: Ord>(v: &mut [T]) {
    debug_assert!(v.len() > 0);
    // Start with the last new/modified node and work our way to
    // the root if necessary...
    let mut node_max = v.len() - 1;
    let mut node_min = left(node_max);
    // The reason for using two variables instead of one is to
    // get around the special case of the last node only containing
    // one element (node_min == node_max).
    if v[node_min] > v[node_max] { v.swap(node_min, node_max); }
    while !is_root(node_min) {
        let par_min = parent_left(node_min);
        let par_max = par_min + 1;
        if v[node_min] < v[par_min] {
            v.swap(par_min, node_min);
        } else if v[par_max] < v[node_max] {
            v.swap(par_max, node_max);
        } else {
            return; // nothing to do anymore
        }
        debug_assert!(v[node_min] <= v[node_max]);
        node_min = par_min;
        node_max = par_max;
    }
}

/// The min element in the root node of an otherwise valid interval heap
/// has been been replaced with some other value without violating rule (1)
/// for the root node. This function restores the interval heap properties.
fn update_min<T: Ord>(v: &mut [T]) {
    // Starting at the root, we go down the tree...
    debug_assert!(v[0] <= v[1]);
    let mut left = 0;
    loop {
        let c1 = left * 2 + 2; // index of 1st child's left element
        let c2 = left * 2 + 4; // index of 2nd child's left element
        if v.len() <= c1 { return; } // No children. We're done.
        // Pick child with lowest min
        let ch = if v.len() <= c2 || v[c1] < v[c2] { c1 }
                 else { c2 };
        if v[ch] < v[left] {
            v.swap(ch, left);
            left = ch;
            let right = left + 1;
            if right < v.len() {
                if v[left] > v[right] { v.swap(left, right); }
            }
        } else {
            break;
        }
    }
}

/// The max element in the root node of an otherwise valid interval heap
/// has been been replaced with some other value without violating rule (1)
/// for the root node. This function restores the interval heap properties.
fn update_max<T: Ord>(v: &mut [T]) {
    debug_assert!(v[0] <= v[1]);
    // Starting at the root, we go down the tree...
    let mut right = 1;
    loop {
        let c1 = right * 2 + 1; // index of 1st child's right element
        let c2 = right * 2 + 3; // index of 2nd child's right element
        if v.len() <= c1 { return; } // No children. We're done.
        // Pick child with greatest max
        let ch = if v.len() <= c2 || v[c1] > v[c2] { c1 }
                 else { c2 };
        if v[ch] > v[right] {
            v.swap(ch, right);
            right = ch;
            let left = right - 1; // always exists
            if v[left] > v[right] { v.swap(left, right); }
        } else {
            break;
        }
    }
}

/// An `IntervalHeap` is an implementation of a double-ended priority queue.
/// As such, it supports the following operations: `push`, `get_min`,
/// `get_max`, `pop_min`, `pop_max` where insertion takes amortized O(log n)
/// time, removal takes O(log n) time and accessing minimum and maximum can
/// be done in constant time. Also, other convenient functions are provided
/// that handle conversion from and into vectors and allow iteration etc.
#[deriving(Clone)]
pub struct IntervalHeap<T> {
    data: Vec<T>
}

impl<T: Ord> Default for IntervalHeap<T> {
    /// Creates an empty `IntervalHeap`.
    #[inline]
    fn default() -> IntervalHeap<T> { IntervalHeap::new() }
}

/// `IntervalHeap` iterator.
pub type Items<'a, T> = slice::Items<'a, T>;

impl<T: Ord> IntervalHeap<T> {
    /// Creates an empty `IntervalHeap`.
    pub fn new() -> IntervalHeap<T> {
        IntervalHeap { data: Vec::new() }
    }

    /// Creates an empty `IntervalHeap` with a specific capacity. This
    /// preallocates enough memory for capacity elements, so that the
    /// `IntervalHeap` does not have to be reallocated until it contains at
    /// least that many values.
    pub fn with_capacity(c: uint) -> IntervalHeap<T> {
        IntervalHeap { data: Vec::with_capacity(c) }
    }

    /// Creates an `IntervalHeap` from a vector.
    pub fn from_vec(mut v: Vec<T>) -> IntervalHeap<T> {
        for to in range(2, v.len()) {
            inteval_heap_push(v.slice_to_mut(to));
        }
        IntervalHeap { data: v }
    }

    /// An iterator visiting all values in underlying vector,
    /// in arbitrary order.
    pub fn iter(&self) -> Items<T> {
        self.data.iter()
    }

    /// Returns a reference to the smallest item or None (if empty).
    pub fn get_min(&self) -> Option<&T> {
        match self.data.len() {
            0 => None,
            _ => Some(&self.data[0])
        }
    }

    /// Returns a reference to the greatest item or None (if empty).
    pub fn get_max(&self) -> Option<&T> {
        match self.data.len() {
            0 => None,
            1 => Some(&self.data[0]),
            _ => Some(&self.data[1])
        }
    }

    /// Returns references to the smallest and greatest item or None (if empty).
    pub fn get_min_max(&self) -> Option<(&T, &T)> {
        match self.data.len() {
            0 => None,
            1 => Some((&self.data[0],&self.data[0])),
            _ => Some((&self.data[0],&self.data[1]))
        }
    }

    /// Returns the number of items the interval heap could hold
    /// without reallocation.
    pub fn capacity(&self) -> uint {
        self.data.capacity()
    }

    /// Reserves the minimum capacity for exactly `additional` more elements
    /// to be inserted in the given `IntervalHeap`. Does nothing if the capacity
    /// is already sufficient.
    ///
    /// Note that the allocator may give the collection more space than it
    /// requests. Therefore capacity can not be relied upon to be precisely
    /// minimal. Prefer `reserve` if future insertions are expected.
    pub fn reserve_exact(&mut self, additional: uint) {
        self.data.reserve_exact(additional);
    }

    /// Reserves capacity for at least `additional` more elements to be inserted
    /// in the `IntervalHeap`. The collection may reserve more space to avoid
    /// frequent reallocations.
    pub fn reserve(&mut self, additional: uint) {
        self.data.reserve(additional);
    }

    /// Discards as much additional capacity as possible.
    pub fn shrink_to_fit(&mut self) {
        self.data.shrink_to_fit()
    }

    /// Removes the smallest item and returns it, or None if is empty.
    pub fn pop_min(&mut self) -> Option<T> {
        match self.data.len() {
            0 => None,
            1...2 => self.data.swap_remove(0),
            _ => {
                let res = self.data.swap_remove(0);
                update_min(self.data.as_mut_slice());
                res
            }
        }
    }

    /// Removes the greatest item and returns it, or None if is empty.
    pub fn pop_max(&mut self) -> Option<T> {
        match self.data.len() {
            0...2 => self.data.pop(),
            _ => {
                let res = self.data.swap_remove(1);
                update_max(self.data.as_mut_slice());
                res
            }
        }
    }

    /// Pushes an item onto the queue.
    pub fn push(&mut self, x: T) {
        self.data.push(x);
        inteval_heap_push(self.data.as_mut_slice());
    }

    /// Consumes the `IntervalHeap` and returns the underlying vector
    /// in arbitrary order.
    pub fn into_vec(self) -> Vec<T> { self.data }

    /// Consumes the `IntervalHeap` and returns a vector in sorted
    /// (ascending) order.
    pub fn into_sorted_vec(self) -> Vec<T> {
        let mut vec = self.data;
        for hsize in range(2, vec.len()).rev() {
            vec.swap(1, hsize);
            update_max(vec.slice_to_mut(hsize));
        }
        vec
    }

    /// Returns the number of items in the interval heap
    pub fn len(&self) -> uint {
        self.data.len()
    }

    /// Returns true if the queue contains no items.
    pub fn is_empty(&self) -> bool {
        self.data.is_empty()
    }

    /// Drops all items from the queue.
    pub fn clear(&mut self) {
        self.data.clear();
    }
}

impl<T: Ord> FromIterator<T> for IntervalHeap<T> {
    /// Creates an interval heap with all the items from an iterator
    fn from_iter<Iter: Iterator<T>>(iter: Iter) -> IntervalHeap<T> {
        let vec: Vec<T> = iter.collect();
        IntervalHeap::from_vec(vec)
    }
}

impl<T: Ord> Extend<T> for IntervalHeap<T> {
    /// Extends the interval heap by a new chunk of items given by
    /// an iterator.
    fn extend<Iter: Iterator<T>>(&mut self, mut iter: Iter) {
        let (lower, _) = iter.size_hint();
        self.reserve(lower);
        for elem in iter {
            self.push(elem);
        }
    }
}

#[cfg(test)]
pub fn as_slice<T>(x: &IntervalHeap<T>) -> &[T] {
    x.data.as_slice()
}

#[cfg(test)]
mod test {
    use std::rand::{ task_rng, Rng };
    use super::{ IntervalHeap, as_slice };

    fn is_interval_heap<T: Ord>(x: &[T]) -> bool {
        if x.len() < 2 { return true; }
        if x[1] < x[0] { return false; }
        let mut ofs = 2;
        while ofs < x.len() {
            let ofz = ofs + (ofs + 1 < x.len()) as uint;
            if x[ofz] < x[ofs] { return false; }
            let parent = (ofs / 2 - 1) & !1u;
            if x[ofs] < x[parent] { return false; }
            if x[parent+1] < x[ofz] { return false; }
            ofs += 2;
        }
        true
    }

    #[test]
    fn fuzz_push_into_sorted_vec() {
        let mut rng = task_rng();
        let mut tmp = Vec::with_capacity(100);
        for _ in range(0, 100u) {
            tmp.clear();
            let mut ih: IntervalHeap<u32> = IntervalHeap::from_vec(tmp);
            for _ in range(0, 100u) {
                ih.push(rng.next_u32());
                assert!(is_interval_heap(as_slice(&ih)));
            }
            tmp = ih.into_sorted_vec();
            for pair in tmp.windows(2) {
                assert!(pair[0] <= pair[1]);
            }
        }
    }

    #[test]
    fn fuzz_pop_min() {
        let mut rng = task_rng();
        let mut tmp = Vec::with_capacity(100);
        for _ in range(0, 100u) {
            tmp.clear();
            let mut ih: IntervalHeap<u32> = IntervalHeap::from_vec(tmp);
            for _ in range(0, 100u) {
                ih.push(rng.next_u32());
            }
            let mut tmpx: Option<u32> = None;
            loop {
                let tmpy = ih.pop_min();
                assert!(is_interval_heap(as_slice(&ih)));
                match (tmpx, tmpy) {
                    (_, None) => break,
                    (Some(x), Some(y)) => assert!(x <= y),
                    _ => ()
                }
                tmpx = tmpy;
            }
            tmp = ih.into_vec();
        }
    }

    #[test]
    fn fuzz_pop_max() {
        let mut rng = task_rng();
        let mut tmp = Vec::with_capacity(100);
        for _ in range(0, 100u) {
            tmp.clear();
            let mut ih: IntervalHeap<u32> = IntervalHeap::from_vec(tmp);
            for _ in range(0, 100u) {
                ih.push(rng.next_u32());
            }
            let mut tmpx: Option<u32> = None;
            loop {
                let tmpy = ih.pop_max();
                assert!(is_interval_heap(as_slice(&ih)));
                match (tmpx, tmpy) {
                    (_, None) => break,
                    (Some(x), Some(y)) => assert!(x >= y),
                    _ => ()
                }
                tmpx = tmpy;
            }
            tmp = ih.into_vec();
        }
    }

} // mod test

<|endoftext|>


<commit_before>use super::internal::*;
use super::*;

/// `Flatten` turns each element to an iterator, then flattens these iterators
/// together. This struct is created by the [`flatten()`] method on
/// [`ParallelIterator`].
///
/// [`flatten()`]: trait.ParallelIterator.html#method.flatten
/// [`ParallelIterator`]: trait.ParallelIterator.html
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct Flatten<I: ParallelIterator> {
    base: I,
}

/// Create a new `Flatten` iterator.
pub fn new<I, PI>(base: I) -> Flatten<I>
    where I: ParallelIterator<Item = PI>,
          PI: IntoParallelIterator + Send
{
    Flatten {
        base: base,
    }
}

impl<I, PI> ParallelIterator for Flatten<I>
    where I: ParallelIterator<Item = PI>,
          PI: IntoParallelIterator + Send
{
    type Item = PI::Item;

    fn drive_unindexed<C>(self, consumer: C) -> C::Result
        where C: UnindexedConsumer<Self::Item>
    {
        self.base.flat_map(|x| x).drive_unindexed(consumer)
    }
}
<commit_msg>note that `flatten::new` is crate local<commit_after>use super::internal::*;
use super::*;

/// `Flatten` turns each element to an iterator, then flattens these iterators
/// together. This struct is created by the [`flatten()`] method on
/// [`ParallelIterator`].
///
/// [`flatten()`]: trait.ParallelIterator.html#method.flatten
/// [`ParallelIterator`]: trait.ParallelIterator.html
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct Flatten<I: ParallelIterator> {
    base: I,
}

/// Create a new `Flatten` iterator.
///
/// NB: Not part of the public API.
pub fn new<I, PI>(base: I) -> Flatten<I>
    where I: ParallelIterator<Item = PI>,
          PI: IntoParallelIterator + Send
{
    Flatten {
        base: base,
    }
}

impl<I, PI> ParallelIterator for Flatten<I>
    where I: ParallelIterator<Item = PI>,
          PI: IntoParallelIterator + Send
{
    type Item = PI::Item;

    fn drive_unindexed<C>(self, consumer: C) -> C::Result
        where C: UnindexedConsumer<Self::Item>
    {
        self.base.flat_map(|x| x).drive_unindexed(consumer)
    }
}
<|endoftext|>


<commit_before>
extern crate tempdir;
extern crate git2;
extern crate centralgithook;
use centralgithook::migrate;
use self::tempdir::TempDir;
use std::path::{Path, PathBuf};
use std::fs::File;
use std::fs;
use std::str;
use std::io::Write;
use std::io::Read;
use centralgithook::migrate::RepoHost;

fn _oid_to_sha1(oid: &[u8]) -> String {
    oid.iter().fold(Vec::new(), |mut acc, x| {
        acc.push(format!("{0:>02x}", x)); acc
    }).concat()
}

struct TestHost {
    td: TempDir
}

impl TestHost {
    fn new() -> Self {
        TestHost {
            td: TempDir::new("test_host").expect("folder test_host should be created")
        }
    }
}

fn empty_commit(repo: &git2::Repository) {
    let sig = git2::Signature::now("foo", "bar").expect("created signature");
    let commit = repo.commit(
        Some("HEAD"),
        &sig,
        &sig,
        "initial",
        &repo.find_tree(repo.treebuilder(None).expect("cannot create empty tree")
             .write().expect("cannot write empty tree")).expect("cannot find empty tree"),
        &[]

    ).expect("cannot commit empty");
}

impl migrate::RepoHost for TestHost {
    fn create_project(&self, module: &str) -> Result<(), git2::Error> {
        let repo_dir = self.td.path().join(&Path::new(module));
        println!("TestHost: create_project {} in {:?}", module, repo_dir);
        let repo = git2::Repository::init_bare(&repo_dir).expect("TestHost: init_bare failed");
        empty_commit(&repo);
        Ok(())
    }

    fn remote_url(&self, module_path: &str) -> String {
        self.td.path().join(&module_path).to_string_lossy().to_string()
    }
}

#[test]
fn test_commit_to_central() {
    let host = TestHost::new();
    let td_play = TempDir::new("play").expect("folder play should be created");
    let central_repo_path = td_play.path().join("central");
    println!("    ");
    println!("    ########### SETUP: create central repository ########### ");
    host.create_project("central").expect("error: create_project");
    let central_repo = create_repository(&central_repo_path);
    migrate::call_command("git", &["status"], Some(&central_repo_path));
    commit_files(&central_repo,
            &central_repo_path,
            &vec!["modules/moduleA/added_in_central.txt",
            "modules/moduleB/added_in_central.txt",
            "modules/moduleC/added_in_central.txt"]);

    println!("    ########### SETUP: create module repositories ########### ");

    let module_names = vec!["moduleA", "moduleB", "moduleC"];
    // let _module_repos: Vec<git2::Repository> = module_names
    //     .iter()
    //     .map(|m| {
    //             let x = td.path().join(&m);
    //             let repo = create_repository(&x);
    //             // point remote to central
    //             let ref_name = "modules/".to_string() + &m;
    //             central_repo.remote(&ref_name, &m).expect("remote as to be added");
    //             commit_files(&repo, &Path::new(&x), &vec!["test/a.txt", "b.txt", "c.txt"]);
    //             migrate::call_command("git", &["checkout","-b", "not_needed"], Some(&x));
    //             repo
    //             })
    // .collect();

    let fp = central_repo_path.clone().join(".git").join("refs").join("heads").join("master");
    let mut f = File::open(&fp).expect("open file master");
    let mut s = String::new();
    f.read_to_string(&mut s).expect("read file shoud work");
    println!("master file in {:?}: {}", central_repo_path, s);

    let h = central_repo.head().expect("get head of central repo");
    let t = h.target().expect("get oid of head reference");
    let central_repo_head_oid = t.as_bytes();

    println!("    ########### START: calling central_submit ########### ");
    let td_scratch = TempDir::new("scratch").expect("folder scratch should be created");

    migrate::central_submit(
        &_oid_to_sha1(&central_repo_head_oid),
        &host,
        "central",
        &central_repo_path,
        &td_scratch.path()
        // &Path::new("/tmp/testscratch")
    ).expect("call central_submit");
    // for m in module_names {
    //     let x = td_scratch.path().join(&m);
    //     migrate::call_command("git", &["checkout", "master"], Some(&x));
    //     assert!(td_scratch.path().join(m).join("added_in_central.txt").exists());
    // }
    // std::thread::sleep_ms(1111111);
}

fn commit_file(repo: &git2::Repository, file: &Path, parents: &[&git2::Commit]) -> git2::Oid {
    let mut index = repo.index().expect("get index of repo");
    index.add_path(file).expect("file should be added");
    index.write().expect("write index");
    let tree_id = index.write_tree().expect("got tree_id");
    let tree = repo.find_tree(tree_id).expect("got tree");
    let sig = git2::Signature::now("foo", "bar").expect("created signature");
    repo.commit(Some("HEAD"),
            &sig,
            &sig,
            &format!("commit for {:?}", &file.as_os_str()),
            &tree,
            &parents)
        .expect("commit to repo")
}

fn commit_files(repo: &git2::Repository, pb: &Path, content: &Vec<&str>) {
    let mut parent_commit = None;
    for file_name in content {
        let foo_file = pb.join(file_name);
        create_dummy_file(&foo_file);
        let oid = match parent_commit {
            Some(parent) => commit_file(&repo, &Path::new(file_name), &[&parent]),
                None => commit_file(&repo, &Path::new(file_name), &[]),
        };
        parent_commit = repo.find_commit(oid).ok();
    }
}

fn create_repository(temp: &Path) -> git2::Repository {
    let repo = git2::Repository::init(temp).expect("init should succeed");

    let path = migrate::get_repo_path(&repo).to_path_buf();
    println!("Initialized empty Git repository in {}", path.display());
    migrate::call_command("git", &["status"], Some(&path));
    repo
}

fn create_dummy_file(f: &PathBuf) {
    let parent_dir = f.as_path().parent().expect("need to get parent");
    fs::create_dir_all(parent_dir).expect("create directories");

    let mut file = File::create(&f.as_path()).expect("create file");
    file.write_all("test content".as_bytes()).expect("write to file");
}
<commit_msg>put assert back in test<commit_after>
extern crate tempdir;
extern crate git2;
extern crate centralgithook;
use centralgithook::migrate;
use self::tempdir::TempDir;
use std::path::{Path, PathBuf};
use std::fs::File;
use std::fs;
use std::str;
use std::io::Write;
use std::io::Read;
use centralgithook::migrate::RepoHost;

fn _oid_to_sha1(oid: &[u8]) -> String {
    oid.iter().fold(Vec::new(), |mut acc, x| {
        acc.push(format!("{0:>02x}", x)); acc
    }).concat()
}

struct TestHost {
    td: TempDir
}

impl TestHost {
    fn new() -> Self {
        TestHost {
            td: TempDir::new("test_host").expect("folder test_host should be created")
        }
    }
}

fn empty_commit(repo: &git2::Repository) {
    let sig = git2::Signature::now("foo", "bar").expect("created signature");
    let commit = repo.commit(
        Some("HEAD"),
        &sig,
        &sig,
        "initial",
        &repo.find_tree(repo.treebuilder(None).expect("cannot create empty tree")
             .write().expect("cannot write empty tree")).expect("cannot find empty tree"),
        &[]

    ).expect("cannot commit empty");
}

impl migrate::RepoHost for TestHost {
    fn create_project(&self, module: &str) -> Result<(), git2::Error> {
        let repo_dir = self.td.path().join(&Path::new(module));
        println!("TestHost: create_project {} in {:?}", module, repo_dir);
        let repo = git2::Repository::init_bare(&repo_dir).expect("TestHost: init_bare failed");
        empty_commit(&repo);
        Ok(())
    }

    fn remote_url(&self, module_path: &str) -> String {
        self.td.path().join(&module_path).to_string_lossy().to_string()
    }
}

#[test]
fn test_commit_to_central() {
    let host = TestHost::new();
    let td_play = TempDir::new("play").expect("folder play should be created");
    let central_repo_path = td_play.path().join("central");
    println!("    ");
    println!("    ########### SETUP: create central repository ########### ");
    host.create_project("central").expect("error: create_project");
    let central_repo = create_repository(&central_repo_path);
    migrate::call_command("git", &["status"], Some(&central_repo_path));
    commit_files(&central_repo,
            &central_repo_path,
            &vec!["modules/moduleA/added_in_central.txt",
            "modules/moduleB/added_in_central.txt",
            "modules/moduleC/added_in_central.txt"]);

    println!("    ########### SETUP: create module repositories ########### ");

    let module_names = vec!["moduleA", "moduleB", "moduleC"];
    // let _module_repos: Vec<git2::Repository> = module_names
    //     .iter()
    //     .map(|m| {
    //             let x = td.path().join(&m);
    //             let repo = create_repository(&x);
    //             // point remote to central
    //             let ref_name = "modules/".to_string() + &m;
    //             central_repo.remote(&ref_name, &m).expect("remote as to be added");
    //             commit_files(&repo, &Path::new(&x), &vec!["test/a.txt", "b.txt", "c.txt"]);
    //             migrate::call_command("git", &["checkout","-b", "not_needed"], Some(&x));
    //             repo
    //             })
    // .collect();

    let fp = central_repo_path.clone().join(".git").join("refs").join("heads").join("master");
    let mut f = File::open(&fp).expect("open file master");
    let mut s = String::new();
    f.read_to_string(&mut s).expect("read file shoud work");
    println!("master file in {:?}: {}", central_repo_path, s);

    let h = central_repo.head().expect("get head of central repo");
    let t = h.target().expect("get oid of head reference");
    let central_repo_head_oid = t.as_bytes();

    println!("    ########### START: calling central_submit ########### ");
    let td_scratch = TempDir::new("scratch").expect("folder scratch should be created");

    migrate::central_submit(
        &_oid_to_sha1(&central_repo_head_oid),
        &host,
        "central",
        &central_repo_path,
        &td_scratch.path()
        // &Path::new("/tmp/testscratch")
    ).expect("call central_submit");

    for m in module_names {
        migrate::call_command(
            "git", &["clone", &host.remote_url(&format!("modules/{}",m))], Some(&td_play.path())
        );
        assert!(td_play.path().join(m).join("added_in_central.txt").exists());
    }
    // std::thread::sleep_ms(1111111);
}

fn commit_file(repo: &git2::Repository, file: &Path, parents: &[&git2::Commit]) -> git2::Oid {
    let mut index = repo.index().expect("get index of repo");
    index.add_path(file).expect("file should be added");
    index.write().expect("write index");
    let tree_id = index.write_tree().expect("got tree_id");
    let tree = repo.find_tree(tree_id).expect("got tree");
    let sig = git2::Signature::now("foo", "bar").expect("created signature");
    repo.commit(Some("HEAD"),
            &sig,
            &sig,
            &format!("commit for {:?}", &file.as_os_str()),
            &tree,
            &parents)
        .expect("commit to repo")
}

fn commit_files(repo: &git2::Repository, pb: &Path, content: &Vec<&str>) {
    let mut parent_commit = None;
    for file_name in content {
        let foo_file = pb.join(file_name);
        create_dummy_file(&foo_file);
        let oid = match parent_commit {
            Some(parent) => commit_file(&repo, &Path::new(file_name), &[&parent]),
                None => commit_file(&repo, &Path::new(file_name), &[]),
        };
        parent_commit = repo.find_commit(oid).ok();
    }
}

fn create_repository(temp: &Path) -> git2::Repository {
    let repo = git2::Repository::init(temp).expect("init should succeed");

    let path = migrate::get_repo_path(&repo).to_path_buf();
    println!("Initialized empty Git repository in {}", path.display());
    migrate::call_command("git", &["status"], Some(&path));
    repo
}

fn create_dummy_file(f: &PathBuf) {
    let parent_dir = f.as_path().parent().expect("need to get parent");
    fs::create_dir_all(parent_dir).expect("create directories");

    let mut file = File::create(&f.as_path()).expect("create file");
    file.write_all("test content".as_bytes()).expect("write to file");
}
<|endoftext|>


<commit_before>use std::sync::atomic::Ordering::{Acquire, Release, Relaxed};
use std::sync::atomic::{AtomicBool, AtomicUsize};
use std::{ptr, mem};
use std::cmp;
use std::cell::UnsafeCell;

use mem::epoch::{self, Atomic, Owned};

const SEG_SIZE: usize = 32;

/// A Michael-Scott queue that allocates "segments" (arrays of nodes)
/// for efficiency.
///
/// Usable with any number of producers and consumers.
pub struct SegQueue<T> {
    head: Atomic<Segment<T>>,
    tail: Atomic<Segment<T>>,
}

struct Segment<T> {
    low: AtomicUsize,
    data: [UnsafeCell<(T, AtomicBool)>; SEG_SIZE],
    high: AtomicUsize,
    next: Atomic<Segment<T>>,
}

unsafe impl<T: Send> Sync for Segment<T> {}

impl<T> Segment<T> {
    fn new() -> Segment<T> {
        let rqueue = Segment {
            data: unsafe { mem::uninitialized() },
            low: AtomicUsize::new(0),
            high: AtomicUsize::new(0),
            next: Atomic::null(),
        };
        for val in rqueue.data.iter() {
            unsafe {
                (*val.get()).1 = AtomicBool::new(false);
            }
        }
        rqueue
    }
}

impl<T> SegQueue<T> {
    /// Create a new, empty queue.
    pub fn new() -> SegQueue<T> {
        let q = SegQueue {
            head: Atomic::null(),
            tail: Atomic::null(),
        };
        let sentinel = Owned::new(Segment::new());
        let guard = epoch::pin();
        let sentinel = q.head.store_and_ref(sentinel, Relaxed, &guard);
        q.tail.store_shared(Some(sentinel), Relaxed);
        q
    }

    /// Add `t` to the back of the queue.
    pub fn push(&self, t: T) {
        let guard = epoch::pin();
        loop {
            let tail = self.tail.load(Acquire, &guard).unwrap();
            if tail.high.load(Relaxed) >= SEG_SIZE { continue }
            let i = tail.high.fetch_add(1, Relaxed);
            unsafe {
                if i < SEG_SIZE {
                    let cell = (*tail).data.get_unchecked(i).get();
                    ptr::write(&mut (*cell).0, t);
                    (*cell).1.store(true, Release);

                    if i + 1 == SEG_SIZE {
                        let tail = tail.next.store_and_ref(Owned::new(Segment::new()), Release, &guard);
                        self.tail.store_shared(Some(tail), Release);
                    }

                    return
                }
            }
        }
    }

    /// Attempt to dequeue from the front.
    ///
    /// Returns `None` if the queue is observed to be empty.
    pub fn try_pop(&self) -> Option<T> {
        let guard = epoch::pin();
        loop {
            let head = self.head.load(Acquire, &guard).unwrap();
            loop {
                let low = head.low.load(Relaxed);
                if low >= cmp::min(head.high.load(Relaxed), SEG_SIZE) { break }
                if head.low.compare_and_swap(low, low+1, Relaxed) == low {
                    unsafe {
                        let cell = (*head).data.get_unchecked(low).get();
                        loop {
                            if (*cell).1.load(Acquire) { break }
                        }
                        if low + 1 == SEG_SIZE {
                            loop {
                                if let Some(next) = head.next.load(Acquire, &guard) {
                                    self.head.store_shared(Some(next), Release);
                                    break
                                }
                            }
                        }
                        return Some(ptr::read(&(*cell).0))
                    }
                }
            }
            if head.next.load(Relaxed, &guard).is_none() { return None }
        }
    }
}

#[cfg(test)]
mod test {
    const CONC_COUNT: i64 = 1000000;

    use scope;
    use super::*;

    #[test]
    fn push_pop_1() {
        let q: SegQueue<i64> = SegQueue::new();
        q.push(37);
        assert_eq!(q.try_pop(), Some(37));
    }

    #[test]
    fn push_pop_2() {
        let q: SegQueue<i64> = SegQueue::new();
        q.push(37);
        q.push(48);
        assert_eq!(q.try_pop(), Some(37));
        assert_eq!(q.try_pop(), Some(48));
    }

    #[test]
    fn push_pop_many_seq() {
        let q: SegQueue<i64> = SegQueue::new();
        for i in 0..200 {
            q.push(i)
        }
        for i in 0..200 {
            assert_eq!(q.try_pop(), Some(i));
        }
    }

    #[test]
    fn push_pop_many_spsc() {
        let q: SegQueue<i64> = SegQueue::new();

        scope(|scope| {
            scope.spawn(|| {
                let mut next = 0;

                while next < CONC_COUNT {
                    if let Some(elem) = q.try_pop() {
                        assert_eq!(elem, next);
                        next += 1;
                    }
                }
            });

            for i in 0..CONC_COUNT {
                q.push(i)
            }
        });
    }

    #[test]
    fn push_pop_many_spmc() {
        fn recv(_t: i32, q: &SegQueue<i64>) {
            let mut cur = -1;
            for _i in 0..CONC_COUNT {
                if let Some(elem) = q.try_pop() {
                    assert!(elem > cur);
                    cur = elem;

                    if cur == CONC_COUNT - 1 { break }
                }
            }
        }

        let q: SegQueue<i64> = SegQueue::new();
        let qr = &q;
        scope(|scope| {
            for i in 0..3 {
                scope.spawn(move || recv(i, qr));
            }

            scope.spawn(|| {
                for i in 0..CONC_COUNT {
                    q.push(i);
                }
            })
        });
    }

    #[test]
    fn push_pop_many_mpmc() {
        enum LR { Left(i64), Right(i64) }

        let q: SegQueue<LR> = SegQueue::new();

        scope(|scope| {
            for _t in 0..2 {
                scope.spawn(|| {
                    for i in CONC_COUNT-1..CONC_COUNT {
                        q.push(LR::Left(i))
                    }
                });
                scope.spawn(|| {
                    for i in CONC_COUNT-1..CONC_COUNT {
                        q.push(LR::Right(i))
                    }
                });
                scope.spawn(|| {
                    let mut vl = vec![];
                    let mut vr = vec![];
                    for _i in 0..CONC_COUNT {
                        match q.try_pop() {
                            Some(LR::Left(x)) => vl.push(x),
                            Some(LR::Right(x)) => vr.push(x),
                            _ => {}
                        }
                    }

                    let mut vl2 = vl.clone();
                    let mut vr2 = vr.clone();
                    vl2.sort();
                    vr2.sort();

                    assert_eq!(vl, vl2);
                    assert_eq!(vr, vr2);
                });
            }
        });
    }
}
<commit_msg>Added GC back<commit_after>use std::sync::atomic::Ordering::{Acquire, Release, Relaxed};
use std::sync::atomic::{AtomicBool, AtomicUsize};
use std::{ptr, mem};
use std::cmp;
use std::cell::UnsafeCell;

use mem::epoch::{self, Atomic, Owned};

const SEG_SIZE: usize = 32;

/// A Michael-Scott queue that allocates "segments" (arrays of nodes)
/// for efficiency.
///
/// Usable with any number of producers and consumers.
pub struct SegQueue<T> {
    head: Atomic<Segment<T>>,
    tail: Atomic<Segment<T>>,
}

struct Segment<T> {
    low: AtomicUsize,
    data: [UnsafeCell<(T, AtomicBool)>; SEG_SIZE],
    high: AtomicUsize,
    next: Atomic<Segment<T>>,
}

unsafe impl<T: Send> Sync for Segment<T> {}

impl<T> Segment<T> {
    fn new() -> Segment<T> {
        let rqueue = Segment {
            data: unsafe { mem::uninitialized() },
            low: AtomicUsize::new(0),
            high: AtomicUsize::new(0),
            next: Atomic::null(),
        };
        for val in rqueue.data.iter() {
            unsafe {
                (*val.get()).1 = AtomicBool::new(false);
            }
        }
        rqueue
    }
}

impl<T> SegQueue<T> {
    /// Create a new, empty queue.
    pub fn new() -> SegQueue<T> {
        let q = SegQueue {
            head: Atomic::null(),
            tail: Atomic::null(),
        };
        let sentinel = Owned::new(Segment::new());
        let guard = epoch::pin();
        let sentinel = q.head.store_and_ref(sentinel, Relaxed, &guard);
        q.tail.store_shared(Some(sentinel), Relaxed);
        q
    }

    /// Add `t` to the back of the queue.
    pub fn push(&self, t: T) {
        let guard = epoch::pin();
        loop {
            let tail = self.tail.load(Acquire, &guard).unwrap();
            if tail.high.load(Relaxed) >= SEG_SIZE { continue }
            let i = tail.high.fetch_add(1, Relaxed);
            unsafe {
                if i < SEG_SIZE {
                    let cell = (*tail).data.get_unchecked(i).get();
                    ptr::write(&mut (*cell).0, t);
                    (*cell).1.store(true, Release);

                    if i + 1 == SEG_SIZE {
                        let tail = tail.next.store_and_ref(Owned::new(Segment::new()), Release, &guard);
                        self.tail.store_shared(Some(tail), Release);
                    }

                    return
                }
            }
        }
    }

    /// Attempt to dequeue from the front.
    ///
    /// Returns `None` if the queue is observed to be empty.
    pub fn try_pop(&self) -> Option<T> {
        let guard = epoch::pin();
        loop {
            let head = self.head.load(Acquire, &guard).unwrap();
            loop {
                let low = head.low.load(Relaxed);
                if low >= cmp::min(head.high.load(Relaxed), SEG_SIZE) { break }
                if head.low.compare_and_swap(low, low+1, Relaxed) == low {
                    unsafe {
                        let cell = (*head).data.get_unchecked(low).get();
                        loop {
                            if (*cell).1.load(Acquire) { break }
                        }
                        if low + 1 == SEG_SIZE {
                            loop {
                                if let Some(next) = head.next.load(Acquire, &guard) {
                                    self.head.store_shared(Some(next), Release);
                                    unsafe { guard.unlinked(head); }
                                    break
                                }
                            }
                        }
                        return Some(ptr::read(&(*cell).0))
                    }
                }
            }
            if head.next.load(Relaxed, &guard).is_none() { return None }
        }
    }
}

#[cfg(test)]
mod test {
    const CONC_COUNT: i64 = 1000000;

    use scope;
    use super::*;

    #[test]
    fn push_pop_1() {
        let q: SegQueue<i64> = SegQueue::new();
        q.push(37);
        assert_eq!(q.try_pop(), Some(37));
    }

    #[test]
    fn push_pop_2() {
        let q: SegQueue<i64> = SegQueue::new();
        q.push(37);
        q.push(48);
        assert_eq!(q.try_pop(), Some(37));
        assert_eq!(q.try_pop(), Some(48));
    }

    #[test]
    fn push_pop_many_seq() {
        let q: SegQueue<i64> = SegQueue::new();
        for i in 0..200 {
            q.push(i)
        }
        for i in 0..200 {
            assert_eq!(q.try_pop(), Some(i));
        }
    }

    #[test]
    fn push_pop_many_spsc() {
        let q: SegQueue<i64> = SegQueue::new();

        scope(|scope| {
            scope.spawn(|| {
                let mut next = 0;

                while next < CONC_COUNT {
                    if let Some(elem) = q.try_pop() {
                        assert_eq!(elem, next);
                        next += 1;
                    }
                }
            });

            for i in 0..CONC_COUNT {
                q.push(i)
            }
        });
    }

    #[test]
    fn push_pop_many_spmc() {
        fn recv(_t: i32, q: &SegQueue<i64>) {
            let mut cur = -1;
            for _i in 0..CONC_COUNT {
                if let Some(elem) = q.try_pop() {
                    assert!(elem > cur);
                    cur = elem;

                    if cur == CONC_COUNT - 1 { break }
                }
            }
        }

        let q: SegQueue<i64> = SegQueue::new();
        let qr = &q;
        scope(|scope| {
            for i in 0..3 {
                scope.spawn(move || recv(i, qr));
            }

            scope.spawn(|| {
                for i in 0..CONC_COUNT {
                    q.push(i);
                }
            })
        });
    }

    #[test]
    fn push_pop_many_mpmc() {
        enum LR { Left(i64), Right(i64) }

        let q: SegQueue<LR> = SegQueue::new();

        scope(|scope| {
            for _t in 0..2 {
                scope.spawn(|| {
                    for i in CONC_COUNT-1..CONC_COUNT {
                        q.push(LR::Left(i))
                    }
                });
                scope.spawn(|| {
                    for i in CONC_COUNT-1..CONC_COUNT {
                        q.push(LR::Right(i))
                    }
                });
                scope.spawn(|| {
                    let mut vl = vec![];
                    let mut vr = vec![];
                    for _i in 0..CONC_COUNT {
                        match q.try_pop() {
                            Some(LR::Left(x)) => vl.push(x),
                            Some(LR::Right(x)) => vr.push(x),
                            _ => {}
                        }
                    }

                    let mut vl2 = vl.clone();
                    let mut vr2 = vr.clone();
                    vl2.sort();
                    vr2.sort();

                    assert_eq!(vl, vl2);
                    assert_eq!(vr, vr2);
                });
            }
        });
    }
}
<|endoftext|>


<commit_before>use {io, Interest, PollOpt, Token};
use event::IoEvent;
use nix::fcntl::Fd;
use nix::sys::epoll::*;
use nix::unistd::close;

#[derive(Debug)]
pub struct Selector {
    epfd: Fd
}

impl Selector {
    pub fn new() -> io::Result<Selector> {
        let epfd = try!(epoll_create().map_err(super::from_nix_error));

        Ok(Selector { epfd: epfd })
    }

    /// Wait for events from the OS
    pub fn select(&mut self, evts: &mut Events, timeout_ms: usize) -> io::Result<()> {
        use std::slice;

        let dst = unsafe {
            slice::from_raw_parts_mut(
                evts.events.as_mut_ptr(),
                evts.events.capacity())
        };

        // Wait for epoll events for at most timeout_ms milliseconds
        let cnt = try!(epoll_wait(self.epfd, dst, timeout_ms)
                           .map_err(super::from_nix_error));

        unsafe { evts.events.set_len(cnt); }

        Ok(())
    }

    /// Register event interests for the given IO handle with the OS
    pub fn register(&mut self, fd: Fd, token: Token, interests: Interest, opts: PollOpt) -> io::Result<()> {
        let info = EpollEvent {
            events: ioevent_to_epoll(interests, opts),
            data: token.as_usize() as u64
        };

        epoll_ctl(self.epfd, EpollOp::EpollCtlAdd, fd, &info)
            .map_err(super::from_nix_error)
    }

    /// Register event interests for the given IO handle with the OS
    pub fn reregister(&mut self, fd: Fd, token: Token, interests: Interest, opts: PollOpt) -> io::Result<()> {
        let info = EpollEvent {
            events: ioevent_to_epoll(interests, opts),
            data: token.as_usize() as u64
        };

        epoll_ctl(self.epfd, EpollOp::EpollCtlMod, fd, &info)
            .map_err(super::from_nix_error)
    }

    /// Deregister event interests for the given IO handle with the OS
    pub fn deregister(&mut self, fd: Fd) -> io::Result<()> {
        // The &info argument should be ignored by the system,
        // but linux < 2.6.9 required it to be not null.
        // For compatibility, we provide a dummy EpollEvent.
        let info = EpollEvent {
            events: EpollEventKind::empty(),
            data: 0
        };

        epoll_ctl(self.epfd, EpollOp::EpollCtlDel, fd, &info)
            .map_err(super::from_nix_error)
    }
}

fn ioevent_to_epoll(interest: Interest, opts: PollOpt) -> EpollEventKind {
    let mut kind = EpollEventKind::empty();

    if interest.is_readable() {
        kind.insert(EPOLLIN);
    }

    if interest.is_writable() {
        kind.insert(EPOLLOUT);
    }

    if interest.is_hup() {
        kind.insert(EPOLLRDHUP);
    }

    if opts.is_edge() {
        kind.insert(EPOLLET);
    }

    if opts.is_oneshot() {
        kind.insert(EPOLLONESHOT);
    }

    if opts.is_level() {
        kind.remove(EPOLLET);
    }

    kind
}

impl Drop for Selector {
    fn drop(&mut self) {
        let _ = close(self.epfd);
    }
}

pub struct Events {
    events: Vec<EpollEvent>,
}

impl Events {
    pub fn new() -> Events {
        Events { events: Vec::with_capacity(1024) }
    }

    #[inline]
    pub fn len(&self) -> usize {
        self.events.len()
    }

    #[inline]
    pub fn get(&self, idx: usize) -> IoEvent {
        if idx >= self.len() {
            panic!("invalid index");
        }

        let epoll = self.events[idx].events;
        let mut kind = Interest::hinted();

        if epoll.contains(EPOLLIN) {
            kind = kind | Interest::readable();
        }

        if epoll.contains(EPOLLOUT) {
            kind = kind | Interest::writable();
        }

        // EPOLLHUP - Usually means a socket error happened
        if epoll.contains(EPOLLERR) {
            kind = kind | Interest::error();
        }

        if epoll.contains(EPOLLRDHUP) | epoll.contains(EPOLLHUP) {
            kind = kind | Interest::hup();
        }

        let token = self.events[idx].data;

        IoEvent::new(kind, token as usize)
    }
}
<commit_msg>Track nix master<commit_after>use {io, Interest, PollOpt, Token};
use event::IoEvent;
use nix::fcntl::Fd;
use nix::sys::epoll::*;
use nix::unistd::close;

#[derive(Debug)]
pub struct Selector {
    epfd: Fd
}

impl Selector {
    pub fn new() -> io::Result<Selector> {
        let epfd = try!(epoll_create().map_err(super::from_nix_error));

        Ok(Selector { epfd: epfd })
    }

    /// Wait for events from the OS
    pub fn select(&mut self, evts: &mut Events, timeout_ms: usize) -> io::Result<()> {
        use std::{isize, slice};

        let timeout_ms = if timeout_ms >= isize::MAX as usize {
            isize::MAX
        } else {
            timeout_ms as isize
        };

        let dst = unsafe {
            slice::from_raw_parts_mut(
                evts.events.as_mut_ptr(),
                evts.events.capacity())
        };

        // Wait for epoll events for at most timeout_ms milliseconds
        let cnt = try!(epoll_wait(self.epfd, dst, timeout_ms)
                           .map_err(super::from_nix_error));

        unsafe { evts.events.set_len(cnt); }

        Ok(())
    }

    /// Register event interests for the given IO handle with the OS
    pub fn register(&mut self, fd: Fd, token: Token, interests: Interest, opts: PollOpt) -> io::Result<()> {
        let info = EpollEvent {
            events: ioevent_to_epoll(interests, opts),
            data: token.as_usize() as u64
        };

        epoll_ctl(self.epfd, EpollOp::EpollCtlAdd, fd, &info)
            .map_err(super::from_nix_error)
    }

    /// Register event interests for the given IO handle with the OS
    pub fn reregister(&mut self, fd: Fd, token: Token, interests: Interest, opts: PollOpt) -> io::Result<()> {
        let info = EpollEvent {
            events: ioevent_to_epoll(interests, opts),
            data: token.as_usize() as u64
        };

        epoll_ctl(self.epfd, EpollOp::EpollCtlMod, fd, &info)
            .map_err(super::from_nix_error)
    }

    /// Deregister event interests for the given IO handle with the OS
    pub fn deregister(&mut self, fd: Fd) -> io::Result<()> {
        // The &info argument should be ignored by the system,
        // but linux < 2.6.9 required it to be not null.
        // For compatibility, we provide a dummy EpollEvent.
        let info = EpollEvent {
            events: EpollEventKind::empty(),
            data: 0
        };

        epoll_ctl(self.epfd, EpollOp::EpollCtlDel, fd, &info)
            .map_err(super::from_nix_error)
    }
}

fn ioevent_to_epoll(interest: Interest, opts: PollOpt) -> EpollEventKind {
    let mut kind = EpollEventKind::empty();

    if interest.is_readable() {
        kind.insert(EPOLLIN);
    }

    if interest.is_writable() {
        kind.insert(EPOLLOUT);
    }

    if interest.is_hup() {
        kind.insert(EPOLLRDHUP);
    }

    if opts.is_edge() {
        kind.insert(EPOLLET);
    }

    if opts.is_oneshot() {
        kind.insert(EPOLLONESHOT);
    }

    if opts.is_level() {
        kind.remove(EPOLLET);
    }

    kind
}

impl Drop for Selector {
    fn drop(&mut self) {
        let _ = close(self.epfd);
    }
}

pub struct Events {
    events: Vec<EpollEvent>,
}

impl Events {
    pub fn new() -> Events {
        Events { events: Vec::with_capacity(1024) }
    }

    #[inline]
    pub fn len(&self) -> usize {
        self.events.len()
    }

    #[inline]
    pub fn get(&self, idx: usize) -> IoEvent {
        if idx >= self.len() {
            panic!("invalid index");
        }

        let epoll = self.events[idx].events;
        let mut kind = Interest::hinted();

        if epoll.contains(EPOLLIN) {
            kind = kind | Interest::readable();
        }

        if epoll.contains(EPOLLOUT) {
            kind = kind | Interest::writable();
        }

        // EPOLLHUP - Usually means a socket error happened
        if epoll.contains(EPOLLERR) {
            kind = kind | Interest::error();
        }

        if epoll.contains(EPOLLRDHUP) | epoll.contains(EPOLLHUP) {
            kind = kind | Interest::hup();
        }

        let token = self.events[idx].data;

        IoEvent::new(kind, token as usize)
    }
}
<|endoftext|>


<commit_before>use std::cmp;
use buffer::{Lexeme, Position, Token};
use syntect::parsing::{ParseState, ScopeStack, ScopeStackOp, SyntaxDefinition};
use buffer::token::line_iterator::LineIterator;

pub struct TokenIterator<'a> {
    scopes: ScopeStack,
    parser: ParseState,
    lines: LineIterator<'a>,
    current_line: Option<&'a str>,
    current_position: Position,
    line_events: Vec<(usize, ScopeStackOp)>,
}

impl<'a> TokenIterator<'a> {
    pub fn new(data: &'a str, def: &SyntaxDefinition) -> TokenIterator<'a> {
        let mut token_iterator = TokenIterator{
            scopes: ScopeStack::new(),
            parser: ParseState::new(def),
            lines: LineIterator::new(data),
            current_line: None,
            current_position: Position{ line: 0, offset: 0 },
            line_events: Vec::new(),
        };

        // Preload the first line
        token_iterator.parse_next_line();

        token_iterator
    }

    fn next_token(&mut self) -> Option<Token<'a>> {
        // Try to fetch a token from the current line.
        if let Some(token) = self.build_next_token() {
            return Some(token)
        }

        // We're done with this line; on to the next.
        self.parse_next_line();
        if self.current_line.is_some() {
            Some(Token::Newline)
        } else {
            None
        }
    }

    fn build_next_token(&mut self) -> Option<Token<'a>> {
        let mut lexeme = None;

        if let Some(line) = self.current_line {
            // Exclude trailing newlines (we have a Newline variant for that).
            let end_of_line = if line.chars().last() == Some('\n') {
                line.len() - 1
            } else {
                line.len()
            };

            while let Some((event_offset, scope_change)) = self.line_events.pop() {
                // We want to capture the full scope for a given token, so we
                // need to make sure we apply all of them and only capture it
                // once we've moved on to another token/offset.
                if event_offset > self.current_position.offset {
                    // Don't include trailing newlines in lexemes.
                    let end_of_token = cmp::min(event_offset, end_of_line);

                    lexeme = Some(
                        Token::Lexeme(Lexeme{
                            value: &line[self.current_position.offset..end_of_token],
                            scope: self.scopes.clone(),
                            position: self.current_position.clone(),
                        })
                    );
                    self.current_position.offset = event_offset;
                }

                // Apply the scope and keep a reference to it, so
                // that we can pair it with a token later on.
                self.scopes.apply(&scope_change);

                if lexeme.is_some() { return lexeme }
            }

            // If the rest of the line hasn't triggered a scope
            // change; categorize it with the last known scope.
            if self.current_position.offset < end_of_line {
                lexeme = Some(
                    Token::Lexeme(Lexeme{
                        value: &line[self.current_position.offset..end_of_line],
                        scope: self.scopes.clone(),
                        position: self.current_position.clone(),
                    })
                );
            }
        }
        self.current_line = None;

        lexeme
    }

    fn parse_next_line(&mut self) {
        if let Some((line_number, line)) = self.lines.next() {
            // We reverse the line elements so that we can pop them off one at a
            // time, handling each event while allowing us to stop at any point.
            let mut line_events = self.parser.parse_line(line);
            line_events.reverse();
            self.line_events = line_events;

            // Keep a reference to the line so that we can create slices of it.
            self.current_line = Some(line);

            // Track our position, which we'll pass to generated tokens.
            self.current_position = Position{ line: line_number, offset: 0 };
        } else {
            self.current_line = None;
        }
    }
}

impl<'a> Iterator for TokenIterator<'a> {
    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        self.next_token()
    }
}

#[cfg(test)]
mod tests {
    use super::TokenIterator;
    use buffer::{Lexeme, Position, ScopeStack, Token};
    use syntect::parsing::{Scope, SyntaxSet};

    #[test]
    fn token_iterator_returns_correct_tokens() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("struct Buffer {\n// comment\n  data: String\n}garbage\n\n", def.unwrap());
        let mut scope_stack = ScopeStack::new();
        let mut expected_tokens = Vec::new();
        scope_stack.push(Scope::new("source.rust").unwrap());
        scope_stack.push(Scope::new("meta.struct.rust").unwrap());
        scope_stack.push(Scope::new("storage.type.struct.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "struct",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 0 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " ",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 6 }
        }));
        scope_stack.push(Scope::new("entity.name.struct.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "Buffer",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 7 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " ",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 13 }
        }));
        scope_stack.push(Scope::new("meta.block.rust").unwrap());
        scope_stack.push(Scope::new("punctuation.section.block.begin.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "{",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 14 }
        }));
        expected_tokens.push(Token::Newline);
        scope_stack.pop();
        scope_stack.push(Scope::new("comment.line.double-slash.rust").unwrap());
        scope_stack.push(Scope::new("punctuation.definition.comment.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "//",
            scope: scope_stack.clone(),
            position: Position{ line: 1, offset: 0 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " comment",
            scope: scope_stack.clone(),
            position: Position{ line: 1, offset: 2 }
        }));
        expected_tokens.push(Token::Newline);
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "  ",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 0 }
        }));
        scope_stack.push(Scope::new("variable.other.member.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "data",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 2 }
        }));
        scope_stack.pop();
        scope_stack.push(Scope::new("punctuation.separator.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: ":",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 6 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " String",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 7 }
        }));
        expected_tokens.push(Token::Newline);
        scope_stack.push(Scope::new("punctuation.section.block.end.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "}",
            scope: scope_stack.clone(),
            position: Position{ line: 3, offset: 0 }
        }));
        scope_stack.pop();
        scope_stack.pop();
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "garbage",
            scope: scope_stack.clone(),
            position: Position{ line: 3, offset: 1 }
        }));
        expected_tokens.push(Token::Newline);
        expected_tokens.push(Token::Newline);
        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }

    #[test]
    fn token_iterator_handles_content_without_trailing_newline() {
        let syntax_set = SyntaxSet::load_defaults_newlines();

        // It's important to use a plain text lexer so that the last token
        // doesn't introduce a scope change, forcing the EOL handling logic.
        let def = syntax_set.find_syntax_plain_text();
        let iterator = TokenIterator::new("struct", def);
        let mut expected_tokens = Vec::new();
        expected_tokens.push(
            Token::Lexeme(Lexeme{
                value: "struct",
                scope: ScopeStack::from_vec(vec![
                    Scope::new("text.plain").unwrap(),
                ]),
                position: Position{ line: 0, offset: 0 }
            })
        );
        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }
}
<commit_msg>Use grapheme cluster-based offsets for token positions<commit_after>use std::cmp;
use buffer::{Lexeme, Position, Token};
use syntect::parsing::{ParseState, ScopeStack, ScopeStackOp, SyntaxDefinition};
use buffer::token::line_iterator::LineIterator;
use unicode_segmentation::UnicodeSegmentation;

pub struct TokenIterator<'a> {
    scopes: ScopeStack,
    parser: ParseState,
    lines: LineIterator<'a>,
    current_line: Option<&'a str>,
    current_byte_offset: usize,
    current_position: Position,
    line_events: Vec<(usize, ScopeStackOp)>,
}

impl<'a> TokenIterator<'a> {
    pub fn new(data: &'a str, def: &SyntaxDefinition) -> TokenIterator<'a> {
        let mut token_iterator = TokenIterator{
            scopes: ScopeStack::new(),
            parser: ParseState::new(def),
            lines: LineIterator::new(data),
            current_line: None,
            current_byte_offset: 0,
            current_position: Position{ line: 0, offset: 0 },
            line_events: Vec::new(),
        };

        // Preload the first line
        token_iterator.parse_next_line();

        token_iterator
    }

    fn next_token(&mut self) -> Option<Token<'a>> {
        // Try to fetch a token from the current line.
        if let Some(token) = self.build_next_token() {
            return Some(token)
        }

        // We're done with this line; on to the next.
        self.parse_next_line();
        if self.current_line.is_some() {
            Some(Token::Newline)
        } else {
            None
        }
    }

    fn build_next_token(&mut self) -> Option<Token<'a>> {
        let mut lexeme = None;

        if let Some(line) = self.current_line {
            // Exclude trailing newlines (we have a Newline variant for that).
            let end_of_line = if line.chars().last() == Some('\n') {
                line.len() - 1
            } else {
                line.len()
            };

            while let Some((event_offset, scope_change)) = self.line_events.pop() {
                // We want to capture the full scope for a given token, so we
                // need to make sure we apply all of them and only capture it
                // once we've moved on to another token/offset.
                if event_offset > self.current_byte_offset {
                    // Don't include trailing newlines in lexemes.
                    let end_of_token = cmp::min(event_offset, end_of_line);

                    lexeme = Some(
                        Token::Lexeme(Lexeme{
                            value: &line[self.current_byte_offset..end_of_token],
                            scope: self.scopes.clone(),
                            position: self.current_position.clone(),
                        })
                    );

                    // The event/current offsets are byte-based, but
                    // position offsets should be grapheme cluster-based.
                    self.current_position.offset +=
                        *&line[self.current_byte_offset..end_of_token]
                        .graphemes(true)
                        .count();

                    self.current_byte_offset = event_offset;
                }

                // Apply the scope and keep a reference to it, so
                // that we can pair it with a token later on.
                self.scopes.apply(&scope_change);

                if lexeme.is_some() { return lexeme }
            }

            // Categorize the rest of the line with the last known scope.
            if self.current_byte_offset < end_of_line {
                lexeme = Some(
                    Token::Lexeme(Lexeme{
                        value: &line[self.current_byte_offset..end_of_line],
                        scope: self.scopes.clone(),
                        position: self.current_position.clone(),
                    })
                );
            }
        }

        // We've finished processing the current line; clean it up.
        self.current_line = None;

        lexeme
    }

    fn parse_next_line(&mut self) {
        if let Some((line_number, line)) = self.lines.next() {
            // We reverse the line elements so that we can pop them off one at a
            // time, handling each event while allowing us to stop at any point.
            let mut line_events = self.parser.parse_line(line);
            line_events.reverse();
            self.line_events = line_events;

            // Keep a reference to the line so that we can create slices of it.
            self.current_line = Some(line);

            // Track our position, which we'll pass to generated tokens.
            self.current_position = Position{ line: line_number, offset: 0 };

            // Reset byte-based line offset.
            self.current_byte_offset = 0;
        } else {
            self.current_line = None;
        }
    }
}

impl<'a> Iterator for TokenIterator<'a> {
    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        self.next_token()
    }
}

#[cfg(test)]
mod tests {
    use super::TokenIterator;
    use buffer::{Lexeme, Position, ScopeStack, Token};
    use syntect::parsing::{Scope, SyntaxSet};

    #[test]
    fn token_iterator_returns_correct_tokens() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("struct Buffer {\n// comment\n  data: String\n}garbage\n\n", def.unwrap());
        let mut scope_stack = ScopeStack::new();
        let mut expected_tokens = Vec::new();
        scope_stack.push(Scope::new("source.rust").unwrap());
        scope_stack.push(Scope::new("meta.struct.rust").unwrap());
        scope_stack.push(Scope::new("storage.type.struct.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "struct",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 0 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " ",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 6 }
        }));
        scope_stack.push(Scope::new("entity.name.struct.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "Buffer",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 7 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " ",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 13 }
        }));
        scope_stack.push(Scope::new("meta.block.rust").unwrap());
        scope_stack.push(Scope::new("punctuation.section.block.begin.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "{",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 14 }
        }));
        expected_tokens.push(Token::Newline);
        scope_stack.pop();
        scope_stack.push(Scope::new("comment.line.double-slash.rust").unwrap());
        scope_stack.push(Scope::new("punctuation.definition.comment.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "//",
            scope: scope_stack.clone(),
            position: Position{ line: 1, offset: 0 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " comment",
            scope: scope_stack.clone(),
            position: Position{ line: 1, offset: 2 }
        }));
        expected_tokens.push(Token::Newline);
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "  ",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 0 }
        }));
        scope_stack.push(Scope::new("variable.other.member.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "data",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 2 }
        }));
        scope_stack.pop();
        scope_stack.push(Scope::new("punctuation.separator.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: ":",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 6 }
        }));
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: " String",
            scope: scope_stack.clone(),
            position: Position{ line: 2, offset: 7 }
        }));
        expected_tokens.push(Token::Newline);
        scope_stack.push(Scope::new("punctuation.section.block.end.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "}",
            scope: scope_stack.clone(),
            position: Position{ line: 3, offset: 0 }
        }));
        scope_stack.pop();
        scope_stack.pop();
        scope_stack.pop();
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "garbage",
            scope: scope_stack.clone(),
            position: Position{ line: 3, offset: 1 }
        }));
        expected_tokens.push(Token::Newline);
        expected_tokens.push(Token::Newline);
        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }

    #[test]
    fn token_iterator_handles_content_without_trailing_newline() {
        let syntax_set = SyntaxSet::load_defaults_newlines();

        // It's important to use a plain text lexer so that the last token
        // doesn't introduce a scope change, forcing the EOL handling logic.
        let def = syntax_set.find_syntax_plain_text();
        let iterator = TokenIterator::new("struct", def);
        let mut expected_tokens = Vec::new();
        expected_tokens.push(
            Token::Lexeme(Lexeme{
                value: "struct",
                scope: ScopeStack::from_vec(vec![
                    Scope::new("text.plain").unwrap(),
                ]),
                position: Position{ line: 0, offset: 0 }
            })
        );
        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }

    #[test]
    fn token_iterator_handles_unicode_characters() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("€16", def.unwrap());
        let mut scope_stack = ScopeStack::new();
        let mut expected_tokens = Vec::new();
        scope_stack.push(Scope::new("source.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "€",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 0 }
        }));
        scope_stack.push(Scope::new("constant.numeric.integer.decimal.rust").unwrap());
        expected_tokens.push(Token::Lexeme(Lexeme{
            value: "16",
            scope: scope_stack.clone(),
            position: Position{ line: 0, offset: 1 }
        }));

        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }
    }
}
<|endoftext|>


<commit_before>use std::env;
use std::fmt;
use std::fs::{self, PathExt};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process;
use std::str::FromStr;
use std::sync::atomic;

use csv;

use Csv;

static XSV_INTEGRATION_TEST_DIR: &'static str = "xit";

static NEXT_ID: atomic::AtomicUsize = atomic::ATOMIC_USIZE_INIT;

pub struct Workdir {
    root: PathBuf,
    dir: PathBuf,
    flexible: bool,
}

impl Workdir {
    pub fn new(name: &str) -> Workdir {
        let id = NEXT_ID.fetch_add(1, atomic::Ordering::SeqCst);
        let root = PathBuf::new(&env::current_exe().unwrap())
                           .parent()
                           .expect("executable's directory")
                           .to_path_buf();
        let dir = root.join(XSV_INTEGRATION_TEST_DIR)
                      .join(name)
                      .join(&format!("test-{}", id));
        if dir.exists() {
            if let Err(err) = fs::remove_dir_all(&dir) {
                panic!("{}", err);
            }
        }
        if let Err(err) = fs::create_dir_all(&dir) {
            panic!("{}", err);
        }
        Workdir { root: root, dir: dir, flexible: false }
    }

    pub fn flexible(mut self, yes: bool) -> Workdir {
        self.flexible = yes;
        self
    }

    pub fn create<T: Csv>(&self, name: &str, rows: T) {
        let mut wtr = csv::Writer::from_file(&self.path(name)).unwrap()
                                  .flexible(self.flexible);
        for row in rows.to_vecs().into_iter() {
            wtr.write(row.iter()).unwrap();
        }
        wtr.flush().unwrap();
    }

    pub fn create_indexed<T: Csv>(&self, name: &str, rows: T) {
        self.create(name, rows);

        let mut cmd = self.command("index");
        cmd.arg(name);
        self.run(&mut cmd);
    }

    pub fn read_stdout<T: Csv>(&self, cmd: &mut process::Command) -> T {
        let mut rdr = csv::Reader::from_string(self.stdout::<String>(cmd))
                                  .has_headers(false);
        Csv::from_vecs(rdr.records().collect::<Result<_, _>>().unwrap())
    }

    pub fn command(&self, sub_command: &str) -> process::Command {
        let mut cmd = process::Command::new(&self.xsv_bin());
        cmd.current_dir(&self.dir).arg(sub_command);
        cmd
    }

    pub fn output(&self, cmd: &mut process::Command) -> process::Output {
        debug!("[{}]: {:?}", self.dir.display(), cmd);
        let o = cmd.output().unwrap();
        if !o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command failed but expected success!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr))
        }
        o
    }

    pub fn run(&self, cmd: &mut process::Command) {
        self.output(cmd);
    }

    pub fn stdout<T: FromStr>(&self, cmd: &mut process::Command) -> T {
        let o = self.output(cmd);
        let stdout = String::from_utf8_lossy(&o.stdout);
        stdout.trim().parse().ok().expect(
            &format!("Could not convert from string: '{}'", stdout))
    }

    pub fn assert_err(&self, cmd: &mut process::Command) {
        let o = cmd.output().unwrap();
        if o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command succeeded but expected failure!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr));
        }
    }

    pub fn from_str<T: FromStr>(&self, name: &Path) -> T {
        let mut o = String::new();
        fs::File::open(name).unwrap().read_to_string(&mut o).unwrap();
        o.parse().ok().expect("fromstr")
    }

    pub fn path(&self, name: &str) -> PathBuf {
        self.dir.join(name)
    }

    pub fn xsv_bin(&self) -> PathBuf {
        self.root.join("xsv")
    }
}

impl fmt::Debug for Workdir {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "path={}", self.dir.display())
    }
}
<commit_msg>Try to appease travis.<commit_after>use std::env;
use std::fmt;
use std::fs::{self, PathExt};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process;
use std::str::FromStr;
use std::sync::atomic;

use csv;

use Csv;

static XSV_INTEGRATION_TEST_DIR: &'static str = "xit";

static NEXT_ID: atomic::AtomicUsize = atomic::ATOMIC_USIZE_INIT;

pub struct Workdir {
    root: PathBuf,
    dir: PathBuf,
    flexible: bool,
}

impl Workdir {
    pub fn new(name: &str) -> Workdir {
        let id = NEXT_ID.fetch_add(1, atomic::Ordering::SeqCst);
        let root = PathBuf::new(&env::current_exe().unwrap())
                           .parent()
                           .expect("executable's directory")
                           .to_path_buf();
        let dir = root.join(XSV_INTEGRATION_TEST_DIR)
                      .join(name)
                      .join(&format!("test-{}", id));
        if dir.exists() {
            if let Err(err) = fs::remove_dir_all(&dir) {
                panic!("{}", err);
            }
        }
        // Explicitly ignoring an error here seems wrong, but for whatever
        // reason, this appears to fail non-deterministically in Travis because
        // the "path already exists." Does that mean `NEXT_ID` isn't really
        // atomic? Really? Very strange and baffling. ---AG
        let _ = fs::create_dir_all(&dir);
        Workdir { root: root, dir: dir, flexible: false }
    }

    pub fn flexible(mut self, yes: bool) -> Workdir {
        self.flexible = yes;
        self
    }

    pub fn create<T: Csv>(&self, name: &str, rows: T) {
        let mut wtr = csv::Writer::from_file(&self.path(name)).unwrap()
                                  .flexible(self.flexible);
        for row in rows.to_vecs().into_iter() {
            wtr.write(row.iter()).unwrap();
        }
        wtr.flush().unwrap();
    }

    pub fn create_indexed<T: Csv>(&self, name: &str, rows: T) {
        self.create(name, rows);

        let mut cmd = self.command("index");
        cmd.arg(name);
        self.run(&mut cmd);
    }

    pub fn read_stdout<T: Csv>(&self, cmd: &mut process::Command) -> T {
        let mut rdr = csv::Reader::from_string(self.stdout::<String>(cmd))
                                  .has_headers(false);
        Csv::from_vecs(rdr.records().collect::<Result<_, _>>().unwrap())
    }

    pub fn command(&self, sub_command: &str) -> process::Command {
        let mut cmd = process::Command::new(&self.xsv_bin());
        cmd.current_dir(&self.dir).arg(sub_command);
        cmd
    }

    pub fn output(&self, cmd: &mut process::Command) -> process::Output {
        debug!("[{}]: {:?}", self.dir.display(), cmd);
        let o = cmd.output().unwrap();
        if !o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command failed but expected success!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr))
        }
        o
    }

    pub fn run(&self, cmd: &mut process::Command) {
        self.output(cmd);
    }

    pub fn stdout<T: FromStr>(&self, cmd: &mut process::Command) -> T {
        let o = self.output(cmd);
        let stdout = String::from_utf8_lossy(&o.stdout);
        stdout.trim().parse().ok().expect(
            &format!("Could not convert from string: '{}'", stdout))
    }

    pub fn assert_err(&self, cmd: &mut process::Command) {
        let o = cmd.output().unwrap();
        if o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command succeeded but expected failure!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr));
        }
    }

    pub fn from_str<T: FromStr>(&self, name: &Path) -> T {
        let mut o = String::new();
        fs::File::open(name).unwrap().read_to_string(&mut o).unwrap();
        o.parse().ok().expect("fromstr")
    }

    pub fn path(&self, name: &str) -> PathBuf {
        self.dir.join(name)
    }

    pub fn xsv_bin(&self) -> PathBuf {
        self.root.join("xsv")
    }
}

impl fmt::Debug for Workdir {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "path={}", self.dir.display())
    }
}
<|endoftext|>