<commit_before>#![allow(dead_code)]

//! The `ParallelIterator` module makes it easy to write parallel
//! programs using an iterator-style interface. To get access to all
//! the methods you want, the easiest is to write `use
//! rayon::par_iter::*;` at the top of your module, which will import
//! the various traits and methods you need.
//!
//! The submodules of this module mostly just contain implementaton
//! details of little interest to an end-user. If you'd like to read
//! the code itself, the `internal` module and `README.md` file are a
//! good place to start.

use std::f64;
use std::ops::Fn;
use self::collect::collect_into;
use self::enumerate::Enumerate;
use self::filter::Filter;
use self::filter_map::FilterMap;
use self::flat_map::FlatMap;
use self::map::Map;
use self::reduce::{reduce, ReduceOp, SumOp, MulOp, MinOp, MaxOp, ReduceWithOp,
                   ReduceWithIdentityOp, SUM, MUL, MIN, MAX};
use self::internal::*;
use self::weight::Weight;
use self::zip::ZipIter;

pub mod collect;
pub mod enumerate;
pub mod filter;
pub mod filter_map;
pub mod flat_map;
pub mod internal;
pub mod len;
pub mod for_each;
#[cfg(feature = "unstable")]
pub mod fold;
pub mod reduce;
pub mod slice;
pub mod slice_mut;
pub mod map;
pub mod weight;
pub mod zip;
pub mod range;
pub mod vec;

#[cfg(test)]
mod test;

pub trait IntoParallelIterator {
    type Iter: ParallelIterator<Item=Self::Item>;
    type Item: Send;

    fn into_par_iter(self) -> Self::Iter;
}

pub trait IntoParallelRefIterator<'data> {
    type Iter: ParallelIterator<Item=&'data Self::Item>;
    type Item: Sync + 'data;

    fn par_iter(&'data self) -> Self::Iter;
}

pub trait IntoParallelRefMutIterator<'data> {
    type Iter: ParallelIterator<Item=&'data mut Self::Item>;
    type Item: Send + 'data;

    fn par_iter_mut(&'data mut self) -> Self::Iter;
}

/// The `ParallelIterator` interface.
pub trait ParallelIterator: Sized {
    type Item: Send;

    /// Indicates the relative "weight" of producing each item in this
    /// parallel iterator. A higher weight will cause finer-grained
    /// parallel subtasks. 1.0 indicates something very cheap and
    /// uniform, like copying a value out of an array, or computing `x
    /// + 1`. If your tasks are either very expensive, or very
    /// unpredictable, you are better off with higher values. See also
    /// `weight_max`, which is a convenient shorthand to force the
    /// finest grained parallel execution posible. Tuning this value
    /// should not affect correctness but can improve (or hurt)
    /// performance.
    fn weight(self, scale: f64) -> Weight<Self> {
        Weight::new(self, scale)
    }

    /// Shorthand for `self.weight(f64::INFINITY)`. This forces the
    /// smallest granularity of parallel execution, which makes sense
    /// when your parallel tasks are (potentially) very expensive to
    /// execute.
    fn weight_max(self) -> Weight<Self> {
        self.weight(f64::INFINITY)
    }

    /// Executes `OP` on each item produced by the iterator, in parallel.
    fn for_each<OP>(self, op: OP)
        where OP: Fn(Self::Item) + Sync
    {
        for_each::for_each(self, &op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn map<MAP_OP,R>(self, map_op: MAP_OP) -> Map<Self, MAP_OP>
        where MAP_OP: Fn(Self::Item) -> R
    {
        Map::new(self, map_op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn filter<FILTER_OP>(self, filter_op: FILTER_OP) -> Filter<Self, FILTER_OP>
        where FILTER_OP: Fn(&Self::Item) -> bool
    {
        Filter::new(self, filter_op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn filter_map<FILTER_OP,R>(self, filter_op: FILTER_OP) -> FilterMap<Self, FILTER_OP>
        where FILTER_OP: Fn(Self::Item) -> Option<R>
    {
        FilterMap::new(self, filter_op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn flat_map<MAP_OP,PI>(self, map_op: MAP_OP) -> FlatMap<Self, MAP_OP>
        where MAP_OP: Fn(Self::Item) -> PI, PI: ParallelIterator
    {
        FlatMap::new(self, map_op)
    }

    /// Reduces the items in the iterator into one item using `op`.
    /// See also `sum`, `mul`, `min`, etc, which are slightly more
    /// efficient. Returns `None` if the iterator is empty.
    ///
    /// Note: unlike in a sequential iterator, the order in which `op`
    /// will be applied to reduce the result is not specified. So `op`
    /// should be commutative and associative or else the results will
    /// be non-deterministic.
    fn reduce_with<OP>(self, op: OP) -> Option<Self::Item>
        where OP: Fn(Self::Item, Self::Item) -> Self::Item + Sync,
    {
        reduce(self.map(Some), &ReduceWithOp::new(&op))
    }

    /// Reduces the items in the iterator into one item using `op`.
    /// The argument `identity` represents an "identity" value which
    /// may be inserted into the sequence as needed to create
    /// opportunities for parallel execution. So, for example, if you
    /// are doing a summation, then `identity` ought to be something
    /// that represents the zero for your type (but consider just
    /// calling `sum()` in that case).
    ///
    /// Example `vectors.par_iter().reduce_with_identity(Vector::zero(), Vector::add)`.
    ///
    /// Note: unlike in a sequential iterator, the order in which `op`
    /// will be applied to reduce the result is not specified. So `op`
    /// should be commutative and associative or else the results will
    /// be non-deterministic. And of course `identity` should be a
    /// true identity.
    fn reduce_with_identity<OP>(self, identity: Self::Item, op: OP) -> Self::Item
        where OP: Fn(Self::Item, Self::Item) -> Self::Item + Sync,
              Self::Item: Clone + Sync,
    {
        reduce(self, &ReduceWithIdentityOp::new(&identity, &op))
    }

    #[cfg(feature = "unstable")]
    fn fold<I,FOLD_OP,REDUCE_OP>(self,
                                 identity: I,
                                 fold_op: FOLD_OP,
                                 reduce_op: REDUCE_OP)
                                 -> I
        where FOLD_OP: Fn(I, Self::Item) -> I + Sync,
              REDUCE_OP: Fn(I, I) -> I + Sync,
              I: Clone + Sync + Send,
    {
        fold::fold(self, &identity, &fold_op, &reduce_op)
    }

    /// Sums up the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `+` operator is not truly commutative and
    /// associative (as is the case for floating point numbers), then
    /// the results are not fully deterministic.
    fn sum(self) -> Self::Item
        where SumOp: ReduceOp<Self::Item>
    {
        reduce(self, SUM)
    }

    /// Multiplies all the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `*` operator is not truly commutative and
    /// associative (as is the case for floating point numbers), then
    /// the results are not fully deterministic.
    fn mul(self) -> Self::Item
        where MulOp: ReduceOp<Self::Item>
    {
        reduce(self, MUL)
    }

    /// Computes the minimum of all the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `Ord` impl is not truly commutative and associative
    /// (as is the case for floating point numbers), then the results
    /// are not deterministic.
    fn min(self) -> Self::Item
        where MinOp: ReduceOp<Self::Item>
    {
        reduce(self, MIN)
    }

    /// Computes the maximum of all the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `Ord` impl is not truly commutative and associative
    /// (as is the case for floating point numbers), then the results
    /// are not deterministic.
    fn max(self) -> Self::Item
        where MaxOp: ReduceOp<Self::Item>
    {
        reduce(self, MAX)
    }

    /// Reduces the items using the given "reduce operator". You may
    /// prefer `reduce_with` for a simpler interface.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `reduce_op` impl is not truly commutative and
    /// associative, then the results are not deterministic.
    fn reduce<REDUCE_OP>(self, reduce_op: &REDUCE_OP) -> Self::Item
        where REDUCE_OP: ReduceOp<Self::Item>
    {
        reduce(self, reduce_op)
    }

    /// Internal method used to define the behavior of this parallel
    /// iterator. You should not need to call this directly.
    #[doc(hidden)]
    fn drive_unindexed<C>(self, consumer: C) -> C::Result
        where C: UnindexedConsumer<Self::Item>;
}

impl<T: ParallelIterator> IntoParallelIterator for T {
    type Iter = T;
    type Item = T::Item;

    fn into_par_iter(self) -> T {
        self
    }
}

/// A trait for parallel iterators items where the precise number of
/// items is not known, but we can at least give an upper-bound. These
/// sorts of iterators result from filtering.
pub trait BoundedParallelIterator: ParallelIterator {
    fn upper_bound(&mut self) -> usize;

    /// Internal method used to define the behavior of this parallel
    /// iterator. You should not need to call this directly.
    #[doc(hidden)]
    fn drive<'c, C: Consumer<Self::Item>>(self,
                                                   consumer: C)
                                                   -> C::Result;

}

/// A trait for parallel iterators items where the precise number of
/// items is known. This occurs when e.g. iterating over a
/// vector. Knowing precisely how many items will be produced is very
/// useful.
pub trait ExactParallelIterator: BoundedParallelIterator {
    /// Produces an exact count of how many items this iterator will
    /// produce, presuming no panic occurs.
    ///
    /// # Safety note
    ///
    /// Returning an incorrect value here could lead to **undefined
    /// behavior**.
    fn len(&mut self) -> usize;

    /// Collects the results of the iterator into the specified
    /// vector. The vector is always truncated before execution
    /// begins. If possible, reusing the vector across calls can lead
    /// to better performance since it reuses the same backing buffer.
    fn collect_into(self, target: &mut Vec<Self::Item>) {
        collect_into(self, target);
    }
}

/// An iterator that supports "random access" to its data, meaning
/// that you can split it at arbitrary indices and draw data from
/// those points.
pub trait IndexedParallelIterator: ExactParallelIterator {
    /// Internal method to convert this parallel iterator into a
    /// producer that can be used to request the items. Users of the
    /// API never need to know about this fn.
    #[doc(hidden)]
    fn with_producer<CB: ProducerCallback<Self::Item>>(self, callback: CB) -> CB::Output;

    /// Iterate over tuples `(A, B)`, where the items `A` are from
    /// this iterator and `B` are from the iterator given as argument.
    /// Like the `zip` method on ordinary iterators, if the two
    /// iterators are of unequal length, you only get the items they
    /// have in common.
    fn zip<ZIP_OP>(self, zip_op: ZIP_OP) -> ZipIter<Self, ZIP_OP::Iter>
        where ZIP_OP: IntoParallelIterator, ZIP_OP::Iter: IndexedParallelIterator
    {
        ZipIter::new(self, zip_op.into_par_iter())
    }

    /// Yields an index along with each item.
    fn enumerate(self) -> Enumerate<Self> {
        Enumerate::new(self)
    }
}

<commit_msg>Talk about fold.<commit_after>#![allow(dead_code)]

//! The `ParallelIterator` module makes it easy to write parallel
//! programs using an iterator-style interface. To get access to all
//! the methods you want, the easiest is to write `use
//! rayon::par_iter::*;` at the top of your module, which will import
//! the various traits and methods you need.
//!
//! The submodules of this module mostly just contain implementaton
//! details of little interest to an end-user. If you'd like to read
//! the code itself, the `internal` module and `README.md` file are a
//! good place to start.

use std::f64;
use std::ops::Fn;
use self::collect::collect_into;
use self::enumerate::Enumerate;
use self::filter::Filter;
use self::filter_map::FilterMap;
use self::flat_map::FlatMap;
use self::map::Map;
use self::reduce::{reduce, ReduceOp, SumOp, MulOp, MinOp, MaxOp, ReduceWithOp,
                   ReduceWithIdentityOp, SUM, MUL, MIN, MAX};
use self::internal::*;
use self::weight::Weight;
use self::zip::ZipIter;

pub mod collect;
pub mod enumerate;
pub mod filter;
pub mod filter_map;
pub mod flat_map;
pub mod internal;
pub mod len;
pub mod for_each;
#[cfg(feature = "unstable")]
pub mod fold;
pub mod reduce;
pub mod slice;
pub mod slice_mut;
pub mod map;
pub mod weight;
pub mod zip;
pub mod range;
pub mod vec;

#[cfg(test)]
mod test;

pub trait IntoParallelIterator {
    type Iter: ParallelIterator<Item=Self::Item>;
    type Item: Send;

    fn into_par_iter(self) -> Self::Iter;
}

pub trait IntoParallelRefIterator<'data> {
    type Iter: ParallelIterator<Item=&'data Self::Item>;
    type Item: Sync + 'data;

    fn par_iter(&'data self) -> Self::Iter;
}

pub trait IntoParallelRefMutIterator<'data> {
    type Iter: ParallelIterator<Item=&'data mut Self::Item>;
    type Item: Send + 'data;

    fn par_iter_mut(&'data mut self) -> Self::Iter;
}

/// The `ParallelIterator` interface.
pub trait ParallelIterator: Sized {
    type Item: Send;

    /// Indicates the relative "weight" of producing each item in this
    /// parallel iterator. A higher weight will cause finer-grained
    /// parallel subtasks. 1.0 indicates something very cheap and
    /// uniform, like copying a value out of an array, or computing `x
    /// + 1`. If your tasks are either very expensive, or very
    /// unpredictable, you are better off with higher values. See also
    /// `weight_max`, which is a convenient shorthand to force the
    /// finest grained parallel execution posible. Tuning this value
    /// should not affect correctness but can improve (or hurt)
    /// performance.
    fn weight(self, scale: f64) -> Weight<Self> {
        Weight::new(self, scale)
    }

    /// Shorthand for `self.weight(f64::INFINITY)`. This forces the
    /// smallest granularity of parallel execution, which makes sense
    /// when your parallel tasks are (potentially) very expensive to
    /// execute.
    fn weight_max(self) -> Weight<Self> {
        self.weight(f64::INFINITY)
    }

    /// Executes `OP` on each item produced by the iterator, in parallel.
    fn for_each<OP>(self, op: OP)
        where OP: Fn(Self::Item) + Sync
    {
        for_each::for_each(self, &op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn map<MAP_OP,R>(self, map_op: MAP_OP) -> Map<Self, MAP_OP>
        where MAP_OP: Fn(Self::Item) -> R
    {
        Map::new(self, map_op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn filter<FILTER_OP>(self, filter_op: FILTER_OP) -> Filter<Self, FILTER_OP>
        where FILTER_OP: Fn(&Self::Item) -> bool
    {
        Filter::new(self, filter_op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn filter_map<FILTER_OP,R>(self, filter_op: FILTER_OP) -> FilterMap<Self, FILTER_OP>
        where FILTER_OP: Fn(Self::Item) -> Option<R>
    {
        FilterMap::new(self, filter_op)
    }

    /// Applies `map_op` to each item of his iterator, producing a new
    /// iterator with the results.
    fn flat_map<MAP_OP,PI>(self, map_op: MAP_OP) -> FlatMap<Self, MAP_OP>
        where MAP_OP: Fn(Self::Item) -> PI, PI: ParallelIterator
    {
        FlatMap::new(self, map_op)
    }

    /// Reduces the items in the iterator into one item using `op`.
    /// See also `sum`, `mul`, `min`, etc, which are slightly more
    /// efficient. Returns `None` if the iterator is empty.
    ///
    /// Note: unlike in a sequential iterator, the order in which `op`
    /// will be applied to reduce the result is not specified. So `op`
    /// should be commutative and associative or else the results will
    /// be non-deterministic.
    fn reduce_with<OP>(self, op: OP) -> Option<Self::Item>
        where OP: Fn(Self::Item, Self::Item) -> Self::Item + Sync,
    {
        reduce(self.map(Some), &ReduceWithOp::new(&op))
    }

    /// Reduces the items in the iterator into one item using `op`.
    /// The argument `identity` represents an "identity" value which
    /// may be inserted into the sequence as needed to create
    /// opportunities for parallel execution. So, for example, if you
    /// are doing a summation, then `identity` ought to be something
    /// that represents the zero for your type (but consider just
    /// calling `sum()` in that case).
    ///
    /// Example `vectors.par_iter().reduce_with_identity(Vector::zero(), Vector::add)`.
    ///
    /// Note: unlike in a sequential iterator, the order in which `op`
    /// will be applied to reduce the result is not specified. So `op`
    /// should be commutative and associative or else the results will
    /// be non-deterministic. And of course `identity` should be a
    /// true identity.
    fn reduce_with_identity<OP>(self, identity: Self::Item, op: OP) -> Self::Item
        where OP: Fn(Self::Item, Self::Item) -> Self::Item + Sync,
              Self::Item: Clone + Sync,
    {
        reduce(self, &ReduceWithIdentityOp::new(&identity, &op))
    }

    /// A variant on the typical `map/reduce` pattern. Parallel fold
    /// is similar to sequential fold except that the sequence of
    /// items may be subdivided before it is folded. The resulting
    /// values are then reduced together using `reduce_op`.  Typically
    /// `fold_op` and `reduce_op` will be doing the same conceptual
    /// operation, but on different types, or with a different twist.
    ///
    /// Here is how to visualize what is happening. Imagine an input
    /// sequence with 7 values as shown:
    ///
    /// ```
    /// [ 0 1 2 3 4 5 6 ]
    ///   |     | |   |
    ///   +--X--+ +-Y-+ // <-- fold_op
    ///      |      |
    ///      +---Z--+   // <-- reduce_op
    /// ```
    ///
    /// These values will be first divided into contiguous chunks of
    /// some size (the precise sizes will depend on how many cores are
    /// present and how active they are). These are folded using
    /// `fold_op`. Here, the chunk `[0, 1, 2, 3]` was folded into `X`
    /// and the chunk `[4, 5, 6]` was folded into `Y`. Note that `X`
    /// and `Y` may, in general, have different types than the
    /// original input sequence. Now the results from these folds are
    /// themselves *reduced* using `reduce_op` (again, in some
    /// unspecified order). So now `X` and `Y` are reduced to `Z`,
    /// which is the final result. Note that `reduce_op` must consume
    /// and produce values of the same type.
    ///
    /// Note that `fold` can always be expressed using map/reduce. For
    /// example, a call `self.fold(identity, fold_op, reduce_op)` could
    /// also be expressed as follows:
    ///
    /// ```
    /// self.map(|elem| fold_op(identity.clone(), elem))
    ///     .reduce_with_identity(identity, reduce_op)
    /// ```
    ///
    /// This is equivalent to an execution of `fold` where the
    /// subsequences that were folded sequentially would up being of
    /// length 1.  However, this would rarely happen in practice,
    /// typically the subsequences would be larger, and hence a call
    /// to `fold` *can* be more efficient than map/reduce,
    /// particularly if the `fold_op` is more efficient when applied
    /// to a large sequence.
    ///
    /// **This method is marked as unstable** because it is
    /// particularly likely to change its name and/or signature, or go
    /// away entirely.
    #[cfg(feature = "unstable")]
    fn fold<I,FOLD_OP,REDUCE_OP>(self,
                                 identity: I,
                                 fold_op: FOLD_OP,
                                 reduce_op: REDUCE_OP)
                                 -> I
        where FOLD_OP: Fn(I, Self::Item) -> I + Sync,
              REDUCE_OP: Fn(I, I) -> I + Sync,
              I: Clone + Sync + Send,
    {
        fold::fold(self, &identity, &fold_op, &reduce_op)
    }

    /// Sums up the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `+` operator is not truly commutative and
    /// associative (as is the case for floating point numbers), then
    /// the results are not fully deterministic.
    fn sum(self) -> Self::Item
        where SumOp: ReduceOp<Self::Item>
    {
        reduce(self, SUM)
    }

    /// Multiplies all the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `*` operator is not truly commutative and
    /// associative (as is the case for floating point numbers), then
    /// the results are not fully deterministic.
    fn mul(self) -> Self::Item
        where MulOp: ReduceOp<Self::Item>
    {
        reduce(self, MUL)
    }

    /// Computes the minimum of all the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `Ord` impl is not truly commutative and associative
    /// (as is the case for floating point numbers), then the results
    /// are not deterministic.
    fn min(self) -> Self::Item
        where MinOp: ReduceOp<Self::Item>
    {
        reduce(self, MIN)
    }

    /// Computes the maximum of all the items in the iterator.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `Ord` impl is not truly commutative and associative
    /// (as is the case for floating point numbers), then the results
    /// are not deterministic.
    fn max(self) -> Self::Item
        where MaxOp: ReduceOp<Self::Item>
    {
        reduce(self, MAX)
    }

    /// Reduces the items using the given "reduce operator". You may
    /// prefer `reduce_with` for a simpler interface.
    ///
    /// Note that the order in items will be reduced is not specified,
    /// so if the `reduce_op` impl is not truly commutative and
    /// associative, then the results are not deterministic.
    fn reduce<REDUCE_OP>(self, reduce_op: &REDUCE_OP) -> Self::Item
        where REDUCE_OP: ReduceOp<Self::Item>
    {
        reduce(self, reduce_op)
    }

    /// Internal method used to define the behavior of this parallel
    /// iterator. You should not need to call this directly.
    #[doc(hidden)]
    fn drive_unindexed<C>(self, consumer: C) -> C::Result
        where C: UnindexedConsumer<Self::Item>;
}

impl<T: ParallelIterator> IntoParallelIterator for T {
    type Iter = T;
    type Item = T::Item;

    fn into_par_iter(self) -> T {
        self
    }
}

/// A trait for parallel iterators items where the precise number of
/// items is not known, but we can at least give an upper-bound. These
/// sorts of iterators result from filtering.
pub trait BoundedParallelIterator: ParallelIterator {
    fn upper_bound(&mut self) -> usize;

    /// Internal method used to define the behavior of this parallel
    /// iterator. You should not need to call this directly.
    #[doc(hidden)]
    fn drive<'c, C: Consumer<Self::Item>>(self,
                                                   consumer: C)
                                                   -> C::Result;

}

/// A trait for parallel iterators items where the precise number of
/// items is known. This occurs when e.g. iterating over a
/// vector. Knowing precisely how many items will be produced is very
/// useful.
pub trait ExactParallelIterator: BoundedParallelIterator {
    /// Produces an exact count of how many items this iterator will
    /// produce, presuming no panic occurs.
    ///
    /// # Safety note
    ///
    /// Returning an incorrect value here could lead to **undefined
    /// behavior**.
    fn len(&mut self) -> usize;

    /// Collects the results of the iterator into the specified
    /// vector. The vector is always truncated before execution
    /// begins. If possible, reusing the vector across calls can lead
    /// to better performance since it reuses the same backing buffer.
    fn collect_into(self, target: &mut Vec<Self::Item>) {
        collect_into(self, target);
    }
}

/// An iterator that supports "random access" to its data, meaning
/// that you can split it at arbitrary indices and draw data from
/// those points.
pub trait IndexedParallelIterator: ExactParallelIterator {
    /// Internal method to convert this parallel iterator into a
    /// producer that can be used to request the items. Users of the
    /// API never need to know about this fn.
    #[doc(hidden)]
    fn with_producer<CB: ProducerCallback<Self::Item>>(self, callback: CB) -> CB::Output;

    /// Iterate over tuples `(A, B)`, where the items `A` are from
    /// this iterator and `B` are from the iterator given as argument.
    /// Like the `zip` method on ordinary iterators, if the two
    /// iterators are of unequal length, you only get the items they
    /// have in common.
    fn zip<ZIP_OP>(self, zip_op: ZIP_OP) -> ZipIter<Self, ZIP_OP::Iter>
        where ZIP_OP: IntoParallelIterator, ZIP_OP::Iter: IndexedParallelIterator
    {
        ZipIter::new(self, zip_op.into_par_iter())
    }

    /// Yields an index along with each item.
    fn enumerate(self) -> Enumerate<Self> {
        Enumerate::new(self)
    }
}

<|endoftext|>


<commit_before>//! Ethereum DPT protocol implementation

extern crate bigint;
extern crate rlp;
extern crate hexutil;
extern crate sha3;
extern crate secp256k1;
#[macro_use]
extern crate log;
#[macro_use]
extern crate futures;
extern crate tokio_io;
extern crate tokio_core;
extern crate time;
extern crate rand;
extern crate url;

mod proto;
mod message;
mod util;

use message::*;
use proto::{DPTCodec, DPTCodecMessage};
use futures::future;
use futures::{Poll, Async, StartSend, AsyncSink, Future, Stream, Sink};
use tokio_io::{AsyncRead, AsyncWrite};
use tokio_io::codec::{Framed, Encoder, Decoder};
use tokio_core::reactor::{Timeout, Handle};
use tokio_core::net::{UdpSocket, UdpFramed};
use std::net::{IpAddr, SocketAddr, Ipv4Addr, Ipv6Addr};
use std::io;
use std::str::FromStr;
use bigint::{H256, H512};
use rlp::UntrustedRlp;
use secp256k1::SECP256K1;
use secp256k1::key::{PublicKey, SecretKey};
use util::{keccak256, pk2id};
use rand::{Rng, thread_rng};
use url::{Host, Url};

fn retain_mut<T, F>(vec: &mut Vec<T>, mut f: F)
    where F: FnMut(&mut T) -> bool
{
    let len = vec.len();
    let mut del = 0;
    {
        let v = &mut **vec;

        for i in 0..len {
            if !f(&mut v[i]) {
                del += 1;
            } else if del > 0 {
                v.swap(i - del, i);
            }
        }
    }
    if del > 0 {
        vec.truncate(len - del);
    }
}

/// DPT message for requesting new peers or ping with timeout
pub enum DPTMessage {
    RequestNewPeer,
    Ping(Timeout),
}

/// DPT stream for sending DPT messages or receiving new peers
pub struct DPTStream {
    stream: UdpFramed<DPTCodec>,
    id: H512,
    connected: Vec<DPTNode>,
    pingponged: Vec<DPTNode>,
    bootstrapped: bool,
    timeout: Option<(Timeout, Vec<H512>)>,
    incoming: Vec<DPTNode>,
    address: IpAddr,
    udp_port: u16,
    tcp_port: u16,
}

#[derive(Debug, Clone, PartialEq, Eq)]
/// DPT node used by a DPT stream
pub struct DPTNode {
    pub address: IpAddr,
    pub tcp_port: u16,
    pub udp_port: u16,
    pub id: H512,
}

#[derive(Debug, Clone)]
pub enum DPTNodeParseError {
    UrlError,
    HexError,
}

impl DPTNode {
    /// The TCP socket address of this node
    pub fn tcp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.tcp_port)
    }

    /// The UDP socket address of this node
    pub fn udp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.udp_port)
    }

    pub fn from_url(url: &Url) -> Result<DPTNode, DPTNodeParseError> {
        println!("{:?}",url.host());
        let address = match url.host() {
            Some(Host::Ipv4(ip)) => IpAddr::V4(ip),
            Some(Host::Ipv6(ip)) => IpAddr::V6(ip),
            Some(Host::Domain(ip)) => IpAddr::V4(Ipv4Addr::from_str(ip).unwrap()),
            _ => return Err(DPTNodeParseError::UrlError),
        };
        println!("{}",url);
        let port = match url.port() {
            Some(port) => port,
            _ => return Err(DPTNodeParseError::UrlError),
        };
        println!("{}",url);
        let id = match H512::from_str(url.username()) {
            Ok(id) => id,
            _ => return Err(DPTNodeParseError::HexError),
        };

        Ok(DPTNode {
            address, id,
            tcp_port: port,
            udp_port: port,
        })
    }
}

impl DPTStream {
    /// Create a new DPT stream
    pub fn new(addr: &SocketAddr, handle: &Handle,
               secret_key: SecretKey,
               bootstrap_nodes: Vec<DPTNode>,
               public_address: &IpAddr, tcp_port: u16) -> Result<Self, io::Error> {
        let id = pk2id(&match PublicKey::from_secret_key(&SECP256K1, &secret_key) {
            Ok(val) => val,
            Err(_) => return Err(io::Error::new(io::ErrorKind::Other, "converting pub key failed")),
        });
        debug!("self id: {:x}", id);
        Ok(Self {
            stream: UdpSocket::bind(addr, handle)?.framed(DPTCodec::new(secret_key)),
            id, connected: bootstrap_nodes.clone(), incoming: bootstrap_nodes,
            pingponged: Vec::new(),
            bootstrapped: false,
            timeout: None,
            address: public_address.clone(), udp_port: addr.port(), tcp_port
        })
    }

    /// Get all connected peers
    pub fn connected_peers(&self) -> &[DPTNode] {
        &self.pingponged
    }

    /// Disconnect from a node
    pub fn disconnect_peer(&mut self, remote_id: H512) {
        self.connected.retain(|node| {
            node.id != remote_id
        });
        self.pingponged.retain(|node| {
            node.id != remote_id
        });
    }

    /// Get the peer by its id
    pub fn get_peer(&self, remote_id: H512) -> Option<DPTNode> {
        for i in 0..self.connected.len() {
            if self.connected[i].id == remote_id {
                return Some(self.connected[i].clone());
            }
        }
        return None;
    }

    fn default_expire(&self) -> u64 {
        time::now_utc().to_timespec().sec as u64 + 60
    }

    fn send_ping(&mut self, addr: SocketAddr, to: DPTNode) -> Poll<(), io::Error> {
        let typ = 0x01u8;
        let message = PingMessage {
            from: Endpoint {
                address: self.address,
                udp_port: self.udp_port,
                tcp_port: self.tcp_port,
            },
            to: Endpoint {
                address: to.address,
                udp_port: to.udp_port,
                tcp_port: to.tcp_port,
            },
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }

    fn send_pong(&mut self, addr: SocketAddr, echo: H256, to: Endpoint) -> Poll<(), io::Error> {
        let typ = 0x02u8;
        let message = PongMessage {
            echo, to,
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        debug!("sending pong ...");
        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }

    fn send_find_neighbours(&mut self, addr: SocketAddr) -> Poll<(), io::Error> {
        let typ = 0x03u8;
        let message = FindNeighboursMessage {
            id: self.id,
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }

    fn send_neighbours(&mut self, addr: SocketAddr) -> Poll<(), io::Error> {
        let typ = 0x04u8;
        // Return at most 3 nodes at a time.
        let mut nodes = Vec::new();
        for i in 0..self.connected.len() {
            if nodes.len() >= 3 {
                break;
            }

            let address = self.connected[i].address;
            let udp_port = self.connected[i].udp_port;
            let tcp_port = self.connected[i].tcp_port;
            let id = self.connected[i].id;

            nodes.push(Neighbour {
                address, udp_port, tcp_port, id,
            });
        }
        let message = NeighboursMessage {
            nodes,
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }
}

impl Stream for DPTStream {
    type Item = DPTNode;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
        if !self.bootstrapped {
            for node in self.connected.clone() {
                self.send_ping(node.udp_addr(), node)?;
            }
            self.bootstrapped = true;
        }

        let mut timeoutted = false;
        if self.timeout.is_some() {
            let &mut (ref mut timeout, ref hs) = self.timeout.as_mut().unwrap();
            timeoutted = match timeout.poll() {
                Ok(Async::Ready(())) => true,
                Ok(Async::NotReady) => false,
                Err(e) => return Err(e),
            };

            if timeoutted {
                debug!("{} endpoints timeoutted", hs.len());
                for h in hs {
                    self.connected.retain(|v| v.id != *h);
                }
            }
        }
        if timeoutted {
            self.timeout = None;
        }

        loop {
            let (message, remote_id, hash) = match self.stream.poll()? {
                Async::Ready(Some(Some(val))) => val,
                Async::Ready(Some(None)) => continue,
                Async::NotReady => {
                    if self.incoming.len() > 0 {
                        return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                    } else {
                        return Ok(Async::NotReady);
                    }
                },
                Async::Ready(None) => return Ok(Async::Ready(None)),
            };

            match message.typ {
                0x01 /* ping */ => {
                    debug!("got ping message");
                    let ping_message: PingMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                        Err(_) => continue,
                    };

                    self.send_pong(message.addr, hash, ping_message.to)?;

                    let v = self.connected.iter().find(|v| v.id == remote_id).map(|v| v.clone());
                    if v.is_some() {
                        let v = v.unwrap();
                        if !self.pingponged.contains(&v) {
                            self.pingponged.push(v);
                        }
                    }
                },
                0x02 /* pong */ => {
                    debug!("got pong message");
                    let pong_message: PongMessage = match UntrustedRlp::new(&message.data).as_val() {
                        Ok(val) => val,
                        Err(_) => continue,
                    };

                    if self.timeout.is_some() {
                        self.timeout.as_mut().unwrap().1.retain(|v| {
                            *v != remote_id
                        });
                    }

                    let v = self.connected.iter().find(|v| v.id == remote_id).map(|v| v.clone());
                    if v.is_some() {
                        let v = v.unwrap();
                        if !self.pingponged.contains(&v) {
                            debug!("pushing pingponged: {:?}", v);
                            self.pingponged.push(v);
                        }
                    }
                },
                0x03 /* find neighbours */ => {
                    debug!("got find neighbours message");
                    self.send_neighbours(message.addr)?;
                },
                0x04 /* neighbours */ => {
                    debug!("got neighbours message");
                    let incoming_message: NeighboursMessage =
                        match UntrustedRlp::new(&message.data).as_val() {
                            Ok(val) => val,
                            Err(_) => continue,
                        };
                    debug!("neighbouts message len {}", incoming_message.nodes.len());
                    for node in incoming_message.nodes {
                        let node = DPTNode {
                            address: node.address,
                            udp_port: node.udp_port,
                            tcp_port: node.tcp_port,
                            id: node.id,
                    };
                        if !self.connected.contains(&node) {
                            self.send_ping(node.udp_addr(), node.clone())?;

                            debug!("pushing new node {:?}", node);
                            self.connected.push(node.clone());
                            self.incoming.push(node.clone());
                            debug!("connected {}", self.connected.len());
                        }
                    }
                },
                _ => { }
            }

            if self.incoming.len() > 0 {
                return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
            }
        }
    }
}

impl Sink for DPTStream {
    type SinkItem = DPTMessage;
    type SinkError = io::Error;

    fn poll_complete(&mut self) -> Poll<(), Self::SinkError> {
        self.stream.poll_complete()
    }

    fn start_send(&mut self, message: DPTMessage) -> StartSend<Self::SinkItem, Self::SinkError> {
        match message {

            DPTMessage::RequestNewPeer => {
                debug!("randomly selecting one peer from {}", self.pingponged.len());
                thread_rng().shuffle(&mut self.pingponged);

                if self.pingponged.len() == 0 {
                    debug!("no peers available to find node");
                    for node in self.connected.clone() {
                        self.send_ping(node.udp_addr(), node)?;
                    }
                    return Ok(AsyncSink::Ready);
                }

                let addr = self.pingponged[0].udp_addr();
                self.send_find_neighbours(addr)?;

                return Ok(AsyncSink::Ready);
            },

            DPTMessage::Ping(timeout) => {
                let mut timeoutting = Vec::new();
                for node in self.connected.clone() {
                    self.send_ping(node.udp_addr(), node.clone())?;
                    timeoutting.push(node.id);
                }

                self.timeout = Some((timeout, timeoutting));

                return Ok(AsyncSink::Ready);
            }

        }
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {
    }
}
<commit_msg>nit: remove println<commit_after>//! Ethereum DPT protocol implementation

extern crate bigint;
extern crate rlp;
extern crate hexutil;
extern crate sha3;
extern crate secp256k1;
#[macro_use]
extern crate log;
#[macro_use]
extern crate futures;
extern crate tokio_io;
extern crate tokio_core;
extern crate time;
extern crate rand;
extern crate url;

mod proto;
mod message;
mod util;

use message::*;
use proto::{DPTCodec, DPTCodecMessage};
use futures::future;
use futures::{Poll, Async, StartSend, AsyncSink, Future, Stream, Sink};
use tokio_io::{AsyncRead, AsyncWrite};
use tokio_io::codec::{Framed, Encoder, Decoder};
use tokio_core::reactor::{Timeout, Handle};
use tokio_core::net::{UdpSocket, UdpFramed};
use std::net::{IpAddr, SocketAddr, Ipv4Addr, Ipv6Addr};
use std::io;
use std::str::FromStr;
use bigint::{H256, H512};
use rlp::UntrustedRlp;
use secp256k1::SECP256K1;
use secp256k1::key::{PublicKey, SecretKey};
use util::{keccak256, pk2id};
use rand::{Rng, thread_rng};
use url::{Host, Url};

fn retain_mut<T, F>(vec: &mut Vec<T>, mut f: F)
    where F: FnMut(&mut T) -> bool
{
    let len = vec.len();
    let mut del = 0;
    {
        let v = &mut **vec;

        for i in 0..len {
            if !f(&mut v[i]) {
                del += 1;
            } else if del > 0 {
                v.swap(i - del, i);
            }
        }
    }
    if del > 0 {
        vec.truncate(len - del);
    }
}

/// DPT message for requesting new peers or ping with timeout
pub enum DPTMessage {
    RequestNewPeer,
    Ping(Timeout),
}

/// DPT stream for sending DPT messages or receiving new peers
pub struct DPTStream {
    stream: UdpFramed<DPTCodec>,
    id: H512,
    connected: Vec<DPTNode>,
    pingponged: Vec<DPTNode>,
    bootstrapped: bool,
    timeout: Option<(Timeout, Vec<H512>)>,
    incoming: Vec<DPTNode>,
    address: IpAddr,
    udp_port: u16,
    tcp_port: u16,
}

#[derive(Debug, Clone, PartialEq, Eq)]
/// DPT node used by a DPT stream
pub struct DPTNode {
    pub address: IpAddr,
    pub tcp_port: u16,
    pub udp_port: u16,
    pub id: H512,
}

#[derive(Debug, Clone)]
pub enum DPTNodeParseError {
    UrlError,
    HexError,
}

impl DPTNode {
    /// The TCP socket address of this node
    pub fn tcp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.tcp_port)
    }

    /// The UDP socket address of this node
    pub fn udp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.udp_port)
    }

    pub fn from_url(url: &Url) -> Result<DPTNode, DPTNodeParseError> {
        let address = match url.host() {
            Some(Host::Ipv4(ip)) => IpAddr::V4(ip),
            Some(Host::Ipv6(ip)) => IpAddr::V6(ip),
            Some(Host::Domain(ip)) => IpAddr::V4(Ipv4Addr::from_str(ip).unwrap()),
            _ => return Err(DPTNodeParseError::UrlError),
        };
        let port = match url.port() {
            Some(port) => port,
            _ => return Err(DPTNodeParseError::UrlError),
        };
        let id = match H512::from_str(url.username()) {
            Ok(id) => id,
            _ => return Err(DPTNodeParseError::HexError),
        };

        Ok(DPTNode {
            address, id,
            tcp_port: port,
            udp_port: port,
        })
    }
}

impl DPTStream {
    /// Create a new DPT stream
    pub fn new(addr: &SocketAddr, handle: &Handle,
               secret_key: SecretKey,
               bootstrap_nodes: Vec<DPTNode>,
               public_address: &IpAddr, tcp_port: u16) -> Result<Self, io::Error> {
        let id = pk2id(&match PublicKey::from_secret_key(&SECP256K1, &secret_key) {
            Ok(val) => val,
            Err(_) => return Err(io::Error::new(io::ErrorKind::Other, "converting pub key failed")),
        });
        debug!("self id: {:x}", id);
        Ok(Self {
            stream: UdpSocket::bind(addr, handle)?.framed(DPTCodec::new(secret_key)),
            id, connected: bootstrap_nodes.clone(), incoming: bootstrap_nodes,
            pingponged: Vec::new(),
            bootstrapped: false,
            timeout: None,
            address: public_address.clone(), udp_port: addr.port(), tcp_port
        })
    }

    /// Get all connected peers
    pub fn connected_peers(&self) -> &[DPTNode] {
        &self.pingponged
    }

    /// Disconnect from a node
    pub fn disconnect_peer(&mut self, remote_id: H512) {
        self.connected.retain(|node| {
            node.id != remote_id
        });
        self.pingponged.retain(|node| {
            node.id != remote_id
        });
    }

    /// Get the peer by its id
    pub fn get_peer(&self, remote_id: H512) -> Option<DPTNode> {
        for i in 0..self.connected.len() {
            if self.connected[i].id == remote_id {
                return Some(self.connected[i].clone());
            }
        }
        return None;
    }

    fn default_expire(&self) -> u64 {
        time::now_utc().to_timespec().sec as u64 + 60
    }

    fn send_ping(&mut self, addr: SocketAddr, to: DPTNode) -> Poll<(), io::Error> {
        let typ = 0x01u8;
        let message = PingMessage {
            from: Endpoint {
                address: self.address,
                udp_port: self.udp_port,
                tcp_port: self.tcp_port,
            },
            to: Endpoint {
                address: to.address,
                udp_port: to.udp_port,
                tcp_port: to.tcp_port,
            },
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }

    fn send_pong(&mut self, addr: SocketAddr, echo: H256, to: Endpoint) -> Poll<(), io::Error> {
        let typ = 0x02u8;
        let message = PongMessage {
            echo, to,
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        debug!("sending pong ...");
        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }

    fn send_find_neighbours(&mut self, addr: SocketAddr) -> Poll<(), io::Error> {
        let typ = 0x03u8;
        let message = FindNeighboursMessage {
            id: self.id,
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }

    fn send_neighbours(&mut self, addr: SocketAddr) -> Poll<(), io::Error> {
        let typ = 0x04u8;
        // Return at most 3 nodes at a time.
        let mut nodes = Vec::new();
        for i in 0..self.connected.len() {
            if nodes.len() >= 3 {
                break;
            }

            let address = self.connected[i].address;
            let udp_port = self.connected[i].udp_port;
            let tcp_port = self.connected[i].tcp_port;
            let id = self.connected[i].id;

            nodes.push(Neighbour {
                address, udp_port, tcp_port, id,
            });
        }
        let message = NeighboursMessage {
            nodes,
            expire: self.default_expire(),
        };
        let data = rlp::encode(&message).to_vec();

        self.stream.start_send(DPTCodecMessage {
            typ, data, addr
        })?;
        self.stream.poll_complete()?;

        Ok(Async::Ready(()))
    }
}

impl Stream for DPTStream {
    type Item = DPTNode;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
        if !self.bootstrapped {
            for node in self.connected.clone() {
                self.send_ping(node.udp_addr(), node)?;
            }
            self.bootstrapped = true;
        }

        let mut timeoutted = false;
        if self.timeout.is_some() {
            let &mut (ref mut timeout, ref hs) = self.timeout.as_mut().unwrap();
            timeoutted = match timeout.poll() {
                Ok(Async::Ready(())) => true,
                Ok(Async::NotReady) => false,
                Err(e) => return Err(e),
            };

            if timeoutted {
                debug!("{} endpoints timeoutted", hs.len());
                for h in hs {
                    self.connected.retain(|v| v.id != *h);
                }
            }
        }
        if timeoutted {
            self.timeout = None;
        }

        loop {
            let (message, remote_id, hash) = match self.stream.poll()? {
                Async::Ready(Some(Some(val))) => val,
                Async::Ready(Some(None)) => continue,
                Async::NotReady => {
                    if self.incoming.len() > 0 {
                        return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                    } else {
                        return Ok(Async::NotReady);
                    }
                },
                Async::Ready(None) => return Ok(Async::Ready(None)),
            };

            match message.typ {
                0x01 /* ping */ => {
                    debug!("got ping message");
                    let ping_message: PingMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                        Err(_) => continue,
                    };

                    self.send_pong(message.addr, hash, ping_message.to)?;

                    let v = self.connected.iter().find(|v| v.id == remote_id).map(|v| v.clone());
                    if v.is_some() {
                        let v = v.unwrap();
                        if !self.pingponged.contains(&v) {
                            self.pingponged.push(v);
                        }
                    }
                },
                0x02 /* pong */ => {
                    debug!("got pong message");
                    let pong_message: PongMessage = match UntrustedRlp::new(&message.data).as_val() {
                        Ok(val) => val,
                        Err(_) => continue,
                    };

                    if self.timeout.is_some() {
                        self.timeout.as_mut().unwrap().1.retain(|v| {
                            *v != remote_id
                        });
                    }

                    let v = self.connected.iter().find(|v| v.id == remote_id).map(|v| v.clone());
                    if v.is_some() {
                        let v = v.unwrap();
                        if !self.pingponged.contains(&v) {
                            debug!("pushing pingponged: {:?}", v);
                            self.pingponged.push(v);
                        }
                    }
                },
                0x03 /* find neighbours */ => {
                    debug!("got find neighbours message");
                    self.send_neighbours(message.addr)?;
                },
                0x04 /* neighbours */ => {
                    debug!("got neighbours message");
                    let incoming_message: NeighboursMessage =
                        match UntrustedRlp::new(&message.data).as_val() {
                            Ok(val) => val,
                            Err(_) => continue,
                        };
                    debug!("neighbouts message len {}", incoming_message.nodes.len());
                    for node in incoming_message.nodes {
                        let node = DPTNode {
                            address: node.address,
                            udp_port: node.udp_port,
                            tcp_port: node.tcp_port,
                            id: node.id,
                    };
                        if !self.connected.contains(&node) {
                            self.send_ping(node.udp_addr(), node.clone())?;

                            debug!("pushing new node {:?}", node);
                            self.connected.push(node.clone());
                            self.incoming.push(node.clone());
                            debug!("connected {}", self.connected.len());
                        }
                    }
                },
                _ => { }
            }

            if self.incoming.len() > 0 {
                return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
            }
        }
    }
}

impl Sink for DPTStream {
    type SinkItem = DPTMessage;
    type SinkError = io::Error;

    fn poll_complete(&mut self) -> Poll<(), Self::SinkError> {
        self.stream.poll_complete()
    }

    fn start_send(&mut self, message: DPTMessage) -> StartSend<Self::SinkItem, Self::SinkError> {
        match message {

            DPTMessage::RequestNewPeer => {
                debug!("randomly selecting one peer from {}", self.pingponged.len());
                thread_rng().shuffle(&mut self.pingponged);

                if self.pingponged.len() == 0 {
                    debug!("no peers available to find node");
                    for node in self.connected.clone() {
                        self.send_ping(node.udp_addr(), node)?;
                    }
                    return Ok(AsyncSink::Ready);
                }

                let addr = self.pingponged[0].udp_addr();
                self.send_find_neighbours(addr)?;

                return Ok(AsyncSink::Ready);
            },

            DPTMessage::Ping(timeout) => {
                let mut timeoutting = Vec::new();
                for node in self.connected.clone() {
                    self.send_ping(node.udp_addr(), node.clone())?;
                    timeoutting.push(node.id);
                }

                self.timeout = Some((timeout, timeoutting));

                return Ok(AsyncSink::Ready);
            }

        }
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {
    }
}
<|endoftext|>


<commit_before>use super::wasm;
use anyhow::Context;
use std::{
    borrow::Cow,
    env, fs,
    net::TcpListener,
    path::{Path, PathBuf},
    str::{self, FromStr as _},
};
use tiny_http::{Header, Response, Server};
use webbrowser;

macro_rules! optional_resource {
    ($name: tt, $path: tt) => {
        #[cfg(TREE_SITTER_EMBED_WASM_BINDING)]
        fn $name(tree_sitter_dir: &Option<PathBuf>) -> Cow<'static, [u8]> {
            if let Some(tree_sitter_dir) = tree_sitter_dir {
                Cow::Owned(fs::read(tree_sitter_dir.join($path)).unwrap())
            } else {
                Cow::Borrowed(include_bytes!(concat!("../../", $path)))
            }
        }

        #[cfg(not(TREE_SITTER_EMBED_WASM_BINDING))]
        fn $name(tree_sitter_dir: &Option<PathBuf>) -> Cow<'static, [u8]> {
            if let Some(tree_sitter_dir) = tree_sitter_dir {
                Cow::Owned(fs::read(tree_sitter_dir.join($path)).unwrap())
            } else {
                Cow::Borrowed(&[])
            }
        }
    };
}

optional_resource!(get_playground_js, "docs/assets/js/playground.js");
optional_resource!(get_lib_js, "lib/binding_web/tree-sitter.js");
optional_resource!(get_lib_wasm, "lib/binding_web/tree-sitter.wasm");

fn get_main_html(tree_sitter_dir: &Option<PathBuf>) -> Cow<'static, [u8]> {
    if let Some(tree_sitter_dir) = tree_sitter_dir {
        Cow::Owned(fs::read(tree_sitter_dir.join("cli/src/playground.html")).unwrap())
    } else {
        Cow::Borrowed(include_bytes!("playground.html"))
    }
}

pub fn serve(grammar_path: &Path, open_in_browser: bool) {
    let port = get_available_port().expect("Couldn't find an available port");
    let addr = format!("127.0.0.1:{}", port);
    let server = Server::http(&addr).expect("Failed to start web server");
    let grammar_name = wasm::get_grammar_name(&grammar_path.join("src"))
        .with_context(|| "Failed to get wasm filename")
        .unwrap();
    let wasm_filename = format!("tree-sitter-{}.wasm", grammar_name);
    let language_wasm = fs::read(grammar_path.join(&wasm_filename))
        .with_context(|| {
            format!(
                "Failed to read {}. Run `tree-sitter build-wasm` first.",
                wasm_filename
            )
        })
        .unwrap();
    let url = format!("http://{}", addr);
    println!("Started playground on: {}", url);
    if open_in_browser {
        if let Err(_) = webbrowser::open(&url) {
            eprintln!("Failed to open '{}' in a web browser", url);
        }
    }

    let tree_sitter_dir = env::var("TREE_SITTER_BASE_DIR").map(PathBuf::from).ok();
    let main_html = str::from_utf8(&get_main_html(&tree_sitter_dir))
        .unwrap()
        .replace("THE_LANGUAGE_NAME", &grammar_name)
        .into_bytes();
    let playground_js = get_playground_js(&tree_sitter_dir);
    let lib_js = get_lib_js(&tree_sitter_dir);
    let lib_wasm = get_lib_wasm(&tree_sitter_dir);

    let html_header = Header::from_str("Content-Type: text/html").unwrap();
    let js_header = Header::from_str("Content-Type: application/javascript").unwrap();
    let wasm_header = Header::from_str("Content-Type: application/wasm").unwrap();

    for request in server.incoming_requests() {
        let res = match request.url() {
            "/" => response(&main_html, &html_header),
            "/tree-sitter-parser.wasm" => response(&language_wasm, &wasm_header),
            "/playground.js" => {
                if playground_js.is_empty() {
                    redirect("https://tree-sitter.github.io/tree-sitter/assets/js/playground.js")
                } else {
                    response(&playground_js, &js_header)
                }
            }
            "/tree-sitter.js" => {
                if lib_js.is_empty() {
                    redirect("https://tree-sitter.github.io/tree-sitter.js")
                } else {
                    response(&lib_js, &js_header)
                }
            }
            "/tree-sitter.wasm" => {
                if lib_wasm.is_empty() {
                    redirect("https://tree-sitter.github.io/tree-sitter.wasm")
                } else {
                    response(&lib_wasm, &wasm_header)
                }
            }
            _ => response(b"Not found", &html_header).with_status_code(404),
        };
        request.respond(res).expect("Failed to write HTTP response");
    }
}

fn redirect<'a>(url: &'a str) -> Response<&'a [u8]> {
    Response::empty(302)
        .with_data("".as_bytes(), Some(0))
        .with_header(Header::from_bytes("Location", url.as_bytes()).unwrap())
}

fn response<'a>(data: &'a [u8], header: &Header) -> Response<&'a [u8]> {
    Response::empty(200)
        .with_data(data, Some(data.len()))
        .with_header(header.clone())
}

fn get_available_port() -> Option<u16> {
    (8000..12000).find(port_is_available)
}

fn port_is_available(port: &u16) -> bool {
    TcpListener::bind(("127.0.0.1", *port)).is_ok()
}
<commit_msg>Add env vars for port and address<commit_after>use super::wasm;
use anyhow::Context;
use std::{
    borrow::Cow,
    env, fs,
    net::TcpListener,
    path::{Path, PathBuf},
    str::{self, FromStr as _},
};
use tiny_http::{Header, Response, Server};
use webbrowser;

macro_rules! optional_resource {
    ($name: tt, $path: tt) => {
        #[cfg(TREE_SITTER_EMBED_WASM_BINDING)]
        fn $name(tree_sitter_dir: &Option<PathBuf>) -> Cow<'static, [u8]> {
            if let Some(tree_sitter_dir) = tree_sitter_dir {
                Cow::Owned(fs::read(tree_sitter_dir.join($path)).unwrap())
            } else {
                Cow::Borrowed(include_bytes!(concat!("../../", $path)))
            }
        }

        #[cfg(not(TREE_SITTER_EMBED_WASM_BINDING))]
        fn $name(tree_sitter_dir: &Option<PathBuf>) -> Cow<'static, [u8]> {
            if let Some(tree_sitter_dir) = tree_sitter_dir {
                Cow::Owned(fs::read(tree_sitter_dir.join($path)).unwrap())
            } else {
                Cow::Borrowed(&[])
            }
        }
    };
}

optional_resource!(get_playground_js, "docs/assets/js/playground.js");
optional_resource!(get_lib_js, "lib/binding_web/tree-sitter.js");
optional_resource!(get_lib_wasm, "lib/binding_web/tree-sitter.wasm");

fn get_main_html(tree_sitter_dir: &Option<PathBuf>) -> Cow<'static, [u8]> {
    if let Some(tree_sitter_dir) = tree_sitter_dir {
        Cow::Owned(fs::read(tree_sitter_dir.join("cli/src/playground.html")).unwrap())
    } else {
        Cow::Borrowed(include_bytes!("playground.html"))
    }
}

pub fn serve(grammar_path: &Path, open_in_browser: bool) {
    let port = env::var("TREE_SITTER_PLAYGROUND_PORT")
        .map(|v| v.parse::<u16>().expect("Invalid port specification"))
        .unwrap_or_else(
            |_| get_available_port().expect(
                "Couldn't find an available port, try providing a port number via the TREE_SITTER_PLAYGROUND_PORT \
                 environment variable"
            )
        );
    let addr = format!(
        "{}:{}",
        env::var("TREE_SITTER_PLAYGROUND_ADDR").unwrap_or("127.0.0.1".to_owned()),
        port
    );
    let server = Server::http(&addr).expect("Failed to start web server");
    let grammar_name = wasm::get_grammar_name(&grammar_path.join("src"))
        .with_context(|| "Failed to get wasm filename")
        .unwrap();
    let wasm_filename = format!("tree-sitter-{}.wasm", grammar_name);
    let language_wasm = fs::read(grammar_path.join(&wasm_filename))
        .with_context(|| {
            format!(
                "Failed to read {}. Run `tree-sitter build-wasm` first.",
                wasm_filename
            )
        })
        .unwrap();
    let url = format!("http://{}", addr);
    println!("Started playground on: {}", url);
    if open_in_browser {
        if let Err(_) = webbrowser::open(&url) {
            eprintln!("Failed to open '{}' in a web browser", url);
        }
    }

    let tree_sitter_dir = env::var("TREE_SITTER_BASE_DIR").map(PathBuf::from).ok();
    let main_html = str::from_utf8(&get_main_html(&tree_sitter_dir))
        .unwrap()
        .replace("THE_LANGUAGE_NAME", &grammar_name)
        .into_bytes();
    let playground_js = get_playground_js(&tree_sitter_dir);
    let lib_js = get_lib_js(&tree_sitter_dir);
    let lib_wasm = get_lib_wasm(&tree_sitter_dir);

    let html_header = Header::from_str("Content-Type: text/html").unwrap();
    let js_header = Header::from_str("Content-Type: application/javascript").unwrap();
    let wasm_header = Header::from_str("Content-Type: application/wasm").unwrap();

    for request in server.incoming_requests() {
        let res = match request.url() {
            "/" => response(&main_html, &html_header),
            "/tree-sitter-parser.wasm" => response(&language_wasm, &wasm_header),
            "/playground.js" => {
                if playground_js.is_empty() {
                    redirect("https://tree-sitter.github.io/tree-sitter/assets/js/playground.js")
                } else {
                    response(&playground_js, &js_header)
                }
            }
            "/tree-sitter.js" => {
                if lib_js.is_empty() {
                    redirect("https://tree-sitter.github.io/tree-sitter.js")
                } else {
                    response(&lib_js, &js_header)
                }
            }
            "/tree-sitter.wasm" => {
                if lib_wasm.is_empty() {
                    redirect("https://tree-sitter.github.io/tree-sitter.wasm")
                } else {
                    response(&lib_wasm, &wasm_header)
                }
            }
            _ => response(b"Not found", &html_header).with_status_code(404),
        };
        request.respond(res).expect("Failed to write HTTP response");
    }
}

fn redirect<'a>(url: &'a str) -> Response<&'a [u8]> {
    Response::empty(302)
        .with_data("".as_bytes(), Some(0))
        .with_header(Header::from_bytes("Location", url.as_bytes()).unwrap())
}

fn response<'a>(data: &'a [u8], header: &Header) -> Response<&'a [u8]> {
    Response::empty(200)
        .with_data(data, Some(data.len()))
        .with_header(header.clone())
}

fn get_available_port() -> Option<u16> {
    (8000..12000).find(port_is_available)
}

fn port_is_available(port: &u16) -> bool {
    TcpListener::bind(("127.0.0.1", *port)).is_ok()
}
<|endoftext|>


<commit_before>use std::old_io::{File, BufferedReader};
use std::str::from_utf8;
use std::iter::Iterator;

use racer;
use racer::util;
use racer::codecleaner;
use racer::codeiter;
use racer::typeinf;
use racer::ast;

fn find_close<'a, A>(mut iter: A, open: u8, close: u8, level_end: u32) -> Option<usize>
    where A: Iterator<Item=&'a u8> {

    let mut count = 0us;
    let mut levels = 0u32;
    for &b in iter {
        if b == close {
            if levels == level_end { return Some(count); }
            levels -= 1;
        } else if b == open { levels += 1; }
        count += 1;
    }
    None
}

pub fn find_closing_paren(src:&str, pos:usize) -> usize {
    match find_close(src.as_bytes()[pos..].iter(), b'(', b')', 0) {
        Some(count) => pos + count,
        None => src.len()
    }
}

pub fn scope_start(src:&str, point:usize) -> usize {
    let masked_src = mask_comments(&src[..point]);
    match find_close(masked_src.as_bytes().iter().rev(), b'}', b'{', 0) {
        Some(count) => point - count,
        None => 0
    }
}

pub fn find_stmt_start(msrc: &str, point: usize) -> Option<usize> {
    // iterate the scope to find the start of the statement
    let scopestart = scope_start(msrc, point);
    for (start, end) in codeiter::iter_stmts(&msrc[scopestart..]) {
        if (scopestart + end) > point {
            return Some(scopestart+start);
        }
    }
    None
}

pub fn get_local_module_path(msrc: &str, point: usize) -> Vec<String> {
    let mut v = Vec::new();
    get_local_module_path_(msrc, point, &mut v);
    v
}

fn get_local_module_path_(msrc: &str, point: usize, out: &mut Vec<String>) {
    for (start, end) in codeiter::iter_stmts(msrc) {
        if start < point && end > point {
            let blob = &msrc[start..end];
            if blob.starts_with("pub mod "){
                let p = typeinf::generate_skeleton_for_parsing(blob);
                ast::parse_mod(p).name.map(|name|{

                    let newstart = blob.find_str("{").unwrap() + 1;
                    out.push(name);
                    get_local_module_path_(&blob[newstart..],
                                       point - start - newstart, out);
                });
            }
        }
    }
}

pub fn find_impl_start(msrc: &str, point: usize, scopestart: usize) -> Option<usize> {

    let len = point-scopestart;
    match codeiter::iter_stmts(&msrc[scopestart..]).find(|&(_, end)| end > len) {
        Some((start, _)) => {
            let blob = &msrc[(scopestart + start)..];
            // TODO:: the following is a bit weak at matching traits. make this better
            if blob.starts_with("impl") ||
                blob.starts_with("trait") || blob.starts_with("pub trait") {
                Some(scopestart + start)
            } else {
                let newstart = blob.find_str("{").unwrap() + 1;
                find_impl_start(msrc, point, scopestart+start+newstart)
            }
        },
        None => None
    }
}

#[test]
fn finds_subnested_module() {
    let src = "
    pub mod foo {
        pub mod bar {
            here
        }
    }";
    let point = coords_to_point(src, 4, 12);
    let v = get_local_module_path(src, point);
    assert_eq!("foo", &v[0][]);
    assert_eq!("bar", &v[1][]);

    let point = coords_to_point(src, 2, 8);
    let v = get_local_module_path(src, point);
    assert_eq!("foo", &v[0][]);
}

pub fn split_into_context_and_completion<'a>(s: &'a str) -> (&'a str, &'a str, racer::CompletionType) {

    match s.char_indices().rev().find(|&(_, c)| !util::is_ident_char(c)) {
        Some((i,c)) => {
            match c {
                '.' => (&s[..i], &s[(i+1)..], racer::CompletionType::CompleteField),
                ':' => (&s[..(i-1)], &s[(i+1)..], racer::CompletionType::CompletePath),
                _   => (&s[..(i+1)], &s[(i+1)..], racer::CompletionType::CompletePath)
            }
        },
        None => ("", s, racer::CompletionType::CompletePath)
    }

}

pub fn get_start_of_search_expr(src: &str, point: usize) -> usize {

    let mut i = point-1;
    let mut levels = 0u32;
    for &b in src.as_bytes()[..point].iter().rev() {
        match b {
            b'(' => {
                if levels == 0 { return i+1; }
                levels -= 1;
            },
            b')' => { levels += 1; },
            _ => {
                if levels == 0 &&
                    !util::is_search_expr_char(src.char_at(i)) ||
                    util::is_double_dot(src,i) {
                    return i+1;
                }
            }
        }
        i -= 1;
    }
    0
}

pub fn get_start_of_pattern(src: &str, point: usize) -> usize {
    let mut i = point-1;
    let mut levels = 0u32;
    for &b in src.as_bytes()[..point].iter().rev() {
        match b {
            b'(' => {
                if levels == 0 { return i+1; }
                levels -= 1;
            },
            b')' => { levels += 1; },
            _ => {
                if levels == 0 &&
                    !util::is_pattern_char(src.char_at(i)) {
                    return i+1;
                }
            }
        }
        i -= 1;
    }
    0
}

#[test]
fn get_start_of_pattern_handles_variant() {
    assert_eq!(4, get_start_of_pattern("foo, Some(a) =>",13));
}

#[test]
fn get_start_of_pattern_handles_variant2() {
    assert_eq!(4, get_start_of_pattern("bla, ast::PatTup(ref tuple_elements) => {",36));
}

pub fn expand_search_expr(msrc: &str, point: usize) -> (usize,usize) {
    let start = get_start_of_search_expr(msrc, point);
    return (start, util::find_ident_end(msrc, point));
}

#[test]
fn expand_search_expr_finds_ident() {
    assert_eq!((0, 7), expand_search_expr("foo.bar", 5))
}

#[test]
fn expand_search_expr_handles_chained_calls() {
    assert_eq!((0, 20), expand_search_expr("yeah::blah.foo().bar", 18))
}

#[test]
fn expand_search_expr_handles_inline_closures() {
    assert_eq!((0, 24), expand_search_expr("yeah::blah.foo(||{}).bar", 22))
}
#[test]
fn expand_search_expr_handles_a_function_arg() {
    assert_eq!((5, 25), expand_search_expr("myfn(foo::new().baz().com)", 23))
}

#[test]
fn expand_search_expr_handles_pos_at_end_of_search_str() {
    assert_eq!((0, 7), expand_search_expr("foo.bar", 7))
}

pub fn mask_comments(src: &str) -> String {
    let mut result = String::with_capacity(src.len());
    let buf_byte = &[b' '; 128][];
    let buffer = &from_utf8(buf_byte).unwrap()[];
    let mut prev: usize = 0;
    for (start, end) in codecleaner::code_chunks(src) {
        for _ in 0..((start-prev)/128) { result.push_str(buffer); }
        result.push_str(&buffer[..((start-prev)%128)]);
        result.push_str(&src[start..end]);
        prev = end;
    }
    result
}

pub fn mask_sub_scopes(src:&str) -> String {
    let mut result = String::with_capacity(src.len());
    let buf_byte = &[b' '; 128][];
    let buffer = &from_utf8(buf_byte).unwrap()[];
    let mut levels = 0i32;
    let mut start = 0us;
    let mut pos = 0us;

    for &b in src.as_bytes().iter() {
        pos += 1;
        match b {
            b'{' => {
                if levels == 0 {
                    result.push_str(&src[start..(pos-1)]);
                    start = pos;
                }
                levels += 1;
            },
            b'}' => {
                if levels == 1 {
                    for _ in 0..((pos-start)/128) { result.push_str(buffer); }
                    result.push_str(&buffer[..((pos-start)%128)]);
                    start = pos+1;
                }
                levels -= 1;
            },
            b'\n' if levels > 0 => {
                for _ in 0..((pos-start-1)/128) { result.push_str(buffer); }
                result.push_str(&buffer[..((pos-start-1)%128)]);
                result.push('\n');
                start = pos;
            },
            _ => {}
        }
    }
    if levels > 0 {
        for _ in 0..((pos - start)/128) { result.push_str(buffer); }
        result.push_str(&buffer[..((pos-start)%128)]);
    } else {
        result.push_str(&src[start..pos]);
    }
    result
}

pub fn end_of_next_scope<'a>(src: &'a str) -> &'a str {
    match find_close(src.as_bytes().iter(), b'{', b'}', 1) {
        Some(count) => &src[..count+1],
        None => ""
    }
}

pub fn coords_to_point(src: &str, mut linenum: usize, col: usize) -> usize {
    let mut point=0;
    for line in src.lines() {
        linenum -= 1;
        if linenum == 0 { break }
        point+=line.len() + 1;  // +1 for the \n
    }
    point + col
}

pub fn point_to_coords(src:&str, point:usize) -> (usize, usize) {
    let mut i = 0;
    let mut linestart = 0;
    let mut nlines = 1;  // lines start at 1
    for &b in src[..point].as_bytes().iter() {
        i += 1;
        if b == b'\n' {
            nlines += 1;
            linestart = i;
        }
    }
    (nlines, point - linestart)
}

pub fn point_to_coords_from_file(path: &Path, point:usize) -> Option<(usize, usize)> {
    let mut lineno = 0;
    let mut file = BufferedReader::new(File::open(path));
    let mut p = 0;
    for line_r in file.lines() {
        let line = line_r.unwrap();
        lineno += 1;
        if point < (p + line.len()) {
            return Some((lineno, point - p));
        }
        p += line.len();
    }
    None
}

#[test]
fn coords_to_point_works() {
    let src = "
fn myfn() {
    let a = 3;
    print(a);
}";
    assert!(coords_to_point(src, 3, 5) == 18);
}

#[test]
fn test_scope_start() {
    let src = "
fn myfn() {
    let a = 3;
    print(a);
}
";
    let point = coords_to_point(src, 4, 10);
    let start = scope_start(src,point);
    assert!(start == 12);
}

#[test]
fn test_scope_start_handles_sub_scopes() {
    let src = "
fn myfn() {
    let a = 3;
    {
      let b = 4;
    }
    print(a);
}
";
    let point = coords_to_point(src, 7, 10);
    let start = scope_start(src,point);
    assert!(start == 12);
}

#[test]
fn masks_out_comments() {
    let src = "
this is some code
this is a line // with a comment
some more
";
    let r = mask_comments(src);

    assert!(src.len() == r.len());
    // characters at the start are the same
    assert!(src.as_bytes()[5] == r.as_bytes()[5]);
    // characters in the comments are masked
    let commentoffset = coords_to_point(src,3,23);
    assert!((&r[]).char_at(commentoffset) == ' ');
    assert!(src.as_bytes()[commentoffset] != r.as_bytes()[commentoffset]);
    // characters afterwards are the same
    assert!(src.as_bytes()[src.len()-3] == r.as_bytes()[src.len()-3]);
}

#[test]
fn test_point_to_coords() {
    let src = "
fn myfn(b:usize) {
   let a = 3;
   if b == 12 {
       let a = 24;
       do_something_with(a);
   }
   do_something_with(a);
}
";
    round_trip_point_and_coords(src, 4, 5);
}

pub fn round_trip_point_and_coords(src:&str, lineno:usize, charno:usize) {
    let (a,b) = point_to_coords(src, coords_to_point(src, lineno, charno));
     assert_eq!((a,b),(lineno,charno));
}

#[test]
fn finds_end_of_struct_scope() {
    let src="
struct foo {
   a: usize,
   blah: ~str
}
Some other junk";

    let expected="
struct foo {
   a: usize,
   blah: ~str
}";
    let s = end_of_next_scope(src);
    assert_eq!(expected, s);
}

<commit_msg>remove mut iter warning<commit_after>use std::old_io::{File, BufferedReader};
use std::str::from_utf8;
use std::iter::Iterator;

use racer;
use racer::util;
use racer::codecleaner;
use racer::codeiter;
use racer::typeinf;
use racer::ast;

fn find_close<'a, A>(iter: A, open: u8, close: u8, level_end: u32) -> Option<usize>
    where A: Iterator<Item=&'a u8> {

    let mut count = 0us;
    let mut levels = 0u32;
    for &b in iter {
        if b == close {
            if levels == level_end { return Some(count); }
            levels -= 1;
        } else if b == open { levels += 1; }
        count += 1;
    }
    None
}

pub fn find_closing_paren(src:&str, pos:usize) -> usize {
    match find_close(src.as_bytes()[pos..].iter(), b'(', b')', 0) {
        Some(count) => pos + count,
        None => src.len()
    }
}

pub fn scope_start(src:&str, point:usize) -> usize {
    let masked_src = mask_comments(&src[..point]);
    match find_close(masked_src.as_bytes().iter().rev(), b'}', b'{', 0) {
        Some(count) => point - count,
        None => 0
    }
}

pub fn find_stmt_start(msrc: &str, point: usize) -> Option<usize> {
    // iterate the scope to find the start of the statement
    let scopestart = scope_start(msrc, point);
    for (start, end) in codeiter::iter_stmts(&msrc[scopestart..]) {
        if (scopestart + end) > point {
            return Some(scopestart+start);
        }
    }
    None
}

pub fn get_local_module_path(msrc: &str, point: usize) -> Vec<String> {
    let mut v = Vec::new();
    get_local_module_path_(msrc, point, &mut v);
    v
}

fn get_local_module_path_(msrc: &str, point: usize, out: &mut Vec<String>) {
    for (start, end) in codeiter::iter_stmts(msrc) {
        if start < point && end > point {
            let blob = &msrc[start..end];
            if blob.starts_with("pub mod "){
                let p = typeinf::generate_skeleton_for_parsing(blob);
                ast::parse_mod(p).name.map(|name|{

                    let newstart = blob.find_str("{").unwrap() + 1;
                    out.push(name);
                    get_local_module_path_(&blob[newstart..],
                                       point - start - newstart, out);
                });
            }
        }
    }
}

pub fn find_impl_start(msrc: &str, point: usize, scopestart: usize) -> Option<usize> {

    let len = point-scopestart;
    match codeiter::iter_stmts(&msrc[scopestart..]).find(|&(_, end)| end > len) {
        Some((start, _)) => {
            let blob = &msrc[(scopestart + start)..];
            // TODO:: the following is a bit weak at matching traits. make this better
            if blob.starts_with("impl") ||
                blob.starts_with("trait") || blob.starts_with("pub trait") {
                Some(scopestart + start)
            } else {
                let newstart = blob.find_str("{").unwrap() + 1;
                find_impl_start(msrc, point, scopestart+start+newstart)
            }
        },
        None => None
    }
}

#[test]
fn finds_subnested_module() {
    let src = "
    pub mod foo {
        pub mod bar {
            here
        }
    }";
    let point = coords_to_point(src, 4, 12);
    let v = get_local_module_path(src, point);
    assert_eq!("foo", &v[0][]);
    assert_eq!("bar", &v[1][]);

    let point = coords_to_point(src, 2, 8);
    let v = get_local_module_path(src, point);
    assert_eq!("foo", &v[0][]);
}

pub fn split_into_context_and_completion<'a>(s: &'a str) -> (&'a str, &'a str, racer::CompletionType) {

    match s.char_indices().rev().find(|&(_, c)| !util::is_ident_char(c)) {
        Some((i,c)) => {
            match c {
                '.' => (&s[..i], &s[(i+1)..], racer::CompletionType::CompleteField),
                ':' => (&s[..(i-1)], &s[(i+1)..], racer::CompletionType::CompletePath),
                _   => (&s[..(i+1)], &s[(i+1)..], racer::CompletionType::CompletePath)
            }
        },
        None => ("", s, racer::CompletionType::CompletePath)
    }

}

pub fn get_start_of_search_expr(src: &str, point: usize) -> usize {

    let mut i = point-1;
    let mut levels = 0u32;
    for &b in src.as_bytes()[..point].iter().rev() {
        match b {
            b'(' => {
                if levels == 0 { return i+1; }
                levels -= 1;
            },
            b')' => { levels += 1; },
            _ => {
                if levels == 0 &&
                    !util::is_search_expr_char(src.char_at(i)) ||
                    util::is_double_dot(src,i) {
                    return i+1;
                }
            }
        }
        i -= 1;
    }
    0
}

pub fn get_start_of_pattern(src: &str, point: usize) -> usize {
    let mut i = point-1;
    let mut levels = 0u32;
    for &b in src.as_bytes()[..point].iter().rev() {
        match b {
            b'(' => {
                if levels == 0 { return i+1; }
                levels -= 1;
            },
            b')' => { levels += 1; },
            _ => {
                if levels == 0 &&
                    !util::is_pattern_char(src.char_at(i)) {
                    return i+1;
                }
            }
        }
        i -= 1;
    }
    0
}

#[test]
fn get_start_of_pattern_handles_variant() {
    assert_eq!(4, get_start_of_pattern("foo, Some(a) =>",13));
}

#[test]
fn get_start_of_pattern_handles_variant2() {
    assert_eq!(4, get_start_of_pattern("bla, ast::PatTup(ref tuple_elements) => {",36));
}

pub fn expand_search_expr(msrc: &str, point: usize) -> (usize,usize) {
    let start = get_start_of_search_expr(msrc, point);
    return (start, util::find_ident_end(msrc, point));
}

#[test]
fn expand_search_expr_finds_ident() {
    assert_eq!((0, 7), expand_search_expr("foo.bar", 5))
}

#[test]
fn expand_search_expr_handles_chained_calls() {
    assert_eq!((0, 20), expand_search_expr("yeah::blah.foo().bar", 18))
}

#[test]
fn expand_search_expr_handles_inline_closures() {
    assert_eq!((0, 24), expand_search_expr("yeah::blah.foo(||{}).bar", 22))
}
#[test]
fn expand_search_expr_handles_a_function_arg() {
    assert_eq!((5, 25), expand_search_expr("myfn(foo::new().baz().com)", 23))
}

#[test]
fn expand_search_expr_handles_pos_at_end_of_search_str() {
    assert_eq!((0, 7), expand_search_expr("foo.bar", 7))
}

pub fn mask_comments(src: &str) -> String {
    let mut result = String::with_capacity(src.len());
    let buf_byte = &[b' '; 128][];
    let buffer = &from_utf8(buf_byte).unwrap()[];
    let mut prev: usize = 0;
    for (start, end) in codecleaner::code_chunks(src) {
        for _ in 0..((start-prev)/128) { result.push_str(buffer); }
        result.push_str(&buffer[..((start-prev)%128)]);
        result.push_str(&src[start..end]);
        prev = end;
    }
    result
}

pub fn mask_sub_scopes(src:&str) -> String {
    let mut result = String::with_capacity(src.len());
    let buf_byte = &[b' '; 128][];
    let buffer = &from_utf8(buf_byte).unwrap()[];
    let mut levels = 0i32;
    let mut start = 0us;
    let mut pos = 0us;

    for &b in src.as_bytes().iter() {
        pos += 1;
        match b {
            b'{' => {
                if levels == 0 {
                    result.push_str(&src[start..(pos-1)]);
                    start = pos;
                }
                levels += 1;
            },
            b'}' => {
                if levels == 1 {
                    for _ in 0..((pos-start)/128) { result.push_str(buffer); }
                    result.push_str(&buffer[..((pos-start)%128)]);
                    start = pos+1;
                }
                levels -= 1;
            },
            b'\n' if levels > 0 => {
                for _ in 0..((pos-start-1)/128) { result.push_str(buffer); }
                result.push_str(&buffer[..((pos-start-1)%128)]);
                result.push('\n');
                start = pos;
            },
            _ => {}
        }
    }
    if levels > 0 {
        for _ in 0..((pos - start)/128) { result.push_str(buffer); }
        result.push_str(&buffer[..((pos-start)%128)]);
    } else {
        result.push_str(&src[start..pos]);
    }
    result
}

pub fn end_of_next_scope<'a>(src: &'a str) -> &'a str {
    match find_close(src.as_bytes().iter(), b'{', b'}', 1) {
        Some(count) => &src[..count+1],
        None => ""
    }
}

pub fn coords_to_point(src: &str, mut linenum: usize, col: usize) -> usize {
    let mut point=0;
    for line in src.lines() {
        linenum -= 1;
        if linenum == 0 { break }
        point+=line.len() + 1;  // +1 for the \n
    }
    point + col
}

pub fn point_to_coords(src:&str, point:usize) -> (usize, usize) {
    let mut i = 0;
    let mut linestart = 0;
    let mut nlines = 1;  // lines start at 1
    for &b in src[..point].as_bytes().iter() {
        i += 1;
        if b == b'\n' {
            nlines += 1;
            linestart = i;
        }
    }
    (nlines, point - linestart)
}

pub fn point_to_coords_from_file(path: &Path, point:usize) -> Option<(usize, usize)> {
    let mut lineno = 0;
    let mut file = BufferedReader::new(File::open(path));
    let mut p = 0;
    for line_r in file.lines() {
        let line = line_r.unwrap();
        lineno += 1;
        if point < (p + line.len()) {
            return Some((lineno, point - p));
        }
        p += line.len();
    }
    None
}

#[test]
fn coords_to_point_works() {
    let src = "
fn myfn() {
    let a = 3;
    print(a);
}";
    assert!(coords_to_point(src, 3, 5) == 18);
}

#[test]
fn test_scope_start() {
    let src = "
fn myfn() {
    let a = 3;
    print(a);
}
";
    let point = coords_to_point(src, 4, 10);
    let start = scope_start(src,point);
    assert!(start == 12);
}

#[test]
fn test_scope_start_handles_sub_scopes() {
    let src = "
fn myfn() {
    let a = 3;
    {
      let b = 4;
    }
    print(a);
}
";
    let point = coords_to_point(src, 7, 10);
    let start = scope_start(src,point);
    assert!(start == 12);
}

#[test]
fn masks_out_comments() {
    let src = "
this is some code
this is a line // with a comment
some more
";
    let r = mask_comments(src);

    assert!(src.len() == r.len());
    // characters at the start are the same
    assert!(src.as_bytes()[5] == r.as_bytes()[5]);
    // characters in the comments are masked
    let commentoffset = coords_to_point(src,3,23);
    assert!((&r[]).char_at(commentoffset) == ' ');
    assert!(src.as_bytes()[commentoffset] != r.as_bytes()[commentoffset]);
    // characters afterwards are the same
    assert!(src.as_bytes()[src.len()-3] == r.as_bytes()[src.len()-3]);
}

#[test]
fn test_point_to_coords() {
    let src = "
fn myfn(b:usize) {
   let a = 3;
   if b == 12 {
       let a = 24;
       do_something_with(a);
   }
   do_something_with(a);
}
";
    round_trip_point_and_coords(src, 4, 5);
}

pub fn round_trip_point_and_coords(src:&str, lineno:usize, charno:usize) {
    let (a,b) = point_to_coords(src, coords_to_point(src, lineno, charno));
     assert_eq!((a,b),(lineno,charno));
}

#[test]
fn finds_end_of_struct_scope() {
    let src="
struct foo {
   a: usize,
   blah: ~str
}
Some other junk";

    let expected="
struct foo {
   a: usize,
   blah: ~str
}";
    let s = end_of_next_scope(src);
    assert_eq!(expected, s);
}

<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! A handler and associated types for the file stat action.
//!
//! A file stat action responses with stat of a given file

use std::fs::Metadata;
use std::path::PathBuf;

use log::warn;

use crate::session::{self, Session};

#[derive(Debug)]
pub struct Request {
    path: PathBuf,
    collect_ext_attrs: bool,
    follow_symlink: bool,
}

#[derive(Debug)]
pub struct Response {
    path: PathBuf,
    metadata: Metadata,
    symlink: Option<PathBuf>,
    #[cfg(target_family = "unix")]
    ext_attrs: Vec<crate::fs::unix::ExtAttr>,
    #[cfg(target_os = "linux")]
    flags_linux: Option<u32>,
}

#[derive(Debug)]
enum Error {
    Metadata(std::io::Error),
}

impl std::error::Error for Error {

    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        use Error::*;

        match *self {
            Metadata(ref error) => Some(error),
        }
    }
}

impl std::fmt::Display for Error {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        use Error::*;

        match *self {
            Metadata(ref error) => {
                write!(fmt, "unable to collect metadata: {}", error)
            }
        }
    }
}

impl From<Error> for session::Error {

    fn from(error: Error) -> session::Error {
        session::Error::action(error)
    }
}

pub fn handle<S: Session>(session: &mut S, request: Request) -> session::Result<()> {
    let metadata = if request.follow_symlink {
        std::fs::metadata(&request.path)
    } else {
        std::fs::symlink_metadata(&request.path)
    }.map_err(Error::Metadata)?;

    let symlink = if metadata.file_type().is_symlink() {
        std::fs::read_link(&request.path).map_err(|error| {
            // TODO: Make the `ack!` macro more expressive and rewrite it.
            warn! {
                "failed to read symlink for '{path}': {cause}",
                path = request.path.display(),
                cause = error,
            }
        }).ok()
    } else {
        None
    };

    #[cfg(target_family = "unix")]
    let ext_attrs = if request.collect_ext_attrs {
        // TODO: Make the `ack!` macro more expressive and then simplify it.
        match crate::fs::unix::ext_attrs(&request.path) {
            Ok(ext_attrs) => ext_attrs.collect(),
            Err(error) => {
                warn! {
                    "failed to collect attributes for '{path}': {cause}",
                    path = request.path.display(),
                    cause = error,
                };
                vec!()
            },
        }
    } else {
        vec!()
    };

    #[cfg(target_os = "linux")]
    let flags_linux = crate::fs::linux::flags(&request.path).map_err(|error| {
        // TODO: Make the `ack!` macro more expressive and rewrite it.
        warn! {
            "failed to collect flags for '{path}': {cause}",
            path = request.path.display(),
            cause = error,
        }
    }).ok();

    let response = Response {
        path: request.path,
        metadata: metadata,
        symlink: symlink,
        #[cfg(target_family = "unix")]
        ext_attrs: ext_attrs,
        #[cfg(target_os = "linux")]
        flags_linux: flags_linux,
    };

    session.reply(response)?;
    Ok(())
}

impl super::Request for Request {

    type Proto = rrg_proto::GetFileStatRequest;

    fn from_proto(proto: Self::Proto) -> Result<Self, session::ParseError> {
        use std::convert::TryInto as _;

        let path = proto.pathspec
            .ok_or(session::MissingFieldError::new("path spec"))?
            .try_into().map_err(session::ParseError::malformed)?;

        Ok(Request {
            path: path,
            follow_symlink: proto.follow_symlink.unwrap_or(false),
            collect_ext_attrs: proto.collect_ext_attrs.unwrap_or(false),
        })
    }
}

impl super::Response for Response {

    const RDF_NAME: Option<&'static str> = Some("StatEntry");

    type Proto = rrg_proto::StatEntry;

    fn into_proto(self) -> Self::Proto {
        use rrg_proto::convert::IntoLossy as _;

        rrg_proto::StatEntry {
            pathspec: Some(self.path.into()),
            #[cfg(target_family = "unix")]
            ext_attrs: self.ext_attrs.into_iter().map(Into::into).collect(),
            #[cfg(target_os = "linux")]
            st_flags_linux: self.flags_linux,
            ..self.metadata.into_lossy()
        }
    }
}

#[cfg(test)]
mod tests {
}
<commit_msg>Improve code formatting.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! A handler and associated types for the file stat action.
//!
//! A file stat action responses with stat of a given file

use std::fs::Metadata;
use std::path::PathBuf;

use log::warn;

use crate::session::{self, Session};

#[derive(Debug)]
pub struct Request {
    path: PathBuf,
    collect_ext_attrs: bool,
    follow_symlink: bool,
}

#[derive(Debug)]
pub struct Response {
    path: PathBuf,
    metadata: Metadata,
    symlink: Option<PathBuf>,
    #[cfg(target_family = "unix")]
    ext_attrs: Vec<crate::fs::unix::ExtAttr>,
    #[cfg(target_os = "linux")]
    flags_linux: Option<u32>,
}

#[derive(Debug)]
enum Error {
    Metadata(std::io::Error),
}

impl std::error::Error for Error {

    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        use Error::*;

        match *self {
            Metadata(ref error) => Some(error),
        }
    }
}

impl std::fmt::Display for Error {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        use Error::*;

        match *self {
            Metadata(ref error) => {
                write!(fmt, "unable to collect metadata: {}", error)
            }
        }
    }
}

impl From<Error> for session::Error {

    fn from(error: Error) -> session::Error {
        session::Error::action(error)
    }
}

pub fn handle<S>(session: &mut S, request: Request) -> session::Result<()>
where
    S: Session,
{
    let metadata = if request.follow_symlink {
        std::fs::metadata(&request.path)
    } else {
        std::fs::symlink_metadata(&request.path)
    }.map_err(Error::Metadata)?;

    let symlink = if metadata.file_type().is_symlink() {
        std::fs::read_link(&request.path).map_err(|error| {
            // TODO: Make the `ack!` macro more expressive and rewrite it.
            warn! {
                "failed to read symlink for '{path}': {cause}",
                path = request.path.display(),
                cause = error,
            }
        }).ok()
    } else {
        None
    };

    #[cfg(target_family = "unix")]
    let ext_attrs = if request.collect_ext_attrs {
        // TODO: Make the `ack!` macro more expressive and then simplify it.
        match crate::fs::unix::ext_attrs(&request.path) {
            Ok(ext_attrs) => ext_attrs.collect(),
            Err(error) => {
                warn! {
                    "failed to collect attributes for '{path}': {cause}",
                    path = request.path.display(),
                    cause = error,
                };
                vec!()
            },
        }
    } else {
        vec!()
    };

    #[cfg(target_os = "linux")]
    let flags_linux = crate::fs::linux::flags(&request.path).map_err(|error| {
        // TODO: Make the `ack!` macro more expressive and rewrite it.
        warn! {
            "failed to collect flags for '{path}': {cause}",
            path = request.path.display(),
            cause = error,
        }
    }).ok();

    let response = Response {
        path: request.path,
        metadata: metadata,
        symlink: symlink,
        #[cfg(target_family = "unix")]
        ext_attrs: ext_attrs,
        #[cfg(target_os = "linux")]
        flags_linux: flags_linux,
    };

    session.reply(response)?;

    Ok(())
}

impl super::Request for Request {

    type Proto = rrg_proto::GetFileStatRequest;

    fn from_proto(proto: Self::Proto) -> Result<Self, session::ParseError> {
        use std::convert::TryInto as _;

        let path = proto.pathspec
            .ok_or(session::MissingFieldError::new("path spec"))?
            .try_into().map_err(session::ParseError::malformed)?;

        Ok(Request {
            path: path,
            follow_symlink: proto.follow_symlink.unwrap_or(false),
            collect_ext_attrs: proto.collect_ext_attrs.unwrap_or(false),
        })
    }
}

impl super::Response for Response {

    const RDF_NAME: Option<&'static str> = Some("StatEntry");

    type Proto = rrg_proto::StatEntry;

    fn into_proto(self) -> Self::Proto {
        use rrg_proto::convert::IntoLossy as _;

        rrg_proto::StatEntry {
            pathspec: Some(self.path.into()),
            #[cfg(target_family = "unix")]
            ext_attrs: self.ext_attrs.into_iter().map(Into::into).collect(),
            #[cfg(target_os = "linux")]
            st_flags_linux: self.flags_linux,
            ..self.metadata.into_lossy()
        }
    }
}

#[cfg(test)]
mod tests {
}
<|endoftext|>