<commit_before>//! Internal traits and functions used to implement parallel
//! iteration. These should be considered highly unstable: users of
//! parallel iterators should not need to interact with them directly.
//! See `README.md` for a high-level overview.

use join;
use super::IndexedParallelIterator;
use super::len::*;
use registry::global_registry;

pub trait ProducerCallback<ITEM> {
    type Output;
    fn callback
(self, producer: P) -> Self::Output where P: Producer<Item = ITEM>;
}

/// A producer which will produce a fixed number of items N. This is
/// not queryable through the API; the consumer is expected to track
/// it.
pub trait Producer: IntoIterator + Send + Sized {
    /// Reports whether the producer has explicit weights.
    fn weighted(&self) -> bool {
        false
    }

    /// Cost to produce `len` items, where `len` must be `N`.
    fn cost(&mut self, len: usize) -> f64;

    /// Split into two producers; one produces items `0..index`, the
    /// other `index..N`. Index must be less than `N`.
    fn split_at(self, index: usize) -> (Self, Self);

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    ///
    /// The provided implementation is sufficient for most iterables.
    fn fold_with<F>(self, folder: F) -> F
        where F: Folder<Self::Item>,
    {
        folder.consume_iter(self)
    }
}

/// A consumer which consumes items that are fed to it.
pub trait Consumer<Item>: Send + Sized {
    type Folder: Folder<Item, Result = Self::Result>;
    type Reducer: Reducer<Self::Result>;
    type Result: Send;

    /// Reports whether the consumer has explicit weights.
    fn weighted(&self) -> bool {
        false
    }

    /// If it costs `producer_cost` to produce the items we will
    /// consume, returns cost adjusted to account for consuming them.
    fn cost(&mut self, producer_cost: f64) -> f64;

    /// Divide the consumer into two consumers, one processing items
    /// `0..index` and one processing items from `index..`. Also
    /// produces a reducer that can be used to reduce the results at
    /// the end.
    fn split_at(self, index: usize) -> (Self, Self, Self::Reducer);

    /// Convert the consumer into a folder that can consume items
    /// sequentially, eventually producing a final result.
    fn into_folder(self) -> Self::Folder;

    /// Hint whether this `Consumer` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Folder<Item>: Sized {
    type Result;

    /// Consume next item and return new sequential state.
    fn consume(self, item: Item) -> Self;

    /// Consume items from the iterator until full, and return new sequential state.
    fn consume_iter(mut self, iter: I) -> Self
        where I: IntoIterator<Item=Item>
    {
        for item in iter {
            self = self.consume(item);
            if self.full() {
                break;
            }
        }
        self
    }

    /// Finish consuming items, produce final result.
    fn complete(self) -> Self::Result;

    /// Hint whether this `Folder` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Reducer<Result> {
    /// Reduce two final results into one; this is executed after a
    /// split.
    fn reduce(self, left: Result, right: Result) -> Result;
}

/// A stateless consumer can be freely copied.
pub trait UnindexedConsumer<ITEM>: Consumer<ITEM> {
    // The result of split_off_left should be used for the left side of the
    // data it consumes, and the remaining consumer for the right side
    // (this matters for methods like find_first).
    fn split_off_left(&self) -> Self;
    fn to_reducer(&self) -> Self::Reducer;
}

/// An unindexed producer that doesn't know its exact length.
/// (or can't represent its known length in a `usize`)
pub trait UnindexedProducer: Send + Sized {
    type Item;

    /// Split midway into a new producer if possible, otherwise return `None`.
    fn split(&mut self) -> Option<Self>;

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    fn fold_with<F>(self, folder: F) -> F where F: Folder<Self::Item>;
}

/// A splitter controls the policy for splitting into smaller work items.
#[derive(Clone, Copy)]
enum Splitter {
    /// Classic cost-splitting uses weights to split until below a threshold.
    Cost(f64),

    /// Thief-splitting is an adaptive policy that starts by splitting into
    /// enough jobs for every worker thread, and then resets itself whenever a
    /// job is actually stolen into a different thread.
    Thief(usize, usize),
}

impl Splitter {
    #[inline]
    fn thief_id() -> usize {
        // The actual `ID` value is irrelevant.  We're just using its TLS
        // address as a unique thread key, faster than a real thread-id call.
        thread_local!{ static ID: bool = false }
        ID.with(|id| id as *const bool as usize)
    }

    #[inline]
    fn new_thief() -> Splitter {
        Splitter::Thief(Splitter::thief_id(), global_registry().num_threads())
    }

    #[inline]
    fn try(&mut self) -> bool {
        match *self {
            Splitter::Cost(ref mut cost) => {
                if *cost > THRESHOLD {
                    *cost /= 2.0;
                    true
                } else {
                    false
                }
            }

            Splitter::Thief(ref mut origin, ref mut splits) => {
                let id = Splitter::thief_id();
                if *origin != id {
                    *origin = id;
                    *splits = global_registry().num_threads();
                    true
                } else if *splits > 0 {
                    *splits /= 2;
                    true
                } else {
                    false
                }
            }
        }
    }

    #[inline]
    fn try_unindexed
(&mut self, producer: &mut P) -> Option

        where P: UnindexedProducer
    {
        if self.try() { producer.split() } else { None }
    }
}

pub fn bridge<PAR_ITER, C>(mut par_iter: PAR_ITER, consumer: C) -> C::Result
    where PAR_ITER: IndexedParallelIterator,
          C: Consumer<PAR_ITER::Item>
{
    let len = par_iter.len();
    return par_iter.with_producer(Callback {
        len: len,
        consumer: consumer,
    });

    struct Callback<C> {
        len: usize,
        consumer: C,
    }

    impl<C, ITEM> ProducerCallback<ITEM> for Callback<C>
        where C: Consumer<ITEM>
    {
        type Output = C::Result;
        fn callback
(self, producer: P) -> C::Result
            where P: Producer<Item = ITEM>
        {
            bridge_producer_consumer(self.len, producer, self.consumer)
        }
    }
}

pub fn bridge_producer_consumer
(len: usize, mut producer: P, mut consumer: C) -> C::Result
    where P: Producer,
          C: Consumer

{
    let splitter = if producer.weighted() || consumer.weighted() {
        let producer_cost = producer.cost(len);
        let cost = consumer.cost(producer_cost);
        Splitter::Cost(cost)
    } else {
        Splitter::new_thief()
    };
    return helper(len, splitter, producer, consumer);

    fn helper
(len: usize, mut splitter: Splitter, producer: P, consumer: C) -> C::Result
        where P: Producer,
              C: Consumer

    {
        if consumer.full() {
            consumer.into_folder().complete()
        } else if len > 1 && splitter.try() {
            let mid = len / 2;
            let (left_producer, right_producer) = producer.split_at(mid);
            let (left_consumer, right_consumer, reducer) = consumer.split_at(mid);
            let (left_result, right_result) =
                join(|| helper(mid, splitter, left_producer, left_consumer),
                     || helper(len - mid, splitter, right_producer, right_consumer));
            reducer.reduce(left_result, right_result)
        } else {
            producer.fold_with(consumer.into_folder()).complete()
        }
    }
}

pub fn bridge_unindexed
(producer: P, consumer: C) -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer

{
    let splitter = Splitter::new_thief();
    bridge_unindexed_producer_consumer(splitter, producer, consumer)
}

fn bridge_unindexed_producer_consumer
(mut splitter: Splitter,
                                            mut producer: P,
                                            consumer: C)
                                            -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer

{
    if consumer.full() {
        consumer.into_folder().complete()
    } else if let Some(right_producer) = splitter.try_unindexed(&mut producer) {
        let (reducer, left_consumer, right_consumer) =
            (consumer.to_reducer(), consumer.split_off_left(), consumer);
        let (left_result, right_result) =
            join(|| bridge_unindexed_producer_consumer(splitter, producer, left_consumer),
                 || bridge_unindexed_producer_consumer(splitter, right_producer, right_consumer));
        reducer.reduce(left_result, right_result)
    } else {
        producer.fold_with(consumer.into_folder()).complete()
    }
}
<commit_msg>doc: the `Producer::split_at` index can equal the length<commit_after>//! Internal traits and functions used to implement parallel
//! iteration. These should be considered highly unstable: users of
//! parallel iterators should not need to interact with them directly.
//! See `README.md` for a high-level overview.

use join;
use super::IndexedParallelIterator;
use super::len::*;
use registry::global_registry;

pub trait ProducerCallback<ITEM> {
    type Output;
    fn callback
(self, producer: P) -> Self::Output where P: Producer<Item = ITEM>;
}

/// A producer which will produce a fixed number of items N. This is
/// not queryable through the API; the consumer is expected to track
/// it.
pub trait Producer: IntoIterator + Send + Sized {
    /// Reports whether the producer has explicit weights.
    fn weighted(&self) -> bool {
        false
    }

    /// Cost to produce `len` items, where `len` must be `N`.
    fn cost(&mut self, len: usize) -> f64;

    /// Split into two producers; one produces items `0..index`, the
    /// other `index..N`. Index must be less than or equal to `N`.
    fn split_at(self, index: usize) -> (Self, Self);

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    ///
    /// The provided implementation is sufficient for most iterables.
    fn fold_with<F>(self, folder: F) -> F
        where F: Folder<Self::Item>,
    {
        folder.consume_iter(self)
    }
}

/// A consumer which consumes items that are fed to it.
pub trait Consumer<Item>: Send + Sized {
    type Folder: Folder<Item, Result = Self::Result>;
    type Reducer: Reducer<Self::Result>;
    type Result: Send;

    /// Reports whether the consumer has explicit weights.
    fn weighted(&self) -> bool {
        false
    }

    /// If it costs `producer_cost` to produce the items we will
    /// consume, returns cost adjusted to account for consuming them.
    fn cost(&mut self, producer_cost: f64) -> f64;

    /// Divide the consumer into two consumers, one processing items
    /// `0..index` and one processing items from `index..`. Also
    /// produces a reducer that can be used to reduce the results at
    /// the end.
    fn split_at(self, index: usize) -> (Self, Self, Self::Reducer);

    /// Convert the consumer into a folder that can consume items
    /// sequentially, eventually producing a final result.
    fn into_folder(self) -> Self::Folder;

    /// Hint whether this `Consumer` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Folder<Item>: Sized {
    type Result;

    /// Consume next item and return new sequential state.
    fn consume(self, item: Item) -> Self;

    /// Consume items from the iterator until full, and return new sequential state.
    fn consume_iter(mut self, iter: I) -> Self
        where I: IntoIterator<Item=Item>
    {
        for item in iter {
            self = self.consume(item);
            if self.full() {
                break;
            }
        }
        self
    }

    /// Finish consuming items, produce final result.
    fn complete(self) -> Self::Result;

    /// Hint whether this `Folder` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Reducer<Result> {
    /// Reduce two final results into one; this is executed after a
    /// split.
    fn reduce(self, left: Result, right: Result) -> Result;
}

/// A stateless consumer can be freely copied.
pub trait UnindexedConsumer<ITEM>: Consumer<ITEM> {
    // The result of split_off_left should be used for the left side of the
    // data it consumes, and the remaining consumer for the right side
    // (this matters for methods like find_first).
    fn split_off_left(&self) -> Self;
    fn to_reducer(&self) -> Self::Reducer;
}

/// An unindexed producer that doesn't know its exact length.
/// (or can't represent its known length in a `usize`)
pub trait UnindexedProducer: Send + Sized {
    type Item;

    /// Split midway into a new producer if possible, otherwise return `None`.
    fn split(&mut self) -> Option<Self>;

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    fn fold_with<F>(self, folder: F) -> F where F: Folder<Self::Item>;
}

/// A splitter controls the policy for splitting into smaller work items.
#[derive(Clone, Copy)]
enum Splitter {
    /// Classic cost-splitting uses weights to split until below a threshold.
    Cost(f64),

    /// Thief-splitting is an adaptive policy that starts by splitting into
    /// enough jobs for every worker thread, and then resets itself whenever a
    /// job is actually stolen into a different thread.
    Thief(usize, usize),
}

impl Splitter {
    #[inline]
    fn thief_id() -> usize {
        // The actual `ID` value is irrelevant.  We're just using its TLS
        // address as a unique thread key, faster than a real thread-id call.
        thread_local!{ static ID: bool = false }
        ID.with(|id| id as *const bool as usize)
    }

    #[inline]
    fn new_thief() -> Splitter {
        Splitter::Thief(Splitter::thief_id(), global_registry().num_threads())
    }

    #[inline]
    fn try(&mut self) -> bool {
        match *self {
            Splitter::Cost(ref mut cost) => {
                if *cost > THRESHOLD {
                    *cost /= 2.0;
                    true
                } else {
                    false
                }
            }

            Splitter::Thief(ref mut origin, ref mut splits) => {
                let id = Splitter::thief_id();
                if *origin != id {
                    *origin = id;
                    *splits = global_registry().num_threads();
                    true
                } else if *splits > 0 {
                    *splits /= 2;
                    true
                } else {
                    false
                }
            }
        }
    }

    #[inline]
    fn try_unindexed
(&mut self, producer: &mut P) -> Option

        where P: UnindexedProducer
    {
        if self.try() { producer.split() } else { None }
    }
}

pub fn bridge<PAR_ITER, C>(mut par_iter: PAR_ITER, consumer: C) -> C::Result
    where PAR_ITER: IndexedParallelIterator,
          C: Consumer<PAR_ITER::Item>
{
    let len = par_iter.len();
    return par_iter.with_producer(Callback {
        len: len,
        consumer: consumer,
    });

    struct Callback<C> {
        len: usize,
        consumer: C,
    }

    impl<C, ITEM> ProducerCallback<ITEM> for Callback<C>
        where C: Consumer<ITEM>
    {
        type Output = C::Result;
        fn callback
(self, producer: P) -> C::Result
            where P: Producer<Item = ITEM>
        {
            bridge_producer_consumer(self.len, producer, self.consumer)
        }
    }
}

pub fn bridge_producer_consumer
(len: usize, mut producer: P, mut consumer: C) -> C::Result
    where P: Producer,
          C: Consumer

{
    let splitter = if producer.weighted() || consumer.weighted() {
        let producer_cost = producer.cost(len);
        let cost = consumer.cost(producer_cost);
        Splitter::Cost(cost)
    } else {
        Splitter::new_thief()
    };
    return helper(len, splitter, producer, consumer);

    fn helper
(len: usize, mut splitter: Splitter, producer: P, consumer: C) -> C::Result
        where P: Producer,
              C: Consumer

    {
        if consumer.full() {
            consumer.into_folder().complete()
        } else if len > 1 && splitter.try() {
            let mid = len / 2;
            let (left_producer, right_producer) = producer.split_at(mid);
            let (left_consumer, right_consumer, reducer) = consumer.split_at(mid);
            let (left_result, right_result) =
                join(|| helper(mid, splitter, left_producer, left_consumer),
                     || helper(len - mid, splitter, right_producer, right_consumer));
            reducer.reduce(left_result, right_result)
        } else {
            producer.fold_with(consumer.into_folder()).complete()
        }
    }
}

pub fn bridge_unindexed
(producer: P, consumer: C) -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer

{
    let splitter = Splitter::new_thief();
    bridge_unindexed_producer_consumer(splitter, producer, consumer)
}

fn bridge_unindexed_producer_consumer
(mut splitter: Splitter,
                                            mut producer: P,
                                            consumer: C)
                                            -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer

{
    if consumer.full() {
        consumer.into_folder().complete()
    } else if let Some(right_producer) = splitter.try_unindexed(&mut producer) {
        let (reducer, left_consumer, right_consumer) =
            (consumer.to_reducer(), consumer.split_off_left(), consumer);
        let (left_result, right_result) =
            join(|| bridge_unindexed_producer_consumer(splitter, producer, left_consumer),
                 || bridge_unindexed_producer_consumer(splitter, right_producer, right_consumer));
        reducer.reduce(left_result, right_result)
    } else {
        producer.fold_with(consumer.into_folder()).complete()
    }
}
<|endoftext|>


<commit_before>//! FBX binary parser.

use std::io;
use std::io::Read;

pub use self::error::{Result, Error, Warning};
pub use self::event::{Event, FbxHeader, FbxFooter, StartNode};
pub use self::event::{Attributes, Attribute, SpecialAttributeType};
pub use self::event::{PrimitiveAttribute, ArrayAttribute, SpecialAttribute};
pub use self::event::ArrayAttributeReader;
use self::event::{EventBuilder, NodeHeader, StartNodeBuilder};
use self::event::read_fbx_header;
pub use self::reader::{ParserSource, BasicSource, SeekableSource, LimitedSeekReader};

mod error;
mod event;
mod reader;
pub mod utils;

/// Parser state without error.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
    /// Reading file header.
    Header,
    /// A node started.
    ///
    /// This status means:
    ///
    /// - there might be some attributes remain unread, or
    /// - the most recent opened node might close without no null node header.
    NodeStarted,
    /// A node ended.
    ///
    /// This status means:
    ///
    /// - if the next event is `NodeEnd`, there must be a null node header, and
    /// - if the parser got an extra null header, it indicates end of implicit root node.
    NodeEnded,
}

/// Information about opened (but not yet closed) node.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
struct OpenNode {
    /// Start offset of the node attribute.
    ///
    /// Note that this doesn't mean start offset of node header.
    begin: u64,
    /// End offset of the node.
    end: u64,
    /// End offset of attributes of the node.
    attributes_end: u64,
}

/// Pull parser for whole FBX with binary format.
#[derive(Debug)]
pub struct RootParser<R> {
    /// Source reader.
    source: R,
    /// Parser state.
    ///
    /// `Ok(State)` if the parser is working without critical error,
    /// `Err(Error)` if the parsing failed and cannot be continued.
    state: Result<State>,
    /// Warnings.
    warnings: Vec<Warning>,
    /// FBX version.
    fbx_version: Option<u32>,
    /// Open nodes stack.
    open_nodes: Vec<OpenNode>,
}

impl<R: Read> RootParser<BasicSource<R>> {
    /// Creates a new binary parser.
    pub fn new(source: R) -> Self {
        RootParser {
            source: BasicSource::new(source),
            state: Ok(State::Header),
            warnings: Vec::new(),
            fbx_version: None,
            open_nodes: Vec::new(),
        }
    }
}

impl<R: Read + io::Seek> RootParser<SeekableSource<R>> {
    /// Creates a new binary parser.
    pub fn from_seekable(source: R) -> Self {
        RootParser {
            source: SeekableSource::new(source),
            state: Ok(State::Header),
            warnings: Vec::new(),
            fbx_version: None,
            open_nodes: Vec::new(),
        }
    }
}

impl<R: ParserSource> RootParser<R> {
    /// Returns FBX version of the reading input.
    ///
    /// Returns `None` if unknown yet.
    pub fn fbx_version(&self) -> Option<u32> {
        self.fbx_version
    }

    /// Returns the parser error if available.
    pub fn error(&self) -> Option<&Error> {
        self.state.as_ref().err()
    }

    /// Returns reference to the warnings.
    pub fn warnings(&self) -> &Vec<Warning> {
        &self.warnings
    }

    /// Parses FBX from the given stream and returns the next event.
    pub fn next_event(&mut self) -> Result<Event<R>> {
        let builder = match self.state.clone()? {
            State::Header => self.read_fbx_header(),
            State::NodeStarted => self.read_after_node_start(),
            State::NodeEnded => self.read_after_node_end(),
        };
        if let Err(ref err) = builder {
            self.set_error(err);
        }
        Ok(builder?.build(self))
    }

    /// Skips to the end of the current node.
    ///
    /// Returns `Ok(true)` if the current node is skipped and closed,
    /// `Ok(false)` if no nodes are open (i.e. the parser is reading under implicit root node),
    /// `Err(err)` if error happened.
    pub fn skip_current_node(&mut self) -> Result<bool> {
        if let Some(end) = self.open_nodes.pop().map(|v| v.end) {
            self.source.skip_to(end)?;
            self.state = Ok(State::NodeEnded);
            Ok(true)
        } else {
            Ok(false)
        }
    }

    /// Returns the number of the opened (and not closed) node.
    pub fn num_open_nodes(&self) -> usize {
        self.open_nodes.len()
    }

    /// Set the parser state as finished parsing.
    fn set_finish(&mut self) {
        self.state = Err(Error::Finished);
    }

    /// Set the parser state as error.
    fn set_error(&mut self, err: &Error) {
        error!("FBX binary parser error: {}", err);
        debug!("Parser: {:#?}", self);
        self.state = Err(err.clone());
    }

    /// Add warning.
    fn warn(&mut self, warning: Warning) {
        warn!("FBX binary parser warning: {}", warning);
        debug!("Parser: {:#?}", self);
        self.warnings.push(warning);
    }

    /// Reads FBX header.
    fn read_fbx_header(&mut self) -> Result<EventBuilder> {
        let header = read_fbx_header(self)?;
        self.fbx_version = Some(header.version);
        self.state = Ok(State::NodeEnded);
        Ok(header.into())
    }

    /// Gets event after node start.
    fn read_after_node_start(&mut self) -> Result<EventBuilder> {
        // Attributes of recent opened node might remain partially unread.
        // They should be skipped before getting a next node event.
        self.skip_attributes()?;

        // Most recent opened node might ends here without a null node header.
        if let Some(end) = self.open_nodes.last().map(|v| v.end) {
            if self.source.position() == end {
                // Most recent opened node ends here (without a null node header).
                self.state = Ok(State::NodeEnded);
                self.open_nodes.pop();
                return Ok(EventBuilder::EndNode);
            }
        }

        let builder = self.read_node_event()?;
        Ok(builder)
    }

    /// Gets event after node end.
    fn read_after_node_end(&mut self) -> Result<EventBuilder> {
        self.read_node_event()
    }

    /// Reads a next node-related event from the source.
    ///
    /// This always returns `Ok(EventBuilder::StartNode)`, `Ok(EventBuilder::EndNode)`,
    /// `Ok(EventBuilder::EndFbx)` or `Err(_)`.
    fn read_node_event(&mut self) -> Result<EventBuilder> {
        let header = NodeHeader::read_from_parser(self)?;
        if header.is_node_end() {
            if let Some(last_node) = self.open_nodes.pop() {
                // There is open nodes, so this is not end of the FBX.
                let current_pos = self.source.position();
                if current_pos != last_node.end {
                    // Invalid node header.
                    return Err(Error::WrongNodeEndOffset {
                        begin: last_node.begin,
                        expected_end: last_node.end,
                        real_end: current_pos,
                    });
                }
            } else {
                assert_eq!(self.state.as_ref().ok(),
                           Some(&State::NodeEnded),
                           "End of implicit root node is read with unexpected parser state {:?}",
                           self.state);
                // No open nodes, so this `EndNode` event indicates the end of
                // the implicit root node.
                // FBX file has no more nodes.
                let footer = self.read_fbx_footer();
                return Ok(footer.into());
            }

            self.state = Ok(State::NodeEnded);
            Ok(EventBuilder::EndNode)
        } else {
            let mut name_vec = vec![0u8; header.bytelen_name as usize];
            self.source.read_exact(&mut name_vec)?;
            let name = String::from_utf8(name_vec).map_err(Error::node_name_invalid_utf8)?;
            let current_pos = self.source.position();
            self.open_nodes.push(OpenNode {
                begin: current_pos,
                end: header.end_offset,
                attributes_end: current_pos + header.bytelen_attributes,
            });

            // Zero or more attributes come after node start.
            self.state = Ok(State::NodeStarted);
            Ok(StartNodeBuilder {
                name: name,
                header: header,
            }
            .into())
        }
    }

    /// Reads an FBX footer.
    fn read_fbx_footer(&mut self) -> Result<FbxFooter> {
        self.set_finish();
        FbxFooter::read_from_parser(self)
    }

    /// Skip attributes of the most recent opened node.
    fn skip_attributes(&mut self) -> io::Result<()> {
        let attributes_end = self.open_nodes
            .last()
            .expect("`RootParser::skip_attributes()` is called but no nodes are open")
            .attributes_end;
        self.source.skip_to(attributes_end)
    }
}

/// Pull parser for a subtree of the FBX binary.
pub struct SubtreeParser<'a, R: 'a> {
    /// Root parser.
    root_parser: &'a mut RootParser<R>,
    /// Initial depth.
    ///
    /// Depth of the implicit root node is `0`.
    initial_depth: usize,
}

impl<'a, R: 'a + ParserSource> SubtreeParser<'a, R> {
    /// Creates a new `SubtreeParser`.
    pub fn new(root_parser: &'a mut RootParser<R>) -> Self {
        let initial_depth = root_parser.num_open_nodes();
        SubtreeParser {
            root_parser: root_parser,
            initial_depth: initial_depth,
        }
    }

    /// Checks if the subtree parser can emit more events.
    ///
    /// Returns `Ok(())` if more events can be read,
    /// `Err(Error::Finished)` if the subtree is all read,
    /// `Err(_)` if error happened.
    fn check_finished(&self) -> Result<()> {
        if let Some(err) = self.root_parser.error() {
            return Err(err.clone());
        }
        if self.root_parser.num_open_nodes() < self.initial_depth {
            return Err(Error::Finished);
        }
        Ok(())
    }

    /// Parses FBX from the given stream and returns the next event.
    pub fn next_event(&mut self) -> Result<Event<R>> {
        self.check_finished()?;
        self.root_parser.next_event()
    }

    /// Skips to the end of the current node.
    ///
    /// Returns `Ok(true)` if the current node is skipped and closed,
    /// `Ok(false)` if no nodes are open (i.e. the parser is reading under implicit root node),
    /// `Err(err)` if error happened.
    pub fn skip_current_node(&mut self) -> Result<bool> {
        self.check_finished()?;
        self.root_parser.skip_current_node()
    }
}
<commit_msg>Create a trait for FBX parser event emitter types<commit_after>//! FBX binary parser.

use std::io;
use std::io::Read;

pub use self::error::{Result, Error, Warning};
pub use self::event::{Event, FbxHeader, FbxFooter, StartNode};
pub use self::event::{Attributes, Attribute, SpecialAttributeType};
pub use self::event::{PrimitiveAttribute, ArrayAttribute, SpecialAttribute};
pub use self::event::ArrayAttributeReader;
use self::event::{EventBuilder, NodeHeader, StartNodeBuilder};
use self::event::read_fbx_header;
pub use self::reader::{ParserSource, BasicSource, SeekableSource, LimitedSeekReader};

mod error;
mod event;
mod reader;
pub mod utils;

/// Parser of a FBX binary node.
pub trait Parser<R: ParserSource> {
    /// Returns the root parser.
    fn root_parser(&self) -> &RootParser<R>;
    /// Parses FBX from the given stream and returns the next event.
    fn next_event(&mut self) -> Result<Event<R>>;
    /// Skips to the end of the current node.
    ///
    /// Returns `Ok(true)` if the current node is skipped and closed,
    /// `Ok(false)` if no nodes are open (i.e. the parser is reading under implicit root node),
    /// `Err(err)` if error happened.
    fn skip_current_node(&mut self) -> Result<bool>;
}

/// Parser state without error.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
    /// Reading file header.
    Header,
    /// A node started.
    ///
    /// This status means:
    ///
    /// - there might be some attributes remain unread, or
    /// - the most recent opened node might close without no null node header.
    NodeStarted,
    /// A node ended.
    ///
    /// This status means:
    ///
    /// - if the next event is `NodeEnd`, there must be a null node header, and
    /// - if the parser got an extra null header, it indicates end of implicit root node.
    NodeEnded,
}

/// Information about opened (but not yet closed) node.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
struct OpenNode {
    /// Start offset of the node attribute.
    ///
    /// Note that this doesn't mean start offset of node header.
    begin: u64,
    /// End offset of the node.
    end: u64,
    /// End offset of attributes of the node.
    attributes_end: u64,
}

/// Pull parser for whole FBX with binary format.
#[derive(Debug)]
pub struct RootParser<R> {
    /// Source reader.
    source: R,
    /// Parser state.
    ///
    /// `Ok(State)` if the parser is working without critical error,
    /// `Err(Error)` if the parsing failed and cannot be continued.
    state: Result<State>,
    /// Warnings.
    warnings: Vec<Warning>,
    /// FBX version.
    fbx_version: Option<u32>,
    /// Open nodes stack.
    open_nodes: Vec<OpenNode>,
}

impl<R: Read> RootParser<BasicSource<R>> {
    /// Creates a new binary parser.
    pub fn new(source: R) -> Self {
        RootParser {
            source: BasicSource::new(source),
            state: Ok(State::Header),
            warnings: Vec::new(),
            fbx_version: None,
            open_nodes: Vec::new(),
        }
    }
}

impl<R: Read + io::Seek> RootParser<SeekableSource<R>> {
    /// Creates a new binary parser.
    pub fn from_seekable(source: R) -> Self {
        RootParser {
            source: SeekableSource::new(source),
            state: Ok(State::Header),
            warnings: Vec::new(),
            fbx_version: None,
            open_nodes: Vec::new(),
        }
    }
}

impl<R: ParserSource> RootParser<R> {
    /// Returns FBX version of the reading input.
    ///
    /// Returns `None` if unknown yet.
    pub fn fbx_version(&self) -> Option<u32> {
        self.fbx_version
    }

    /// Returns the parser error if available.
    pub fn error(&self) -> Option<&Error> {
        self.state.as_ref().err()
    }

    /// Returns reference to the warnings.
    pub fn warnings(&self) -> &Vec<Warning> {
        &self.warnings
    }

    /// Returns the number of the opened (and not closed) node.
    pub fn num_open_nodes(&self) -> usize {
        self.open_nodes.len()
    }

    /// Set the parser state as finished parsing.
    fn set_finish(&mut self) {
        self.state = Err(Error::Finished);
    }

    /// Set the parser state as error.
    fn set_error(&mut self, err: &Error) {
        error!("FBX binary parser error: {}", err);
        debug!("Parser: {:#?}", self);
        self.state = Err(err.clone());
    }

    /// Add warning.
    fn warn(&mut self, warning: Warning) {
        warn!("FBX binary parser warning: {}", warning);
        debug!("Parser: {:#?}", self);
        self.warnings.push(warning);
    }

    /// Reads FBX header.
    fn read_fbx_header(&mut self) -> Result<EventBuilder> {
        let header = read_fbx_header(self)?;
        self.fbx_version = Some(header.version);
        self.state = Ok(State::NodeEnded);
        Ok(header.into())
    }

    /// Gets event after node start.
    fn read_after_node_start(&mut self) -> Result<EventBuilder> {
        // Attributes of recent opened node might remain partially unread.
        // They should be skipped before getting a next node event.
        self.skip_attributes()?;

        // Most recent opened node might ends here without a null node header.
        if let Some(end) = self.open_nodes.last().map(|v| v.end) {
            if self.source.position() == end {
                // Most recent opened node ends here (without a null node header).
                self.state = Ok(State::NodeEnded);
                self.open_nodes.pop();
                return Ok(EventBuilder::EndNode);
            }
        }

        let builder = self.read_node_event()?;
        Ok(builder)
    }

    /// Gets event after node end.
    fn read_after_node_end(&mut self) -> Result<EventBuilder> {
        self.read_node_event()
    }

    /// Reads a next node-related event from the source.
    ///
    /// This always returns `Ok(EventBuilder::StartNode)`, `Ok(EventBuilder::EndNode)`,
    /// `Ok(EventBuilder::EndFbx)` or `Err(_)`.
    fn read_node_event(&mut self) -> Result<EventBuilder> {
        let header = NodeHeader::read_from_parser(self)?;
        if header.is_node_end() {
            if let Some(last_node) = self.open_nodes.pop() {
                // There is open nodes, so this is not end of the FBX.
                let current_pos = self.source.position();
                if current_pos != last_node.end {
                    // Invalid node header.
                    return Err(Error::WrongNodeEndOffset {
                        begin: last_node.begin,
                        expected_end: last_node.end,
                        real_end: current_pos,
                    });
                }
            } else {
                assert_eq!(self.state.as_ref().ok(),
                           Some(&State::NodeEnded),
                           "End of implicit root node is read with unexpected parser state {:?}",
                           self.state);
                // No open nodes, so this `EndNode` event indicates the end of
                // the implicit root node.
                // FBX file has no more nodes.
                let footer = self.read_fbx_footer();
                return Ok(footer.into());
            }

            self.state = Ok(State::NodeEnded);
            Ok(EventBuilder::EndNode)
        } else {
            let mut name_vec = vec![0u8; header.bytelen_name as usize];
            self.source.read_exact(&mut name_vec)?;
            let name = String::from_utf8(name_vec).map_err(Error::node_name_invalid_utf8)?;
            let current_pos = self.source.position();
            self.open_nodes.push(OpenNode {
                begin: current_pos,
                end: header.end_offset,
                attributes_end: current_pos + header.bytelen_attributes,
            });

            // Zero or more attributes come after node start.
            self.state = Ok(State::NodeStarted);
            Ok(StartNodeBuilder {
                name: name,
                header: header,
            }
            .into())
        }
    }

    /// Reads an FBX footer.
    fn read_fbx_footer(&mut self) -> Result<FbxFooter> {
        self.set_finish();
        FbxFooter::read_from_parser(self)
    }

    /// Skip attributes of the most recent opened node.
    fn skip_attributes(&mut self) -> io::Result<()> {
        let attributes_end = self.open_nodes
            .last()
            .expect("`RootParser::skip_attributes()` is called but no nodes are open")
            .attributes_end;
        self.source.skip_to(attributes_end)
    }
}

impl<R: ParserSource> Parser<R> for RootParser<R> {
    fn root_parser(&self) -> &RootParser<R> {
        self
    }

    fn next_event(&mut self) -> Result<Event<R>> {
        let builder = match self.state.clone()? {
            State::Header => self.read_fbx_header(),
            State::NodeStarted => self.read_after_node_start(),
            State::NodeEnded => self.read_after_node_end(),
        };
        if let Err(ref err) = builder {
            self.set_error(err);
        }
        Ok(builder?.build(self))
    }

    fn skip_current_node(&mut self) -> Result<bool> {
        if let Some(end) = self.open_nodes.pop().map(|v| v.end) {
            self.source.skip_to(end)?;
            self.state = Ok(State::NodeEnded);
            Ok(true)
        } else {
            Ok(false)
        }
    }
}

/// Pull parser for a subtree of the FBX binary.
pub struct SubtreeParser<'a, R: 'a> {
    /// Root parser.
    root_parser: &'a mut RootParser<R>,
    /// Initial depth.
    ///
    /// Depth of the implicit root node is `0`.
    initial_depth: usize,
}

impl<'a, R: 'a + ParserSource> SubtreeParser<'a, R> {
    /// Creates a new `SubtreeParser`.
    pub fn new(root_parser: &'a mut RootParser<R>) -> Self {
        let initial_depth = root_parser.num_open_nodes();
        SubtreeParser {
            root_parser: root_parser,
            initial_depth: initial_depth,
        }
    }

    /// Checks if the subtree parser can emit more events.
    ///
    /// Returns `Ok(())` if more events can be read,
    /// `Err(Error::Finished)` if the subtree is all read,
    /// `Err(_)` if error happened.
    fn check_finished(&self) -> Result<()> {
        if let Some(err) = self.root_parser.error() {
            return Err(err.clone());
        }
        if self.root_parser.num_open_nodes() < self.initial_depth {
            return Err(Error::Finished);
        }
        Ok(())
    }
}

impl<'a, R: 'a + ParserSource> Parser<R> for SubtreeParser<'a, R> {
    fn root_parser(&self) -> &RootParser<R> {
        self.root_parser
    }

    fn next_event(&mut self) -> Result<Event<R>> {
        self.check_finished()?;
        self.root_parser.next_event()
    }

    fn skip_current_node(&mut self) -> Result<bool> {
        self.check_finished()?;
        self.root_parser.skip_current_node()
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! aws ec2 metadata fetcher
//!
use retry;

use metadata::Metadata;

use errors::*;

fn url_for_key(key: &str) -> String {
    format!("http://169.254.169.254/2009-04-04/{}", key)
}

#[allow(non_snake_case)]
#[derive(Debug, Deserialize)]
struct InstanceIdDoc {
    region: String,
}

pub fn fetch_metadata() -> Result<Metadata> {
    let client = retry::Client::new()
        .chain_err(|| format!("ec2: failed to create http client"))?
        .return_on_404(true);

    let instance_id: Option<String> = client.get(retry::Raw, url_for_key("meta-data/instance-id")).send()?;
    let public: Option<String> = client.get(retry::Raw, url_for_key("meta-data/public-ipv4")).send()?;
    let local: Option<String> = client.get(retry::Raw, url_for_key("meta-data/local-ipv4")).send()?;
    let hostname: Option<String> = client.get(retry::Raw, url_for_key("meta-data/hostname")).send()?;
    let availability_zone: Option<String> = client.get(retry::Raw, url_for_key("meta-data/placement/availability-zone")).send()?;
    let region: Option<String> = client.get(retry::Json, url_for_key("dynamic/instance-identity/document")).send()?
        .map(|instance_id_doc: InstanceIdDoc| instance_id_doc.region);

    Ok(Metadata::builder()
        .add_attribute_if_exists("EC2_REGION".to_owned(), region)
        .add_attribute_if_exists("EC2_INSTANCE_ID".to_owned(), instance_id)
        .add_attribute_if_exists("EC2_IPV4_PUBLIC".to_owned(), public)
        .add_attribute_if_exists("EC2_IPV4_LOCAL".to_owned(), local)
        .add_attribute_if_exists("EC2_HOSTNAME".to_owned(), hostname.clone())
        .add_attribute_if_exists("EC2_AVAILABILITY_ZONE".to_owned(), availability_zone)
        .set_hostname_if_exists(hostname)
        .build())
}
<commit_msg>providers/ec2: add ssh keys implementation<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! aws ec2 metadata fetcher
//!
use retry;

use metadata::Metadata;

use errors::*;

fn url_for_key(key: &str) -> String {
    format!("http://169.254.169.254/2009-04-04/{}", key)
}

#[allow(non_snake_case)]
#[derive(Debug, Deserialize)]
struct InstanceIdDoc {
    region: String,
}

pub fn fetch_metadata() -> Result<Metadata> {
    let client = retry::Client::new()
        .chain_err(|| format!("ec2: failed to create http client"))?
        .return_on_404(true);

    let instance_id: Option<String> = client.get(retry::Raw, url_for_key("meta-data/instance-id")).send()?;
    let public: Option<String> = client.get(retry::Raw, url_for_key("meta-data/public-ipv4")).send()?;
    let local: Option<String> = client.get(retry::Raw, url_for_key("meta-data/local-ipv4")).send()?;
    let hostname: Option<String> = client.get(retry::Raw, url_for_key("meta-data/hostname")).send()?;
    let availability_zone: Option<String> = client.get(retry::Raw, url_for_key("meta-data/placement/availability-zone")).send()?;
    let region: Option<String> = client.get(retry::Json, url_for_key("dynamic/instance-identity/document")).send()?
        .map(|instance_id_doc: InstanceIdDoc| instance_id_doc.region);

    let mut ssh_keys: Vec<String> = fetch_ssh_keys(client)?;

    Ok(Metadata::builder()
        .add_attribute_if_exists("EC2_REGION".to_owned(), region)
        .add_attribute_if_exists("EC2_INSTANCE_ID".to_owned(), instance_id)
        .add_attribute_if_exists("EC2_IPV4_PUBLIC".to_owned(), public)
        .add_attribute_if_exists("EC2_IPV4_LOCAL".to_owned(), local)
        .add_attribute_if_exists("EC2_HOSTNAME".to_owned(), hostname.clone())
        .add_attribute_if_exists("EC2_AVAILABILITY_ZONE".to_owned(), availability_zone)
        .set_hostname_if_exists(hostname)
        .add_ssh_keys(ssh_keys)
        .build())
}

fn fetch_ssh_keys(client: retry::Client) -> Result<Vec<String>> {
    let keydata: Option<String> = client.get(retry::Raw, url_for_key("meta-data/public-keys")).send()?;
    match keydata {
        Some(s) => {
            let mut res = Ok(Vec::new());
            for keyname in s.split_whitespace() {
                let tokens: Vec<&str> = keyname.split('=').collect();
                if tokens.len() != 2 {
                    res = Err("malformed public key".into());
                    break;
                }
                let ssh_key: String = client.get(retry::Raw, url_for_key(&format!("meta-data/public-keys/{}/openssh-key", tokens[0]))).send()?
                    .ok_or("missing ssh key")?;
                match res.as_mut() {
                    Ok(keys) => (*keys).push(ssh_key),
                    Err(_) => (),
                }
            }
            res
        }
        None => {
            Ok(Vec::new())
        }
    }
}
<|endoftext|>


<commit_before>use rand::Rng;
use std::fmt;

/// Simple implementation of [Reservoir Sampling](https://en.wikipedia.org/wiki/Reservoir_sampling)
/// with [fast approximation](https://erikerlandson.github.io/blog/2015/11/20/very-fast-reservoir-sampling/)
pub struct ReservoirSampling<T, R>
where
    R: Rng,
{
    k: usize,
    rng: R,
    reservoir: Vec<T>,
    i: usize,
    skip_until: usize,
}

impl<T, R> ReservoirSampling<T, R>
where
    R: Rng,
{
    /// Create new reservoir sampler that keeps `k` samples and uses `rng` for its random
    /// decisions.
    pub fn new(k: usize, rng: R) -> ReservoirSampling<T, R> {
        ReservoirSampling {
            k: k,
            rng: rng,
            reservoir: vec![],
            i: 0,
            skip_until: 0,
        }
    }

    /// Number of samples that should be kept.
    pub fn k(&self) -> usize {
        self.k
    }

    /// Read-only copy of the reservoir. Contains at most `k` entries.
    pub fn reservoir(&self) -> &Vec<T> {
        &self.reservoir
    }

    /// Number of data points seen.
    pub fn i(&self) -> usize {
        self.i
    }

    /// Observe new data point.
    pub fn add(&mut self, obj: T) {
        let t = self.k * 4; // TODO: make this a parameter

        if self.i < self.k {
            // initial fill-up
            self.reservoir.push(obj)
        } else if self.i < t {
            // normal reservoir sampling
            let j = self.rng.gen_range::<usize>(0, self.i);
            if j < self.k {
                self.reservoir[j] = obj;
            }
        } else if self.i >= self.skip_until {
            // fast skipping approximation
            let j = self.rng.gen_range::<usize>(0, self.k);
            self.reservoir[j] = obj;

            // calculate next skip
            let p = (self.k as f64) / ((self.i + 1) as f64);
            let u = 1. - self.rng.gen_range::<f64>(0., 1.); // (0.0, 1.0]
            let g = (u.ln() / (1. - p).ln()).floor() as usize;
            self.skip_until = self.i + g;
        }

        self.i += 1;
    }

    /// Checks if reservoir is empty (i.e. no data points where observed)
    pub fn is_empty(&self) -> bool {
        self.i == 0
    }
}

impl<T, R> fmt::Debug for ReservoirSampling<T, R>
where
    R: Rng,
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "ReservoirSampling {{ k: {} }}", self.k)
    }
}

#[cfg(test)]
mod tests {
    use super::ReservoirSampling;
    use rand::ChaChaRng;

    #[test]
    fn getter() {
        let rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        assert_eq!(rs.k(), 10);
        assert_eq!(rs.i(), 0);
        assert_eq!(rs.reservoir(), &vec![]);
    }

    #[test]
    fn empty() {
        let rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        assert!(rs.is_empty());
        assert!(rs.reservoir().is_empty());
    }

    #[test]
    fn add_k() {
        let mut rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        for i in 0..10 {
            rs.add(i);
        }
        assert_eq!(rs.i(), 10);
        assert_eq!(rs.reservoir(), &vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
    }

    #[test]
    fn add_parts() {
        let mut rs = ReservoirSampling::<u64, ChaChaRng>::new(100, ChaChaRng::new_unseeded());
        for _ in 0..1500 {
            rs.add(0);
        }
        for _ in 0..7500 {
            rs.add(1);
        }
        for _ in 0..1000 {
            rs.add(0);
        }
        assert!((rs.reservoir().iter().sum::<u64>() as i64 - 75).abs() < 5);
    }

    #[test]
    fn debug() {
        let rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        assert_eq!(format!("{:?}", rs), "ReservoirSampling { k: 10 }");
    }
}
<commit_msg>reservoirsampling: implement clear method<commit_after>use rand::Rng;
use std::fmt;

/// Simple implementation of [Reservoir Sampling](https://en.wikipedia.org/wiki/Reservoir_sampling)
/// with [fast approximation](https://erikerlandson.github.io/blog/2015/11/20/very-fast-reservoir-sampling/)
pub struct ReservoirSampling<T, R>
where
    R: Rng,
{
    k: usize,
    rng: R,
    reservoir: Vec<T>,
    i: usize,
    skip_until: usize,
}

impl<T, R> ReservoirSampling<T, R>
where
    R: Rng,
{
    /// Create new reservoir sampler that keeps `k` samples and uses `rng` for its random
    /// decisions.
    pub fn new(k: usize, rng: R) -> ReservoirSampling<T, R> {
        ReservoirSampling {
            k: k,
            rng: rng,
            reservoir: vec![],
            i: 0,
            skip_until: 0,
        }
    }

    /// Number of samples that should be kept.
    pub fn k(&self) -> usize {
        self.k
    }

    /// Read-only copy of the reservoir. Contains at most `k` entries.
    pub fn reservoir(&self) -> &Vec<T> {
        &self.reservoir
    }

    /// Number of data points seen.
    pub fn i(&self) -> usize {
        self.i
    }

    /// Observe new data point.
    pub fn add(&mut self, obj: T) {
        let t = self.k * 4; // TODO: make this a parameter

        if self.i < self.k {
            // initial fill-up
            self.reservoir.push(obj)
        } else if self.i < t {
            // normal reservoir sampling
            let j = self.rng.gen_range::<usize>(0, self.i);
            if j < self.k {
                self.reservoir[j] = obj;
            }
        } else if self.i >= self.skip_until {
            // fast skipping approximation
            let j = self.rng.gen_range::<usize>(0, self.k);
            self.reservoir[j] = obj;

            // calculate next skip
            let p = (self.k as f64) / ((self.i + 1) as f64);
            let u = 1. - self.rng.gen_range::<f64>(0., 1.); // (0.0, 1.0]
            let g = (u.ln() / (1. - p).ln()).floor() as usize;
            self.skip_until = self.i + g;
        }

        self.i += 1;
    }

    /// Checks if reservoir is empty (i.e. no data points where observed)
    pub fn is_empty(&self) -> bool {
        self.i == 0
    }

    /// Clear sampler state. It now behaves like a fresh one (i.e. no data points seen so far).
    pub fn clear(&mut self) {
        self.i = 0;
        self.skip_until = 0;
        self.reservoir.clear();
    }
}

impl<T, R> fmt::Debug for ReservoirSampling<T, R>
where
    R: Rng,
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "ReservoirSampling {{ k: {} }}", self.k)
    }
}

#[cfg(test)]
mod tests {
    use super::ReservoirSampling;
    use rand::ChaChaRng;

    #[test]
    fn getter() {
        let rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        assert_eq!(rs.k(), 10);
        assert_eq!(rs.i(), 0);
        assert_eq!(rs.reservoir(), &vec![]);
    }

    #[test]
    fn empty() {
        let rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        assert!(rs.is_empty());
        assert!(rs.reservoir().is_empty());
    }

    #[test]
    fn add_k() {
        let mut rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        for i in 0..10 {
            rs.add(i);
        }
        assert_eq!(rs.i(), 10);
        assert_eq!(rs.reservoir(), &vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
    }

    #[test]
    fn add_parts() {
        let mut rs = ReservoirSampling::<u64, ChaChaRng>::new(100, ChaChaRng::new_unseeded());
        for _ in 0..1500 {
            rs.add(0);
        }
        for _ in 0..7500 {
            rs.add(1);
        }
        for _ in 0..1000 {
            rs.add(0);
        }
        assert!((rs.reservoir().iter().sum::<u64>() as i64 - 75).abs() < 5);
    }

    #[test]
    fn debug() {
        let rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        assert_eq!(format!("{:?}", rs), "ReservoirSampling { k: 10 }");
    }

    #[test]
    fn add_clear() {
        let mut rs = ReservoirSampling::<u64, ChaChaRng>::new(10, ChaChaRng::new_unseeded());
        for i in 0..10 {
            rs.add(i);
        }
        rs.clear();
        for i in 10..20 {
            rs.add(i);
        }
        assert_eq!(rs.i(), 10);
        assert_eq!(
            rs.reservoir(),
            &vec![10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
        );
    }
}
<|endoftext|>


<commit_before>use crate::lib_configs::{
    global_config, MOQT_INSTALL_DIR_ENV_VAR_NAME, MOQT_TEMPLATE_DIR_ENV_VAR_NAME,
};
use ritual::cli::{self, Options};
use ritual_common::cpp_lib_builder::{BuildType, CppLibBuilder};
use ritual_common::env_var_names;
use ritual_common::errors::{FancyUnwrap, Result};
use ritual_common::file_utils::{
    canonicalize, copy_recursively, create_dir, create_dir_all, create_file, file_to_string,
    read_dir, remove_dir_all, repo_dir_path,
};
use ritual_common::utils::add_env_path_item;
use std::env;
use std::io::Write;
use std::path::{Path, PathBuf};

#[derive(Debug)]
pub enum TempTestDir {
    System(::tempdir::TempDir),
    Custom(PathBuf),
}

impl TempTestDir {
    pub fn new(name: &str) -> TempTestDir {
        if let Ok(value) = ::std::env::var("RITUAL_TEMP_TEST_DIR") {
            let path = PathBuf::from(value);
            create_dir_all(&path).unwrap();
            let path = canonicalize(path).unwrap().join(name);
            create_dir_all(&path).unwrap();
            TempTestDir::Custom(path)
        } else {
            TempTestDir::System(::tempdir::TempDir::new(name).unwrap())
        }
    }

    pub fn path(&self) -> &Path {
        match self {
            TempTestDir::System(dir) => dir.path(),
            TempTestDir::Custom(path) => path,
        }
    }
}

fn build_cpp_lib() -> Result<TempTestDir> {
    let cpp_lib_source_dir = repo_dir_path("qt_ritual/test_assets/moqt")?;

    let temp_dir = TempTestDir::new("test_full_run");
    let build_dir = temp_dir.path().join("build");
    let install_dir = temp_dir.path().join("install");
    let template_dir = temp_dir.path().join("template");
    create_dir_all(&build_dir)?;
    create_dir_all(&install_dir)?;
    remove_dir_all(&template_dir)?;
    create_dir_all(&template_dir)?;
    CppLibBuilder {
        cmake_source_dir: cpp_lib_source_dir,
        build_dir,
        build_type: BuildType::Release,
        install_dir: Some(install_dir.clone()),
        num_jobs: None,
        cmake_vars: Vec::new(),
        capture_output: false,
        skip_cmake: false,
        skip_cmake_after_first_run: false,
    }
    .run()?;

    let repo_template_dir = repo_dir_path("qt_ritual/crate_templates")?;
    let add_env = |name, path: &Path| -> Result<()> {
        let value = add_env_path_item(name, vec![path.to_path_buf()])?;
        env::set_var(name, value);
        Ok(())
    };

    for lib in &["moqt_core", "moqt_gui"] {
        let path = install_dir.join("include").join(lib);
        add_env("CPLUS_INCLUDE_PATH", &path)?;
        add_env("INCLUDE", &path)?; // for Windows

        copy_recursively(&repo_template_dir.join(lib), &template_dir.join(lib))?;
        let real_lib = lib.replace("mo", "");
        let real_template_src_dir = repo_template_dir.join(&real_lib).join("src");
        let template_src_dir = template_dir.join(&lib).join("src");
        create_dir(&template_src_dir)?;
        for item in read_dir(&real_template_src_dir)? {
            let item = item?;
            let content = file_to_string(item.path())?;
            let new_content = content
                .replace("qt_core", "moqt_core")
                .replace("qt_gui", "moqt_gui");
            let mut file = create_file(template_src_dir.join(item.file_name()))?;
            write!(file, "{}", new_content)?;
        }
    }
    let lib_path = install_dir.join("lib");
    add_env("LIBRARY_PATH", &lib_path)?;
    add_env("LD_LIBRARY_PATH", &lib_path)?;
    add_env("DYLD_LIBRARY_PATH", &lib_path)?;
    add_env("PATH", &lib_path)?;
    add_env(env_var_names::LIBRARY_PATH, &lib_path)?;
    env::set_var(MOQT_INSTALL_DIR_ENV_VAR_NAME, &install_dir);
    env::set_var(MOQT_TEMPLATE_DIR_ENV_VAR_NAME, &template_dir);
    Ok(temp_dir)
}

#[test]
fn test_moqt() {
    let temp_dir = build_cpp_lib().fancy_unwrap();
    let workspace = temp_dir.path().join("workspace");
    create_dir_all(&workspace).unwrap();

    cli::run(
        Options {
            workspace,
            local_paths: Some(true),
            crates: vec!["moqt_core".into(), "moqt_gui".into()],
            operations: vec!["discard".into(), "main".into()],
            cluster: None,
        },
        global_config(),
    )
    .fancy_unwrap();
}
<commit_msg>fix moqt test<commit_after>use crate::lib_configs::{
    global_config, MOQT_INSTALL_DIR_ENV_VAR_NAME, MOQT_TEMPLATE_DIR_ENV_VAR_NAME,
};
use ritual::cli::{self, Options};
use ritual_common::cpp_lib_builder::{BuildType, CppLibBuilder};
use ritual_common::env_var_names;
use ritual_common::errors::{FancyUnwrap, Result};
use ritual_common::file_utils::{
    canonicalize, copy_recursively, create_dir, create_dir_all, create_file, file_to_string,
    read_dir, remove_dir_all, repo_dir_path,
};
use ritual_common::utils::add_env_path_item;
use std::env;
use std::io::Write;
use std::path::{Path, PathBuf};

#[derive(Debug)]
pub enum TempTestDir {
    System(::tempdir::TempDir),
    Custom(PathBuf),
}

impl TempTestDir {
    pub fn new(name: &str) -> TempTestDir {
        if let Ok(value) = ::std::env::var("RITUAL_TEMP_TEST_DIR") {
            let path = PathBuf::from(value);
            create_dir_all(&path).unwrap();
            let path = canonicalize(path).unwrap().join(name);
            create_dir_all(&path).unwrap();
            TempTestDir::Custom(path)
        } else {
            TempTestDir::System(::tempdir::TempDir::new(name).unwrap())
        }
    }

    pub fn path(&self) -> &Path {
        match self {
            TempTestDir::System(dir) => dir.path(),
            TempTestDir::Custom(path) => path,
        }
    }
}

fn build_cpp_lib() -> Result<TempTestDir> {
    let cpp_lib_source_dir = repo_dir_path("qt_ritual/test_assets/moqt")?;

    let temp_dir = TempTestDir::new("test_full_run");
    let build_dir = temp_dir.path().join("build");
    let install_dir = temp_dir.path().join("install");
    let template_dir = temp_dir.path().join("template");
    create_dir_all(&build_dir)?;
    create_dir_all(&install_dir)?;
    if template_dir.exists() {
        remove_dir_all(&template_dir)?;
    }
    create_dir_all(&template_dir)?;
    CppLibBuilder {
        cmake_source_dir: cpp_lib_source_dir,
        build_dir,
        build_type: BuildType::Release,
        install_dir: Some(install_dir.clone()),
        num_jobs: None,
        cmake_vars: Vec::new(),
        capture_output: false,
        skip_cmake: false,
        skip_cmake_after_first_run: false,
    }
    .run()?;

    let repo_template_dir = repo_dir_path("qt_ritual/crate_templates")?;
    let add_env = |name, path: &Path| -> Result<()> {
        let value = add_env_path_item(name, vec![path.to_path_buf()])?;
        env::set_var(name, value);
        Ok(())
    };

    for lib in &["moqt_core", "moqt_gui"] {
        let path = install_dir.join("include").join(lib);
        add_env("CPLUS_INCLUDE_PATH", &path)?;
        add_env("INCLUDE", &path)?; // for Windows

        copy_recursively(&repo_template_dir.join(lib), &template_dir.join(lib))?;
        let real_lib = lib.replace("mo", "");
        let real_template_src_dir = repo_template_dir.join(&real_lib).join("src");
        let template_src_dir = template_dir.join(&lib).join("src");
        create_dir(&template_src_dir)?;
        for item in read_dir(&real_template_src_dir)? {
            let item = item?;
            let content = file_to_string(item.path())?;
            let new_content = content
                .replace("qt_core", "moqt_core")
                .replace("qt_gui", "moqt_gui");
            let mut file = create_file(template_src_dir.join(item.file_name()))?;
            write!(file, "{}", new_content)?;
        }
    }
    let lib_path = install_dir.join("lib");
    add_env("LIBRARY_PATH", &lib_path)?;
    add_env("LD_LIBRARY_PATH", &lib_path)?;
    add_env("DYLD_LIBRARY_PATH", &lib_path)?;
    add_env("PATH", &lib_path)?;
    add_env(env_var_names::LIBRARY_PATH, &lib_path)?;
    env::set_var(MOQT_INSTALL_DIR_ENV_VAR_NAME, &install_dir);
    env::set_var(MOQT_TEMPLATE_DIR_ENV_VAR_NAME, &template_dir);
    Ok(temp_dir)
}

#[test]
fn test_moqt() {
    let temp_dir = build_cpp_lib().fancy_unwrap();
    let workspace = temp_dir.path().join("workspace");
    create_dir_all(&workspace).unwrap();

    cli::run(
        Options {
            workspace,
            local_paths: Some(true),
            crates: vec!["moqt_core".into(), "moqt_gui".into()],
            operations: vec!["discard".into(), "main".into()],
            cluster: None,
        },
        global_config(),
    )
    .fancy_unwrap();
}
<|endoftext|>


<commit_before>use super::{
    directory_stack::DirectoryStack,
    escape::{escape, unescape},
    variables::Variables,
};
use glob::glob;
use liner::{Completer, FilenameCompleter};
use smallvec::SmallVec;
use std::{iter, str};

/// Performs escaping to an inner `FilenameCompleter` to enable a handful of special cases
/// needed by the shell, such as expanding '~' to a home directory, or adding a backslash
/// when a special character is contained within an expanded filename.
pub(crate) struct IonFileCompleter {
    /// The completer that this completer is  handling.
    inner: FilenameCompleter,
    /// A pointer to the directory stack in the shell.
    dir_stack: *const DirectoryStack,
    /// A pointer to the variables map in the shell.
    vars: *const Variables,
}

impl IonFileCompleter {
    pub(crate) fn new(
        path: Option<&str>,
        dir_stack: *const DirectoryStack,
        vars: *const Variables,
    ) -> IonFileCompleter {
        IonFileCompleter {
            inner: FilenameCompleter::new(path),
            dir_stack,
            vars,
        }
    }
}

impl Completer for IonFileCompleter {
    /// When the tab key is pressed, **Liner** will use this method to perform completions of
    /// filenames. As our `IonFileCompleter` is a wrapper around **Liner**'s
    /// `FilenameCompleter`,
    /// the purpose of our custom `Completer` is to expand possible `~` characters in the
    /// `start`
    /// value that we receive from the prompt, grab completions from the inner
    /// `FilenameCompleter`,
    /// and then escape the resulting filenames, as well as remove the expanded form of the `~`
    /// character and re-add the `~` character in it's place.
    fn completions(&self, start: &str) -> Vec<String> {
        // Only if the first character is a tilde character will we perform expansions
        if start.starts_with('~') {
            // Dereferencing the raw pointers here should be entirely safe, theoretically,
            // because no changes will occur to either of the underlying references in the
            // duration between creation of the completers and execution of their
            // completions.
            if let Some(expanded) = unsafe { (*self.vars).tilde_expansion(start, &*self.dir_stack) }
            {
                // Now we obtain completions for the `expanded` form of the `start` value.
                let mut iterator = filename_completion(&expanded, |x| self.inner.completions(x));

                // And then we will need to take those completions and remove the expanded form
                // of the tilde pattern and replace it with that pattern yet again.
                let mut completions = Vec::new();

                // We can do that by obtaining the index position where the tilde character
                // ends. We don't search with `~` because we also want to
                // handle other tilde variants.
                let t_index = start.find('/').unwrap_or(1);
                // `tilde` is the tilde pattern, and `search` is the pattern that follows.
                let (tilde, search) = start.split_at(t_index as usize);

                if search.len() < 2 {
                    // If the length of the search pattern is less than 2, the search pattern is
                    // empty, and thus the completions actually contain files and directories in
                    // the home directory.

                    // The tilde pattern will actually be our `start` command in itself,
                    // and the completed form will be all of the characters beyond the length of
                    // the expanded form of the tilde pattern.
                    for completion in iterator {
                        completions.push([start, &completion[expanded.len()..]].concat());
                    }
                // To save processing time, we should get obtain the index position where our
                // search pattern begins, and re-use that index to slice the completions so
                // that we may re-add the tilde character with the completion that follows.
                } else if let Some(completion) = iterator.next() {
                    if let Some(e_index) = expanded.rfind(search) {
                        completions.push(escape(&[tilde, &completion[e_index..]].concat()));
                        for completion in iterator {
                            let expanded = &completion[e_index..];
                            completions.push(escape(&[tilde, expanded].concat()));
                        }
                    }
                }

                return completions;
            }
        }

        filename_completion(&start, |x| self.inner.completions(x)).collect()
    }
}

fn filename_completion<'a, LC>(
    start: &'a str,
    liner_complete: LC,
) -> impl Iterator<Item = String> + 'a
where
    LC: Fn(&str) -> Vec<String> + 'a,
{
    let unescaped_start = unescape(start);

    let split_start = unescaped_start.split('/');
    let mut string: SmallVec<[u8; 128]> = SmallVec::with_capacity(128);

    // When 'start' is an absolute path, "/..." gets split to ["", "..."]
    // So we skip the first element and add "/" to the start of the string
    let skip = if unescaped_start.starts_with('/') {
        string.push(b'/');
        1
    } else {
        0
    };

    for element in split_start.skip(skip) {
        string.extend_from_slice(element.as_bytes());
        string.extend_from_slice(b"*/");
    }

    string.pop(); // pop out the last '/' character
    let string = unsafe { &str::from_utf8_unchecked(&string) };

    let globs = glob(string).ok().and_then(|completions| {
        let mut completions = completions
            .filter_map(Result::ok)
            .map(|x| x.to_string_lossy().into_owned());

        if let Some(first) = completions.next() {
            Some(iter::once(first).chain(completions))
        } else {
            None
        }
    });

    let iter_inner_glob: Box<Iterator<Item = String>> = match globs {
        Some(iter) => Box::new(iter),
        None => Box::new(iter::once(escape(start))),
    };

    // Use Liner::Completer as well, to preserve the previous behaviour
    // around single-directory completions
    iter_inner_glob.flat_map(move |path| {
        liner_complete(&path)
            .into_iter()
            .map(|x| escape(x.as_str()))
    })
}

/// A completer that combines suggestions from multiple completers.
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct MultiCompleter<A, B>
where
    A: Completer,
    B: Completer,
{
    a: Vec<A>,
    b: B,
}

impl<A, B> MultiCompleter<A, B>
where
    A: Completer,
    B: Completer,
{
    pub(crate) fn new(a: Vec<A>, b: B) -> MultiCompleter<A, B> { MultiCompleter { a, b } }
}

impl<A, B> Completer for MultiCompleter<A, B>
where
    A: Completer,
    B: Completer,
{
    fn completions(&self, start: &str) -> Vec<String> {
        let mut completions = self.b.completions(start);
        for x in &self.a {
            completions.extend_from_slice(&x.completions(start));
        }
        completions
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::env;

    #[test]
    fn filename_completion() {
        let current_dir = env::current_dir().expect("Unable to get current directory");

        let completer = IonFileCompleter::new(
            current_dir.to_str(),
            &DirectoryStack::new(),
            &Variables::default(),
        );
        assert_eq!(completer.completions("testing"), vec!["testing/"]);
        assert_eq!(
            completer.completions("testing/file"),
            vec!["testing/file_with_text"]
        );

        assert_eq!(completer.completions("~"), vec!["~/"]);

        assert_eq!(
            completer.completions("tes/fil"),
            vec!["testing/file_with_text"]
        );
    }
}
<commit_msg>Fix glob expansion bug and refactor<commit_after>use super::{
    directory_stack::DirectoryStack,
    escape::{escape, unescape},
    variables::Variables,
};
use glob::glob;
use liner::{Completer, FilenameCompleter};
use smallvec::SmallVec;
use std::{iter, str};

/// Performs escaping to an inner `FilenameCompleter` to enable a handful of special cases
/// needed by the shell, such as expanding '~' to a home directory, or adding a backslash
/// when a special character is contained within an expanded filename.
pub(crate) struct IonFileCompleter {
    /// The completer that this completer is  handling.
    inner: FilenameCompleter,
    /// A pointer to the directory stack in the shell.
    dir_stack: *const DirectoryStack,
    /// A pointer to the variables map in the shell.
    vars: *const Variables,
}

impl IonFileCompleter {
    pub(crate) fn new(
        path: Option<&str>,
        dir_stack: *const DirectoryStack,
        vars: *const Variables,
    ) -> IonFileCompleter {
        IonFileCompleter {
            inner: FilenameCompleter::new(path),
            dir_stack,
            vars,
        }
    }
}

impl Completer for IonFileCompleter {
    /// When the tab key is pressed, **Liner** will use this method to perform completions of
    /// filenames. As our `IonFileCompleter` is a wrapper around **Liner**'s
    /// `FilenameCompleter`,
    /// the purpose of our custom `Completer` is to expand possible `~` characters in the
    /// `start`
    /// value that we receive from the prompt, grab completions from the inner
    /// `FilenameCompleter`,
    /// and then escape the resulting filenames, as well as remove the expanded form of the `~`
    /// character and re-add the `~` character in it's place.
    fn completions(&self, start: &str) -> Vec<String> {
        // Only if the first character is a tilde character will we perform expansions
        if start.starts_with('~') {
            // Dereferencing the raw pointers here should be entirely safe, theoretically,
            // because no changes will occur to either of the underlying references in the
            // duration between creation of the completers and execution of their
            // completions.
            if let Some(expanded) = unsafe { (*self.vars).tilde_expansion(start, &*self.dir_stack) }
            {
                // Now we obtain completions for the `expanded` form of the `start` value.
                let mut iterator = filename_completion(&expanded, |x| self.inner.completions(x));

                // And then we will need to take those completions and remove the expanded form
                // of the tilde pattern and replace it with that pattern yet again.
                let mut completions = Vec::new();

                // We can do that by obtaining the index position where the tilde character
                // ends. We don't search with `~` because we also want to
                // handle other tilde variants.
                let t_index = start.find('/').unwrap_or(1);
                // `tilde` is the tilde pattern, and `search` is the pattern that follows.
                let (tilde, search) = start.split_at(t_index as usize);

                if search.len() < 2 {
                    // If the length of the search pattern is less than 2, the search pattern is
                    // empty, and thus the completions actually contain files and directories in
                    // the home directory.

                    // The tilde pattern will actually be our `start` command in itself,
                    // and the completed form will be all of the characters beyond the length of
                    // the expanded form of the tilde pattern.
                    for completion in iterator {
                        completions.push([start, &completion[expanded.len()..]].concat());
                    }
                // To save processing time, we should get obtain the index position where our
                // search pattern begins, and re-use that index to slice the completions so
                // that we may re-add the tilde character with the completion that follows.
                } else if let Some(completion) = iterator.next() {
                    if let Some(e_index) = expanded.rfind(search) {
                        completions.push(escape(&[tilde, &completion[e_index..]].concat()));
                        for completion in iterator {
                            let expanded = &completion[e_index..];
                            completions.push(escape(&[tilde, expanded].concat()));
                        }
                    }
                }

                return completions;
            }
        }

        filename_completion(&start, |x| self.inner.completions(x)).collect()
    }
}

fn filename_completion<'a, LC>(
    start: &'a str,
    liner_complete: LC,
) -> impl Iterator<Item = String> + 'a
where
    LC: Fn(&str) -> Vec<String> + 'a,
{
    let unescaped_start = unescape(start);

    let split_start = unescaped_start.split('/');
    let mut string: SmallVec<[u8; 128]> = SmallVec::with_capacity(128);

    // When 'start' is an absolute path, "/..." gets split to ["", "..."]
    // So we skip the first element and add "/" to the start of the string
    let skip = if unescaped_start.starts_with('/') {
        string.push(b'/');
        1
    } else {
        0
    };

    for element in split_start.skip(skip) {
        string.extend_from_slice(element.as_bytes());

        if element != ".." && element != "." {
            string.extend_from_slice(b"*/");
        }
    }

    string.pop(); // pop out the last '/' character
    let string = unsafe { &str::from_utf8_unchecked(&string) };

    let globs = glob(string).ok().and_then(|completions| {
        let mut completions = completions
            .filter_map(Result::ok)
            .map(|x| x.to_string_lossy().into_owned())
            .peekable();

        if completions.peek().is_some() {
            Some(completions)
        } else {
            None
        }
    });

    let iter_inner_glob: Box<Iterator<Item = String>> = match globs {
        Some(iter) => Box::new(iter),
        None => Box::new(iter::once(escape(start))),
    };

    // Use Liner::Completer as well, to preserve the previous behaviour
    // around single-directory completions
    iter_inner_glob.flat_map(move |path| {
        liner_complete(&path)
            .into_iter()
            .map(|x| escape(x.as_str()))
    })
}

/// A completer that combines suggestions from multiple completers.
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct MultiCompleter<A, B>
where
    A: Completer,
    B: Completer,
{
    a: Vec<A>,
    b: B,
}

impl<A, B> MultiCompleter<A, B>
where
    A: Completer,
    B: Completer,
{
    pub(crate) fn new(a: Vec<A>, b: B) -> MultiCompleter<A, B> { MultiCompleter { a, b } }
}

impl<A, B> Completer for MultiCompleter<A, B>
where
    A: Completer,
    B: Completer,
{
    fn completions(&self, start: &str) -> Vec<String> {
        let mut completions = self.b.completions(start);
        for x in &self.a {
            completions.extend_from_slice(&x.completions(start));
        }
        completions
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::env;

    #[test]
    fn filename_completion() {
        let current_dir = env::current_dir().expect("Unable to get current directory");

        let completer = IonFileCompleter::new(
            current_dir.to_str(),
            &DirectoryStack::new(),
            &Variables::default(),
        );
        assert_eq!(completer.completions("testing"), vec!["testing/"]);
        assert_eq!(
            completer.completions("testing/file"),
            vec!["testing/file_with_text"]
        );

        assert_eq!(completer.completions("~"), vec!["~/"]);

        assert_eq!(
            completer.completions("tes/fil"),
            vec!["testing/file_with_text"]
        );
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use middle::def_id::DefId;
use middle::privacy::LastPrivate;
use middle::subst::ParamSpace;
use util::nodemap::NodeMap;
use syntax::ast;
use rustc_front::hir;

#[derive(Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
pub enum Def {
    Fn(DefId),
    SelfTy(Option<DefId>,                    // trait id
              Option<(ast::NodeId, ast::NodeId)>),   // (impl id, self type id)
    Mod(DefId),
    ForeignMod(DefId),
    Static(DefId, bool /* is_mutbl */),
    Const(DefId),
    AssociatedConst(DefId),
    Local(DefId, // def id of variable
             ast::NodeId), // node id of variable
    Variant(DefId /* enum */, DefId /* variant */),
    Enum(DefId),
    TyAlias(DefId),
    AssociatedTy(DefId /* trait */, DefId),
    Trait(DefId),
    PrimTy(hir::PrimTy),
    TyParam(ParamSpace, u32, DefId, ast::Name),
    Upvar(DefId,        // def id of closed over local
             ast::NodeId,  // node id of closed over local
             usize,        // index in the freevars list of the closure
             ast::NodeId), // expr node that creates the closure

    // If Def::Struct lives in type namespace it denotes a struct item and its DefId refers
    // to NodeId of the struct itself.
    // If Def::Struct lives in value namespace (e.g. tuple struct, unit struct expressions)
    // it denotes a constructor and its DefId refers to NodeId of the struct's constructor.
    Struct(DefId),
    Label(ast::NodeId),
    Method(DefId),
    Err,
}

/// The result of resolving a path.
/// Before type checking completes, `depth` represents the number of
/// trailing segments which are yet unresolved. Afterwards, if there
/// were no errors, all paths should be fully resolved, with `depth`
/// set to `0` and `base_def` representing the final resolution.
///
///     module::Type::AssocX::AssocY::MethodOrAssocType
///     ^~~~~~~~~~~~  ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
///     base_def      depth = 3
///
///     <T as Trait>::AssocX::AssocY::MethodOrAssocType
///           ^~~~~~~~~~~~~~  ^~~~~~~~~~~~~~~~~~~~~~~~~
///           base_def        depth = 2
#[derive(Copy, Clone, Debug)]
pub struct PathResolution {
    pub base_def: Def,
    pub last_private: LastPrivate,
    pub depth: usize
}

impl PathResolution {
    /// Get the definition, if fully resolved, otherwise panic.
    pub fn full_def(&self) -> Def {
        if self.depth != 0 {
            panic!("path not fully resolved: {:?}", self);
        }
        self.base_def
    }

    /// Get the DefId, if fully resolved, otherwise panic.
    pub fn def_id(&self) -> DefId {
        self.full_def().def_id()
    }

    pub fn new(base_def: Def,
               last_private: LastPrivate,
               depth: usize)
               -> PathResolution {
        PathResolution {
            base_def: base_def,
            last_private: last_private,
            depth: depth,
        }
    }
}

// Definition mapping
pub type DefMap = NodeMap<PathResolution>;
// This is the replacement export map. It maps a module to all of the exports
// within.
pub type ExportMap = NodeMap<Vec<Export>>;

#[derive(Copy, Clone)]
pub struct Export {
    pub name: ast::Name,    // The name of the target.
    pub def_id: DefId, // The definition of the target.
}

impl Def {
    pub fn var_id(&self) -> ast::NodeId {
        match *self {
            Def::Local(_, id) |
            Def::Upvar(_, id, _, _) => {
                id
            }

            Def::Fn(..) | Def::Mod(..) | Def::ForeignMod(..) | Def::Static(..) |
            Def::Variant(..) | Def::Enum(..) | Def::TyAlias(..) | Def::AssociatedTy(..) |
            Def::TyParam(..) | Def::Struct(..) | Def::Trait(..) |
            Def::Method(..) | Def::Const(..) | Def::AssociatedConst(..) |
            Def::PrimTy(..) | Def::Label(..) | Def::SelfTy(..) | Def::Err => {
                panic!("attempted .def_id() on invalid {:?}", self)
            }
        }
    }

    pub fn def_id(&self) -> DefId {
        match *self {
            Def::Fn(id) | Def::Mod(id) | Def::ForeignMod(id) | Def::Static(id, _) |
            Def::Variant(_, id) | Def::Enum(id) | Def::TyAlias(id) | Def::AssociatedTy(_, id) |
            Def::TyParam(_, _, id, _) | Def::Struct(id) | Def::Trait(id) |
            Def::Method(id) | Def::Const(id) | Def::AssociatedConst(id) |
            Def::Local(id, _) | Def::Upvar(id, _, _, _) => {
                id
            }

            Def::Label(..)  |
            Def::PrimTy(..) |
            Def::SelfTy(..) |
            Def::Err => {
                panic!("attempted .def_id() on invalid def: {:?}", self)
            }
        }
    }

    pub fn variant_def_ids(&self) -> Option<(DefId, DefId)> {
        match *self {
            Def::Variant(enum_id, var_id) => {
                Some((enum_id, var_id))
            }
            _ => None
        }
    }
}
<commit_msg>Spelling fix in `middle::def::Def`<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use middle::def_id::DefId;
use middle::privacy::LastPrivate;
use middle::subst::ParamSpace;
use util::nodemap::NodeMap;
use syntax::ast;
use rustc_front::hir;

#[derive(Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
pub enum Def {
    Fn(DefId),
    SelfTy(Option<DefId>,                    // trait id
              Option<(ast::NodeId, ast::NodeId)>),   // (impl id, self type id)
    Mod(DefId),
    ForeignMod(DefId),
    Static(DefId, bool /* is_mutbl */),
    Const(DefId),
    AssociatedConst(DefId),
    Local(DefId, // def id of variable
             ast::NodeId), // node id of variable
    Variant(DefId /* enum */, DefId /* variant */),
    Enum(DefId),
    TyAlias(DefId),
    AssociatedTy(DefId /* trait */, DefId),
    Trait(DefId),
    PrimTy(hir::PrimTy),
    TyParam(ParamSpace, u32, DefId, ast::Name),
    Upvar(DefId,        // def id of closed over local
             ast::NodeId,  // node id of closed over local
             usize,        // index in the freevars list of the closure
             ast::NodeId), // expr node that creates the closure

    // If Def::Struct lives in type namespace it denotes a struct item and its DefId refers
    // to NodeId of the struct itself.
    // If Def::Struct lives in value namespace (e.g. tuple struct, unit struct expressions)
    // it denotes a constructor and its DefId refers to NodeId of the struct's constructor.
    Struct(DefId),
    Label(ast::NodeId),
    Method(DefId),
    Err,
}

/// The result of resolving a path.
/// Before type checking completes, `depth` represents the number of
/// trailing segments which are yet unresolved. Afterwards, if there
/// were no errors, all paths should be fully resolved, with `depth`
/// set to `0` and `base_def` representing the final resolution.
///
///     module::Type::AssocX::AssocY::MethodOrAssocType
///     ^~~~~~~~~~~~  ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
///     base_def      depth = 3
///
///     <T as Trait>::AssocX::AssocY::MethodOrAssocType
///           ^~~~~~~~~~~~~~  ^~~~~~~~~~~~~~~~~~~~~~~~~
///           base_def        depth = 2
#[derive(Copy, Clone, Debug)]
pub struct PathResolution {
    pub base_def: Def,
    pub last_private: LastPrivate,
    pub depth: usize
}

impl PathResolution {
    /// Get the definition, if fully resolved, otherwise panic.
    pub fn full_def(&self) -> Def {
        if self.depth != 0 {
            panic!("path not fully resolved: {:?}", self);
        }
        self.base_def
    }

    /// Get the DefId, if fully resolved, otherwise panic.
    pub fn def_id(&self) -> DefId {
        self.full_def().def_id()
    }

    pub fn new(base_def: Def,
               last_private: LastPrivate,
               depth: usize)
               -> PathResolution {
        PathResolution {
            base_def: base_def,
            last_private: last_private,
            depth: depth,
        }
    }
}

// Definition mapping
pub type DefMap = NodeMap<PathResolution>;
// This is the replacement export map. It maps a module to all of the exports
// within.
pub type ExportMap = NodeMap<Vec<Export>>;

#[derive(Copy, Clone)]
pub struct Export {
    pub name: ast::Name,    // The name of the target.
    pub def_id: DefId, // The definition of the target.
}

impl Def {
    pub fn var_id(&self) -> ast::NodeId {
        match *self {
            Def::Local(_, id) |
            Def::Upvar(_, id, _, _) => {
                id
            }

            Def::Fn(..) | Def::Mod(..) | Def::ForeignMod(..) | Def::Static(..) |
            Def::Variant(..) | Def::Enum(..) | Def::TyAlias(..) | Def::AssociatedTy(..) |
            Def::TyParam(..) | Def::Struct(..) | Def::Trait(..) |
            Def::Method(..) | Def::Const(..) | Def::AssociatedConst(..) |
            Def::PrimTy(..) | Def::Label(..) | Def::SelfTy(..) | Def::Err => {
                panic!("attempted .var_id() on invalid {:?}", self)
            }
        }
    }

    pub fn def_id(&self) -> DefId {
        match *self {
            Def::Fn(id) | Def::Mod(id) | Def::ForeignMod(id) | Def::Static(id, _) |
            Def::Variant(_, id) | Def::Enum(id) | Def::TyAlias(id) | Def::AssociatedTy(_, id) |
            Def::TyParam(_, _, id, _) | Def::Struct(id) | Def::Trait(id) |
            Def::Method(id) | Def::Const(id) | Def::AssociatedConst(id) |
            Def::Local(id, _) | Def::Upvar(id, _, _, _) => {
                id
            }

            Def::Label(..)  |
            Def::PrimTy(..) |
            Def::SelfTy(..) |
            Def::Err => {
                panic!("attempted .def_id() on invalid def: {:?}", self)
            }
        }
    }

    pub fn variant_def_ids(&self) -> Option<(DefId, DefId)> {
        match *self {
            Def::Variant(enum_id, var_id) => {
                Some((enum_id, var_id))
            }
            _ => None
        }
    }
}
<|endoftext|>