<commit_before>//! Reminisce is a lightweight library intended to be used for detecting and
//! reading from joysticks.
extern crate libc;
#[cfg(target_os = "windows")]
#[macro_use] extern crate rustc_bitflags;

#[cfg(feature = "sdl")]
extern crate sdl2;

extern crate glob;

/// Someday, somehow
#[cfg(all(feature = "emscripten", not(feature = "sdl")))]
pub mod emscripten;

#[cfg(all(feature = "emscripten", not(feature = "sdl")))]
pub use emscripten as native;

#[cfg(all(target_os = "linux", not(feature = "sdl")))]
pub mod linux;

#[cfg(all(target_os = "linux", not(feature = "sdl")))]
pub use linux as native;

#[cfg(all(target_os = "windows", not(feature = "sdl")))]
pub mod windows;

#[cfg(all(target_os = "windows", not(feature = "sdl")))]
pub use windows as native;

#[cfg(feature = "sdl")]
pub mod sdl;

#[cfg(feature = "sdl")]
pub use sdl as native;

pub use native::{NativeJoystick, NativeContext};

/// The maximum axis value
pub static MAX_AXIS_VALUE:i16 = 32767;
/// The minimum axis value
pub static MIN_AXIS_VALUE:i16 = -32767;

use std::borrow::Cow;
use std::fmt::Debug;

#[cfg(feature = "mappings")]
macro_rules! text_enum(
    ($name:ident, $($enumer:ident => $text:expr),+) => (
        impl ::std::fmt::Display for $name {
            fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
                write!(fmt, "{}", match *self {
                    $($name::$enumer => $text),+
                })
            }
        }
        impl ::std::str::FromStr for $name {
            type Err = String;
            fn from_str(s: &str) -> Result<$name, String> {
                match s {
                    $($text => Ok($name::$enumer),)+
                    _ => Err(format!("Could not make {} from {}", stringify!($name), s))
                }
            }
        }
    )
);
/// A direction on a joystick.
pub type Axis = u8;

/// A button on a joystick.
pub type Button = u8;

/// A joystick index.
pub type JoystickIndex = u8;
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
/// An event emitted by a joystick
pub enum Event {
    /// Fired when a joystick is connected with the index of the joystick
    /// that was connected.
    Connected(JoystickIndex),
    /// Fired when a joystick is disconnected with the index of the joystick
    /// that was disconnected.
    Disconnected(JoystickIndex),
    /// Fired when a button is pressed with the joystick's index and the
    /// button's index.
    ButtonPressed(JoystickIndex, Button),
    /// Fired when a button is released with the joystick's index and the
    /// button's index.
    ButtonReleased(JoystickIndex, Button),
    /// Fired when a axis is moved with the joystick index, axis index
    /// and its value, which is between `MIN_JOYSTICK_VALUE` and
    /// `MAX_JOYSTICK_VALUE`
    AxisMoved(JoystickIndex, Axis, i16)
}

/// A lightweight context that tracks and polls all the available joysticks.
///
/// Each `Context` has its own event queue tied to the events of the joysticks.
///
/// ``` rust
/// use reminisce::*;
/// let mut context = NativeContext::new();
/// let mut dir = 0;
/// let mut left = 30;
/// for event in &mut context {
///     left -= 1;
///     if(left <= 0) { break }
///     match event {
///         Event::AxisMoved(_, 0, nx) =>
///             dir = -1,
///         Event::AxisMoved(_, 1, ny) =>
///             dir = 1,
///         _ => ()
///     }
/// }
/// ```
pub trait Context: Sized {
    /// The kind of joystick this context tracks.
    type Joystick : Joystick;

    /// Create a new context.
    fn new() -> Self;

    /// Returns the number of joysticks connected.
    fn num_joysticks(&self) -> usize;

    /// Return a reference to the joysticks connected.
    fn get_joysticks(&self) -> &[Self::Joystick];

    /// Poll this context for events.
    fn poll(&mut self) -> Option<Event>;

    /// Iterate through the events that haven't been processed yet
    fn iter(&mut self) -> Poller<Self> {
        Poller { context: self }
    }
}

impl<'a> IntoIterator for &'a mut NativeContext {
    type Item = Event;
    type IntoIter = Poller<'a, NativeContext>;
    fn into_iter(self) -> Poller<'a, NativeContext> {
        self.iter()
    }
}

/// A joystick or gamepad.
pub trait Joystick : Sized {
    /// The error that could be thrown while trying to open the joystick
    type OpenError: Debug;

    /// Attempts to open a joystick from its index
    ///
    /// If an error occurs, this will return the textual representation of that error.
    ///
    /// ``` rust
    /// use reminisce::*;
    /// if let Ok(joystick) = NativeJoystick::open(0) {
    ///     println!("{}", joystick.get_id())
    /// } else {
    ///     println!("No joystick plugged in")
    /// }
    /// ```
    fn open(index: JoystickIndex) -> Result<Self, Self::OpenError>;

    /// Check if the joystick is still connected.
    fn is_connected(&self) -> bool;

    /// Get the identifier of this joystick.
    ///
    /// This is copy-on-write because it can be a borrowed or owned String,
    /// depending on the implementation.
    fn get_id(&self) -> Cow<str>;

    /// Get the index of this joystick.
    fn get_index(&self) -> JoystickIndex;

    /// Get the number of axes this joystick has
    ///
    /// This is capped at 6 axes for now.
    fn get_num_axes(&self) -> Axis;

    /// Get the number of buttons this joystick has
    ///
    /// This is capped at 16 buttons currently.
    fn get_num_buttons(&self) -> Button;

    /// Get the battery level of this joystick
    ///
    /// Returns none if the joystick is wired or this operation is not supported
    /// by the backend
    fn get_battery(&self) -> Option<f32>;
}

/// A joystick that tracks its state.
pub struct StatefulJoystick<J> where J: Joystick {
    joystick: J,
    buttons: i32,
    axes: Vec<i16>
}
impl<J> StatefulJoystick<J> where J: Joystick {
    /// Get the position of the axis.
    pub fn get_axis(&self, axis: Axis) -> Option<i16> {
        self.axes.get(axis as usize).cloned()
    }
    /// Check if the button given is down.
    pub fn get_button(&self, button: Button) -> Option<bool> {
        if button < self.joystick.get_num_buttons() {
            Some(self.buttons & (1 << button) > 0)
        } else {
            None
        }
    }
    /// Update this joystick with the event given.
    pub fn process(&mut self, event: Event) {
        let index = self.joystick.get_index();
        match event {
            Event::ButtonPressed(i, b) if i == index => {
                self.buttons |= (i as i32) << b;
            },
            Event::ButtonReleased(i, b) if i == index => {
                self.buttons &= !((i as i32) << b);
            },
            Event::AxisMoved(i, a, v) if i == index => {
                self.axes[a as usize] = v;
            },
            _ => ()
        }
    }
}
impl<J> Joystick for StatefulJoystick<J> where J: Joystick {
    type OpenError = J::OpenError;
    fn open(index: JoystickIndex) -> Result<Self, Self::OpenError> {
        let joystick = try!(J::open(index));
        let axes = (0..joystick.get_num_axes()).map(|_| 0).collect();
        Ok(StatefulJoystick {
            joystick: joystick,
            buttons: 0,
            axes: axes
        })
    }
    fn is_connected(&self) -> bool {
        self.joystick.is_connected()
    }
    fn get_id(&self) -> Cow<str> {
        self.joystick.get_id()
    }
    fn get_index(&self) -> JoystickIndex {
        self.joystick.get_index()
    }
    fn get_num_axes(&self) -> Axis {
        self.joystick.get_num_axes()
    }
    fn get_num_buttons(&self) -> Button {
        self.joystick.get_num_buttons()
    }
    fn get_battery(&self) -> Option<f32> {
        self.joystick.get_battery()
    }
}

/// An iterator over a context's event queue
pub struct Poller<'a, J> where J:Context+'a {
    context: &'a mut J
}

impl<'a, C> Iterator for Poller<'a, C> where C:Context {
    type Item = Event;
    /// This calls the `context.poll()` method to poll for the next event
    fn next(&mut self) -> Option<Event> {
        self.context.poll()
    }
}
<commit_msg>Get rid of unnoticed mappings stuff<commit_after>//! Reminisce is a lightweight library intended to be used for detecting and
//! reading from joysticks.
extern crate libc;
#[cfg(target_os = "windows")]
#[macro_use] extern crate rustc_bitflags;

#[cfg(feature = "sdl")]
extern crate sdl2;

extern crate glob;

/// Someday, somehow
#[cfg(all(feature = "emscripten", not(feature = "sdl")))]
pub mod emscripten;

#[cfg(all(feature = "emscripten", not(feature = "sdl")))]
pub use emscripten as native;

#[cfg(all(target_os = "linux", not(feature = "sdl")))]
pub mod linux;

#[cfg(all(target_os = "linux", not(feature = "sdl")))]
pub use linux as native;

#[cfg(all(target_os = "windows", not(feature = "sdl")))]
pub mod windows;

#[cfg(all(target_os = "windows", not(feature = "sdl")))]
pub use windows as native;

#[cfg(feature = "sdl")]
pub mod sdl;

#[cfg(feature = "sdl")]
pub use sdl as native;

pub use native::{NativeJoystick, NativeContext};

/// The maximum axis value
pub static MAX_AXIS_VALUE:i16 = 32767;
/// The minimum axis value
pub static MIN_AXIS_VALUE:i16 = -32767;

use std::borrow::Cow;
use std::fmt::Debug;

/// A direction on a joystick.
pub type Axis = u8;

/// A button on a joystick.
pub type Button = u8;

/// A joystick index.
pub type JoystickIndex = u8;
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
/// An event emitted by a joystick
pub enum Event {
    /// Fired when a joystick is connected with the index of the joystick
    /// that was connected.
    Connected(JoystickIndex),
    /// Fired when a joystick is disconnected with the index of the joystick
    /// that was disconnected.
    Disconnected(JoystickIndex),
    /// Fired when a button is pressed with the joystick's index and the
    /// button's index.
    ButtonPressed(JoystickIndex, Button),
    /// Fired when a button is released with the joystick's index and the
    /// button's index.
    ButtonReleased(JoystickIndex, Button),
    /// Fired when a axis is moved with the joystick index, axis index
    /// and its value, which is between `MIN_JOYSTICK_VALUE` and
    /// `MAX_JOYSTICK_VALUE`
    AxisMoved(JoystickIndex, Axis, i16)
}

/// A lightweight context that tracks and polls all the available joysticks.
///
/// Each `Context` has its own event queue tied to the events of the joysticks.
///
/// ``` rust
/// use reminisce::*;
/// let mut context = NativeContext::new();
/// let mut dir = 0;
/// let mut left = 30;
/// for event in &mut context {
///     left -= 1;
///     if(left <= 0) { break }
///     match event {
///         Event::AxisMoved(_, 0, nx) =>
///             dir = -1,
///         Event::AxisMoved(_, 1, ny) =>
///             dir = 1,
///         _ => ()
///     }
/// }
/// ```
pub trait Context: Sized {
    /// The kind of joystick this context tracks.
    type Joystick : Joystick;

    /// Create a new context.
    fn new() -> Self;

    /// Returns the number of joysticks connected.
    fn num_joysticks(&self) -> usize;

    /// Return a reference to the joysticks connected.
    fn get_joysticks(&self) -> &[Self::Joystick];

    /// Poll this context for events.
    fn poll(&mut self) -> Option<Event>;

    /// Iterate through the events that haven't been processed yet
    fn iter(&mut self) -> Poller<Self> {
        Poller { context: self }
    }
}

impl<'a> IntoIterator for &'a mut NativeContext {
    type Item = Event;
    type IntoIter = Poller<'a, NativeContext>;
    fn into_iter(self) -> Poller<'a, NativeContext> {
        self.iter()
    }
}

/// A joystick or gamepad.
pub trait Joystick : Sized {
    /// The error that could be thrown while trying to open the joystick
    type OpenError: Debug;

    /// Attempts to open a joystick from its index
    ///
    /// If an error occurs, this will return the textual representation of that error.
    ///
    /// ``` rust
    /// use reminisce::*;
    /// if let Ok(joystick) = NativeJoystick::open(0) {
    ///     println!("{}", joystick.get_id())
    /// } else {
    ///     println!("No joystick plugged in")
    /// }
    /// ```
    fn open(index: JoystickIndex) -> Result<Self, Self::OpenError>;

    /// Check if the joystick is still connected.
    fn is_connected(&self) -> bool;

    /// Get the identifier of this joystick.
    ///
    /// This is copy-on-write because it can be a borrowed or owned String,
    /// depending on the implementation.
    fn get_id(&self) -> Cow<str>;

    /// Get the index of this joystick.
    fn get_index(&self) -> JoystickIndex;

    /// Get the number of axes this joystick has
    ///
    /// This is capped at 6 axes for now.
    fn get_num_axes(&self) -> Axis;

    /// Get the number of buttons this joystick has
    ///
    /// This is capped at 16 buttons currently.
    fn get_num_buttons(&self) -> Button;

    /// Get the battery level of this joystick
    ///
    /// Returns none if the joystick is wired or this operation is not supported
    /// by the backend
    fn get_battery(&self) -> Option<f32>;
}

/// A joystick that tracks its state.
pub struct StatefulJoystick<J> where J: Joystick {
    joystick: J,
    buttons: i32,
    axes: Vec<i16>
}
impl<J> StatefulJoystick<J> where J: Joystick {
    /// Get the position of the axis.
    pub fn get_axis(&self, axis: Axis) -> Option<i16> {
        self.axes.get(axis as usize).cloned()
    }
    /// Check if the button given is down.
    pub fn get_button(&self, button: Button) -> Option<bool> {
        if button < self.joystick.get_num_buttons() {
            Some(self.buttons & (1 << button) > 0)
        } else {
            None
        }
    }
    /// Update this joystick with the event given.
    pub fn process(&mut self, event: Event) {
        let index = self.joystick.get_index();
        match event {
            Event::ButtonPressed(i, b) if i == index => {
                self.buttons |= (i as i32) << b;
            },
            Event::ButtonReleased(i, b) if i == index => {
                self.buttons &= !((i as i32) << b);
            },
            Event::AxisMoved(i, a, v) if i == index => {
                self.axes[a as usize] = v;
            },
            _ => ()
        }
    }
}
impl<J> Joystick for StatefulJoystick<J> where J: Joystick {
    type OpenError = J::OpenError;
    fn open(index: JoystickIndex) -> Result<Self, Self::OpenError> {
        let joystick = try!(J::open(index));
        let axes = (0..joystick.get_num_axes()).map(|_| 0).collect();
        Ok(StatefulJoystick {
            joystick: joystick,
            buttons: 0,
            axes: axes
        })
    }
    fn is_connected(&self) -> bool {
        self.joystick.is_connected()
    }
    fn get_id(&self) -> Cow<str> {
        self.joystick.get_id()
    }
    fn get_index(&self) -> JoystickIndex {
        self.joystick.get_index()
    }
    fn get_num_axes(&self) -> Axis {
        self.joystick.get_num_axes()
    }
    fn get_num_buttons(&self) -> Button {
        self.joystick.get_num_buttons()
    }
    fn get_battery(&self) -> Option<f32> {
        self.joystick.get_battery()
    }
}

/// An iterator over a context's event queue
pub struct Poller<'a, J> where J:Context+'a {
    context: &'a mut J
}

impl<'a, C> Iterator for Poller<'a, C> where C:Context {
    type Item = Event;
    /// This calls the `context.poll()` method to poll for the next event
    fn next(&mut self) -> Option<Event> {
        self.context.poll()
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A wrapper around another PRNG that reseeds it after it
//! generates a certain number of random bytes.

use rand_core::{RngCore, BlockRngCore, SeedableRng, Error, ErrorKind};
use rand_core::impls::BlockRng;

/// A wrapper around any PRNG which reseeds the underlying PRNG after it has
/// generated a certain number of random bytes.
///
/// Reseeding is never strictly *necessary*. Cryptographic PRNGs don't have a
/// limited number of bytes they can output, or at least not a limit reachable
/// in any practical way. There is no such thing as 'running out of entropy'.
///
/// Some small non-cryptographic PRNGs can have very small periods, for
/// example less than 264. Would reseeding help to ensure that you do
/// not wrap around at the end of the period? A period of 264 still
/// takes several centuries of CPU-years on current hardware. Reseeding will
/// actually make things worse, because the reseeded PRNG will just continue
/// somewhere else *in the same period*, with a high chance of overlapping with
/// previously used parts of it.
///
/// # When should you use `ReseedingRng`?
///
/// - Reseeding can be seen as some form of 'security in depth'. Even if in the
///   future a cryptographic weakness is found in the CSPRNG being used,
///   occasionally reseeding should make exploiting it much more difficult or
///   even impossible.
/// - It can be used as a poor man's cryptography (not recommended, just use a
///   good CSPRNG). Previous implementations of `thread_rng` for example used
///   `ReseedingRng` with the ISAAC RNG. That algorithm, although apparently
///   strong and with no known attack, does not come with any proof of security
///   and does not meet the current standards for a cryptographically secure
///   PRNG. By reseeding it frequently (every 32 kiB) it seems safe to assume
///   there is no attack that can operate on the tiny window between reseeds.
///
/// # Error handling
///
/// If reseeding fails, `try_fill_bytes` is the only `Rng` method to report it.
/// For all other `Rng` methods, `ReseedingRng` will not panic but try to
/// handle the error intelligently through some combination of retrying and
/// delaying reseeding until later. If handling the source error fails these
/// methods will continue generating data from the wrapped PRNG without
/// reseeding.
#[derive(Debug)]
pub struct ReseedingRng<R, Rsdr>(BlockRng<Reseeder<R, Rsdr>>)
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore;

impl<R, Rsdr> ReseedingRng<R, Rsdr>
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore
{
    /// Create a new `ReseedingRng` with the given parameters.
    ///
    /// # Arguments
    ///
    /// * `rng`: the random number generator to use.
    /// * `threshold`: the number of generated bytes after which to reseed the RNG.
    /// * `reseeder`: the RNG to use for reseeding.
    pub fn new(rng: R, threshold: u64, reseeder: Rsdr)
        -> ReseedingRng<R, Rsdr>
    {
        assert!(threshold <= ::core::i64::MAX as u64);
        let results_empty = R::Results::default();
        ReseedingRng(
            BlockRng {
                core: Reseeder {
                    core: rng,
                    reseeder: reseeder,
                    threshold: threshold as i64,
                    bytes_until_reseed: threshold as i64,
                },
                index: results_empty.as_ref().len(), // generate on first use
                results: results_empty,
            }
        )
    }

    /// Reseed the internal PRNG.
    pub fn reseed(&mut self) -> Result<(), Error> {
        self.0.core.reseed()
    }
}

impl<R: BlockRngCore<u32> + SeedableRng, Rsdr: RngCore> RngCore for ReseedingRng<R, Rsdr> {
    #[inline]
    fn next_u32(&mut self) -> u32 {
        self.0.next_u32()
    }

    #[inline]
    fn next_u64(&mut self) -> u64 {
        self.0.next_u64()
    }

    #[inline]
    fn fill_bytes(&mut self, dest: &mut [u8]) {
        self.0.fill_bytes(dest)
    }

    #[inline]
    fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
        self.0.try_fill_bytes(dest)
    }
}

#[derive(Debug)]
struct Reseeder<R, Rsdr> {
    core: R,
    reseeder: Rsdr,
    threshold: i64,
    bytes_until_reseed: i64,
}

impl<R, Rsdr> BlockRngCore<u32> for Reseeder<R, Rsdr>
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore
{
    type Results = <R as BlockRngCore<u32>>::Results;

    fn generate(&mut self, results: &mut Self::Results) -> Result<(), Error> {
        if self.bytes_until_reseed <= 0 {
            // In the unlikely event the internal PRNG fails, we don't know
            // whether this is resolvable; schedule to reseed on next use and
            // return original error kind.
            return self.reseed_and_generate(results);
        }
        self.bytes_until_reseed -= results.as_ref().len() as i64 * 4;
        self.core.generate(results)
    }
}

impl<R, Rsdr> Reseeder<R, Rsdr>
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore
{
    /// Reseed the internal PRNG.
    fn reseed(&mut self) -> Result<(), Error> {
        R::from_rng(&mut self.reseeder).map(|result| self.core = result)
    }

    /// Reseed the internal PRNG.
    ///
    /// If reseeding fails, this will try to work around errors intelligently
    /// through some combination of retrying and delaying reseeding until later.
    /// It will also report the error with `ErrorKind::Transient` with the
    /// original error as cause.
    fn auto_reseed(&mut self) -> Result<(), Error> {
        trace!("Reseeding RNG after {} generated bytes",
               self.threshold - self.bytes_until_reseed);
        if let Err(mut e) = self.reseed()  {
            let delay = match e.kind {
                ErrorKind::Transient => 0,
                kind @ _ if kind.should_retry() => self.threshold >> 8,
                _ => self.threshold,
            };
            warn!("Reseeding RNG delayed reseeding by {} bytes due to \
                    error from source: {}", delay, e);
            self.bytes_until_reseed = delay;
            e.kind = ErrorKind::Transient;
            Err(e)
        } else {
            self.bytes_until_reseed = self.threshold;
            Ok(())
        }
    }

    #[inline(never)]
    fn reseed_and_generate(&mut self,
                           results: &mut <Self as BlockRngCore<u32>>::Results)
        -> Result<(), Error>
    {
        let res1 = self.auto_reseed();
        self.bytes_until_reseed -= results.as_ref().len() as i64 * 4;
        let res2 = self.core.generate(results);
        if res2.is_err() { res2 } else { res1 }
    }
}

#[cfg(test)]
mod test {
    use {Rng, SeedableRng};
    use prng::chacha::ChaChaCore;
    use mock::StepRng;
    use super::ReseedingRng;

    #[test]
    fn test_reseeding() {
        let mut zero = StepRng::new(0, 0);
        let rng = ChaChaCore::from_rng(&mut zero).unwrap();
        let mut reseeding = ReseedingRng::new(rng, 32*4, zero);

        // Currently we only support for arrays up to length 32.
        // TODO: cannot generate seq via Rng::gen because it uses different alg
        let mut buf = [0u32; 32]; // Needs to be a multiple of the RNGs result
                                  // size to test exactly.
        reseeding.fill(&mut buf);
        let seq = buf;
        for _ in 0..10 {
            reseeding.fill(&mut buf);
            assert_eq!(buf, seq);
        }
    }
}
<commit_msg>Fix comment<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A wrapper around another PRNG that reseeds it after it
//! generates a certain number of random bytes.

use rand_core::{RngCore, BlockRngCore, SeedableRng, Error, ErrorKind};
use rand_core::impls::BlockRng;

/// A wrapper around any PRNG which reseeds the underlying PRNG after it has
/// generated a certain number of random bytes.
///
/// Reseeding is never strictly *necessary*. Cryptographic PRNGs don't have a
/// limited number of bytes they can output, or at least not a limit reachable
/// in any practical way. There is no such thing as 'running out of entropy'.
///
/// Some small non-cryptographic PRNGs can have very small periods, for
/// example less than 264. Would reseeding help to ensure that you do
/// not wrap around at the end of the period? A period of 264 still
/// takes several centuries of CPU-years on current hardware. Reseeding will
/// actually make things worse, because the reseeded PRNG will just continue
/// somewhere else *in the same period*, with a high chance of overlapping with
/// previously used parts of it.
///
/// # When should you use `ReseedingRng`?
///
/// - Reseeding can be seen as some form of 'security in depth'. Even if in the
///   future a cryptographic weakness is found in the CSPRNG being used,
///   occasionally reseeding should make exploiting it much more difficult or
///   even impossible.
/// - It can be used as a poor man's cryptography (not recommended, just use a
///   good CSPRNG). Previous implementations of `thread_rng` for example used
///   `ReseedingRng` with the ISAAC RNG. That algorithm, although apparently
///   strong and with no known attack, does not come with any proof of security
///   and does not meet the current standards for a cryptographically secure
///   PRNG. By reseeding it frequently (every 32 kiB) it seems safe to assume
///   there is no attack that can operate on the tiny window between reseeds.
///
/// # Error handling
///
/// If reseeding fails, `try_fill_bytes` is the only `Rng` method to report it.
/// For all other `Rng` methods, `ReseedingRng` will not panic but try to
/// handle the error intelligently through some combination of retrying and
/// delaying reseeding until later. If handling the source error fails these
/// methods will continue generating data from the wrapped PRNG without
/// reseeding.
#[derive(Debug)]
pub struct ReseedingRng<R, Rsdr>(BlockRng<Reseeder<R, Rsdr>>)
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore;

impl<R, Rsdr> ReseedingRng<R, Rsdr>
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore
{
    /// Create a new `ReseedingRng` with the given parameters.
    ///
    /// # Arguments
    ///
    /// * `rng`: the random number generator to use.
    /// * `threshold`: the number of generated bytes after which to reseed the RNG.
    /// * `reseeder`: the RNG to use for reseeding.
    pub fn new(rng: R, threshold: u64, reseeder: Rsdr)
        -> ReseedingRng<R, Rsdr>
    {
        assert!(threshold <= ::core::i64::MAX as u64);
        let results_empty = R::Results::default();
        ReseedingRng(
            BlockRng {
                core: Reseeder {
                    core: rng,
                    reseeder: reseeder,
                    threshold: threshold as i64,
                    bytes_until_reseed: threshold as i64,
                },
                index: results_empty.as_ref().len(), // generate on first use
                results: results_empty,
            }
        )
    }

    /// Reseed the internal PRNG.
    pub fn reseed(&mut self) -> Result<(), Error> {
        self.0.core.reseed()
    }
}

impl<R: BlockRngCore<u32> + SeedableRng, Rsdr: RngCore> RngCore for ReseedingRng<R, Rsdr> {
    #[inline]
    fn next_u32(&mut self) -> u32 {
        self.0.next_u32()
    }

    #[inline]
    fn next_u64(&mut self) -> u64 {
        self.0.next_u64()
    }

    #[inline]
    fn fill_bytes(&mut self, dest: &mut [u8]) {
        self.0.fill_bytes(dest)
    }

    #[inline]
    fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
        self.0.try_fill_bytes(dest)
    }
}

#[derive(Debug)]
struct Reseeder<R, Rsdr> {
    core: R,
    reseeder: Rsdr,
    threshold: i64,
    bytes_until_reseed: i64,
}

impl<R, Rsdr> BlockRngCore<u32> for Reseeder<R, Rsdr>
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore
{
    type Results = <R as BlockRngCore<u32>>::Results;

    fn generate(&mut self, results: &mut Self::Results) -> Result<(), Error> {
        if self.bytes_until_reseed <= 0 {
            // We want to reseed here, and generate results later in the
            // function. If generating results fail, we should return the error
            // from that. If generating results succeeded, but reseeding failed,
            // we should return the error from reseeding.
            // The only way to get this behaviour without destroying performance
            // was to split part of the function out into a
            // `reseed_and_generate` method.
            return self.reseed_and_generate(results);
        }
        self.bytes_until_reseed -= results.as_ref().len() as i64 * 4;
        self.core.generate(results)
    }
}

impl<R, Rsdr> Reseeder<R, Rsdr>
where R: BlockRngCore<u32> + SeedableRng,
      Rsdr: RngCore
{
    /// Reseed the internal PRNG.
    fn reseed(&mut self) -> Result<(), Error> {
        R::from_rng(&mut self.reseeder).map(|result| self.core = result)
    }

    /// Reseed the internal PRNG.
    ///
    /// If reseeding fails, this will try to work around errors intelligently
    /// through some combination of retrying and delaying reseeding until later.
    /// It will also report the error with `ErrorKind::Transient` with the
    /// original error as cause.
    fn auto_reseed(&mut self) -> Result<(), Error> {
        trace!("Reseeding RNG after {} generated bytes",
               self.threshold - self.bytes_until_reseed);
        if let Err(mut e) = self.reseed()  {
            let delay = match e.kind {
                ErrorKind::Transient => 0,
                kind @ _ if kind.should_retry() => self.threshold >> 8,
                _ => self.threshold,
            };
            warn!("Reseeding RNG delayed reseeding by {} bytes due to \
                    error from source: {}", delay, e);
            self.bytes_until_reseed = delay;
            e.kind = ErrorKind::Transient;
            Err(e)
        } else {
            self.bytes_until_reseed = self.threshold;
            Ok(())
        }
    }

    #[inline(never)]
    fn reseed_and_generate(&mut self,
                           results: &mut <Self as BlockRngCore<u32>>::Results)
        -> Result<(), Error>
    {
        let res1 = self.auto_reseed();
        self.bytes_until_reseed -= results.as_ref().len() as i64 * 4;
        let res2 = self.core.generate(results);
        if res2.is_err() { res2 } else { res1 }
    }
}

#[cfg(test)]
mod test {
    use {Rng, SeedableRng};
    use prng::chacha::ChaChaCore;
    use mock::StepRng;
    use super::ReseedingRng;

    #[test]
    fn test_reseeding() {
        let mut zero = StepRng::new(0, 0);
        let rng = ChaChaCore::from_rng(&mut zero).unwrap();
        let mut reseeding = ReseedingRng::new(rng, 32*4, zero);

        // Currently we only support for arrays up to length 32.
        // TODO: cannot generate seq via Rng::gen because it uses different alg
        let mut buf = [0u32; 32]; // Needs to be a multiple of the RNGs result
                                  // size to test exactly.
        reseeding.fill(&mut buf);
        let seq = buf;
        for _ in 0..10 {
            reseeding.fill(&mut buf);
            assert_eq!(buf, seq);
        }
    }
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate commands;
extern crate rustyline;

use commands::parser::{Command, CommandTree, ParseError, Parser};
use commands::tokenizer::tokenize;

fn main() {
    let mut tree = CommandTree::new();
    tree.command(Command::new("show"));
    let root = tree.finalize();

    let mut rl = rustyline::Editor::new();
    while let Ok(line) = rl.readline(">> ") {
        if let Ok(tokens) = tokenize(&line) {
            let mut parser = Parser::new(root.clone());
            if let Err(err) = parser.parse(tokens) {
                match err {
                    ParseError::NoMatches(_, acceptable) => {
                        println!("No match for '{}'", line);
                        println!("\nPossible options:");
                        for ref option in acceptable {
                            let n = option.node();
                            println!("  {} - {}", n.help_symbol, n.help_text);
                        }
                    }
                    ParseError::AmbiguousMatch(_, matches) => {
                        println!("\nCan be interpreted as:");
                        for ref option in matches {
                            let n = option.node();
                            println!("  {} - {}", n.help_symbol, n.help_text);
                        }
                    }
                }
            } else if let Err(err) = parser.verify() {
                println!("{}", err);
            } else {
                parser.execute();
            }
        }
        println!("");
    }
    println!("\nExiting.");
}
<commit_msg>rustyline: Add each line to the history.<commit_after>// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate commands;
extern crate rustyline;

use commands::parser::{Command, CommandTree, ParseError, Parser};
use commands::tokenizer::tokenize;

fn main() {
    let mut tree = CommandTree::new();
    tree.command(Command::new("show"));
    let root = tree.finalize();

    let mut rl = rustyline::Editor::new();
    while let Ok(line) = rl.readline(">> ") {
        rl.add_history_entry(&line);
        if let Ok(tokens) = tokenize(&line) {
            let mut parser = Parser::new(root.clone());
            if let Err(err) = parser.parse(tokens) {
                match err {
                    ParseError::NoMatches(_, acceptable) => {
                        println!("No match for '{}'", line);
                        println!("\nPossible options:");
                        for ref option in acceptable {
                            let n = option.node();
                            println!("  {} - {}", n.help_symbol, n.help_text);
                        }
                    }
                    ParseError::AmbiguousMatch(_, matches) => {
                        println!("\nCan be interpreted as:");
                        for ref option in matches {
                            let n = option.node();
                            println!("  {} - {}", n.help_symbol, n.help_text);
                        }
                    }
                }
            } else if let Err(err) = parser.verify() {
                println!("{}", err);
            } else {
                parser.execute();
            }
        }
        println!("");
    }
    println!("\nExiting.");
}
<|endoftext|>


<commit_before>use std::cmp::min;
use super::token::Token;
use super::token::Category;

pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store
/// tokens for the various language and format lexers.
pub struct Tokenizer {
    pub data: String,
    char_count: usize,
    pub token_start: usize,
    pub token_position: usize,
    tokens: Vec<Token>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let lexer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.to_string(),
      char_count: data.chars().count(),
      token_start: 0,
      token_position: 0,
      tokens: vec![]
    }
}

impl Tokenizer {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// let lexer = luthor::tokenizer::new("luthor");
    /// lexer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut lexer = luthor::tokenizer::new("luthor");
    /// assert_eq!(lexer.current_char().unwrap(), 'l');
    /// lexer.advance();
    /// assert_eq!(lexer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        if self.has_more_data() {
            self.token_position += 1;
        }
    }

    /// Determines whether or not there is more unprocessed data.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut lexer = luthor::tokenizer::new("l");
    /// assert_eq!(lexer.has_more_data(), true);
    /// lexer.advance();
    /// assert_eq!(lexer.has_more_data(), false);
    /// ```
    pub fn has_more_data(&self) -> bool {
        self.token_position < self.char_count
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut lexer = luthor::tokenizer::new("l");
    /// assert_eq!(lexer.current_char().unwrap(), 'l');
    /// lexer.advance();
    /// assert_eq!(lexer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        if self.has_more_data() {
            Some(self.data.chars().nth(self.token_position).unwrap())
        } else {
            None
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// let mut lexer = luthor::tokenizer::new("luthor");
    /// lexer.advance();
    /// lexer.advance();
    /// lexer.tokenize(Category::Text);
    /// assert_eq!(lexer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if self.token_start != self.token_position {
            let token = Token{
                lexeme: self.data.slice_chars(self.token_start, self.token_position).to_string(),
                category: category,
            };
            self.tokens.push(token);
            self.token_start = self.token_position;
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// Category::Text category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// let mut lexer = luthor::tokenizer::new("luthor");
    /// lexer.advance();
    /// lexer.tokenize_next(5, Category::Keyword);
    /// assert_eq!(lexer.tokens()[0], Token{ lexeme: "l".to_string(), category: Category::Text});
    /// assert_eq!(lexer.tokens()[1], Token{ lexeme: "uthor".to_string(), category: Category::Keyword});
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        self.tokenize(Category::Text);
        self.token_position = min(self.token_position + amount, self.char_count);
        self.tokenize(category);
    }
}

mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    #[test]
    fn new_initializes_correctly_with_unicode_data() {
        let lexer_data = "différent";
        let lexer = new(lexer_data);
        assert_eq!(lexer.data, lexer_data);
        assert_eq!(lexer.char_count, 9);
        assert_eq!(lexer.token_start, 0);
        assert_eq!(lexer.token_position, 0);
        assert_eq!(lexer.tokens, vec![]);
    }

    #[test]
    fn advance_increments_the_cursor_by_one() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        assert_eq!(lexer.token_position, 1);
        lexer.advance();
        assert_eq!(lexer.token_position, 2);
    }

    #[test]
    fn advance_stops_when_there_is_no_more_data() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);

        // Try to go beyond the last character.
        for _ in 0..lexer.char_count {
            lexer.advance();
        }

        assert_eq!(lexer.token_position, lexer.char_count);
    }

    #[test]
    fn has_more_data_works() {
        let lexer_data = "él";
        let mut lexer = new(lexer_data);

        lexer.advance();
        assert!(lexer.has_more_data());

        lexer.advance();
        assert_eq!(lexer.has_more_data(), false);
    }

    #[test]
    fn current_char_returns_the_char_at_token_position() {
        let lexer_data = "él";
        let mut lexer = new(lexer_data);

        assert_eq!(lexer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let lexer_data = "él";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();

        assert_eq!(lexer.current_char(), None);
    }

    #[test]
    fn tokenize_advances_token_start_to_cursor() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize(Category::Text);

        assert_eq!(lexer.token_start, 2);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize(Category::Text);

        let token = lexer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.tokenize(Category::Text);

        assert_eq!(lexer.tokens.len(), 0);
        assert_eq!(lexer.token_start, 0);
        assert_eq!(lexer.token_position, 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize_next(1, Category::Keyword);

        let token = lexer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize_next(5, Category::Keyword);

        let token = lexer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize_next(15, Category::Keyword);

        let token = lexer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }
}
<commit_msg>Add a state stack to the tokenizer for ambiguous parent states.<commit_after>use std::cmp::min;
use super::token::Token;
use super::token::Category;

pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store
/// tokens for the various language and format lexers.
pub struct Tokenizer {
    pub data: String,
    char_count: usize,
    pub token_start: usize,
    pub token_position: usize,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let lexer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.to_string(),
      char_count: data.chars().count(),
      token_start: 0,
      token_position: 0,
      tokens: vec![],
      states: vec![]
    }
}

impl Tokenizer {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// let lexer = luthor::tokenizer::new("luthor");
    /// lexer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut lexer = luthor::tokenizer::new("luthor");
    /// assert_eq!(lexer.current_char().unwrap(), 'l');
    /// lexer.advance();
    /// assert_eq!(lexer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        if self.has_more_data() {
            self.token_position += 1;
        }
    }

    /// Determines whether or not there is more unprocessed data.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut lexer = luthor::tokenizer::new("l");
    /// assert_eq!(lexer.has_more_data(), true);
    /// lexer.advance();
    /// assert_eq!(lexer.has_more_data(), false);
    /// ```
    pub fn has_more_data(&self) -> bool {
        self.token_position < self.char_count
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut lexer = luthor::tokenizer::new("l");
    /// assert_eq!(lexer.current_char().unwrap(), 'l');
    /// lexer.advance();
    /// assert_eq!(lexer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        if self.has_more_data() {
            Some(self.data.chars().nth(self.token_position).unwrap())
        } else {
            None
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// let mut lexer = luthor::tokenizer::new("luthor");
    /// lexer.advance();
    /// lexer.advance();
    /// lexer.tokenize(Category::Text);
    /// assert_eq!(lexer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if self.token_start != self.token_position {
            let token = Token{
                lexeme: self.data.slice_chars(self.token_start, self.token_position).to_string(),
                category: category,
            };
            self.tokens.push(token);
            self.token_start = self.token_position;
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// Category::Text category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// let mut lexer = luthor::tokenizer::new("luthor");
    /// lexer.advance();
    /// lexer.tokenize_next(5, Category::Keyword);
    /// assert_eq!(lexer.tokens()[0], Token{ lexeme: "l".to_string(), category: Category::Text});
    /// assert_eq!(lexer.tokens()[1], Token{ lexeme: "uthor".to_string(), category: Category::Keyword});
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        self.tokenize(Category::Text);
        self.token_position = min(self.token_position + amount, self.char_count);
        self.tokenize(category);
    }
}

mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    #[test]
    fn new_initializes_correctly_with_unicode_data() {
        let lexer_data = "différent";
        let lexer = new(lexer_data);
        assert_eq!(lexer.data, lexer_data);
        assert_eq!(lexer.char_count, 9);
        assert_eq!(lexer.token_start, 0);
        assert_eq!(lexer.token_position, 0);
        assert_eq!(lexer.tokens, vec![]);
    }

    #[test]
    fn advance_increments_the_cursor_by_one() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        assert_eq!(lexer.token_position, 1);
        lexer.advance();
        assert_eq!(lexer.token_position, 2);
    }

    #[test]
    fn advance_stops_when_there_is_no_more_data() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);

        // Try to go beyond the last character.
        for _ in 0..lexer.char_count {
            lexer.advance();
        }

        assert_eq!(lexer.token_position, lexer.char_count);
    }

    #[test]
    fn has_more_data_works() {
        let lexer_data = "él";
        let mut lexer = new(lexer_data);

        lexer.advance();
        assert!(lexer.has_more_data());

        lexer.advance();
        assert_eq!(lexer.has_more_data(), false);
    }

    #[test]
    fn current_char_returns_the_char_at_token_position() {
        let lexer_data = "él";
        let mut lexer = new(lexer_data);

        assert_eq!(lexer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let lexer_data = "él";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();

        assert_eq!(lexer.current_char(), None);
    }

    #[test]
    fn tokenize_advances_token_start_to_cursor() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize(Category::Text);

        assert_eq!(lexer.token_start, 2);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize(Category::Text);

        let token = lexer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.tokenize(Category::Text);

        assert_eq!(lexer.tokens.len(), 0);
        assert_eq!(lexer.token_start, 0);
        assert_eq!(lexer.token_position, 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize_next(1, Category::Keyword);

        let token = lexer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize_next(5, Category::Keyword);

        let token = lexer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let lexer_data = "élégant";
        let mut lexer = new(lexer_data);
        lexer.advance();
        lexer.advance();
        lexer.tokenize_next(15, Category::Keyword);

        let token = lexer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }
}
<|endoftext|>


<commit_before>use event::{Event, State, SassRule, SassComment, SassSelector, TopLevelEvent};
use std::borrow::Cow::Borrowed;

fn is_ascii_whitespace(c: u8) -> bool {
   is_newline(c) || is_ascii_whitespace_no_nl(c)
}

fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

fn is_newline(c: u8) -> bool {
    c == b'\n' || c == b'\r'
}

fn isnt_newline(c: u8) -> bool {
    !is_newline(c)
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}

#[derive(Debug)]
pub struct Tokenizer<'a> {
    sass: &'a str,
    bytes: &'a [u8],
    offset: usize,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule: SassRule<'a>,
}

impl<'a> Tokenizer<'a> {
    pub fn new(sass: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            sass: &sass,
            bytes: &sass.as_bytes(),
            offset: 0,
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule: SassRule::new(),
        }
    }

    fn start_something(&mut self) -> Option<TopLevelEvent<'a>> {
        self.current_sass_rule = SassRule::new();

        self.pick_something();
        while self.state != State::OutsideRules {
            let curr_state = self.state.clone();
            match curr_state {
                State::Eof => {
                    return None
                },
                State::InSelectors => {
                    let sel = self.next_selector();
                    match sel {
                        Some(Event::Selector(sass_selector)) => self.current_sass_rule.selectors.push(sass_selector),
                        _ => {},
                    }
                },
                State::InProperties => {
                    let prop = self.next_property();
                    if prop.is_some() {
                        self.current_sass_rule.children.push(prop.unwrap());
                    }
                },
                State::EndRule => {
                    // consume '}', should probably be checking that's what we actually have
                    self.offset += 1;

                    match self.sass_rule_stack.pop() {
                        Some(ref mut rule) => {
                            rule.children.push(Event::ChildRule(self.current_sass_rule.clone()));
                            self.current_sass_rule = (*rule).clone();
                            self.pick_something();
                        },
                        None => self.state = State::OutsideRules,
                    }
                },
                State::InRule => {
                    self.sass_rule_stack.push(self.current_sass_rule.clone());
                    self.current_sass_rule = SassRule::new();
                    self.pick_something();
                },
                State::InVariable => {
                    let var = self.next_variable();
                    if var.is_some() {
                        return var
                    } else {
                        // is this really what we should be doing here? reachable?
                        self.state = State::Eof;
                    }
                },
                State::InComment => {
                    let comment = self.next_comment();
                    if comment.is_some() {
                        if self.sass_rule_stack.len() == 0 &&
                           self.current_sass_rule.selectors.len() == 0 &&
                           self.current_sass_rule.children.len() == 0 {
                               return Some(TopLevelEvent::Comment(SassComment { comment: comment.unwrap() }))
                        } else {
                            self.current_sass_rule.children.push(comment.unwrap());
                            self.pick_something();
                        }
                    } else {
                        // is this really what we should be doing here? reachable?
                        self.state = State::Eof;
                    }
                }
                unknown_state => {
                    println!("i dont know what to do for {:?}", unknown_state);
                    println!("current sass rule = {:?}", self.current_sass_rule);
                    self.state = State::Eof;
                },
            }
        }

        Some(TopLevelEvent::Rule(self.current_sass_rule.clone()))
    }

    fn pick_something(&mut self) {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return
        }

        let c = self.bytes[self.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.offset + 1) < self.sass.len() {
            let d = self.bytes[self.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        self.state = State::InSelectors;
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let mut i = self.offset;
        let limit = self.sass.len();

        while i < limit {
            let c = self.bytes[i];
            if is_ascii_whitespace(c) {
                i += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
            } else if c == b'/' && i + 1 < limit && self.bytes[i + 1] == b'/' {
                i += self.scan_while(&self.sass[i..self.sass.len()], isnt_newline);
            } else {
                self.offset = i;
                return
            }
        }
        self.offset = limit;
    }

    fn next_comment(&mut self) -> Option<Event<'a>> {
        let comment_body_beginning = self.offset;
        let mut i = comment_body_beginning + 2;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b'*' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            if self.bytes[i+1] == b'/' {
                self.offset = i + 2;
                return Some(Event::Comment(Borrowed(&self.sass[comment_body_beginning..i + 2])))
            } else {
                i += 1;
            }
        }
        None
    }

    fn next_variable(&mut self) -> Option<TopLevelEvent<'a>> {
        // TODO: can parts of this be deduplicated with properties?
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(TopLevelEvent::SassVariable {
                    name: Borrowed(&self.sass[name_beginning..name_end]),
                    value: Borrowed(&self.sass[value_beginning..value_end]),
                })
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_property(&mut self) -> Option<Event<'a>> {
        self.skip_leading_whitespace();

        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = self.bytes[i];
        if c == b'}' {
            self.state = State::EndRule;
            return None
        }

        let d = self.bytes[i + 1];
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return None
        }

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            // Inefficient since we already skipped the whitespace and we'll have to
            // do it again but oh well
            let c = self.bytes[i];
            if c == b'{' {
                self.state = State::InRule;
                return None
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(Event::Property(
                    Borrowed(&self.sass[name_beginning..name_end]),
                    Borrowed(&self.sass[value_beginning..value_end])
                ))

            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_selector(&mut self) -> Option<Event<'a>> {
        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b',' || c == b'{' || c == b':') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = self.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return None
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule.selectors_done {
                            self.state = State::InRule;
                            return None
                        } else {
                            self.current_sass_rule.selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.offset = i + 1;
                    return Some(Event::Selector(SassSelector { name: Borrowed(&self.sass[beginning..end]) }))
                }
            }

            self.offset = i;
            if i > beginning {
                return Some(Event::Selector(SassSelector { name: Borrowed(&self.sass[beginning..i]) }))
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Some(Event::Selector(SassSelector { name: Borrowed(&self.sass[beginning..i]) }))
        } else {
            self.state = State::Eof;
            None
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = TopLevelEvent<'a>;

    fn next(&mut self) -> Option<TopLevelEvent<'a>> {
        if self.offset < self.sass.len() {
            return self.start_something()
        }
        None
    }
}
<commit_msg>Add a length check; this gets most of the tests running again<commit_after>use event::{Event, State, SassRule, SassComment, SassSelector, TopLevelEvent};
use std::borrow::Cow::Borrowed;

fn is_ascii_whitespace(c: u8) -> bool {
   is_newline(c) || is_ascii_whitespace_no_nl(c)
}

fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

fn is_newline(c: u8) -> bool {
    c == b'\n' || c == b'\r'
}

fn isnt_newline(c: u8) -> bool {
    !is_newline(c)
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}

#[derive(Debug)]
pub struct Tokenizer<'a> {
    sass: &'a str,
    bytes: &'a [u8],
    offset: usize,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule: SassRule<'a>,
}

impl<'a> Tokenizer<'a> {
    pub fn new(sass: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            sass: &sass,
            bytes: &sass.as_bytes(),
            offset: 0,
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule: SassRule::new(),
        }
    }

    fn start_something(&mut self) -> Option<TopLevelEvent<'a>> {
        self.current_sass_rule = SassRule::new();

        self.pick_something();
        while self.state != State::OutsideRules {
            let curr_state = self.state.clone();
            match curr_state {
                State::Eof => {
                    return None
                },
                State::InSelectors => {
                    let sel = self.next_selector();
                    match sel {
                        Some(Event::Selector(sass_selector)) => self.current_sass_rule.selectors.push(sass_selector),
                        _ => {},
                    }
                },
                State::InProperties => {
                    let prop = self.next_property();
                    if prop.is_some() {
                        self.current_sass_rule.children.push(prop.unwrap());
                    }
                },
                State::EndRule => {
                    // consume '}', should probably be checking that's what we actually have
                    self.offset += 1;

                    match self.sass_rule_stack.pop() {
                        Some(ref mut rule) => {
                            rule.children.push(Event::ChildRule(self.current_sass_rule.clone()));
                            self.current_sass_rule = (*rule).clone();
                            self.pick_something();
                        },
                        None => self.state = State::OutsideRules,
                    }
                },
                State::InRule => {
                    self.sass_rule_stack.push(self.current_sass_rule.clone());
                    self.current_sass_rule = SassRule::new();
                    self.pick_something();
                },
                State::InVariable => {
                    let var = self.next_variable();
                    if var.is_some() {
                        return var
                    } else {
                        // is this really what we should be doing here? reachable?
                        self.state = State::Eof;
                    }
                },
                State::InComment => {
                    let comment = self.next_comment();
                    if comment.is_some() {
                        if self.sass_rule_stack.len() == 0 &&
                           self.current_sass_rule.selectors.len() == 0 &&
                           self.current_sass_rule.children.len() == 0 {
                               return Some(TopLevelEvent::Comment(SassComment { comment: comment.unwrap() }))
                        } else {
                            self.current_sass_rule.children.push(comment.unwrap());
                            self.pick_something();
                        }
                    } else {
                        // is this really what we should be doing here? reachable?
                        self.state = State::Eof;
                    }
                }
                unknown_state => {
                    println!("i dont know what to do for {:?}", unknown_state);
                    println!("current sass rule = {:?}", self.current_sass_rule);
                    self.state = State::Eof;
                },
            }
        }

        Some(TopLevelEvent::Rule(self.current_sass_rule.clone()))
    }

    fn pick_something(&mut self) {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return
        }

        let c = self.bytes[self.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.offset + 1) < self.sass.len() {
            let d = self.bytes[self.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        self.state = State::InSelectors;
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let mut i = self.offset;
        let limit = self.sass.len();

        while i < limit {
            let c = self.bytes[i];
            if is_ascii_whitespace(c) {
                i += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
            } else if c == b'/' && i + 1 < limit && self.bytes[i + 1] == b'/' {
                i += self.scan_while(&self.sass[i..self.sass.len()], isnt_newline);
            } else {
                self.offset = i;
                return
            }
        }
        self.offset = limit;
    }

    fn next_comment(&mut self) -> Option<Event<'a>> {
        let comment_body_beginning = self.offset;
        let mut i = comment_body_beginning + 2;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b'*' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            if self.bytes[i+1] == b'/' {
                self.offset = i + 2;
                return Some(Event::Comment(Borrowed(&self.sass[comment_body_beginning..i + 2])))
            } else {
                i += 1;
            }
        }
        None
    }

    fn next_variable(&mut self) -> Option<TopLevelEvent<'a>> {
        // TODO: can parts of this be deduplicated with properties?
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(TopLevelEvent::SassVariable {
                    name: Borrowed(&self.sass[name_beginning..name_end]),
                    value: Borrowed(&self.sass[value_beginning..value_end]),
                })
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_property(&mut self) -> Option<Event<'a>> {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return None
        }

        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = self.bytes[i];
        if c == b'}' {
            self.state = State::EndRule;
            return None
        }

        let d = self.bytes[i + 1];
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return None
        }

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            // Inefficient since we already skipped the whitespace and we'll have to
            // do it again but oh well
            let c = self.bytes[i];
            if c == b'{' {
                self.state = State::InRule;
                return None
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(Event::Property(
                    Borrowed(&self.sass[name_beginning..name_end]),
                    Borrowed(&self.sass[value_beginning..value_end])
                ))

            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_selector(&mut self) -> Option<Event<'a>> {
        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b',' || c == b'{' || c == b':') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = self.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return None
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule.selectors_done {
                            self.state = State::InRule;
                            return None
                        } else {
                            self.current_sass_rule.selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.offset = i + 1;
                    return Some(Event::Selector(SassSelector { name: Borrowed(&self.sass[beginning..end]) }))
                }
            }

            self.offset = i;
            if i > beginning {
                return Some(Event::Selector(SassSelector { name: Borrowed(&self.sass[beginning..i]) }))
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Some(Event::Selector(SassSelector { name: Borrowed(&self.sass[beginning..i]) }))
        } else {
            self.state = State::Eof;
            None
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = TopLevelEvent<'a>;

    fn next(&mut self) -> Option<TopLevelEvent<'a>> {
        if self.offset < self.sass.len() {
            return self.start_something()
        }
        None
    }
}
<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: usize,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn next(&self) -> Option<&Rc<Token>> {
        match self {
            &Token::Identifier { next: ref next, .. } => next.as_ref(),
            &Token::Index { next: ref next, .. }      => next.as_ref(),
        }
    }

    /// Convenience function for `token.next().is_some()`
    pub fn has_next(&self) -> bool {
        self.next().is_some()
    }

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> usize {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Replace Rc with Box in token data structure<commit_after>/// The tokenizer for the query interpreter

use error::*;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Box<Token>>
    },

    Index {
        idx: usize,
        next: Option<Box<Token>>
    }
}

impl Token {

    pub fn next(&self) -> Option<&Box<Token>> {
        match self {
            &Token::Identifier { next: ref next, .. } => next.as_ref(),
            &Token::Index { next: ref next, .. }      => next.as_ref(),
        }
    }

    /// Convenience function for `token.next().is_some()`
    pub fn has_next(&self) -> bool {
        self.next().is_some()
    }

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Box::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Box::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> usize {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Box::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Box::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Box::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>