<commit_before>use std::sync::atomic::{self, AtomicPtr};
use std::cell::Cell;

static GARBAGE: Stack<Box<Drop>> = Stack::new();
static ACTIVE_SNAPSHOTS: Stack<RawSnapshot> = Stack::new();
static STATE: State = State::new();
static TICKS_BEFORE_GC: u16 = 2000;

thread_local!(static CLOCK: Cell<u16> = Cell::new(0));
thread_local!(static ACTIVE_READERS: Cell<usize> = Cell::new(0));
thread_local!(static GARBAGE_COLLECTING: Cell<bool> = Cell::new(false));

pub fn gc() {
    // Skip garbage collection if one is already happening in this thread.
    if GARBAGE_COLLECTING.get() {
        return;
    }

    // Check if there are any active readers.
    if ACTIVE_READERS.get() != 0 {
        // There are. We cannot garbage collect with active readers, hence we must skip it.
        return;
    }

    // Set the clock back to zero to delay the next garbage collection.
    CLOCK.set(0);

    // Start the garbage collection.
    if !STATE.start_gc() {
        // Another thread is garbage collecting, so we short-circuit. We won't set back the clock,
        // as the other thread will have collected the garbage.
        return;
    }

    // Set the garbage collecting flag to avoid nested, unbounded recursion in garbage collection
    // from destructors.
    GARBAGE_COLLECTING.set(true);

    // Initially, every garbage is marked unused.
    let mut unused = GARBAGE.collect();

    // Traverse the active snapshots and update the reference counts.
    ACTIVE_SNAPSHOTS.take_each(|reader| {
        if reader.active.load() {
            // The reader is not released yet, and is thus considered active.

            // Remove the reader from the unused set and insert it back into the log (if it
            // exists in the unused set), as the garbage is active.
            unused.remove(reader.ptr).map(|x| GARBAGE.insert(x));
            // Put the reader back in the structure.
            ACTIVE_SNAPSHOTS.insert(reader);
        } else {
            // The reader was released. Destroy it.
            reader.destroy();
        }
    });

    // End the garbage collection cycle.
    STATE.end_gc();
    GARBAGE_COLLECTING.set(false);
}

pub fn tick() {
    let clock = CLOCK.get();
    if clock == TICKS_BEFORE_GC {
        gc();
    } else {
        CLOCK.set(clock + 1);
    }
}

pub fn read<T, F>(f: F) -> T
where F: Fn(Reader) -> T {
    let mut active = ACTIVE_READERS.get();
    ACTIVE_READERS.set(active + 1);
    if active == 0 {
        STATE.start_read();
    }

    let reader = Reader;
    let ret = f(&reader);

    active = ACTIVE_READERS.get();
    if active == 1 {
        STATE.end_read();
    }
    ACTIVE_READERS.set(active - 1);

    ret
}

pub struct Reader;

impl Reader {
    pub fn load<T>(&self, a: &Atomic<T>) -> Snapshot<T> {
        // Construct the raw reader.
        let reader = RawSnapshot {
            // Load a snapshot of the pointer.
            ptr: self.inner.load(atomic::Ordering::Relaxed),
            // We allocate the atomic boolean on the heap as it is shared between the returned RAII
            // guard and the reader stack.
            released: Box::into_raw(Box::new(AtomicBool::new(false))),
        };

        // Register the reader through the reader stack, ensuring that it is not freed before the
        // RAII guard drops (`reader.release` is set to `true`).
        ACTIVE_SNAPSHOTS.push(reader);

        Snapshot {
            raw: reader,
            _marker: PhantomData,
        }
    }
}

pub struct Stack<T> {
    head: AtomicPtr<Node<T>>,
}

struct Node<T> {
    data: T,
    next: *mut Node<T>,
}

impl<T> Stack<T> {
    const fn new() -> Stack<T> {
        Stack {
            head: AtomicPtr::new(0 as *const T),
        }
    }

    fn push(&self, t: T) {
        // Allocate the node, and immediately turn it into a `*mut` pointer.
        let n = Box::into_raw(Box::new(Node {
            data: t,
            next: null_mut(),
        }));

        loop {
            // Snapshot current head.
            let head = self.head.load(atomic::Ordering::Relaxed);

            // Update `next` pointer with snapshot.
            unsafe { (*n).next = head; }

            // If snapshot is still good, link in new node.
            if self.head.compare_and_swap(head, n, atomic::Ordering::Release) == head {
                break
            }
        }
    }

    fn take_each(&self, f: F)
    where F: Fn(T) {
        // Replace the old head with a null pointer.
        let mut node = self.head.swap(AtomicPtr::default(), atomic::Ordering::Acquire);

        // We traverse every node until the pointer is null.
        while !node.is_null() {
            // Read the node into an owned box.
            let bx = unsafe { Box::from_raw(head) };
            // Apply the provided closure.
            f(bx.data);
            // Go to the next link.
            node = bx.next;
        }
    }

    fn collect(&self) -> HashSet<T> {
        let mut hs = HashSet::new();
        self.take_each(|x| hs.insert(x));

        hs
    }
}

struct RawSnapshot<T> {
    active: *const AtomicBool,
    ptr: *const T,
}

impl RawSnapshot {
    unsafe fn destroy(self) {
        // Drop the atomic boolean stored on the heap.
        mem::drop_in_place(self.active);
    }
}

struct Snapshot<'a, T> {
    raw: RawSnapshot,
    _marker: PhantomData<'a>,
}

impl<'a, T> Snapshot<'a, T> {
    fn drop(&mut self) {
        self.raw.active.store(true);
    }
}

#[derive(Default)]
struct State {
    flags: AtomicUsize,
}

impl State {
    fn start_gc(&self) -> bool {
        // Mark that a garbage collection is pending.
        if self.flags.fetch_or(1, atomic::Ordering::Relaxed) & 1 != 0 {
            // Another thread is pending to or currently garbage collecting, so we won't do the
            // same.
            return false;
        }

        // Spin until no thread is currently modifying the stacks. This prevents premature frees in
        // the thread which is currently pushing to the active snapshots.
        loop {
            // Read the flags, and if no active snapshots or garbage collectors, activate garbage
            // collection.
            let flags = self.flags.compare_and_swap(1, 0b11, atomic::Ordering::Relaxed);
            if flags == 1 {
                // Currently, no one accesses the active snapshot stack and the CAS above means
                // that the lowest bitflag have been set, indicating that a garbage collection is
                // now active.
                return true;
            }
        }
    }

    fn end_gc(&self) {
        self.flags.fetch_sub(1, atomic::Ordering::Relaxed);
    }

    fn start_read(&self) {
        // Increment the number of threads currently pushing to the active snapshot stack. We add
        // two to account for the LSB being a separate bitflag.
        self.flags.fetch_add(2, atomic::Ordering::Relaxed);
    }

    fn end_read(&self) {
        self.flags.fetch_sub(2, atomic::Ordering::Relaxed);
    }
}

pub struct Atomic<T> {
    inner: AtomicPtr<T>,
}

impl<T> Atomic<T> {
    pub fn new(inner: T) -> Atomic<T> {
        Atomic {
            inner: AtomicPtr::new(Box::into_raw(Box::new(inner))),
        }
    }

    pub fn load(&self) -> Snapshot<T> {
        read(|r| r.load(self))
    }

    pub fn store(&self, new: Box<T>) {
        // Replace the inner by the new value.
        let old = self.inner.swap(Box::into_raw(new), atomic::Ordering::Relaxed);
        // Push the old pointer to the garbage stack.
        GARBAGE.push(Box::from_raw(old));

        tick();
    }
}
<commit_msg>Allow overlapping memory reclamation cycles/garbage collection.<commit_after>use std::sync::atomic::{self, AtomicPtr};
use std::cell::Cell;

static GARBAGE: Stack<Box<Drop>> = Stack::new();
static ACTIVE_SNAPSHOTS: Stack<RawSnapshot> = Stack::new();
// TODO: rename, name's shite
static PUSHING_TO_ACTIVE_SNAPSHOTS: AtomicUsize = AtomicUsize::new(0);
static TICKS_BEFORE_GC: u16 = 2000;

thread_local!(static CLOCK: Cell<u16> = Cell::new(0));
thread_local!(static ACTIVE_READERS: Cell<usize> = Cell::new(0));

pub fn gc() {
    // Check if there are any active readers.
    if ACTIVE_READERS.get() != 0 {
        // There are. We cannot garbage collect with active readers, hence we must skip it.
        return;
    }

    // Set the clock back to zero to delay the next garbage collection.
    CLOCK.set(0);

    // Spin until no other thread is pushing snapshots, ensuring the stack of active snapshot is
    // not incomplete, and therefore collectable.
    while PUSHING_TO_ACTIVE_SNAPSHOTS.compare_and_swap(0, 1) != 0 {}

    // Initially, every garbage is marked unused.
    let mut unused = GARBAGE.collect();
    let snapshots = ACTIVE_SNAPSHOTS.take();

    // Set back the counter as our collection of the stacks is over.
    PUSHING_TO_ACTIVE_SNAPSHOTS.store(0);

    // Traverse the active snapshots and update the reference counts.
    snapshots.for_each(|reader| {
        if reader.active.load() {
            // The reader is not released yet, and is thus considered active.

            // Remove the reader from the unused set and insert it back into the log (if it
            // exists in the unused set), as the garbage is active.
            unused.remove(reader.ptr).map(|x| GARBAGE.insert(x));
            // Put the reader back in the structure.
            ACTIVE_SNAPSHOTS.insert(reader);
        } else {
            // The reader was released. Destroy it.
            reader.destroy();
        }
    });
}

pub fn tick() {
    let clock = CLOCK.get();
    if clock == TICKS_BEFORE_GC {
        gc();
    } else {
        CLOCK.set(clock + 1);
    }
}

pub fn read<T, F>(f: F) -> T
where F: Fn(Reader) -> T {
    let mut active = ACTIVE_READERS.get();
    ACTIVE_READERS.set(active + 1);
    if active == 0 {
        PUSHING_TO_ACTIVE_SNAPSHOTS.fetch_add(1);
    }

    let reader = Reader;
    let ret = f(&reader);

    active = ACTIVE_READERS.get();
    if active == 1 {
        PUSHING_TO_ACTIVE_SNAPSHOTS.fetch_sub(1);
    }
    ACTIVE_READERS.set(active - 1);

    ret
}

pub struct Reader;

impl Reader {
    pub fn load<T>(&self, a: &Atomic<T>) -> Snapshot<T> {
        // Construct the raw reader.
        let reader = RawSnapshot {
            // Load a snapshot of the pointer.
            ptr: self.inner.load(atomic::Ordering::Relaxed),
            // We allocate the atomic boolean on the heap as it is shared between the returned RAII
            // guard and the reader stack.
            released: Box::into_raw(Box::new(AtomicBool::new(false))),
        };

        // Register the reader through the reader stack, ensuring that it is not freed before the
        // RAII guard drops (`reader.release` is set to `true`).
        ACTIVE_SNAPSHOTS.push(reader);

        Snapshot {
            raw: reader,
            _marker: PhantomData,
        }
    }
}

pub struct Stack<T> {
    head: AtomicPtr<Node<T>>,
}

struct Node<T> {
    data: T,
    next: *mut Node<T>,
}

impl<T> Stack<T> {
    const fn new() -> Stack<T> {
        Stack {
            head: AtomicPtr::new(0 as *const T),
        }
    }

    fn push(&self, t: T) {
        // Allocate the node, and immediately turn it into a `*mut` pointer.
        let n = Box::into_raw(Box::new(Node {
            data: t,
            next: null_mut(),
        }));

        loop {
            // Snapshot current head.
            let head = self.head.load(atomic::Ordering::Relaxed);

            // Update `next` pointer with snapshot.
            unsafe { (*n).next = head; }

            // If snapshot is still good, link in new node.
            if self.head.compare_and_swap(head, n, atomic::Ordering::Release) == head {
                break
            }
        }
    }

    fn take(&self) -> Stack<T> {
        // Replace the old head with a null pointer.
        self.head.swap(AtomicPtr::default(), atomic::Ordering::Acquire)
    }

    fn for_each(self, f: F)
    where F: Fn(T) {
        let mut node = self.head;
        // We traverse every node until the pointer is null.
        while !node.is_null() {
            // Read the node into an owned box.
            let bx = unsafe { Box::from_raw(head) };
            // Apply the provided closure.
            f(bx.data);
            // Go to the next link.
            node = bx.next;
        }
    }

    fn collect(&self) -> HashSet<T> {
        let mut hs = HashSet::new();
        self.take_each(|x| hs.insert(x));

        hs
    }
}

struct RawSnapshot<T> {
    active: *const AtomicBool,
    ptr: *const T,
}

impl RawSnapshot {
    unsafe fn destroy(self) {
        // Drop the atomic boolean stored on the heap.
        mem::drop_in_place(self.active);
    }
}

struct Snapshot<'a, T> {
    raw: RawSnapshot,
    _marker: PhantomData<'a>,
}

impl<'a, T> Snapshot<'a, T> {
    fn drop(&mut self) {
        self.raw.active.store(true);
    }
}

pub struct Atomic<T> {
    inner: AtomicPtr<T>,
}

impl<T> Atomic<T> {
    pub fn new(inner: T) -> Atomic<T> {
        Atomic {
            inner: AtomicPtr::new(Box::into_raw(Box::new(inner))),
        }
    }

    pub fn load(&self) -> Snapshot<T> {
        read(|r| r.load(self))
    }

    pub fn store(&self, new: Box<T>) {
        // Replace the inner by the new value.
        let old = self.inner.swap(Box::into_raw(new), atomic::Ordering::Relaxed);
        // Push the old pointer to the garbage stack.
        GARBAGE.push(Box::from_raw(old));

        tick();
    }
}
<|endoftext|>


<commit_before>/// "Signaling" trait used in impl trait to tag lifetimes that you may
/// need to capture but don't really need for other reasons.
/// Basically a workaround; see [this comment] for details.
///
/// [this comment]: https://github.com/rust-lang/rust/issues/34511#issuecomment-373423999
// FIXME(eddyb) false positive, the lifetime parameter is "phantom" but needed.
#[allow(unused_lifetimes)]
pub trait Captures<'a> {}

impl<'a, T: ?Sized> Captures<'a> for T {}
<commit_msg>Remove `#[allow(unused_lifetimes)]` which is now unnecessary<commit_after>/// "Signaling" trait used in impl trait to tag lifetimes that you may
/// need to capture but don't really need for other reasons.
/// Basically a workaround; see [this comment] for details.
///
/// [this comment]: https://github.com/rust-lang/rust/issues/34511#issuecomment-373423999
pub trait Captures<'a> {}

impl<'a, T: ?Sized> Captures<'a> for T {}
<|endoftext|>


<commit_before>use super::ffi;
use { Lua, Push, CopyRead };

// this function is the main entry point when Lua wants to call one of our functions
extern fn wrapper1(lua: *mut ffi::lua_State) -> ::libc::c_int {
    // we load the pointer to the wrapper2 function from an upvalue (an upvalue is a value that was pushed alongside our function)
    let wrapper2raw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(2)) };
    let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = unsafe { ::std::mem::transmute(wrapper2raw) };

    wrapper2(lua)
}

// this function is called when Lua wants to call one of our functions
fn wrapper2<T: AnyCallable>(lua: *mut ffi::lua_State) -> ::libc::c_int {
    // loading the object that we want to call from the Lua context
    let dataRaw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(1)) };
    let data: &mut T = unsafe { ::std::mem::transmute(dataRaw) };

    data.load_args_and_call(lua)
}

// this trait should be implemented on objects that are pushed to be callbacks
trait AnyCallable {
    fn load_args_and_call(&mut self, lua: *mut ffi::lua_State) -> ::libc::c_int;
}

// should be implemented by objects that can be called
// this will be removed in favor of std::ops::Fn when it is widely supported
// "Args" should be a tuple containing the parameters
trait Callable<'lua, Args: CopyRead<Lua<'lua>>, Ret: Push<Lua<'lua>>> {
    fn do_call(&mut self, args: Args) -> Ret;
}

impl<'lua, Args: CopyRead<Lua<'lua>>, Ret: Push<Lua<'lua>>, T: Callable<'lua, Args, Ret>> AnyCallable for T {
    fn load_args_and_call(&mut self, lua: *mut ffi::lua_State)
        -> ::libc::c_int
    {
        // creating a temporary Lua context in order to pass it to push & read functions
        let mut tmpLua = Lua { lua: lua, marker: ::std::kinds::marker::ContravariantLifetime, must_be_closed: false, inside_callback: true } ;

        // trying to read the arguments
        let argumentsCount = unsafe { ffi::lua_gettop(lua) } as int;
        let args = match CopyRead::read_from_lua(&mut tmpLua, -argumentsCount as ::libc::c_int) {      // TODO: what if the user has the wrong params?
            None => {
                let errMsg = format!("wrong parameter types for callback function");
                errMsg.push_to_lua(&mut tmpLua);
                unsafe { ffi::lua_error(lua); }
                unreachable!()
            },
            Some(a) => a
        };

        let retValue = self.do_call(args);

        // pushing back the result of the function on the stack
        let nb = retValue.push_to_lua(&mut tmpLua);

        nb as ::libc::c_int
    }
}

// this macro will allow us to handle multiple parameters count
macro_rules! Push_function(
    ($s:ident, $args:ident, $b:block | $($ty:ident),*) => (
        impl<'lua, Ret: Push<Lua<'lua>> $(, $ty : CopyRead<Lua<'lua>>+Clone)*> Push<Lua<'lua>> for fn($($ty),*)->Ret {
            fn push_to_lua(self, lua: &mut Lua) -> uint {
                // pushing the function pointer as a userdata
                let luaDataRaw = unsafe { ffi::lua_newuserdata(lua.lua, ::std::mem::size_of_val(&self) as ::libc::size_t) };
                let luaData: *mut fn($($ty),*)->Ret = unsafe { ::std::mem::transmute(luaDataRaw) };
                unsafe { ::std::ptr::write(luaData, self) };

                // pushing wrapper2 as a lightuserdata
                let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = wrapper2::<fn($($ty),*)->Ret>;
                unsafe { ffi::lua_pushlightuserdata(lua.lua, ::std::mem::transmute(wrapper2)) };

                // pushing wrapper1 as a closure
                unsafe { ffi::lua_pushcclosure(lua.lua, ::std::mem::transmute(wrapper1), 2) };

                1
            }
        }

        #[allow(unused_variable)]
        impl<'lua, Ret: Push<Lua<'lua>> $(, $ty : CopyRead<Lua<'lua>>+Clone)*> Callable<'lua,($($ty),*),Ret> for fn($($ty),*)->Ret {
            fn do_call(&mut $s, $args: ($($ty),*)) -> Ret {
                $b
            }
        }

        impl<'lua, Ret: Push<Lua<'lua>> $(, $ty : CopyRead<Lua<'lua>>+Clone)*> Push<Lua<'lua>> for |$($ty),*|:'lua->Ret {
            fn push_to_lua(self, lua: &mut Lua) -> uint {
                // pushing the function pointer as a userdata
                let luaDataRaw = unsafe { ffi::lua_newuserdata(lua.lua, ::std::mem::size_of_val(&self) as ::libc::size_t) };
                let luaData: *mut |$($ty),*|->Ret = unsafe { ::std::mem::transmute(luaDataRaw) };
                unsafe { ::std::ptr::write(luaData, self) };

                // pushing wrapper2 as a lightuserdata
                let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = wrapper2::<|$($ty),*|:'lua->Ret>;
                unsafe { ffi::lua_pushlightuserdata(lua.lua, ::std::mem::transmute(wrapper2)) };

                // pushing wrapper1 as a closure
                unsafe { ffi::lua_pushcclosure(lua.lua, ::std::mem::transmute(wrapper1), 2) };

                1
            }
        }

        #[allow(unused_variable)]
        impl<'lua, Ret: Push<Lua<'lua>> $(, $ty : CopyRead<Lua<'lua>>+Clone)*> Callable<'lua,($($ty),*),Ret> for |$($ty),*|:'lua->Ret {
            fn do_call(&mut $s, $args: ($($ty),*)) -> Ret {
                $b
            }
        }
    );
)

Push_function!(self, args, { (*self)() } | )
Push_function!(self, args, { (*self)(args) } | Arg1 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone()) } | Arg1, Arg2 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone(), args.ref2().clone()) } | Arg1, Arg2, Arg3 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone(), args.ref2().clone(), args.ref3().clone()) } | Arg1, Arg2, Arg3, Arg4 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone(), args.ref2().clone(), args.ref3().clone(), args.ref4().clone()) } | Arg1, Arg2, Arg3, Arg4, Arg5 )
// TODO: finish

impl<'lua, T: Push<Lua<'lua>>, E: ::std::fmt::Show> Push<Lua<'lua>> for Result<T,E> {
    fn push_to_lua(self, lua: &mut Lua<'lua>) -> uint {
        if !lua.inside_callback {
            fail!("cannot push a Result object except as a function return type")
        }

        match self {
            Ok(val) => val.push_to_lua(lua),
            Err(val) => {
                let msg = format!("{}", val);
                msg.push_to_lua(lua);
                unsafe { ffi::lua_error(lua.lua); }
                unreachable!()
            }
        }
    }
}
<commit_msg>Add InsideCallback struct<commit_after>use super::ffi;
use {HasLua, Lua, Push, CopyRead};
use std::kinds::marker::ContravariantLifetime;

// this function is the main entry point when Lua wants to call one of our functions
extern fn wrapper1(lua: *mut ffi::lua_State) -> ::libc::c_int {
    // we load the pointer to the wrapper2 function from an upvalue (an upvalue is a value that was pushed alongside our function)
    let wrapper2raw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(2)) };
    let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = unsafe { ::std::mem::transmute(wrapper2raw) };

    wrapper2(lua)
}

// this function is called when Lua wants to call one of our functions
fn wrapper2<T: AnyCallable>(lua: *mut ffi::lua_State) -> ::libc::c_int {
    // loading the object that we want to call from the Lua context
    let dataRaw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(1)) };
    let data: &mut T = unsafe { ::std::mem::transmute(dataRaw) };

    data.load_args_and_call(lua)
}

// this trait should be implemented on objects that are pushed to be callbacks
trait AnyCallable {
    fn load_args_and_call(&mut self, lua: *mut ffi::lua_State) -> ::libc::c_int;
}

// lua context used inside callbacks
struct InsideCallback<'lua> {
    lua: *mut ffi::lua_State,
    marker: ContravariantLifetime<'lua>,
}

impl<'lua> HasLua<'lua> for InsideCallback<'lua> {
    fn use_lua(&mut self) -> *mut ffi::lua_State {
        self.lua
    }
}

// should be implemented by objects that can be called
// this will be removed in favor of std::ops::Fn when it is widely supported
// "Args" should be a tuple containing the parameters
trait Callable<'lua, Args: CopyRead<InsideCallback<'lua>>, Ret: Push<InsideCallback<'lua>>> {
    fn do_call(&mut self, args: Args) -> Ret;
}

impl<'lua, Args: CopyRead<InsideCallback<'lua>>, Ret: Push<InsideCallback<'lua>>, T: Callable<'lua, Args, Ret>> AnyCallable for T {
    fn load_args_and_call(&mut self, lua: *mut ffi::lua_State)
        -> ::libc::c_int
    {
        // creating a temporary Lua context in order to pass it to push & read functions
        let mut tmpLua = InsideCallback { lua: lua, marker: ::std::kinds::marker::ContravariantLifetime } ;

        // trying to read the arguments
        let argumentsCount = unsafe { ffi::lua_gettop(lua) } as int;
        let args = match CopyRead::read_from_lua(&mut tmpLua, -argumentsCount as ::libc::c_int) {      // TODO: what if the user has the wrong params?
            None => {
                let errMsg = format!("wrong parameter types for callback function");
                errMsg.push_to_lua(&mut tmpLua);
                unsafe { ffi::lua_error(lua); }
                unreachable!()
            },
            Some(a) => a
        };

        let retValue = self.do_call(args);

        // pushing back the result of the function on the stack
        let nb = retValue.push_to_lua(&mut tmpLua);

        nb as ::libc::c_int
    }
}

// this macro will allow us to handle multiple parameters count
macro_rules! Push_function(
    ($s:ident, $args:ident, $b:block | $($ty:ident),*) => (
        impl<'lua, L: HasLua<'lua>, Ret: Push<InsideCallback<'lua>> $(, $ty : CopyRead<InsideCallback<'lua>>+Clone)*> Push<L> for fn($($ty),*)->Ret {
            fn push_to_lua(self, lua: &mut L) -> uint {
                // pushing the function pointer as a userdata
                let luaDataRaw = unsafe { ffi::lua_newuserdata(lua.use_lua(), ::std::mem::size_of_val(&self) as ::libc::size_t) };
                let luaData: *mut fn($($ty),*)->Ret = unsafe { ::std::mem::transmute(luaDataRaw) };
                unsafe { ::std::ptr::write(luaData, self) };

                // pushing wrapper2 as a lightuserdata
                let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = wrapper2::<fn($($ty),*)->Ret>;
                unsafe { ffi::lua_pushlightuserdata(lua.use_lua(), ::std::mem::transmute(wrapper2)) };

                // pushing wrapper1 as a closure
                unsafe { ffi::lua_pushcclosure(lua.use_lua(), ::std::mem::transmute(wrapper1), 2) };

                1
            }
        }

        #[allow(unused_variable)]
        impl<'lua, Ret: Push<InsideCallback<'lua>> $(, $ty : CopyRead<InsideCallback<'lua>>+Clone)*> Callable<'lua,($($ty),*),Ret> for fn($($ty),*)->Ret {
            fn do_call(&mut $s, $args: ($($ty),*)) -> Ret {
                $b
            }
        }

        impl<'lua, L: HasLua<'lua>, Ret: Push<InsideCallback<'lua>> $(, $ty : CopyRead<InsideCallback<'lua>>+Clone)*> Push<L> for |$($ty),*|:'lua->Ret {
            fn push_to_lua(self, lua: &mut L) -> uint {
                // pushing the function pointer as a userdata
                let luaDataRaw = unsafe { ffi::lua_newuserdata(lua.use_lua(), ::std::mem::size_of_val(&self) as ::libc::size_t) };
                let luaData: *mut |$($ty),*|->Ret = unsafe { ::std::mem::transmute(luaDataRaw) };
                unsafe { ::std::ptr::write(luaData, self) };

                // pushing wrapper2 as a lightuserdata
                let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = wrapper2::<|$($ty),*|:'lua->Ret>;
                unsafe { ffi::lua_pushlightuserdata(lua.use_lua(), ::std::mem::transmute(wrapper2)) };

                // pushing wrapper1 as a closure
                unsafe { ffi::lua_pushcclosure(lua.use_lua(), ::std::mem::transmute(wrapper1), 2) };

                1
            }
        }

        #[allow(unused_variable)]
        impl<'lua, Ret: Push<InsideCallback<'lua>> $(, $ty : CopyRead<InsideCallback<'lua>>+Clone)*> Callable<'lua,($($ty),*),Ret> for |$($ty),*|:'lua->Ret {
            fn do_call(&mut $s, $args: ($($ty),*)) -> Ret {
                $b
            }
        }
    );
)

Push_function!(self, args, { (*self)() } | )
Push_function!(self, args, { (*self)(args) } | Arg1 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone()) } | Arg1, Arg2 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone(), args.ref2().clone()) } | Arg1, Arg2, Arg3 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone(), args.ref2().clone(), args.ref3().clone()) } | Arg1, Arg2, Arg3, Arg4 )
Push_function!(self, args, { (*self)(args.ref0().clone(), args.ref1().clone(), args.ref2().clone(), args.ref3().clone(), args.ref4().clone()) } | Arg1, Arg2, Arg3, Arg4, Arg5 )
// TODO: finish

impl<'lua, T: Push<InsideCallback<'lua>>, E: ::std::fmt::Show> Push<InsideCallback<'lua>> for Result<T,E> {
    fn push_to_lua(self, lua: &mut InsideCallback<'lua>) -> uint {
        match self {
            Ok(val) => val.push_to_lua(lua),
            Err(val) => {
                let msg = format!("{}", val);
                msg.push_to_lua(lua);
                unsafe { ffi::lua_error(lua.use_lua()); }
                unreachable!()
            }
        }
    }
}
<|endoftext|>


<commit_before>use clap::ArgMatches;
use hyper::Client;
use hyper::Url;
use std::io::prelude::*;
use rustc_serialize::json::Json;
use pact_mock_server::{
    lookup_mock_server,
    lookup_mock_server_by_port,
    MockServer
};

pub fn verify_mock_server(host: &str, port: u16, matches: &ArgMatches) -> Result<(), i32> {
    let mock_server_id = matches.value_of("mock-server-id");
    let mock_server_port = matches.value_of("mock-server-port");

    let client = Client::new();
    let url = Url::parse(format!("http://{}:{}/mockserver/{}/verify", host, port,
        if mock_server_id.is_some() { mock_server_id.unwrap() } else { mock_server_port.unwrap() })
        .as_str()).unwrap();
    let res = client.post(url.clone()).send();

    match res {
        Ok(mut result) => {
            let mut body = String::new();
            result.read_to_string(&mut body).unwrap();
            let json_result = Json::from_str(body.as_str());
            match json_result {
                Ok(json) => {
                    let mock_server = json.find("mockServer").unwrap();
                    let id = mock_server.find("id").unwrap().as_string().unwrap();
                    let port = mock_server.find("port").unwrap().as_u64().unwrap();
                    let status = mock_server.find("status").unwrap().as_string().unwrap();
                    if status == "ok" {
                        println!("Mock server {}/{} verified ok", id, port);
                        Ok(())
                    } else {
                        display_verification_errors(id, port, &json);
                        Err(2)
                    }
                },
                Err(err) => {
                    error!("Failed to parse JSON: {}\n{}", err, body);
                    ::display_error(format!("Failed to parse JSON: {}\n{}", err, body), matches);
                }
            }
        },
        Err(err) => {
            ::display_error(format!("Failed to connect to the master mock server '{}': {}", url, err), matches);
        }
    }
}

fn validate_port(id: u16) -> Result<MockServer, String> {
    lookup_mock_server_by_port(id as i32, &|ref ms| {
        MockServer {
            id: ms.id.clone(),
            port: ms.port,
            server: ms.server,
            matches: ms.matches.clone(),
            resources: vec![],
            pact: ms.pact.clone()
        }
    }).ok_or(format!("No mock server running with port '{}'", id))
}

fn validate_uuid(id: &String) -> Result<MockServer, String> {
    lookup_mock_server(id.clone(), &|ref ms| {
        MockServer {
            id: ms.id.clone(),
            port: ms.port,
            server: ms.server,
            matches: ms.matches.clone(),
            resources: vec![],
            pact: ms.pact.clone()
        }
    }).ok_or(format!("No mock server running with id '{}'", id))
}

pub fn validate_id(id: &str) -> Result<MockServer, String> {
    if id.chars().all(|ch| ch.is_digit(10)) {
        validate_port(id.parse::<u16>().unwrap())
    } else {
        validate_uuid(&s!(id))
    }
}

fn display_verification_errors(id: &str, port: u64, json: &Json) {
    let mismatches = json.find("mismatches").unwrap().as_array().unwrap();
    println!("Mock server {}/{} failed verification with {} errors\n", id, port, mismatches.len());

    for (i, mismatch) in mismatches.iter().enumerate() {
        match mismatch.find("type").unwrap().as_string().unwrap() {
            "missing-request" => {
                let request = mismatch.find("request").unwrap();
                println!("{} - Expected request was not received - {}", i, request)
            },
            "request-not-found" => {
                let request = mismatch.find("request").unwrap();
                println!("{} - Received a request that was not expected - {}", i, request)
            },
            "request-mismatch" => {
                let path = mismatch.find("path").unwrap().as_string().unwrap();
                let method = mismatch.find("method").unwrap().as_string().unwrap();
                println!("{} - Received a request that did not match with expected - {} {}", i, method, path);
                let request_mismatches = mismatch.find("mismatches").unwrap().as_array().unwrap();
                for request_mismatch in request_mismatches {
                    println!("        {}", request_mismatch.find("mismatch").unwrap().as_string().unwrap())
                }
            },
            _ => println!("{} - Known failure - {}", i, mismatch),
        }
    }
}
<commit_msg>correctly handle the status codes from the master mock server<commit_after>use clap::ArgMatches;
use hyper::Client;
use hyper::Url;
use hyper::status::*;
use std::io::prelude::*;
use rustc_serialize::json::Json;
use pact_mock_server::{
    lookup_mock_server,
    lookup_mock_server_by_port,
    MockServer
};

pub fn verify_mock_server(host: &str, port: u16, matches: &ArgMatches) -> Result<(), i32> {
    let mock_server_id = matches.value_of("mock-server-id");
    let mock_server_port = matches.value_of("mock-server-port");
    let id = if mock_server_id.is_some() {
        (mock_server_id.unwrap(), "id")
    } else {
        (mock_server_port.unwrap(), "port")
    };

    let client = Client::new();
    let url = Url::parse(format!("http://{}:{}/mockserver/{}/verify", host, port, id.0)
        .as_str()).unwrap();
    let res = client.post(url.clone()).send();

    match res {
        Ok(mut result) => {
            if !result.status.is_success() {
                match result.status {
                    StatusCode::NotFound => {
                        println!("No mock server found with {} '{}', use the 'list' command to get a list of available mock servers.",
                            id.1, id.0);
                        Err(3)
                    },
                    StatusCode::UnprocessableEntity => {
                        let mut body = String::new();
                        result.read_to_string(&mut body).unwrap();
                        let json_result = Json::from_str(body.as_str());
                        match json_result {
                            Ok(json) => {
                                let mock_server = json.find("mockServer").unwrap();
                                let id = mock_server.find("id").unwrap().as_string().unwrap();
                                let port = mock_server.find("port").unwrap().as_u64().unwrap();
                                display_verification_errors(id, port, &json);
                                Err(2)
                            },
                            Err(err) => {
                                error!("Failed to parse JSON: {}\n{}", err, body);
                                ::display_error(format!("Failed to parse JSON: {}\n{}", err, body), matches);
                            }
                        }
                    },
                    _ => ::display_error(format!("Unexpected response from master mock server '{}': {}", url, result.status), matches)
                }
            } else {
                println!("Mock server with {} '{}' verified ok", id.1, id.0);
                Ok(())
            }
        },
        Err(err) => {
            ::display_error(format!("Failed to connect to the master mock server '{}': {}", url, err), matches);
        }
    }
}

fn validate_port(id: u16) -> Result<MockServer, String> {
    lookup_mock_server_by_port(id as i32, &|ref ms| {
        MockServer {
            id: ms.id.clone(),
            port: ms.port,
            server: ms.server,
            matches: ms.matches.clone(),
            resources: vec![],
            pact: ms.pact.clone()
        }
    }).ok_or(format!("No mock server running with port '{}'", id))
}

fn validate_uuid(id: &String) -> Result<MockServer, String> {
    lookup_mock_server(id.clone(), &|ref ms| {
        MockServer {
            id: ms.id.clone(),
            port: ms.port,
            server: ms.server,
            matches: ms.matches.clone(),
            resources: vec![],
            pact: ms.pact.clone()
        }
    }).ok_or(format!("No mock server running with id '{}'", id))
}

pub fn validate_id(id: &str) -> Result<MockServer, String> {
    if id.chars().all(|ch| ch.is_digit(10)) {
        validate_port(id.parse::<u16>().unwrap())
    } else {
        validate_uuid(&s!(id))
    }
}

fn display_verification_errors(id: &str, port: u64, json: &Json) {
    let mismatches = json.find("mismatches").unwrap().as_array().unwrap();
    println!("Mock server {}/{} failed verification with {} errors\n", id, port, mismatches.len());

    for (i, mismatch) in mismatches.iter().enumerate() {
        match mismatch.find("type").unwrap().as_string().unwrap() {
            "missing-request" => {
                let request = mismatch.find("request").unwrap();
                println!("{} - Expected request was not received - {}", i, request)
            },
            "request-not-found" => {
                let request = mismatch.find("request").unwrap();
                println!("{} - Received a request that was not expected - {}", i, request)
            },
            "request-mismatch" => {
                let path = mismatch.find("path").unwrap().as_string().unwrap();
                let method = mismatch.find("method").unwrap().as_string().unwrap();
                println!("{} - Received a request that did not match with expected - {} {}", i, method, path);
                let request_mismatches = mismatch.find("mismatches").unwrap().as_array().unwrap();
                for request_mismatch in request_mismatches {
                    println!("        {}", request_mismatch.find("mismatch").unwrap().as_string().unwrap())
                }
            },
            _ => println!("{} - Known failure - {}", i, mismatch),
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! A base for xi plugins. Will be split out into its own crate once it's a bit more stable.

use std::io;
use std::path::{PathBuf, Path};

use serde_json::{self, Value};

use xi_core::{ViewIdentifier, PluginPid, plugin_rpc, SyntaxDefinition,
ConfigTable, BufferConfig};
use xi_rpc::{self, RpcLoop, RpcCtx, RemoteError, ReadError};

#[derive(Debug)]
pub enum Error {
    RpcError(xi_rpc::Error),
    WrongReturnType,
}

pub trait Handler {
    fn handle_notification(&mut self, ctx: PluginCtx,
                           rpc: plugin_rpc::HostNotification);
    fn handle_request(&mut self, ctx: PluginCtx, rpc: plugin_rpc::HostRequest)
                      -> Result<Value, RemoteError>;
    #[allow(unused_variables)]
    fn idle(&mut self, ctx: PluginCtx, token: usize) {}
}

/// A container for general view information, shared between all plugin layers.
pub struct ViewState {
    pub view_id: ViewIdentifier,
    pub syntax: SyntaxDefinition,
    config_table: ConfigTable,
    pub config: Option<BufferConfig>,
    pub path: Option<PathBuf>,
}

pub struct PluginCtx<'a> {
    inner: &'a RpcCtx,
    pub view: &'a ViewState,
    plugin_id: PluginPid,
}

impl ViewState {
    fn new(init_info: &plugin_rpc::PluginBufferInfo) -> Self {

        let &plugin_rpc::PluginBufferInfo {
            ref views, ref path, ref syntax, ref config, ..
        } = init_info;

        ViewState {
            view_id: *views.first().unwrap(),
            syntax: *syntax,
            config_table: config.clone(),
            config: serde_json::from_value(Value::Object(config.clone())).unwrap(),
            path: path.as_ref().map(PathBuf::from)
        }
    }

    fn update_config(&mut self, changes: &ConfigTable) {
        for (key, value) in changes.iter() {
            self.config_table.insert(key.to_owned(), value.to_owned());
        }
        let conf = serde_json::from_value(Value::Object(self.config_table.clone()));
        self.config = conf.unwrap();
    }

    fn update_path(&mut self, path: &Path) {
        self.path = Some(path.to_owned())
    }
}

impl<'a> PluginCtx<'a> {
    fn new(inner: &'a RpcCtx, view: &'a ViewState, plugin_id: PluginPid) -> Self {
        PluginCtx { inner, view, plugin_id }
    }

    pub fn get_data(&self, offset: usize, max_size: usize, rev: u64)
        -> Result<String, Error> {
        let params = json!({
            "plugin_id": self.plugin_id,
            "view_id": self.view.view_id,
            "offset": offset,
            "max_size": max_size,
            "rev": rev,
        });
        let result = self.send_rpc_request("get_data", &params);
        match result {
            Ok(Value::String(s)) => Ok(s),
            Ok(_) => Err(Error::WrongReturnType),
            Err(err) => Err(Error::RpcError(err)),
        }
    }

    pub fn add_scopes(&self, scopes: &Vec<Vec<String>>) {
        let params = json!({
            "plugin_id": self.plugin_id,
            "view_id": self.view.view_id,
            "scopes": scopes,
        });
        self.send_rpc_notification("add_scopes", &params);
    }

    pub fn update_spans(&self, start: usize, len: usize, rev: u64, spans: &[plugin_rpc::ScopeSpan]) {
        let params = json!({
            "plugin_id": self.plugin_id,
            "view_id": self.view.view_id,
            "start": start,
            "len": len,
            "rev": rev,
            "spans": spans,
        });
        self.send_rpc_notification("update_spans", &params);
    }

    fn send_rpc_notification(&self, method: &str, params: &Value) {
        self.inner.get_peer().send_rpc_notification(method, params)
    }

    fn send_rpc_request(&self, method: &str, params: &Value) -> Result<Value, xi_rpc::Error> {
        self.inner.get_peer().send_rpc_request(method, params)
    }

    /// Determines whether an incoming request (or notification) is pending. This
    /// is intended to reduce latency for bulk operations done in the background.
    pub fn request_is_pending(&self) -> bool {
        self.inner.get_peer().request_is_pending()
    }

    /// Schedule the idle handler to be run when there are no requests pending.
    pub fn schedule_idle(&mut self, token: usize) {
        self.inner.schedule_idle(token);
    }
}

struct MyHandler<'a, H: 'a> {
    inner: &'a mut H,
    plugin_id: Option<PluginPid>,
    state: Option<ViewState>,
}

impl<'a, H: 'a> MyHandler<'a, H> {
    fn new(inner: &'a mut H) -> Self {
        MyHandler {
            inner: inner,
            plugin_id: None,
            state: None,
        }
    }

    fn expect_state_mut(&mut self) -> &mut ViewState {
        self.state.as_mut()
            .expect("missing state; was plugin init RPC sent?")
    }
}

impl<'a, H: Handler> xi_rpc::Handler for MyHandler<'a, H> {
    type Notification = plugin_rpc::HostNotification;
    type Request = plugin_rpc::HostRequest;
    fn handle_notification(&mut self, ctx: &RpcCtx, rpc: Self::Notification) {
        use self::plugin_rpc::HostNotification::*;
        // we handle a few RPCs here, updating basic view information
        // before forwarding to the actual handler.
        match rpc {
            // don't forward ping before we're initialized
            Ping( .. ) => { if self.state.is_none() { return } }
            Initialize { ref plugin_id, ref buffer_info } => {
                assert!(self.state.is_none());
                self.state = Some(ViewState::new(buffer_info.first().as_ref().expect("missing buffer info?")));
                self.plugin_id = Some(*plugin_id);
            }

            ConfigChanged { ref changes, .. } =>
                self.expect_state_mut().update_config(changes),

            DidSave { ref path, .. } =>
                self.expect_state_mut().update_path(path),

            TracingConfig {enabled} => {
                use xi_trace;

                if enabled {
                    eprintln!("Enabling tracing in {:?}", self.plugin_id);
                    xi_trace::enable_tracing();
                } else {
                    eprintln!("Disabling tracing in {:?}",  self.plugin_id);
                    xi_trace::disable_tracing();
                }
            }
            _ => (),
        }

        let plugin_ctx = PluginCtx::new(
            ctx, self.state.as_ref().unwrap(), self.plugin_id.unwrap());
        self.inner.handle_notification(plugin_ctx, rpc)
    }

    fn handle_request(&mut self, ctx: &RpcCtx, rpc: Self::Request)
                      -> Result<Value, RemoteError> {
        assert!(self.state.is_some(), "request received before init: {:?}", &rpc);
        let plugin_ctx = PluginCtx::new(
            ctx, self.state.as_ref().unwrap(), self.plugin_id.unwrap());
        self.inner.handle_request(plugin_ctx, rpc)
    }

    fn idle(&mut self, ctx: &RpcCtx, token: usize) {
        let plugin_ctx = PluginCtx::new(
            ctx, self.state.as_ref().unwrap(), self.plugin_id.unwrap());
        self.inner.idle(plugin_ctx, token);
    }
}

pub fn mainloop<H: Handler>(handler: &mut H) -> Result<(), ReadError> {
    let stdin = io::stdin();
    let stdout = io::stdout();
    let mut rpc_looper = RpcLoop::new(stdout);
    let mut my_handler = MyHandler::new(handler);

    rpc_looper.mainloop(|| stdin.lock(), &mut my_handler)
}
<commit_msg>Minor doc and naming changes in plugin lib<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! A base for xi plugins. Will be split out into its own crate once it's a bit more stable.

use std::io;
use std::path::{PathBuf, Path};

use serde_json::{self, Value};

use xi_core::{ViewIdentifier, PluginPid, plugin_rpc, SyntaxDefinition,
ConfigTable, BufferConfig};
use xi_rpc::{self, RpcLoop, RpcCtx, RemoteError, ReadError};

#[derive(Debug)]
pub enum Error {
    RpcError(xi_rpc::Error),
    WrongReturnType,
}

pub trait Handler {
    fn handle_notification(&mut self, ctx: PluginCtx,
                           rpc: plugin_rpc::HostNotification);
    fn handle_request(&mut self, ctx: PluginCtx, rpc: plugin_rpc::HostRequest)
                      -> Result<Value, RemoteError>;
    #[allow(unused_variables)]
    fn idle(&mut self, ctx: PluginCtx, token: usize) {}
}

/// A container for general view information, shared between all plugin layers.
pub struct ViewState {
    pub view_id: ViewIdentifier,
    pub syntax: SyntaxDefinition,
    config_table: ConfigTable,
    pub config: Option<BufferConfig>,
    pub path: Option<PathBuf>,
}

pub struct PluginCtx<'a> {
    inner: &'a RpcCtx,
    /// Information about the view initiating this RPC.
    pub view: &'a ViewState,
    plugin_id: PluginPid,
}

/// The handler that does low level plugin setup, and then forwards RPC calls
/// to another `Handler` type.
struct BaseHandler<'a, H: 'a> {
    inner: &'a mut H,
    plugin_id: Option<PluginPid>,
    state: Option<ViewState>,
}

impl ViewState {
    fn new(init_info: &plugin_rpc::PluginBufferInfo) -> Self {

        let &plugin_rpc::PluginBufferInfo {
            ref views, ref path, ref syntax, ref config, ..
        } = init_info;

        ViewState {
            view_id: *views.first().unwrap(),
            syntax: *syntax,
            config_table: config.clone(),
            config: serde_json::from_value(Value::Object(config.clone())).unwrap(),
            path: path.as_ref().map(PathBuf::from)
        }
    }

    fn update_config(&mut self, changes: &ConfigTable) {
        for (key, value) in changes.iter() {
            self.config_table.insert(key.to_owned(), value.to_owned());
        }
        let conf = serde_json::from_value(Value::Object(self.config_table.clone()));
        self.config = conf.unwrap();
    }

    fn update_path(&mut self, path: &Path) {
        self.path = Some(path.to_owned())
    }
}

impl<'a> PluginCtx<'a> {
    fn new(inner: &'a RpcCtx, view: &'a ViewState, plugin_id: PluginPid) -> Self {
        PluginCtx { inner, view, plugin_id }
    }

    pub fn get_data(&self, offset: usize, max_size: usize, rev: u64)
        -> Result<String, Error> {
        let params = json!({
            "plugin_id": self.plugin_id,
            "view_id": self.view.view_id,
            "offset": offset,
            "max_size": max_size,
            "rev": rev,
        });
        let result = self.send_rpc_request("get_data", &params);
        match result {
            Ok(Value::String(s)) => Ok(s),
            Ok(_) => Err(Error::WrongReturnType),
            Err(err) => Err(Error::RpcError(err)),
        }
    }

    pub fn add_scopes(&self, scopes: &Vec<Vec<String>>) {
        let params = json!({
            "plugin_id": self.plugin_id,
            "view_id": self.view.view_id,
            "scopes": scopes,
        });
        self.send_rpc_notification("add_scopes", &params);
    }

    pub fn update_spans(&self, start: usize, len: usize, rev: u64, spans: &[plugin_rpc::ScopeSpan]) {
        let params = json!({
            "plugin_id": self.plugin_id,
            "view_id": self.view.view_id,
            "start": start,
            "len": len,
            "rev": rev,
            "spans": spans,
        });
        self.send_rpc_notification("update_spans", &params);
    }

    fn send_rpc_notification(&self, method: &str, params: &Value) {
        self.inner.get_peer().send_rpc_notification(method, params)
    }

    fn send_rpc_request(&self, method: &str, params: &Value) -> Result<Value, xi_rpc::Error> {
        self.inner.get_peer().send_rpc_request(method, params)
    }

    /// Determines whether an incoming request (or notification) is pending. This
    /// is intended to reduce latency for bulk operations done in the background.
    pub fn request_is_pending(&self) -> bool {
        self.inner.get_peer().request_is_pending()
    }

    /// Schedule the idle handler to be run when there are no requests pending.
    pub fn schedule_idle(&mut self, token: usize) {
        self.inner.schedule_idle(token);
    }
}

impl<'a, H: 'a> BaseHandler<'a, H> {
    fn new(inner: &'a mut H) -> Self {
        BaseHandler {
            inner: inner,
            plugin_id: None,
            state: None,
        }
    }

    fn expect_state_mut(&mut self) -> &mut ViewState {
        self.state.as_mut()
            .expect("missing state; was plugin init RPC sent?")
    }
}

impl<'a, H: Handler> xi_rpc::Handler for BaseHandler<'a, H> {
    type Notification = plugin_rpc::HostNotification;
    type Request = plugin_rpc::HostRequest;
    fn handle_notification(&mut self, ctx: &RpcCtx, rpc: Self::Notification) {
        use self::plugin_rpc::HostNotification::*;
        // we handle a few RPCs here, updating basic view information
        // before forwarding to the actual handler.
        match rpc {
            // don't forward ping before we're initialized
            Ping( .. ) => { if self.state.is_none() { return } }
            Initialize { ref plugin_id, ref buffer_info } => {
                assert!(self.state.is_none());
                self.state = Some(ViewState::new(buffer_info.first().as_ref().expect("missing buffer info?")));
                self.plugin_id = Some(*plugin_id);
            }

            ConfigChanged { ref changes, .. } =>
                self.expect_state_mut().update_config(changes),

            DidSave { ref path, .. } =>
                self.expect_state_mut().update_path(path),

            TracingConfig {enabled} => {
                use xi_trace;

                if enabled {
                    eprintln!("Enabling tracing in {:?}", self.plugin_id);
                    xi_trace::enable_tracing();
                } else {
                    eprintln!("Disabling tracing in {:?}",  self.plugin_id);
                    xi_trace::disable_tracing();
                }
            }
            _ => (),
        }

        let plugin_ctx = PluginCtx::new(
            ctx, self.state.as_ref().unwrap(), self.plugin_id.unwrap());
        self.inner.handle_notification(plugin_ctx, rpc)
    }

    fn handle_request(&mut self, ctx: &RpcCtx, rpc: Self::Request)
                      -> Result<Value, RemoteError> {
        assert!(self.state.is_some(), "request received before init: {:?}", &rpc);
        let plugin_ctx = PluginCtx::new(
            ctx, self.state.as_ref().unwrap(), self.plugin_id.unwrap());
        self.inner.handle_request(plugin_ctx, rpc)
    }

    fn idle(&mut self, ctx: &RpcCtx, token: usize) {
        let plugin_ctx = PluginCtx::new(
            ctx, self.state.as_ref().unwrap(), self.plugin_id.unwrap());
        self.inner.idle(plugin_ctx, token);
    }
}

pub fn mainloop<H: Handler>(handler: &mut H) -> Result<(), ReadError> {
    let stdin = io::stdin();
    let stdout = io::stdout();
    let mut rpc_looper = RpcLoop::new(stdout);
    let mut my_handler = BaseHandler::new(handler);

    rpc_looper.mainloop(|| stdin.lock(), &mut my_handler)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::fs::File;
use std::io::{self, Write};
use std::path::Path;

use flate2;
use flate2::write::GzEncoder;
use tar::{Builder, Header};
use walkdir::WalkDir;
use xz2::write::XzEncoder;

use errors::*;
use util::*;

actor!{
    #[derive(Debug)]
    pub struct Tarballer {
        /// The input folder to be compressed
        input: String = "package",

        /// The prefix of the tarballs
        output: String = "./dist",

        /// The folder in which the input is to be found
        work_dir: String = "./workdir",
    }
}

impl Tarballer {
    /// Generate the actual tarballs
    pub fn run(self) -> Result<()> {
        let tar_gz = self.output.clone() + ".tar.gz";
        let tar_xz = self.output.clone() + ".tar.xz";

        // Remove any existing files
        for file in &[&tar_gz, &tar_xz] {
            if Path::new(file).exists() {
                remove_file(file)?;
            }
        }

        // Sort files by their suffix, to group files with the same name from
        // different locations (likely identical) and files with the same
        // extension (likely containing similar data).
        let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input)
            .chain_err(|| "failed to collect file paths")?;
        files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev()));

        // Prepare the .tar.gz file
        let gz = GzEncoder::new(create_new_file(tar_gz)?, flate2::Compression::Best);

        // Prepare the .tar.xz file
        let xz = XzEncoder::new(create_new_file(tar_xz)?, 9);

        // Write the tar into both encoded files.  We write all directories
        // first, so files may be directly created. (see rustup.rs#1092)
        let mut builder = Builder::new(Tee(gz, xz));
        for path in dirs {
            let src = Path::new(&self.work_dir).join(&path);
            builder.append_dir(&path, &src)
                .chain_err(|| format!("failed to tar dir '{}'", src.display()))?;
        }
        for path in files {
            let src = Path::new(&self.work_dir).join(&path);
            let file = open_file(&src)?;
            builder.append_data(&mut header(&src, &file)?, &path, &file)
                .chain_err(|| format!("failed to tar file '{}'", src.display()))?;
        }
        let Tee(gz, xz) = builder.into_inner()
            .chain_err(|| "failed to finish writing .tar stream")?;

        // Finish both encoded files
        gz.finish().chain_err(|| "failed to finish .tar.gz file")?;
        xz.finish().chain_err(|| "failed to finish .tar.xz file")?;

        Ok(())
    }
}

fn header(src: &Path, file: &File) -> Result<Header> {
    let mut header = Header::new_gnu();
    header.set_metadata(&file.metadata()?);
    if cfg!(windows) {
        // Windows doesn't really have a mode, so `tar` never marks files executable.
        // Use an extension whitelist to update files that usually should be so.
        const EXECUTABLES: [&'static str; 4] = ["exe", "dll", "py", "sh"];
        if let Some(ext) = src.extension().and_then(|s| s.to_str()) {
            if EXECUTABLES.contains(&ext) {
                let mode = header.mode()?;
                header.set_mode(mode | 0o111);
            }
        }
    }
    Ok(header)
}

/// Returns all `(directories, files)` under the source path
fn get_recursive_paths<P, Q>(root: P, name: Q) -> Result<(Vec<String>, Vec<String>)>
    where P: AsRef<Path>, Q: AsRef<Path>
{
    let root = root.as_ref();
    let name = name.as_ref();

    if !name.is_relative() && !name.starts_with(root) {
        bail!("input '{}' is not in work dir '{}'", name.display(), root.display());
    }

    let mut dirs = vec![];
    let mut files = vec![];
    for entry in WalkDir::new(root.join(name)) {
        let entry = entry?;
        let path = entry.path().strip_prefix(root)?;
        let path = path_to_str(&path)?;

        if entry.file_type().is_dir() {
            dirs.push(path.to_owned());
        } else {
            files.push(path.to_owned());
        }
    }
    Ok((dirs, files))
}

struct Tee<A, B>(A, B);

impl<A: Write, B: Write> Write for Tee<A, B> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.0.write_all(buf)
            .and(self.1.write_all(buf))
            .and(Ok(buf.len()))
    }

    fn flush(&mut self) -> io::Result<()> {
        self.0.flush().and(self.1.flush())
    }
}
<commit_msg>Create xz/gz files in parallel<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use std::thread;

use flate2;
use flate2::write::GzEncoder;
use tar::{Builder, Header};
use walkdir::WalkDir;
use xz2::write::XzEncoder;

use errors::*;
use util::*;

actor!{
    #[derive(Debug)]
    pub struct Tarballer {
        /// The input folder to be compressed
        input: String = "package",

        /// The prefix of the tarballs
        output: String = "./dist",

        /// The folder in which the input is to be found
        work_dir: String = "./workdir",
    }
}

impl Tarballer {
    /// Generate the actual tarballs
    pub fn run(self) -> Result<()> {
        let tar_gz = self.output.clone() + ".tar.gz";
        let tar_xz = self.output.clone() + ".tar.xz";

        // Remove any existing files
        for file in &[&tar_gz, &tar_xz] {
            if Path::new(file).exists() {
                remove_file(file)?;
            }
        }

        // Sort files by their suffix, to group files with the same name from
        // different locations (likely identical) and files with the same
        // extension (likely containing similar data).
        let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input)
            .chain_err(|| "failed to collect file paths")?;
        files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev()));

        // Write the tar into both encoded files.  We write all directories
        // first, so files may be directly created. (see rustup.rs#1092)
        let mut builder = Builder::new(Vec::new());
        for path in dirs {
            let src = Path::new(&self.work_dir).join(&path);
            builder.append_dir(&path, &src)
                .chain_err(|| format!("failed to tar dir '{}'", src.display()))?;
        }
        for path in files {
            let src = Path::new(&self.work_dir).join(&path);
            let file = open_file(&src)?;
            builder.append_data(&mut header(&src, &file)?, &path, &file)
                .chain_err(|| format!("failed to tar file '{}'", src.display()))?;
        }
        let contents = builder.into_inner()
            .chain_err(|| "failed to finish writing .tar stream")?;
        let contents = Arc::new(contents);

        // Prepare the .tar.gz file
        let contents2 = contents.clone();
        let t = thread::spawn(move || {
            let mut gz = GzEncoder::new(create_new_file(tar_gz)?,
                                        flate2::Compression::Best);
            gz.write_all(&contents2).chain_err(|| "failed to write .gz")?;
            gz.finish().chain_err(|| "failed to finish .gz")
        });

        // Prepare the .tar.xz file
        let mut xz = XzEncoder::new(create_new_file(tar_xz)?, 9);
        xz.write_all(&contents).chain_err(|| "failed to write .xz")?;
        xz.finish().chain_err(|| "failed to finish .xz")?;

        t.join().unwrap()?;

        Ok(())
    }
}

fn header(src: &Path, file: &File) -> Result<Header> {
    let mut header = Header::new_gnu();
    header.set_metadata(&file.metadata()?);
    if cfg!(windows) {
        // Windows doesn't really have a mode, so `tar` never marks files executable.
        // Use an extension whitelist to update files that usually should be so.
        const EXECUTABLES: [&'static str; 4] = ["exe", "dll", "py", "sh"];
        if let Some(ext) = src.extension().and_then(|s| s.to_str()) {
            if EXECUTABLES.contains(&ext) {
                let mode = header.mode()?;
                header.set_mode(mode | 0o111);
            }
        }
    }
    Ok(header)
}

/// Returns all `(directories, files)` under the source path
fn get_recursive_paths<P, Q>(root: P, name: Q) -> Result<(Vec<String>, Vec<String>)>
    where P: AsRef<Path>, Q: AsRef<Path>
{
    let root = root.as_ref();
    let name = name.as_ref();

    if !name.is_relative() && !name.starts_with(root) {
        bail!("input '{}' is not in work dir '{}'", name.display(), root.display());
    }

    let mut dirs = vec![];
    let mut files = vec![];
    for entry in WalkDir::new(root.join(name)) {
        let entry = entry?;
        let path = entry.path().strip_prefix(root)?;
        let path = path_to_str(&path)?;

        if entry.file_type().is_dir() {
            dirs.push(path.to_owned());
        } else {
            files.push(path.to_owned());
        }
    }
    Ok((dirs, files))
}
<|endoftext|>


<commit_before><commit_msg>Rename protocol::{Read, Write} to protocol::{Encode, Decode}.<commit_after><|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A tokenizer is a pattern matcher that is intended to turn a stream of symbols into another stream of symbols based on the patterns
//! that are matched. Every pattern can produce a different output symbol. If two input strings can ndfa in two different output
//! symbols, then the output symbol that is ordered lower is the one that's produced (ie, if the output symbols are numbers, then '0' will
//! be produced instead of '1' in the event of a clash)
//!

use super::countable::*;
use super::symbol_range::*;
use super::regular_pattern::*;
use super::state_machine::*;
use super::ndfa::*;
use super::prepare::*;
use super::symbol_range_dfa::*;
use super::symbol_reader::*;
use super::pattern_matcher::*;
use super::matches::*;
use super::tape::*;

///
/// Used for generating tokenizing pattern matchers
///
pub struct TokenMatcher<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord> {
    patterns: Vec<(Pattern<InputSymbol>, OutputSymbol)>
}

impl<InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> TokenMatcher<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new TokenMatcher
    ///
    pub fn new() -> TokenMatcher<InputSymbol, OutputSymbol> {
        TokenMatcher { patterns: vec![] }
    }

    ///
    /// Adds a new pattern that will generate the specified output symbol
    ///
    pub fn add_pattern<TPattern: ToPattern<InputSymbol>>(&mut self, pattern: TPattern, output: OutputSymbol) {
        self.patterns.push((pattern.to_pattern(), output));
    }

    ///
    /// Compiles an NDFA from this TokenMatcher
    ///
    pub fn to_ndfa(&self) -> Box<StateMachine<SymbolRange<InputSymbol>, OutputSymbol>> {
        let mut ndfa = Ndfa::new();

        for &(ref pattern, ref output) in &self.patterns {
            // Compile each pattern starting at state 0
            let end_state = pattern.compile(&mut ndfa, 0);

            // Set the output for this pattern
            ndfa.set_output_symbol(end_state, output.clone());
        }

        // Clear out any overlapping ranges so we can build a valid DFA
        ndfa.fix_overlapping_ranges();

        Box::new(ndfa)
    }
}

impl<'a, InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>
for &'a TokenMatcher<InputSymbol, OutputSymbol> {
    #[inline]
    fn prepare_to_match(self) -> SymbolRangeDfa<InputSymbol, OutputSymbol> {
        let ndfa = self.to_ndfa();

        ndfa.prepare_to_match()
    }
}

///
/// A tokenizer is a type of symbol stream that uses a pattern matcher to convert a symbol stream into a stream of tokens
///
pub struct Tokenizer<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> {
    /// The pattern matcher for this tokenizer
    dfa: SymbolRangeDfa<InputSymbol, OutputSymbol>,

    /// Tape of input symbols that will be used to generate the result
    tape: Tape<InputSymbol, Reader>,
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> Tokenizer<InputSymbol, OutputSymbol, Reader> {
    ///
    /// Creates a new tokenizer from a pattern (usually a TokenMatcher)
    ///
    pub fn new<Prepare: PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>>(source: Reader, pattern: Prepare) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern.prepare_to_match(), tape: Tape::new(source) }
    }

    ///
    /// Creates a new tokenizer from a prepared pattern
    ///
    pub fn new_prepared(source: Reader, pattern: SymbolRangeDfa<InputSymbol, OutputSymbol>) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern, tape: Tape::new(source) }
    }

    ///
    /// Returns the current position in the source (the position after the last matched symbol)
    ///
    pub fn get_source_position(&self) -> usize {
        self.tape.get_source_position()
    }

    ///
    /// Skips an input symbol (returning the symbol that was skipped)
    ///
    pub fn skip_input(&mut self) -> Option<InputSymbol> {
        self.tape.next_symbol()
    }

    ///
    /// True if we've reached the end of the source reader
    ///
    /// If `next_symbol` returns `None` and `at_end_of_reader` is false, then the input stream does not contain a symbol matching the DFA
    ///
    pub fn at_end_of_reader(&self) -> bool {
        self.tape.at_end_of_reader()
    }
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord+'static, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for Tokenizer<InputSymbol, OutputSymbol, Reader> {
    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        // Start of the next symbol
        let start_pos = self.tape.get_source_position();

        // Match against it
        let match_result = match_pattern(self.dfa.start(), &mut self.tape);

        let end_pos = self.tape.get_source_position();
        match match_result {
            Accept(length, outputsymbol) => {
                if length > 0 {
                    // Rewind the tape to after the accepted symbol
                    self.tape.rewind(end_pos-start_pos - length);

                    // Won't try to match anything before this position
                    self.tape.cut();

                    // Result is the oputput symbol
                    Some(outputsymbol.clone())
                } else {
                    // Zero-length match
                    // If we accepted matches of length 0 we'd get an infinite stream when we hit a symbol that doesn't match
                    self.tape.rewind(end_pos-start_pos);

                    // Return no match
                    None
                }
            },

            Reject => {
                // Rewind back to the start position
                self.tape.rewind(end_pos-start_pos);

                // No match
                None
            },

            _ => {
                panic!("Unexpected output state of ");
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_match_tokens_like_any_other_pattern() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        assert!(matches("aaaa", &token_matcher) == Some(4));
        assert!(matches("bbbbb", &token_matcher) == Some(5));
        assert!(matches("abbb", &token_matcher) == Some(1));
        assert!(matches("bbaaa", &token_matcher) == Some(2));
    }

    #[test]
    fn can_distinguish_simple_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn clashes_producer_lower_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Abbb,
            Aaab
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1).append("b"), TestToken::Aaab);
        token_matcher.add_pattern("a".append("b".repeat_forever(1)), TestToken::Abbb);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaab".read_symbols()).is_accepted(&TestToken::Aaab));
        assert!(match_pattern(matcher.start(), &mut "ab".read_symbols()).is_accepted(&TestToken::Abbb));
        assert!(match_pattern(matcher.start(), &mut "abbbb".read_symbols()).is_accepted(&TestToken::Abbb));
    }

    #[test]
    fn can_match_number_stream() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 390  32 ".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 2);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 3);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 6);
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 8);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 10);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
    }

    #[test]
    fn will_skip_bad_symbols() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }

    #[test]
    fn wont_match_zero_length() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(0), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(0), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }
}
<commit_msg>Change test name<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A tokenizer is a pattern matcher that is intended to turn a stream of symbols into another stream of symbols based on the patterns
//! that are matched. Every pattern can produce a different output symbol. If two input strings can ndfa in two different output
//! symbols, then the output symbol that is ordered lower is the one that's produced (ie, if the output symbols are numbers, then '0' will
//! be produced instead of '1' in the event of a clash)
//!

use super::countable::*;
use super::symbol_range::*;
use super::regular_pattern::*;
use super::state_machine::*;
use super::ndfa::*;
use super::prepare::*;
use super::symbol_range_dfa::*;
use super::symbol_reader::*;
use super::pattern_matcher::*;
use super::matches::*;
use super::tape::*;

///
/// Used for generating tokenizing pattern matchers
///
pub struct TokenMatcher<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord> {
    patterns: Vec<(Pattern<InputSymbol>, OutputSymbol)>
}

impl<InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> TokenMatcher<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new TokenMatcher
    ///
    pub fn new() -> TokenMatcher<InputSymbol, OutputSymbol> {
        TokenMatcher { patterns: vec![] }
    }

    ///
    /// Adds a new pattern that will generate the specified output symbol
    ///
    pub fn add_pattern<TPattern: ToPattern<InputSymbol>>(&mut self, pattern: TPattern, output: OutputSymbol) {
        self.patterns.push((pattern.to_pattern(), output));
    }

    ///
    /// Compiles an NDFA from this TokenMatcher
    ///
    pub fn to_ndfa(&self) -> Box<StateMachine<SymbolRange<InputSymbol>, OutputSymbol>> {
        let mut ndfa = Ndfa::new();

        for &(ref pattern, ref output) in &self.patterns {
            // Compile each pattern starting at state 0
            let end_state = pattern.compile(&mut ndfa, 0);

            // Set the output for this pattern
            ndfa.set_output_symbol(end_state, output.clone());
        }

        // Clear out any overlapping ranges so we can build a valid DFA
        ndfa.fix_overlapping_ranges();

        Box::new(ndfa)
    }
}

impl<'a, InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>
for &'a TokenMatcher<InputSymbol, OutputSymbol> {
    #[inline]
    fn prepare_to_match(self) -> SymbolRangeDfa<InputSymbol, OutputSymbol> {
        let ndfa = self.to_ndfa();

        ndfa.prepare_to_match()
    }
}

///
/// A tokenizer is a type of symbol stream that uses a pattern matcher to convert a symbol stream into a stream of tokens
///
pub struct Tokenizer<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> {
    /// The pattern matcher for this tokenizer
    dfa: SymbolRangeDfa<InputSymbol, OutputSymbol>,

    /// Tape of input symbols that will be used to generate the result
    tape: Tape<InputSymbol, Reader>,
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> Tokenizer<InputSymbol, OutputSymbol, Reader> {
    ///
    /// Creates a new tokenizer from a pattern (usually a TokenMatcher)
    ///
    pub fn new<Prepare: PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>>(source: Reader, pattern: Prepare) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern.prepare_to_match(), tape: Tape::new(source) }
    }

    ///
    /// Creates a new tokenizer from a prepared pattern
    ///
    pub fn new_prepared(source: Reader, pattern: SymbolRangeDfa<InputSymbol, OutputSymbol>) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern, tape: Tape::new(source) }
    }

    ///
    /// Returns the current position in the source (the position after the last matched symbol)
    ///
    pub fn get_source_position(&self) -> usize {
        self.tape.get_source_position()
    }

    ///
    /// Skips an input symbol (returning the symbol that was skipped)
    ///
    pub fn skip_input(&mut self) -> Option<InputSymbol> {
        self.tape.next_symbol()
    }

    ///
    /// True if we've reached the end of the source reader
    ///
    /// If `next_symbol` returns `None` and `at_end_of_reader` is false, then the input stream does not contain a symbol matching the DFA
    ///
    pub fn at_end_of_reader(&self) -> bool {
        self.tape.at_end_of_reader()
    }
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord+'static, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for Tokenizer<InputSymbol, OutputSymbol, Reader> {
    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        // Start of the next symbol
        let start_pos = self.tape.get_source_position();

        // Match against it
        let match_result = match_pattern(self.dfa.start(), &mut self.tape);

        let end_pos = self.tape.get_source_position();
        match match_result {
            Accept(length, outputsymbol) => {
                if length > 0 {
                    // Rewind the tape to after the accepted symbol
                    self.tape.rewind(end_pos-start_pos - length);

                    // Won't try to match anything before this position
                    self.tape.cut();

                    // Result is the oputput symbol
                    Some(outputsymbol.clone())
                } else {
                    // Zero-length match
                    // If we accepted matches of length 0 we'd get an infinite stream when we hit a symbol that doesn't match
                    self.tape.rewind(end_pos-start_pos);

                    // Return no match
                    None
                }
            },

            Reject => {
                // Rewind back to the start position
                self.tape.rewind(end_pos-start_pos);

                // No match
                None
            },

            _ => {
                panic!("Unexpected output state of ");
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_match_tokens_like_any_other_pattern() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        assert!(matches("aaaa", &token_matcher) == Some(4));
        assert!(matches("bbbbb", &token_matcher) == Some(5));
        assert!(matches("abbb", &token_matcher) == Some(1));
        assert!(matches("bbaaa", &token_matcher) == Some(2));
    }

    #[test]
    fn can_distinguish_simple_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn clashes_producer_lower_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Abbb,
            Aaab
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1).append("b"), TestToken::Aaab);
        token_matcher.add_pattern("a".append("b".repeat_forever(1)), TestToken::Abbb);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaab".read_symbols()).is_accepted(&TestToken::Aaab));
        assert!(match_pattern(matcher.start(), &mut "ab".read_symbols()).is_accepted(&TestToken::Abbb));
        assert!(match_pattern(matcher.start(), &mut "abbbb".read_symbols()).is_accepted(&TestToken::Abbb));
    }

    #[test]
    fn can_match_number_stream() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 390  32 ".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 2);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 3);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 6);
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 8);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 10);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
    }

    #[test]
    fn can_skip_bad_symbols() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }

    #[test]
    fn wont_match_zero_length() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(0), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(0), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }
}
<|endoftext|>