<commit_before>//! IO module
//!
//! ```
//! // Load HDF5 example here
//! ```

extern crate std;

use libc::{c_char, c_void, c_ulonglong, c_int};
use std::path::Path;
use hdf5_sys as ffi;

use tensor::Tensor;

#[allow(non_camel_case_types)]
type hsize_t = c_ulonglong;
#[allow(non_camel_case_types)]
type hid_t = c_int;

extern fn error_handler(_: hid_t, _: *const c_void) {
    // Suppress errors. We will rely on return statuses alone.
}

macro_rules! add_save {
    ($t:ty, $h5type:expr) => (
        impl Tensor<$t> {
            pub fn save_hdf5(&self, path: &Path) -> std::io::Result<()> {
                let filename = match path.to_str() {
                    Some(v) => v,
                    None => {
                        let err = std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("Path could not be converted to string: {:?}", path));
                        return Err(err);
                    },
                };
                // This could be made an option
                let group = "data";

                unsafe {
                    let filename_cstr = try!(::std::ffi::CString::new(filename));
                    let group_cstr = try!(::std::ffi::CString::new(group));

                    ffi::H5Eset_auto2(0, error_handler, 0 as *const c_void);

                    let file = ffi::H5Fcreate(filename_cstr.as_ptr() as *const c_char,
                                   ffi::H5F_ACC_TRUNC, ffi::H5P_DEFAULT, ffi::H5P_DEFAULT);

                    let mut shape: Vec<u64> = Vec::new();
                    for s in self.shape().iter() {
                        shape.push(*s as u64);
                    }

                    let space = ffi::H5Screate_simple(shape.len() as i32, shape.as_ptr(), std::ptr::null());

                    let dset = ffi::H5Dcreate2(file, group_cstr.as_ptr() as *const c_char,
                                               $h5type, space,
                                               ffi::H5P_DEFAULT,
                                               ffi::H5P_DEFAULT,
                                               ffi::H5P_DEFAULT);

                    let status = ffi::H5Dwrite(dset, $h5type, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                               self.as_ptr() as * const c_void);

                    if status < 0 {
                        let err = std::io::Error::new(std::io::ErrorKind::Other, format!("Failed to write '{}': {:?}", group, path));
                        return Err(err);
                    }

                    ffi::H5Dclose(dset);
                    ffi::H5Fclose(file);
                }
                Ok(())
            }
        }
    )
}

add_save!(u8, ffi::H5T_NATIVE_UINT8);
add_save!(u16, ffi::H5T_NATIVE_UINT16);
add_save!(u32, ffi::H5T_NATIVE_UINT32);
add_save!(u64, ffi::H5T_NATIVE_UINT64);
add_save!(i8, ffi::H5T_NATIVE_INT8);
add_save!(i16, ffi::H5T_NATIVE_INT16);
add_save!(i32, ffi::H5T_NATIVE_INT32);
add_save!(i64, ffi::H5T_NATIVE_INT64);
add_save!(f32, ffi::H5T_NATIVE_FLOAT);
add_save!(f64, ffi::H5T_NATIVE_DOUBLE);

macro_rules! add_load {
    ($name:ident, $t:ty) => (
        pub fn $name(path: &Path, group: &str) -> std::io::Result<Tensor<$t>> {
            let filename = match path.to_str() {
                Some(v) => v,
                None => {
                    let err = std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("Path could not be converted to string: {:?}", path));
                    return Err(err);
                },
            };
            unsafe {
                let filename_cstr = try!(::std::ffi::CString::new(filename));
                let group_cstr = try!(::std::ffi::CString::new(group));

                ffi::H5Eset_auto2(0, error_handler, 0 as *const c_void);

                let file = ffi::H5Fopen(filename_cstr.as_ptr() as *const c_char,
                               ffi::H5F_ACC_RDONLY, ffi::H5P_DEFAULT);

                if file < 0 {
                    let err = std::io::Error::new(std::io::ErrorKind::NotFound, format!("File not found: {:?}", path));
                    return Err(err);
                }

                let dset = ffi::H5Dopen2(file, group_cstr.as_ptr() as *const c_char,
                                        ffi::H5P_DEFAULT);

                if dset < 0 {
                    let err = std::io::Error::new(std::io::ErrorKind::NotFound, format!("Group '{}' not found: {}", group, filename));
                    return Err(err);
                }

                let datatype = ffi::H5Dget_type(dset);

                let space = ffi::H5Dget_space(dset);
                let ndims = ffi::H5Sget_simple_extent_ndims(space);

                let mut shape: Tensor<hsize_t> = Tensor::zeros(&[ndims as usize]);

                if ffi::H5Sget_simple_extent_dims(space, shape.as_mut_ptr(), 0 as *mut hsize_t) != ndims {
                    let err = std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Could not read shape of tesor: {}", filename));
                    return Err(err);
                }

                //let unsigned_shape: Vec<usize> = shape.iter().map(|x| x as usize).collect();
                let unsigned_tensor = shape.convert::<usize>();
                let unsigned_shape = &unsigned_tensor.data();

                let data: Tensor<$t> = {
                    if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT8) == 1 {
                        let mut native_data: Tensor<u8> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT8, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT8) == 1 {
                        let mut native_data: Tensor<i8> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT8, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT16) == 1 {
                        let mut native_data: Tensor<u16> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT16, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT16) == 1 {
                        let mut native_data: Tensor<i16> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT16, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT32) == 1 {
                        let mut native_data: Tensor<u32> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT32, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT32) == 1 {
                        let mut native_data: Tensor<i32> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT32, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT64) == 1 {
                        let mut native_data: Tensor<u64> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT64, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT64) == 1 {
                        let mut native_data: Tensor<i64> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT64, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_FLOAT) == 1 {
                        let mut native_data: Tensor<f32> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_FLOAT, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_DOUBLE) == 1 {
                        let mut native_data: Tensor<f64> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_DOUBLE, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else {
                        let err = std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Unable to convert '{}' to {}: {}", group, "f64", filename));
                        return Err(err);
                    }
                };

                ffi::H5Tclose(datatype);
                ffi::H5Dclose(dset);
                ffi::H5Fclose(file);

                Ok(data)
            }
        }
    )
}

add_load!(load_hdf5_as_u8, u8);
add_load!(load_hdf5_as_u16, u16);
add_load!(load_hdf5_as_u32, u32);
add_load!(load_hdf5_as_u64, u64);
add_load!(load_hdf5_as_i8, i8);
add_load!(load_hdf5_as_i16, i16);
add_load!(load_hdf5_as_i32, i32);
add_load!(load_hdf5_as_i64, i64);
add_load!(load_hdf5_as_f32, f32);
add_load!(load_hdf5_as_f64, f64);
add_load!(load_hdf5_as_isize, isize);
add_load!(load_hdf5_as_usize, usize);
<commit_msg>DOC: Updated io<commit_after>//! Saving and loading data to and from disk.
//!
//! # HDF5
//!
//! The recommended way to save/load data in Numeric is using HDF5.
//!
//! **Note:** The HDF5 library will by default not be thread-safe (it depends on how you compiled
//! it), so do not call either of these functions concurrently.
//!
//! ##Saving to an HDF5 file:
//!
//! ```
//! let path = Path::new("output.h5");
//! let t1 = tensor![1_i32, 23; 20, 10];
//! try!(t.save_hdf5(&path));
//! ```
//! The data will be saved to the group `/data`. Now, we can load this using:
//!
//! ## Loading from HDF5 file
//!
//! Now, we can load this file:
//!
//! ```
//! let t2 = try!(numeric::io::load_hdf5_as_f64(&path, "/data"));
//! ```
//!
//! Note that since we need to know the type of `t2` at compile time, it doesn't matter that we
//! saved the file as `i32`, we have to specify how to load it. The way this is done is that it
//! will load the `i32` natively and then convert it to `f64`. If your data converted, you simply
//! have to load it as the same type as you know is in the file.

extern crate std;

use libc::{c_char, c_void, c_ulonglong, c_int};
use std::path::Path;
use hdf5_sys as ffi;

use tensor::Tensor;

#[allow(non_camel_case_types)]
type hsize_t = c_ulonglong;
#[allow(non_camel_case_types)]
type hid_t = c_int;

extern fn error_handler(_: hid_t, _: *const c_void) {
    // Suppress errors. We will rely on return statuses alone.
}

macro_rules! add_save {
    ($t:ty, $h5type:expr) => (
        impl Tensor<$t> {
            /// Saves tensor to an HDF5 file.
            ///
            /// **Warning**: This function is not thread-safe (unless you compiled HDF5 to be
            /// thread-safe). Do no call this function concurrently from multiple threads.
            pub fn save_hdf5(&self, path: &Path) -> std::io::Result<()> {
                let filename = match path.to_str() {
                    Some(v) => v,
                    None => {
                        let err = std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("Path could not be converted to string: {:?}", path));
                        return Err(err);
                    },
                };
                // This could be made an option
                let group = "data";

                unsafe {
                    let filename_cstr = try!(::std::ffi::CString::new(filename));
                    let group_cstr = try!(::std::ffi::CString::new(group));

                    //ffi::H5Eset_auto2(0, error_handler, 0 as *const c_void);

                    let file = ffi::H5Fcreate(filename_cstr.as_ptr() as *const c_char,
                                   ffi::H5F_ACC_TRUNC, ffi::H5P_DEFAULT, ffi::H5P_DEFAULT);

                    let mut shape: Vec<u64> = Vec::new();
                    for s in self.shape().iter() {
                        shape.push(*s as u64);
                    }

                    let space = ffi::H5Screate_simple(shape.len() as i32, shape.as_ptr(), std::ptr::null());

                    let dset = ffi::H5Dcreate2(file, group_cstr.as_ptr() as *const c_char,
                                               $h5type, space,
                                               ffi::H5P_DEFAULT,
                                               ffi::H5P_DEFAULT,
                                               ffi::H5P_DEFAULT);

                    let status = ffi::H5Dwrite(dset, $h5type, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                               self.as_ptr() as * const c_void);

                    if status < 0 {
                        let err = std::io::Error::new(std::io::ErrorKind::Other,
                                                      format!("Failed to write '{}': {:?}", group, path));
                        return Err(err);
                    }

                    ffi::H5Dclose(dset);
                    ffi::H5Fclose(file);
                }
                Ok(())
            }
        }
    )
}

add_save!(u8, ffi::H5T_NATIVE_UINT8);
add_save!(u16, ffi::H5T_NATIVE_UINT16);
add_save!(u32, ffi::H5T_NATIVE_UINT32);
add_save!(u64, ffi::H5T_NATIVE_UINT64);
add_save!(i8, ffi::H5T_NATIVE_INT8);
add_save!(i16, ffi::H5T_NATIVE_INT16);
add_save!(i32, ffi::H5T_NATIVE_INT32);
add_save!(i64, ffi::H5T_NATIVE_INT64);
add_save!(f32, ffi::H5T_NATIVE_FLOAT);
add_save!(f64, ffi::H5T_NATIVE_DOUBLE);

macro_rules! add_load {
    ($name:ident, $t:ty) => (
        /// Load HDF5 file and convert to specified type.
        pub fn $name(path: &Path, group: &str) -> std::io::Result<Tensor<$t>> {
            let filename = match path.to_str() {
                Some(v) => v,
                None => {
                    let err = std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("Path could not be converted to string: {:?}", path));
                    return Err(err);
                },
            };
            unsafe {
                let filename_cstr = try!(::std::ffi::CString::new(filename));
                let group_cstr = try!(::std::ffi::CString::new(group));

                ffi::H5Eset_auto2(0, error_handler, 0 as *const c_void);

                let file = ffi::H5Fopen(filename_cstr.as_ptr() as *const c_char,
                               ffi::H5F_ACC_RDONLY, ffi::H5P_DEFAULT);

                if file < 0 {
                    let err = std::io::Error::new(std::io::ErrorKind::NotFound, format!("File not found: {:?}", path));
                    return Err(err);
                }

                let dset = ffi::H5Dopen2(file, group_cstr.as_ptr() as *const c_char,
                                        ffi::H5P_DEFAULT);

                if dset < 0 {
                    let err = std::io::Error::new(std::io::ErrorKind::NotFound, format!("Group '{}' not found: {}", group, filename));
                    return Err(err);
                }

                let datatype = ffi::H5Dget_type(dset);

                let space = ffi::H5Dget_space(dset);
                let ndims = ffi::H5Sget_simple_extent_ndims(space);

                let mut shape: Tensor<hsize_t> = Tensor::zeros(&[ndims as usize]);

                if ffi::H5Sget_simple_extent_dims(space, shape.as_mut_ptr(), 0 as *mut hsize_t) != ndims {
                    let err = std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Could not read shape of tesor: {}", filename));
                    return Err(err);
                }

                //let unsigned_shape: Vec<usize> = shape.iter().map(|x| x as usize).collect();
                let unsigned_tensor = shape.convert::<usize>();
                let unsigned_shape = &unsigned_tensor.data();

                let data: Tensor<$t> = {
                    if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT8) == 1 {
                        let mut native_data: Tensor<u8> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT8, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT8) == 1 {
                        let mut native_data: Tensor<i8> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT8, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT16) == 1 {
                        let mut native_data: Tensor<u16> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT16, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT16) == 1 {
                        let mut native_data: Tensor<i16> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT16, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT32) == 1 {
                        let mut native_data: Tensor<u32> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT32, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT32) == 1 {
                        let mut native_data: Tensor<i32> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT32, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_UINT64) == 1 {
                        let mut native_data: Tensor<u64> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_UINT64, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_INT64) == 1 {
                        let mut native_data: Tensor<i64> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_INT64, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_FLOAT) == 1 {
                        let mut native_data: Tensor<f32> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_FLOAT, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else if ffi::H5Tequal(datatype, ffi::H5T_NATIVE_DOUBLE) == 1 {
                        let mut native_data: Tensor<f64> = Tensor::empty(&unsigned_shape[..]);
                        // Finally load the actual data
                        ffi::H5Dread(dset, ffi::H5T_NATIVE_DOUBLE, ffi::H5S_ALL, ffi::H5S_ALL, ffi::H5P_DEFAULT,
                                     native_data.as_mut_ptr() as *mut c_void);
                        native_data.convert::<$t>()
                    } else {
                        let err = std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Unable to convert '{}' to {}: {}", group, "f64", filename));
                        return Err(err);
                    }
                };

                ffi::H5Tclose(datatype);
                ffi::H5Dclose(dset);
                ffi::H5Fclose(file);

                Ok(data)
            }
        }
    )
}

add_load!(load_hdf5_as_u8, u8);
add_load!(load_hdf5_as_u16, u16);
add_load!(load_hdf5_as_u32, u32);
add_load!(load_hdf5_as_u64, u64);
add_load!(load_hdf5_as_i8, i8);
add_load!(load_hdf5_as_i16, i16);
add_load!(load_hdf5_as_i32, i32);
add_load!(load_hdf5_as_i64, i64);
add_load!(load_hdf5_as_f32, f32);
add_load!(load_hdf5_as_f64, f64);
add_load!(load_hdf5_as_isize, isize);
add_load!(load_hdf5_as_usize, usize);
<|endoftext|>


<commit_before>#![cfg(feature = "use_std")]

extern crate futures;

mod support;

use futures::prelude::*;
use futures::unsync::oneshot;
use futures::unsync::mpsc::{self, SendError};
use futures::future::lazy;
use futures::stream::{iter_ok, unfold};

use support::local_executor::Core;

#[test]
fn mpsc_send_recv() {
    let (tx, rx) = mpsc::channel::<i32>(1);
    let mut rx = rx.wait();

    tx.send(42).wait().unwrap();

    assert_eq!(rx.next(), Some(Ok(42)));
    assert_eq!(rx.next(), None);
}

#[test]
fn mpsc_rx_notready() {
    let (_tx, mut rx) = mpsc::channel::<i32>(1);

    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::NotReady);
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_rx_end() {
    let (_, mut rx) = mpsc::channel::<i32>(1);

    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::Ready(None));
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_tx_notready() {
    let (tx, _rx) = mpsc::channel::<i32>(1);
    let tx = tx.send(1).wait().unwrap();
    lazy(move || {
        assert!(tx.send(2).poll().unwrap().is_not_ready());
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_tx_err() {
    let (tx, _) = mpsc::channel::<i32>(1);
    lazy(move || {
        assert!(tx.send(2).poll().is_err());
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_backpressure() {
    let (tx, rx) = mpsc::channel::<i32>(1);
    lazy(move || {
        iter_ok(vec![1, 2, 3])
            .forward(tx)
            .map_err(|e: SendError<i32>| panic!("{}", e))
            .join(rx.take(3).collect().map(|xs| {
                assert_eq!(xs, [1, 2, 3]);
            }))
    }).wait().unwrap();
}

#[test]
fn mpsc_unbounded() {
    let (tx, rx) = mpsc::unbounded::<i32>();
    lazy(move || {
        iter_ok(vec![1, 2, 3])
            .forward(tx)
            .map_err(|e: SendError<i32>| panic!("{}", e))
            .join(rx.take(3).collect().map(|xs| {
                assert_eq!(xs, [1, 2, 3]);
            }))
    }).wait().unwrap();
}

#[test]
fn mpsc_recv_unpark() {
    let core = Core::new();
    let (tx, rx) = mpsc::channel::<i32>(1);
    let tx2 = tx.clone();
    core.spawn(rx.collect().map(|xs| assert_eq!(xs, [1, 2])));
    core.spawn(lazy(move || tx.send(1).map(|_| ()).map_err(|e| panic!("{}", e))));
    core.run(lazy(move || tx2.send(2))).unwrap();
}

#[test]
fn mpsc_send_unpark() {
    let core = Core::new();
    let (tx, rx) = mpsc::channel::<i32>(1);
    let (donetx, donerx) = oneshot::channel();
    core.spawn(iter_ok(vec![1, 2]).forward(tx)
        .then(|x: Result<_, SendError<i32>>| {
            assert!(x.is_err());
            donetx.send(()).unwrap();
            Ok(())
        }));
    core.spawn(lazy(move || { let _ = rx; Ok(()) }));
    core.run(donerx).unwrap();
}

#[test]
fn spawn_sends_items() {
    let core = Core::new();
    let stream = unfold(0, |i| Some(Ok::<_,u8>((i, i + 1))));
    let rx = mpsc::spawn(stream, &core, 1);
    assert_eq!(core.run(rx.take(4).collect()).unwrap(),
               [0, 1, 2, 3]);
}

#[test]
fn spawn_kill_dead_stream() {
    use std::thread;
    use std::time::Duration;
    use futures::future::Either;

    // a stream which never returns anything (maybe a remote end isn't
    // responding), but dropping it leads to observable side effects
    // (like closing connections, releasing limited resources, ...)
    #[derive(Debug)]
    struct Dead {
        // when dropped you should get Err(oneshot::Canceled) on the
        // receiving end
        done: oneshot::Sender<()>,
    }
    impl Stream for Dead {
        type Item = ();
        type Error = ();

        fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
            Ok(Async::NotReady)
        }
    }

    // need to implement a timeout for the test, as it would hang
    // forever right now
    let (timeout_tx, timeout_rx) = futures::sync::oneshot::channel();
    thread::spawn(move || {
        thread::sleep(Duration::from_millis(1000));
        let _ = timeout_tx.send(());
    });

    let core = Core::new();
    let (done_tx, done_rx) = oneshot::channel();
    let stream = Dead{done: done_tx};
    let rx = mpsc::spawn(stream, &core, 1);
    let res = core.run(
        Ok::<_, ()>(())
        .into_future()
        .then(move |_| {
            // now drop the spawned stream: maybe some timeout exceeded,
            // or some connection on this end was closed by the remote
            // end.
            drop(rx);
            // and wait for the spawned stream to release its resources
            done_rx
        })
        .select2(timeout_rx)
    );
    match res {
        Err(Either::A((oneshot::Canceled, _))) => (),
        _ => {
            panic!("dead stream wasn't canceled");
        },
    }
}
<commit_msg>Add test for unsync::mpsc to ensure Rc works for cloned senders<commit_after>#![cfg(feature = "use_std")]

extern crate futures;

mod support;

use futures::prelude::*;
use futures::unsync::oneshot;
use futures::unsync::mpsc::{self, SendError};
use futures::future::lazy;
use futures::stream::{iter_ok, unfold};

use support::local_executor::Core;

#[test]
fn mpsc_send_recv() {
    let (tx, rx) = mpsc::channel::<i32>(1);
    let mut rx = rx.wait();

    tx.send(42).wait().unwrap();

    assert_eq!(rx.next(), Some(Ok(42)));
    assert_eq!(rx.next(), None);
}

#[test]
fn mpsc_rx_notready() {
    let (_tx, mut rx) = mpsc::channel::<i32>(1);

    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::NotReady);
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_rx_end() {
    let (_, mut rx) = mpsc::channel::<i32>(1);

    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::Ready(None));
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_tx_clone_weak_rc() {
    let (tx, mut rx) = mpsc::channel::<i32>(1); // rc = 1

    let tx_clone = tx.clone(); // rc = 2
    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::NotReady);
        Ok(()) as Result<(), ()>
    }).wait().unwrap();

    drop(tx); // rc = 1
    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::NotReady);
        Ok(()) as Result<(), ()>
    }).wait().unwrap();

    drop(tx_clone); // rc = 0
    lazy(|| {
        assert_eq!(rx.poll().unwrap(), Async::Ready(None));
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_tx_notready() {
    let (tx, _rx) = mpsc::channel::<i32>(1);
    let tx = tx.send(1).wait().unwrap();
    lazy(move || {
        assert!(tx.send(2).poll().unwrap().is_not_ready());
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_tx_err() {
    let (tx, _) = mpsc::channel::<i32>(1);
    lazy(move || {
        assert!(tx.send(2).poll().is_err());
        Ok(()) as Result<(), ()>
    }).wait().unwrap();
}

#[test]
fn mpsc_backpressure() {
    let (tx, rx) = mpsc::channel::<i32>(1);
    lazy(move || {
        iter_ok(vec![1, 2, 3])
            .forward(tx)
            .map_err(|e: SendError<i32>| panic!("{}", e))
            .join(rx.take(3).collect().map(|xs| {
                assert_eq!(xs, [1, 2, 3]);
            }))
    }).wait().unwrap();
}

#[test]
fn mpsc_unbounded() {
    let (tx, rx) = mpsc::unbounded::<i32>();
    lazy(move || {
        iter_ok(vec![1, 2, 3])
            .forward(tx)
            .map_err(|e: SendError<i32>| panic!("{}", e))
            .join(rx.take(3).collect().map(|xs| {
                assert_eq!(xs, [1, 2, 3]);
            }))
    }).wait().unwrap();
}

#[test]
fn mpsc_recv_unpark() {
    let core = Core::new();
    let (tx, rx) = mpsc::channel::<i32>(1);
    let tx2 = tx.clone();
    core.spawn(rx.collect().map(|xs| assert_eq!(xs, [1, 2])));
    core.spawn(lazy(move || tx.send(1).map(|_| ()).map_err(|e| panic!("{}", e))));
    core.run(lazy(move || tx2.send(2))).unwrap();
}

#[test]
fn mpsc_send_unpark() {
    let core = Core::new();
    let (tx, rx) = mpsc::channel::<i32>(1);
    let (donetx, donerx) = oneshot::channel();
    core.spawn(iter_ok(vec![1, 2]).forward(tx)
        .then(|x: Result<_, SendError<i32>>| {
            assert!(x.is_err());
            donetx.send(()).unwrap();
            Ok(())
        }));
    core.spawn(lazy(move || { let _ = rx; Ok(()) }));
    core.run(donerx).unwrap();
}

#[test]
fn spawn_sends_items() {
    let core = Core::new();
    let stream = unfold(0, |i| Some(Ok::<_,u8>((i, i + 1))));
    let rx = mpsc::spawn(stream, &core, 1);
    assert_eq!(core.run(rx.take(4).collect()).unwrap(),
               [0, 1, 2, 3]);
}

#[test]
fn spawn_kill_dead_stream() {
    use std::thread;
    use std::time::Duration;
    use futures::future::Either;

    // a stream which never returns anything (maybe a remote end isn't
    // responding), but dropping it leads to observable side effects
    // (like closing connections, releasing limited resources, ...)
    #[derive(Debug)]
    struct Dead {
        // when dropped you should get Err(oneshot::Canceled) on the
        // receiving end
        done: oneshot::Sender<()>,
    }
    impl Stream for Dead {
        type Item = ();
        type Error = ();

        fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
            Ok(Async::NotReady)
        }
    }

    // need to implement a timeout for the test, as it would hang
    // forever right now
    let (timeout_tx, timeout_rx) = futures::sync::oneshot::channel();
    thread::spawn(move || {
        thread::sleep(Duration::from_millis(1000));
        let _ = timeout_tx.send(());
    });

    let core = Core::new();
    let (done_tx, done_rx) = oneshot::channel();
    let stream = Dead{done: done_tx};
    let rx = mpsc::spawn(stream, &core, 1);
    let res = core.run(
        Ok::<_, ()>(())
        .into_future()
        .then(move |_| {
            // now drop the spawned stream: maybe some timeout exceeded,
            // or some connection on this end was closed by the remote
            // end.
            drop(rx);
            // and wait for the spawned stream to release its resources
            done_rx
        })
        .select2(timeout_rx)
    );
    match res {
        Err(Either::A((oneshot::Canceled, _))) => (),
        _ => {
            panic!("dead stream wasn't canceled");
        },
    }
}
<|endoftext|>


<commit_before>use std::io;
use byteorder::{WriteBytesExt, ReadBytesExt, BigEndian};

#[allow(dead_code)]
pub enum Opcode {
    Get = 0x00,
    Set = 0x01,
    Add = 0x02,
    Repalce = 0x03,
    Flush = 0x08,
    Version = 0x0b,
}

pub enum Magic {
    Request = 0x80,
    Response = 0x81,
}

#[allow(dead_code)]
pub enum ResponseStatus {
    NoError = 0x0000,
    KeyNotFound = 0x0001,
    KeyExits = 0x0002,
    ValueTooLarge = 0x003,
    InvalidArguments = 0x004,
}

#[allow(dead_code)]
#[derive(Debug, Default)]
pub struct PacketHeader {
    pub magic: u8,
    pub opcode: u8,
    pub key_length: u16,
    pub extras_length: u8,
    pub data_type: u8,
    pub vbucket_id_or_status: u16,
    pub total_body_length: u32,
    pub opaque: u32,
    pub cas: u64,
}

#[derive(Debug)]
pub struct StoreExtras {
    pub flags: u32,
    pub expiration: u32,
}

impl PacketHeader {
    pub fn write<T: io::Write>(self, mut writer: T) -> Result<(), io::Error> {
        writer.write_u8(self.magic)?;
        writer.write_u8(self.opcode)?;
        writer.write_u16::<BigEndian>(self.key_length)?;
        writer.write_u8(self.extras_length)?;
        writer.write_u8(self.data_type)?;
        writer.write_u16::<BigEndian>(self.vbucket_id_or_status)?;
        writer.write_u32::<BigEndian>(self.total_body_length)?;
        writer.write_u32::<BigEndian>(self.opaque)?;
        writer.write_u64::<BigEndian>(self.cas)?;
        return Ok(());
    }

    pub fn read<T: io::Read>(mut reader: T) -> Result<PacketHeader, io::Error> {
        let magic = reader.read_u8()?;
        if magic != Magic::Response as u8 {
            // TODO: raise error
        }
        let header = PacketHeader {
            magic: magic,
            opcode: reader.read_u8()?,
            key_length: reader.read_u16::<BigEndian>()?,
            extras_length: reader.read_u8()?,
            data_type: reader.read_u8()?,
            vbucket_id_or_status: reader.read_u16::<BigEndian>()?,
            total_body_length: reader.read_u32::<BigEndian>()?,
            opaque: reader.read_u32::<BigEndian>()?,
            cas: reader.read_u64::<BigEndian>()?,
        };
        return Ok(header);
    }
}
<commit_msg>:sparkles: throw error while got an invalid response<commit_after>use std::io;
use byteorder::{WriteBytesExt, ReadBytesExt, BigEndian};
use error::MemcacheError;

#[allow(dead_code)]
pub enum Opcode {
    Get = 0x00,
    Set = 0x01,
    Add = 0x02,
    Repalce = 0x03,
    Flush = 0x08,
    Version = 0x0b,
}

pub enum Magic {
    Request = 0x80,
    Response = 0x81,
}

#[allow(dead_code)]
pub enum ResponseStatus {
    NoError = 0x0000,
    KeyNotFound = 0x0001,
    KeyExits = 0x0002,
    ValueTooLarge = 0x003,
    InvalidArguments = 0x004,
}

#[allow(dead_code)]
#[derive(Debug, Default)]
pub struct PacketHeader {
    pub magic: u8,
    pub opcode: u8,
    pub key_length: u16,
    pub extras_length: u8,
    pub data_type: u8,
    pub vbucket_id_or_status: u16,
    pub total_body_length: u32,
    pub opaque: u32,
    pub cas: u64,
}

#[derive(Debug)]
pub struct StoreExtras {
    pub flags: u32,
    pub expiration: u32,
}

impl PacketHeader {
    pub fn write<T: io::Write>(self, mut writer: T) -> Result<(), io::Error> {
        writer.write_u8(self.magic)?;
        writer.write_u8(self.opcode)?;
        writer.write_u16::<BigEndian>(self.key_length)?;
        writer.write_u8(self.extras_length)?;
        writer.write_u8(self.data_type)?;
        writer.write_u16::<BigEndian>(self.vbucket_id_or_status)?;
        writer.write_u32::<BigEndian>(self.total_body_length)?;
        writer.write_u32::<BigEndian>(self.opaque)?;
        writer.write_u64::<BigEndian>(self.cas)?;
        return Ok(());
    }

    pub fn read<T: io::Read>(mut reader: T) -> Result<PacketHeader, MemcacheError> {
        let magic = reader.read_u8()?;
        if magic != Magic::Response as u8 {
            return Err(MemcacheError::ClientError(
                String::from("Bad magic number in response header"),
            ));
        }
        let header = PacketHeader {
            magic: magic,
            opcode: reader.read_u8()?,
            key_length: reader.read_u16::<BigEndian>()?,
            extras_length: reader.read_u8()?,
            data_type: reader.read_u8()?,
            vbucket_id_or_status: reader.read_u16::<BigEndian>()?,
            total_body_length: reader.read_u32::<BigEndian>()?,
            opaque: reader.read_u32::<BigEndian>()?,
            cas: reader.read_u64::<BigEndian>()?,
        };
        return Ok(header);
    }
}
<|endoftext|>


<commit_before>use super::lexer::Lexer;
use super::token::Token;
use super::token::TokenType;
use super::ast::Program;
use super::ast::Node;
use super::ast::Statement;
use super::ast::Expression;

enum Precedence {
    Lowest,
    Equals,
    LessGreater,
    Sum,
    Product,
    Prefix,
    Call,
}

struct Parser<'a> {
    lexer: &'a mut Lexer,
    cur_token: Token,
    peek_token: Token,
    errors: Vec<String>,
}

impl<'a> Parser<'a> {
    fn new(lexer: &'a mut Lexer) -> Self {
        let cur_token = lexer.next_token();
        let peek_token = lexer.next_token();

        Parser {
            lexer: lexer,
            cur_token: cur_token,
            peek_token: peek_token,
            errors: Vec::new(),
        }
    }

    fn next_token(&mut self) {
        self.cur_token = self.peek_token.clone();
        self.peek_token = self.lexer.next_token();
    }

    fn parse_stmt(&mut self) -> Option<Statement> {
        match self.cur_token.token_type {
            TokenType::Let => self.parse_let_stmt(),
            TokenType::Return => self.parse_return_stmt(),
            _ => self.parse_expr_stmt(),
        }
    }

    fn parse_let_stmt(&mut self) -> Option<Statement> {
        let let_token = self.cur_token.clone();

        if !self.expect_peek(TokenType::Ident) {
            return None;
        }

        let name = Expression::Identifier {
            token: self.cur_token.clone(),
            value: self.cur_token.literal.clone(),
        };

        if !self.expect_peek(TokenType::Assign) {
            return None;
        }

        while self.cur_token.token_type != TokenType::Semicolon {
            self.next_token();
        }

        Some(Statement::Let {
            token: let_token,
            name: name,
            value: unsafe { ::std::mem::uninitialized() },
        })
    }

    fn parse_return_stmt(&mut self) -> Option<Statement> {
        let return_token = self.cur_token.clone();

        self.next_token();

        while self.cur_token.token_type != TokenType::Semicolon {
            self.next_token();
        }

        Some(Statement::Return {
            token: return_token,
            value: unsafe { ::std::mem::uninitialized() },
        })
    }

    fn parse_expr_stmt(&mut self) -> Option<Statement> {
        let expr_token = self.cur_token.clone();
        let expr = self.parse_expr(Precedence::Lowest)
            .expect("Unable to parse expression");

        if self.peek_token.token_type == TokenType::Semicolon {
            self.next_token();
        }

        Some(Statement::Expression {
            token: expr_token,
            expression: expr,
        })
    }

    fn parse_expr(&mut self, precedence: Precedence) -> Option<Expression> {
        self.parse_prefix()
    }

    fn parse_prefix(&mut self) -> Option<Expression> {
        match self.cur_token.token_type {
            TokenType::Ident => self.parse_identifier(),
            TokenType::Int => self.parse_integer_literal(),
            TokenType::Bang | TokenType::Minus => self.parse_prefix_expr(),
            ref tt @ _ => {
                let err_msg = format!("No prefix parse fn for {:?} found.", tt);
                self.errors.push(err_msg);

                None
            }
        }
    }

    fn parse_identifier(&self) -> Option<Expression> {
        Some(Expression::Identifier {
            token: self.cur_token.clone(),
            value: self.cur_token.literal.clone(),
        })
    }

    fn parse_integer_literal(&mut self) -> Option<Expression> {
        let literal_token = self.cur_token.clone();
        let parse_result = self.cur_token.literal.parse();

        match parse_result {
            Ok(value) => {
                Some(Expression::IntegerLiteral {
                    token: literal_token,
                    value: value,
                })
            }
            Err(_) => {
                let msg = format!("Could not parse {} as integer", literal_token.literal);
                self.errors.push(msg);

                None
            }
        }
    }

    fn parse_prefix_expr(&mut self) -> Option<Expression> {
        let token = self.cur_token.clone();
        let operator = self.cur_token.literal.clone();

        self.next_token();

        let right = self.parse_expr(Precedence::Prefix);

        if let Some(r) = right {
            Some(Expression::Prefix {
                token: token,
                operator: operator,
                right: Box::new(r),
            })
        } else {
            None
        }
    }

    fn expect_peek(&mut self, tt: TokenType) -> bool {
        if self.peek_token.token_type == tt {
            self.next_token();
            true
        } else {
            self.peek_error(tt);
            false
        }
    }

    fn peek_error(&mut self, tt: TokenType) {
        let err_msg = format!("Expected next token to be {:?}, got {:?} instead",
                              tt,
                              self.peek_token.token_type);
        self.errors.push(err_msg);
    }

    fn parse_program(&mut self) -> Program {
        let mut program = Program::new();

        while self.cur_token.token_type != TokenType::EOF {
            let maybe_stmt = self.parse_stmt();
            if let Some(stmt) = maybe_stmt {
                program.statements.push(stmt);
            }
            self.next_token();
        }

        program
    }
}

#[cfg(test)]
fn check_parser_errors<'a>(parser: &'a Parser) {
    if parser.errors.len() == 0 {
        return;
    }
    for err in &parser.errors {
        println!("{}", err);
    }
    panic!("There are parser errors!");
}

#[cfg(test)]
fn check_integer_literal(il: &Expression, test_value: i64) {
    match *il {
        Expression::IntegerLiteral { ref value, .. } => {
            assert!(*value == test_value, "il.value is not {}. Got {}", test_value, value);
            assert!(il.token_literal() == test_value.to_string(),
                    "il.token_literal() is not {}. Got {}",
                    test_value,
                    il.token_literal());
        }
        _ => panic!("il is not Expression::IntegerLiteral. Got {:?}", il),
    }
}

#[test]
fn test_let_statements() {
    let input = String::from("
let x = 5;
let y = 10;
let foobar = 838383;
    ");

    let mut l = Lexer::new(input);
    let mut p = Parser::new(&mut l);

    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 3,
            "program.statements does not contain 3 statements. Got {}",
            program.statements.len());

    let tests = vec!["x", "y", "foobar"];

    for ((i, expected), stmt) in tests.iter().enumerate().zip(program.statements) {
        assert!(stmt.token_literal() == "let",
                "[{}] stmt.token_literal not 'let'. Got {}",
                i,
                stmt.token_literal());

        if let Statement::Let { name, .. } = stmt {
            if let Expression::Identifier { ref value, .. } = name {
                assert!(value == *expected, "stmt.name.value is not {}. Got {}", expected, value);

                assert!(name.token_literal() == *expected,
                        "stmt.name is not {}. Got {:?}",
                        expected,
                        name);
            } else {
                panic!("name is not Expression::Identifier. Got {:?}", name);
            }
        } else {
            panic!("stmt is not Statement::Let. Got {:?}", stmt);
        }
    }
}

#[test]
fn test_return_statements() {
    let input = String::from("
return 5;
return 10;
return 993322;
    ");

    let mut l = Lexer::new(input);
    let mut p = Parser::new(&mut l);

    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 3,
            "program.statements does not contain 3 statements. Got {}",
            program.statements.len());

    for stmt in program.statements {
        if let Statement::Return { .. } = stmt {
            assert!(stmt.token_literal() == "return",
                    "stmt.token_literal() isn't 'return'. Got {}",
                    stmt.token_literal());
        } else {
            panic!("stmt is not Statement::Return. Got {:?}", stmt);
        }
    }
}

#[test]
fn test_identifier_expression() {
    let input = "foobar;";

    let mut l = Lexer::new(input.to_string());
    let mut p = Parser::new(&mut l);
    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 1,
            "program has not enough statements. Got {}",
            program.statements.len());

    if let Statement::Expression { ref expression, .. } = program.statements[0] {
        if let Expression::Identifier { ref value, .. } = *expression {
            assert!(value == "foobar", "value not 'foobar'. Got '{}'", value);
            assert!(expression.token_literal() == "foobar",
                    "token_literal is not 'foobar'. Got {}",
                    expression.token_literal());
        } else {
            panic!("expression is not Expression::Identifier. Got {:?}", expression);
        }
    } else {
        panic!("program.statements[0] is not Statement::Expression. Got {:?}",
               program.statements[0]);
    }
}

#[test]
fn test_integer_literal_expression() {
    let input = "5;";

    let mut l = Lexer::new(input.to_string());
    let mut p = Parser::new(&mut l);
    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 1,
            "program has not enough statements. Got {}",
            program.statements.len());

    if let Statement::Expression { ref expression, .. } = program.statements[0] {
        if let Expression::IntegerLiteral { ref value, .. } = *expression {
            assert!(*value == 5, "value is not 5. Got {}", value);
            assert!(expression.token_literal() == "5",
                    "token_literal() is not '5'. Got {}",
                    expression.token_literal());
        } else {
            panic!("expression is not Expression::IntegerLiteral. Got {:?}", expression);
        }
    } else {
        panic!("program.statements[0] is not Statement::Expression. Got {:?}",
               program.statements[0]);
    }
}

#[test]
fn test_parsing_prefix_expressions() {
    let tests = vec![("!5", "!", 5), ("-15", "-", 15)];

    for (input, op, value) in tests {
        let mut l = Lexer::new(input.to_string());
        let mut p = Parser::new(&mut l);
        let program = p.parse_program();
        check_parser_errors(&p);

        assert!(program.statements.len() == 1,
                "program.statements does not contain 1 statement. Got {}",
                program.statements.len());

        if let Statement::Expression { ref expression, .. } = program.statements[0] {
            if let Expression::Prefix { ref operator, .. } = *expression {
                assert!(operator == op, "operator is not {}. Got {}", op, operator);
                check_integer_literal(expression, value);

            } else {
                panic!("expression is not Expression::Prefix. Got {:?}", expression);
            }
        } else {
            panic!("program.statements[0] is not Statement::Expression. Got {:?}",
                   program.statements[0]);
        }
    }
}

#[test]
fn test_parsing_infix_expressions() {
    let tests = vec![("5 + 5;", 5, "+", 5),
                     ("5 - 5;", 5, "-", 5),
                     ("5 * 5;", 5, "*", 5),
                     ("5 / 5;", 5, "/", 5),
                     ("5 > 5;", 5, ">", 5),
                     ("5 < 5;", 5, "<", 5),
                     ("5 == 5;", 5, "==", 5),
                     ("5 != 5;", 5, "!=", 5)];

    for (input, left_val, op, right_val) in tests {
        let mut l = Lexer::new(input.to_string());
        let mut p = Parser::new(&mut l);
        let program = p.parse_program();
        check_parser_errors(&p);

        assert!(program.statements.len() == 1,
                "program.statements does not contain 1 statement. Got {}",
                program.statements.len());

        if let Statement::Expression { ref expression, .. } = program.statements[0] {
            if let Expression::Infix { ref left, ref operator, ref right, .. } = *expression {

                check_integer_literal(left, left_val);
                assert!(operator == op, "operator is not {}. Got {}", op, operator);
                check_integer_literal(right, right_val);

            } else {
                panic!("expression is not Expression::Infix. Got {:?}", expression);
            }
        } else {
            panic!("program.statements[0] is not Statement::Expression. Got {:?}",
                   program.statements[0]);
        }
    }
}
<commit_msg>Fixed prefix expr parsing test.<commit_after>use super::lexer::Lexer;
use super::token::Token;
use super::token::TokenType;
use super::ast::Program;
use super::ast::Node;
use super::ast::Statement;
use super::ast::Expression;

enum Precedence {
    Lowest,
    Equals,
    LessGreater,
    Sum,
    Product,
    Prefix,
    Call,
}

struct Parser<'a> {
    lexer: &'a mut Lexer,
    cur_token: Token,
    peek_token: Token,
    errors: Vec<String>,
}

impl<'a> Parser<'a> {
    fn new(lexer: &'a mut Lexer) -> Self {
        let cur_token = lexer.next_token();
        let peek_token = lexer.next_token();

        Parser {
            lexer: lexer,
            cur_token: cur_token,
            peek_token: peek_token,
            errors: Vec::new(),
        }
    }

    fn next_token(&mut self) {
        self.cur_token = self.peek_token.clone();
        self.peek_token = self.lexer.next_token();
    }

    fn parse_stmt(&mut self) -> Option<Statement> {
        match self.cur_token.token_type {
            TokenType::Let => self.parse_let_stmt(),
            TokenType::Return => self.parse_return_stmt(),
            _ => self.parse_expr_stmt(),
        }
    }

    fn parse_let_stmt(&mut self) -> Option<Statement> {
        let let_token = self.cur_token.clone();

        if !self.expect_peek(TokenType::Ident) {
            return None;
        }

        let name = Expression::Identifier {
            token: self.cur_token.clone(),
            value: self.cur_token.literal.clone(),
        };

        if !self.expect_peek(TokenType::Assign) {
            return None;
        }

        while self.cur_token.token_type != TokenType::Semicolon {
            self.next_token();
        }

        Some(Statement::Let {
            token: let_token,
            name: name,
            value: unsafe { ::std::mem::uninitialized() },
        })
    }

    fn parse_return_stmt(&mut self) -> Option<Statement> {
        let return_token = self.cur_token.clone();

        self.next_token();

        while self.cur_token.token_type != TokenType::Semicolon {
            self.next_token();
        }

        Some(Statement::Return {
            token: return_token,
            value: unsafe { ::std::mem::uninitialized() },
        })
    }

    fn parse_expr_stmt(&mut self) -> Option<Statement> {
        let expr_token = self.cur_token.clone();
        let expr = self.parse_expr(Precedence::Lowest)
            .expect("Unable to parse expression");

        if self.peek_token.token_type == TokenType::Semicolon {
            self.next_token();
        }

        Some(Statement::Expression {
            token: expr_token,
            expression: expr,
        })
    }

    fn parse_expr(&mut self, precedence: Precedence) -> Option<Expression> {
        self.parse_prefix()
    }

    fn parse_prefix(&mut self) -> Option<Expression> {
        match self.cur_token.token_type {
            TokenType::Ident => self.parse_identifier(),
            TokenType::Int => self.parse_integer_literal(),
            TokenType::Bang | TokenType::Minus => self.parse_prefix_expr(),
            ref tt @ _ => {
                let err_msg = format!("No prefix parse fn for {:?} found.", tt);
                self.errors.push(err_msg);

                None
            }
        }
    }

    fn parse_identifier(&self) -> Option<Expression> {
        Some(Expression::Identifier {
            token: self.cur_token.clone(),
            value: self.cur_token.literal.clone(),
        })
    }

    fn parse_integer_literal(&mut self) -> Option<Expression> {
        let literal_token = self.cur_token.clone();
        let parse_result = self.cur_token.literal.parse();

        match parse_result {
            Ok(value) => {
                Some(Expression::IntegerLiteral {
                    token: literal_token,
                    value: value,
                })
            }
            Err(_) => {
                let msg = format!("Could not parse {} as integer", literal_token.literal);
                self.errors.push(msg);

                None
            }
        }
    }

    fn parse_prefix_expr(&mut self) -> Option<Expression> {
        let token = self.cur_token.clone();
        let operator = self.cur_token.literal.clone();

        self.next_token();

        let right = self.parse_expr(Precedence::Prefix);

        if let Some(r) = right {
            Some(Expression::Prefix {
                token: token,
                operator: operator,
                right: Box::new(r),
            })
        } else {
            None
        }
    }

    fn expect_peek(&mut self, tt: TokenType) -> bool {
        if self.peek_token.token_type == tt {
            self.next_token();
            true
        } else {
            self.peek_error(tt);
            false
        }
    }

    fn peek_error(&mut self, tt: TokenType) {
        let err_msg = format!("Expected next token to be {:?}, got {:?} instead",
                              tt,
                              self.peek_token.token_type);
        self.errors.push(err_msg);
    }

    fn parse_program(&mut self) -> Program {
        let mut program = Program::new();

        while self.cur_token.token_type != TokenType::EOF {
            let maybe_stmt = self.parse_stmt();
            if let Some(stmt) = maybe_stmt {
                program.statements.push(stmt);
            }
            self.next_token();
        }

        program
    }
}

#[cfg(test)]
fn check_parser_errors<'a>(parser: &'a Parser) {
    if parser.errors.len() == 0 {
        return;
    }
    for err in &parser.errors {
        println!("{}", err);
    }
    panic!("There are parser errors!");
}

#[cfg(test)]
fn check_integer_literal(il: &Expression, test_value: i64) {
    match *il {
        Expression::IntegerLiteral { ref value, .. } => {
            assert!(*value == test_value, "il.value is not {}. Got {}", test_value, value);
            assert!(il.token_literal() == test_value.to_string(),
                    "il.token_literal() is not {}. Got {}",
                    test_value,
                    il.token_literal());
        }
        _ => panic!("il is not Expression::IntegerLiteral. Got {:?}", il),
    }
}

#[test]
fn test_let_statements() {
    let input = String::from("
let x = 5;
let y = 10;
let foobar = 838383;
    ");

    let mut l = Lexer::new(input);
    let mut p = Parser::new(&mut l);

    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 3,
            "program.statements does not contain 3 statements. Got {}",
            program.statements.len());

    let tests = vec!["x", "y", "foobar"];

    for ((i, expected), stmt) in tests.iter().enumerate().zip(program.statements) {
        assert!(stmt.token_literal() == "let",
                "[{}] stmt.token_literal not 'let'. Got {}",
                i,
                stmt.token_literal());

        if let Statement::Let { name, .. } = stmt {
            if let Expression::Identifier { ref value, .. } = name {
                assert!(value == *expected, "stmt.name.value is not {}. Got {}", expected, value);

                assert!(name.token_literal() == *expected,
                        "stmt.name is not {}. Got {:?}",
                        expected,
                        name);
            } else {
                panic!("name is not Expression::Identifier. Got {:?}", name);
            }
        } else {
            panic!("stmt is not Statement::Let. Got {:?}", stmt);
        }
    }
}

#[test]
fn test_return_statements() {
    let input = String::from("
return 5;
return 10;
return 993322;
    ");

    let mut l = Lexer::new(input);
    let mut p = Parser::new(&mut l);

    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 3,
            "program.statements does not contain 3 statements. Got {}",
            program.statements.len());

    for stmt in program.statements {
        if let Statement::Return { .. } = stmt {
            assert!(stmt.token_literal() == "return",
                    "stmt.token_literal() isn't 'return'. Got {}",
                    stmt.token_literal());
        } else {
            panic!("stmt is not Statement::Return. Got {:?}", stmt);
        }
    }
}

#[test]
fn test_identifier_expression() {
    let input = "foobar;";

    let mut l = Lexer::new(input.to_string());
    let mut p = Parser::new(&mut l);
    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 1,
            "program has not enough statements. Got {}",
            program.statements.len());

    if let Statement::Expression { ref expression, .. } = program.statements[0] {
        if let Expression::Identifier { ref value, .. } = *expression {
            assert!(value == "foobar", "value not 'foobar'. Got '{}'", value);
            assert!(expression.token_literal() == "foobar",
                    "token_literal is not 'foobar'. Got {}",
                    expression.token_literal());
        } else {
            panic!("expression is not Expression::Identifier. Got {:?}", expression);
        }
    } else {
        panic!("program.statements[0] is not Statement::Expression. Got {:?}",
               program.statements[0]);
    }
}

#[test]
fn test_integer_literal_expression() {
    let input = "5;";

    let mut l = Lexer::new(input.to_string());
    let mut p = Parser::new(&mut l);
    let program = p.parse_program();
    check_parser_errors(&p);

    assert!(program.statements.len() == 1,
            "program has not enough statements. Got {}",
            program.statements.len());

    if let Statement::Expression { ref expression, .. } = program.statements[0] {
        if let Expression::IntegerLiteral { ref value, .. } = *expression {
            assert!(*value == 5, "value is not 5. Got {}", value);
            assert!(expression.token_literal() == "5",
                    "token_literal() is not '5'. Got {}",
                    expression.token_literal());
        } else {
            panic!("expression is not Expression::IntegerLiteral. Got {:?}", expression);
        }
    } else {
        panic!("program.statements[0] is not Statement::Expression. Got {:?}",
               program.statements[0]);
    }
}

#[test]
fn test_parsing_prefix_expressions() {
    let tests = vec![("!5", "!", 5), ("-15", "-", 15)];

    for (input, op, value) in tests {
        let mut l = Lexer::new(input.to_string());
        let mut p = Parser::new(&mut l);
        let program = p.parse_program();
        check_parser_errors(&p);

        assert!(program.statements.len() == 1,
                "program.statements does not contain 1 statement. Got {}",
                program.statements.len());

        if let Statement::Expression { ref expression, .. } = program.statements[0] {
            if let Expression::Prefix { ref operator, ref right, .. } = *expression {
                assert!(operator == op, "operator is not {}. Got {}", op, operator);
                check_integer_literal(right, value);

            } else {
                panic!("expression is not Expression::Prefix. Got {:?}", expression);
            }
        } else {
            panic!("program.statements[0] is not Statement::Expression. Got {:?}",
                   program.statements[0]);
        }
    }
}

#[test]
fn test_parsing_infix_expressions() {
    let tests = vec![("5 + 5;", 5, "+", 5),
                     ("5 - 5;", 5, "-", 5),
                     ("5 * 5;", 5, "*", 5),
                     ("5 / 5;", 5, "/", 5),
                     ("5 > 5;", 5, ">", 5),
                     ("5 < 5;", 5, "<", 5),
                     ("5 == 5;", 5, "==", 5),
                     ("5 != 5;", 5, "!=", 5)];

    for (input, left_val, op, right_val) in tests {
        let mut l = Lexer::new(input.to_string());
        let mut p = Parser::new(&mut l);
        let program = p.parse_program();
        check_parser_errors(&p);

        assert!(program.statements.len() == 1,
                "program.statements does not contain 1 statement. Got {}",
                program.statements.len());

        if let Statement::Expression { ref expression, .. } = program.statements[0] {
            if let Expression::Infix { ref left, ref operator, ref right, .. } = *expression {

                check_integer_literal(left, left_val);
                assert!(operator == op, "operator is not {}. Got {}", op, operator);
                check_integer_literal(right, right_val);

            } else {
                panic!("expression is not Expression::Infix. Got {:?}", expression);
            }
        } else {
            panic!("program.statements[0] is not Statement::Expression. Got {:?}",
                   program.statements[0]);
        }
    }
}
<|endoftext|>


<commit_before>use chrono::DateTime;
use chrono::prelude::{Local, Utc};
use errors::*;
use nom::{alphanumeric, IResult};
use output::Output;
use std::str;

#[derive(Debug, PartialEq, Eq)]
struct Program {
    steps: Vec<Step>,
}

#[derive(Debug, PartialEq, Eq)]
struct Step {
    op: MacroOp,
    args: Vec<String>
}

#[derive(Debug, PartialEq, Eq)]
enum MacroOp {
    Add,
    Divide,
    Multiply,
    Name(String),
    Roll,
    Say,
    Subtract,
    Whisper,
}

#[derive(Debug, PartialEq, Eq)]
enum RollOp {
    Advantage,
    Disadvantage,
    NdD,
}

#[derive(Debug, PartialEq, Eq)]
enum SayOp {
    Message,
    To,
}

named!(command <&[u8], MacroOp>, alt!(
    map!(tag!("!roll"),     |_| MacroOp::Roll)      |
    map!(tag!("!r"),        |_| MacroOp::Roll)      |
    map!(tag!("!say"),      |_| MacroOp::Say)       |
    map!(tag!("!s"),        |_| MacroOp::Say)       |
    map!(tag!("!whisper"),  |_| MacroOp::Whisper)   |
    map!(tag!("!w"),        |_| MacroOp::Whisper)
));

named!(name <&[u8], MacroOp>, do_parse!(
    tag!("#") >>
    name: map_res!(is_not!(" \t\r\n"), |r: &[u8]| String::from_utf8(r.to_vec())) >>
    (MacroOp::Name(name))
));

named!(primitive <&[u8], MacroOp>, alt!(
    map!(tag!("*"), |_| MacroOp::Multiply)  |
    map!(tag!("+"), |_| MacroOp::Add)       |
    map!(tag!("-"), |_| MacroOp::Subtract)  |
    map!(tag!("/"), |_| MacroOp::Divide)
));

#[test]
fn test_name() {
    let (_, result) = name(b"#macro_name").unwrap();
    assert_eq!(result, MacroOp::Name(String::from("macro_name")));

    let bad_result = name(b"macro_name");
}
<commit_msg>test for parse error<commit_after>use chrono::DateTime;
use chrono::prelude::{Local, Utc};
use errors::*;
use nom::{alphanumeric, ErrorKind, IResult};
use output::Output;
use std::str;

#[derive(Debug, PartialEq, Eq)]
struct Program {
    steps: Vec<Step>,
}

#[derive(Debug, PartialEq, Eq)]
struct Step {
    op: MacroOp,
    args: Vec<String>
}

#[derive(Debug, PartialEq, Eq)]
enum MacroOp {
    Add,
    Divide,
    Multiply,
    Name(String),
    Roll,
    Say,
    Subtract,
    Whisper,
}

#[derive(Debug, PartialEq, Eq)]
enum RollOp {
    Advantage,
    Disadvantage,
    NdD,
}

#[derive(Debug, PartialEq, Eq)]
enum SayOp {
    Message,
    To,
}

named!(command <&[u8], MacroOp>, alt!(
    map!(tag!("!roll"),     |_| MacroOp::Roll)      |
    map!(tag!("!r"),        |_| MacroOp::Roll)      |
    map!(tag!("!say"),      |_| MacroOp::Say)       |
    map!(tag!("!s"),        |_| MacroOp::Say)       |
    map!(tag!("!whisper"),  |_| MacroOp::Whisper)   |
    map!(tag!("!w"),        |_| MacroOp::Whisper)
));

named!(name <&[u8], MacroOp>, do_parse!(
    tag!("#") >>
    name: map_res!(is_not!(" \t\r\n"), |r: &[u8]| String::from_utf8(r.to_vec())) >>
    (MacroOp::Name(name))
));

named!(primitive <&[u8], MacroOp>, alt!(
    map!(tag!("*"), |_| MacroOp::Multiply)  |
    map!(tag!("+"), |_| MacroOp::Add)       |
    map!(tag!("-"), |_| MacroOp::Subtract)  |
    map!(tag!("/"), |_| MacroOp::Divide)
));

#[test]
fn test_name() {
    let (_, result) = name(b"#macro_name").unwrap();
    assert_eq!(result, MacroOp::Name(String::from("macro_name")));

    let (_, result) = name(b"#macro-name").unwrap();
    assert_eq!(result, MacroOp::Name(String::from("macro-name")));

    let (_, result) = name(b"#123macro-name").unwrap();
    assert_eq!(result, MacroOp::Name(String::from("123macro-name")));

    let (_, result) = name(b"#Z123macro-name").unwrap();
    assert_eq!(result, MacroOp::Name(String::from("Z123macro-name")));

    let bad_result = name(b"macro_name");
    match bad_result {
        IResult::Error(e) => assert_eq!(e, ErrorKind::Tag),
        _ => assert_eq!(1, 0),
    }
}
<|endoftext|>


<commit_before>pub struct Captures<'a> {
    pub begin: &'a [u8],
    pub data: &'a [u8],
    pub end: &'a [u8],
}

pub fn parse_captures<'a>(input: &'a [u8]) -> Option<Captures<'a>> {
    parser_inner(input).map(|(_, cap)| cap)
}
pub fn parse_captures_iter<'a>(input: &'a [u8]) -> CaptureMatches<'a> {
    CaptureMatches { input }
}

pub struct CaptureMatches<'a> {
    input: &'a [u8],
}
impl<'a> Iterator for CaptureMatches<'a> {
    type Item = Captures<'a>;
    fn next(&mut self) -> Option<Self::Item> {
        if self.input.is_empty() {
            return None;
        }
        match parser_inner(self.input) {
            Some((remaining, captures)) => {
                self.input = remaining;
                Some(captures)
            }
            None => {
                self.input = &[];
                None
            }
        }
    }
}

fn parser_inner<'a>(input: &'a [u8]) -> Option<(&'a [u8], Captures<'a>)> {
    // Should be equivalent to the regex
    // "(?s)-----BEGIN (?P<begin>.*?)-----[ \t\n\r]*(?P<data>.*?)-----END (?P<end>.*?)-----[ \t\n\r]*"

    // (?s)                                      # Enable dotall (. matches all characters incl \n)
    // -----BEGIN (?P<begin>.*?)-----[ \t\n\r]*  # Parse begin
    // (?P<data>.*?)                             # Parse data
    // -----END (?P<end>.*?)-----[ \t\n\r]*      # Parse end

    let (input, _) = read_until(input, b"-----BEGIN ")?;
    let (input, begin) = read_until(input, b"-----")?;
    let input = skip_whitespace(input);
    let (input, data) = read_until(input, b"-----END ")?;
    let (remaining, end) = read_until(input, b"-----")?;
    let remaining = skip_whitespace(remaining);

    let captures = Captures { begin, data, end };
    Some((remaining, captures))
}

// Equivalent to the regex [ \t\n\r]*
fn skip_whitespace(mut input: &[u8]) -> &[u8] {
    while let Some(b' ' | b'\t' | b'\n' | b'\r') = input.first() {
        input = &input[1..];
    }
    input
}
// Equivalent to (.*?) followed by a string
// Returns the remaining input (after the secondary matched string) and the matched data
fn read_until<'a, 'b>(input: &'a [u8], marker: &'b [u8]) -> Option<(&'a [u8], &'a [u8])> {
    // If there is no end condition, short circuit
    if marker.is_empty() {
        return Some((&[], input));
    }
    let mut index = 0;
    let mut found = 0;
    while input.len() - index >= marker.len() - found {
        if input[index] == marker[found] {
            found += 1;
        } else {
            found = 0;
        }
        index += 1;
        if found == marker.len() {
            let remaining = &input[index..];
            let matched = &input[..index - found];
            return Some((remaining, matched));
        }
    }
    None
}
<commit_msg>Remove syntax requiring Rust 1.53<commit_after>pub struct Captures<'a> {
    pub begin: &'a [u8],
    pub data: &'a [u8],
    pub end: &'a [u8],
}

pub fn parse_captures<'a>(input: &'a [u8]) -> Option<Captures<'a>> {
    parser_inner(input).map(|(_, cap)| cap)
}
pub fn parse_captures_iter<'a>(input: &'a [u8]) -> CaptureMatches<'a> {
    CaptureMatches { input }
}

pub struct CaptureMatches<'a> {
    input: &'a [u8],
}
impl<'a> Iterator for CaptureMatches<'a> {
    type Item = Captures<'a>;
    fn next(&mut self) -> Option<Self::Item> {
        if self.input.is_empty() {
            return None;
        }
        match parser_inner(self.input) {
            Some((remaining, captures)) => {
                self.input = remaining;
                Some(captures)
            }
            None => {
                self.input = &[];
                None
            }
        }
    }
}

fn parser_inner<'a>(input: &'a [u8]) -> Option<(&'a [u8], Captures<'a>)> {
    // Should be equivalent to the regex
    // "(?s)-----BEGIN (?P<begin>.*?)-----[ \t\n\r]*(?P<data>.*?)-----END (?P<end>.*?)-----[ \t\n\r]*"

    // (?s)                                      # Enable dotall (. matches all characters incl \n)
    // -----BEGIN (?P<begin>.*?)-----[ \t\n\r]*  # Parse begin
    // (?P<data>.*?)                             # Parse data
    // -----END (?P<end>.*?)-----[ \t\n\r]*      # Parse end

    let (input, _) = read_until(input, b"-----BEGIN ")?;
    let (input, begin) = read_until(input, b"-----")?;
    let input = skip_whitespace(input);
    let (input, data) = read_until(input, b"-----END ")?;
    let (remaining, end) = read_until(input, b"-----")?;
    let remaining = skip_whitespace(remaining);

    let captures = Captures { begin, data, end };
    Some((remaining, captures))
}

// Equivalent to the regex [ \t\n\r]*
fn skip_whitespace(mut input: &[u8]) -> &[u8] {
    while let Some(b) = input.first() {
        match b {
            b' ' | b'\t' | b'\n' | b'\r' => {
                input = &input[1..];
            }
            _ => break,
        }
    }
    input
}
// Equivalent to (.*?) followed by a string
// Returns the remaining input (after the secondary matched string) and the matched data
fn read_until<'a, 'b>(input: &'a [u8], marker: &'b [u8]) -> Option<(&'a [u8], &'a [u8])> {
    // If there is no end condition, short circuit
    if marker.is_empty() {
        return Some((&[], input));
    }
    let mut index = 0;
    let mut found = 0;
    while input.len() - index >= marker.len() - found {
        if input[index] == marker[found] {
            found += 1;
        } else {
            found = 0;
        }
        index += 1;
        if found == marker.len() {
            let remaining = &input[index..];
            let matched = &input[..index - found];
            return Some((remaining, matched));
        }
    }
    None
}
<|endoftext|>


<commit_before>//! A parser for lambda expressions

use crate::term::{Term, Notation, abs, app};
use crate::term::Term::*;
use self::Token::*;
use self::CToken::*;
use self::ParseError::*;
use self::Expression::*;
pub use crate::term::Notation::*;

/// An error returned by `parse()` when a parsing issue is encountered.
#[derive(Debug, PartialEq)]
pub enum ParseError {
    /// lexical error; contains the invalid character and its index
    InvalidCharacter((usize, char)),
    /// syntax error; the expression is invalid
    InvalidExpression,
    /// syntax error; the expression is empty
    EmptyExpression
}

#[derive(Debug, PartialEq)]
#[doc(hidden)]
pub enum Token {
    /// the lambda symbol ('λ' or '\')
    Lambda,
    /// left parenthesis
    Lparen,
    /// right parenthesis
    Rparen,
    /// a hex-encoded digit
    Number(usize)
}

#[derive(Debug, PartialEq)]
#[doc(hidden)]
pub enum CToken {
    /// an abstraction with a bound variable
    CLambda(String),
    /// left parenthesis
    CLparen,
    /// right parenthesis
    CRparen,
    /// a variable with an identifier
    CName(String)
}

#[doc(hidden)]
pub fn tokenize_dbr(input: &str) -> Result<Vec<Token>, ParseError> {
    let chars = input.chars().enumerate();
    let mut tokens = Vec::with_capacity(input.len());

    for (i, c) in chars {
        match c {
     '\\' | 'λ' => { tokens.push(Lambda) },
            '(' => { tokens.push(Lparen) },
            ')' => { tokens.push(Rparen) },
             _  => {
                if let Some(n) = c.to_digit(16) {
                    tokens.push(Number(n as usize))
                } else if c.is_whitespace() {
                    ()
                } else {
                    return Err(InvalidCharacter((i, c)))
                }
            }
        }
    }

    Ok(tokens)
}

#[doc(hidden)]
pub fn tokenize_cla(input: &str) -> Result<Vec<CToken>, ParseError> {
    let mut chars = input.chars().enumerate().peekable();
    let mut tokens = Vec::with_capacity(input.len());

    while let Some((i, c)) = chars.next() {
        match c {
            '\\' | 'λ' => {
                let mut name = String::new();
                while let Some((i, c)) = chars.next() {
                    if c == '.' {
                        break
                    } else if c.is_alphabetic() {
                        name.push(c)
                    } else {
                        return Err(InvalidCharacter((i, c)))
                    }
                }
                tokens.push(CLambda(name))
            },
            '(' => { tokens.push(CLparen) },
            ')' => { tokens.push(CRparen) },
             _  => {
                if c.is_whitespace() {
                    ()
                } else if c.is_alphabetic() {
                    let mut name = c.to_string();
                    while let Some(&(_, c)) = chars.peek() {
                        if c.is_whitespace() || c == '(' || c == ')' {
                            break
                        } else {
                            name.push(c);
                            chars.next();
                        }
                    }
                    tokens.push(CName(name))
                } else {
                    return Err(InvalidCharacter((i, c)))
                }
            }
        }
    }

    Ok(tokens)
}

#[doc(hidden)]
pub fn convert_classic_tokens(tokens: &[CToken]) -> Vec<Token> {
    _convert_classic_tokens(tokens, &mut Vec::with_capacity(tokens.len()), &mut 0)
}

fn _convert_classic_tokens<'t, 's>(tokens: &'t [CToken], stack: &'s mut Vec<&'t str>, pos: &mut usize) -> Vec<Token>
{
    let mut output = Vec::with_capacity(tokens.len() - *pos);
    let mut inner_stack_count = 0;

    while let Some(token) = tokens.get(*pos) {
        match *token {
            CLambda(ref name) => {
                output.push(Lambda);
                stack.push(name);
                inner_stack_count += 1;
            },
            CLparen => {
                output.push(Lparen);
                *pos += 1;
                output.append(&mut _convert_classic_tokens(tokens, stack, pos));
            },
            CRparen => {
                output.push(Rparen);
                let l = stack.len(); // TODO: move when NLL hits stable
                stack.truncate(l - inner_stack_count);
                return output
            },
            CName(ref name) => {
                if let Some(index) = stack.iter().rev().position(|t| t == name) {
                    output.push(Number(index + 1))
                } else {
                    output.push(Number(stack.len() + 1))
                }
            }
        }
        *pos += 1;
    }

    output
}

#[derive(Debug, PartialEq)]
#[doc(hidden)]
pub enum Expression {
    /// an abstraction
    Abstraction,
    /// a sequence of `Expression`s
    Sequence(Vec<Expression>),
    /// a variable with a De Bruijn index
    Variable(usize)
}

#[doc(hidden)]
pub fn get_ast(tokens: &[Token]) -> Result<Expression, ParseError> {
    _get_ast(tokens, &mut 0)
}

fn _get_ast(tokens: &[Token], pos: &mut usize) -> Result<Expression, ParseError> {
    if tokens.is_empty() { return Err(EmptyExpression) }

    let mut expr = Vec::new();

    while let Some(token) = tokens.get(*pos) {
        match *token {
            Lambda => {
                expr.push(Abstraction)
            },
            Number(i) => {
                expr.push(Variable(i))
            },
            Lparen => {
                *pos += 1;
                let subtree = _get_ast(tokens, pos)?;
                expr.push(subtree);
            },
            Rparen => {
                return Ok(Sequence(expr))
            }
        }
        *pos += 1;
    }

    Ok(Sequence(expr))
}

/// Attempts to parse the input `&str` as a lambda `Term` encoded in the given `Notation`.
///
/// - lambdas can be represented either with the greek letter (λ) or a backslash (\\ -
/// less aesthetic, but only one byte in size)
/// - the identifiers in `Classic` notation are `String`s of alphabetic Unicode characters
/// - `Classic` notation ignores whitespaces where unambiguous
/// - the indices in the `DeBruijn` notation start with 1 and are hexadecimal digits
/// - `DeBruijn` notation ignores all whitespaces (since indices > 15 are very unlikely)
///
/// # Examples
/// ```
/// use lambda_calculus::*;
/// use lambda_calculus::combinators::{S, Y};
///
/// assert_eq!(parse(&"λf.(λx.f (x x)) (λx.f (x x))", Classic), Ok(Y()));
/// assert_eq!(parse(&"λƒ.(λℵ.ƒ(ℵ ℵ))(λℵ.ƒ(ℵ ℵ))", Classic),  Ok(Y()));
///
/// assert_eq!(parse(  &"λλλ31(21)",     DeBruijn), Ok(S()));
/// assert_eq!(parse(&r#"\\\3 1 (2 1)"#, DeBruijn), Ok(S()));
/// ```
///
/// # Errors
///
/// Returns a `ParseError` when a lexing or syntax error is encountered.
pub fn parse(input: &str, notation: Notation) -> Result<Term, ParseError> {
    let tokens = if notation == DeBruijn {
        tokenize_dbr(input)?
    } else {
        convert_classic_tokens(&tokenize_cla(input)?)
    };
    let ast = get_ast(&tokens)?;

    let exprs = if let Sequence(exprs) = ast {
        Ok(exprs)
    } else {
        Err(InvalidExpression)
    };

    fold_exprs(&exprs?)
}

#[doc(hidden)]
pub fn fold_exprs(exprs: &[Expression]) -> Result<Term, ParseError> {
    let mut depth  = 0;
    let mut output = Vec::new();

    for expr in exprs.iter() {
        match *expr {
            Abstraction         => depth += 1,
            Variable(i)         => output.push(Var(i)),
            Sequence(ref exprs) => output.push(fold_exprs(exprs)?)
        }
    }

    Ok(abs!(depth, fold_terms(output)?))
}

fn fold_terms(mut terms: Vec<Term>) -> Result<Term, ParseError> {
    if terms.is_empty() {
        Err(EmptyExpression)
    } else {
        let fst = terms.remove(0);
        if terms.is_empty() {
            Ok( fst )
        } else {
            Ok( terms.into_iter().fold(fst, app) )
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokenization_error() {
        assert_eq!(tokenize_dbr(&"λλx2"),    Err(InvalidCharacter((2, 'x'))));
        assert_eq!(tokenize_cla(&"λa.λb a"), Err(InvalidCharacter((5, ' '))));
    }

    #[test]
    fn tokenization_success() {
        let quine = "λ 1 ( (λ 1 1) (λ λ λ λ λ 1 4 (3 (5 5) 2) ) ) 1";
        let tokens = tokenize_dbr(&quine);

        assert!(tokens.is_ok());
        assert_eq!(tokens.unwrap(), vec![Lambda, Number(1), Lparen, Lparen, Lambda, Number(1),
            Number(1), Rparen, Lparen, Lambda, Lambda, Lambda, Lambda, Lambda, Number(1),
            Number(4), Lparen, Number(3), Lparen, Number(5), Number(5), Rparen, Number(2),
            Rparen, Rparen, Rparen, Number(1)]);
    }

    #[test]
    fn tokenization_success_classic() {
        let blc_dbr = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))\
            (λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        let blc_cla = parse(&blc_dbr, DeBruijn).unwrap().to_string();

        let tokens_cla = tokenize_cla(&blc_cla);
        let tokens_dbr = tokenize_dbr(&blc_dbr);

        assert!(tokens_cla.is_ok());
        assert!(tokens_dbr.is_ok());

        assert_eq!(convert_classic_tokens(&tokens_cla.unwrap()), tokens_dbr.unwrap());
    }

    #[test]
    fn alternative_lambda_parsing() {
        assert_eq!(parse(&r#"\\\2(321)"#, DeBruijn), parse(&"λλλ2(321)", DeBruijn))
    }

    #[test]
    fn succ_ast() {
        let tokens = tokenize_dbr(&"λλλ2(321)").unwrap();
        let ast = get_ast(&tokens);

        assert_eq!(ast,
            Ok(Sequence(vec![
                Abstraction,
                Abstraction,
                Abstraction,
                Variable(2),
                Sequence(vec![
                    Variable(3),
                    Variable(2),
                    Variable(1)
                ])
            ])
        ));
    }

    #[test]
    fn parse_y() {
        let y = "λ(λ2(11))(λ2(11))";
        assert_eq!(parse(&y, DeBruijn).unwrap(),
            abs(
                app(
                    abs(app(Var(2), app(Var(1), Var(1)))),
                    abs(app(Var(2), app(Var(1), Var(1))))
                )
            )
        );
    }

    #[test]
    fn parse_quine() {
        let quine = "λ1((λ11)(λλλλλ14(3(55)2)))1";
        assert_eq!(parse(&quine, DeBruijn).unwrap(),
            abs(app(app(Var(1), app(abs(app(Var(1), Var(1))), abs!(5, app(app(Var(1),
            Var(4)), app(app(Var(3), app(Var(5), Var(5))), Var(2)))))), Var(1)))
        );
    }

    #[test]
    fn parse_blc() {
        let blc = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))\
                   (λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        assert_eq!(parse(&blc, DeBruijn).unwrap(),
            app(app(abs(app(Var(1), Var(1))), abs!(3, app(app(app(Var(1),
            abs!(4, app(Var(3), abs(app(app(Var(5), app(Var(3), abs(app(app(Var(2),
            app(Var(3), abs!(2, app(Var(3), abs(app(app(Var(1), Var(2)), Var(3))))))), app(Var(4),
            abs(app(Var(4), abs(app(app(Var(3), Var(1)), app(Var(2), Var(1))))))))))),
            app(app(Var(1), app(Var(2), abs(app(Var(1), Var(2))))), abs(app(app(Var(4),
            abs(app(Var(4), abs(app(Var(2), app(Var(1), Var(4))))))), Var(5))))))))),
            app(Var(3), Var(3))), Var(2)))), abs(app(Var(1), app(abs(app(Var(1), Var(1))),
            abs(app(Var(1), Var(1)))))))
        );
    }
}
<commit_msg>take advantage of now-stable NLL<commit_after>//! A parser for lambda expressions

use crate::term::{Term, Notation, abs, app};
use crate::term::Term::*;
use self::Token::*;
use self::CToken::*;
use self::ParseError::*;
use self::Expression::*;
pub use crate::term::Notation::*;

/// An error returned by `parse()` when a parsing issue is encountered.
#[derive(Debug, PartialEq)]
pub enum ParseError {
    /// lexical error; contains the invalid character and its index
    InvalidCharacter((usize, char)),
    /// syntax error; the expression is invalid
    InvalidExpression,
    /// syntax error; the expression is empty
    EmptyExpression
}

#[derive(Debug, PartialEq)]
#[doc(hidden)]
pub enum Token {
    /// the lambda symbol ('λ' or '\')
    Lambda,
    /// left parenthesis
    Lparen,
    /// right parenthesis
    Rparen,
    /// a hex-encoded digit
    Number(usize)
}

#[derive(Debug, PartialEq)]
#[doc(hidden)]
pub enum CToken {
    /// an abstraction with a bound variable
    CLambda(String),
    /// left parenthesis
    CLparen,
    /// right parenthesis
    CRparen,
    /// a variable with an identifier
    CName(String)
}

#[doc(hidden)]
pub fn tokenize_dbr(input: &str) -> Result<Vec<Token>, ParseError> {
    let chars = input.chars().enumerate();
    let mut tokens = Vec::with_capacity(input.len());

    for (i, c) in chars {
        match c {
     '\\' | 'λ' => { tokens.push(Lambda) },
            '(' => { tokens.push(Lparen) },
            ')' => { tokens.push(Rparen) },
             _  => {
                if let Some(n) = c.to_digit(16) {
                    tokens.push(Number(n as usize))
                } else if c.is_whitespace() {
                    ()
                } else {
                    return Err(InvalidCharacter((i, c)))
                }
            }
        }
    }

    Ok(tokens)
}

#[doc(hidden)]
pub fn tokenize_cla(input: &str) -> Result<Vec<CToken>, ParseError> {
    let mut chars = input.chars().enumerate().peekable();
    let mut tokens = Vec::with_capacity(input.len());

    while let Some((i, c)) = chars.next() {
        match c {
            '\\' | 'λ' => {
                let mut name = String::new();
                while let Some((i, c)) = chars.next() {
                    if c == '.' {
                        break
                    } else if c.is_alphabetic() {
                        name.push(c)
                    } else {
                        return Err(InvalidCharacter((i, c)))
                    }
                }
                tokens.push(CLambda(name))
            },
            '(' => { tokens.push(CLparen) },
            ')' => { tokens.push(CRparen) },
             _  => {
                if c.is_whitespace() {
                    ()
                } else if c.is_alphabetic() {
                    let mut name = c.to_string();
                    while let Some(&(_, c)) = chars.peek() {
                        if c.is_whitespace() || c == '(' || c == ')' {
                            break
                        } else {
                            name.push(c);
                            chars.next();
                        }
                    }
                    tokens.push(CName(name))
                } else {
                    return Err(InvalidCharacter((i, c)))
                }
            }
        }
    }

    Ok(tokens)
}

#[doc(hidden)]
pub fn convert_classic_tokens(tokens: &[CToken]) -> Vec<Token> {
    _convert_classic_tokens(tokens, &mut Vec::with_capacity(tokens.len()), &mut 0)
}

fn _convert_classic_tokens<'t, 's>(tokens: &'t [CToken], stack: &'s mut Vec<&'t str>, pos: &mut usize) -> Vec<Token>
{
    let mut output = Vec::with_capacity(tokens.len() - *pos);
    let mut inner_stack_count = 0;

    while let Some(token) = tokens.get(*pos) {
        match *token {
            CLambda(ref name) => {
                output.push(Lambda);
                stack.push(name);
                inner_stack_count += 1;
            },
            CLparen => {
                output.push(Lparen);
                *pos += 1;
                output.append(&mut _convert_classic_tokens(tokens, stack, pos));
            },
            CRparen => {
                output.push(Rparen);
                stack.truncate(stack.len() - inner_stack_count);
                return output
            },
            CName(ref name) => {
                if let Some(index) = stack.iter().rev().position(|t| t == name) {
                    output.push(Number(index + 1))
                } else {
                    output.push(Number(stack.len() + 1))
                }
            }
        }
        *pos += 1;
    }

    output
}

#[derive(Debug, PartialEq)]
#[doc(hidden)]
pub enum Expression {
    /// an abstraction
    Abstraction,
    /// a sequence of `Expression`s
    Sequence(Vec<Expression>),
    /// a variable with a De Bruijn index
    Variable(usize)
}

#[doc(hidden)]
pub fn get_ast(tokens: &[Token]) -> Result<Expression, ParseError> {
    _get_ast(tokens, &mut 0)
}

fn _get_ast(tokens: &[Token], pos: &mut usize) -> Result<Expression, ParseError> {
    if tokens.is_empty() { return Err(EmptyExpression) }

    let mut expr = Vec::new();

    while let Some(token) = tokens.get(*pos) {
        match *token {
            Lambda => {
                expr.push(Abstraction)
            },
            Number(i) => {
                expr.push(Variable(i))
            },
            Lparen => {
                *pos += 1;
                let subtree = _get_ast(tokens, pos)?;
                expr.push(subtree);
            },
            Rparen => {
                return Ok(Sequence(expr))
            }
        }
        *pos += 1;
    }

    Ok(Sequence(expr))
}

/// Attempts to parse the input `&str` as a lambda `Term` encoded in the given `Notation`.
///
/// - lambdas can be represented either with the greek letter (λ) or a backslash (\\ -
/// less aesthetic, but only one byte in size)
/// - the identifiers in `Classic` notation are `String`s of alphabetic Unicode characters
/// - `Classic` notation ignores whitespaces where unambiguous
/// - the indices in the `DeBruijn` notation start with 1 and are hexadecimal digits
/// - `DeBruijn` notation ignores all whitespaces (since indices > 15 are very unlikely)
///
/// # Examples
/// ```
/// use lambda_calculus::*;
/// use lambda_calculus::combinators::{S, Y};
///
/// assert_eq!(parse(&"λf.(λx.f (x x)) (λx.f (x x))", Classic), Ok(Y()));
/// assert_eq!(parse(&"λƒ.(λℵ.ƒ(ℵ ℵ))(λℵ.ƒ(ℵ ℵ))", Classic),  Ok(Y()));
///
/// assert_eq!(parse(  &"λλλ31(21)",     DeBruijn), Ok(S()));
/// assert_eq!(parse(&r#"\\\3 1 (2 1)"#, DeBruijn), Ok(S()));
/// ```
///
/// # Errors
///
/// Returns a `ParseError` when a lexing or syntax error is encountered.
pub fn parse(input: &str, notation: Notation) -> Result<Term, ParseError> {
    let tokens = if notation == DeBruijn {
        tokenize_dbr(input)?
    } else {
        convert_classic_tokens(&tokenize_cla(input)?)
    };
    let ast = get_ast(&tokens)?;

    let exprs = if let Sequence(exprs) = ast {
        Ok(exprs)
    } else {
        Err(InvalidExpression)
    };

    fold_exprs(&exprs?)
}

#[doc(hidden)]
pub fn fold_exprs(exprs: &[Expression]) -> Result<Term, ParseError> {
    let mut depth  = 0;
    let mut output = Vec::new();

    for expr in exprs.iter() {
        match *expr {
            Abstraction         => depth += 1,
            Variable(i)         => output.push(Var(i)),
            Sequence(ref exprs) => output.push(fold_exprs(exprs)?)
        }
    }

    Ok(abs!(depth, fold_terms(output)?))
}

fn fold_terms(mut terms: Vec<Term>) -> Result<Term, ParseError> {
    if terms.is_empty() {
        Err(EmptyExpression)
    } else {
        let fst = terms.remove(0);
        if terms.is_empty() {
            Ok( fst )
        } else {
            Ok( terms.into_iter().fold(fst, app) )
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokenization_error() {
        assert_eq!(tokenize_dbr(&"λλx2"),    Err(InvalidCharacter((2, 'x'))));
        assert_eq!(tokenize_cla(&"λa.λb a"), Err(InvalidCharacter((5, ' '))));
    }

    #[test]
    fn tokenization_success() {
        let quine = "λ 1 ( (λ 1 1) (λ λ λ λ λ 1 4 (3 (5 5) 2) ) ) 1";
        let tokens = tokenize_dbr(&quine);

        assert!(tokens.is_ok());
        assert_eq!(tokens.unwrap(), vec![Lambda, Number(1), Lparen, Lparen, Lambda, Number(1),
            Number(1), Rparen, Lparen, Lambda, Lambda, Lambda, Lambda, Lambda, Number(1),
            Number(4), Lparen, Number(3), Lparen, Number(5), Number(5), Rparen, Number(2),
            Rparen, Rparen, Rparen, Number(1)]);
    }

    #[test]
    fn tokenization_success_classic() {
        let blc_dbr = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))\
            (λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        let blc_cla = parse(&blc_dbr, DeBruijn).unwrap().to_string();

        let tokens_cla = tokenize_cla(&blc_cla);
        let tokens_dbr = tokenize_dbr(&blc_dbr);

        assert!(tokens_cla.is_ok());
        assert!(tokens_dbr.is_ok());

        assert_eq!(convert_classic_tokens(&tokens_cla.unwrap()), tokens_dbr.unwrap());
    }

    #[test]
    fn alternative_lambda_parsing() {
        assert_eq!(parse(&r#"\\\2(321)"#, DeBruijn), parse(&"λλλ2(321)", DeBruijn))
    }

    #[test]
    fn succ_ast() {
        let tokens = tokenize_dbr(&"λλλ2(321)").unwrap();
        let ast = get_ast(&tokens);

        assert_eq!(ast,
            Ok(Sequence(vec![
                Abstraction,
                Abstraction,
                Abstraction,
                Variable(2),
                Sequence(vec![
                    Variable(3),
                    Variable(2),
                    Variable(1)
                ])
            ])
        ));
    }

    #[test]
    fn parse_y() {
        let y = "λ(λ2(11))(λ2(11))";
        assert_eq!(parse(&y, DeBruijn).unwrap(),
            abs(
                app(
                    abs(app(Var(2), app(Var(1), Var(1)))),
                    abs(app(Var(2), app(Var(1), Var(1))))
                )
            )
        );
    }

    #[test]
    fn parse_quine() {
        let quine = "λ1((λ11)(λλλλλ14(3(55)2)))1";
        assert_eq!(parse(&quine, DeBruijn).unwrap(),
            abs(app(app(Var(1), app(abs(app(Var(1), Var(1))), abs!(5, app(app(Var(1),
            Var(4)), app(app(Var(3), app(Var(5), Var(5))), Var(2)))))), Var(1)))
        );
    }

    #[test]
    fn parse_blc() {
        let blc = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))\
                   (λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        assert_eq!(parse(&blc, DeBruijn).unwrap(),
            app(app(abs(app(Var(1), Var(1))), abs!(3, app(app(app(Var(1),
            abs!(4, app(Var(3), abs(app(app(Var(5), app(Var(3), abs(app(app(Var(2),
            app(Var(3), abs!(2, app(Var(3), abs(app(app(Var(1), Var(2)), Var(3))))))), app(Var(4),
            abs(app(Var(4), abs(app(app(Var(3), Var(1)), app(Var(2), Var(1))))))))))),
            app(app(Var(1), app(Var(2), abs(app(Var(1), Var(2))))), abs(app(app(Var(4),
            abs(app(Var(4), abs(app(Var(2), app(Var(1), Var(4))))))), Var(5))))))))),
            app(Var(3), Var(3))), Var(2)))), abs(app(Var(1), app(abs(app(Var(1), Var(1))),
            abs(app(Var(1), Var(1)))))))
        );
    }
}
<|endoftext|>