<commit_before>// External modules:
use walkdir::WalkDir;
use error::{Result, ResultExt};
use image;
use image::imageops::resize;
use image::{GenericImage, FilterType};

// Std modules:
use std::fs::File;
use std::path::Path;

// Internal modules:
use config::PanolutionConfig;
use util::is_supported_format;

fn split_path(path: &str) -> (&str, &str) {
    let path = Path::new(path);

    (path.parent().unwrap().to_str().unwrap(),
     path.file_name().unwrap().to_str().unwrap())
}

pub fn create_thumbnails(config: &PanolutionConfig) -> Result<(Vec<String>, Vec<Vec<String>>)> {
    let mut all_image_paths = Vec::new();
    let mut all_thumbnail_paths = Vec::new();

    for entry in WalkDir::new(&config.input_path) {
        let entry = entry.chain_err(|| "error in WalkDir")?;
        if entry.file_type().is_file() {
            let entry = entry.file_name().to_str().unwrap();
            if is_supported_format(entry) {
                if entry.starts_with("thumb_") {
                    info!("Ignore thumbnail file: {}", entry);
                } else {
                    let full_path = format!("{}/{}", config.input_path, entry); // TODO: windows path separator
                    all_image_paths.push(full_path);
                }
            } else {
                info!("Image format not supported yet: {}", entry);
            }
        }
    }

    for scale_factor in &config.scale_factors {
        let mut thumb_paths = Vec::new();
        for full_path in &all_image_paths {
            info!("Loading image file: '{}'", full_path);
            let orig_img = image::open(&full_path).chain_err(|| format!("can't open image: '{}", full_path))?;
            let (base_path, file_name) = split_path(&full_path);
            let thumb_path = format!("{}/thumb_{}_{}", base_path, scale_factor, file_name);
            if Path::new(&thumb_path).exists() {
                info!("Thumbnail was already generated, using old one: '{}'", &thumb_path)
            } else {
                info!("No thumbnail found for scale factor '{}', generating new one", scale_factor);
                let (orig_w, orig_h) = orig_img.dimensions();
                let (thumb_w, thumb_h) = (((orig_w as f64) * scale_factor) as u32, ((orig_h as f64) * scale_factor) as u32);
                info!("image sizes: ({}, {}) -> ({}, {})", orig_w, orig_h, thumb_w, thumb_h);
                let thumb_img = resize(&orig_img, thumb_w, thumb_h, FilterType::Nearest);
                let _ = thumb_img.save(&thumb_path).chain_err(|| "can't save thumbnail image")?;
            }
            thumb_paths.push(thumb_path);
        }
        all_thumbnail_paths.push(thumb_paths);
    }

    Ok((all_image_paths, all_thumbnail_paths))
}
<commit_msg>avoid loading images if not needed<commit_after>// External modules:
use walkdir::WalkDir;
use error::{Result, ResultExt};
use image;
use image::imageops::resize;
use image::{GenericImage, FilterType};

// Std modules:
use std::fs::File;
use std::path::Path;

// Internal modules:
use config::PanolutionConfig;
use util::is_supported_format;

fn split_path(path: &str) -> (&str, &str) {
    let path = Path::new(path);

    (path.parent().unwrap().to_str().unwrap(),
     path.file_name().unwrap().to_str().unwrap())
}

pub fn create_thumbnails(config: &PanolutionConfig) -> Result<(Vec<String>, Vec<Vec<String>>)> {
    let mut all_image_paths = Vec::new();
    let mut all_thumbnail_paths = Vec::new();

    for entry in WalkDir::new(&config.input_path) {
        let entry = entry.chain_err(|| "error in WalkDir")?;
        if entry.file_type().is_file() {
            let entry = entry.file_name().to_str().unwrap();
            if is_supported_format(entry) {
                if entry.starts_with("thumb_") {
                    info!("Ignore thumbnail file: {}", entry);
                } else {
                    let full_path = format!("{}/{}", config.input_path, entry); // TODO: windows path separator
                    all_image_paths.push(full_path);
                }
            } else {
                info!("Image format not supported yet: {}", entry);
            }
        }
    }

    for scale_factor in &config.scale_factors {
        let mut thumb_paths = Vec::new();
        for full_path in &all_image_paths {
            let (base_path, file_name) = split_path(&full_path);
            let thumb_path = format!("{}/thumb_{}_{}", base_path, scale_factor, file_name);
            if Path::new(&thumb_path).exists() {
                info!("Thumbnail was already generated, using old one: '{}'", &thumb_path)
            } else {
                info!("Loading image file: '{}'", full_path);
                let orig_img = image::open(&full_path).chain_err(|| format!("can't open image: '{}", full_path))?;
                info!("No thumbnail found for scale factor '{}', generating new one", scale_factor);
                let (orig_w, orig_h) = orig_img.dimensions();
                let (thumb_w, thumb_h) = (((orig_w as f64) * scale_factor) as u32, ((orig_h as f64) * scale_factor) as u32);
                info!("image sizes: ({}, {}) -> ({}, {})", orig_w, orig_h, thumb_w, thumb_h);
                let thumb_img = resize(&orig_img, thumb_w, thumb_h, FilterType::Nearest);
                let _ = thumb_img.save(&thumb_path).chain_err(|| "can't save thumbnail image")?;
            }
            thumb_paths.push(thumb_path);
        }
        all_thumbnail_paths.push(thumb_paths);
    }

    Ok((all_image_paths, all_thumbnail_paths))
}
<|endoftext|>


<commit_before>use chrono;                                     //Datetime Handling
use time;
use byteorder::{ReadBytesExt, LittleEndian};    //Reading little endian data structs
use std::io::{Error};
use std::fmt;
use std::fmt::{Display,Debug};
use std::io::Read;
use serde::{ser};

pub static mut TIMESTAMP_FORMAT: &'static str = "%Y-%m-%d %H:%M:%S%.3f";

pub struct WinTimestamp(
    pub u64
);
impl WinTimestamp {
    pub fn to_datetime(&self) -> chrono::NaiveDateTime {
        // Get nanoseconds (100-nanosecond intervals)
        let t_micro = self.0 / 10;
        // Add microseconds to timestamp via Duration
        (
            chrono::NaiveDate::from_ymd(1601, 1, 1).and_hms_nano(0, 0, 0, 0) + // Win Epoc = 1601-01-01
            time::Duration::microseconds(t_micro as i64)
        ) as chrono::NaiveDateTime
    }
}
impl Display for WinTimestamp {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.to_datetime())
    }
}
impl Debug for WinTimestamp {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.to_datetime())
    }
}
impl ser::Serialize for WinTimestamp {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: ser::Serializer
    {
        serializer.serialize_str(
            &format!("{}",
            self.to_datetime().format(unsafe{TIMESTAMP_FORMAT}).to_string())
        )
    }
}

#[allow(dead_code)]
pub fn raw_to_wintimestamp<R: Read>(mut buffer: R)->Result<WinTimestamp,Error>{
    let win_timestamp: WinTimestamp = WinTimestamp(
        buffer.read_u64::<LittleEndian>().unwrap()
    );
    Ok(win_timestamp)
}

#[test]
fn test_timestamp() {
    let raw_timestamp: &[u8] = &[0x53,0xC7,0x8B,0x18,0xC5,0xCC,0xCE,0x01];

    let time_stamp: WinTimestamp = match raw_to_wintimestamp(raw_timestamp){
        Ok(time_stamp) => time_stamp,
        Err(error) => panic!(error)
    };

    assert_eq!(format!("{}",time_stamp),"2013-10-19 12:16:53.276040");
    assert_eq!(format!("{:?}",time_stamp),"2013-10-19 12:16:53.276040");
    assert_eq!(time_stamp.0,130266586132760403);
}
<commit_msg>Added clone trait for WinTimestamp<commit_after>use chrono;                                     //Datetime Handling
use time;
use byteorder::{ReadBytesExt, LittleEndian};    //Reading little endian data structs
use std::io::{Error};
use std::fmt;
use std::fmt::{Display,Debug};
use std::io::Read;
use serde::{ser};

pub static mut TIMESTAMP_FORMAT: &'static str = "%Y-%m-%d %H:%M:%S%.3f";

#[derive(Clone)]
pub struct WinTimestamp(
    pub u64
);
impl WinTimestamp {
    pub fn to_datetime(&self) -> chrono::NaiveDateTime {
        // Get nanoseconds (100-nanosecond intervals)
        let t_micro = self.0 / 10;
        // Add microseconds to timestamp via Duration
        (
            chrono::NaiveDate::from_ymd(1601, 1, 1).and_hms_nano(0, 0, 0, 0) + // Win Epoc = 1601-01-01
            time::Duration::microseconds(t_micro as i64)
        ) as chrono::NaiveDateTime
    }
}
impl Display for WinTimestamp {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.to_datetime())
    }
}
impl Debug for WinTimestamp {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.to_datetime())
    }
}
impl ser::Serialize for WinTimestamp {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: ser::Serializer
    {
        serializer.serialize_str(
            &format!("{}",
            self.to_datetime().format(unsafe{TIMESTAMP_FORMAT}).to_string())
        )
    }
}

#[allow(dead_code)]
pub fn raw_to_wintimestamp<R: Read>(mut buffer: R)->Result<WinTimestamp,Error>{
    let win_timestamp: WinTimestamp = WinTimestamp(
        buffer.read_u64::<LittleEndian>().unwrap()
    );
    Ok(win_timestamp)
}

#[test]
fn test_timestamp() {
    let raw_timestamp: &[u8] = &[0x53,0xC7,0x8B,0x18,0xC5,0xCC,0xCE,0x01];

    let time_stamp: WinTimestamp = match raw_to_wintimestamp(raw_timestamp){
        Ok(time_stamp) => time_stamp,
        Err(error) => panic!(error)
    };

    assert_eq!(format!("{}",time_stamp),"2013-10-19 12:16:53.276040");
    assert_eq!(format!("{:?}",time_stamp),"2013-10-19 12:16:53.276040");
    assert_eq!(time_stamp.0,130266586132760403);
}
<|endoftext|>


<commit_before>use std::default::Default;
use super::len_while;

pub trait Tokenizer: Clone + Default + ::std::fmt::Show {
	fn token_len(&self, s: &str) -> Option<uint>;
}

#[deriving(Clone, Default, Show)]
pub struct WordsAndInts;

impl Tokenizer for WordsAndInts {
	fn token_len(&self, s: &str) -> Option<uint> {
		if s.len() == 0 {
			return None;
		}

		let ch0 = s.char_at(0);

		if ch0.is_alphabetic() {
			len_while(s, |ch| ch.is_alphabetic())
		} else if ch0.is_digit() {
			len_while(s, |ch| ch.is_digit())
		} else {
			None
		}
	}
}
<commit_msg>Added tests for WordsAndInts tokeniser.<commit_after>use std::default::Default;
use super::len_while;

pub trait Tokenizer: Clone + Default + ::std::fmt::Show {
	fn token_len(&self, s: &str) -> Option<uint>;
}

#[deriving(Clone, Default, Show)]
pub struct WordsAndInts;

impl Tokenizer for WordsAndInts {
	fn token_len(&self, s: &str) -> Option<uint> {
		if s.len() == 0 {
			return None;
		}

		let ch0 = s.char_at(0);

		if ch0.is_alphabetic() {
			len_while(s, |ch| ch.is_alphabetic())
		} else if ch0.is_digit() {
			len_while(s, |ch| ch.is_digit())
		} else {
			None
		}
	}
}

#[test]
fn test_words_and_ints() {
	let tl = |s:&str| WordsAndInts.token_len(s);

	assert_eq!(tl(""), None);
	assert_eq!(tl("_"), None);
	assert_eq!(tl("abc"), Some(3));
	assert_eq!(tl("abc def"), Some(3));
	assert_eq!(tl("abc123"), Some(3));
	assert_eq!(tl("abc_def"), Some(3));
	assert_eq!(tl("123"), Some(3));
	assert_eq!(tl("123 456"), Some(3));
	assert_eq!(tl("123abc"), Some(3));
	assert_eq!(tl("123_456"), Some(3));
	assert_eq!(tl("123.456"), Some(3));
}
<|endoftext|>


<commit_before>use encoding::types::EncodingRef;

use error::Error;
use history::{Cursor, History};
use buffer::Buffer;
use parser::{parse, ParseError, ParseSuccess};
use instr;

#[derive(Copy,Clone)]
pub enum EditMode {
    Emacs,
    Vi,
}

#[derive(Copy, Clone, PartialEq)]
pub enum ViMode {
    Insert,
    Normal,
    Replace,
}

pub struct EditCtx<'a> {
    buf: Buffer,
    history_cursor: Cursor<'a>,
    prompt: &'a str,
    seq: Vec<u8>,
    enc: EncodingRef,
    mode: EditMode,
    vi_mode: ViMode,
    vi_count: u32,
}

impl<'a> EditCtx<'a> {

    pub fn new(prompt: &'a str, history: &'a History, enc: EncodingRef, mode: EditMode) -> Self {
        EditCtx {
            buf: Buffer::new(),
            history_cursor: Cursor::new(history),
            prompt: prompt,
            seq: Vec::new(),
            enc: enc,
            mode: mode,
            // always start in insert mode
            vi_mode: ViMode::Insert,
            vi_count: 0,
        }
    }

    pub fn fill(&mut self, byte: u8) {
        self.seq.push(byte)
    }

}

pub enum EditResult<C> {
    Cont(C),
    Halt(Result<String, Error>)
}

macro_rules! vi_repeat {
    ( $ctx:ident, $x:expr ) => {
        match $ctx.vi_count {
            0 => { $x; }
            _ => for _ in 0..$ctx.vi_count { $x; }
        }
        $ctx.vi_count = 0;
    };
    ( $ctx:ident, $operation:expr, $step:expr ) => {
        match $ctx.vi_count {
            0 => { $operation; }
            _ => {
                $operation;
                for _ in 1..$ctx.vi_count { $step; $operation; }
            }
        }
        $ctx.vi_count = 0;
    };
}

pub fn edit<'a>(ctx: &mut EditCtx<'a>) -> EditResult<Vec<u8>> {
    use self::EditResult::*;

    let res = match parse(&ctx.seq, ctx.enc) {
        Err(ParseError::Error(len)) => {
            for _ in (0..len) {
                ctx.seq.remove(0);
            };
            Cont(false)
        },
        Err(ParseError::Incomplete) => Cont(false),
        Ok(ParseSuccess(token, len)) => {
            let res = match instr::interpret_token(token, ctx.mode, ctx.vi_mode) {
                instr::Instr::Done => {
                    Halt(Ok(ctx.buf.drain()))
                },
                instr::Instr::DeleteCharLeftOfCursor => {
                    ctx.buf.delete_char_left_of_cursor();
                    Cont(false)
                },
                instr::Instr::DeleteCharRightOfCursor => {
                    ctx.buf.delete_char_right_of_cursor();
                    Cont(false)
                },
                instr::Instr::DeleteCharRightOfCursorOrEOF => {
                    if !ctx.buf.delete_char_right_of_cursor() {
                        Halt(Err(Error::EndOfFile))
                    } else {
                        Cont(false)
                    }
                },
                instr::Instr::MoveCursorLeft => {
                    ctx.buf.move_left();
                    Cont(false)
                },
                instr::Instr::MoveCursorRight => {
                    ctx.buf.move_right();
                    Cont(false)
                },
                instr::Instr::MoveCursorStart => {
                    ctx.buf.move_start();
                    Cont(false)
                },
                instr::Instr::MoveCursorEnd => {
                    ctx.buf.move_end();
                    Cont(false)
                },
                instr::Instr::HistoryPrev => {
                    if ctx.history_cursor.incr() {
                        ctx.buf.swap()
                    }
                    ctx.history_cursor.get().map(|s| ctx.buf.replace(s));
                    Cont(false)
                },
                instr::Instr::HistoryNext => {
                    if ctx.history_cursor.decr() {
                        ctx.buf.swap()
                    }
                    ctx.history_cursor.get().map(|s| ctx.buf.replace(s));
                    Cont(false)
                },
                instr::Instr::NormalMode => {
                    if ctx.vi_mode == ViMode::Insert {
                        // cursor moves left when leaving insert mode
                        ctx.buf.move_left();
                    }
                    ctx.vi_mode = ViMode::Normal;
                    ctx.vi_count = 0;
                    Cont(false)
                }
                instr::Instr::ReplaceMode => {
                    ctx.vi_mode = ViMode::Replace;
                    Cont(false)
                }
                instr::Instr::Insert => {
                    ctx.vi_mode = ViMode::Insert;
                    Cont(false)
                }
                instr::Instr::InsertStart => {
                    ctx.vi_mode = ViMode::Insert;
                    ctx.buf.move_start();
                    Cont(false)
                }
                instr::Instr::Append => {
                    ctx.vi_mode = ViMode::Insert;
                    ctx.buf.move_right();
                    Cont(false)
                }
                instr::Instr::AppendEnd => {
                    ctx.vi_mode = ViMode::Insert;
                    ctx.buf.move_end();
                    Cont(false)
                }
                instr::Instr::Digit(i) => {
                    match (ctx.vi_count, i) {
                        // if count is 0, then 0 moves to the start of a line
                        (0, 0) => ctx.buf.move_start(),
                        // otherwise add a digit to the count
                        (_, i) => ctx.vi_count = ctx.vi_count * 10 + i,
                    }
                    Cont(false)
                }
                instr::Instr::MoveEndOfWordRight => {
                    vi_repeat!(ctx, ctx.buf.move_to_end_of_word());
                    Cont(false)
                }
                instr::Instr::Substitute => {
                    ctx.buf.delete_char_right_of_cursor();
                    ctx.vi_mode = ViMode::Insert;
                    Cont(false)
                }
                instr::Instr::Noop => {
                    Cont(false)
                },
                instr::Instr::Cancel => Halt(Err(Error::Cancel)),
                instr::Instr::Clear => {
                    Cont(true)
                },
                instr::Instr::InsertAtCursor(text) => {
                    ctx.buf.insert_chars_at_cursor(text);
                    Cont(false)
                }
                instr::Instr::ReplaceAtCursor(text) => {
                    ctx.buf.replace_chars_at_cursor(text);
                    ctx.vi_mode = ViMode::Normal;
                    Cont(false)
                }
            };
            for _ in (0..len) {
                ctx.seq.remove(0);
            };
            res
        }
    };
    match res {
        Cont(clear) => EditResult::Cont(ctx.buf.get_line(ctx.prompt, clear)),
        Halt(res) => EditResult::Halt(res)
    }
}
<commit_msg>add repeat to 'r' command<commit_after>use encoding::types::EncodingRef;

use error::Error;
use history::{Cursor, History};
use buffer::Buffer;
use parser::{parse, ParseError, ParseSuccess};
use instr;

#[derive(Copy,Clone)]
pub enum EditMode {
    Emacs,
    Vi,
}

#[derive(Copy, Clone, PartialEq)]
pub enum ViMode {
    Insert,
    Normal,
    Replace,
}

pub struct EditCtx<'a> {
    buf: Buffer,
    history_cursor: Cursor<'a>,
    prompt: &'a str,
    seq: Vec<u8>,
    enc: EncodingRef,
    mode: EditMode,
    vi_mode: ViMode,
    vi_count: u32,
}

impl<'a> EditCtx<'a> {

    pub fn new(prompt: &'a str, history: &'a History, enc: EncodingRef, mode: EditMode) -> Self {
        EditCtx {
            buf: Buffer::new(),
            history_cursor: Cursor::new(history),
            prompt: prompt,
            seq: Vec::new(),
            enc: enc,
            mode: mode,
            // always start in insert mode
            vi_mode: ViMode::Insert,
            vi_count: 0,
        }
    }

    pub fn fill(&mut self, byte: u8) {
        self.seq.push(byte)
    }

}

pub enum EditResult<C> {
    Cont(C),
    Halt(Result<String, Error>)
}

macro_rules! vi_repeat {
    ( $ctx:ident, $x:expr ) => {
        match $ctx.vi_count {
            0 => { $x; }
            _ => for _ in 0..$ctx.vi_count { $x; }
        }
        $ctx.vi_count = 0;
    };
    ( $ctx:ident, $operation:expr, $step:expr ) => {
        match $ctx.vi_count {
            0 => { $operation; }
            _ => {
                $operation;
                for _ in 1..$ctx.vi_count { $step; $operation; }
            }
        }
        $ctx.vi_count = 0;
    };
}

pub fn edit<'a>(ctx: &mut EditCtx<'a>) -> EditResult<Vec<u8>> {
    use self::EditResult::*;

    let res = match parse(&ctx.seq, ctx.enc) {
        Err(ParseError::Error(len)) => {
            for _ in (0..len) {
                ctx.seq.remove(0);
            };
            Cont(false)
        },
        Err(ParseError::Incomplete) => Cont(false),
        Ok(ParseSuccess(token, len)) => {
            let res = match instr::interpret_token(token, ctx.mode, ctx.vi_mode) {
                instr::Instr::Done => {
                    Halt(Ok(ctx.buf.drain()))
                },
                instr::Instr::DeleteCharLeftOfCursor => {
                    ctx.buf.delete_char_left_of_cursor();
                    Cont(false)
                },
                instr::Instr::DeleteCharRightOfCursor => {
                    ctx.buf.delete_char_right_of_cursor();
                    Cont(false)
                },
                instr::Instr::DeleteCharRightOfCursorOrEOF => {
                    if !ctx.buf.delete_char_right_of_cursor() {
                        Halt(Err(Error::EndOfFile))
                    } else {
                        Cont(false)
                    }
                },
                instr::Instr::MoveCursorLeft => {
                    ctx.buf.move_left();
                    Cont(false)
                },
                instr::Instr::MoveCursorRight => {
                    ctx.buf.move_right();
                    Cont(false)
                },
                instr::Instr::MoveCursorStart => {
                    ctx.buf.move_start();
                    Cont(false)
                },
                instr::Instr::MoveCursorEnd => {
                    ctx.buf.move_end();
                    Cont(false)
                },
                instr::Instr::HistoryPrev => {
                    if ctx.history_cursor.incr() {
                        ctx.buf.swap()
                    }
                    ctx.history_cursor.get().map(|s| ctx.buf.replace(s));
                    Cont(false)
                },
                instr::Instr::HistoryNext => {
                    if ctx.history_cursor.decr() {
                        ctx.buf.swap()
                    }
                    ctx.history_cursor.get().map(|s| ctx.buf.replace(s));
                    Cont(false)
                },
                instr::Instr::NormalMode => {
                    if ctx.vi_mode == ViMode::Insert {
                        // cursor moves left when leaving insert mode
                        ctx.buf.move_left();
                    }
                    ctx.vi_mode = ViMode::Normal;
                    ctx.vi_count = 0;
                    Cont(false)
                }
                instr::Instr::ReplaceMode => {
                    ctx.vi_mode = ViMode::Replace;
                    Cont(false)
                }
                instr::Instr::Insert => {
                    ctx.vi_mode = ViMode::Insert;
                    Cont(false)
                }
                instr::Instr::InsertStart => {
                    ctx.vi_mode = ViMode::Insert;
                    ctx.buf.move_start();
                    Cont(false)
                }
                instr::Instr::Append => {
                    ctx.vi_mode = ViMode::Insert;
                    ctx.buf.move_right();
                    Cont(false)
                }
                instr::Instr::AppendEnd => {
                    ctx.vi_mode = ViMode::Insert;
                    ctx.buf.move_end();
                    Cont(false)
                }
                instr::Instr::Digit(i) => {
                    match (ctx.vi_count, i) {
                        // if count is 0, then 0 moves to the start of a line
                        (0, 0) => ctx.buf.move_start(),
                        // otherwise add a digit to the count
                        (_, i) => ctx.vi_count = ctx.vi_count * 10 + i,
                    }
                    Cont(false)
                }
                instr::Instr::MoveEndOfWordRight => {
                    vi_repeat!(ctx, ctx.buf.move_to_end_of_word());
                    Cont(false)
                }
                instr::Instr::Substitute => {
                    ctx.buf.delete_char_right_of_cursor();
                    ctx.vi_mode = ViMode::Insert;
                    Cont(false)
                }
                instr::Instr::Noop => {
                    Cont(false)
                },
                instr::Instr::Cancel => Halt(Err(Error::Cancel)),
                instr::Instr::Clear => {
                    Cont(true)
                },
                instr::Instr::InsertAtCursor(text) => {
                    ctx.buf.insert_chars_at_cursor(text);
                    Cont(false)
                }
                instr::Instr::ReplaceAtCursor(text) => {
                    vi_repeat!(
                        ctx,
                        ctx.buf.replace_chars_at_cursor(text.clone()),
                        ctx.buf.move_right()
                    );
                    ctx.vi_mode = ViMode::Normal;
                    Cont(false)
                }
            };
            for _ in (0..len) {
                ctx.seq.remove(0);
            };
            res
        }
    };
    match res {
        Cont(clear) => EditResult::Cont(ctx.buf.get_line(ctx.prompt, clear)),
        Halt(res) => EditResult::Halt(res)
    }
}
<|endoftext|>


<commit_before>use std::str::from_utf8;
use std::result::Result;
use nom::*;

/*
 * Core structs
 */

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     &'a str,
    pub mode:     &'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   &'a str,
    pub typeflag: TypeFlag,
    pub linkname: &'a str,
    pub ustar:    ExtraHeader<'a>
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    PaxInterexchangeFormat,
    PaxExtendedAttributes,
    VendorSpecific
}

#[derive(Debug,PartialEq,Eq)]
pub enum ExtraHeader<'a> {
    UStar(UStarHeader<'a>),
    Padding
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    &'a str,
    pub version:  &'a str,
    pub uname:    &'a str,
    pub gname:    &'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub extra:    UStarExtraHeader<'a>
}

#[derive(Debug,PartialEq,Eq)]
pub enum UStarExtraHeader<'a> {
    PosixUStar(PosixUStarHeader<'a>),
    Pax(PaxHeader<'a>)
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixUStarHeader<'a> {
    pub prefix: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PaxHeader<'a> {
    pub atime:      u64,
    pub ctime:      u64,
    pub offset:     u64,
    pub longnames:  &'a str,
    pub sparses:    Vec<Sparse>,
    pub isextended: bool,
    pub realsize:   u64,
}

#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub struct Sparse {
    pub offset:   u64,
    pub numbytes: u64
}

#[derive(Debug,PartialEq,Eq)]
pub struct Padding;

/*
 * Useful macros
 */

macro_rules! take_str_eat_garbage (
    ( $i:expr, $size:expr ) => (
        chain!($i,
            s: map_res!(take_until!("\0"), from_utf8) ~
            take!($size - s.len()),
            ||{
                s
            }
        )
    );
);

named!(parse_str4<&[u8], &str>, take_str_eat_garbage!(4));
named!(parse_str8<&[u8], &str>, take_str_eat_garbage!(8));
named!(parse_str32<&[u8], &str>, take_str_eat_garbage!(32));
named!(parse_str100<&[u8], &str>, take_str_eat_garbage!(100));
named!(parse_str155<&[u8], &str>, take_str_eat_garbage!(155));

macro_rules! take_until_expr_with_limit(
  ($i:expr, $submac:ident!( $($args:tt)* ), $stop: expr, $limit: expr) => (
    {
      let mut begin = 0;
      let mut remaining = $i.len();
      let mut res = Vec::new();
      let mut cnt = 0;
      let mut err = false;
      loop {
        match $submac!(&$i[begin..], $($args)*) {
          IResult::Done(i,o) => {
            if $stop(o) {
              break
            }
            res.push(o);
            begin += remaining - i.len();
            remaining = i.len();
            cnt = cnt + 1;
            if cnt == $limit {
              break
            }
          },
          IResult::Error(_)  => {
            err = true;
            break;
          },
          IResult::Incomplete(_) => {
            break;
          }
        }
      }
      if err {
        IResult::Error(Err::Position(ErrorCode::Count as u32,$i))
      } else if cnt == $limit {
        IResult::Done(&$i[begin..], res)
      } else {
        IResult::Incomplete(Needed::Unknown)
      }
    }
  );
  ($i:expr, $f:expr, $stop: expr, $limit: expr) => (
    take_until_expr_with_limit!($i, call!($f), $stop, $limit);
  );
);

/*
 * Octal string parsing
 */

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;

    for c in s.chars() {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u *= 8;
        u += (c as u64) - ('0' as u64);
    }

    Ok(u)
}

fn parse_octal(i: &[u8], n: usize) -> IResult<&[u8], u64> {
    map_res!(i, take_str_eat_garbage!(n), octal_to_u64)
}

named!(parse_octal8<&[u8], u64>, apply!(parse_octal, 8));
named!(parse_octal12<&[u8], u64>, apply!(parse_octal, 12));

/*
 * TypeFlag parsing
 */

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::PaxInterexchangeFormat,
        'x' => TypeFlag::PaxExtendedAttributes,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::NormalFile
    }
}

fn bytes_to_type_flag(i: &[u8]) -> Result<TypeFlag, &'static str> {
    Ok(char_to_type_flag(i[0] as char))
}

named!(parse_type_flag<&[u8], TypeFlag>, map_res!(take!(1), bytes_to_type_flag));

/*
 * Sparse parsing
 */

fn parse_one_sparse(i: &[u8]) -> IResult<&[u8], Sparse> {
    chain!(i,
        offset:   parse_octal12 ~
        numbytes: parse_octal12,
        ||{
            Sparse {
                offset:   offset,
                numbytes: numbytes
            }
        }
    )
}

fn parse_sparses_with_limit(i: &[u8], limit: usize) -> IResult<&[u8], Vec<Sparse>> {
    take_until_expr_with_limit!(i, parse_one_sparse, |s: Sparse| s.offset == 0 && s.numbytes == 0, limit)
}

fn add_to_vec(extra: Vec<Sparse>, sparses: &mut Vec<Sparse>) -> Result<&'static str, &'static str> {
    for sparse in &extra {
        sparses.push(*sparse);
    }
    Ok("")
}

fn parse_extra_sparses<'a>(i: &'a [u8], isextended: bool, sparses: &'a mut Vec<Sparse>) -> IResult<'a, &'a [u8], &'a mut Vec<Sparse>> {
    if isextended {
        chain!(i,
            map_res!(apply!(parse_sparses_with_limit, 21), apply!(add_to_vec, sparses)) ~
            extended:      parse_bool                                                   ~
            take!(7) /* padding to 512 */                                               ~
            extra_sparses: apply!(parse_extra_sparses, extended, sparses),
            ||{
                extra_sparses
            }
        )
    } else {
        IResult::Done(i, sparses)
    }
}

/*
 * Boolean parsing
 */

fn to_bool(i: &[u8]) -> Result<bool, &'static str> {
    Ok(i[0] != 0)
}

named!(parse_bool<&[u8], bool>, map_res!(take!(1), to_bool));

/*
 * UStar PAX extended parsing
 */

fn parse_ustar00_extra_pax(i: &[u8]) -> IResult<&[u8], PaxHeader> {
    chain!(i,
        atime:       parse_octal12                       ~
        ctime:       parse_octal12                       ~
        offset:      parse_octal12                       ~
        longnames:   parse_str4                          ~
        take!(1)                                         ~
        sparses:     apply!(parse_sparses_with_limit, 4) ~
        isextended:  parse_bool                          ~
        realsize:    parse_octal12                       ~
        take!(17), /* padding to 512 */
        ||{
            PaxHeader {
                atime:         atime,
                ctime:         ctime,
                offset:        offset,
                longnames:     longnames,
                sparses:       sparses,
                isextended:    isextended,
                realsize:      realsize,
            }
        }
    )
}

/*
 * UStar Posix parsing
 */

fn parse_ustar00_extra_posix(i: &[u8]) -> IResult<&[u8], UStarExtraHeader> {
    chain!(i,
        prefix: parse_str155 ~
        take!(12),
        ||{
            UStarExtraHeader::PosixUStar(PosixUStarHeader {
                prefix: prefix
            })
        }
    )
}

fn parse_ustar00_extra(i: &[u8], flag: TypeFlag) -> IResult<&[u8], UStarExtraHeader> {
    match flag {
        TypeFlag::PaxInterexchangeFormat => {
            chain!(i,
                header: parse_ustar00_extra_pax ~
                apply!(parse_extra_sparses, header.isextended, &mut header.sparses),
                ||{
                    UStarExtraHeader::Pax(header)
                }
            )
        },
        _ => parse_ustar00_extra_posix(i)
    }
}

fn parse_ustar00(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("00")             ~
        uname:    parse_str32  ~
        gname:    parse_str32  ~
        devmajor: parse_octal8 ~
        devminor: parse_octal8 ~
        extra:    apply!(parse_ustar00_extra, flag),
        ||{
            ExtraHeader::UStar(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                extra:    extra
            })
        }
    )
}

fn parse_ustar(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: apply!(parse_ustar00, flag),
        ||{
            ustar
        }
    )
}

/*
 * Posix tar archive header parsing
 */

fn parse_posix(i: &[u8]) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            ExtraHeader::Padding
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     parse_str100    ~
        mode:     parse_str8      ~
        uid:      parse_octal8    ~
        gid:      parse_octal8    ~
        size:     parse_octal12   ~
        mtime:    parse_octal12   ~
        chksum:   parse_str8      ~
        typeflag: parse_type_flag ~
        linkname: parse_str100    ~
        ustar:    alt!(apply!(parse_ustar, typeflag) | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: typeflag,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

/*
 * Contents parsing
 */

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

/*
 * Tar entry header + contents parsing
 */

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

/*
 * Tar archive parsing
 */

fn filter_entries(entries: Vec<TarEntry>) -> Result<Vec<TarEntry>, &'static str> {
    /* Filter out empty entries */
    Ok(entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>())
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    chain!(i,
        entries: map_res!(many0!(parse_entry), filter_entries) ~
        eof,
        ||{
            entries
        }
    )
}

/*
 * Tests
 */

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<commit_msg>fix missing consumption<commit_after>use std::str::from_utf8;
use std::result::Result;
use nom::*;

/*
 * Core structs
 */

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     &'a str,
    pub mode:     &'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   &'a str,
    pub typeflag: TypeFlag,
    pub linkname: &'a str,
    pub ustar:    ExtraHeader<'a>
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    PaxInterexchangeFormat,
    PaxExtendedAttributes,
    VendorSpecific
}

#[derive(Debug,PartialEq,Eq)]
pub enum ExtraHeader<'a> {
    UStar(UStarHeader<'a>),
    Padding
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    &'a str,
    pub version:  &'a str,
    pub uname:    &'a str,
    pub gname:    &'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub extra:    UStarExtraHeader<'a>
}

#[derive(Debug,PartialEq,Eq)]
pub enum UStarExtraHeader<'a> {
    PosixUStar(PosixUStarHeader<'a>),
    Pax(PaxHeader<'a>)
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixUStarHeader<'a> {
    pub prefix: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PaxHeader<'a> {
    pub atime:      u64,
    pub ctime:      u64,
    pub offset:     u64,
    pub longnames:  &'a str,
    pub sparses:    Vec<Sparse>,
    pub isextended: bool,
    pub realsize:   u64,
}

#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub struct Sparse {
    pub offset:   u64,
    pub numbytes: u64
}

#[derive(Debug,PartialEq,Eq)]
pub struct Padding;

/*
 * Useful macros
 */

macro_rules! take_str_eat_garbage (
    ( $i:expr, $size:expr ) => (
        chain!($i,
            s: map_res!(take_until!("\0"), from_utf8) ~
            take!($size - s.len()),
            ||{
                s
            }
        )
    );
);

named!(parse_str4<&[u8], &str>, take_str_eat_garbage!(4));
named!(parse_str8<&[u8], &str>, take_str_eat_garbage!(8));
named!(parse_str32<&[u8], &str>, take_str_eat_garbage!(32));
named!(parse_str100<&[u8], &str>, take_str_eat_garbage!(100));
named!(parse_str155<&[u8], &str>, take_str_eat_garbage!(155));

macro_rules! take_until_expr_with_limit_consume(
  ($i:expr, $submac:ident!( $($args:tt)* ), $stop: expr, $limit: expr) => (
    {
      let mut begin = 0;
      let mut remaining = $i.len();
      let mut res = Vec::new();
      let mut cnt = 0;
      let mut err = false;
      let mut append = true;
      loop {
        match $submac!(&$i[begin..], $($args)*) {
          IResult::Done(i,o) => {
            if append {
              if $stop(o) {
                append = false;
              } else {
                res.push(o);
              }
            }
            begin += remaining - i.len();
            remaining = i.len();
            cnt = cnt + 1;
            if cnt == $limit {
              break
            }
          },
          IResult::Error(_)  => {
            err = true;
            break;
          },
          IResult::Incomplete(_) => {
            break;
          }
        }
      }
      if err {
        IResult::Error(Err::Position(ErrorCode::Count as u32,$i))
      } else if cnt == $limit {
        IResult::Done(&$i[begin..], res)
      } else {
        IResult::Incomplete(Needed::Unknown)
      }
    }
  );
  ($i:expr, $f:expr, $stop: expr, $limit: expr) => (
    take_until_expr_with_limit_consume!($i, call!($f), $stop, $limit);
  );
);

/*
 * Octal string parsing
 */

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;

    for c in s.chars() {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u *= 8;
        u += (c as u64) - ('0' as u64);
    }

    Ok(u)
}

fn parse_octal(i: &[u8], n: usize) -> IResult<&[u8], u64> {
    map_res!(i, take_str_eat_garbage!(n), octal_to_u64)
}

named!(parse_octal8<&[u8], u64>, apply!(parse_octal, 8));
named!(parse_octal12<&[u8], u64>, apply!(parse_octal, 12));

/*
 * TypeFlag parsing
 */

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::PaxInterexchangeFormat,
        'x' => TypeFlag::PaxExtendedAttributes,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::NormalFile
    }
}

fn bytes_to_type_flag(i: &[u8]) -> Result<TypeFlag, &'static str> {
    Ok(char_to_type_flag(i[0] as char))
}

named!(parse_type_flag<&[u8], TypeFlag>, map_res!(take!(1), bytes_to_type_flag));

/*
 * Sparse parsing
 */

fn parse_one_sparse(i: &[u8]) -> IResult<&[u8], Sparse> {
    chain!(i,
        offset:   parse_octal12 ~
        numbytes: parse_octal12,
        ||{
            Sparse {
                offset:   offset,
                numbytes: numbytes
            }
        }
    )
}

fn parse_sparses_with_limit(i: &[u8], limit: usize) -> IResult<&[u8], Vec<Sparse>> {
    take_until_expr_with_limit_consume!(i, parse_one_sparse, |s: Sparse| s.offset == 0 && s.numbytes == 0, limit)
}

fn add_to_vec(extra: Vec<Sparse>, sparses: &mut Vec<Sparse>) -> Result<&'static str, &'static str> {
    for sparse in &extra {
        sparses.push(*sparse);
    }
    Ok("")
}

fn parse_extra_sparses<'a>(i: &'a [u8], isextended: bool, sparses: &'a mut Vec<Sparse>) -> IResult<'a, &'a [u8], &'a mut Vec<Sparse>> {
    if isextended {
        chain!(i,
            map_res!(apply!(parse_sparses_with_limit, 21), apply!(add_to_vec, sparses)) ~
            extended:      parse_bool                                                   ~
            take!(7) /* padding to 512 */                                               ~
            extra_sparses: apply!(parse_extra_sparses, extended, sparses),
            ||{
                extra_sparses
            }
        )
    } else {
        IResult::Done(i, sparses)
    }
}

/*
 * Boolean parsing
 */

fn to_bool(i: &[u8]) -> Result<bool, &'static str> {
    Ok(i[0] != 0)
}

named!(parse_bool<&[u8], bool>, map_res!(take!(1), to_bool));

/*
 * UStar PAX extended parsing
 */

fn parse_ustar00_extra_pax(i: &[u8]) -> IResult<&[u8], PaxHeader> {
    chain!(i,
        atime:       parse_octal12                       ~
        ctime:       parse_octal12                       ~
        offset:      parse_octal12                       ~
        longnames:   parse_str4                          ~
        take!(1)                                         ~
        sparses:     apply!(parse_sparses_with_limit, 4) ~
        isextended:  parse_bool                          ~
        realsize:    parse_octal12                       ~
        take!(17), /* padding to 512 */
        ||{
            PaxHeader {
                atime:         atime,
                ctime:         ctime,
                offset:        offset,
                longnames:     longnames,
                sparses:       sparses,
                isextended:    isextended,
                realsize:      realsize,
            }
        }
    )
}

/*
 * UStar Posix parsing
 */

fn parse_ustar00_extra_posix(i: &[u8]) -> IResult<&[u8], UStarExtraHeader> {
    chain!(i,
        prefix: parse_str155 ~
        take!(12),
        ||{
            UStarExtraHeader::PosixUStar(PosixUStarHeader {
                prefix: prefix
            })
        }
    )
}

fn parse_ustar00_extra(i: &[u8], flag: TypeFlag) -> IResult<&[u8], UStarExtraHeader> {
    match flag {
        TypeFlag::PaxInterexchangeFormat => {
            chain!(i,
                header: parse_ustar00_extra_pax ~
                apply!(parse_extra_sparses, header.isextended, &mut header.sparses),
                ||{
                    UStarExtraHeader::Pax(header)
                }
            )
        },
        _ => parse_ustar00_extra_posix(i)
    }
}

fn parse_ustar00(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("00")             ~
        uname:    parse_str32  ~
        gname:    parse_str32  ~
        devmajor: parse_octal8 ~
        devminor: parse_octal8 ~
        extra:    apply!(parse_ustar00_extra, flag),
        ||{
            ExtraHeader::UStar(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                extra:    extra
            })
        }
    )
}

fn parse_ustar(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: apply!(parse_ustar00, flag),
        ||{
            ustar
        }
    )
}

/*
 * Posix tar archive header parsing
 */

fn parse_posix(i: &[u8]) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            ExtraHeader::Padding
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     parse_str100    ~
        mode:     parse_str8      ~
        uid:      parse_octal8    ~
        gid:      parse_octal8    ~
        size:     parse_octal12   ~
        mtime:    parse_octal12   ~
        chksum:   parse_str8      ~
        typeflag: parse_type_flag ~
        linkname: parse_str100    ~
        ustar:    alt!(apply!(parse_ustar, typeflag) | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: typeflag,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

/*
 * Contents parsing
 */

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

/*
 * Tar entry header + contents parsing
 */

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

/*
 * Tar archive parsing
 */

fn filter_entries(entries: Vec<TarEntry>) -> Result<Vec<TarEntry>, &'static str> {
    /* Filter out empty entries */
    Ok(entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>())
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    chain!(i,
        entries: map_res!(many0!(parse_entry), filter_entries) ~
        eof,
        ||{
            entries
        }
    )
}

/*
 * Tests
 */

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<|endoftext|>


<commit_before>use ast::*;
use scanner::{Lexeme, Position, Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub enum ParseError {
    UnexpectedEndOfFile,
    UnexpectedToken(Lexeme, Position),
    MissingSubexpression(Lexeme, Position),
    MissingClosingParen(Lexeme, Position),
    MissingSemicolon(Lexeme, Position),
}

pub fn parse(tokens: &[TokenWithContext]) -> Result<Vec<Statement>, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    let mut statements = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = parse_statement(&mut iter) {
        match result {
            Ok(statement) => {
                statements.push(statement);
            }
            Err(error) => {
                errors.push(error);
                synchronise(&mut iter);
            }
        }
    }
    if errors.is_empty() {
        Ok(statements)
    } else {
        Err(errors)
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum PositionInConstruct {
        Start,
        Body,
        End,
    };
    fn classify(token: &Token) -> PositionInConstruct {
        match *token {
            Token::Semicolon => PositionInConstruct::End,
            Token::Class | Token::Fun | Token::Var | Token::For | Token::If | Token::While |
            Token::Print | Token::Return => PositionInConstruct::Start,
            _ => PositionInConstruct::Body,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            PositionInConstruct::Start => {
                break;
            }
            PositionInConstruct::Body => {
                let _ = tokens.next();
            }
            PositionInConstruct::End => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let result = match tokens.peek().map(|t| &t.token) {
        Some(&Token::Print) => {
            let _ = tokens.next();
            parse_print_statement(tokens)
        }
        Some(_) => parse_expression_statement(tokens),
        None => None,
    };
    if let Some(Ok(statement)) = result {
        match tokens.peek(){
            Some(&&TokenWithContext{token: Token::Semicolon, lexeme: _, position: _}) => {
                let _ = tokens.next();
                Some(Ok(statement))
            },
            Some(&&TokenWithContext{token: _, ref lexeme, ref position}) => {
                Some(Err(ParseError::MissingSemicolon(lexeme.clone(), position.clone())))
            },
            None => Some(Err(ParseError::UnexpectedEndOfFile)),
        }
    } else {
        result
    }
}

fn parse_print_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Print(e)))
}

fn parse_expression_statement<'a, I>(tokens: &mut Peekable<I>)
                                     -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Expression(e)))
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<BinaryOperator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_subexpression(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::BangEqual => Some(BinaryOperator::NotEqual),
            Token::EqualEqual => Some(BinaryOperator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Greater => Some(BinaryOperator::Greater),
            Token::GreaterEqual => Some(BinaryOperator::GreaterEqual),
            Token::Less => Some(BinaryOperator::Less),
            Token::LessEqual => Some(BinaryOperator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Minus => Some(BinaryOperator::Minus),
            Token::Plus => Some(BinaryOperator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Slash => Some(BinaryOperator::Slash),
            Token::Star => Some(BinaryOperator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<UnaryOperator> {
        match *token {
            Token::Minus => Some(UnaryOperator::Minus),
            Token::Bang => Some(UnaryOperator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_unary(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr = if let Some(result) = parse_expression(tokens) {
                    try_wrap_err!(result)
                } else {
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::MissingClosingParen(token.lexeme.clone(),
                                                                            token.position)));
                        }

                    }
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                }
            }
            _ => {
                return Some(Err(ParseError::UnexpectedToken(primary_token.lexeme.clone(),
                                                            primary_token.position)));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let string = String::from("123");
        let (tokens, _) = scan(&string);
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!(&string, &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2");
        assert!(parse(&tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)");
        assert!(parse(&tokens).is_err());
    }
}
<commit_msg>cargo fmt and parse_declaration<commit_after>use ast::*;
use scanner::{Lexeme, Position, Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub enum ParseError {
    UnexpectedEndOfFile,
    UnexpectedToken(Lexeme, Position),
    MissingSubexpression(Lexeme, Position),
    MissingClosingParen(Lexeme, Position),
    MissingSemicolon(Lexeme, Position),
}

pub fn parse(tokens: &[TokenWithContext]) -> Result<Vec<Statement>, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    let mut statements = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = parse_statement(&mut iter) {
        match result {
            Ok(statement) => {
                statements.push(statement);
            }
            Err(error) => {
                errors.push(error);
                synchronise(&mut iter);
            }
        }
    }
    if errors.is_empty() {
        Ok(statements)
    } else {
        Err(errors)
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum PositionInConstruct {
        Start,
        Body,
        End,
    };
    fn classify(token: &Token) -> PositionInConstruct {
        match *token {
            Token::Semicolon => PositionInConstruct::End,
            Token::Class | Token::Fun | Token::Var | Token::For | Token::If | Token::While |
            Token::Print | Token::Return => PositionInConstruct::Start,
            _ => PositionInConstruct::Body,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            PositionInConstruct::Start => {
                break;
            }
            PositionInConstruct::Body => {
                let _ = tokens.next();
            }
            PositionInConstruct::End => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_declaration<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    unimplemented!();
}

fn parse_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let result = match tokens.peek().map(|t| &t.token) {
        Some(&Token::Print) => {
            let _ = tokens.next();
            parse_print_statement(tokens)
        }
        Some(_) => parse_expression_statement(tokens),
        None => None,
    };
    if let Some(Ok(statement)) = result {
        match tokens.peek() {
            Some(&&TokenWithContext { token: Token::Semicolon, lexeme: _, position: _ }) => {
                let _ = tokens.next();
                Some(Ok(statement))
            }
            Some(&&TokenWithContext { token: _, ref lexeme, ref position }) => {
                Some(Err(ParseError::MissingSemicolon(lexeme.clone(), position.clone())))
            }
            None => Some(Err(ParseError::UnexpectedEndOfFile)),
        }
    } else {
        result
    }
}

fn parse_print_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Print(e)))
}

fn parse_expression_statement<'a, I>(tokens: &mut Peekable<I>)
                                     -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Expression(e)))
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<BinaryOperator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_subexpression(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::BangEqual => Some(BinaryOperator::NotEqual),
            Token::EqualEqual => Some(BinaryOperator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Greater => Some(BinaryOperator::Greater),
            Token::GreaterEqual => Some(BinaryOperator::GreaterEqual),
            Token::Less => Some(BinaryOperator::Less),
            Token::LessEqual => Some(BinaryOperator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Minus => Some(BinaryOperator::Minus),
            Token::Plus => Some(BinaryOperator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Slash => Some(BinaryOperator::Slash),
            Token::Star => Some(BinaryOperator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<UnaryOperator> {
        match *token {
            Token::Minus => Some(UnaryOperator::Minus),
            Token::Bang => Some(UnaryOperator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_unary(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr = if let Some(result) = parse_expression(tokens) {
                    try_wrap_err!(result)
                } else {
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::MissingClosingParen(token.lexeme.clone(),
                                                                            token.position)));
                        }

                    }
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                }
            }
            _ => {
                return Some(Err(ParseError::UnexpectedToken(primary_token.lexeme.clone(),
                                                            primary_token.position)));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let string = String::from("123");
        let (tokens, _) = scan(&string);
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!(&string, &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2");
        assert!(parse(&tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)");
        assert!(parse(&tokens).is_err());
    }
}
<|endoftext|>


<commit_before>use std::io::{BufRead, BufReader};
use std::str;

use types::{make_extension_error, ErrorKind, RedisError, RedisResult, Value};

use futures::{Async, Future, Poll};
use tokio_io::AsyncRead;

use combine;
use combine::byte::{byte, crlf, newline};
use combine::combinator::{any_partial_state, AnyPartialState};
#[allow(unused_imports)] // See https://github.com/rust-lang/rust/issues/43970
use combine::error::StreamError;
use combine::parser::choice::choice;
use combine::range::{recognize, take, take_until_range};
use combine::stream::{RangeStream, StreamErrorFor};

struct ResultExtend<T, E>(Result<T, E>);

impl<T, E> Default for ResultExtend<T, E>
where
    T: Default,
{
    fn default() -> Self {
        ResultExtend(Ok(T::default()))
    }
}

impl<T, U, E> Extend<Result<U, E>> for ResultExtend<T, E>
where
    T: Extend<U>,
{
    fn extend<I>(&mut self, iter: I)
    where
        I: IntoIterator<Item = Result<U, E>>,
    {
        let mut returned_err = None;
        match self.0 {
            Ok(ref mut elems) => elems.extend(iter.into_iter().scan((), |_, item| match item {
                Ok(item) => Some(item),
                Err(err) => {
                    returned_err = Some(err);
                    None
                }
            })),
            Err(_) => (),
        }
        if let Some(err) = returned_err {
            self.0 = Err(err);
        }
    }
}

parser!{
    type PartialState = AnyPartialState;
    fn value['a, I]()(I) -> RedisResult<Value>
        where [I: RangeStream<Item = u8, Range = &'a [u8]> ]
    {
        let end_of_line: fn () -> _ = || crlf().or(newline());
        let line = || recognize(take_until_range(&b"\r\n"[..]).with(end_of_line()))
            .and_then(|line: &[u8]| {
                str::from_utf8(line)
                    .map(|line| line.trim_right_matches(|c: char| c == '\r' || c == '\n'))
                    .map_err(StreamErrorFor::<I>::other)
            });

        let status = || line().map(|line| {
            if line == "OK" {
                Value::Okay
            } else {
                Value::Status(line.into())
            }
        });

        let int = || line().and_then(|line| {
            match line.trim().parse::<i64>() {
                Err(_) => Err(StreamErrorFor::<I>::message_static_message("Expected integer, got garbage")),
                Ok(value) => Ok(value),
            }
        });

        let data = || int().then_partial(move |size| {
            if *size < 0 {
                combine::value(Value::Nil).left()
            } else {
                take(*size as usize)
                    .map(|bs: &[u8]| Value::Data(bs.to_vec()))
                    .skip(end_of_line())
                    .right()
            }
        });

        let bulk = || {
            int().then_partial(|&mut length| {
                if length < 0 {
                    combine::value(Value::Nil).map(Ok).left()
                } else {
                    let length = length as usize;
                    combine::count_min_max(length, length, value()).map(|result: ResultExtend<_, _>| {
                        result.0.map(Value::Bulk)
                    }).right()
                }
            })
        };

        let error = || {
            line()
                .map(|line: &str| {
                    let desc = "An error was signalled by the server";
                    let mut pieces = line.splitn(2, ' ');
                    let kind = match pieces.next().unwrap() {
                        "ERR" => ErrorKind::ResponseError,
                        "EXECABORT" => ErrorKind::ExecAbortError,
                        "LOADING" => ErrorKind::BusyLoadingError,
                        "NOSCRIPT" => ErrorKind::NoScriptError,
                        code => {
                            return make_extension_error(code, pieces.next())
                        }
                    };
                    match pieces.next() {
                        Some(detail) => RedisError::from((kind, desc, detail.to_string())),
                        None => RedisError::from((kind, desc)),
                    }
                })
        };

        any_partial_state(choice((
           byte(b'+').with(status().map(Ok)),
           byte(b':').with(int().map(Value::Int).map(Ok)),
           byte(b'$').with(data().map(Ok)),
           byte(b'*').with(bulk()),
           byte(b'-').with(error().map(Err))
        )))
    }
}

pub struct ValueFuture<R> {
    reader: Option<R>,
    state: AnyPartialState,
    // Intermediate storage for data we know that we need to parse a value but we haven't been able
    // to parse completely yet
    remaining: Vec<u8>,
}

impl<R> Future for ValueFuture<R>
where
    R: BufRead,
{
    type Item = (R, Value);
    type Error = RedisError;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        loop {
            assert!(
                self.reader.is_some(),
                "ValueFuture: poll called on completed future"
            );
            let remaining_data = self.remaining.len();

            let (opt, mut removed) = {
                let buffer = try_nb!(self.reader.as_mut().unwrap().fill_buf());
                if buffer.len() == 0 {
                    fail!((ErrorKind::ResponseError, "Could not read enough bytes"))
                }
                let buffer = if !self.remaining.is_empty() {
                    self.remaining.extend(buffer);
                    &self.remaining[..]
                } else {
                    buffer
                };
                let stream = combine::easy::Stream(combine::stream::PartialStream(buffer));
                match combine::stream::decode(value(), stream, &mut self.state) {
                    Ok(x) => x,
                    Err(err) => {
                        let err = err.map_position(|pos| pos.translate_position(buffer))
                            .map_range(|range| format!("{:?}", range))
                            .to_string();
                        return Err(RedisError::from((
                            ErrorKind::ResponseError,
                            "parse error",
                            err,
                        )));
                    }
                }
            };

            if !self.remaining.is_empty() {
                // Remove the data we have parsed and adjust `removed` to be the amount of data we
                // consumed from `self.reader`
                self.remaining.drain(..removed);
                if removed >= remaining_data {
                    removed = removed - remaining_data;
                } else {
                    removed = 0;
                }
            }

            match opt {
                Some(value) => {
                    self.reader.as_mut().unwrap().consume(removed);
                    let reader = self.reader.take().unwrap();
                    return Ok(Async::Ready((reader, value?)));
                }
                None => {
                    // We have not enough data to produce a Value but we know that all the data of
                    // the current buffer are necessary. Consume all the buffered data to ensure
                    // that the next iteration actually reads more data.
                    let buffer_len = {
                        let buffer = try!(self.reader.as_mut().unwrap().fill_buf());
                        if remaining_data == 0 {
                            self.remaining.extend(&buffer[removed..]);
                        }
                        buffer.len()
                    };
                    self.reader.as_mut().unwrap().consume(buffer_len);
                }
            }
        }
    }
}

pub fn parse_async<R>(reader: R) -> ValueFuture<R>
where
    R: AsyncRead + BufRead,
{
    ValueFuture {
        reader: Some(reader),
        state: Default::default(),
        remaining: Vec::new(),
    }
}

/// The internal redis response parser.
pub struct Parser<T> {
    reader: T,
}

/// The parser can be used to parse redis responses into values.  Generally
/// you normally do not use this directly as it's already done for you by
/// the client but in some more complex situations it might be useful to be
/// able to parse the redis responses.
impl<'a, T: BufRead> Parser<T> {
    /// Creates a new parser that parses the data behind the reader.  More
    /// than one value can be behind the reader in which case the parser can
    /// be invoked multiple times.  In other words: the stream does not have
    /// to be terminated.
    pub fn new(reader: T) -> Parser<T> {
        Parser { reader: reader }
    }

    // public api

    pub fn parse_value(&mut self) -> RedisResult<Value> {
        let mut parser = ValueFuture {
            reader: Some(&mut self.reader),
            state: AnyPartialState::default(),
            remaining: Vec::new(),
        };
        match parser.poll()? {
            Async::NotReady => panic!("Blocking read received WouldBlock error"),
            Async::Ready((_, value)) => return Ok(value),
        }
    }
}

/// Parses bytes into a redis value.
///
/// This is the most straightforward way to parse something into a low
/// level redis value instead of having to use a whole parser.
pub fn parse_redis_value(bytes: &[u8]) -> RedisResult<Value> {
    let mut parser = Parser::new(BufReader::new(bytes));
    parser.parse_value()
}
<commit_msg>Return WouldBlock instead of panicking for synchronous reads on an async Read instance<commit_after>use std::io::{self, BufRead, BufReader};
use std::str;

use types::{make_extension_error, ErrorKind, RedisError, RedisResult, Value};

use futures::{Async, Future, Poll};
use tokio_io::AsyncRead;

use combine;
use combine::byte::{byte, crlf, newline};
use combine::combinator::{any_partial_state, AnyPartialState};
#[allow(unused_imports)] // See https://github.com/rust-lang/rust/issues/43970
use combine::error::StreamError;
use combine::parser::choice::choice;
use combine::range::{recognize, take, take_until_range};
use combine::stream::{RangeStream, StreamErrorFor};

struct ResultExtend<T, E>(Result<T, E>);

impl<T, E> Default for ResultExtend<T, E>
where
    T: Default,
{
    fn default() -> Self {
        ResultExtend(Ok(T::default()))
    }
}

impl<T, U, E> Extend<Result<U, E>> for ResultExtend<T, E>
where
    T: Extend<U>,
{
    fn extend<I>(&mut self, iter: I)
    where
        I: IntoIterator<Item = Result<U, E>>,
    {
        let mut returned_err = None;
        match self.0 {
            Ok(ref mut elems) => elems.extend(iter.into_iter().scan((), |_, item| match item {
                Ok(item) => Some(item),
                Err(err) => {
                    returned_err = Some(err);
                    None
                }
            })),
            Err(_) => (),
        }
        if let Some(err) = returned_err {
            self.0 = Err(err);
        }
    }
}

parser!{
    type PartialState = AnyPartialState;
    fn value['a, I]()(I) -> RedisResult<Value>
        where [I: RangeStream<Item = u8, Range = &'a [u8]> ]
    {
        let end_of_line: fn () -> _ = || crlf().or(newline());
        let line = || recognize(take_until_range(&b"\r\n"[..]).with(end_of_line()))
            .and_then(|line: &[u8]| {
                str::from_utf8(line)
                    .map(|line| line.trim_right_matches(|c: char| c == '\r' || c == '\n'))
                    .map_err(StreamErrorFor::<I>::other)
            });

        let status = || line().map(|line| {
            if line == "OK" {
                Value::Okay
            } else {
                Value::Status(line.into())
            }
        });

        let int = || line().and_then(|line| {
            match line.trim().parse::<i64>() {
                Err(_) => Err(StreamErrorFor::<I>::message_static_message("Expected integer, got garbage")),
                Ok(value) => Ok(value),
            }
        });

        let data = || int().then_partial(move |size| {
            if *size < 0 {
                combine::value(Value::Nil).left()
            } else {
                take(*size as usize)
                    .map(|bs: &[u8]| Value::Data(bs.to_vec()))
                    .skip(end_of_line())
                    .right()
            }
        });

        let bulk = || {
            int().then_partial(|&mut length| {
                if length < 0 {
                    combine::value(Value::Nil).map(Ok).left()
                } else {
                    let length = length as usize;
                    combine::count_min_max(length, length, value()).map(|result: ResultExtend<_, _>| {
                        result.0.map(Value::Bulk)
                    }).right()
                }
            })
        };

        let error = || {
            line()
                .map(|line: &str| {
                    let desc = "An error was signalled by the server";
                    let mut pieces = line.splitn(2, ' ');
                    let kind = match pieces.next().unwrap() {
                        "ERR" => ErrorKind::ResponseError,
                        "EXECABORT" => ErrorKind::ExecAbortError,
                        "LOADING" => ErrorKind::BusyLoadingError,
                        "NOSCRIPT" => ErrorKind::NoScriptError,
                        code => {
                            return make_extension_error(code, pieces.next())
                        }
                    };
                    match pieces.next() {
                        Some(detail) => RedisError::from((kind, desc, detail.to_string())),
                        None => RedisError::from((kind, desc)),
                    }
                })
        };

        any_partial_state(choice((
           byte(b'+').with(status().map(Ok)),
           byte(b':').with(int().map(Value::Int).map(Ok)),
           byte(b'$').with(data().map(Ok)),
           byte(b'*').with(bulk()),
           byte(b'-').with(error().map(Err))
        )))
    }
}

pub struct ValueFuture<R> {
    reader: Option<R>,
    state: AnyPartialState,
    // Intermediate storage for data we know that we need to parse a value but we haven't been able
    // to parse completely yet
    remaining: Vec<u8>,
}

impl<R> Future for ValueFuture<R>
where
    R: BufRead,
{
    type Item = (R, Value);
    type Error = RedisError;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        loop {
            assert!(
                self.reader.is_some(),
                "ValueFuture: poll called on completed future"
            );
            let remaining_data = self.remaining.len();

            let (opt, mut removed) = {
                let buffer = try_nb!(self.reader.as_mut().unwrap().fill_buf());
                if buffer.len() == 0 {
                    fail!((ErrorKind::ResponseError, "Could not read enough bytes"))
                }
                let buffer = if !self.remaining.is_empty() {
                    self.remaining.extend(buffer);
                    &self.remaining[..]
                } else {
                    buffer
                };
                let stream = combine::easy::Stream(combine::stream::PartialStream(buffer));
                match combine::stream::decode(value(), stream, &mut self.state) {
                    Ok(x) => x,
                    Err(err) => {
                        let err = err.map_position(|pos| pos.translate_position(buffer))
                            .map_range(|range| format!("{:?}", range))
                            .to_string();
                        return Err(RedisError::from((
                            ErrorKind::ResponseError,
                            "parse error",
                            err,
                        )));
                    }
                }
            };

            if !self.remaining.is_empty() {
                // Remove the data we have parsed and adjust `removed` to be the amount of data we
                // consumed from `self.reader`
                self.remaining.drain(..removed);
                if removed >= remaining_data {
                    removed = removed - remaining_data;
                } else {
                    removed = 0;
                }
            }

            match opt {
                Some(value) => {
                    self.reader.as_mut().unwrap().consume(removed);
                    let reader = self.reader.take().unwrap();
                    return Ok(Async::Ready((reader, value?)));
                }
                None => {
                    // We have not enough data to produce a Value but we know that all the data of
                    // the current buffer are necessary. Consume all the buffered data to ensure
                    // that the next iteration actually reads more data.
                    let buffer_len = {
                        let buffer = try!(self.reader.as_mut().unwrap().fill_buf());
                        if remaining_data == 0 {
                            self.remaining.extend(&buffer[removed..]);
                        }
                        buffer.len()
                    };
                    self.reader.as_mut().unwrap().consume(buffer_len);
                }
            }
        }
    }
}

pub fn parse_async<R>(reader: R) -> ValueFuture<R>
where
    R: AsyncRead + BufRead,
{
    ValueFuture {
        reader: Some(reader),
        state: Default::default(),
        remaining: Vec::new(),
    }
}

/// The internal redis response parser.
pub struct Parser<T> {
    reader: T,
}

/// The parser can be used to parse redis responses into values.  Generally
/// you normally do not use this directly as it's already done for you by
/// the client but in some more complex situations it might be useful to be
/// able to parse the redis responses.
impl<'a, T: BufRead> Parser<T> {
    /// Creates a new parser that parses the data behind the reader.  More
    /// than one value can be behind the reader in which case the parser can
    /// be invoked multiple times.  In other words: the stream does not have
    /// to be terminated.
    pub fn new(reader: T) -> Parser<T> {
        Parser { reader: reader }
    }

    // public api

    pub fn parse_value(&mut self) -> RedisResult<Value> {
        let mut parser = ValueFuture {
            reader: Some(&mut self.reader),
            state: AnyPartialState::default(),
            remaining: Vec::new(),
        };
        match parser.poll()? {
            Async::NotReady => Err(io::Error::from(io::ErrorKind::WouldBlock).into()),
            Async::Ready((_, value)) => Ok(value),
        }
    }
}

/// Parses bytes into a redis value.
///
/// This is the most straightforward way to parse something into a low
/// level redis value instead of having to use a whole parser.
pub fn parse_redis_value(bytes: &[u8]) -> RedisResult<Value> {
    let mut parser = Parser::new(BufReader::new(bytes));
    parser.parse_value()
}
<|endoftext|>