<commit_before>const HEAP_START: usize = 0xC0_00_00_00;
// const HEAP_END: usize = 0xD0_00_00_00;

use core::cmp;
use core::ptr::null_mut;
use core::option::Option::{self, Some, None};
use core::atomic::{AtomicBool, ATOMIC_BOOL_INIT, Ordering};
use core::intrinsics::copy_nonoverlapping;

use mutex::Mutex;

// This mutex does not guard anything.
// However, all top-level functions that manipulate the heap must take it
static HEAP_MUTEX: Mutex<()> = mutex!(());

const MAX_ORDER: u8 = 22;
const BLOCK_BITS: u8 = 5;

const USED_MAGIC: [u8; 4] = [0; 4];
const FREE_MAGIC: [u8; 4] = ['F' as u8, 'R' as u8, 'E' as u8, 'E' as u8];

#[repr(packed)]
struct Block {
    pub order: u8,
    pub magic: [u8; 4],
    prev: *mut Block,
    next: *mut Block,
}

// Guarded by the big fat heap lock
static mut free_lists: [*mut Block; (MAX_ORDER + 1) as usize] =
    [0 as *mut Block; (MAX_ORDER + 1) as usize];

static INITIALIZED: AtomicBool = ATOMIC_BOOL_INIT;

unsafe fn to_block<'r>(ptr: *mut u8) -> &'r mut Block {
    &mut *(ptr.offset(-1) as *mut Block)
}

impl Block {
    fn to_pointer(&mut self) -> *mut u8 {
        &mut self.magic[0] as *mut u8
    }

    fn to_const_pointer(&self) -> *const u8 {
        &self.magic[0] as *const u8
    }

    fn buddy<'r>(&self) -> &'r mut Block {
        unsafe {
            let addr = self.to_const_pointer() as usize;
            let buddy_addr = addr ^ ((1 << (self.order + BLOCK_BITS)) as usize);
            to_block(buddy_addr as *mut u8)
        }
    }

    unsafe fn free_insert(&mut self) {
        let mut prev = null_mut();
        let mut next = free_lists[self.order as usize];
        let self_ptr = self as *mut Block;
        let addr = self_ptr as usize;
        while !next.is_null() && (next as usize) < addr {
            prev = next;
            next = (*next).next;
        }
        self.prev = prev;
        self.next = next;
        if !next.is_null() {
            (*next).prev = self_ptr;
        }
        if prev.is_null() {
            free_lists[self.order as usize] = self_ptr;
        }
        else {
            (*prev).next = self_ptr
        }
    }

    unsafe fn free_remove(&mut self) {
        if !self.prev.is_null() {
            (*self.prev).next = self.next;
        }
        if !self.next.is_null() {
            (*self.next).prev = self.prev;
        }
        if free_lists[self.order as usize] == self as *mut Block {
            free_lists[self.order as usize] = self.next;
        }
    }

    fn make_used(&mut self) {
        self.magic = USED_MAGIC;
        unsafe { self.free_remove() ;}
        self.prev = null_mut();
        self.next = null_mut();
    }

    fn free(&mut self) {
        self.magic = FREE_MAGIC;
        unsafe { self.free_insert(); }
    }

    fn split<'r>(&mut self) -> &'r mut Block {
        self.make_used();
        self.order -= 1;
        self.free();
        let ret = self.buddy();
        ret.order = self.order;
        ret.free();
        ret
    }

    fn maybe_merge<'r>(&mut self){
        let buddy = self.buddy();
        if self.magic != FREE_MAGIC || buddy.magic != FREE_MAGIC || self.order == MAX_ORDER {
            return;
        }
        let is_self_smaller = (self as *mut Block as usize) < (buddy as *mut Block as usize);
        let (smaller, larger) = if is_self_smaller { (self, buddy) } else { (buddy, self) };
        if smaller.next != larger || larger.prev != smaller {
            return // Not actually free blocks, unless weird things happened
        }
        smaller.make_used();
        larger.make_used();
        larger.order = 0;
        smaller.order += 1;
        smaller.free();
        smaller.maybe_merge(); // Get as much merging in as we can
    }
}

unsafe fn init() {
    if INITIALIZED.compare_and_swap(false, true, Ordering::SeqCst) == false {
        let top_addr = HEAP_START - 1;
        let top = &mut *(top_addr as *mut u8 as *mut Block);
        top.order = MAX_ORDER;
        top.free();

        let buddy = top.buddy();
        buddy.order = MAX_ORDER;
        buddy.free();
    }
}

fn order_of(size: usize) -> u8 {
    let rounded = size.next_power_of_two();
    let log = rounded.trailing_zeros() as u8;
    cmp::max(log, BLOCK_BITS) - BLOCK_BITS
}

#[inline]
fn block_size(order: u8) -> usize {
    (1 << (order + BLOCK_BITS)) - 1
}

fn find_block_of_order<'r>(order: u8) -> Option<&'r mut Block> {
    if order > MAX_ORDER {
        return None
    }
    unsafe {
        if !INITIALIZED.load(Ordering::Relaxed) {
            init();
        }
        let maybe_head = free_lists[order as usize];
        if !maybe_head.is_null() {
            Some(&mut *maybe_head)
        }
        else {
            match find_block_of_order(order + 1) {
                Some(block) => {
                    block.split();
                    Some(block)
                }
                None => None
            }
        }
    }
}

#[inline]
fn to_order(size: usize, align: usize) -> u8 {
    cmp::max(order_of(size), order_of(align))
}

#[no_mangle]
pub unsafe extern fn rust_allocate(size: usize, align: usize) -> *mut u8 {
    let _lock = HEAP_MUTEX.lock();
    let order = to_order(size + 1, align);
    match find_block_of_order(order) {
        Some(block) => {
            block.make_used();
            block.to_pointer()
        }
        None => null_mut()
    }
}

#[inline]
pub unsafe fn malloc(size: usize) -> *mut u8 {
    rust_allocate(size, 1)
}

#[inline]
#[no_mangle]
pub unsafe extern fn rust_deallocate(ptr: *mut u8,
                                     _old_size: usize, _align: usize) {
    free(ptr)
}

pub unsafe fn free(ptr: *mut u8) {
    let _lock = HEAP_MUTEX.lock();
    let block = to_block(ptr);
    block.free();
    block.maybe_merge();
}

#[inline]
pub unsafe fn realloc(ptr: *mut u8, size: usize) -> *mut u8 {
    rust_reallocate(ptr, 1, size, 1)
}

#[no_mangle]
pub unsafe extern fn rust_reallocate(ptr: *mut u8, _old_size: usize,
                                     size: usize, align: usize) -> *mut u8 {
    let _lock = HEAP_MUTEX.lock();
    let block = to_block(ptr);
    let new_order = to_order(size + 1, align);
    if block.order >= new_order {
        ptr
    }
    else {
        match find_block_of_order(new_order) {
            Some(new_block) => {
                new_block.make_used();
                let ret = new_block.to_pointer();
                copy_nonoverlapping(ptr, ret, block_size(block.order));
                block.free();
                block.maybe_merge();
                ret
            }
            None => null_mut()
        }
    }
}

#[no_mangle]
pub unsafe extern fn rust_reallocate_inplace(ptr: *mut u8, old_size: usize,
                                             size: usize, align: usize) -> usize {
    let _lock = HEAP_MUTEX.lock();
    let block = to_block(ptr);
    let new_order = to_order(size + 1, align);
    if block.order >= new_order {
        block_size(block.order)
    }
    else {
        rust_usable_size(old_size, align)
    }
}

#[no_mangle]
pub extern fn rust_usable_size(size: usize, align: usize) -> usize {
    let order = to_order(size + 1, align);
    block_size(order)
}

#[no_mangle]
pub extern fn rust_stats_print() {}
<commit_msg>Make find_block_of_order a bit more functional<commit_after>const HEAP_START: usize = 0xC0_00_00_00;
// const HEAP_END: usize = 0xD0_00_00_00;

use core::cmp;
use core::ptr::null_mut;
use core::option::Option::{self, Some, None};
use core::atomic::{AtomicBool, ATOMIC_BOOL_INIT, Ordering};
use core::intrinsics::copy_nonoverlapping;

use mutex::Mutex;

// This mutex does not guard anything.
// However, all top-level functions that manipulate the heap must take it
static HEAP_MUTEX: Mutex<()> = mutex!(());

const MAX_ORDER: u8 = 22;
const BLOCK_BITS: u8 = 5;

const USED_MAGIC: [u8; 4] = [0; 4];
const FREE_MAGIC: [u8; 4] = ['F' as u8, 'R' as u8, 'E' as u8, 'E' as u8];

#[repr(packed)]
struct Block {
    pub order: u8,
    pub magic: [u8; 4],
    prev: *mut Block,
    next: *mut Block,
}

// Guarded by the big fat heap lock
static mut free_lists: [*mut Block; (MAX_ORDER + 1) as usize] =
    [0 as *mut Block; (MAX_ORDER + 1) as usize];

static INITIALIZED: AtomicBool = ATOMIC_BOOL_INIT;

unsafe fn to_block<'r>(ptr: *mut u8) -> &'r mut Block {
    &mut *(ptr.offset(-1) as *mut Block)
}

impl Block {
    fn to_pointer(&mut self) -> *mut u8 {
        &mut self.magic[0] as *mut u8
    }

    fn to_const_pointer(&self) -> *const u8 {
        &self.magic[0] as *const u8
    }

    fn buddy<'r>(&self) -> &'r mut Block {
        unsafe {
            let addr = self.to_const_pointer() as usize;
            let buddy_addr = addr ^ ((1 << (self.order + BLOCK_BITS)) as usize);
            to_block(buddy_addr as *mut u8)
        }
    }

    unsafe fn free_insert(&mut self) {
        let mut prev = null_mut();
        let mut next = free_lists[self.order as usize];
        let self_ptr = self as *mut Block;
        let addr = self_ptr as usize;
        while !next.is_null() && (next as usize) < addr {
            prev = next;
            next = (*next).next;
        }
        self.prev = prev;
        self.next = next;
        if !next.is_null() {
            (*next).prev = self_ptr;
        }
        if prev.is_null() {
            free_lists[self.order as usize] = self_ptr;
        }
        else {
            (*prev).next = self_ptr
        }
    }

    unsafe fn free_remove(&mut self) {
        if !self.prev.is_null() {
            (*self.prev).next = self.next;
        }
        if !self.next.is_null() {
            (*self.next).prev = self.prev;
        }
        if free_lists[self.order as usize] == self as *mut Block {
            free_lists[self.order as usize] = self.next;
        }
    }

    fn make_used(&mut self) {
        self.magic = USED_MAGIC;
        unsafe { self.free_remove() ;}
        self.prev = null_mut();
        self.next = null_mut();
    }

    fn free(&mut self) {
        self.magic = FREE_MAGIC;
        unsafe { self.free_insert(); }
    }

    fn split<'r>(&mut self) -> &'r mut Block {
        self.make_used();
        self.order -= 1;
        self.free();
        let ret = self.buddy();
        ret.order = self.order;
        ret.free();
        ret
    }

    fn maybe_merge<'r>(&mut self){
        let buddy = self.buddy();
        if self.magic != FREE_MAGIC || buddy.magic != FREE_MAGIC || self.order == MAX_ORDER {
            return;
        }
        let is_self_smaller = (self as *mut Block as usize) < (buddy as *mut Block as usize);
        let (smaller, larger) = if is_self_smaller { (self, buddy) } else { (buddy, self) };
        if smaller.next != larger || larger.prev != smaller {
            return // Not actually free blocks, unless weird things happened
        }
        smaller.make_used();
        larger.make_used();
        larger.order = 0;
        smaller.order += 1;
        smaller.free();
        smaller.maybe_merge(); // Get as much merging in as we can
    }
}

unsafe fn init() {
    if INITIALIZED.compare_and_swap(false, true, Ordering::SeqCst) == false {
        let top_addr = HEAP_START - 1;
        let top = &mut *(top_addr as *mut u8 as *mut Block);
        top.order = MAX_ORDER;
        top.free();

        let buddy = top.buddy();
        buddy.order = MAX_ORDER;
        buddy.free();
    }
}

fn order_of(size: usize) -> u8 {
    let rounded = size.next_power_of_two();
    let log = rounded.trailing_zeros() as u8;
    cmp::max(log, BLOCK_BITS) - BLOCK_BITS
}

#[inline]
fn block_size(order: u8) -> usize {
    (1 << (order + BLOCK_BITS)) - 1
}

fn find_block_of_order<'r>(order: u8) -> Option<&'r mut Block> {
    if order > MAX_ORDER {
        return None
    }
    unsafe {
        if !INITIALIZED.load(Ordering::Relaxed) {
            init();
        }
        let maybe_head = free_lists[order as usize];
        if !maybe_head.is_null() {
            Some(&mut *maybe_head)
        }
        else {
            find_block_of_order(order + 1).map(|block| block.split())
        }
    }
}

#[inline]
fn to_order(size: usize, align: usize) -> u8 {
    cmp::max(order_of(size), order_of(align))
}

#[no_mangle]
pub unsafe extern fn rust_allocate(size: usize, align: usize) -> *mut u8 {
    let _lock = HEAP_MUTEX.lock();
    let order = to_order(size + 1, align);
    match find_block_of_order(order) {
        Some(block) => {
            block.make_used();
            block.to_pointer()
        }
        None => null_mut()
    }
}

#[inline]
pub unsafe fn malloc(size: usize) -> *mut u8 {
    rust_allocate(size, 1)
}

#[inline]
#[no_mangle]
pub unsafe extern fn rust_deallocate(ptr: *mut u8,
                                     _old_size: usize, _align: usize) {
    free(ptr)
}

pub unsafe fn free(ptr: *mut u8) {
    let _lock = HEAP_MUTEX.lock();
    let block = to_block(ptr);
    block.free();
    block.maybe_merge();
}

#[inline]
pub unsafe fn realloc(ptr: *mut u8, size: usize) -> *mut u8 {
    rust_reallocate(ptr, 1, size, 1)
}

#[no_mangle]
pub unsafe extern fn rust_reallocate(ptr: *mut u8, _old_size: usize,
                                     size: usize, align: usize) -> *mut u8 {
    let _lock = HEAP_MUTEX.lock();
    let block = to_block(ptr);
    let new_order = to_order(size + 1, align);
    if block.order >= new_order {
        ptr
    }
    else {
        match find_block_of_order(new_order) {
            Some(new_block) => {
                new_block.make_used();
                let ret = new_block.to_pointer();
                copy_nonoverlapping(ptr, ret, block_size(block.order));
                block.free();
                block.maybe_merge();
                ret
            }
            None => null_mut()
        }
    }
}

#[no_mangle]
pub unsafe extern fn rust_reallocate_inplace(ptr: *mut u8, old_size: usize,
                                             size: usize, align: usize) -> usize {
    let _lock = HEAP_MUTEX.lock();
    let block = to_block(ptr);
    let new_order = to_order(size + 1, align);
    if block.order >= new_order {
        block_size(block.order)
    }
    else {
        rust_usable_size(old_size, align)
    }
}

#[no_mangle]
pub extern fn rust_usable_size(size: usize, align: usize) -> usize {
    let order = to_order(size + 1, align);
    block_size(order)
}

#[no_mangle]
pub extern fn rust_stats_print() {}
<|endoftext|>


<commit_before>use std::str;
use std::io::prelude::*;
use std::fs::File;

use nom::*;

use error::*;

named!(string_between_quotes, delimited!(char!('\"'), is_not!("\""), char!('\"')));
named!(get_cell, take_while!(is_not_cell_end));
named!(get_line, take_until!("\n"));
named!(consume_useless_chars, take_while!(is_whitespace));

fn is_whitespace(c: u8) -> bool {
    c as char == ' ' || c as char == '\t'
}

fn is_not_cell_end(c: u8) -> bool {
    c as char != ',' && c as char != '\n'
}

fn get_column_value(input: &[u8], pos: Position) -> IResult<&[u8], &[u8], CsvError> {
    let (i, cell) = try_parse!(input,
        fix_error!(CsvError,
            preceded!(
            consume_useless_chars,
                alt!(
                    string_between_quotes
                  | get_cell
                )
            )
        )
    );

    if i.len() == 0 {
        IResult::Incomplete(Needed::Unknown)
    } else if is_not_cell_end(i[0]) {
      let p = Position { line: pos.line, column: pos.column + input.offset(i) };
        IResult::Error(Err::Code(ErrorKind::Custom(
            CsvError::InvalidCharacter(CharError::new(',', i[0] as char, &p))
        )))
    } else {
      IResult::Done(i, cell)
    }

    /*match consume_useless_chars(entry) {
        IResult::Done(out, _) => {
            if out.len() < 1 {
                Ok(IResult::Done(b"", b""))
            } else if out[0] as char == '\"' {
                match string_between_quotes(out) {
                    IResult::Done(i, out) => {
                        if is_not_cell_end(i[0]) {
                            Err(CsvError::InvalidCharacter(CharError::new(',', i[0] as char, &pos)))
                            //panic!("Expected `,`, found `{}`", i[0] as char)
                        } else {
                            Ok(IResult::Done(i, out))
                        }
                    }
                    x => Ok(x),
                }
            } else if out[0] as char == '\n' {
                Ok(IResult::Done(b"", b""))
            } else {
                Ok(get_cell(out))
            }
        },
        x => Ok(x),
    }*/

}

fn get_line_values<'a>(ret: &mut Vec<String>, entry: &'a [u8],
    line: usize) -> Result<IResult<&'a [u8], &'a [u8], CsvError>, CsvError> {
    if entry.len() < 1 {
        Ok(IResult::Done(b"", b""))
    } else {
        match get_column_value(entry, Position::new(line, ret.len())) {
          IResult::Done(in_, out) => {
                if out.len() < 1 && in_.len() < 1 {
                    ret.push(String::new());
                    Ok(IResult::Done(b"", b""))
                } else {
                    if let Ok(s) = str::from_utf8(out) {
                        ret.push(s.to_owned());
                    }
                    if in_.len() > 0 && in_[0] as char != '\n' {
                        match fix_error!(in_, CsvError, take!(1)) {
                            IResult::Done(in_, _) => get_line_values(ret, in_, line),
                            x => Ok(x),
                        }
                    } else {
                        Ok(IResult::Done(b"", b""))
                    }
                }
            }
            x => Ok(x)
            /*
            n if n.is_incomplete() => {
                if let Some(out) = n.remaining_input() {
                    if out.len() < 1 {
                        Ok(IResult::Done(b"", b""))
                    } else {
                        if let Ok(s) = str::from_utf8(out) {
                            ret.push(s.to_owned());
                        }
                        Ok(IResult::Done(b"", b""))
                    }
                } else {
                    Ok(IResult::Done(b"", b""))
                }
            }
            x => Ok(x),
            */
        }
    }
}

fn get_lines_values(mut ret: Vec<Vec<String>>, entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    match get_line(entry) {
        IResult::Done(in_, out) => {
            let mut line = vec!();

            match get_line_values(&mut line, out, ret.len()) {
                Err(e) => return Err(e),
                _ => {}
            }
            // we stop at first empty line
            if line.len() < 1 {
                return Ok(ret)
            }
            if ret.len() > 0 && line.len() != ret[0].len() {
                panic!("Line `{}` has `{}` value{}, `{}` {} expected.",
                       ret.len(), line.len(), if line.len() > 1 { "s" } else { "" },
                       ret[0].len(), if ret[0].len() > 1 { "were" } else { "was" });
            }
            ret.push(line);
            if in_.len() > 0 && in_[0] as char == '\n' {
                match take!(in_, 1) {
                    IResult::Done(in_, _) => get_lines_values(ret, in_),
                    _ => Ok(ret),
                }
            } else {
                Ok(ret)
            }
        }
        _ => Ok(ret),
    }
}

pub fn parse_csv_from_slice(entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    get_lines_values(vec!(), entry)
}

pub fn parse_csv_from_file(filename: &str) -> Result<Vec<Vec<String>>, CsvError> {
    let mut f = File::open(filename).unwrap();
    let mut buffer = vec!();

    f.read_to_end(&mut buffer).unwrap();
    parse_csv_from_slice(&buffer)
}

pub fn parse_csv(entry: &str) -> Result<Vec<Vec<String>>, CsvError> {
    parse_csv_from_slice(entry.as_bytes())
}

#[test]
fn check_string_between_quotes() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match string_between_quotes(f) {
        IResult::Done(in_, out) => {
            assert_eq!(out, b"nom");
            assert_eq!(in_, b",age\ncarles,30\nlaure,28\n");
        },
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_cell() {
    let f = b"age\ncarles,30\n";
    let g = b"age2,carles,30\n";

    match get_cell(f) {
        IResult::Done(_, out) => assert_eq!(out, b"age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
    match get_cell(g) {
        IResult::Done(_, out) => assert_eq!(out, b"age2"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match get_line(f) {
        IResult::Done(_, out) => assert_eq!(out, b"\"nom\",age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line_values() {
    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",,age", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",,age\n", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",age,\n", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned()));

    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",age,,\"hoho\",,end\n", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned(), "hoho".to_owned(), "".to_owned(), "end".to_owned()));

    let mut cells = vec!();
    let e = get_line_values(&mut cells, b"\"nom\" ,age,\"hoho\"", 0);
    assert_eq!(e, Err(CsvError::InvalidCharacter(CharError::new(',', ' ', &Position::new(0, 0)))));
}

#[test]
fn check_get_lines_values() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(get_lines_values(vec!(), f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}

#[test]
fn check_parse_csv() {
    let f = "\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(parse_csv(f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}
<commit_msg>Leading whitespace is optional<commit_after>use std::str;
use std::io::prelude::*;
use std::fs::File;

use nom::*;

use error::*;

named!(string_between_quotes, delimited!(char!('\"'), is_not!("\""), char!('\"')));
named!(get_cell, take_while!(is_not_cell_end));
named!(get_line, take_until!("\n"));
named!(consume_useless_chars, take_while!(is_whitespace));

fn is_whitespace(c: u8) -> bool {
    c as char == ' ' || c as char == '\t'
}

fn is_not_cell_end(c: u8) -> bool {
    c as char != ',' && c as char != '\n'
}

fn get_column_value(input: &[u8], pos: Position) -> IResult<&[u8], &[u8], CsvError> {
    let (i, cell) = try_parse!(input,
        fix_error!(CsvError,
            preceded!(
            opt!(consume_useless_chars),
                alt!(
                    string_between_quotes
                  | get_cell
                )
            )
        )
    );

    if i.len() == 0 {
        IResult::Incomplete(Needed::Unknown)
    } else if is_not_cell_end(i[0]) {
      let p = Position { line: pos.line, column: pos.column + input.offset(i) };
        IResult::Error(Err::Code(ErrorKind::Custom(
            CsvError::InvalidCharacter(CharError::new(',', i[0] as char, &p))
        )))
    } else {
      IResult::Done(i, cell)
    }

    /*match consume_useless_chars(entry) {
        IResult::Done(out, _) => {
            if out.len() < 1 {
                Ok(IResult::Done(b"", b""))
            } else if out[0] as char == '\"' {
                match string_between_quotes(out) {
                    IResult::Done(i, out) => {
                        if is_not_cell_end(i[0]) {
                            Err(CsvError::InvalidCharacter(CharError::new(',', i[0] as char, &pos)))
                            //panic!("Expected `,`, found `{}`", i[0] as char)
                        } else {
                            Ok(IResult::Done(i, out))
                        }
                    }
                    x => Ok(x),
                }
            } else if out[0] as char == '\n' {
                Ok(IResult::Done(b"", b""))
            } else {
                Ok(get_cell(out))
            }
        },
        x => Ok(x),
    }*/

}

fn get_line_values<'a>(ret: &mut Vec<String>, entry: &'a [u8],
    line: usize) -> Result<IResult<&'a [u8], &'a [u8], CsvError>, CsvError> {
    if entry.len() < 1 {
        Ok(IResult::Done(b"", b""))
    } else {
        match get_column_value(entry, Position::new(line, ret.len())) {
          IResult::Done(in_, out) => {
                if out.len() < 1 && in_.len() < 1 {
                    ret.push(String::new());
                    Ok(IResult::Done(b"", b""))
                } else {
                    if let Ok(s) = str::from_utf8(out) {
                        ret.push(s.to_owned());
                    }
                    if in_.len() > 0 && in_[0] as char != '\n' {
                        match fix_error!(in_, CsvError, take!(1)) {
                            IResult::Done(in_, _) => get_line_values(ret, in_, line),
                            x => Ok(x),
                        }
                    } else {
                        Ok(IResult::Done(b"", b""))
                    }
                }
            }
            x => Ok(x)
            /*
            n if n.is_incomplete() => {
                if let Some(out) = n.remaining_input() {
                    if out.len() < 1 {
                        Ok(IResult::Done(b"", b""))
                    } else {
                        if let Ok(s) = str::from_utf8(out) {
                            ret.push(s.to_owned());
                        }
                        Ok(IResult::Done(b"", b""))
                    }
                } else {
                    Ok(IResult::Done(b"", b""))
                }
            }
            x => Ok(x),
            */
        }
    }
}

fn get_lines_values(mut ret: Vec<Vec<String>>, entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    match get_line(entry) {
        IResult::Done(in_, out) => {
            let mut line = vec!();

            match get_line_values(&mut line, out, ret.len()) {
                Err(e) => return Err(e),
                _ => {}
            }
            // we stop at first empty line
            if line.len() < 1 {
                return Ok(ret)
            }
            if ret.len() > 0 && line.len() != ret[0].len() {
                panic!("Line `{}` has `{}` value{}, `{}` {} expected.",
                       ret.len(), line.len(), if line.len() > 1 { "s" } else { "" },
                       ret[0].len(), if ret[0].len() > 1 { "were" } else { "was" });
            }
            ret.push(line);
            if in_.len() > 0 && in_[0] as char == '\n' {
                match take!(in_, 1) {
                    IResult::Done(in_, _) => get_lines_values(ret, in_),
                    _ => Ok(ret),
                }
            } else {
                Ok(ret)
            }
        }
        _ => Ok(ret),
    }
}

pub fn parse_csv_from_slice(entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    get_lines_values(vec!(), entry)
}

pub fn parse_csv_from_file(filename: &str) -> Result<Vec<Vec<String>>, CsvError> {
    let mut f = File::open(filename).unwrap();
    let mut buffer = vec!();

    f.read_to_end(&mut buffer).unwrap();
    parse_csv_from_slice(&buffer)
}

pub fn parse_csv(entry: &str) -> Result<Vec<Vec<String>>, CsvError> {
    parse_csv_from_slice(entry.as_bytes())
}

#[test]
fn check_string_between_quotes() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match string_between_quotes(f) {
        IResult::Done(in_, out) => {
            assert_eq!(out, b"nom");
            assert_eq!(in_, b",age\ncarles,30\nlaure,28\n");
        },
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_cell() {
    let f = b"age\ncarles,30\n";
    let g = b"age2,carles,30\n";

    match get_cell(f) {
        IResult::Done(_, out) => assert_eq!(out, b"age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
    match get_cell(g) {
        IResult::Done(_, out) => assert_eq!(out, b"age2"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match get_line(f) {
        IResult::Done(_, out) => assert_eq!(out, b"\"nom\",age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line_values() {
    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",,age", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",,age\n", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",age,\n", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned()));

    let mut cells = vec!();
    get_line_values(&mut cells, b"\"nom\",age,,\"hoho\",,end\n", 0).unwrap();
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned(), "hoho".to_owned(), "".to_owned(), "end".to_owned()));

    let mut cells = vec!();
    let e = get_line_values(&mut cells, b"\"nom\" ,age,\"hoho\"", 0);
    assert_eq!(e, Err(CsvError::InvalidCharacter(CharError::new(',', ' ', &Position::new(0, 0)))));
}

#[test]
fn check_get_lines_values() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(get_lines_values(vec!(), f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}

#[test]
fn check_parse_csv() {
    let f = "\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(parse_csv(f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}
<|endoftext|>


<commit_before>use super::{Token, String, Placeholder, Conditional, ContentConditional, Generated};

use std::iter::{Iterator, Peekable};
use std::fmt;

#[deriving(Eq)]
enum LexToken {
	Begin,
	End,
	Colon,
	Questionmark,
	Exclamation,
	Plus,
	Slash,
	Quote,
	Character(char)
}

struct Parser<T, V> {
    tokens: Peekable<V, T>
}

impl<T: Iterator<V>, V: Eq> Parser<T, V> {
	#[inline]
	fn eat(&mut self, expected: V) -> bool {
		let eaten = match self.peek() {
			Some(t) if *t == expected => {
				true
			},
			_ => false
		};

		if eaten {
			self.next();
		}

		eaten
	}

	fn eat_if(&mut self, is_edible: |&V| -> bool) -> bool {
		let eaten = match self.peek() {
			Some(t) => is_edible(t),
			_ => false
		};

		if eaten {
			self.next();
		}

		eaten
	}

	#[inline]
	fn peek<'a>(&'a mut self) -> Option<&'a V> {
		self.tokens.peek()
	}
}

impl<T: Iterator<V>, V> Iterator<V> for Parser<T, V> {
	#[inline]
	fn next(&mut self) -> Option<V> {
		self.tokens.by_ref().next()
	}
}

impl fmt::Show for LexToken {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		match *self {
			Begin => "[[".fmt(f),
			End => "]]".fmt(f),
			Colon => ":".fmt(f),
			Questionmark => "?".fmt(f),
			Exclamation => "!".fmt(f),
			Plus => "+".fmt(f),
			Slash => "/".fmt(f),
			Quote => "\"".fmt(f),
			Character(c) => c.fmt(f)
		}
	}
}

pub fn parse<T: Iterator<Result<char, ~str>>>(chars: &mut T) -> Result<Vec<Token>, ~str> {
	let tokens = try!(lex(chars));
	parse_block(&mut Parser{
		tokens: tokens.move_iter().by_ref().peekable()
	})
}

fn lex<T: Iterator<Result<char, ~str>>>(chars: &mut T) -> Result<Vec<LexToken>, ~str> {
	let mut chars = chars.by_ref().peekable();
	let mut tokens = Vec::new();

	loop {
		match chars.next() {
			Some(Ok(c)) => match c {
				'[' => match chars.peek() {
					Some(&Ok('[')) => {
						tokens.push(Begin);
						chars.next();
					},
					_ => tokens.push(Character('['))
				},
				']' => match chars.peek() {
					Some(&Ok(']')) => {
						tokens.push(End);
						chars.next();
					},
					_ => tokens.push(Character(']'))
				},
				':' => tokens.push(Colon),
				'?' => tokens.push(Questionmark),
				'!' => tokens.push(Exclamation),
				'+' => tokens.push(Plus),
				'/' => tokens.push(Slash),
				'"' => tokens.push(Quote),
				'\\' => match chars.next() {
					Some(Ok(c)) => {
						tokens.push(Character(c));
					},
					Some(Err(e)) => return Err(e.to_str()),
					None => break
				},
				c => tokens.push(Character(c))
			},
			Some(Err(e)) => return Err(e),
			None => break
		}
	}

	Ok(tokens)
}

fn parse_block<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Vec<Token>, ~str> {
	let mut result = Vec::new();
	let mut string = StrBuf::new();

	loop {
		match tokens.next() {
			Some(Begin) => match tokens.next() {
				Some(Colon) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					result.push(try!(parse_placeholder(tokens)));
				},
				Some(Questionmark) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					result.push(try!(parse_conditional(tokens)));
				},
				Some(Plus) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					result.push(try!(parse_generator(tokens)));
				},
				Some(Slash) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					parse_block_end(tokens);
					break
				},
				Some(t) => {
					return Err(format!("parse error: unknown token type: '{}'", t))
				},
				None => string.push_str(Begin.to_str())
			},
			Some(t) => string.push_str(t.to_str()),
			None => break
		}
	}

	if string.len() > 0 {
		result.push(String(string.into_owned()));
	}

	Ok(result)
}

fn parse_placeholder<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, ~str> {
	let mut label = StrBuf::new();

	for t in tokens.by_ref().take_while(|&t| t != End).map(|t| t.to_str()) {
		label.push_str(t.to_str());
	}

	Ok(Placeholder(label.into_owned()))
}

fn parse_conditional<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, ~str> {
	let negative = tokens.eat(Exclamation);
	let content_cond = tokens.eat(Colon);
	let mut label = StrBuf::new();

	for t in tokens.by_ref().take_while(|&t| t != End).map(|t| t.to_str()) {
		label.push_str(t.to_str());
	}

	let content = try!(parse_block(tokens));

	if content_cond {
		Ok(ContentConditional(label.into_owned(), !negative, content))
	} else {
		Ok(Conditional(label.into_owned(), !negative, content))
	}
}

fn parse_generator<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, ~str> {
	let mut label = StrBuf::new();
	let mut args = Vec::new();

	if tokens.eat(Quote) {
		for t in tokens.by_ref().take_while(|&t| t != Quote).map(|t| t.to_str()) {
			label.push_str(t.to_str());
		}
	} else {
		loop {
			match tokens.next() {
				Some(End) => return Ok(Generated(label.into_owned(), Vec::new())),
				Some(Character(c)) if c.is_whitespace() => break,
				Some(t) => label.push_str(t.to_str()),
				None => break
			}
		}
	}

	'arg_list: loop {
		let mut new_arg = StrBuf::new();

		while tokens.eat_if(|&t| match t {Character(c) if c.is_whitespace() => true, _ => false}) {};

		if tokens.eat(Quote) {
			for t in tokens.by_ref().take_while(|&t| t != Quote).map(|t| t.to_str()) {
				new_arg.push_str(t.to_str());
			}
		} else {
			'arg: loop {
				match tokens.next() {
					Some(End) => if new_arg.len() > 0 {
						args.push(new_arg.into_owned());
						break 'arg_list;
					},
					Some(Character(c)) if c.is_whitespace() => {
						while tokens.eat_if(|&t| match t {Character(c) if c.is_whitespace() => true, _ => false}) {}
						break 'arg
					},
					Some(t) => new_arg.push_str(t.to_str()),
					None => break 'arg_list
				}
			}
		}

		args.push(new_arg.into_owned());
	}

	Ok(Generated(label.into_owned(), args))
}

fn parse_block_end<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) {
	tokens.by_ref().advance(|t| t != End);
}

<commit_msg>Changed eat_if to eat_while. That's how it's used, anyway<commit_after>use super::{Token, String, Placeholder, Conditional, ContentConditional, Generated};

use std::iter::{Iterator, Peekable};
use std::fmt;

#[deriving(Eq)]
enum LexToken {
	Begin,
	End,
	Colon,
	Questionmark,
	Exclamation,
	Plus,
	Slash,
	Quote,
	Character(char)
}

struct Parser<T, V> {
    tokens: Peekable<V, T>
}

impl<T: Iterator<V>, V: Eq> Parser<T, V> {
	#[inline]
	fn eat(&mut self, expected: V) -> bool {
		let eaten = match self.peek() {
			Some(t) if *t == expected => {
				true
			},
			_ => false
		};

		if eaten {
			self.next();
		}

		eaten
	}

	fn eat_while(&mut self, is_edible: |&V| -> bool) {
		loop {
			let eaten = match self.peek() {
				Some(t) => is_edible(t),
				_ => false
			};

			if eaten {
				self.next();
			} else {
				break;
			}
		}
	}

	#[inline]
	fn peek<'a>(&'a mut self) -> Option<&'a V> {
		self.tokens.peek()
	}
}

impl<T: Iterator<V>, V> Iterator<V> for Parser<T, V> {
	#[inline]
	fn next(&mut self) -> Option<V> {
		self.tokens.by_ref().next()
	}
}

impl fmt::Show for LexToken {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		match *self {
			Begin => "[[".fmt(f),
			End => "]]".fmt(f),
			Colon => ":".fmt(f),
			Questionmark => "?".fmt(f),
			Exclamation => "!".fmt(f),
			Plus => "+".fmt(f),
			Slash => "/".fmt(f),
			Quote => "\"".fmt(f),
			Character(c) => c.fmt(f)
		}
	}
}

pub fn parse<T: Iterator<Result<char, ~str>>>(chars: &mut T) -> Result<Vec<Token>, ~str> {
	let tokens = try!(lex(chars));
	parse_block(&mut Parser{
		tokens: tokens.move_iter().by_ref().peekable()
	})
}

fn lex<T: Iterator<Result<char, ~str>>>(chars: &mut T) -> Result<Vec<LexToken>, ~str> {
	let mut chars = chars.by_ref().peekable();
	let mut tokens = Vec::new();

	loop {
		match chars.next() {
			Some(Ok(c)) => match c {
				'[' => match chars.peek() {
					Some(&Ok('[')) => {
						tokens.push(Begin);
						chars.next();
					},
					_ => tokens.push(Character('['))
				},
				']' => match chars.peek() {
					Some(&Ok(']')) => {
						tokens.push(End);
						chars.next();
					},
					_ => tokens.push(Character(']'))
				},
				':' => tokens.push(Colon),
				'?' => tokens.push(Questionmark),
				'!' => tokens.push(Exclamation),
				'+' => tokens.push(Plus),
				'/' => tokens.push(Slash),
				'"' => tokens.push(Quote),
				'\\' => match chars.next() {
					Some(Ok(c)) => {
						tokens.push(Character(c));
					},
					Some(Err(e)) => return Err(e.to_str()),
					None => break
				},
				c => tokens.push(Character(c))
			},
			Some(Err(e)) => return Err(e),
			None => break
		}
	}

	Ok(tokens)
}

fn parse_block<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Vec<Token>, ~str> {
	let mut result = Vec::new();
	let mut string = StrBuf::new();

	loop {
		match tokens.next() {
			Some(Begin) => match tokens.next() {
				Some(Colon) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					result.push(try!(parse_placeholder(tokens)));
				},
				Some(Questionmark) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					result.push(try!(parse_conditional(tokens)));
				},
				Some(Plus) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					result.push(try!(parse_generator(tokens)));
				},
				Some(Slash) => {
					if string.len() > 0 {
						result.push(String(string.into_owned()));
						string = StrBuf::new();
					}

					parse_block_end(tokens);
					break
				},
				Some(t) => {
					return Err(format!("parse error: unknown token type: '{}'", t))
				},
				None => string.push_str(Begin.to_str())
			},
			Some(t) => string.push_str(t.to_str()),
			None => break
		}
	}

	if string.len() > 0 {
		result.push(String(string.into_owned()));
	}

	Ok(result)
}

fn parse_placeholder<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, ~str> {
	let mut label = StrBuf::new();

	for t in tokens.by_ref().take_while(|&t| t != End).map(|t| t.to_str()) {
		label.push_str(t.to_str());
	}

	Ok(Placeholder(label.into_owned()))
}

fn parse_conditional<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, ~str> {
	let negative = tokens.eat(Exclamation);
	let content_cond = tokens.eat(Colon);
	let mut label = StrBuf::new();

	for t in tokens.by_ref().take_while(|&t| t != End).map(|t| t.to_str()) {
		label.push_str(t.to_str());
	}

	let content = try!(parse_block(tokens));

	if content_cond {
		Ok(ContentConditional(label.into_owned(), !negative, content))
	} else {
		Ok(Conditional(label.into_owned(), !negative, content))
	}
}

fn parse_generator<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, ~str> {
	let mut label = StrBuf::new();
	let mut args = Vec::new();

	if tokens.eat(Quote) {
		for t in tokens.by_ref().take_while(|&t| t != Quote).map(|t| t.to_str()) {
			label.push_str(t.to_str());
		}
	} else {
		loop {
			match tokens.next() {
				Some(End) => return Ok(Generated(label.into_owned(), Vec::new())),
				Some(Character(c)) if c.is_whitespace() => break,
				Some(t) => label.push_str(t.to_str()),
				None => break
			}
		}
	}

	'arg_list: loop {
		let mut new_arg = StrBuf::new();

		tokens.eat_while(|&t| match t {Character(c) if c.is_whitespace() => true, _ => false});

		if tokens.eat(Quote) {
			for t in tokens.by_ref().take_while(|&t| t != Quote).map(|t| t.to_str()) {
				new_arg.push_str(t.to_str());
			}
		} else {
			'arg: loop {
				match tokens.next() {
					Some(End) => if new_arg.len() > 0 {
						args.push(new_arg.into_owned());
						break 'arg_list;
					},
					Some(Character(c)) if c.is_whitespace() => {
						tokens.eat_while(|&t| match t {Character(c) if c.is_whitespace() => true, _ => false});
						break 'arg
					},
					Some(t) => new_arg.push_str(t.to_str()),
					None => break 'arg_list
				}
			}
		}

		args.push(new_arg.into_owned());
	}

	Ok(Generated(label.into_owned(), args))
}

fn parse_block_end<T: Iterator<LexToken>>(tokens: &mut Parser<T, LexToken>) {
	tokens.by_ref().advance(|t| t != End);
}

<|endoftext|>


<commit_before>use nom::{IResult, alpha, digit, space, rest};
use regex::{Regex, RegexBuilder};
use chrono::{Date, UTC, Datelike};

use std::str;
use std::fmt;

#[derive(Debug)]
pub enum ParserError {
    SummaryParsing(String),
    FooterParsing(String),
    CommitMessageLength,
}

impl fmt::Display for ParserError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ParserError::SummaryParsing(ref line) => write!(f, "Could not parse commit summary: {}", line),
            ParserError::FooterParsing(ref line) => write!(f, "Could not parse commit footer: {}", line),
            ParserError::CommitMessageLength => write!(f, "Commit message length too small."),
        }
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ParsedTag {
    pub name: String,
    pub date: Date<UTC>,
}

impl fmt::Display for ParsedTag {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,
               "{} ({}-{:02}-{:02})",
               self.name,
               self.date.year(),
               self.date.month(),
               self.date.day())
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ParsedCommit {
    pub summary: SummaryElement,
    pub body: Vec<BodyElement>,
    pub footer: Vec<FooterElement>,
}

impl fmt::Display for ParsedCommit {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(writeln!(f, "{}", self.summary));
        for item in &self.body {
            match *item {
                BodyElement::List(ref vec) => {
                    for item in vec {
                        try!(writeln!(f, "    {}", item.text));
                    }
                },
                BodyElement::Paragraph(ref par) => {
                    for line in par.text.lines().map(|x| format!("    {}", x)).collect::<Vec<String>>() {
                        try!(writeln!(f, "{}", line));
                    }
                }
            }
        }
        Ok(())
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct SummaryElement {
    pub prefix: String,
    pub category: String,
    pub text: String,
    pub tags: Vec<String>,
}

impl fmt::Display for SummaryElement {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "- [{}]{}", self.category, self.text)
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub enum BodyElement {
    List(Vec<ListElement>),
    Paragraph(ParagraphElement),
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ListElement {
    pub category: String,
    pub text: String,
    pub tags: Vec<String>,
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ParagraphElement {
    pub text: String,
    pub tags: Vec<String>,
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct FooterElement {
    pub key: String,
    pub value: String,
}

lazy_static! {
    static ref RE_TAGS: Regex = Regex::new(r" :(.*?):").unwrap();
    static ref RE_FOOTER: Regex = RegexBuilder::new(r"^([\w-]+):\s(.*)$").multi_line(true).compile().unwrap();
    static ref RE_LIST: Regex = RegexBuilder::new(r"^-\s.*$(\n^\s+-\s.*)*").multi_line(true).compile().unwrap();
}

pub struct Parser;
impl Parser {
    /// Parses a single commit message and returns a changelog ready form
    pub fn parse_commit_message(&self, message: &str) -> Result<ParsedCommit, ParserError> {

        /// Parses for tags and returns them with the resulting string
        fn parse_and_consume_tags(i: &[u8]) -> (Vec<String>, String) {
            let string = str::from_utf8(i).unwrap_or("");
            let mut tags = vec![];
            for cap in RE_TAGS.captures_iter(string) {
                tags.extend(cap.at(1).unwrap_or("").split(',').map(|x| x.trim().to_owned()).collect::<Vec<String>>());
            }
            (tags, RE_TAGS.replace_all(string, ""))
        }

        named!(parse_category<&str>,
            chain!(
                tag!("[")? ~
                p_category: map_res!(
                    alt!(
                        tag!("Added") |
                        tag!("Changed") |
                        tag!("Fixed") |
                        tag!("Improved") |
                        tag!("Removed")
                    ),
                    str::from_utf8
                ) ~
                tag!("]")? ,
                || p_category
            )
        );

        named!(parse_list_category<&str>,
            chain!(
                many0!(space) ~
                tag!("- ") ~
                p_category: parse_category ,
                || p_category
            )
        );

        // Every block is split by two newlines
        let mut commit_parts = message.split("\n\n");

        // Parse the summary line
        let summary_line = try!(commit_parts.nth(0).ok_or(ParserError::CommitMessageLength)).trim();
        named!(parse_summary<SummaryElement>,
            chain!(
                p_prefix: separated_pair!(alpha, char!('-'), digit)? ~
                space? ~
                p_category: parse_category ~
                tag!("]")? ~
                p_tags_rest: map!(rest, parse_and_consume_tags),
            || SummaryElement {
                prefix: p_prefix.map_or("".to_owned(), |p| {
                    format!("{}-{}", str::from_utf8(p.0).unwrap_or(""), str::from_utf8(p.1).unwrap_or(""))
                }),
                category: p_category.to_owned(),
                tags: p_tags_rest.0.clone(),
                text: p_tags_rest.1.clone(),
            })
        );
        let parsed_summary = match parse_summary(summary_line.as_bytes()) {
            IResult::Done(_, parsed) => parsed,
            _ => return Err(ParserError::SummaryParsing(summary_line.to_owned())),
        };

        // Parse the body and the footer, the summary is already consumed
        let mut parsed_footer = vec![];
        let mut parsed_body = vec![];
        for part in commit_parts {
            // Parse footer
            if RE_FOOTER.is_match(part) {
                for cap in RE_FOOTER.captures_iter(part) {
                    parsed_footer.push(FooterElement {
                        key: try!(cap.at(1).ok_or(ParserError::FooterParsing(part.to_owned()))).to_owned(),
                        value: try!(cap.at(2).ok_or(ParserError::FooterParsing(part.to_owned()))).to_owned(),
                    });
                }
            } else if RE_LIST.is_match(part) {
                // Parse list items
                let mut list = vec![];
                for list_item in part.lines() {
                    let (parsed_tags, parsed_text) = parse_and_consume_tags(list_item.as_bytes());
                    let parsed_category = match parse_list_category(list_item.as_bytes()) {
                        IResult::Done(_, cat) => cat,
                        _ => "",
                    };
                    list.push(ListElement {
                        category: parsed_category.to_owned(),
                        text: parsed_text,
                        tags: parsed_tags,
                    });
                }
                parsed_body.push(BodyElement::List(list));
            } else {
                // Assume paragraph
                let (parsed_tags, parsed_text) = parse_and_consume_tags(part.as_bytes());
                parsed_body.push(BodyElement::Paragraph(ParagraphElement {
                    text: parsed_text.trim().to_owned(),
                    tags: parsed_tags,
                }));
            }
        }

        Ok(ParsedCommit {
            summary: parsed_summary,
            body: parsed_body,
            footer: parsed_footer,
        })
    }
}
<commit_msg>Improved newline handling<commit_after>use nom::{IResult, alpha, digit, space, rest};
use regex::{Regex, RegexBuilder};
use chrono::{Date, UTC, Datelike};

use std::str;
use std::fmt;

#[derive(Debug)]
pub enum ParserError {
    SummaryParsing(String),
    FooterParsing(String),
    CommitMessageLength,
}

impl fmt::Display for ParserError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ParserError::SummaryParsing(ref line) => write!(f, "Could not parse commit summary: {}", line),
            ParserError::FooterParsing(ref line) => write!(f, "Could not parse commit footer: {}", line),
            ParserError::CommitMessageLength => write!(f, "Commit message length too small."),
        }
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ParsedTag {
    pub name: String,
    pub date: Date<UTC>,
}

impl fmt::Display for ParsedTag {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,
               "{} ({}-{:02}-{:02})",
               self.name,
               self.date.year(),
               self.date.month(),
               self.date.day())
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ParsedCommit {
    pub summary: SummaryElement,
    pub body: Vec<BodyElement>,
    pub footer: Vec<FooterElement>,
}

impl fmt::Display for ParsedCommit {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(write!(f, "{}", self.summary));
        for item in &self.body {
            match *item {
                BodyElement::List(ref vec) => {
                    for item in vec {
                        try!(write!(f, "\n    {}", item.text));
                    }
                },
                BodyElement::Paragraph(ref par) => {
                    for line in par.text.lines().map(|x| format!("    {}", x)).collect::<Vec<String>>() {
                        try!(write!(f, "\n{}", line));
                    }
                }
            }
        }
        Ok(())
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct SummaryElement {
    pub prefix: String,
    pub category: String,
    pub text: String,
    pub tags: Vec<String>,
}

impl fmt::Display for SummaryElement {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "- [{}]{}", self.category, self.text)
    }
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub enum BodyElement {
    List(Vec<ListElement>),
    Paragraph(ParagraphElement),
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ListElement {
    pub category: String,
    pub text: String,
    pub tags: Vec<String>,
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct ParagraphElement {
    pub text: String,
    pub tags: Vec<String>,
}

#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub struct FooterElement {
    pub key: String,
    pub value: String,
}

lazy_static! {
    static ref RE_TAGS: Regex = Regex::new(r" :(.*?):").unwrap();
    static ref RE_FOOTER: Regex = RegexBuilder::new(r"^([\w-]+):\s(.*)$").multi_line(true).compile().unwrap();
    static ref RE_LIST: Regex = RegexBuilder::new(r"^-\s.*$(\n^\s+-\s.*)*").multi_line(true).compile().unwrap();
}

pub struct Parser;
impl Parser {
    /// Parses a single commit message and returns a changelog ready form
    pub fn parse_commit_message(&self, message: &str) -> Result<ParsedCommit, ParserError> {

        /// Parses for tags and returns them with the resulting string
        fn parse_and_consume_tags(i: &[u8]) -> (Vec<String>, String) {
            let string = str::from_utf8(i).unwrap_or("");
            let mut tags = vec![];
            for cap in RE_TAGS.captures_iter(string) {
                tags.extend(cap.at(1).unwrap_or("").split(',').map(|x| x.trim().to_owned()).collect::<Vec<String>>());
            }
            (tags, RE_TAGS.replace_all(string, ""))
        }

        named!(parse_category<&str>,
            chain!(
                tag!("[")? ~
                p_category: map_res!(
                    alt!(
                        tag!("Added") |
                        tag!("Changed") |
                        tag!("Fixed") |
                        tag!("Improved") |
                        tag!("Removed")
                    ),
                    str::from_utf8
                ) ~
                tag!("]")? ,
                || p_category
            )
        );

        named!(parse_list_category<&str>,
            chain!(
                many0!(space) ~
                tag!("- ") ~
                p_category: parse_category ,
                || p_category
            )
        );

        // Every block is split by two newlines
        let mut commit_parts = message.split("\n\n");

        // Parse the summary line
        let summary_line = try!(commit_parts.nth(0).ok_or(ParserError::CommitMessageLength)).trim();
        named!(parse_summary<SummaryElement>,
            chain!(
                p_prefix: separated_pair!(alpha, char!('-'), digit)? ~
                space? ~
                p_category: parse_category ~
                tag!("]")? ~
                p_tags_rest: map!(rest, parse_and_consume_tags),
            || SummaryElement {
                prefix: p_prefix.map_or("".to_owned(), |p| {
                    format!("{}-{}", str::from_utf8(p.0).unwrap_or(""), str::from_utf8(p.1).unwrap_or(""))
                }),
                category: p_category.to_owned(),
                tags: p_tags_rest.0.clone(),
                text: p_tags_rest.1.clone(),
            })
        );
        let parsed_summary = match parse_summary(summary_line.as_bytes()) {
            IResult::Done(_, parsed) => parsed,
            _ => return Err(ParserError::SummaryParsing(summary_line.to_owned())),
        };

        // Parse the body and the footer, the summary is already consumed
        let mut parsed_footer = vec![];
        let mut parsed_body = vec![];
        for part in commit_parts {
            // Parse footer
            if RE_FOOTER.is_match(part) {
                for cap in RE_FOOTER.captures_iter(part) {
                    parsed_footer.push(FooterElement {
                        key: try!(cap.at(1).ok_or(ParserError::FooterParsing(part.to_owned()))).to_owned(),
                        value: try!(cap.at(2).ok_or(ParserError::FooterParsing(part.to_owned()))).to_owned(),
                    });
                }
            } else if RE_LIST.is_match(part) {
                // Parse list items
                let mut list = vec![];
                for list_item in part.lines() {
                    let (parsed_tags, parsed_text) = parse_and_consume_tags(list_item.as_bytes());
                    let parsed_category = match parse_list_category(list_item.as_bytes()) {
                        IResult::Done(_, cat) => cat,
                        _ => "",
                    };
                    list.push(ListElement {
                        category: parsed_category.to_owned(),
                        text: parsed_text,
                        tags: parsed_tags,
                    });
                }
                parsed_body.push(BodyElement::List(list));
            } else {
                // Assume paragraph
                let (parsed_tags, parsed_text) = parse_and_consume_tags(part.as_bytes());
                parsed_body.push(BodyElement::Paragraph(ParagraphElement {
                    text: parsed_text.trim().to_owned(),
                    tags: parsed_tags,
                }));
            }
        }

        Ok(ParsedCommit {
            summary: parsed_summary,
            body: parsed_body,
            footer: parsed_footer,
        })
    }
}
<|endoftext|>


<commit_before>use std::iter::{Peekable};
use ast;

/* Grammar
    main {  }

*/

#[derive(Debug, Eq, PartialEq)]
enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    SingleQuote,
    DoubleQuote,
    Dot,
    Plus,
    Asterik,
    Question,
    Exclamation,
    Ambersand,
    Slash,
    Name(usize, usize)
}

fn tokenize(grammar : &String) -> Vec<Token> {
    let mut iterator = grammar.chars();
    let mut tokens = vec![];
    let mut name = (-1, -1);
    let mut i = 0;

    while let Some(item) = iterator.next() {
        match item {
            '{' => tokens.push(Token::OpenBrace),
            '}' => tokens.push(Token::CloseBrace),
            '(' => tokens.push(Token::OpenParen),
            ')' => tokens.push(Token::CloseParen),
            '[' => tokens.push(Token::OpenBracket),
            ']' => tokens.push(Token::CloseBracket),
            '\'' => tokens.push(Token::SingleQuote),
            '\"' => tokens.push(Token::DoubleQuote),
            '.' => tokens.push(Token::Dot),
            '+' => tokens.push(Token::Plus),
            '*' => tokens.push(Token::Asterik),
            '?' => tokens.push(Token::Question),
            '!' => tokens.push(Token::Exclamation),
            '&' => tokens.push(Token::Ambersand),
            '/' => tokens.push(Token::Slash),
            _ if item.is_alphanumeric() => {
                if name == (-1, -1) {
                    name = (i, i);
                } else {
                    name.1 += 1;
                }
            },
            _ if item.is_whitespace() && name != (-1, -1) => {
                tokens.push(Token::Name(name.0 as usize, name.1 as usize));
                name = (-1, -1);
            }
            _ => { }
        }
        i += 1;
    }
    tokens
}

fn parse(grammar : &String, tokens : Vec<Token>) -> Result<ast::Grammar, u8> {
    let mut iterator = tokens.iter().peekable();
    let mut grammar_object = ast::Grammar { rules: vec![], main: 0 };

    while let Some(token) = iterator.next() {
        if let &Token::Name(name_start, name_end) = token {
            if let Some(brace_token) = iterator.next() {
                if brace_token == &Token::OpenBrace {
                    let pattern = parse_pattern(grammar, &mut iterator, false);
                    match pattern {
                        Ok(p) => grammar_object.rules.push(p),
                        Err(x) => return Err(x)
                    }
                } else {
                    return Err(0);
                }
            } else {
                return Err(0);
            }
        } else {
            return Err(0);
        }
    }
    Ok(grammar_object)
}

fn parse_pattern<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut sequence = vec![];

    while let Some(token) = iterator.next() {
        let mut initial_pattern = Err(0);
        match token {
            &Token::OpenParen => { initial_pattern = initial_pattern.or(parse_pattern(grammar, iterator, true)); },
            &Token::OpenBracket => { initial_pattern = initial_pattern.or(parse_char_class(grammar, iterator)); },
            &Token::DoubleQuote | &Token::SingleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator)); },
            &Token::Dot => { initial_pattern = initial_pattern.or(Ok(ast::Pattern::CharAny)); },
            &Token::Name(le, ri) => { initial_pattern = initial_pattern.or(parse_variable(grammar, le, ri)); },
            &Token::Ambersand => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, true)); },
            &Token::Exclamation => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, false)); },
            _ => { }
        }

        if initial_pattern.is_ok() {
            if let Some(&suffix_token) = iterator.peek() {
                match suffix_token {
                    &Token::Plus | &Token::Asterik | &Token::Question => {
                        initial_pattern = parse_suffix(grammar, iterator, initial_pattern.ok().unwrap());
                    }
                    _ => { }
                }
            }
        }

        match initial_pattern {
            Ok(p) => sequence.push(p),
            Err(x) => return Err(x)
        }

        if let Some(&test_token) = iterator.peek() {
            match test_token {
                &Token::CloseParen => if is_subpattern {
                    iterator.next();
                } else {
                    return Err(0);
                },
                &Token::CloseBrace => if !is_subpattern {
                    iterator.next();
                    break;
                } else {
                    return Err(0);
                },
                &Token::Slash => {
                    let sub_pattern = parse_choice(grammar, iterator, &mut sequence, is_subpattern);
                    match sub_pattern {
                        Ok(p) => sequence.push(p),
                        Err(x) => return Err(x)
                    }
                },
                _ => { }
            }
        }
    }

    let mut boxed_vec = vec![];
    while let Some(item) = sequence.pop() {
        boxed_vec.push(Box::new(item));
    }

    Ok(ast::Pattern::Sequence(boxed_vec))
}

fn parse_choice<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, patterns : &mut Vec<ast::Pattern>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut boxed_vec = vec![];
    while let Some(item) = patterns.pop() {
        boxed_vec.push(Box::new(item));
    }

    iterator.next();

    let right = parse_pattern(grammar, iterator, is_subpattern);

    match right {
        Ok(p) => Ok(ast::Pattern::Choice(
            Box::new(ast::Pattern::Sequence(boxed_vec)),
            Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_suffix<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, pattern : ast::Pattern) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    if let Some(token) = iterator.next() {
        match token {
            &Token::Asterik => Ok(ast::Pattern::ZeroOrMore(Box::new(pattern))),
            &Token::Plus => Ok(ast::Pattern::OneOrMore(Box::new(pattern))),
            &Token::Question => Ok(ast::Pattern::Optional(Box::new(pattern))),
            _ => Err(0)
        }
    } else {
        Err(0)
    }
}

fn parse_lookahead<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, lookahead : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let sub_pattern = parse_pattern(grammar, iterator, false);
    match sub_pattern {
        Ok(p) => Ok(ast::Pattern::Lookahead(lookahead, Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_char_class<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_char_sequence<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_variable(grammar : &String, left : usize, right : usize) -> Result<ast::Pattern, u8>
{
    Err(0)
}

<commit_msg>Add Letter token and escaped chars to tokenizer<commit_after>use std::iter::{Peekable};
use ast;

/* Grammar
    main {  }

*/

#[derive(Debug, Eq, PartialEq)]
enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    SingleQuote,
    DoubleQuote,
    Dot,
    Plus,
    Asterik,
    Question,
    Exclamation,
    Ambersand,
    Slash,
    Name(usize, usize),
    Letter(u8)
}

fn tokenize(grammar : &String) -> Vec<Token> {
    let mut iterator = grammar.chars();
    let mut tokens = vec![];
    let mut name = (-1, -1);
    let mut i = 0;
    let mut in_quote = false;
    let mut escaped = false;

    while let Some(item) = iterator.next() {
        if in_quote {
            if escaped {
                tokens.push(Token::Letter(item as u8));
                escaped = false;
            } else {
                match item {
                    '\'' => {
                        tokens.push(Token::SingleQuote);
                        in_quote = false;
                    },
                    '\"' => {
                        tokens.push(Token::DoubleQuote);
                        in_quote = false;
                    },
                    '\\' => { escaped = true; },
                    _ => tokens.push(Token::Letter(item as u8))
                }
            }
        } else {
            match item {
                '{' => tokens.push(Token::OpenBrace),
                '}' => tokens.push(Token::CloseBrace),
                '(' => tokens.push(Token::OpenParen),
                ')' => tokens.push(Token::CloseParen),
                '[' => tokens.push(Token::OpenBracket),
                ']' => tokens.push(Token::CloseBracket),
                '.' => tokens.push(Token::Dot),
                '+' => tokens.push(Token::Plus),
                '*' => tokens.push(Token::Asterik),
                '?' => tokens.push(Token::Question),
                '!' => tokens.push(Token::Exclamation),
                '&' => tokens.push(Token::Ambersand),
                '/' => tokens.push(Token::Slash),
                '\'' => {
                    tokens.push(Token::SingleQuote);
                    in_quote = true;
                },
                '\"' => {
                    tokens.push(Token::DoubleQuote);
                    in_quote = true;
                },
                _ if item.is_alphanumeric() => {
                    if name == (-1, -1) {
                        name = (i, i);
                    } else {
                        name.1 += 1;
                    }
                },
                _ if item.is_whitespace() && name != (-1, -1) => {
                    tokens.push(Token::Name(name.0 as usize, name.1 as usize));
                    name = (-1, -1);
                }
                _ => { }
            }
        }
        i += 1;
    }
    tokens
}

fn parse(grammar : &String, tokens : Vec<Token>) -> Result<ast::Grammar, u8> {
    let mut iterator = tokens.iter().peekable();
    let mut grammar_object = ast::Grammar { rules: vec![], main: 0 };

    while let Some(token) = iterator.next() {
        if let &Token::Name(name_start, name_end) = token {
            if let Some(brace_token) = iterator.next() {
                if brace_token == &Token::OpenBrace {
                    let pattern = parse_pattern(grammar, &mut iterator, false);
                    match pattern {
                        Ok(p) => grammar_object.rules.push(p),
                        Err(x) => return Err(x)
                    }
                } else {
                    return Err(0);
                }
            } else {
                return Err(0);
            }
        } else {
            return Err(0);
        }
    }
    Ok(grammar_object)
}

fn parse_pattern<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut sequence = vec![];

    while let Some(token) = iterator.next() {
        let mut initial_pattern = Err(0);
        match token {
            &Token::OpenParen => { initial_pattern = initial_pattern.or(parse_pattern(grammar, iterator, true)); },
            &Token::OpenBracket => { initial_pattern = initial_pattern.or(parse_char_class(grammar, iterator)); },
            &Token::DoubleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator, false)); },
            &Token::SingleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator, true)); },
            &Token::Dot => { initial_pattern = initial_pattern.or(Ok(ast::Pattern::CharAny)); },
            &Token::Name(le, ri) => { initial_pattern = initial_pattern.or(parse_variable(grammar, le, ri)); },
            &Token::Ambersand => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, true)); },
            &Token::Exclamation => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, false)); },
            _ => { }
        }

        if initial_pattern.is_ok() {
            if let Some(&suffix_token) = iterator.peek() {
                match suffix_token {
                    &Token::Plus | &Token::Asterik | &Token::Question => {
                        initial_pattern = parse_suffix(grammar, iterator, initial_pattern.ok().unwrap());
                    }
                    _ => { }
                }
            }
        }

        match initial_pattern {
            Ok(p) => sequence.push(p),
            Err(x) => return Err(x)
        }

        if let Some(&test_token) = iterator.peek() {
            match test_token {
                &Token::CloseParen => if is_subpattern {
                    iterator.next();
                } else {
                    return Err(0);
                },
                &Token::CloseBrace => if !is_subpattern {
                    iterator.next();
                    break;
                } else {
                    return Err(0);
                },
                &Token::Slash => {
                    let sub_pattern = parse_choice(grammar, iterator, &mut sequence, is_subpattern);
                    match sub_pattern {
                        Ok(p) => sequence.push(p),
                        Err(x) => return Err(x)
                    }
                },
                _ => { }
            }
        }
    }

    let mut boxed_vec = vec![];
    while let Some(item) = sequence.pop() {
        boxed_vec.push(Box::new(item));
    }

    Ok(ast::Pattern::Sequence(boxed_vec))
}

fn parse_choice<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, patterns : &mut Vec<ast::Pattern>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut boxed_vec = vec![];
    while let Some(item) = patterns.pop() {
        boxed_vec.push(Box::new(item));
    }

    iterator.next();

    let right = parse_pattern(grammar, iterator, is_subpattern);

    match right {
        Ok(p) => Ok(ast::Pattern::Choice(
            Box::new(ast::Pattern::Sequence(boxed_vec)),
            Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_suffix<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, pattern : ast::Pattern) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    if let Some(token) = iterator.next() {
        match token {
            &Token::Asterik => Ok(ast::Pattern::ZeroOrMore(Box::new(pattern))),
            &Token::Plus => Ok(ast::Pattern::OneOrMore(Box::new(pattern))),
            &Token::Question => Ok(ast::Pattern::Optional(Box::new(pattern))),
            _ => Err(0)
        }
    } else {
        Err(0)
    }
}

fn parse_lookahead<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, lookahead : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let sub_pattern = parse_pattern(grammar, iterator, false);
    match sub_pattern {
        Ok(p) => Ok(ast::Pattern::Lookahead(lookahead, Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_char_class<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_char_sequence<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, single_quote : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{

    Err(0)
}

fn parse_variable(grammar : &String, left : usize, right : usize) -> Result<ast::Pattern, u8>
{
    Err(0)
}

<|endoftext|>


<commit_before>//============================================================================
//
//  A Tagged Image File Format (TIFF) Library for Rust
//
//  Based on the TIFF 6.0 specification:
//
//      https://partners.adobe.com/public/developer/en/tiff/TIFF6.pdf
//
//  Copyright (c) 2014 by Gavin Baker <gavinb@antonym.org>
//  Published under the MIT License
//
//============================================================================

use std::io::{Result, Error, ErrorKind, Read, Seek, SeekFrom};
use std::path::Path;
use std::fs::File;

use byteorder::{ReadBytesExt, WriteBytesExt, ByteOrder, BigEndian, LittleEndian};

use {TIFFByteOrder, HeaderMagic, TIFFHeader, IFD, IFDEntry, decode_tag, decode_tag_type, type_and_count_for_tag, SeekableReader, BYTE, SBYTE, SHORT, SSHORT, LONG, SLONG, FLOAT, TagType, TagValue};

pub struct TIFFReader;

impl TIFFReader {

    pub fn load(&self, filename: &str) -> Result<Box<TIFFHeader>> {

        let filepath = Path::new(filename);
        let mut reader = File::open(&filepath).unwrap();

        self.read(&mut reader)
    }

    pub fn read(&self, reader: &mut SeekableReader) -> Result<Box<TIFFHeader>> {

        let byte_order = self.read_byte_order(reader);

        let magic = match byte_order {
            Ok(TIFFByteOrder::LittleEndian) => self.read_magic(reader),
            Ok(TIFFByteOrder::BigEndian) => self.read_magic(reader),
            Err(e) => Err(e)
        };

        self.read_::<BigEndian>(reader)
    }

    pub fn read_byte_order(&self, reader: &mut SeekableReader) -> Result<TIFFByteOrder> {

        // Bytes 0-1: "II" or "MM"
        // Read and validate ByteOrder

        let byte_order_field = try!(reader.read_u16::<LittleEndian>());
        let byte_order: TIFFByteOrder;

        if byte_order_field == TIFFByteOrder::LittleEndian as u16 {
            byte_order = TIFFByteOrder::LittleEndian;
        } else if byte_order_field == TIFFByteOrder::BigEndian as u16 {
            byte_order = TIFFByteOrder::BigEndian;
        } else {
            return Err(Error::new(ErrorKind::Other,
                                  format!("Invalid byte order in header: {:04x}", byte_order_field)));
        }
        println!("byte_order {:?}", byte_order);

        Ok(byte_order)
    }

    pub fn read_magic(&self, reader: &mut SeekableReader) -> Result<HeaderMagic> {

        // Bytes 2-3: 0042
        // Read and validate HeaderMagic

        let magic_field = try!(reader.read_u16::<LittleEndian>());

        let magic: HeaderMagic;

        if magic_field == HeaderMagic::LittleEndian as u16 {
            Ok(HeaderMagic::LittleEndian)
        }
        else if magic_field == HeaderMagic::BigEndian as u16 {
            Ok(HeaderMagic::BigEndian)
        } else {
            Err(Error::new(ErrorKind::Other, "Invalid magic number in header"))
        }
    }

    pub fn read_<Endian: ByteOrder>(&self, reader: &mut SeekableReader) -> Result<Box<TIFFHeader>> {

        // @todo Ensure file is >= min size

        // Bytes 4-7: offset
        // Offset from start of file to first IFD

        let ifd_offset_field = try!(reader.read_u32::<Endian>());

        // Assemble validated header

        let header = Box::new(TIFFHeader {
            byte_order: TIFFByteOrder::LittleEndian,
            magic: HeaderMagic::LittleEndian,
            ifd_offset: ifd_offset_field,
        });

        try!(reader.seek(SeekFrom::Start(ifd_offset_field as u64)));
        println!("IFD offset: {:?}", ifd_offset_field);

        try!(self.read_IFD::<Endian>(reader));

        Ok(header)
    }

    #[allow(non_snake_case)]
    fn read_IFD<Endian: ByteOrder>(&self, reader: &mut SeekableReader) -> Result<Box<IFD>> {

        // 2 byte count of IFD entries
        let entry_count = try!(reader.read_u16::<Endian>());

        println!("IFD entry count: {}", entry_count);

        let mut ifd = Box::new(IFD { count: entry_count, entries: Vec::with_capacity(entry_count as usize) });

        for entry_number in 0..entry_count as usize {
            ifd.entries.push(*self.read_tag::<Endian>(entry_number, reader).unwrap());
        }

        Ok(ifd)
    }

    fn read_tag<Endian: ByteOrder>(&self, entry_number: usize, reader: &mut SeekableReader) -> Result<Box<IFDEntry>> {

        // Bytes 0..1: u16 tag ID
        let tag_value = try!(reader.read_u16::<Endian>());

        // Bytes 2..3: u16 field Type
        let typ_value = try!(reader.read_u16::<Endian>());

        // Bytes 4..7: u32 number of Values of type
        let count_value = try!(reader.read_u32::<Endian>());

        // Bytes 8..11: u32 offset in file to Value
        let value_offset_value = try!(reader.read_u32::<Endian>());

        // Decode tag
        let tag_msg = format!("Invalid tag {:x}", tag_value);
        let tag = decode_tag(tag_value).expect(&tag_msg);

        // Decode type
        let typ_msg = format!("Invalid tag type {:x}", typ_value);
        let typ = decode_tag_type(typ_value).expect(&typ_msg);

        // Create entry
        let mut e0 = Box::new(IFDEntry {
            tag: tag,
            typ: typ,
            count: count_value,
            value_offset: value_offset_value,
            value: None,
        });

        let maybe_tac = type_and_count_for_tag(e0.tag);

        let (expected_typ, expected_count) = maybe_tac.unwrap();

        println!("IFD[{:?}] tag: {:?} type: {:?} count: {} offset: {:08x}",
                 entry_number, e0.tag, e0.typ, e0.count, e0.value_offset);

        let valid_short_or_long = expected_typ == TagType::ShortOrLongTag &&
            (e0.typ == TagType::ShortTag ||
             e0.typ == TagType::LongTag);

        if  ! valid_short_or_long && e0.typ != expected_typ {
            println!("    *** ERROR: expected typ: {:?} found: {:?}", expected_typ, e0.typ);
        }

        if expected_count != 0 && e0.count != expected_count {
            println!("    *** ERROR: expected count: {:?} found: {:?}", expected_count, e0.count);
        }

        /*
            p15: Value/Offset

            To save time and space the Value Offset contains the Value instead
            of pointing to the Value if and only if the Value fits into 4
            bytes. If the Value is shorter than 4 bytes, it is left-justified
            within the 4-byte Value Offset, i.e., stored in the lower- numbered
            bytes. Whether the Value fits within 4 bytes is determined by the
            Type and Count of the field.
        */

        // Try to read values
        if e0.count == 1 {
            e0.value = match e0.typ {
                TagType::ByteTag => Some(TagValue::ByteValue(e0.value_offset as BYTE)),
                TagType::ShortTag => Some(TagValue::ShortValue(e0.value_offset as SHORT)),
                TagType::LongTag => Some(TagValue::LongValue(e0.value_offset)),
                TagType::SignedByteTag => Some(TagValue::SignedByteValue(e0.value_offset as SBYTE)),
                TagType::SignedShortTag => Some(TagValue::SignedShortValue(e0.value_offset as SSHORT)),
                TagType::SignedLongTag => Some(TagValue::SignedLongValue(e0.value_offset as SLONG)),
                TagType::FloatTag => Some(TagValue::FloatValue(e0.value_offset as FLOAT)),
                TagType::ShortOrLongTag => Some(TagValue::LongValue(e0.value_offset as LONG)), // @todo FIXME
                _ => None
            };
        }

        println!("    {:?}", e0.value);

        Ok(e0)
    }
}
<commit_msg>Improve unrecognised tag handling<commit_after>//============================================================================
//
//  A Tagged Image File Format (TIFF) Library for Rust
//
//  Based on the TIFF 6.0 specification:
//
//      https://partners.adobe.com/public/developer/en/tiff/TIFF6.pdf
//
//  Copyright (c) 2014 by Gavin Baker <gavinb@antonym.org>
//  Published under the MIT License
//
//============================================================================

use std::io::{Result, Error, ErrorKind, Read, Seek, SeekFrom};
use std::path::Path;
use std::fs::File;

use byteorder::{ReadBytesExt, WriteBytesExt, ByteOrder, BigEndian, LittleEndian};

use {TIFFByteOrder, HeaderMagic, TIFFHeader, IFD, IFDEntry, decode_tag, decode_tag_type, type_and_count_for_tag, SeekableReader, BYTE, SBYTE, SHORT, SSHORT, LONG, SLONG, FLOAT, TagType, TagValue};

pub struct TIFFReader;

impl TIFFReader {

    pub fn load(&self, filename: &str) -> Result<Box<TIFFHeader>> {

        let filepath = Path::new(filename);
        let mut reader = File::open(&filepath).unwrap();

        self.read(&mut reader)
    }

    pub fn read(&self, reader: &mut SeekableReader) -> Result<Box<TIFFHeader>> {

        let byte_order = self.read_byte_order(reader);

        let magic = match byte_order {
            Ok(TIFFByteOrder::LittleEndian) => self.read_magic(reader),
            Ok(TIFFByteOrder::BigEndian) => self.read_magic(reader),
            Err(e) => Err(e)
        };

        self.read_::<BigEndian>(reader)
    }

    pub fn read_byte_order(&self, reader: &mut SeekableReader) -> Result<TIFFByteOrder> {

        // Bytes 0-1: "II" or "MM"
        // Read and validate ByteOrder

        let byte_order_field = try!(reader.read_u16::<LittleEndian>());
        let byte_order: TIFFByteOrder;

        if byte_order_field == TIFFByteOrder::LittleEndian as u16 {
            byte_order = TIFFByteOrder::LittleEndian;
        } else if byte_order_field == TIFFByteOrder::BigEndian as u16 {
            byte_order = TIFFByteOrder::BigEndian;
        } else {
            return Err(Error::new(ErrorKind::Other,
                                  format!("Invalid byte order in header: {:04x}", byte_order_field)));
        }
        println!("byte_order {:?}", byte_order);

        Ok(byte_order)
    }

    pub fn read_magic(&self, reader: &mut SeekableReader) -> Result<HeaderMagic> {

        // Bytes 2-3: 0042
        // Read and validate HeaderMagic

        let magic_field = try!(reader.read_u16::<LittleEndian>());

        let magic: HeaderMagic;

        if magic_field == HeaderMagic::LittleEndian as u16 {
            Ok(HeaderMagic::LittleEndian)
        }
        else if magic_field == HeaderMagic::BigEndian as u16 {
            Ok(HeaderMagic::BigEndian)
        } else {
            Err(Error::new(ErrorKind::Other, "Invalid magic number in header"))
        }
    }

    pub fn read_<Endian: ByteOrder>(&self, reader: &mut SeekableReader) -> Result<Box<TIFFHeader>> {

        // @todo Ensure file is >= min size

        // Bytes 4-7: offset
        // Offset from start of file to first IFD

        let ifd_offset_field = try!(reader.read_u32::<Endian>());

        // Assemble validated header

        let header = Box::new(TIFFHeader {
            byte_order: TIFFByteOrder::LittleEndian,
            magic: HeaderMagic::LittleEndian,
            ifd_offset: ifd_offset_field,
        });

        try!(reader.seek(SeekFrom::Start(ifd_offset_field as u64)));
        println!("IFD offset: {:?}", ifd_offset_field);

        try!(self.read_IFD::<Endian>(reader));

        Ok(header)
    }

    #[allow(non_snake_case)]
    fn read_IFD<Endian: ByteOrder>(&self, reader: &mut SeekableReader) -> Result<Box<IFD>> {

        // 2 byte count of IFD entries
        let entry_count = try!(reader.read_u16::<Endian>());

        println!("IFD entry count: {}", entry_count);

        let mut ifd = Box::new(IFD { count: entry_count, entries: Vec::with_capacity(entry_count as usize) });

        for entry_number in 0..entry_count as usize {
            let entry = self.read_tag::<Endian>(entry_number, reader);
            match entry {
                Ok(e) => ifd.entries.push(e),
                Err(err) => println!("Invalid tag at index {}: {}", entry_number, err),
            }
        }

        Ok(ifd)
    }

    fn read_tag<Endian: ByteOrder>(&self, entry_number: usize, reader: &mut SeekableReader) -> Result<IFDEntry> {

        // Bytes 0..1: u16 tag ID
        let tag_value = try!(reader.read_u16::<Endian>());

        // Bytes 2..3: u16 field Type
        let typ_value = try!(reader.read_u16::<Endian>());

        // Bytes 4..7: u32 number of Values of type
        let count_value = try!(reader.read_u32::<Endian>());

        // Bytes 8..11: u32 offset in file to Value
        let value_offset_value = try!(reader.read_u32::<Endian>());

        // Decode tag
        let tag_msg = format!("Invalid tag {:x}", tag_value);
        let tag = decode_tag(tag_value).expect(&tag_msg);

        // Decode type
        let typ_msg = format!("Invalid tag type {:x}", typ_value);
        let typ = decode_tag_type(typ_value).expect(&typ_msg);

        // Create entry
        let mut e0 = IFDEntry {
            tag: tag,
            typ: typ,
            count: count_value,
            value_offset: value_offset_value,
            value: None,
        };

        let maybe_tac = type_and_count_for_tag(e0.tag);

        if maybe_tac.is_none() {
            return Err(Error::new(ErrorKind::Other,
                                  format!("Unknown tag {:?} in IFD", e0.tag)));
        }

        let (expected_typ, expected_count) = maybe_tac.unwrap();

        println!("IFD[{:?}] tag: {:?} type: {:?} count: {} offset: {:08x}",
                 entry_number, e0.tag, e0.typ, e0.count, e0.value_offset);

        let valid_short_or_long = expected_typ == TagType::ShortOrLongTag &&
            (e0.typ == TagType::ShortTag ||
             e0.typ == TagType::LongTag);

        if  ! valid_short_or_long && e0.typ != expected_typ {
            println!("    *** ERROR: expected typ: {:?} found: {:?}", expected_typ, e0.typ);
        }

        if expected_count != 0 && e0.count != expected_count {
            println!("    *** ERROR: expected count: {:?} found: {:?}", expected_count, e0.count);
        }

        /*
            p15: Value/Offset

            To save time and space the Value Offset contains the Value instead
            of pointing to the Value if and only if the Value fits into 4
            bytes. If the Value is shorter than 4 bytes, it is left-justified
            within the 4-byte Value Offset, i.e., stored in the lower- numbered
            bytes. Whether the Value fits within 4 bytes is determined by the
            Type and Count of the field.
        */

        // Try to read values
        if e0.count == 1 {
            e0.value = match e0.typ {
                TagType::ByteTag => Some(TagValue::ByteValue(e0.value_offset as BYTE)),
                TagType::ShortTag => Some(TagValue::ShortValue(e0.value_offset as SHORT)),
                TagType::LongTag => Some(TagValue::LongValue(e0.value_offset)),
                TagType::SignedByteTag => Some(TagValue::SignedByteValue(e0.value_offset as SBYTE)),
                TagType::SignedShortTag => Some(TagValue::SignedShortValue(e0.value_offset as SSHORT)),
                TagType::SignedLongTag => Some(TagValue::SignedLongValue(e0.value_offset as SLONG)),
                TagType::FloatTag => Some(TagValue::FloatValue(e0.value_offset as FLOAT)),
                TagType::ShortOrLongTag => Some(TagValue::LongValue(e0.value_offset as LONG)), // @todo FIXME
                _ => None
            };
        }

        println!("    {:?}", e0.value);

        Ok(e0)
    }
}
<|endoftext|>