<commit_before>#![allow(dead_code)]

use regex::Regex;
use std::collections::HashMap;
use lexer;

pub type Token = lexer::Token;
pub type Lexer = lexer::Lexer;

pub struct Parser {
    lexer: Lexer,
    variables: HashMap<String, f64>,
}

impl Parser {
    pub fn new() -> Self {
        Parser{lexer: Lexer::new(), variables: HashMap::new()}
    }
    pub fn parse(&mut self, input: String) -> Result<f64, String> {
        lazy_static! {
            static ref VAR_ASSIGN_RE: Regex = Regex::new(r"\A(\w+)=").unwrap();
            static ref VAR_USAGE_RE: Regex = Regex::new(r"[a-zA-Z]+").unwrap();
            // Theoretical function matching might look like [a-zA-Z]+\( etc...
        }
        let mut internal_input = input.clone();
        let mut new_variable: String = String::default();
        if VAR_ASSIGN_RE.is_match(internal_input.as_ref()) {
            new_variable = String::from(VAR_ASSIGN_RE.captures(internal_input.as_ref()).unwrap().at(1).unwrap());
            internal_input = VAR_ASSIGN_RE.replace(internal_input.as_ref(), "");
        }
        while VAR_USAGE_RE.is_match(internal_input.as_ref()) {
            let variable = String::from(VAR_USAGE_RE.captures(internal_input.as_ref()).unwrap().at(0).unwrap());
            if let Some(value) = self.variables.get(&variable) {
                internal_input = VAR_USAGE_RE.replace(internal_input.as_ref(), &(value.to_string())[..]);
            } else {
                let mut msg = String::from("Unknown variable: ");
                msg.push_str(variable.as_ref());
                return Err(msg);
            }
        }
        self.lexer.set_input(internal_input);
        let expression_value: f64 = match self.expression() {
            Ok(v) => v,
            Err(msg) => return Err(msg)
        };
        let token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        match token.get_kind() {
            lexer::SubToken::End => {
                if new_variable != String::default() {
                    self.variables.insert(new_variable, expression_value);
                }
                return Ok(expression_value);
            }
            _ => {
                return Err(String::from("End expected"));
            }
        }
    }
    pub fn expression(&mut self) -> Result<f64, String> {
        let mut component1: f64 = match self.factor() {
            Ok(x) => x,
            Err(msg) => return Err(msg)
        };
        let mut token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        loop {
            match token.get_kind() {
                lexer::SubToken::Plus => {
                    let component2 = match self.factor() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    component1 += component2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Minus => {
                    let component2 = match self.factor() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    component1 -= component2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                _ => {
                    break;
                }
            }
        }
        self.lexer.revert();
        Ok(component1)
    }
    pub fn factor(&mut self) -> Result<f64, String> {
        let mut factor1: f64 = match self.number() {
            Ok(x) => x,
            Err(msg) => return Err(msg)
        };
        let mut token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        loop {
            match token.get_kind() {
                lexer::SubToken::Multiply => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 *= factor2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Divide => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 /= factor2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Power => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 = factor1.powf(factor2);
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Modulo => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 = factor1 % factor2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                _ => {
                    break;
                }
            }
        }
        self.lexer.revert();
        Ok(factor1)
    }
    pub fn number(&mut self) -> Result<f64, String> {
        let token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        let value: f64;
        match token.get_kind() {
            lexer::SubToken::LParen => {
                value = match self.expression() {
                    Ok(v) => v,
                    Err(msg) => return Err(msg)
                };
                match self.lexer.get_next_token() {
                    Err(msg) => return Err(msg),
                    _ => ()
                };
            }
            lexer::SubToken::Number => {
                value = match token.get_value() {
                    Some(v) => v,
                    None => return Err(String::from("No value"))
                };
            }
            lexer::SubToken::Minus => {
                value = match self.factor() {
                    Ok(x) => x*-1.0,
                    Err(msg) => return Err(msg)
                };
            }
            _ => {
                return Err(String::from("Not a number"));
            }
        }
        Ok(value)
    }
}<commit_msg>Allow for variables with underscores<commit_after>#![allow(dead_code)]

use regex::Regex;
use std::collections::HashMap;
use lexer;

pub type Token = lexer::Token;
pub type Lexer = lexer::Lexer;

pub struct Parser {
    lexer: Lexer,
    variables: HashMap<String, f64>,
}

impl Parser {
    pub fn new() -> Self {
        Parser{lexer: Lexer::new(), variables: HashMap::new()}
    }
    pub fn parse(&mut self, input: String) -> Result<f64, String> {
        lazy_static! {
            static ref VAR_ASSIGN_RE: Regex = Regex::new(r"\A(([a-zA-Z]|_)+)=").unwrap();
            static ref VAR_USAGE_RE: Regex = Regex::new(r"([a-zA-Z]|_)+").unwrap();
            // Theoretical function matching might look like [a-zA-Z]+\( etc...
        }
        let mut internal_input = input.clone();
        let mut new_variable: String = String::default();
        if VAR_ASSIGN_RE.is_match(internal_input.as_ref()) {
            new_variable = String::from(VAR_ASSIGN_RE.captures(internal_input.as_ref()).unwrap().at(1).unwrap());
            internal_input = VAR_ASSIGN_RE.replace(internal_input.as_ref(), "");
        }
        while VAR_USAGE_RE.is_match(internal_input.as_ref()) {
            let variable = String::from(VAR_USAGE_RE.captures(internal_input.as_ref()).unwrap().at(0).unwrap());
            if let Some(value) = self.variables.get(&variable) {
                internal_input = VAR_USAGE_RE.replace(internal_input.as_ref(), &(value.to_string())[..]);
            } else {
                let mut msg = String::from("Unknown variable: ");
                msg.push_str(variable.as_ref());
                return Err(msg);
            }
        }
        self.lexer.set_input(internal_input);
        let expression_value: f64 = match self.expression() {
            Ok(v) => v,
            Err(msg) => return Err(msg)
        };
        let token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        match token.get_kind() {
            lexer::SubToken::End => {
                if new_variable != String::default() {
                    self.variables.insert(new_variable, expression_value);
                }
                return Ok(expression_value);
            }
            _ => {
                return Err(String::from("End expected"));
            }
        }
    }
    pub fn expression(&mut self) -> Result<f64, String> {
        let mut component1: f64 = match self.factor() {
            Ok(x) => x,
            Err(msg) => return Err(msg)
        };
        let mut token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        loop {
            match token.get_kind() {
                lexer::SubToken::Plus => {
                    let component2 = match self.factor() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    component1 += component2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Minus => {
                    let component2 = match self.factor() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    component1 -= component2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                _ => {
                    break;
                }
            }
        }
        self.lexer.revert();
        Ok(component1)
    }
    pub fn factor(&mut self) -> Result<f64, String> {
        let mut factor1: f64 = match self.number() {
            Ok(x) => x,
            Err(msg) => return Err(msg)
        };
        let mut token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        loop {
            match token.get_kind() {
                lexer::SubToken::Multiply => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 *= factor2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Divide => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 /= factor2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Power => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 = factor1.powf(factor2);
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                lexer::SubToken::Modulo => {
                    let factor2: f64 = match self.number() {
                        Ok(x) => x,
                        Err(msg) => return Err(msg)
                    };
                    factor1 = factor1 % factor2;
                    token = match self.lexer.get_next_token() {
                        Ok(t) => t,
                        Err(msg) => return Err(msg)
                    };
                }
                _ => {
                    break;
                }
            }
        }
        self.lexer.revert();
        Ok(factor1)
    }
    pub fn number(&mut self) -> Result<f64, String> {
        let token: Token = match self.lexer.get_next_token() {
            Ok(t) => t,
            Err(msg) => return Err(msg)
        };
        let value: f64;
        match token.get_kind() {
            lexer::SubToken::LParen => {
                value = match self.expression() {
                    Ok(v) => v,
                    Err(msg) => return Err(msg)
                };
                match self.lexer.get_next_token() {
                    Err(msg) => return Err(msg),
                    _ => ()
                };
            }
            lexer::SubToken::Number => {
                value = match token.get_value() {
                    Some(v) => v,
                    None => return Err(String::from("No value"))
                };
            }
            lexer::SubToken::Minus => {
                value = match self.factor() {
                    Ok(x) => x*-1.0,
                    Err(msg) => return Err(msg)
                };
            }
            _ => {
                return Err(String::from("Not a number"));
            }
        }
        Ok(value)
    }
}<|endoftext|>


<commit_before>extern crate byteorder;
extern crate encoding;

use std::borrow::Cow;
use std::error;
use std::fmt;
use std::io;

use self::byteorder::{ByteOrder, BigEndian, LittleEndian};
use self::encoding::label::encoding_from_whatwg_label;
use self::encoding::types::EncodingRef;
use self::encoding::types::DecoderTrap::Strict;

use super::{Catalog, Message};
use metadata::parse_metadata;

#[allow(non_upper_case_globals)]
static utf8_encoding: EncodingRef = &encoding::codec::utf_8::UTF8Encoding;

/// Represents an error encountered while parsing an MO file.
#[derive(Debug)]
pub enum Error {
    /// An incorrect magic number has been encountered
    BadMagic,
    /// An invalid byte sequence for the given encoding has been encountered
    DecodingError,
    /// An unexpected EOF occured
    Eof,
    /// An I/O error occured
    Io(io::Error),
    /// Incorrect syntax encountered while parsing the meta information
    MalformedMetadata,
    /// Meta information string was not the first string in the catalog
    MisplacedMetadata,
    /// An unknown encoding was specified in the metadata
    UnknownEncoding,
}
// Can not use use `Error::*` as per this issue:
// (https://github.com/rust-lang/rust/issues/4865)
use Error::{BadMagic, DecodingError, Eof, Io, MalformedMetadata, MisplacedMetadata,
            UnknownEncoding};

impl error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            BadMagic => "bad magic number",
            DecodingError => "invalid byte sequence in a string",
            Eof => "unxpected end of file",
            Io(ref err) => err.description(),
            MalformedMetadata => "metadata syntax error",
            MisplacedMetadata => "misplaced metadata",
            UnknownEncoding => "unknown encoding specified",
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let self_err: &error::Error = self;
        write!(fmt, "{}", self_err.description())
    }
}

impl From<io::Error> for Error {
    fn from(inner: io::Error) -> Error {
        Io(inner)
    }
}

impl From<Cow<'static, str>> for Error {
    fn from(_: Cow<'static, str>) -> Error {
        DecodingError
    }
}

/// ParseOptions allows setting options for parsing MO catalogs.
///
/// # Examples
/// ```ignore
/// use std::fs::File;
///
/// extern crate encoding;
/// use encoding::all::ISO_8859_1;
///
/// let file = File::open("french.mo").unwrap();
/// let catalog = ParseOptions::new().force_encoding(ISO_8859_1).parse(file).unwrap();
/// ```
#[allow(missing_debug_implementations)]
pub struct ParseOptions {
    force_encoding: Option<EncodingRef>,
}

impl ParseOptions {
    /// Returns a new instance of ParseOptions with default options.
    pub fn new() -> Self {
        ParseOptions { force_encoding: None }
    }

    /// Tries to parse the catalog from the given reader using the specified options.
    pub fn parse<R: io::Read>(&self, reader: R) -> Result<Catalog, Error> {
        parse_catalog(reader, self)
    }

    /// Forces a use of a specific encoding
    /// when parsing strings from a catalog.
    /// If this option is not enabled,
    /// the parser tries to use the encoding specified in the metadata
    /// or UTF-8 if metadata is non-existent.
    pub fn force_encoding(&mut self, encoding: EncodingRef) -> &mut Self {
        self.force_encoding = Some(encoding);
        self
    }
}

/// According to the given magic number of a MO file,
/// returns the function which reads a `u32` in the relevant endianness.
fn get_read_u32_fn(magic: &[u8]) -> Option<fn(&[u8]) -> u32> {
    if magic == [0xde, 0x12, 0x04, 0x95] {
        Some(LittleEndian::read_u32)
    } else if magic == [0x95, 0x04, 0x12, 0xde] {
        Some(BigEndian::read_u32)
    } else {
        None
    }
}

pub fn parse_catalog<R: io::Read>(mut file: R, opts: &ParseOptions) -> Result<Catalog, Error> {
    let mut contents = vec![];
    let n = try!(file.read_to_end(&mut contents));
    if n < 28 {
        return Err(Eof);
    }

    let read_u32 = match get_read_u32_fn(&contents[0..4]) {
        Some(f) => f,
        None => return Err(BadMagic),
    };

    // ignore hashing tables (bytes at 20..28)
    let num_strings = read_u32(&contents[8..12]) as usize;
    let mut off_otable = read_u32(&contents[12..16]) as usize;
    let mut off_ttable = read_u32(&contents[16..20]) as usize;
    if n < off_otable || n < off_ttable {
        return Err(Eof);
    }

    let mut catalog = Catalog::new();
    let mut encoding = opts.force_encoding.unwrap_or(utf8_encoding);

    for i in 0..num_strings {
        let id;
        let context;
        let translated: Vec<String>;
        // Parse the original string
        {
            if n < off_otable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_otable..off_otable + 4]) as usize;
            let off = read_u32(&contents[off_otable + 4..off_otable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            let mut original = &contents[off..off + len + 1];
            // check for context
            context = match original.iter().position(|x| *x == 4) {
                Some(idx) => {
                    let ctx = &original[..idx];
                    original = &original[idx + 1..];
                    Some(try!(encoding.decode(ctx, Strict)))
                }
                None => None,
            };
            // extract msg_id singular, ignoring the plural
            id = match original.iter().position(|x| *x == 0).map(|i| &original[..i]) {
                Some(b) => try!(encoding.decode(b, Strict)),
                None => return Err(Eof),
            };
            if id == "" && i != 0 {
                return Err(MisplacedMetadata);
            }
        }

        // Parse the translation strings
        {
            if n < off_ttable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_ttable..off_ttable + 4]) as usize;
            let off = read_u32(&contents[off_ttable + 4..off_ttable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            translated = try!((&contents[off..off + len])
                                  .split(|x| *x == 0)
                                  .map(|b| encoding.decode(b, Strict))
                                  .collect::<Result<Vec<_>, _>>());
            if id == "" {
                let map = parse_metadata(&*translated[0]).unwrap();
                if let (Some(c), None) = (map.charset(), opts.force_encoding) {
                    encoding = match encoding_from_whatwg_label(c) {
                        Some(enc_ref) => enc_ref,
                        None => return Err(UnknownEncoding),
                    }
                }
            }
        }

        catalog.insert(Message::new(id, context, translated));

        off_otable += 8;
        off_ttable += 8;
    }

    Ok(catalog)
}

#[test]
fn test_get_read_u32_fn() {
    use std::mem;

    assert!(get_read_u32_fn(&[]).is_none());
    assert!(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95, 0x00]).is_none());

    {
        let le_ptr: *const ();
        let ret_ptr;
        unsafe {
            le_ptr = mem::transmute(LittleEndian::read_u32 as usize);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95]).unwrap());
        }
        assert_eq!(le_ptr, ret_ptr);
    }

    {
        let be_ptr: *const ();
        let ret_ptr;
        unsafe {
            be_ptr = mem::transmute(BigEndian::read_u32 as usize);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0x95, 0x04, 0x12, 0xde]).unwrap());
        }
        assert_eq!(be_ptr, ret_ptr);
    }
}

#[test]
fn test_parse_catalog() {
    macro_rules! assert_variant {
        ($value:expr, $variant:path) => {
            match $value {
                $variant => (),
                _ => panic!("Expected {:?}, got {:?}", $variant, $value),
            }
        };
    }

    let fluff = [0; 24]; // zeros to pad our magic test cases to satisfy the length requirements

    {
        let mut reader = vec![1u8, 2, 3];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..], &ParseOptions::new()).unwrap_err();
        assert_variant!(err, Eof);
    }

    {
        let mut reader = vec![1u8, 2, 3, 4];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..], &ParseOptions::new()).unwrap_err();
        assert_variant!(err, BadMagic);
    }

    {
        let mut reader = vec![0x95, 0x04, 0x12, 0xde];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..], &ParseOptions::new()).is_ok());
    }

    {
        let mut reader = vec![0xde, 0x12, 0x04, 0x95];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..], &ParseOptions::new()).is_ok());
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/1.mo");
        let catalog = parse_catalog(reader, &ParseOptions::new()).unwrap();
        assert_eq!(catalog.strings.len(), 1);
        assert_eq!(catalog.strings["this is context\x04Text"],
                   Message::new("Text", Some("this is context"), vec!["Tekstas", "Tekstai"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/2.mo");
        let catalog = parse_catalog(reader, &ParseOptions::new()).unwrap();
        assert_eq!(catalog.strings.len(), 2);
        assert_eq!(catalog.strings["Image"],
                   Message::new("Image", None, vec!["Nuotrauka", "Nuotraukos"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/invalid_utf8.mo");
        let err = parse_catalog(reader, &ParseOptions::new()).unwrap_err();
        assert_variant!(err, DecodingError);
    }

    // cp1257_meta
    {
        let reader: &[u8] = include_bytes!("../test_cases/cp1257_meta.mo");
        let catalog = parse_catalog(reader, &ParseOptions::new()).unwrap();
        assert_eq!(catalog.gettext("Garlic"), "Česnakas");
    }

    // cp1257_forced
    {
        let reader: &[u8] = include_bytes!("../test_cases/cp1257_forced.mo");
        for enc_name in &["cp1257", "windows-1257", "x-cp1257"] {
            let encoding = encoding_from_whatwg_label(enc_name).unwrap();
            let catalog = ParseOptions::new().force_encoding(encoding).parse(reader).unwrap();
            assert_eq!(catalog.gettext("Garlic"), "Česnakas");
        }
    }
}
<commit_msg>ParseOptions::force_plural<commit_after>extern crate byteorder;
extern crate encoding;

use std::borrow::Cow;
use std::default::Default;
use std::error;
use std::fmt;
use std::io;

use self::byteorder::{ByteOrder, BigEndian, LittleEndian};
use self::encoding::label::encoding_from_whatwg_label;
use self::encoding::types::EncodingRef;
use self::encoding::types::DecoderTrap::Strict;

use super::{Catalog, Message};
use metadata::parse_metadata;

#[allow(non_upper_case_globals)]
static utf8_encoding: EncodingRef = &encoding::codec::utf_8::UTF8Encoding;

/// Represents an error encountered while parsing an MO file.
#[derive(Debug)]
pub enum Error {
    /// An incorrect magic number has been encountered
    BadMagic,
    /// An invalid byte sequence for the given encoding has been encountered
    DecodingError,
    /// An unexpected EOF occured
    Eof,
    /// An I/O error occured
    Io(io::Error),
    /// Incorrect syntax encountered while parsing the meta information
    MalformedMetadata,
    /// Meta information string was not the first string in the catalog
    MisplacedMetadata,
    /// An unknown encoding was specified in the metadata
    UnknownEncoding,
}
// Can not use use `Error::*` as per this issue:
// (https://github.com/rust-lang/rust/issues/4865)
use Error::{BadMagic, DecodingError, Eof, Io, MalformedMetadata, MisplacedMetadata,
            UnknownEncoding};

impl error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            BadMagic => "bad magic number",
            DecodingError => "invalid byte sequence in a string",
            Eof => "unxpected end of file",
            Io(ref err) => err.description(),
            MalformedMetadata => "metadata syntax error",
            MisplacedMetadata => "misplaced metadata",
            UnknownEncoding => "unknown encoding specified",
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let self_err: &error::Error = self;
        write!(fmt, "{}", self_err.description())
    }
}

impl From<io::Error> for Error {
    fn from(inner: io::Error) -> Error {
        Io(inner)
    }
}

impl From<Cow<'static, str>> for Error {
    fn from(_: Cow<'static, str>) -> Error {
        DecodingError
    }
}

/// ParseOptions allows setting options for parsing MO catalogs.
///
/// # Examples
/// ```ignore
/// use std::fs::File;
///
/// extern crate encoding;
/// use encoding::all::ISO_8859_1;
///
/// let file = File::open("french.mo").unwrap();
/// let catalog = ParseOptions::new().force_encoding(ISO_8859_1).parse(file).unwrap();
/// ```
#[allow(missing_debug_implementations)]
#[derive(Default)]
pub struct ParseOptions {
    force_encoding: Option<EncodingRef>,
    force_plural: Option<Box<Fn(u64) -> usize + 'static>>,
}

impl ParseOptions {
    /// Returns a new instance of ParseOptions with default options.
    pub fn new() -> Self {
        Default::default()
    }

    /// Tries to parse the catalog from the given reader using the specified options.
    pub fn parse<R: io::Read>(&self, reader: R) -> Result<Catalog, Error> {
        parse_catalog(reader, self)
    }

    /// Forces a use of a specific encoding
    /// when parsing strings from a catalog.
    /// If this option is not enabled,
    /// the parser tries to use the encoding specified in the metadata
    /// or UTF-8 if metadata is non-existent.
    pub fn force_encoding(&mut self, encoding: EncodingRef) -> &mut Self {
        self.force_encoding = Some(encoding);
        self
    }

    /// Forces a use of the given plural formula
    /// for deciding the proper plural form for a message.
    /// If this option is not enabled,
    /// the parser tries to use the plural formula
    /// specified in the catalog,
    /// falling back on `n != 1`
    /// in case it fails to find or parse one.
    pub fn force_plural<T: Fn(u64) -> usize + 'static>(&mut self, plural: T) -> &mut Self {
        self.force_plural = Some(Box::new(plural));
        self
    }
}

/// According to the given magic number of a MO file,
/// returns the function which reads a `u32` in the relevant endianness.
fn get_read_u32_fn(magic: &[u8]) -> Option<fn(&[u8]) -> u32> {
    if magic == [0xde, 0x12, 0x04, 0x95] {
        Some(LittleEndian::read_u32)
    } else if magic == [0x95, 0x04, 0x12, 0xde] {
        Some(BigEndian::read_u32)
    } else {
        None
    }
}

pub fn parse_catalog<R: io::Read>(mut file: R, opts: &ParseOptions) -> Result<Catalog, Error> {
    let mut contents = vec![];
    let n = try!(file.read_to_end(&mut contents));
    if n < 28 {
        return Err(Eof);
    }

    let read_u32 = match get_read_u32_fn(&contents[0..4]) {
        Some(f) => f,
        None => return Err(BadMagic),
    };

    // ignore hashing tables (bytes at 20..28)
    let num_strings = read_u32(&contents[8..12]) as usize;
    let mut off_otable = read_u32(&contents[12..16]) as usize;
    let mut off_ttable = read_u32(&contents[16..20]) as usize;
    if n < off_otable || n < off_ttable {
        return Err(Eof);
    }

    let mut catalog = Catalog::new();
    let mut encoding = opts.force_encoding.unwrap_or(utf8_encoding);

    for i in 0..num_strings {
        let id;
        let context;
        let translated: Vec<String>;
        // Parse the original string
        {
            if n < off_otable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_otable..off_otable + 4]) as usize;
            let off = read_u32(&contents[off_otable + 4..off_otable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            let mut original = &contents[off..off + len + 1];
            // check for context
            context = match original.iter().position(|x| *x == 4) {
                Some(idx) => {
                    let ctx = &original[..idx];
                    original = &original[idx + 1..];
                    Some(try!(encoding.decode(ctx, Strict)))
                }
                None => None,
            };
            // extract msg_id singular, ignoring the plural
            id = match original.iter().position(|x| *x == 0).map(|i| &original[..i]) {
                Some(b) => try!(encoding.decode(b, Strict)),
                None => return Err(Eof),
            };
            if id == "" && i != 0 {
                return Err(MisplacedMetadata);
            }
        }

        // Parse the translation strings
        {
            if n < off_ttable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_ttable..off_ttable + 4]) as usize;
            let off = read_u32(&contents[off_ttable + 4..off_ttable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            translated = try!((&contents[off..off + len])
                                  .split(|x| *x == 0)
                                  .map(|b| encoding.decode(b, Strict))
                                  .collect::<Result<Vec<_>, _>>());
            if id == "" {
                let map = parse_metadata(&*translated[0]).unwrap();
                if let (Some(c), None) = (map.charset(), opts.force_encoding) {
                    encoding = match encoding_from_whatwg_label(c) {
                        Some(enc_ref) => enc_ref,
                        None => return Err(UnknownEncoding),
                    }
                }
            }
        }

        catalog.insert(Message::new(id, context, translated));

        off_otable += 8;
        off_ttable += 8;
    }

    Ok(catalog)
}

#[test]
fn test_get_read_u32_fn() {
    use std::mem;

    assert!(get_read_u32_fn(&[]).is_none());
    assert!(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95, 0x00]).is_none());

    {
        let le_ptr: *const ();
        let ret_ptr;
        unsafe {
            le_ptr = mem::transmute(LittleEndian::read_u32 as usize);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95]).unwrap());
        }
        assert_eq!(le_ptr, ret_ptr);
    }

    {
        let be_ptr: *const ();
        let ret_ptr;
        unsafe {
            be_ptr = mem::transmute(BigEndian::read_u32 as usize);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0x95, 0x04, 0x12, 0xde]).unwrap());
        }
        assert_eq!(be_ptr, ret_ptr);
    }
}

#[test]
fn test_parse_catalog() {
    macro_rules! assert_variant {
        ($value:expr, $variant:path) => {
            match $value {
                $variant => (),
                _ => panic!("Expected {:?}, got {:?}", $variant, $value),
            }
        };
    }

    let fluff = [0; 24]; // zeros to pad our magic test cases to satisfy the length requirements

    {
        let mut reader = vec![1u8, 2, 3];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..], &ParseOptions::new()).unwrap_err();
        assert_variant!(err, Eof);
    }

    {
        let mut reader = vec![1u8, 2, 3, 4];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..], &ParseOptions::new()).unwrap_err();
        assert_variant!(err, BadMagic);
    }

    {
        let mut reader = vec![0x95, 0x04, 0x12, 0xde];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..], &ParseOptions::new()).is_ok());
    }

    {
        let mut reader = vec![0xde, 0x12, 0x04, 0x95];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..], &ParseOptions::new()).is_ok());
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/1.mo");
        let catalog = parse_catalog(reader, &ParseOptions::new()).unwrap();
        assert_eq!(catalog.strings.len(), 1);
        assert_eq!(catalog.strings["this is context\x04Text"],
                   Message::new("Text", Some("this is context"), vec!["Tekstas", "Tekstai"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/2.mo");
        let catalog = parse_catalog(reader, &ParseOptions::new()).unwrap();
        assert_eq!(catalog.strings.len(), 2);
        assert_eq!(catalog.strings["Image"],
                   Message::new("Image", None, vec!["Nuotrauka", "Nuotraukos"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/invalid_utf8.mo");
        let err = parse_catalog(reader, &ParseOptions::new()).unwrap_err();
        assert_variant!(err, DecodingError);
    }

    // cp1257_meta
    {
        let reader: &[u8] = include_bytes!("../test_cases/cp1257_meta.mo");
        let catalog = parse_catalog(reader, &ParseOptions::new()).unwrap();
        assert_eq!(catalog.gettext("Garlic"), "Česnakas");
    }

    // cp1257_forced
    {
        let reader: &[u8] = include_bytes!("../test_cases/cp1257_forced.mo");
        for enc_name in &["cp1257", "windows-1257", "x-cp1257"] {
            let encoding = encoding_from_whatwg_label(enc_name).unwrap();
            let catalog = ParseOptions::new().force_encoding(encoding).parse(reader).unwrap();
            assert_eq!(catalog.gettext("Garlic"), "Česnakas");
        }
    }
}
<|endoftext|>


<commit_before>use std::str::from_utf8;
use std::result::Result;
use nom::*;

/*
 * Core structs
 */

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     &'a str,
    pub mode:     &'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   &'a str,
    pub typeflag: TypeFlag,
    pub linkname: &'a str,
    pub ustar:    ExtraHeader<'a>
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    PaxInterexchangeFormat,
    PaxExtendedAttributes,
    VendorSpecific
}

#[derive(Debug,PartialEq,Eq)]
pub enum ExtraHeader<'a> {
    UStar(UStarHeader<'a>),
    Padding
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    &'a str,
    pub version:  &'a str,
    pub uname:    &'a str,
    pub gname:    &'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub extra:    UStarExtraHeader<'a>
}

#[derive(Debug,PartialEq,Eq)]
pub enum UStarExtraHeader<'a> {
    PosixUStar(PosixUStarHeader<'a>),
    Pax(PaxHeader<'a>)
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixUStarHeader<'a> {
    pub prefix: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PaxHeader<'a> {
    pub atime:      u64,
    pub ctime:      u64,
    pub offset:     u64,
    pub longnames:  &'a str,
    pub sparses:    Vec<Sparse>,
    pub isextended: bool,
    pub realsize:   u64,
}

#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub struct Sparse {
    pub offset:   u64,
    pub numbytes: u64
}

#[derive(Debug,PartialEq,Eq)]
pub struct Padding;

/*
 * Useful macros
 */

macro_rules! take_str_eat_garbage (
    ( $i:expr, $size:expr ) => (
        chain!($i,
            s: map_res!(take_until!("\0"), from_utf8) ~
            take!($size - s.len()),
            ||{
                s
            }
        )
    );
);

named!(parse_str4<&[u8], &str>, take_str_eat_garbage!(4));
named!(parse_str8<&[u8], &str>, take_str_eat_garbage!(8));
named!(parse_str32<&[u8], &str>, take_str_eat_garbage!(32));
named!(parse_str100<&[u8], &str>, take_str_eat_garbage!(100));
named!(parse_str155<&[u8], &str>, take_str_eat_garbage!(155));

macro_rules! take_until_expr_with_limit_consume(
  ($i:expr, $submac:ident!( $($args:tt)* ), $stop: expr, $limit: expr) => (
    {
      let mut begin = 0;
      let mut remaining = $i.len();
      let mut res = Vec::new();
      let mut cnt = 0;
      let mut err = false;
      let mut append = true;
      loop {
        match $submac!(&$i[begin..], $($args)*) {
          IResult::Done(i,o) => {
            if append {
              if $stop(o) {
                append = false;
              } else {
                res.push(o);
              }
            }
            begin += remaining - i.len();
            remaining = i.len();
            cnt = cnt + 1;
            if cnt == $limit {
              break
            }
          },
          IResult::Error(_)  => {
            err = true;
            break;
          },
          IResult::Incomplete(_) => {
            break;
          }
        }
      }
      if err {
        IResult::Error(Err::Position(ErrorCode::Count as u32,$i))
      } else if cnt == $limit {
        IResult::Done(&$i[begin..], res)
      } else {
        IResult::Incomplete(Needed::Unknown)
      }
    }
  );
  ($i:expr, $f:expr, $stop: expr, $limit: expr) => (
    take_until_expr_with_limit_consume!($i, call!($f), $stop, $limit);
  );
);

/*
 * Octal string parsing
 */

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;

    for c in s.chars() {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u *= 8;
        u += (c as u64) - ('0' as u64);
    }

    Ok(u)
}

fn parse_octal(i: &[u8], n: usize) -> IResult<&[u8], u64> {
    map_res!(i, take_str_eat_garbage!(n), octal_to_u64)
}

named!(parse_octal8<&[u8], u64>, apply!(parse_octal, 8));
named!(parse_octal12<&[u8], u64>, apply!(parse_octal, 12));

/*
 * TypeFlag parsing
 */

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::PaxInterexchangeFormat,
        'x' => TypeFlag::PaxExtendedAttributes,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::NormalFile
    }
}

fn bytes_to_type_flag(i: &[u8]) -> Result<TypeFlag, &'static str> {
    Ok(char_to_type_flag(i[0] as char))
}

named!(parse_type_flag<&[u8], TypeFlag>, map_res!(take!(1), bytes_to_type_flag));

/*
 * Sparse parsing
 */

fn parse_one_sparse(i: &[u8]) -> IResult<&[u8], Sparse> {
    chain!(i,
        offset:   parse_octal12 ~
        numbytes: parse_octal12,
        ||{
            Sparse {
                offset:   offset,
                numbytes: numbytes
            }
        }
    )
}

fn parse_sparses_with_limit(i: &[u8], limit: usize) -> IResult<&[u8], Vec<Sparse>> {
    take_until_expr_with_limit_consume!(i, parse_one_sparse, |s: Sparse| s.offset == 0 && s.numbytes == 0, limit)
}

fn add_to_vec(extra: Vec<Sparse>, sparses: &mut Vec<Sparse>) -> Result<&'static str, &'static str> {
    for sparse in &extra {
        sparses.push(*sparse);
    }
    Ok("")
}

fn parse_extra_sparses<'a>(i: &'a [u8], isextended: bool, sparses: &'a mut Vec<Sparse>) -> IResult<'a, &'a [u8], &'a mut Vec<Sparse>> {
    if isextended {
        chain!(i,
            map_res!(apply!(parse_sparses_with_limit, 21), apply!(add_to_vec, sparses)) ~
            extended:      parse_bool                                                   ~
            take!(7) /* padding to 512 */                                               ~
            extra_sparses: apply!(parse_extra_sparses, extended, sparses),
            ||{
                extra_sparses
            }
        )
    } else {
        IResult::Done(i, sparses)
    }
}

/*
 * Boolean parsing
 */

fn to_bool(i: &[u8]) -> Result<bool, &'static str> {
    Ok(i[0] != 0)
}

named!(parse_bool<&[u8], bool>, map_res!(take!(1), to_bool));

/*
 * UStar PAX extended parsing
 */

fn parse_ustar00_extra_pax(i: &[u8]) -> IResult<&[u8], PaxHeader> {
    chain!(i,
        atime:       parse_octal12                       ~
        ctime:       parse_octal12                       ~
        offset:      parse_octal12                       ~
        longnames:   parse_str4                          ~
        take!(1)                                         ~
        sparses:     apply!(parse_sparses_with_limit, 4) ~
        isextended:  parse_bool                          ~
        realsize:    parse_octal12                       ~
        take!(17), /* padding to 512 */
        ||{
            PaxHeader {
                atime:         atime,
                ctime:         ctime,
                offset:        offset,
                longnames:     longnames,
                sparses:       sparses,
                isextended:    isextended,
                realsize:      realsize,
            }
        }
    )
}

/*
 * UStar Posix parsing
 */

fn parse_ustar00_extra_posix(i: &[u8]) -> IResult<&[u8], UStarExtraHeader> {
    chain!(i,
        prefix: parse_str155 ~
        take!(12),
        ||{
            UStarExtraHeader::PosixUStar(PosixUStarHeader {
                prefix: prefix
            })
        }
    )
}

fn parse_ustar00_extra(i: &[u8], flag: TypeFlag) -> IResult<&[u8], UStarExtraHeader> {
    match flag {
        TypeFlag::PaxInterexchangeFormat => {
            chain!(i,
                header: parse_ustar00_extra_pax ~
                apply!(parse_extra_sparses, header.isextended, &mut header.sparses),
                ||{
                    UStarExtraHeader::Pax(header)
                }
            )
        },
        _ => parse_ustar00_extra_posix(i)
    }
}

fn parse_ustar00(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("00")             ~
        uname:    parse_str32  ~
        gname:    parse_str32  ~
        devmajor: parse_octal8 ~
        devminor: parse_octal8 ~
        extra:    apply!(parse_ustar00_extra, flag),
        ||{
            ExtraHeader::UStar(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                extra:    extra
            })
        }
    )
}

fn parse_ustar(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: apply!(parse_ustar00, flag),
        ||{
            ustar
        }
    )
}

/*
 * Posix tar archive header parsing
 */

fn parse_posix(i: &[u8]) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            ExtraHeader::Padding
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     parse_str100    ~
        mode:     parse_str8      ~
        uid:      parse_octal8    ~
        gid:      parse_octal8    ~
        size:     parse_octal12   ~
        mtime:    parse_octal12   ~
        chksum:   parse_str8      ~
        typeflag: parse_type_flag ~
        linkname: parse_str100    ~
        ustar:    alt!(apply!(parse_ustar, typeflag) | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: typeflag,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

/*
 * Contents parsing
 */

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

/*
 * Tar entry header + contents parsing
 */

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

/*
 * Tar archive parsing
 */

fn filter_entries(entries: Vec<TarEntry>) -> Result<Vec<TarEntry>, &'static str> {
    /* Filter out empty entries */
    Ok(entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>())
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    chain!(i,
        entries: map_res!(many0!(parse_entry), filter_entries) ~
        eof,
        ||{
            entries
        }
    )
}

/*
 * Tests
 */

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<commit_msg>sparses: make it work<commit_after>use std::str::from_utf8;
use std::result::Result;
use nom::*;

/*
 * Core structs
 */

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     &'a str,
    pub mode:     &'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   &'a str,
    pub typeflag: TypeFlag,
    pub linkname: &'a str,
    pub ustar:    ExtraHeader<'a>
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    PaxInterexchangeFormat,
    PaxExtendedAttributes,
    VendorSpecific
}

#[derive(Debug,PartialEq,Eq)]
pub enum ExtraHeader<'a> {
    UStar(UStarHeader<'a>),
    Padding
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    &'a str,
    pub version:  &'a str,
    pub uname:    &'a str,
    pub gname:    &'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub extra:    UStarExtraHeader<'a>
}

#[derive(Debug,PartialEq,Eq)]
pub enum UStarExtraHeader<'a> {
    PosixUStar(PosixUStarHeader<'a>),
    Pax(PaxHeader<'a>)
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixUStarHeader<'a> {
    pub prefix: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PaxHeader<'a> {
    pub atime:      u64,
    pub ctime:      u64,
    pub offset:     u64,
    pub longnames:  &'a str,
    pub sparses:    Vec<Sparse>,
    pub isextended: bool,
    pub realsize:   u64,
}

#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub struct Sparse {
    pub offset:   u64,
    pub numbytes: u64
}

#[derive(Debug,PartialEq,Eq)]
pub struct Padding;

/*
 * Useful macros
 */

macro_rules! take_str_eat_garbage (
    ( $i:expr, $size:expr ) => (
        chain!($i,
            s: map_res!(take_until!("\0"), from_utf8) ~
            take!($size - s.len()),
            ||{
                s
            }
        )
    );
);

named!(parse_str4<&[u8], &str>, take_str_eat_garbage!(4));
named!(parse_str8<&[u8], &str>, take_str_eat_garbage!(8));
named!(parse_str32<&[u8], &str>, take_str_eat_garbage!(32));
named!(parse_str100<&[u8], &str>, take_str_eat_garbage!(100));
named!(parse_str155<&[u8], &str>, take_str_eat_garbage!(155));

macro_rules! take_until_expr_with_limit_consume(
  ($i:expr, $submac:ident!( $($args:tt)* ), $stop: expr, $limit: expr) => (
    {
      let mut begin = 0;
      let mut remaining = $i.len();
      let mut res = Vec::new();
      let mut cnt = 0;
      let mut err = false;
      let mut append = true;
      loop {
        match $submac!(&$i[begin..], $($args)*) {
          IResult::Done(i,o) => {
            if append {
              if $stop(o) {
                append = false;
              } else {
                res.push(o);
              }
            }
            begin += remaining - i.len();
            remaining = i.len();
            cnt = cnt + 1;
            if cnt == $limit {
              break
            }
          },
          IResult::Error(_)  => {
            err = true;
            break;
          },
          IResult::Incomplete(_) => {
            break;
          }
        }
      }
      if err {
        IResult::Error(Err::Position(ErrorCode::Count as u32,$i))
      } else if cnt == $limit {
        IResult::Done(&$i[begin..], res)
      } else {
        IResult::Incomplete(Needed::Unknown)
      }
    }
  );
  ($i:expr, $f:expr, $stop: expr, $limit: expr) => (
    take_until_expr_with_limit_consume!($i, call!($f), $stop, $limit);
  );
);

/*
 * Octal string parsing
 */

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;

    for c in s.chars() {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u *= 8;
        u += (c as u64) - ('0' as u64);
    }

    Ok(u)
}

fn parse_octal(i: &[u8], n: usize) -> IResult<&[u8], u64> {
    map_res!(i, take_str_eat_garbage!(n), octal_to_u64)
}

named!(parse_octal8<&[u8], u64>, apply!(parse_octal, 8));
named!(parse_octal12<&[u8], u64>, apply!(parse_octal, 12));

/*
 * TypeFlag parsing
 */

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::PaxInterexchangeFormat,
        'x' => TypeFlag::PaxExtendedAttributes,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::NormalFile
    }
}

fn bytes_to_type_flag(i: &[u8]) -> Result<TypeFlag, &'static str> {
    Ok(char_to_type_flag(i[0] as char))
}

named!(parse_type_flag<&[u8], TypeFlag>, map_res!(take!(1), bytes_to_type_flag));

/*
 * Sparse parsing
 */

fn parse_one_sparse(i: &[u8]) -> IResult<&[u8], Sparse> {
    chain!(i,
        offset:   parse_octal12 ~
        numbytes: parse_octal12,
        ||{
            Sparse {
                offset:   offset,
                numbytes: numbytes
            }
        }
    )
}

fn parse_sparses_with_limit(i: &[u8], limit: usize) -> IResult<&[u8], Vec<Sparse>> {
    take_until_expr_with_limit_consume!(i, parse_one_sparse, |s: Sparse| s.offset == 0 && s.numbytes == 0, limit)
}

fn add_to_vec(extra: Vec<Sparse>, sparses: &mut Vec<Sparse>) -> Result<&'static str, &'static str> {
    for sparse in &extra {
        sparses.push(*sparse);
    }
    Ok("")
}

fn parse_extra_sparses<'a, 'b>(i: &'a [u8], isextended: bool, sparses: &'b mut Vec<Sparse>) -> IResult<'a, &'a [u8], &'b mut Vec<Sparse>> {
    if isextended {
        chain!(i,
            map_res!(apply!(parse_sparses_with_limit, 21), apply!(add_to_vec, sparses)) ~
            extended:      parse_bool                                                   ~
            take!(7) /* padding to 512 */                                               ~
            extra_sparses: apply!(parse_extra_sparses, extended, sparses),
            ||{
                extra_sparses
            }
        )
    } else {
        IResult::Done(i, sparses)
    }
}

fn parse_pax_extra_sparses<'a, 'b>(i: &'a [u8], h: &'b mut PaxHeader) -> IResult<'a, &'a [u8], &'b mut Vec<Sparse>> {
    parse_extra_sparses(i, h.isextended, &mut h.sparses)
}

/*
 * Boolean parsing
 */

fn to_bool(i: &[u8]) -> Result<bool, &'static str> {
    Ok(i[0] != 0)
}

named!(parse_bool<&[u8], bool>, map_res!(take!(1), to_bool));

/*
 * UStar PAX extended parsing
 */

fn parse_ustar00_extra_pax(i: &[u8]) -> IResult<&[u8], PaxHeader> {
    chain!(i,
        atime:       parse_octal12                       ~
        ctime:       parse_octal12                       ~
        offset:      parse_octal12                       ~
        longnames:   parse_str4                          ~
        take!(1)                                         ~
        sparses:     apply!(parse_sparses_with_limit, 4) ~
        isextended:  parse_bool                          ~
        realsize:    parse_octal12                       ~
        take!(17), /* padding to 512 */
        ||{
            PaxHeader {
                atime:         atime,
                ctime:         ctime,
                offset:        offset,
                longnames:     longnames,
                sparses:       sparses,
                isextended:    isextended,
                realsize:      realsize,
            }
        }
    )
}

/*
 * UStar Posix parsing
 */

fn parse_ustar00_extra_posix(i: &[u8]) -> IResult<&[u8], UStarExtraHeader> {
    chain!(i,
        prefix: parse_str155 ~
        take!(12),
        ||{
            UStarExtraHeader::PosixUStar(PosixUStarHeader {
                prefix: prefix
            })
        }
    )
}

fn parse_ustar00_extra(i: &[u8], flag: TypeFlag) -> IResult<&[u8], UStarExtraHeader> {
    let mut header : PaxHeader;
    match flag {
        TypeFlag::PaxInterexchangeFormat => {
            chain!(i,
                header = parse_ustar00_extra_pax ~
                apply!(parse_pax_extra_sparses, &mut header),
                ||{
                    UStarExtraHeader::Pax(header)
                }
            )
        },
        _ => parse_ustar00_extra_posix(i)
    }
}

fn parse_ustar00(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("00")             ~
        uname:    parse_str32  ~
        gname:    parse_str32  ~
        devmajor: parse_octal8 ~
        devminor: parse_octal8 ~
        extra:    apply!(parse_ustar00_extra, flag),
        ||{
            ExtraHeader::UStar(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                extra:    extra
            })
        }
    )
}

fn parse_ustar(i: &[u8], flag: TypeFlag) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: apply!(parse_ustar00, flag),
        ||{
            ustar
        }
    )
}

/*
 * Posix tar archive header parsing
 */

fn parse_posix(i: &[u8]) -> IResult<&[u8], ExtraHeader> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            ExtraHeader::Padding
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     parse_str100    ~
        mode:     parse_str8      ~
        uid:      parse_octal8    ~
        gid:      parse_octal8    ~
        size:     parse_octal12   ~
        mtime:    parse_octal12   ~
        chksum:   parse_str8      ~
        typeflag: parse_type_flag ~
        linkname: parse_str100    ~
        ustar:    alt!(apply!(parse_ustar, typeflag) | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: typeflag,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

/*
 * Contents parsing
 */

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

/*
 * Tar entry header + contents parsing
 */

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

/*
 * Tar archive parsing
 */

fn filter_entries(entries: Vec<TarEntry>) -> Result<Vec<TarEntry>, &'static str> {
    /* Filter out empty entries */
    Ok(entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>())
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    chain!(i,
        entries: map_res!(many0!(parse_entry), filter_entries) ~
        eof,
        ||{
            entries
        }
    )
}

/*
 * Tests
 */

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<|endoftext|>


<commit_before>use ast::*;
use scanner::{Lexeme, Position, Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub enum ParseError {
    UnexpectedEndOfFile,
    UnexpectedToken(Lexeme, Position),
    MissingSubexpression(Lexeme, Position),
    MissingClosingParen(Lexeme, Position),
    MissingSemicolon(Lexeme, Position),
}

pub fn parse(tokens: &[TokenWithContext]) -> Result<Vec<Statement>, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    let mut statements = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = parse_statement(&mut iter) {
        match result {
            Ok(statement) => {
                statements.push(statement);
            }
            Err(error) => {
                errors.push(error);
                synchronise(&mut iter);
            }
        }
    }
    if errors.is_empty() {
        Ok(statements)
    } else {
        Err(errors)
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum PositionInConstruct {
        Start,
        Body,
        End,
    };
    fn classify(token: &Token) -> PositionInConstruct {
        match *token {
            Token::Semicolon => PositionInConstruct::End,
            Token::Class | Token::Fun | Token::Var | Token::For | Token::If | Token::While |
            Token::Print | Token::Return => PositionInConstruct::Start,
            _ => PositionInConstruct::Body,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            PositionInConstruct::Start => {
                break;
            }
            PositionInConstruct::Body => {
                let _ = tokens.next();
            }
            PositionInConstruct::End => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_declaration<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    unimplemented!();
}

fn parse_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let result = match tokens.peek().map(|t| &t.token) {
        Some(&Token::Print) => {
            let _ = tokens.next();
            parse_print_statement(tokens)
        }
        Some(_) => parse_expression_statement(tokens),
        None => None,
    };
    if let Some(Ok(statement)) = result {
        match tokens.peek() {
            Some(&&TokenWithContext { token: Token::Semicolon, lexeme: _, position: _ }) => {
                let _ = tokens.next();
                Some(Ok(statement))
            }
            Some(&&TokenWithContext { token: _, ref lexeme, ref position }) => {
                Some(Err(ParseError::MissingSemicolon(lexeme.clone(), position.clone())))
            }
            None => Some(Err(ParseError::UnexpectedEndOfFile)),
        }
    } else {
        result
    }
}

fn parse_print_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Print(e)))
}

fn parse_expression_statement<'a, I>(tokens: &mut Peekable<I>)
                                     -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Expression(e)))
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<BinaryOperator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_subexpression(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::BangEqual => Some(BinaryOperator::NotEqual),
            Token::EqualEqual => Some(BinaryOperator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Greater => Some(BinaryOperator::Greater),
            Token::GreaterEqual => Some(BinaryOperator::GreaterEqual),
            Token::Less => Some(BinaryOperator::Less),
            Token::LessEqual => Some(BinaryOperator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Minus => Some(BinaryOperator::Minus),
            Token::Plus => Some(BinaryOperator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Slash => Some(BinaryOperator::Slash),
            Token::Star => Some(BinaryOperator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<UnaryOperator> {
        match *token {
            Token::Minus => Some(UnaryOperator::Minus),
            Token::Bang => Some(UnaryOperator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_unary(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr = if let Some(result) = parse_expression(tokens) {
                    try_wrap_err!(result)
                } else {
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::MissingClosingParen(token.lexeme.clone(),
                                                                            token.position)));
                        }

                    }
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                }
            }
            _ => {
                return Some(Err(ParseError::UnexpectedToken(primary_token.lexeme.clone(),
                                                            primary_token.position)));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let string = String::from("123");
        let (tokens, _) = scan(&string);
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!(&string, &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2");
        assert!(parse(&tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)");
        assert!(parse(&tokens).is_err());
    }
}
<commit_msg>prepared var declaration<commit_after>use ast::*;
use scanner::{Lexeme, Position, Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub enum ParseError {
    UnexpectedEndOfFile,
    UnexpectedToken(Lexeme, Position),
    MissingSubexpression(Lexeme, Position),
    MissingClosingParen(Lexeme, Position),
    MissingSemicolon(Lexeme, Position),
}

pub fn parse(tokens: &[TokenWithContext]) -> Result<Vec<Statement>, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    let mut statements = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = parse_declaration(&mut iter) {
        match result {
            Ok(statement) => {
                statements.push(statement);
            }
            Err(error) => {
                errors.push(error);
                synchronise(&mut iter);
            }
        }
    }
    if errors.is_empty() {
        Ok(statements)
    } else {
        Err(errors)
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum PositionInConstruct {
        Start,
        Body,
        End,
    };
    fn classify(token: &Token) -> PositionInConstruct {
        match *token {
            Token::Semicolon => PositionInConstruct::End,
            Token::Class | Token::Fun | Token::Var | Token::For | Token::If | Token::While |
            Token::Print | Token::Return => PositionInConstruct::Start,
            _ => PositionInConstruct::Body,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            PositionInConstruct::Start => {
                break;
            }
            PositionInConstruct::Body => {
                let _ = tokens.next();
            }
            PositionInConstruct::End => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_declaration<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    match tokens.peek().map(|t| &t.token) {
        Some(&Token::Var) => {
            let _ = tokens.next();
            parse_var_declaration(tokens)
        }
        Some(_) => parse_statement(tokens),
        None => None,
    }
}

fn parse_var_declaration<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    unimplemented!()
}

fn parse_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let result = match tokens.peek().map(|t| &t.token) {
        Some(&Token::Print) => {
            let _ = tokens.next();
            parse_print_statement(tokens)
        }
        Some(_) => parse_expression_statement(tokens),
        None => None,
    };
    if let Some(Ok(statement)) = result {
        match tokens.peek() {
            Some(&&TokenWithContext { token: Token::Semicolon, lexeme: _, position: _ }) => {
                let _ = tokens.next();
                Some(Ok(statement))
            }
            Some(&&TokenWithContext { token: _, ref lexeme, ref position }) => {
                Some(Err(ParseError::MissingSemicolon(lexeme.clone(), position.clone())))
            }
            None => Some(Err(ParseError::UnexpectedEndOfFile)),
        }
    } else {
        result
    }
}

fn parse_print_statement<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Print(e)))
}

fn parse_expression_statement<'a, I>(tokens: &mut Peekable<I>)
                                     -> Option<Result<Statement, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_expression(tokens).map(|r| r.map(|e| Statement::Expression(e)))
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<BinaryOperator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_subexpression(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::BangEqual => Some(BinaryOperator::NotEqual),
            Token::EqualEqual => Some(BinaryOperator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Greater => Some(BinaryOperator::Greater),
            Token::GreaterEqual => Some(BinaryOperator::GreaterEqual),
            Token::Less => Some(BinaryOperator::Less),
            Token::LessEqual => Some(BinaryOperator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Minus => Some(BinaryOperator::Minus),
            Token::Plus => Some(BinaryOperator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Slash => Some(BinaryOperator::Slash),
            Token::Star => Some(BinaryOperator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<UnaryOperator> {
        match *token {
            Token::Minus => Some(UnaryOperator::Minus),
            Token::Bang => Some(UnaryOperator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_unary(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr = if let Some(result) = parse_expression(tokens) {
                    try_wrap_err!(result)
                } else {
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::MissingClosingParen(token.lexeme.clone(),
                                                                            token.position)));
                        }

                    }
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                }
            }
            _ => {
                return Some(Err(ParseError::UnexpectedToken(primary_token.lexeme.clone(),
                                                            primary_token.position)));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let string = String::from("123");
        let (tokens, _) = scan(&string);
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!(&string, &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789");
        let expr = parse_expression(&mut tokens.iter().peekable()).unwrap().unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2");
        assert!(parse(&tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)");
        assert!(parse(&tokens).is_err());
    }
}
<|endoftext|>