<commit_before>use std::mem;

pub use self::Token::*;

use self::TokenClass::*;
use self::ParserState::*;

/// `Token` is a section of a compiled mustache string.
#[derive(Clone, Debug)]
pub enum Token {
    Text(String),
    ETag(Vec<String>, String),
    UTag(Vec<String>, String),
    Section(Vec<String>, bool, Vec<Token>, String, String, String, String, String),
    IncompleteSection(Vec<String>, bool, String, bool),
    Partial(String, String, String),
}

enum TokenClass {
    Normal,
    StandAlone,
    WhiteSpace(String, usize),
    NewLineWhiteSpace(String, usize),
}

/// `Parser` parses a string into a series of `Token`s.
pub struct Parser<'a, T: 'a> {
    reader: &'a mut T,
    ch: Option<char>,
    lookahead: Option<char>,
    line: usize,
    col: usize,
    content: String,
    state: ParserState,
    otag: String,
    ctag: String,
    otag_chars: Vec<char>,
    ctag_chars: Vec<char>,
    tag_position: usize,
    tokens: Vec<Token>,
    partials: Vec<String>,
}

enum ParserState { TEXT, OTAG, TAG, CTAG }

impl<'a, T: Iterator<Item=char>> Parser<'a, T> {
    pub fn new(reader: &'a mut T, otag: &str, ctag: &str) -> Parser<'a, T> {
        let mut parser = Parser {
            reader: reader,
            ch: None,
            lookahead: None,
            line: 1,
            col: 1,
            content: String::new(),
            state: TEXT,
            otag: otag.to_string(),
            ctag: ctag.to_string(),
            otag_chars: otag.chars().collect(),
            ctag_chars: ctag.chars().collect(),
            tag_position: 0,
            tokens: Vec::new(),
            partials: Vec::new(),
        };

        parser.bump();
        parser
    }

    fn bump(&mut self) {
        match self.lookahead.take() {
            None => { self.ch = self.reader.next(); }
            Some(ch) => { self.ch = Some(ch); }
        }

        match self.ch {
            Some(ch) => {
                if ch == '\n' {
                    self.line += 1;
                    self.col = 1;
                } else {
                    self.col += 1;
                }
            }
            None => { }
        }
    }

    fn peek(&mut self) -> Option<char> {
        match self.lookahead {
            None => {
                self.lookahead = self.reader.next();
                self.lookahead
            }
            Some(ch) => Some(ch),
        }
    }

    fn ch_is(&self, ch: char) -> bool {
        match self.ch {
            Some(c) => c == ch,
            None => false,
        }
    }

    /// Parse the template into tokens and a list of partial files.
    pub fn parse(mut self) -> (Vec<Token>, Vec<String>) {
        let mut curly_brace_tag = false;

        loop {
            let ch = match self.ch {
                Some(ch) => ch,
                None => { break; }
            };

            match self.state {
                TEXT => {
                    if ch == self.otag_chars[0] {
                        if self.otag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = OTAG;
                        } else {
                            self.add_text();
                            self.state = TAG;
                        }
                    } else {
                        self.content.push(ch);
                    }
                    self.bump();
                }
                OTAG => {
                    if ch == self.otag_chars[self.tag_position] {
                        if self.tag_position == self.otag_chars.len() - 1 {
                            self.add_text();
                            curly_brace_tag = false;
                            self.state = TAG;
                        } else {
                            self.tag_position = self.tag_position + 1;
                        }
                    } else {
                        // We don't have a tag, so add all the tag parts we've seen
                        // so far to the string.
                        self.state = TEXT;
                        self.not_otag();
                        self.content.push(ch);
                    }
                    self.bump();
                }
                TAG => {
                    if self.content.is_empty() && ch == '{' {
                        curly_brace_tag = true;
                        self.content.push(ch);
                        self.bump();
                    } else if curly_brace_tag && ch == '}' {
                        curly_brace_tag = false;
                        self.content.push(ch);
                        self.bump();
                    } else if ch == self.ctag_chars[0] {
                        if self.ctag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = CTAG;
                            self.bump();
                        } else {
                            self.add_tag();
                            self.state = TEXT;
                        }
                    } else {
                        self.content.push(ch);
                        self.bump();
                    }
                }
                CTAG => {
                    if ch == self.ctag_chars[self.tag_position] {
                        if self.tag_position == self.ctag_chars.len() - 1 {
                            self.add_tag();
                            self.state = TEXT;
                        } else {
                            self.state = TAG;
                            self.not_ctag();
                            self.content.push(ch);
                            self.bump();
                        }
                    } else {
                        panic!("character {} is not part of CTAG: {}",
                              ch,
                              self.ctag_chars[self.tag_position]);
                    }
                }
            }
        }

        match self.state {
            TEXT => { self.add_text(); }
            OTAG => { self.not_otag(); self.add_text(); }
            TAG => { panic!("unclosed tag"); }
            CTAG => { self.not_ctag(); self.add_text(); }
        }

        // Check that we don't have any incomplete sections.
        for token in self.tokens.iter() {
            match *token {
                IncompleteSection(ref path, _, _, _) => {
                    panic!("Unclosed mustache section {}", path.join("."));
              }
              _ => {}
            }
        };

        let Parser { tokens, partials, .. } = self;

        (tokens, partials)
    }

    fn add_text(&mut self) {
        if !self.content.is_empty() {
            let mut content = String::new();
            mem::swap(&mut content, &mut self.content);

            self.tokens.push(Text(content.to_string()));
        }
    }

    // This function classifies whether or not a token is standalone, or if it
    // has trailing whitespace. It's looking for this pattern:
    //
    //   ("\n" | "\r\n") whitespace* token ("\n" | "\r\n")
    //
    fn classify_token(&mut self) -> TokenClass {
        // Exit early if the next character is not '\n' or '\r\n'.
        match self.ch {
            None => { }
            Some(ch) => {
                if !(ch == '\n' || (ch == '\r' && self.peek() == Some('\n'))) {
                    return Normal;
                }
            }
        }

        match self.tokens.last() {
            // If the last token ends with a newline (or there is no previous
            // token), then this token is standalone.
            None => { StandAlone }

            Some(&IncompleteSection(_, _, _, true)) => { StandAlone }

            Some(&Text(ref s)) if !s.is_empty() => {
                // Look for the last newline character that may have whitespace
                // following it.
                match s.rfind(| c:char| c == '\n' || !c.is_whitespace()) {
                    // It's all whitespace.
                    None => {
                        if self.tokens.len() == 1 {
                            WhiteSpace(s.to_string(), 0)
                        } else {
                            Normal
                        }
                    }
                    Some(pos) => {
                        if s.as_bytes()[pos] == b'\n' {
                            if pos == s.len() - 1 {
                                StandAlone
                            } else {
                                WhiteSpace(s.to_string(), pos + 1)
                            }
                        } else { Normal }
                    }
                }
            }
            Some(_) => Normal,
        }
    }

    fn eat_whitespace(&mut self) -> bool {
        // If the next character is a newline, and the last token ends with a
        // newline and whitespace, clear out the whitespace.

        match self.classify_token() {
            Normal => { false }
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                true
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                true
            }
        }
    }

    fn add_tag(&mut self) {
        self.bump();

        let tag = self.otag.clone() + &self.content + &self.ctag;

        // Move the content to avoid a copy.
        let mut content = String::new();
        mem::swap(&mut content, &mut self.content);
        let len = content.len();
        let content = content;

        match content.as_bytes()[0] as char {
            '!' => {
                // ignore comments
                self.eat_whitespace();
            }
            '&' => {
                let name = &content[1..len];
                let name = self.check_content(name);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(UTag(name, tag));
            }
            '{' => {
                if content.ends_with("}") {
                    let name = &content[1..len - 1];
                    let name = self.check_content(name);
                    let name = name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect();
                    self.tokens.push(UTag(name, tag));
                } else { panic!("unbalanced \"{\" in tag"); }
            }
            '#' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, false, tag, newlined));
            }
            '^' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, true, tag, newlined));
            }
            '/' => {
                self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                let mut children: Vec<Token> = Vec::new();

                loop {
                    if self.tokens.is_empty() {
                        panic!("closing unopened section");
                    }

                    let last = self.tokens.pop();

                    match last {
                        Some(IncompleteSection(section_name, inverted, osection, _)) => {
                            children.reverse();

                            // Collect all the children's sources.
                            let mut srcs = Vec::new();
                            for child in children.iter() {
                                match *child {
                                    Text(ref s)
                                    | ETag(_, ref s)
                                    | UTag(_, ref s)
                                    | Partial(_, _, ref s) => {
                                        srcs.push(s.clone())
                                    }
                                    Section(_, _, _, _, ref osection, ref src, ref csection, _) => {
                                        srcs.push(osection.clone());
                                        srcs.push(src.clone());
                                        srcs.push(csection.clone());
                                    }
                                    _ => panic!(),
                                }
                            }

                            if section_name == name {
                                // Cache the combination of all the sources in the
                                // section. It's unfortunate, but we need to do this in
                                // case the user uses a function to instantiate the
                                // tag.
                                let mut src = String::new();
                                for s in srcs.iter() { src.push_str(s); }

                                self.tokens.push(
                                    Section(
                                        name,
                                        inverted,
                                        children,
                                        self.otag.to_string(),
                                        osection,
                                        src.to_string(),
                                        tag,
                                        self.ctag.to_string()));
                                break;
                            } else {
                                panic!("Unclosed section");
                            }
                        }
                        _ => { match last {
                            Some(last_token) => {children.push(last_token); }
                            None => ()
                            }
                        }
                    }
                }
            }
            '>' => { self.add_partial(&content, tag); }
            '=' => {
                self.eat_whitespace();

                if len > 2usize && content.ends_with("=") {
                    let s = self.check_content(&content[1..len - 1]);

                    fn is_whitespace(c: char) -> bool { c.is_whitespace() }
                    let pos = s.find(is_whitespace);
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.otag = s[0..pos].to_string();
                    self.otag_chars = self.otag.chars().collect();

                    let s2 = &s[pos..];
                    let pos = s2.find(| c : char| !c.is_whitespace());
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.ctag = s2[pos..].to_string();
                    self.ctag_chars = self.ctag.chars().collect();
                } else {
                    panic!("invalid change delimiter tag content");
                }
            }
            _ => {
                // If the name is "." then we want the top element, which we represent with
                // an empty name.
                let name = self.check_content(&content);
                let name = if name == "." {
                    Vec::new()
                } else {
                    name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect()
                };

                self.tokens.push(ETag(name, tag));
            }
        }
    }

    fn add_partial(&mut self, content: &str, tag: String) {
        let indent = match self.classify_token() {
            Normal => "".to_string(),
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                "".to_string()
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                let ws = &s[pos..];

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                ws.to_string()
            }
        };

        // We can't inline the tokens directly as we may have a recursive
        // partial. So instead, we'll cache the partials we used and look them
        // up later.
        let name = &content[1..content.len()];
        let name = self.check_content(name);

        self.tokens.push(Partial(name.to_string(), indent, tag));
        self.partials.push(name);
    }

    fn not_otag(&mut self) {
        for (i, ch) in self.otag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn not_ctag(&mut self) {
        for (i, ch) in self.ctag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn check_content(&self, content: &str) -> String {
        let trimmed = content.trim();
        if trimmed.is_empty() {
            panic!("empty tag");
        }
        trimmed.to_string()
    }
}
<commit_msg>chore(clippy): fix Clippy’s `while_let_loop` warning<commit_after>use std::mem;

pub use self::Token::*;

use self::TokenClass::*;
use self::ParserState::*;

/// `Token` is a section of a compiled mustache string.
#[derive(Clone, Debug)]
pub enum Token {
    Text(String),
    ETag(Vec<String>, String),
    UTag(Vec<String>, String),
    Section(Vec<String>, bool, Vec<Token>, String, String, String, String, String),
    IncompleteSection(Vec<String>, bool, String, bool),
    Partial(String, String, String),
}

enum TokenClass {
    Normal,
    StandAlone,
    WhiteSpace(String, usize),
    NewLineWhiteSpace(String, usize),
}

/// `Parser` parses a string into a series of `Token`s.
pub struct Parser<'a, T: 'a> {
    reader: &'a mut T,
    ch: Option<char>,
    lookahead: Option<char>,
    line: usize,
    col: usize,
    content: String,
    state: ParserState,
    otag: String,
    ctag: String,
    otag_chars: Vec<char>,
    ctag_chars: Vec<char>,
    tag_position: usize,
    tokens: Vec<Token>,
    partials: Vec<String>,
}

enum ParserState { TEXT, OTAG, TAG, CTAG }

impl<'a, T: Iterator<Item=char>> Parser<'a, T> {
    pub fn new(reader: &'a mut T, otag: &str, ctag: &str) -> Parser<'a, T> {
        let mut parser = Parser {
            reader: reader,
            ch: None,
            lookahead: None,
            line: 1,
            col: 1,
            content: String::new(),
            state: TEXT,
            otag: otag.to_string(),
            ctag: ctag.to_string(),
            otag_chars: otag.chars().collect(),
            ctag_chars: ctag.chars().collect(),
            tag_position: 0,
            tokens: Vec::new(),
            partials: Vec::new(),
        };

        parser.bump();
        parser
    }

    fn bump(&mut self) {
        match self.lookahead.take() {
            None => { self.ch = self.reader.next(); }
            Some(ch) => { self.ch = Some(ch); }
        }

        match self.ch {
            Some(ch) => {
                if ch == '\n' {
                    self.line += 1;
                    self.col = 1;
                } else {
                    self.col += 1;
                }
            }
            None => { }
        }
    }

    fn peek(&mut self) -> Option<char> {
        match self.lookahead {
            None => {
                self.lookahead = self.reader.next();
                self.lookahead
            }
            Some(ch) => Some(ch),
        }
    }

    fn ch_is(&self, ch: char) -> bool {
        match self.ch {
            Some(c) => c == ch,
            None => false,
        }
    }

    /// Parse the template into tokens and a list of partial files.
    pub fn parse(mut self) -> (Vec<Token>, Vec<String>) {
        let mut curly_brace_tag = false;

        while let Some(ch) = self.ch {
            match self.state {
                TEXT => {
                    if ch == self.otag_chars[0] {
                        if self.otag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = OTAG;
                        } else {
                            self.add_text();
                            self.state = TAG;
                        }
                    } else {
                        self.content.push(ch);
                    }
                    self.bump();
                }
                OTAG => {
                    if ch == self.otag_chars[self.tag_position] {
                        if self.tag_position == self.otag_chars.len() - 1 {
                            self.add_text();
                            curly_brace_tag = false;
                            self.state = TAG;
                        } else {
                            self.tag_position = self.tag_position + 1;
                        }
                    } else {
                        // We don't have a tag, so add all the tag parts we've seen
                        // so far to the string.
                        self.state = TEXT;
                        self.not_otag();
                        self.content.push(ch);
                    }
                    self.bump();
                }
                TAG => {
                    if self.content.is_empty() && ch == '{' {
                        curly_brace_tag = true;
                        self.content.push(ch);
                        self.bump();
                    } else if curly_brace_tag && ch == '}' {
                        curly_brace_tag = false;
                        self.content.push(ch);
                        self.bump();
                    } else if ch == self.ctag_chars[0] {
                        if self.ctag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = CTAG;
                            self.bump();
                        } else {
                            self.add_tag();
                            self.state = TEXT;
                        }
                    } else {
                        self.content.push(ch);
                        self.bump();
                    }
                }
                CTAG => {
                    if ch == self.ctag_chars[self.tag_position] {
                        if self.tag_position == self.ctag_chars.len() - 1 {
                            self.add_tag();
                            self.state = TEXT;
                        } else {
                            self.state = TAG;
                            self.not_ctag();
                            self.content.push(ch);
                            self.bump();
                        }
                    } else {
                        panic!("character {} is not part of CTAG: {}",
                              ch,
                              self.ctag_chars[self.tag_position]);
                    }
                }
            }
        }

        match self.state {
            TEXT => { self.add_text(); }
            OTAG => { self.not_otag(); self.add_text(); }
            TAG => { panic!("unclosed tag"); }
            CTAG => { self.not_ctag(); self.add_text(); }
        }

        // Check that we don't have any incomplete sections.
        for token in self.tokens.iter() {
            match *token {
                IncompleteSection(ref path, _, _, _) => {
                    panic!("Unclosed mustache section {}", path.join("."));
              }
              _ => {}
            }
        };

        let Parser { tokens, partials, .. } = self;

        (tokens, partials)
    }

    fn add_text(&mut self) {
        if !self.content.is_empty() {
            let mut content = String::new();
            mem::swap(&mut content, &mut self.content);

            self.tokens.push(Text(content.to_string()));
        }
    }

    // This function classifies whether or not a token is standalone, or if it
    // has trailing whitespace. It's looking for this pattern:
    //
    //   ("\n" | "\r\n") whitespace* token ("\n" | "\r\n")
    //
    fn classify_token(&mut self) -> TokenClass {
        // Exit early if the next character is not '\n' or '\r\n'.
        match self.ch {
            None => { }
            Some(ch) => {
                if !(ch == '\n' || (ch == '\r' && self.peek() == Some('\n'))) {
                    return Normal;
                }
            }
        }

        match self.tokens.last() {
            // If the last token ends with a newline (or there is no previous
            // token), then this token is standalone.
            None => { StandAlone }

            Some(&IncompleteSection(_, _, _, true)) => { StandAlone }

            Some(&Text(ref s)) if !s.is_empty() => {
                // Look for the last newline character that may have whitespace
                // following it.
                match s.rfind(| c:char| c == '\n' || !c.is_whitespace()) {
                    // It's all whitespace.
                    None => {
                        if self.tokens.len() == 1 {
                            WhiteSpace(s.to_string(), 0)
                        } else {
                            Normal
                        }
                    }
                    Some(pos) => {
                        if s.as_bytes()[pos] == b'\n' {
                            if pos == s.len() - 1 {
                                StandAlone
                            } else {
                                WhiteSpace(s.to_string(), pos + 1)
                            }
                        } else { Normal }
                    }
                }
            }
            Some(_) => Normal,
        }
    }

    fn eat_whitespace(&mut self) -> bool {
        // If the next character is a newline, and the last token ends with a
        // newline and whitespace, clear out the whitespace.

        match self.classify_token() {
            Normal => { false }
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                true
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                true
            }
        }
    }

    fn add_tag(&mut self) {
        self.bump();

        let tag = self.otag.clone() + &self.content + &self.ctag;

        // Move the content to avoid a copy.
        let mut content = String::new();
        mem::swap(&mut content, &mut self.content);
        let len = content.len();
        let content = content;

        match content.as_bytes()[0] as char {
            '!' => {
                // ignore comments
                self.eat_whitespace();
            }
            '&' => {
                let name = &content[1..len];
                let name = self.check_content(name);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(UTag(name, tag));
            }
            '{' => {
                if content.ends_with("}") {
                    let name = &content[1..len - 1];
                    let name = self.check_content(name);
                    let name = name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect();
                    self.tokens.push(UTag(name, tag));
                } else { panic!("unbalanced \"{\" in tag"); }
            }
            '#' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, false, tag, newlined));
            }
            '^' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, true, tag, newlined));
            }
            '/' => {
                self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                let mut children: Vec<Token> = Vec::new();

                loop {
                    if self.tokens.is_empty() {
                        panic!("closing unopened section");
                    }

                    let last = self.tokens.pop();

                    match last {
                        Some(IncompleteSection(section_name, inverted, osection, _)) => {
                            children.reverse();

                            // Collect all the children's sources.
                            let mut srcs = Vec::new();
                            for child in children.iter() {
                                match *child {
                                    Text(ref s)
                                    | ETag(_, ref s)
                                    | UTag(_, ref s)
                                    | Partial(_, _, ref s) => {
                                        srcs.push(s.clone())
                                    }
                                    Section(_, _, _, _, ref osection, ref src, ref csection, _) => {
                                        srcs.push(osection.clone());
                                        srcs.push(src.clone());
                                        srcs.push(csection.clone());
                                    }
                                    _ => panic!(),
                                }
                            }

                            if section_name == name {
                                // Cache the combination of all the sources in the
                                // section. It's unfortunate, but we need to do this in
                                // case the user uses a function to instantiate the
                                // tag.
                                let mut src = String::new();
                                for s in srcs.iter() { src.push_str(s); }

                                self.tokens.push(
                                    Section(
                                        name,
                                        inverted,
                                        children,
                                        self.otag.to_string(),
                                        osection,
                                        src.to_string(),
                                        tag,
                                        self.ctag.to_string()));
                                break;
                            } else {
                                panic!("Unclosed section");
                            }
                        }
                        _ => { match last {
                            Some(last_token) => {children.push(last_token); }
                            None => ()
                            }
                        }
                    }
                }
            }
            '>' => { self.add_partial(&content, tag); }
            '=' => {
                self.eat_whitespace();

                if len > 2usize && content.ends_with("=") {
                    let s = self.check_content(&content[1..len - 1]);

                    fn is_whitespace(c: char) -> bool { c.is_whitespace() }
                    let pos = s.find(is_whitespace);
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.otag = s[0..pos].to_string();
                    self.otag_chars = self.otag.chars().collect();

                    let s2 = &s[pos..];
                    let pos = s2.find(| c : char| !c.is_whitespace());
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.ctag = s2[pos..].to_string();
                    self.ctag_chars = self.ctag.chars().collect();
                } else {
                    panic!("invalid change delimiter tag content");
                }
            }
            _ => {
                // If the name is "." then we want the top element, which we represent with
                // an empty name.
                let name = self.check_content(&content);
                let name = if name == "." {
                    Vec::new()
                } else {
                    name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect()
                };

                self.tokens.push(ETag(name, tag));
            }
        }
    }

    fn add_partial(&mut self, content: &str, tag: String) {
        let indent = match self.classify_token() {
            Normal => "".to_string(),
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                "".to_string()
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                let ws = &s[pos..];

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                ws.to_string()
            }
        };

        // We can't inline the tokens directly as we may have a recursive
        // partial. So instead, we'll cache the partials we used and look them
        // up later.
        let name = &content[1..content.len()];
        let name = self.check_content(name);

        self.tokens.push(Partial(name.to_string(), indent, tag));
        self.partials.push(name);
    }

    fn not_otag(&mut self) {
        for (i, ch) in self.otag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn not_ctag(&mut self) {
        for (i, ch) in self.ctag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn check_content(&self, content: &str) -> String {
        let trimmed = content.trim();
        if trimmed.is_empty() {
            panic!("empty tag");
        }
        trimmed.to_string()
    }
}
<|endoftext|>


<commit_before>use ast::*;

mod ast;

macro_rules! parse_subexprs (
	($expfn:ident, $($others:ident),+) => (
		match self.$expfn() {
			Ok(m) => m,
			Err(f) => parse_subexprs!($($others)+)
		}
	);
	($expfn:ident) => (
		if_ok!(self.$expfn())
	)
)

pub struct Parser {
	code: ~str,
	pos: uint,
	line: uint,
	column: uint
}

pub struct ParseError<'a> {
	line: uint,
	column: uint,
	desc: &'a str
}

pub type ParseResult<'a, T> = Result<T, ParseError<'a>>;

impl<'a> ParseError<'a> {
	pub fn new(line: uint, col: uint, desc: &'a str) -> ParseError<'a> {
		ParseError {
			line: line,
			column: col,
			desc: desc
		}
	}
}

impl Parser {
	pub fn new() -> Parser {
		Parser {
			code: ~"",
			pos: 0,
			line: 1,
			column: 1
		}
	}

	pub fn load_code(&mut self, code: ~str) {
		self.code = code;
		self.pos = 0;
		self.line = 1;
		self.column = 1;
	}

	pub fn parse_code(&mut self, code: ~str) -> ~Ast {
		self.load_code(code);
		self.parse()
	}

	pub fn parse(&mut self) -> ~Ast {
		let mut root = RootAst::new();
		while self.code.len() > 0 {
			let expr = match self.parse_expr() {
				Ok(m) => m,
				Err(f) => {
					error!("error at line {}, column {}: {}", f.line, f.column, f.desc);
					fail!(); // fix fail! later
				}
			};
			root.push(expr);
		}
		~root as ~Ast
	}

	fn parse_expr(&mut self) -> ParseResult<~Ast> {
		let expr = parse_subexprs!(parse_sexpr, parse_integer);
		Ok(expr)
	}

	fn parse_sexpr(&mut self) -> ParseResult<~Ast> {
		Err(ParseError::new(self.line, self.column, "not implemented"))
	}

	fn parse_integer(&mut self) -> ParseResult<~Ast> {
		if self.pos == self.code.len() {
			return Err(ParseError::new(self.line, self.column, "end of file"));
		}
		let mut number = 0;
		let mut neg = false;
		let mut digits = 0;
		while {
			match self.code.char_at(self.pos) {
				num @ '0'..'9' => {
					digits += 1;
					number = number * 10 + num.to_digit(10).unwrap() as i64;
					true
				}
				'-' => {
					if digits == 0 {
						neg = true;
					} else {
						return Err(ParseError::new(self.line, self.column, "expected integer but found '-'"));
					}
					true
				}
				other => {
					if digits == 0 {
						return Err(ParseError::new(self.line, self.column,
							"error")); // THIS SHOULD BE FIXED //format!("expected integer but found '{:c}'", other)));
					}
					false
				}
			}
		} { self.pos += 1; if self.pos == self.code.len() { break } }
		Ok(~IntegerAst::new(if neg { -number } else { number }) as ~Ast)
	}

	#[inline(always)]
	fn add_line(&mut self) {
		self.line += 1;
		self.column = 1;
	}
}
<commit_msg>Fix incompatibility between ParseResult and format!<commit_after>use ast::*;

mod ast;

macro_rules! parse_subexprs (
	($expfn:ident, $($others:ident),+) => (
		match self.$expfn() {
			Ok(m) => m,
			Err(f) => parse_subexprs!($($others)+)
		}
	);
	($expfn:ident) => (
		if_ok!(self.$expfn())
	)
)

pub struct Parser {
	code: ~str,
	pos: uint,
	line: uint,
	column: uint
}

pub struct ParseError {
	line: uint,
	column: uint,
	desc: ~str
}

pub type ParseResult<T> = Result<T, ParseError>;

impl ParseError {
	pub fn new(line: uint, col: uint, desc: ~str) -> ParseError {
		ParseError {
			line: line,
			column: col,
			desc: desc
		}
	}
}

impl Parser {
	pub fn new() -> Parser {
		Parser {
			code: ~"",
			pos: 0,
			line: 1,
			column: 1
		}
	}

	pub fn load_code(&mut self, code: ~str) {
		self.code = code;
		self.pos = 0;
		self.line = 1;
		self.column = 1;
	}

	pub fn parse_code(&mut self, code: ~str) -> ~Ast {
		self.load_code(code);
		self.parse()
	}

	pub fn parse(&mut self) -> ~Ast {
		let mut root = RootAst::new();
		while self.code.len() > 0 {
			let expr = match self.parse_expr() {
				Ok(m) => m,
				Err(f) => {
					error!("error at line {}, column {}: {}", f.line, f.column, f.desc);
					fail!(); // fix fail! later
				}
			};
			root.push(expr);
		}
		~root as ~Ast
	}

	fn parse_expr(&mut self) -> ParseResult<~Ast> {
		let expr = parse_subexprs!(parse_sexpr, parse_integer);
		Ok(expr)
	}

	fn parse_sexpr(&mut self) -> ParseResult<~Ast> {
		Err(ParseError::new(self.line, self.column, ~"not implemented"))
	}

	fn parse_integer(&mut self) -> ParseResult<~Ast> {
		if self.pos == self.code.len() {
			return Err(ParseError::new(self.line, self.column, ~"end of file"));
		}
		let mut number = 0;
		let mut neg = false;
		let mut digits = 0;
		while {
			match self.code.char_at(self.pos) {
				num @ '0'..'9' => {
					digits += 1;
					number = number * 10 + num.to_digit(10).unwrap() as i64;
					true
				}
				'-' => {
					if digits == 0 {
						neg = true;
					} else {
						return Err(ParseError::new(self.line, self.column, ~"expected integer but found '-'"));
					}
					true
				}
				other => {
					if digits == 0 {
						return Err(ParseError::new(self.line, self.column,
							format!("expected integer but found '{:c}'", other)));
					}
					false
				}
			}
		} { self.pos += 1; if self.pos == self.code.len() { break } }
		Ok(~IntegerAst::new(if neg { -number } else { number }) as ~Ast)
	}

	#[inline(always)]
	fn add_line(&mut self) {
		self.line += 1;
		self.column = 1;
	}
}
<|endoftext|>


<commit_before>//! A parser for lambda expressions with
//! [De Bruijn indices](https://en.wikipedia.org/wiki/De_Bruijn_index)

use term::*;
use term::Term::*;
use self::Token::*;
use self::Error::*;
use self::Expression::*;

/// A type to represent a parsing error.
#[derive(Debug, PartialEq)]
pub enum Error {
    InvalidCharacter((usize, char)),
    InvalidExpression,
    EmptyExpression
}

#[derive(Debug, PartialEq)]
enum Token {
    Lambda,
    Lparen,
    Rparen,
    Number(usize)
}

fn tokenize(input: &str) -> Result<Vec<Token>, Error> {
    let mut chars = input.chars();
    let mut tokens = Vec::new();
    let mut position = 0;

    while let Some(c) = chars.next() {
        match c {
     '\\' | 'λ' => { tokens.push(Lambda) },
            '(' => { tokens.push(Lparen) },
            ')' => { tokens.push(Rparen) },
             x  => {
                if let Some(n) = x.to_digit(16) {
                    tokens.push(Number(n as usize))
                } else if x.is_whitespace() {
                    ()
                } else {
                    return Err(InvalidCharacter((position, x)))
                }
            }
        }
        position += if c == 'λ' { 2 } else { 1 };
    }

    Ok(tokens)
}

#[derive(Debug, PartialEq)]
enum Expression {
    Abstraction,
    Sequence(Vec<Expression>),
    Variable(usize)
}

fn _get_ast(tokens: &[Token], pos: &mut usize) -> Result<Expression, Error> {
    let mut expr = Vec::new();

    if tokens.is_empty() { return Err(EmptyExpression) }

    while *pos < tokens.len() {
        match tokens[*pos] {
            Lambda => {
                expr.push(Abstraction)
            },
            Number(i) => {
                expr.push(Variable(i))
            },
            Lparen => {
                *pos += 1;
                let subtree = try!(_get_ast(&tokens, pos));
                expr.push(subtree);
            },
            Rparen => {
                return Ok(Sequence(expr))
            }
        }
        *pos += 1;
    }

    Ok(Sequence(expr))
}

fn get_ast(tokens: &[Token]) -> Result<Expression, Error> {
    let mut pos = 0;

    _get_ast(tokens, &mut pos)
}

/// Parses the input lambda expression to a `Term`.
///
/// # Example
/// ```
/// use lambda_calculus::parser::parse;
/// use lambda_calculus::arithmetic::{succ, pred};
///
/// assert_eq!(parse(&"λ λ λ 2 (3 2 1)"), Ok(succ()));
/// assert_eq!(parse(&"λλλ3(λλ1(24))(λ2)(λ1)"), Ok(pred()));
/// ```
pub fn parse(input: &str) -> Result<Term, Error> {
    let tokens = try!(tokenize(input));
    let ast = try!(get_ast(&tokens));

    let exprs = try!(if let Sequence(exprs) = ast { Ok(exprs) } else { Err(InvalidExpression) });

    let mut stack = Vec::new();
    let mut output = Vec::new();
    let term = fold_exprs(&exprs, &mut stack, &mut output);

    term
}

fn fold_exprs(exprs: &[Expression], stack: &mut Vec<Expression>, output: &mut Vec<Term>)
    -> Result<Term, Error>
{
    let mut iter = exprs.iter();

    while let Some(ref expr) = iter.next() {
        match **expr {
            Variable(i) => output.push(Var(i)),
            Abstraction => stack.push(Abstraction),
            Sequence(ref exprs) => {
                let mut stack2 = Vec::new();
                let mut output2 = Vec::new();
                let subexpr = try!(fold_exprs(&exprs, &mut stack2, &mut output2));
                output.push(subexpr)
            }
        }
    }

    let mut ret = try!(fold_terms(output.drain(..).collect()));

    while let Some(Abstraction) = stack.pop() {
        ret = abs(ret);
    }

    Ok(ret)
}

fn fold_terms(mut terms: Vec<Term>) -> Result<Term, Error> {
    if terms.len() > 1 {
        terms.reverse();
        let fst = terms.pop().unwrap();
        terms.reverse();
        Ok( terms.into_iter().fold(fst, |acc, t| app(acc, t)) )
    } else if terms.len() == 1 {
        Ok( terms.pop().unwrap() )
    } else {
        Err(EmptyExpression)
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn tokenization_error() {
        assert_eq!(tokenize(&"λλx2"), Err(InvalidCharacter((4, 'x'))))
    }

    #[test]
    fn tokenization_success() {
        let quine = "λ 1 ( (λ 1 1) (λ λ λ λ λ 1 4 (3 (5 5) 2) ) ) 1";
        let tokens = tokenize(&quine);

        assert!(tokens.is_ok());
        assert_eq!(tokens.unwrap(), vec![Lambda, Number(1), Lparen, Lparen, Lambda, Number(1),
            Number(1), Rparen, Lparen, Lambda, Lambda, Lambda, Lambda, Lambda, Number(1),
            Number(4), Lparen, Number(3), Lparen, Number(5), Number(5), Rparen, Number(2),
            Rparen, Rparen, Rparen, Number(1)]);
    }

    #[test]
    fn alternative_lambda_parsing() {
        assert_eq!(parse(&"\\\\\\2(321)"), parse(&"λλλ2(321)"))
    }

    #[test]
    fn succ_ast() {
        let tokens = tokenize(&"λλλ2(321)").unwrap();
        let ast = get_ast(&tokens);

        assert_eq!(ast,
            Ok(Sequence(vec![
                Abstraction,
                Abstraction,
                Abstraction,
                Variable(2),
                Sequence(vec![
                    Variable(3),
                    Variable(2),
                    Variable(1)
                ])
            ])
        ));
    }

    #[test]
    fn parse_y() {
        let y = "λ(λ2(11))(λ2(11))";
        assert_eq!(&*format!("{}", parse(&y).expect("parsing Y failed!")), y);
    }

    #[test]
    fn parse_quine() {
        let quine = "λ1((λ11)(λλλλλ14(3(55)2)))1";
        assert_eq!(&*format!("{}", parse(&quine).expect("parsing QUINE failed!")), quine);
    }

    #[test]
    fn parse_blc() {
        let blc = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))\
                   (λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        assert_eq!(&*format!("{}", parse(&blc).expect("parsing BLC failed!")), blc);
    }
}
<commit_msg>Document that a backslash is parser-friendly too<commit_after>//! A parser for lambda expressions with
//! [De Bruijn indices](https://en.wikipedia.org/wiki/De_Bruijn_index)

use term::*;
use term::Term::*;
use self::Token::*;
use self::Error::*;
use self::Expression::*;

/// A type to represent a parsing error.
#[derive(Debug, PartialEq)]
pub enum Error {
    InvalidCharacter((usize, char)),
    InvalidExpression,
    EmptyExpression
}

#[derive(Debug, PartialEq)]
enum Token {
    Lambda,
    Lparen,
    Rparen,
    Number(usize)
}

fn tokenize(input: &str) -> Result<Vec<Token>, Error> {
    let mut chars = input.chars();
    let mut tokens = Vec::new();
    let mut position = 0;

    while let Some(c) = chars.next() {
        match c {
     '\\' | 'λ' => { tokens.push(Lambda) },
            '(' => { tokens.push(Lparen) },
            ')' => { tokens.push(Rparen) },
             x  => {
                if let Some(n) = x.to_digit(16) {
                    tokens.push(Number(n as usize))
                } else if x.is_whitespace() {
                    ()
                } else {
                    return Err(InvalidCharacter((position, x)))
                }
            }
        }
        position += if c == 'λ' { 2 } else { 1 };
    }

    Ok(tokens)
}

#[derive(Debug, PartialEq)]
enum Expression {
    Abstraction,
    Sequence(Vec<Expression>),
    Variable(usize)
}

fn _get_ast(tokens: &[Token], pos: &mut usize) -> Result<Expression, Error> {
    let mut expr = Vec::new();

    if tokens.is_empty() { return Err(EmptyExpression) }

    while *pos < tokens.len() {
        match tokens[*pos] {
            Lambda => {
                expr.push(Abstraction)
            },
            Number(i) => {
                expr.push(Variable(i))
            },
            Lparen => {
                *pos += 1;
                let subtree = try!(_get_ast(&tokens, pos));
                expr.push(subtree);
            },
            Rparen => {
                return Ok(Sequence(expr))
            }
        }
        *pos += 1;
    }

    Ok(Sequence(expr))
}

fn get_ast(tokens: &[Token]) -> Result<Expression, Error> {
    let mut pos = 0;

    _get_ast(tokens, &mut pos)
}

/// Parses the input lambda expression to a `Term`; the lambda can be represented either with the
/// greek letter (λ) or a backslash (\\ - less aesthetic, but only one byte in size).
///
/// # Example
/// ```
/// use lambda_calculus::parser::parse;
/// use lambda_calculus::arithmetic::{succ, pred};
///
/// assert_eq!(parse(&"λ λ λ 2 (3 2 1)"), Ok(succ()));
/// assert_eq!(parse(&r#"\ \ \ 2 (3 2 1)"#), Ok(succ()));
/// assert_eq!(parse(&"λλλ3(λλ1(24))(λ2)(λ1)"), Ok(pred()));
/// ```
pub fn parse(input: &str) -> Result<Term, Error> {
    let tokens = try!(tokenize(input));
    let ast = try!(get_ast(&tokens));

    let exprs = try!(if let Sequence(exprs) = ast { Ok(exprs) } else { Err(InvalidExpression) });

    let mut stack = Vec::new();
    let mut output = Vec::new();
    let term = fold_exprs(&exprs, &mut stack, &mut output);

    term
}

fn fold_exprs(exprs: &[Expression], stack: &mut Vec<Expression>, output: &mut Vec<Term>)
    -> Result<Term, Error>
{
    let mut iter = exprs.iter();

    while let Some(ref expr) = iter.next() {
        match **expr {
            Variable(i) => output.push(Var(i)),
            Abstraction => stack.push(Abstraction),
            Sequence(ref exprs) => {
                let mut stack2 = Vec::new();
                let mut output2 = Vec::new();
                let subexpr = try!(fold_exprs(&exprs, &mut stack2, &mut output2));
                output.push(subexpr)
            }
        }
    }

    let mut ret = try!(fold_terms(output.drain(..).collect()));

    while let Some(Abstraction) = stack.pop() {
        ret = abs(ret);
    }

    Ok(ret)
}

fn fold_terms(mut terms: Vec<Term>) -> Result<Term, Error> {
    if terms.len() > 1 {
        terms.reverse();
        let fst = terms.pop().unwrap();
        terms.reverse();
        Ok( terms.into_iter().fold(fst, |acc, t| app(acc, t)) )
    } else if terms.len() == 1 {
        Ok( terms.pop().unwrap() )
    } else {
        Err(EmptyExpression)
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn tokenization_error() {
        assert_eq!(tokenize(&"λλx2"), Err(InvalidCharacter((4, 'x'))))
    }

    #[test]
    fn tokenization_success() {
        let quine = "λ 1 ( (λ 1 1) (λ λ λ λ λ 1 4 (3 (5 5) 2) ) ) 1";
        let tokens = tokenize(&quine);

        assert!(tokens.is_ok());
        assert_eq!(tokens.unwrap(), vec![Lambda, Number(1), Lparen, Lparen, Lambda, Number(1),
            Number(1), Rparen, Lparen, Lambda, Lambda, Lambda, Lambda, Lambda, Number(1),
            Number(4), Lparen, Number(3), Lparen, Number(5), Number(5), Rparen, Number(2),
            Rparen, Rparen, Rparen, Number(1)]);
    }

    #[test]
    fn alternative_lambda_parsing() {
        assert_eq!(parse(&"\\\\\\2(321)"), parse(&"λλλ2(321)"))
    }

    #[test]
    fn succ_ast() {
        let tokens = tokenize(&"λλλ2(321)").unwrap();
        let ast = get_ast(&tokens);

        assert_eq!(ast,
            Ok(Sequence(vec![
                Abstraction,
                Abstraction,
                Abstraction,
                Variable(2),
                Sequence(vec![
                    Variable(3),
                    Variable(2),
                    Variable(1)
                ])
            ])
        ));
    }

    #[test]
    fn parse_y() {
        let y = "λ(λ2(11))(λ2(11))";
        assert_eq!(&*format!("{}", parse(&y).expect("parsing Y failed!")), y);
    }

    #[test]
    fn parse_quine() {
        let quine = "λ1((λ11)(λλλλλ14(3(55)2)))1";
        assert_eq!(&*format!("{}", parse(&quine).expect("parsing QUINE failed!")), quine);
    }

    #[test]
    fn parse_blc() {
        let blc = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))\
                   (λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        assert_eq!(&*format!("{}", parse(&blc).expect("parsing BLC failed!")), blc);
    }
}
<|endoftext|>


<commit_before>use log::{error, warn};
use std::cmp;
use std::fs::File;
use std::io::{Error, ErrorKind, Read, Result};
use std::path::Path;

use super::parser;
use super::{Document, Object, ObjectId};
use crate::object_stream::ObjectStream;
use crate::xref::XrefEntry;

impl Document {
	/// Load PDF document from specified file path.
	#[inline]
	pub fn load<P: AsRef<Path>>(path: P) -> Result<Document> {
		let file = File::open(path)?;
		let buffer = Vec::with_capacity(file.metadata()?.len() as usize);
		Self::load_internal(file, buffer)
	}

	/// Load PDF document from arbitrary source
	#[inline]
	pub fn load_from<R: Read>(source: R) -> Result<Document> {
		let buffer = Vec::<u8>::new();
		Self::load_internal(source, buffer)
	}

	fn load_internal<R: Read>(mut source: R, mut buffer: Vec<u8>) -> Result<Document> {
		source.read_to_end(&mut buffer)?;

		let mut reader = Reader {
			buffer,
			document: Document::new(),
		};

		reader.read()?;
		Ok(reader.document)
	}
}

pub struct Reader {
	buffer: Vec<u8>,
	document: Document,
}

impl Reader {
	/// Read whole document.
	fn read(&mut self) -> Result<()> {
		// The document structure can be expressed in PEG as:
		//   document <- header indirect_object* xref trailer xref_start
		let version = parser::header().parse(&self.buffer).map_err(|_| Error::new(ErrorKind::InvalidData, "Not a valid PDF file (header)."))?;

		let xref_start = Self::get_xref_start(&self.buffer)?;

		let (mut xref, mut trailer) = parser::xref_and_trailer(&self)
			.parse(&self.buffer[xref_start..])
			.map_err(|err| Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (xref_and_trailer).\n{:?}", err)))?;

		// Read previous Xrefs of linearized or incremental updated document.
		let mut prev_xref_start = trailer.remove(b"Prev");
		while let Some(prev) = prev_xref_start.and_then(|offset| offset.as_i64()) {
			let (prev_xref, mut prev_trailer) = parser::xref_and_trailer(&self)
				.parse(&self.buffer[prev as usize..])
				.map_err(|err| Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (prev xref_and_trailer).\n{:?}", err)))?;
			xref.extend(prev_xref);

			// Read xref stream in hybrid-reference file
			let prev_xref_stream_start = trailer.remove(b"XRefStm");
			if let Some(prev) = prev_xref_stream_start.and_then(|offset| offset.as_i64()) {
				let (prev_xref, _) = parser::xref_and_trailer(&self)
					.parse(&self.buffer[prev as usize..])
					.map_err(|_| Error::new(ErrorKind::InvalidData, "Not a valid PDF file (prev xref_and_trailer)."))?;
				xref.extend(prev_xref);
			}

			prev_xref_start = prev_trailer.remove(b"Prev");
		}

		let xref_entry_count = xref.max_id() + 1;
		if xref.size != xref_entry_count {
			warn!("Size entry of trailer dictionary is {}, correct value is {}.", xref.size, xref_entry_count);
			xref.size = xref_entry_count;
		}

		self.document.version = version;
		self.document.max_id = xref.size - 1;
		self.document.trailer = trailer;
		self.document.reference_table = xref;

		let mut zero_length_streams = vec![];
		for entry in self.document.reference_table.entries.values().filter(|entry| entry.is_normal()) {
			if let XrefEntry::Normal { offset, .. } = *entry {
				let read_result = self.read_object(offset as usize);
				match read_result {
					Ok((object_id, mut object)) => {
						if let Object::Stream(ref mut stream) = object {
							if stream.dict.type_is(b"ObjStm") {
								let mut obj_stream = ObjectStream::new(stream);
								self.document.objects.append(&mut obj_stream.objects);
							} else if stream.content.is_empty() {
								zero_length_streams.push(object_id);
							}
						}
						self.document.objects.insert(object_id, object);
					}
					Err(err) => {
						error!("{:?}", err);
					}
				}
			};
		}

		for object_id in zero_length_streams {
			if let Some(length) = self.get_stream_length(object_id) {
				if let Some(ref mut object) = self.document.get_object_mut(object_id) {
					if let Object::Stream(ref mut stream) = object {
						if let Some(start) = stream.start_position {
							let end = start + length as usize;
							stream.set_content(self.buffer[start..end].to_vec());
						}
					}
				}
			}
		}

		Ok(())
	}

	fn get_stream_length(&self, object_id: ObjectId) -> Option<i64> {
		let object = self.document.get_object(object_id).unwrap();
		match object {
			Object::Stream(ref stream) => stream.dict.get(b"Length").and_then(|value| {
				if let Some(id) = value.as_reference() {
					return self.document.get_object(id).and_then(Object::as_i64);
				}
				value.as_i64()
			}),
			_ => None,
		}
	}

	/// Get object offset by object id.
	fn get_offset(&self, id: ObjectId) -> Option<u32> {
		if let Some(entry) = self.document.reference_table.get(id.0) {
			match *entry {
				XrefEntry::Normal { offset, generation } => {
					if id.1 == generation {
						Some(offset)
					} else {
						None
					}
				}
				_ => None,
			}
		} else {
			None
		}
	}

	pub fn get_object(&self, id: ObjectId) -> Option<Object> {
		if let Some(offset) = self.get_offset(id) {
			if let Ok((_, obj)) = self.read_object(offset as usize) {
				return Some(obj);
			}
		}
		None
	}

	fn read_object(&self, offset: usize) -> Result<(ObjectId, Object)> {
		parser::indirect_object(self)
			.parse(&self.buffer[offset..])
			.map_err(|err| Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (read object at {}).\n{:?}", offset, err)))
	}

	fn get_xref_start(buffer: &[u8]) -> Result<usize> {
		let seek_pos = buffer.len() - cmp::min(buffer.len(), 512);
		Self::search_substring(buffer, b"%%EOF", seek_pos)
			.and_then(|eof_pos| Self::search_substring(buffer, b"startxref", eof_pos - 25))
			.and_then(|xref_pos| match parser::xref_start().parse(&buffer[xref_pos..]) {
				Ok(startxref) => Some(startxref as usize),
				_ => None,
			})
			.ok_or_else(|| Error::new(ErrorKind::InvalidData, "Not a valid PDF file (xref_start)."))
	}

	fn search_substring(buffer: &[u8], pattern: &[u8], start_pos: usize) -> Option<usize> {
		let mut seek_pos = start_pos;
		let mut index = 0;

		while seek_pos < buffer.len() && index < pattern.len() {
			if buffer[seek_pos] == pattern[index] {
				index += 1;
			} else if index > 0 {
				seek_pos -= index;
				index = 0;
			}
			seek_pos += 1;

			if index == pattern.len() {
				return Some(seek_pos - index);
			}
		}

		None
	}
}

#[test]
fn load_document() {
	let mut doc = Document::load("assets/example.pdf").unwrap();
	assert_eq!(doc.version, "1.5");
	doc.save("test_2_load.pdf").unwrap();
}
<commit_msg>Check offsets read from file to avoid panics<commit_after>use log::{error, warn};
use std::cmp;
use std::fs::File;
use std::io::{Error, ErrorKind, Read, Result};
use std::path::Path;

use super::parser;
use super::{Document, Object, ObjectId};
use crate::object_stream::ObjectStream;
use crate::xref::XrefEntry;

impl Document {
	/// Load PDF document from specified file path.
	#[inline]
	pub fn load<P: AsRef<Path>>(path: P) -> Result<Document> {
		let file = File::open(path)?;
		let buffer = Vec::with_capacity(file.metadata()?.len() as usize);
		Self::load_internal(file, buffer)
	}

	/// Load PDF document from arbitrary source
	#[inline]
	pub fn load_from<R: Read>(source: R) -> Result<Document> {
		let buffer = Vec::<u8>::new();
		Self::load_internal(source, buffer)
	}

	fn load_internal<R: Read>(mut source: R, mut buffer: Vec<u8>) -> Result<Document> {
		source.read_to_end(&mut buffer)?;

		let mut reader = Reader {
			buffer,
			document: Document::new(),
		};

		reader.read()?;
		Ok(reader.document)
	}
}

pub struct Reader {
	buffer: Vec<u8>,
	document: Document,
}

impl Reader {
	/// Read whole document.
	fn read(&mut self) -> Result<()> {
		// The document structure can be expressed in PEG as:
		//   document <- header indirect_object* xref trailer xref_start
		let version = parser::header().parse(&self.buffer).map_err(|_| Error::new(ErrorKind::InvalidData, "Not a valid PDF file (header)."))?;

		let xref_start = Self::get_xref_start(&self.buffer)?;
		if xref_start > self.buffer.len() {
			return Err(Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (xref_start)")));
		}

		let (mut xref, mut trailer) = parser::xref_and_trailer(&self)
			.parse(&self.buffer[xref_start..])
			.map_err(|err| Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (xref_and_trailer).\n{:?}", err)))?;

		// Read previous Xrefs of linearized or incremental updated document.
		let mut prev_xref_start = trailer.remove(b"Prev");
		while let Some(prev) = prev_xref_start.and_then(|offset| offset.as_i64()) {
			let prev = prev as usize;
			if prev > self.buffer.len() {
				return Err(Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (prev_xref_start)")));
			}
			let (prev_xref, mut prev_trailer) = parser::xref_and_trailer(&self)
				.parse(&self.buffer[prev..])
				.map_err(|err| Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (prev xref_and_trailer).\n{:?}", err)))?;
			xref.extend(prev_xref);

			// Read xref stream in hybrid-reference file
			let prev_xref_stream_start = trailer.remove(b"XRefStm");
			if let Some(prev) = prev_xref_stream_start.and_then(|offset| offset.as_i64()) {
				let prev = prev as usize;
				if prev > self.buffer.len() {
					return Err(Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (prev_xref_stream_start)")));
				}
				let (prev_xref, _) = parser::xref_and_trailer(&self)
					.parse(&self.buffer[prev..])
					.map_err(|_| Error::new(ErrorKind::InvalidData, "Not a valid PDF file (prev xref_and_trailer)."))?;
				xref.extend(prev_xref);
			}

			prev_xref_start = prev_trailer.remove(b"Prev");
		}

		let xref_entry_count = xref.max_id() + 1;
		if xref.size != xref_entry_count {
			warn!("Size entry of trailer dictionary is {}, correct value is {}.", xref.size, xref_entry_count);
			xref.size = xref_entry_count;
		}

		self.document.version = version;
		self.document.max_id = xref.size - 1;
		self.document.trailer = trailer;
		self.document.reference_table = xref;

		let mut zero_length_streams = vec![];
		for entry in self.document.reference_table.entries.values().filter(|entry| entry.is_normal()) {
			if let XrefEntry::Normal { offset, .. } = *entry {
				let read_result = self.read_object(offset as usize);
				match read_result {
					Ok((object_id, mut object)) => {
						if let Object::Stream(ref mut stream) = object {
							if stream.dict.type_is(b"ObjStm") {
								let mut obj_stream = ObjectStream::new(stream);
								self.document.objects.append(&mut obj_stream.objects);
							} else if stream.content.is_empty() {
								zero_length_streams.push(object_id);
							}
						}
						self.document.objects.insert(object_id, object);
					}
					Err(err) => {
						error!("{:?}", err);
					}
				}
			};
		}

		for object_id in zero_length_streams {
			if let Some(length) = self.get_stream_length(object_id) {
				if let Some(ref mut object) = self.document.get_object_mut(object_id) {
					if let Object::Stream(ref mut stream) = object {
						if let Some(start) = stream.start_position {
							let end = start + length as usize;
							stream.set_content(self.buffer[start..end].to_vec());
						}
					}
				}
			}
		}

		Ok(())
	}

	fn get_stream_length(&self, object_id: ObjectId) -> Option<i64> {
		let object = self.document.get_object(object_id).unwrap();
		match object {
			Object::Stream(ref stream) => stream.dict.get(b"Length").and_then(|value| {
				if let Some(id) = value.as_reference() {
					return self.document.get_object(id).and_then(Object::as_i64);
				}
				value.as_i64()
			}),
			_ => None,
		}
	}

	/// Get object offset by object id.
	fn get_offset(&self, id: ObjectId) -> Option<u32> {
		if let Some(entry) = self.document.reference_table.get(id.0) {
			match *entry {
				XrefEntry::Normal { offset, generation } => {
					if id.1 == generation {
						Some(offset)
					} else {
						None
					}
				}
				_ => None,
			}
		} else {
			None
		}
	}

	pub fn get_object(&self, id: ObjectId) -> Option<Object> {
		if let Some(offset) = self.get_offset(id) {
			if let Ok((_, obj)) = self.read_object(offset as usize) {
				return Some(obj);
			}
		}
		None
	}

	fn read_object(&self, offset: usize) -> Result<(ObjectId, Object)> {
		if offset > self.buffer.len() {
			return Err(Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (read at offset {})", offset)));
		}
		parser::indirect_object(self)
			.parse(&self.buffer[offset..])
			.map_err(|err| Error::new(ErrorKind::InvalidData, format!("Not a valid PDF file (read object at {}).\n{:?}", offset, err)))
	}

	fn get_xref_start(buffer: &[u8]) -> Result<usize> {
		let seek_pos = buffer.len() - cmp::min(buffer.len(), 512);
		Self::search_substring(buffer, b"%%EOF", seek_pos)
			.and_then(|eof_pos| Self::search_substring(buffer, b"startxref", eof_pos - 25))
			.ok_or_else(|| Error::new(ErrorKind::InvalidData, "Not a valid PDF file (xref_start)."))
			.and_then(|xref_pos| if xref_pos <= buffer.len() {
				match parser::xref_start().parse(&buffer[xref_pos..]) {
					Ok(startxref) => Ok(startxref as usize),
					Err(_) => Err(Error::new(ErrorKind::InvalidData, "Not a valid PDF file (xref_start).")),
				}
			} else {
				Err(Error::new(ErrorKind::InvalidData, "Not a valid PDF file (xref_pos)"))
			})
	}

	fn search_substring(buffer: &[u8], pattern: &[u8], start_pos: usize) -> Option<usize> {
		let mut seek_pos = start_pos;
		let mut index = 0;

		while seek_pos < buffer.len() && index < pattern.len() {
			if buffer[seek_pos] == pattern[index] {
				index += 1;
			} else if index > 0 {
				seek_pos -= index;
				index = 0;
			}
			seek_pos += 1;

			if index == pattern.len() {
				return Some(seek_pos - index);
			}
		}

		None
	}
}

#[test]
fn load_document() {
	let mut doc = Document::load("assets/example.pdf").unwrap();
	assert_eq!(doc.version, "1.5");
	doc.save("test_2_load.pdf").unwrap();
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use serialize::json::Json;

use template::{Template, TemplateElement};
use template::TemplateElement::{RawString, Expression, Comment, HelperBlock, HTMLExpression};
use registry::Registry;
use context::{Context, JsonRender};

pub static EMPTY: &'static str = "";
static NULL_VALUE: Json = Json::Null;

#[deriving(Show, Copy)]
pub struct RenderError {
    pub desc: &'static str
}

pub struct RenderContext {
    partials: HashMap<String, String>,
    path: String,
    local_variables: HashMap<String, Json>
}

impl RenderContext {
    pub fn new() -> RenderContext {
        RenderContext {
            partials: HashMap::new(),
            path: ".".to_string(),
            local_variables: HashMap::new()
        }
    }

    pub fn get_rendered_partial(&self, name: String) -> Option<&String> {
        self.partials.get(&name)
    }

    pub fn set_rendered_partial(&mut self, name: String, result: String) {
        self.partials.insert(name, result);
    }

    pub fn get_path(&self) -> &String {
        &self.path
    }

    pub fn set_path(&mut self, path: String) {
        self.path = path
    }

    pub fn set_local_var(&mut self, name: String, value: Json) {
        self.local_variables.insert(name, value);
    }

    pub fn clear_local_vars(&mut self){
        self.local_variables.clear();
    }

    pub fn promote_local_vars(&mut self) {
        let mut new_map: HashMap<String, Json> = HashMap::new();
        for key in self.local_variables.keys() {
            let mut new_key = String::new();
            new_key.push_str("@../");
            new_key.push_str(key.slice_from(1));

            let v = self.local_variables.get(key).unwrap().clone();
            new_map.insert(new_key, v);
        }
        self.local_variables = new_map;
    }

    pub fn demote_local_vars(&mut self) {
        let mut new_map: HashMap<String, Json> = HashMap::new();
        for key in self.local_variables.keys() {
            if key.starts_with("@../") {
                let mut new_key = String::new();
                new_key.push('@');
                new_key.push_str(key.slice_from(4));

                let v = self.local_variables.get(key).unwrap().clone();
                new_map.insert(new_key, v);
            }
        }
        self.local_variables = new_map;
    }

    pub fn get_local_var(&self, name: &String) -> &Json {
        match self.local_variables.get(name) {
            Some(ref j) => *j,
            None => &NULL_VALUE
        }
    }
}

pub trait Renderable {
    fn render(&self, ctx: &Context, registry: &Registry, rc: &mut RenderContext) -> Result<String, RenderError>;
}

impl Renderable for Template {
    fn render(&self, ctx: &Context, registry: &Registry, rc: &mut RenderContext) -> Result<String, RenderError> {
        let mut output = String::new();
        let mut iter = self.elements.iter();
        for t in iter {
            let c = ctx;
            match t.render(c, registry, rc) {
                Ok(r) => output.push_str(r.as_slice()),
                Err(e) => return Err(e)
            }
        }
        return Ok(output);
    }
}

pub fn render_error(desc: &'static str) -> RenderError {
    RenderError {
        desc: desc
    }
}

impl Renderable for TemplateElement {
    fn render(&self, ctx: &Context, registry: &Registry, rc: &mut RenderContext) -> Result<String, RenderError> {
        match *self {
            RawString(ref v) => {
                return Ok(v.clone());
            },
            Expression(ref v) => {
                let value = if (*v).starts_with("@") {
                    rc.get_local_var(v)
                } else {
                    ctx.navigate(rc.get_path(), v)
                };
                let rendered = value.render();
                Ok(rendered.replace("&", "&")
                   .replace("\"", """)
                   .replace("<", "<")
                   .replace(">", ">"))
            },
            HTMLExpression(ref v) => {
                let value = if (*v).starts_with("@") {
                    rc.get_local_var(v)
                } else {
                    ctx.navigate(rc.get_path(), v)
                };
                Ok(value.render())
            },
            HelperBlock(ref helper) => {
                match registry.get_helper(helper.name()) {
                    Some(d) => {
                        return (**d).resolve(ctx, helper, registry, rc);
                    },
                    None => {
                        return Err(RenderError{
                            desc: "Helper not defined."
                        });
                    }
                }
            },
            Comment => {
                return Ok(EMPTY.to_string());
            }
        }
    }
}

#[test]
fn test_raw_string() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let raw_string = RawString("
hello world
".to_string());
    assert_eq!(raw_string.render(
        &Context::null(), &r, &mut rc).unwrap(), "
hello world
".to_string());
}

#[test]
fn test_expression() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let element = Expression("hello".to_string());
    let mut m: HashMap<String, String> = HashMap::new();
    let value = "

".to_string();

    m.insert("hello".to_string(), value);

    let ctx = Context::wraps(&m);

    assert_eq!(element.render(&ctx, &r, &mut rc).unwrap(), "<p></p>".to_string());
}

#[test]
fn test_html_expression() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let element = HTMLExpression("hello".to_string());
    let mut m: HashMap<String, String> = HashMap::new();
    let value = "world";
    m.insert("hello".to_string(), value.to_string());

    let ctx = Context::wraps(&m);

    assert_eq!(element.render(&ctx, &r, &mut rc).unwrap(), value.to_string());
}

#[test]
fn test_template() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let mut elements: Vec<TemplateElement> = Vec::new();

    let e1 = RawString("
".to_string());
    elements.push(e1);

    let e2 = Expression("hello".to_string());
    elements.push(e2);

    let e3 = RawString("
".to_string());
    elements.push(e3);

    let e4 = Comment;
    elements.push(e4);

    let template = Template {
        elements: elements
    };

    let mut m: HashMap<String, String> = HashMap::new();
    let value = "world".to_string();
    m.insert("hello".to_string(), value);

    let ctx = Context::wraps(&m);

    assert_eq!(template.render(&ctx, &r, &mut rc).unwrap(), "
world
".to_string());
}
<commit_msg>added test for render context promotion and demotion.<commit_after>use std::collections::HashMap;
use serialize::json::Json;

use template::{Template, TemplateElement};
use template::TemplateElement::{RawString, Expression, Comment, HelperBlock, HTMLExpression};
use registry::Registry;
use context::{Context, JsonRender};

pub static EMPTY: &'static str = "";
static NULL_VALUE: Json = Json::Null;

#[deriving(Show, Copy)]
pub struct RenderError {
    pub desc: &'static str
}

pub struct RenderContext {
    partials: HashMap<String, String>,
    path: String,
    local_variables: HashMap<String, Json>
}

impl RenderContext {
    pub fn new() -> RenderContext {
        RenderContext {
            partials: HashMap::new(),
            path: ".".to_string(),
            local_variables: HashMap::new()
        }
    }

    pub fn get_rendered_partial(&self, name: String) -> Option<&String> {
        self.partials.get(&name)
    }

    pub fn set_rendered_partial(&mut self, name: String, result: String) {
        self.partials.insert(name, result);
    }

    pub fn get_path(&self) -> &String {
        &self.path
    }

    pub fn set_path(&mut self, path: String) {
        self.path = path
    }

    pub fn set_local_var(&mut self, name: String, value: Json) {
        self.local_variables.insert(name, value);
    }

    pub fn clear_local_vars(&mut self){
        self.local_variables.clear();
    }

    pub fn promote_local_vars(&mut self) {
        let mut new_map: HashMap<String, Json> = HashMap::new();
        for key in self.local_variables.keys() {
            let mut new_key = String::new();
            new_key.push_str("@../");
            new_key.push_str(key.slice_from(1));

            let v = self.local_variables.get(key).unwrap().clone();
            new_map.insert(new_key, v);
        }
        self.local_variables = new_map;
    }

    pub fn demote_local_vars(&mut self) {
        let mut new_map: HashMap<String, Json> = HashMap::new();
        for key in self.local_variables.keys() {
            if key.starts_with("@../") {
                let mut new_key = String::new();
                new_key.push('@');
                new_key.push_str(key.slice_from(4));

                let v = self.local_variables.get(key).unwrap().clone();
                new_map.insert(new_key, v);
            }
        }
        self.local_variables = new_map;
    }

    pub fn get_local_var(&self, name: &String) -> &Json {
        match self.local_variables.get(name) {
            Some(ref j) => *j,
            None => &NULL_VALUE
        }
    }
}

pub trait Renderable {
    fn render(&self, ctx: &Context, registry: &Registry, rc: &mut RenderContext) -> Result<String, RenderError>;
}

impl Renderable for Template {
    fn render(&self, ctx: &Context, registry: &Registry, rc: &mut RenderContext) -> Result<String, RenderError> {
        let mut output = String::new();
        let mut iter = self.elements.iter();
        for t in iter {
            let c = ctx;
            match t.render(c, registry, rc) {
                Ok(r) => output.push_str(r.as_slice()),
                Err(e) => return Err(e)
            }
        }
        return Ok(output);
    }
}

pub fn render_error(desc: &'static str) -> RenderError {
    RenderError {
        desc: desc
    }
}

impl Renderable for TemplateElement {
    fn render(&self, ctx: &Context, registry: &Registry, rc: &mut RenderContext) -> Result<String, RenderError> {
        match *self {
            RawString(ref v) => {
                return Ok(v.clone());
            },
            Expression(ref v) => {
                let value = if (*v).starts_with("@") {
                    rc.get_local_var(v)
                } else {
                    ctx.navigate(rc.get_path(), v)
                };
                let rendered = value.render();
                Ok(rendered.replace("&", "&")
                   .replace("\"", """)
                   .replace("<", "<")
                   .replace(">", ">"))
            },
            HTMLExpression(ref v) => {
                let value = if (*v).starts_with("@") {
                    rc.get_local_var(v)
                } else {
                    ctx.navigate(rc.get_path(), v)
                };
                Ok(value.render())
            },
            HelperBlock(ref helper) => {
                match registry.get_helper(helper.name()) {
                    Some(d) => {
                        return (**d).resolve(ctx, helper, registry, rc);
                    },
                    None => {
                        return Err(RenderError{
                            desc: "Helper not defined."
                        });
                    }
                }
            },
            Comment => {
                return Ok(EMPTY.to_string());
            }
        }
    }
}

#[test]
fn test_raw_string() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let raw_string = RawString("
hello world
".to_string());
    assert_eq!(raw_string.render(
        &Context::null(), &r, &mut rc).unwrap(), "
hello world
".to_string());
}

#[test]
fn test_expression() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let element = Expression("hello".to_string());
    let mut m: HashMap<String, String> = HashMap::new();
    let value = "

".to_string();

    m.insert("hello".to_string(), value);

    let ctx = Context::wraps(&m);

    assert_eq!(element.render(&ctx, &r, &mut rc).unwrap(), "<p></p>".to_string());
}

#[test]
fn test_html_expression() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let element = HTMLExpression("hello".to_string());
    let mut m: HashMap<String, String> = HashMap::new();
    let value = "world";
    m.insert("hello".to_string(), value.to_string());

    let ctx = Context::wraps(&m);

    assert_eq!(element.render(&ctx, &r, &mut rc).unwrap(), value.to_string());
}

#[test]
fn test_template() {
    let r = Registry::new();
    let mut rc = RenderContext::new();
    let mut elements: Vec<TemplateElement> = Vec::new();

    let e1 = RawString("
".to_string());
    elements.push(e1);

    let e2 = Expression("hello".to_string());
    elements.push(e2);

    let e3 = RawString("
".to_string());
    elements.push(e3);

    let e4 = Comment;
    elements.push(e4);

    let template = Template {
        elements: elements
    };

    let mut m: HashMap<String, String> = HashMap::new();
    let value = "world".to_string();
    m.insert("hello".to_string(), value);

    let ctx = Context::wraps(&m);

    assert_eq!(template.render(&ctx, &r, &mut rc).unwrap(), "
world
".to_string());
}

#[test]
fn test_render_context_promotion_and_demotion() {
    use serialize::json::ToJson;
    let mut render_context = RenderContext::new();

    render_context.set_local_var("@index".to_string(), 0u.to_json());

    render_context.promote_local_vars();

    assert_eq!(render_context.get_local_var(&"@../index".to_string()),
               &0u.to_json());

    render_context.demote_local_vars();

    assert_eq!(render_context.get_local_var(&"@index".to_string()),
               &0u.to_json());
}
<|endoftext|>