<commit_before>use super::bit;
use super::error::{Error, ErrorKind};
use super::index_builder::IndexBuilder;
use super::query::Query;
use super::result::Result;
use super::utf8::{COMMA, CR, HT, LF, RIGHT_BRACE, SPACE};
use fnv::{FnvHashMap, FnvHashSet};
use std::cell::RefCell;

pub struct Parser {
    pub index_builder: IndexBuilder,
    stats: Vec<FnvHashSet<usize>>,
    colon_positions: RefCell<Vec<Vec<usize>>>,
}

impl Parser {
    pub fn new(max_depth: usize, num_nodes: usize) -> Self {
        let index_builder = IndexBuilder::new(max_depth);
        let colon_positions = RefCell::new(vec![Vec::new(); max_depth]);
        let stats = vec![Default::default(); num_nodes];
        Self {
            index_builder,
            stats,
            colon_positions,
        }
    }

    #[inline]
    pub fn basic_parse<'a>(
        &mut self,
        rec: &'a [u8],
        queries: &FnvHashMap<&[u8], Query>,
        start: usize,
        end: usize,
        level: usize,
        set_stats: bool,
        results: &mut Vec<Option<&'a [u8]>>,
    ) -> Result<()> {
        generate_colon_positions(
            &self.index_builder.index,
            start,
            end,
            level,
            &mut *self.colon_positions.borrow_mut(),
        );

        let mut found_num = 0;
        let mut vei = end;
        let cp_len = self.colon_positions.borrow()[level].len();
        for i in (0..cp_len).rev() {
            let (fsi, fei) = search_pre_field_indices(
                &self.index_builder.b_quote,
                if i > 0 {
                    self.colon_positions.borrow()[level][i - 1]
                } else {
                    start
                },
                self.colon_positions.borrow()[level][i],
            )?;
            let field = &rec[fsi + 1..fei];
            if let Some(query) = queries.get(field) {
                let (vsi, vei) = search_post_value_indices(
                    rec,
                    self.colon_positions.borrow()[level][i] + 1,
                    vei,
                    if i == cp_len - 1 { RIGHT_BRACE } else { COMMA },
                )?;
                found_num += 1;
                if set_stats && !self.stats[query.i].contains(&i) {
                    self.stats[query.i].insert(i);
                }
                if let Some(ref children) = query.children {
                    self.basic_parse(rec, children, vsi, vei, level + 1, set_stats, results)?;
                }
                if query.target {
                    results[query.ri] = Some(&rec[vsi..vei + 1]);
                }
                if found_num == queries.len() {
                    return Ok(());
                }
            }
            vei = fsi - 1;
        }
        Ok(())
    }

    #[inline]
    pub fn speculative_parse<'a>(&self, rec: &'a [u8], queries: &FnvHashMap<&[u8], Query>, start: usize, end: usize, level: usize, results: &mut Vec<Option<&'a [u8]>>) -> Result<bool> {
        generate_colon_positions(
            &self.index_builder.index,
            start,
            end,
            level,
            &mut *self.colon_positions.borrow_mut(),
        );

        for (&s, q) in queries {
            let mut found = false;
            for &i in &self.stats[q.i] {
                let cp_len = self.colon_positions.borrow()[level].len();
                if i >= cp_len {
                    continue;
                }
                let (fsi, fei) = search_pre_field_indices(
                    &self.index_builder.b_quote,
                    if i > 0 {
                        self.colon_positions.borrow()[level][i - 1]
                    } else {
                        start
                    },
                    self.colon_positions.borrow()[level][i],
                )?;
                let field = &rec[fsi + 1..fei];
                if s == field {
                    let vei = if i < cp_len - 1 {
                        let (nfsi, _) = search_pre_field_indices(
                            &self.index_builder.b_quote,
                            self.colon_positions.borrow()[level][i],
                            self.colon_positions.borrow()[level][i + 1],
                        )?;
                        nfsi - 1
                    } else {
                        end
                    };
                    let (vsi, vei) = search_post_value_indices(
                        rec,
                        self.colon_positions.borrow()[level][i] + 1,
                        vei,
                        if i == cp_len - 1 { RIGHT_BRACE } else { COMMA },
                    )?;
                    if let Some(ref children) = q.children {
                        found = self.speculative_parse(rec, children, vsi, vei, level + 1, results)?;
                    } else {
                        found = true;
                    }
                    if q.target {
                        results[q.ri] = Some(&rec[vsi..vei + 1]);
                    }
                    break;
                }
            }
            if !found {
                return Ok(false);
            }
        }
        Ok(true)
    }
}

#[inline]
fn generate_colon_positions(index: &[Vec<u64>], start: usize, end: usize, level: usize, colon_positions: &mut Vec<Vec<usize>>) {
    let cp = &mut colon_positions[level];
    cp.clear();
    for i in start / 64..(end + 63) / 64 {
        let mut m_colon = index[level][i];
        while m_colon != 0 {
            let m_bit = bit::e(m_colon);
            let offset = i * 64 + (m_bit.trailing_zeros() as usize);
            if start <= offset && offset <= end {
                cp.push(offset);
            }
            m_colon = bit::r(m_colon);
        }
    }
}

#[inline]
fn search_pre_field_indices(b_quote: &[u64], start: usize, end: usize) -> Result<(usize, usize)> {
    let mut si = 0;
    let mut ei = 0;
    let mut ei_set = false;
    let mut n_quote = 0;
    for i in (start / 64..(end + 63) / 64).rev() {
        let mut m_quote = b_quote[i];
        while m_quote != 0 {
            let m_bit = bit::e(m_quote);
            let offset = i * 64 + (m_bit.trailing_zeros() as usize);
            if end <= offset {
                break;
            }
            if start < offset {
                if ei_set {
                    si = offset;
                } else {
                    si = ei;
                    ei = offset;
                }
                n_quote += 1;
            }
            m_quote = bit::r(m_quote);
        }
        if n_quote >= 2 {
            break;
        }
        if n_quote == 1 && !ei_set {
            ei_set = true;
        }
    }
    if n_quote >= 2 {
        Ok((si, ei))
    } else {
        Err(Error::from(ErrorKind::InvalidRecord))
    }
}

#[inline]
fn search_post_value_indices(rec: &[u8], si: usize, ei: usize, ignore_once_char: u8) -> Result<(usize, usize)> {
    let mut si = si;
    let mut ei = ei;
    let mut ignore_once_char_ignored = false;
    let n = rec.len();
    while si < n {
        match rec[si] {
            SPACE | HT | LF | CR => {
                si += 1;
            }
            _ => {
                break;
            }
        }
    }
    if si == n {
        return Err(Error::from(ErrorKind::InvalidRecord));
    }
    while si <= ei {
        match rec[ei] {
            SPACE | HT | LF | CR => {
                ei -= 1;
            }
            char if char == ignore_once_char => {
                if ignore_once_char_ignored {
                    break;
                }
                ignore_once_char_ignored = true;
                ei -= 1;
            }
            _ => {
                break;
            }
        }
    }
    if ei < si {
        return Err(Error::from(ErrorKind::InvalidRecord));
    }
    Ok((si, ei))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_basic_parse() {
        let json_rec_str = r#"{ "aaa" : "AAA", "bbb" : 111, "ccc": ["C1", "C2"], "ddd" : { "d1" : "D1", "d2" : "D2", "d3": 333 }, "eee": { "e1": "EEE" } } "#;
        let json_rec = json_rec_str.as_bytes();

        let l = 10;
        let mut parser = Parser::new(l, 7);
        let r = parser.index_builder.build_structural_indices(json_rec);
        assert_eq!(Ok(()), r);

        let mut children = FnvHashMap::default();
        children.insert(
            "d1".as_bytes(),
            Query {
                i: 0,
                ri: 0,
                target: true,
                children: None,
            },
        );
        children.insert(
            "d3".as_bytes(),
            Query {
                i: 1,
                ri: 1,
                target: true,
                children: None,
            },
        );
        let mut queries = FnvHashMap::default();
        queries.insert(
            "aaa".as_bytes(),
            Query {
                i: 2,
                ri: 2,
                target: true,
                children: None,
            },
        );
        queries.insert(
            "bbb".as_bytes(),
            Query {
                i: 3,
                ri: 3,
                target: true,
                children: None,
            },
        );
        queries.insert(
            "ccc".as_bytes(),
            Query {
                i: 4,
                ri: 4,
                target: true,
                children: None,
            },
        );
        queries.insert(
            "ddd".as_bytes(),
            Query {
                i: 5,
                ri: 0,
                target: false,
                children: Some(children),
            },
        );
        queries.insert(
            "eee".as_bytes(),
            Query {
                i: 6,
                ri: 5,
                target: true,
                children: None,
            },
        );
        let mut results = vec![None, None, None, None, None, None];
        let result = parser.basic_parse(
            json_rec,
            &mut queries,
            0,
            json_rec.len() - 1,
            0,
            true,
            &mut results,
        );

        assert_eq!(Ok(()), result);
        assert_eq!(Some(r#""D1""#.as_bytes()), results[0]);
        assert_eq!(Some(r#"333"#.as_bytes()), results[1]);
        assert_eq!(Some(r#""AAA""#.as_bytes()), results[2]);
        assert_eq!(Some(r#"111"#.as_bytes()), results[3]);
        assert_eq!(Some(r#"["C1", "C2"]"#.as_bytes()), results[4]);
        assert_eq!(Some(r#"{ "e1": "EEE" }"#.as_bytes()), results[5]);
    }

    #[test]
    fn test_generate_colon_positions() {
        struct TestCase {
            index: Vec<Vec<u64>>,
            start: usize,
            end: usize,
            level: usize,
            want: Vec<usize>,
        }
        let test_cases = vec![
            TestCase {
                index: vec![vec![]],
                start: 0,
                end: 0,
                level: 0,
                want: vec![],
            },
            TestCase {
                index: vec![
                    vec![
                        0b00000001_11000000_00000000_00000000_00001000_00000000_00000011_10000000,
                    ],
                ],
                start: 8,
                end: 55,
                level: 0,
                want: vec![8, 9, 27, 54, 55],
            },
            TestCase {
                index: vec![
                    vec![
                        0b10000000_00000000_00000000_00000000_00001000_00000000_00000011_10000000,
                        0b00000001_11000001_00000000_00000011_00000000_00000000_00000000_00000001,
                    ],
                ],
                start: 8,
                end: 119,
                level: 0,
                want: vec![8, 9, 27, 63, 64, 96, 97, 112, 118, 119],
            },
        ];
        for t in test_cases {
            let mut cp = vec![Vec::new(); t.level + 1];
            generate_colon_positions(&t.index, t.start, t.end, t.level, &mut cp);
            assert_eq!(t.want, cp[0]);
        }
    }
}
<commit_msg>test(parser): refactor<commit_after>use super::bit;
use super::error::{Error, ErrorKind};
use super::index_builder::IndexBuilder;
use super::query::Query;
use super::result::Result;
use super::utf8::{COMMA, CR, HT, LF, RIGHT_BRACE, SPACE};
use fnv::{FnvHashMap, FnvHashSet};
use std::cell::RefCell;

pub struct Parser {
    pub index_builder: IndexBuilder,
    stats: Vec<FnvHashSet<usize>>,
    colon_positions: RefCell<Vec<Vec<usize>>>,
}

impl Parser {
    pub fn new(max_depth: usize, num_nodes: usize) -> Self {
        let index_builder = IndexBuilder::new(max_depth);
        let colon_positions = RefCell::new(vec![Vec::new(); max_depth]);
        let stats = vec![Default::default(); num_nodes];
        Self {
            index_builder,
            stats,
            colon_positions,
        }
    }

    #[inline]
    pub fn basic_parse<'a>(
        &mut self,
        rec: &'a [u8],
        queries: &FnvHashMap<&[u8], Query>,
        start: usize,
        end: usize,
        level: usize,
        set_stats: bool,
        results: &mut Vec<Option<&'a [u8]>>,
    ) -> Result<()> {
        generate_colon_positions(
            &self.index_builder.index,
            start,
            end,
            level,
            &mut *self.colon_positions.borrow_mut(),
        );

        let mut found_num = 0;
        let mut vei = end;
        let cp_len = self.colon_positions.borrow()[level].len();
        for i in (0..cp_len).rev() {
            let (fsi, fei) = search_pre_field_indices(
                &self.index_builder.b_quote,
                if i > 0 {
                    self.colon_positions.borrow()[level][i - 1]
                } else {
                    start
                },
                self.colon_positions.borrow()[level][i],
            )?;
            let field = &rec[fsi + 1..fei];
            if let Some(query) = queries.get(field) {
                let (vsi, vei) = search_post_value_indices(
                    rec,
                    self.colon_positions.borrow()[level][i] + 1,
                    vei,
                    if i == cp_len - 1 { RIGHT_BRACE } else { COMMA },
                )?;
                found_num += 1;
                if set_stats && !self.stats[query.i].contains(&i) {
                    self.stats[query.i].insert(i);
                }
                if let Some(ref children) = query.children {
                    self.basic_parse(rec, children, vsi, vei, level + 1, set_stats, results)?;
                }
                if query.target {
                    results[query.ri] = Some(&rec[vsi..vei + 1]);
                }
                if found_num == queries.len() {
                    return Ok(());
                }
            }
            vei = fsi - 1;
        }
        Ok(())
    }

    #[inline]
    pub fn speculative_parse<'a>(&self, rec: &'a [u8], queries: &FnvHashMap<&[u8], Query>, start: usize, end: usize, level: usize, results: &mut Vec<Option<&'a [u8]>>) -> Result<bool> {
        generate_colon_positions(
            &self.index_builder.index,
            start,
            end,
            level,
            &mut *self.colon_positions.borrow_mut(),
        );

        for (&s, q) in queries {
            let mut found = false;
            for &i in &self.stats[q.i] {
                let cp_len = self.colon_positions.borrow()[level].len();
                if i >= cp_len {
                    continue;
                }
                let (fsi, fei) = search_pre_field_indices(
                    &self.index_builder.b_quote,
                    if i > 0 {
                        self.colon_positions.borrow()[level][i - 1]
                    } else {
                        start
                    },
                    self.colon_positions.borrow()[level][i],
                )?;
                let field = &rec[fsi + 1..fei];
                if s == field {
                    let vei = if i < cp_len - 1 {
                        let (nfsi, _) = search_pre_field_indices(
                            &self.index_builder.b_quote,
                            self.colon_positions.borrow()[level][i],
                            self.colon_positions.borrow()[level][i + 1],
                        )?;
                        nfsi - 1
                    } else {
                        end
                    };
                    let (vsi, vei) = search_post_value_indices(
                        rec,
                        self.colon_positions.borrow()[level][i] + 1,
                        vei,
                        if i == cp_len - 1 { RIGHT_BRACE } else { COMMA },
                    )?;
                    if let Some(ref children) = q.children {
                        found = self.speculative_parse(rec, children, vsi, vei, level + 1, results)?;
                    } else {
                        found = true;
                    }
                    if q.target {
                        results[q.ri] = Some(&rec[vsi..vei + 1]);
                    }
                    break;
                }
            }
            if !found {
                return Ok(false);
            }
        }
        Ok(true)
    }
}

#[inline]
fn generate_colon_positions(index: &[Vec<u64>], start: usize, end: usize, level: usize, colon_positions: &mut Vec<Vec<usize>>) {
    let cp = &mut colon_positions[level];
    cp.clear();
    for i in start / 64..(end + 63) / 64 {
        let mut m_colon = index[level][i];
        while m_colon != 0 {
            let m_bit = bit::e(m_colon);
            let offset = i * 64 + (m_bit.trailing_zeros() as usize);
            if start <= offset && offset <= end {
                cp.push(offset);
            }
            m_colon = bit::r(m_colon);
        }
    }
}

#[inline]
fn search_pre_field_indices(b_quote: &[u64], start: usize, end: usize) -> Result<(usize, usize)> {
    let mut si = 0;
    let mut ei = 0;
    let mut ei_set = false;
    let mut n_quote = 0;
    for i in (start / 64..(end + 63) / 64).rev() {
        let mut m_quote = b_quote[i];
        while m_quote != 0 {
            let m_bit = bit::e(m_quote);
            let offset = i * 64 + (m_bit.trailing_zeros() as usize);
            if end <= offset {
                break;
            }
            if start < offset {
                if ei_set {
                    si = offset;
                } else {
                    si = ei;
                    ei = offset;
                }
                n_quote += 1;
            }
            m_quote = bit::r(m_quote);
        }
        if n_quote >= 2 {
            break;
        }
        if n_quote == 1 && !ei_set {
            ei_set = true;
        }
    }
    if n_quote >= 2 {
        Ok((si, ei))
    } else {
        Err(Error::from(ErrorKind::InvalidRecord))
    }
}

#[inline]
fn search_post_value_indices(rec: &[u8], si: usize, ei: usize, ignore_once_char: u8) -> Result<(usize, usize)> {
    let mut si = si;
    let mut ei = ei;
    let mut ignore_once_char_ignored = false;
    let n = rec.len();
    while si < n {
        match rec[si] {
            SPACE | HT | LF | CR => {
                si += 1;
            }
            _ => {
                break;
            }
        }
    }
    if si == n {
        return Err(Error::from(ErrorKind::InvalidRecord));
    }
    while si <= ei {
        match rec[ei] {
            SPACE | HT | LF | CR => {
                ei -= 1;
            }
            char if char == ignore_once_char => {
                if ignore_once_char_ignored {
                    break;
                }
                ignore_once_char_ignored = true;
                ei -= 1;
            }
            _ => {
                break;
            }
        }
    }
    if ei < si {
        return Err(Error::from(ErrorKind::InvalidRecord));
    }
    Ok((si, ei))
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::super::query::QueryTree;

    #[test]
    fn test_basic_parse() {
        let json_rec_str = r#"{ "aaa" : "AAA", "bbb" : 111, "ccc": ["C1", "C2"], "ddd" : { "d1" : "D1", "d2" : "D2", "d3": 333 }, "eee": { "e1": "EEE" } } "#;
        let json_rec = json_rec_str.as_bytes();
        let query_strs = &["$.ddd.d1", "$.ddd.d3", "$.aaa", "$.bbb", "$.ccc", "$.eee"];

        let queries = QueryTree::new(query_strs).unwrap();

        let l = 10;
        let mut parser = Parser::new(l, queries.num_nodes);
        let r = parser.index_builder.build_structural_indices(json_rec);
        assert_eq!(Ok(()), r);

        let mut results = vec![None; queries.num_queries];
        let result = parser.basic_parse(
            json_rec,
            &queries.root,
            0,
            json_rec.len() - 1,
            0,
            true,
            &mut results,
        );

        assert_eq!(Ok(()), result);
        assert_eq!(Some(r#""D1""#.as_bytes()), results[0]);
        assert_eq!(Some(r#"333"#.as_bytes()), results[1]);
        assert_eq!(Some(r#""AAA""#.as_bytes()), results[2]);
        assert_eq!(Some(r#"111"#.as_bytes()), results[3]);
        assert_eq!(Some(r#"["C1", "C2"]"#.as_bytes()), results[4]);
        assert_eq!(Some(r#"{ "e1": "EEE" }"#.as_bytes()), results[5]);
    }

    #[test]
    fn test_generate_colon_positions() {
        struct TestCase {
            index: Vec<Vec<u64>>,
            start: usize,
            end: usize,
            level: usize,
            want: Vec<usize>,
        }
        let test_cases = vec![
            TestCase {
                index: vec![vec![]],
                start: 0,
                end: 0,
                level: 0,
                want: vec![],
            },
            TestCase {
                index: vec![
                    vec![
                        0b00000001_11000000_00000000_00000000_00001000_00000000_00000011_10000000,
                    ],
                ],
                start: 8,
                end: 55,
                level: 0,
                want: vec![8, 9, 27, 54, 55],
            },
            TestCase {
                index: vec![
                    vec![
                        0b10000000_00000000_00000000_00000000_00001000_00000000_00000011_10000000,
                        0b00000001_11000001_00000000_00000011_00000000_00000000_00000000_00000001,
                    ],
                ],
                start: 8,
                end: 119,
                level: 0,
                want: vec![8, 9, 27, 63, 64, 96, 97, 112, 118, 119],
            },
        ];
        for t in test_cases {
            let mut cp = vec![Vec::new(); t.level + 1];
            generate_colon_positions(&t.index, t.start, t.end, t.level, &mut cp);
            assert_eq!(t.want, cp[0]);
        }
    }
}
<|endoftext|>


<commit_before>use super::{Document,Root,RootChild,Element};

struct Parser;

struct ParsedAttribute<'a> {
    name: &'a str,
    value: &'a str,
}

impl Parser {
    fn new() -> Parser {
        Parser
    }

    fn parse_preamble<'a>(&self, xml: &'a str) -> &'a str {
        // Parse the preamble
        let idx = xml.find_str("?>").expect("No preamble end");
        let end_of_preamble = idx + "?>".len();
        xml.slice_from(end_of_preamble)
    }

    fn optional_space<'a>(&self, xml: &'a str) -> &'a str {
        match xml.slice_space() {
            Some((_, next_xml)) => next_xml,
            None => xml,
        }
    }

    fn parse_attribute<'a>(&self, xml: &'a str) -> Option<(ParsedAttribute<'a>, &'a str)> {
        let (name, xml) = match xml.slice_name() {
            Some(x) => x,
            None => return None,
        };

        let xml = self.optional_space(xml);
        let (_, xml) = xml.slice_literal("=").expect("No equal sign");
        let xml = self.optional_space(xml);

        let (_, xml) = xml.slice_literal("'").expect("No quote"); // single and double quotes
        let (value, xml) = xml.slice_until("'").expect("No value"); // single and double quotes
        let (_, xml) = xml.slice_literal("'").expect("No quote"); // single and double quotes

        Some((ParsedAttribute{name: name, value: value}, xml))
    }

    fn parse_element<'a>(&self, doc: Document, xml: &'a str) -> (Element, &'a str) {
        let (_, after_start_brace) = xml.slice_literal("<").expect("no start brace");

        let (name, after_name) = after_start_brace.slice_name().expect("failed to parse a name!");

        let (_, after_space) = after_name.slice_space().expect("no space after name");

        let mut attrs = Vec::new();
        let after_attr = match self.parse_attribute(after_space) {
            None => after_space,
            Some((attr, after_attr)) => {
                attrs.push(attr);
                after_attr
            },
        };

        let after_attr = self.optional_space(after_attr);

        let (_, after_end_brace) = after_attr.slice_literal("/>").expect("no end brace");

        let e = doc.new_element(name.to_string());
        for attr in attrs.iter() {
            e.set_attribute(attr.name.to_string(), attr.value.to_string());
        }

        (e, after_end_brace)
    }

    fn parse(&self, xml: &str) -> Document {
        let doc = Document::new();

        let after_preamble = self.parse_preamble(xml);

        let (element, _tail) = self.parse_element(doc.clone(), after_preamble);
        doc.root().append_child(element);

        doc
    }
}

trait XmlStr<'a> {
    fn slice_at(&self, position: uint) -> (&'a str, &'a str);
    fn slice_until(&self, s: &str) -> Option<(&'a str, &'a str)>;
    fn slice_literal(&self, expected: &str) -> Option<(&'a str, &'a str)>;
    fn slice_start_rest(&self, is_first: |char| -> bool, is_rest: |char| -> bool) -> Option<(&'a str, &'a str)>;
    fn slice_name(&self) -> Option<(&'a str, &'a str)>;
    fn slice_space(&self) -> Option<(&'a str, &'a str)>;
}

impl<'a> XmlStr<'a> for &'a str {
    fn slice_at(&self, position: uint) -> (&'a str, &'a str) {
        (self.slice_to(position), self.slice_from(position))
    }

    fn slice_until(&self, s: &str) -> Option<(&'a str, &'a str)> {
        match self.find_str(s) {
            Some(position) => Some(self.slice_at(position)),
            None => None
        }
    }

    fn slice_literal(&self, expected: &str) -> Option<(&'a str, &'a str)> {
        if self.starts_with(expected) {
            Some(self.slice_at(expected.len()))
        } else {
            None
        }
    }

    fn slice_start_rest(&self,
                        is_first: |char| -> bool,
                        is_rest: |char| -> bool)
                        -> Option<(&'a str, &'a str)>
    {
        let mut positions = self.char_indices();

        match positions.next() {
            Some((_, c)) if is_first(c) => (),
            Some((_, _)) => return None,
            None => return None,
        };

        let mut positions = positions.skip_while(|&(_, c)| is_rest(c));
        match positions.next() {
            Some((offset, _)) => Some(self.slice_at(offset)),
            None => Some((self.clone(), "")),
        }
    }

    fn slice_name(&self) -> Option<(&'a str, &'a str)> {
        self.slice_start_rest(|c| c.is_name_start_char(), |c| c.is_name_char())
    }

    fn slice_space(&self) -> Option<(&'a str, &'a str)> {
        self.slice_start_rest(|c| c.is_space_char(), |c| c.is_space_char())
    }
}

trait XmlChar {
    fn is_name_start_char(&self) -> bool;
    fn is_name_char(&self) -> bool;
    fn is_space_char(&self) -> bool;
}

impl XmlChar for char {
    fn is_name_start_char(&self) -> bool {
        match *self {
            ':'                        |
            'A'..'Z'                   |
            '_'                        |
            'a'..'z'                   |
            '\U000000C0'..'\U000000D6' |
            '\U000000D8'..'\U000000F6' |
            '\U000000F8'..'\U000002FF' |
            '\U00000370'..'\U0000037D' |
            '\U0000037F'..'\U00001FFF' |
            '\U0000200C'..'\U0000200D' |
            '\U00002070'..'\U0000218F' |
            '\U00002C00'..'\U00002FEF' |
            '\U00003001'..'\U0000D7FF' |
            '\U0000F900'..'\U0000FDCF' |
            '\U0000FDF0'..'\U0000FFFD' |
            '\U00010000'..'\U000EFFFF' => true,
            _ => false,
        }
    }

    fn is_name_char(&self) -> bool {
        if self.is_name_start_char() { return true; }
        match *self {
            '-'                |
            '.'                |
            '0'..'9'           |
            '\u00B7'           |
            '\u0300'..'\u036F' |
            '\u203F'..'\u2040' => true,
            _ => false
        }
    }

    fn is_space_char(&self) -> bool {
        match *self {
            '\x20' |
            '\x09' |
            '\x0D' |
            '\x0A' => true,
            _ => false,
        }
    }
}

trait Hax {
    fn first_child(&self) -> Option<RootChild>;
}

impl Hax for Root {
    fn first_child(&self) -> Option<RootChild> {
        self.children().remove(0)
    }
}

#[test]
fn parses_a_document_with_a_single_element() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello />");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.name().as_slice(), "hello");
}

#[test]
fn parses_an_element_with_an_attribute() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello scope='world'/>");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.get_attribute("scope").unwrap().as_slice(), "world");
}
<commit_msg>Single or double quoted attribute values<commit_after>use super::{Document,Root,RootChild,Element};

struct Parser;

struct ParsedAttribute<'a> {
    name: &'a str,
    value: &'a str,
}

impl Parser {
    fn new() -> Parser {
        Parser
    }

    fn parse_preamble<'a>(&self, xml: &'a str) -> &'a str {
        // Parse the preamble
        let idx = xml.find_str("?>").expect("No preamble end");
        let end_of_preamble = idx + "?>".len();
        xml.slice_from(end_of_preamble)
    }

    fn optional_space<'a>(&self, xml: &'a str) -> &'a str {
        match xml.slice_space() {
            Some((_, next_xml)) => next_xml,
            None => xml,
        }
    }

    fn parse_attribute_value_quote<'a>(&self, xml: &'a str, quote: &str) -> Option<(&'a str, &'a str)> {
        let (_, xml) = match xml.slice_literal(quote) {
            None => return None,
            Some(x) => x,
        };
        // TODO: don't consume & or <
        let (value, xml) = xml.slice_until(quote).expect("No value");
        let (_, xml) = xml.slice_literal(quote).expect("No quote");

        Some((value, xml))
    }

    fn parse_attribute<'a>(&self, xml: &'a str) -> Option<(ParsedAttribute<'a>, &'a str)> {
        let (name, xml) = match xml.slice_name() {
            Some(x) => x,
            None => return None,
        };

        let xml = self.optional_space(xml);
        let (_, xml) = xml.slice_literal("=").expect("No equal sign");
        let xml = self.optional_space(xml);

        // Pattern: alternate
        let (value, xml) = match self.parse_attribute_value_quote(xml, "'") {
            Some(x) => x,
            None => match self.parse_attribute_value_quote(xml, "\"") {
                Some(x) => x,
                None => fail!("No attribute value"),
            },
        };

        Some((ParsedAttribute{name: name, value: value}, xml))
    }

    fn parse_element<'a>(&self, doc: Document, xml: &'a str) -> (Element, &'a str) {
        let (_, after_start_brace) = xml.slice_literal("<").expect("no start brace");

        let (name, after_name) = after_start_brace.slice_name().expect("failed to parse a name!");

        let (_, after_space) = after_name.slice_space().expect("no space after name");

        let mut attrs = Vec::new();
        let after_attr = match self.parse_attribute(after_space) {
            None => after_space,
            Some((attr, after_attr)) => {
                attrs.push(attr);
                after_attr
            },
        };

        let after_attr = self.optional_space(after_attr);

        let (_, after_end_brace) = after_attr.slice_literal("/>").expect("no end brace");

        let e = doc.new_element(name.to_string());
        for attr in attrs.iter() {
            e.set_attribute(attr.name.to_string(), attr.value.to_string());
        }

        (e, after_end_brace)
    }

    fn parse(&self, xml: &str) -> Document {
        let doc = Document::new();

        let after_preamble = self.parse_preamble(xml);

        let (element, _tail) = self.parse_element(doc.clone(), after_preamble);
        doc.root().append_child(element);

        doc
    }
}

trait XmlStr<'a> {
    fn slice_at(&self, position: uint) -> (&'a str, &'a str);
    fn slice_until(&self, s: &str) -> Option<(&'a str, &'a str)>;
    fn slice_literal(&self, expected: &str) -> Option<(&'a str, &'a str)>;
    fn slice_start_rest(&self, is_first: |char| -> bool, is_rest: |char| -> bool) -> Option<(&'a str, &'a str)>;
    fn slice_name(&self) -> Option<(&'a str, &'a str)>;
    fn slice_space(&self) -> Option<(&'a str, &'a str)>;
}

impl<'a> XmlStr<'a> for &'a str {
    fn slice_at(&self, position: uint) -> (&'a str, &'a str) {
        (self.slice_to(position), self.slice_from(position))
    }

    fn slice_until(&self, s: &str) -> Option<(&'a str, &'a str)> {
        match self.find_str(s) {
            Some(position) => Some(self.slice_at(position)),
            None => None
        }
    }

    fn slice_literal(&self, expected: &str) -> Option<(&'a str, &'a str)> {
        if self.starts_with(expected) {
            Some(self.slice_at(expected.len()))
        } else {
            None
        }
    }

    fn slice_start_rest(&self,
                        is_first: |char| -> bool,
                        is_rest: |char| -> bool)
                        -> Option<(&'a str, &'a str)>
    {
        let mut positions = self.char_indices();

        match positions.next() {
            Some((_, c)) if is_first(c) => (),
            Some((_, _)) => return None,
            None => return None,
        };

        let mut positions = positions.skip_while(|&(_, c)| is_rest(c));
        match positions.next() {
            Some((offset, _)) => Some(self.slice_at(offset)),
            None => Some((self.clone(), "")),
        }
    }

    fn slice_name(&self) -> Option<(&'a str, &'a str)> {
        self.slice_start_rest(|c| c.is_name_start_char(), |c| c.is_name_char())
    }

    fn slice_space(&self) -> Option<(&'a str, &'a str)> {
        self.slice_start_rest(|c| c.is_space_char(), |c| c.is_space_char())
    }
}

trait XmlChar {
    fn is_name_start_char(&self) -> bool;
    fn is_name_char(&self) -> bool;
    fn is_space_char(&self) -> bool;
}

impl XmlChar for char {
    fn is_name_start_char(&self) -> bool {
        match *self {
            ':'                        |
            'A'..'Z'                   |
            '_'                        |
            'a'..'z'                   |
            '\U000000C0'..'\U000000D6' |
            '\U000000D8'..'\U000000F6' |
            '\U000000F8'..'\U000002FF' |
            '\U00000370'..'\U0000037D' |
            '\U0000037F'..'\U00001FFF' |
            '\U0000200C'..'\U0000200D' |
            '\U00002070'..'\U0000218F' |
            '\U00002C00'..'\U00002FEF' |
            '\U00003001'..'\U0000D7FF' |
            '\U0000F900'..'\U0000FDCF' |
            '\U0000FDF0'..'\U0000FFFD' |
            '\U00010000'..'\U000EFFFF' => true,
            _ => false,
        }
    }

    fn is_name_char(&self) -> bool {
        if self.is_name_start_char() { return true; }
        match *self {
            '-'                |
            '.'                |
            '0'..'9'           |
            '\u00B7'           |
            '\u0300'..'\u036F' |
            '\u203F'..'\u2040' => true,
            _ => false
        }
    }

    fn is_space_char(&self) -> bool {
        match *self {
            '\x20' |
            '\x09' |
            '\x0D' |
            '\x0A' => true,
            _ => false,
        }
    }
}

trait Hax {
    fn first_child(&self) -> Option<RootChild>;
}

impl Hax for Root {
    fn first_child(&self) -> Option<RootChild> {
        self.children().remove(0)
    }
}

#[test]
fn parses_a_document_with_a_single_element() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello />");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.name().as_slice(), "hello");
}

#[test]
fn parses_an_element_with_an_attribute() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello scope='world'/>");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.get_attribute("scope").unwrap().as_slice(), "world");
}

#[test]
fn parses_an_element_with_an_attribute_using_double_quotes() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello scope=\"world\"/>");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.get_attribute("scope").unwrap().as_slice(), "world");
}
<|endoftext|>


<commit_before>use ast::*;

#[deriving(Clone, Show)]
pub struct Parser {
    code: String,
    cur: uint,
    col: uint,
    line: uint
}

#[allow(dead_code)]
pub struct ParserError {
    line: uint,
    col: uint,
    desc: String
}

#[allow(dead_code)]
impl ParserError {
    pub fn new(line: uint, col: uint, desc: String) -> ParserError {
        ParserError {
            line: line,
            col: col,
            desc: desc
        }
    }
}

#[allow(dead_code)]
pub type ParseResult<T> = Result<T, ParserError>;

impl Parser {
    pub fn new() -> Parser {
        Parser{
            code: "".to_string(),
            line: 0, cur: 0, col: 0
        }
    }

    pub fn load(&mut self, code: String) -> ExprAst {
        self.code = code;
        self.line = 1;
        self.cur = 0;
        self.col = 0;
        self.skip_space();
        self.read_exp()
    }

    //============= private methods =================
    fn read_exp(&mut self) -> ExprAst {
        let mut cur = self.readc();
        if cur == '#' {
            let next = self.readc();
            match next {
                't' => return ExprAst::Bool(BoolNode::new(true)),
                'f' => return ExprAst::Bool(BoolNode::new(false)),
                '\\' => return self.read_char(),
                _ => panic!("error")
            }
        } else if UnicodeChar::is_numeric(cur) ||
            (cur == '-' && (UnicodeChar::is_numeric(self.peekc()))) {
                let mut sign = 1i;
                let mut num = 0i;
                if cur == '-' {
                    sign = -1;
                } else {
                    self.unread();
                }
                loop {
                    cur = self.readc();
                    if !UnicodeChar::is_numeric(cur) {
                        break;
                    }
                    num = (num * 10i) + (cur as int - '0' as int);
                }
                num *= sign;
                if self.is_delimiter(cur) {
                    self.unread();
                    return ExprAst::Int(IntNode::new(num));
                } else {
                    panic!("number not followed by delimiter");
                }
            } else if cur == '\"' {
                let mut buf = String::new();
                loop {
                    cur = self.readc();
                    if cur == '\"' {
                        break;
                    }
                    buf.push(cur);
                }
                return ExprAst::Str(StrNode::new(buf.as_slice()));
            } else if cur == '(' && cur != ')' {
                // rust-mode bug here
                return self.read_pair();
            } else if self.is_initial(cur) {
                let mut buf = String::new();
                buf.push(cur);
                loop {
                    cur = self.readc();
                    if !(self.is_initial(cur) || UnicodeChar::is_numeric(cur)) {
                        break;
                    }
                    buf.push(cur);
                }
                if self.is_delimiter(cur) {
                    self.unread();
                    return ExprAst::Symbol(SymbolNode::new(buf.as_slice()));
                }
            } else if cur == '\'' {
                let quote_sym = ExprAst::Symbol(SymbolNode::new("quote"));
                let quote_exp = ExprAst::Pair(PairNode::new(self.read_exp(),
                                                            ExprAst::Nil));
                return ExprAst::Pair(PairNode::new(quote_sym,
                                                   quote_exp));
            }
        ExprAst::Int(IntNode::new(0))
    }

    fn read_pair(&mut self) -> ExprAst {
        self.skip_space();
        let mut cur = self.readc();
        // rust-mode bug here
        if cur != '(' && cur == ')' {
            return ExprAst::Nil;
        }
        self.unread();
        let car_obj = self.read_exp();
        self.skip_space();
        cur = self.readc();
        if cur != '.' {
            self.unread();
            let cdr_obj = self.read_pair();
            return ExprAst::Pair(PairNode::new(car_obj, cdr_obj));
        } else {
            return ExprAst::Nil;
        }
    }

    fn is_delimiter(&self, ch: char) -> bool {
        ch.is_whitespace() ||
            ch == '\"' || ch == '(' || ch == ')' ||  ch == ';' ||
            (ch as u32 == 0)
    }

    fn is_initial(&self, ch: char) -> bool {
        ch.is_alphabetic() ||
            ch == '*' || ch == '/' || ch == '+' || ch == '-' ||
            ch == '>' || ch == '<' || ch == '=' || ch == '?' ||
            ch == '!'
    }

    fn skip_space(&mut self) {
        while self.peekc().is_whitespace() {
            self.readc();
        }
    }

    fn peekc(&self) -> char {
        if self.cur >= self.code.len() {
            panic!("invalid position");
        }
        self.code.char_at(self.cur)
    }

    fn prevc(&self) -> char {
        if self.cur <= 0 {
            panic!("invalid position");
        }
        self.code.char_at(self.cur - 1)
    }

    fn readc(&mut self) -> char {
        if self.cur < self.code.len() {
            let res = self.peekc();
            if res == '\n' {
                self.line += 1;
            }
            self.cur += 1;
            res
        } else {
            0 as char
        }
    }

    fn unread(&mut self) {
        if self.cur == 0 {
            panic!("error current position");
        }
        if self.prevc() == '\n' {
            assert!(self.line > 1);
            self.line -= 1;
        }
        self.cur -= 1;
    }

    fn read_char(&mut self) -> ExprAst {
        ExprAst::Char(CharNode::new('a'))
    }
}

#[test]
fn test_parser() {
    let mut parser = Parser::new();
    let res = parser.load("11".to_string());
    assert!(res.as_int() == 11);
    res.print();

    let res = parser.load("-11".to_string());
    assert!(res.as_int() == -11);
    res.print();

    let res = parser.load(r#""hello""#.to_string());
    assert!(res.as_str() == "hello");
    res.print();

    let res = parser.load("()".to_string());
    assert!(res.is_empty());
    res.print();

    let res = parser.load("(1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().as_int() == 1);
    assert!(res.cdr().car().as_int() == 2);
    assert!(res.cdr().cdr().is_empty());

    let res = parser.load("(+ 1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().is_symbol());
    assert!(res.cdr().car().as_int() == 1);
    assert!(res.cdr().cdr().car().as_int() == 2);
}
<commit_msg>update rust-mode, one indent bug fixed<commit_after>use ast::*;

#[deriving(Clone, Show)]
pub struct Parser {
    code: String,
    cur: uint,
    col: uint,
    line: uint
}

#[allow(dead_code)]
pub struct ParserError {
    line: uint,
    col: uint,
    desc: String
}

#[allow(dead_code)]
impl ParserError {
    pub fn new(line: uint, col: uint, desc: String) -> ParserError {
        ParserError {
            line: line,
            col: col,
            desc: desc
        }
    }
}

#[allow(dead_code)]
pub type ParseResult<T> = Result<T, ParserError>;

impl Parser {
    pub fn new() -> Parser {
        Parser{
            code: "".to_string(),
            line: 0, cur: 0, col: 0
        }
    }

    pub fn load(&mut self, code: String) -> ExprAst {
        self.code = code;
        self.line = 1;
        self.cur = 0;
        self.col = 0;
        self.skip_space();
        self.read_exp()
    }

    //============= private methods =================
    fn read_exp(&mut self) -> ExprAst {
        let mut cur = self.readc();
        if cur == '#' {
            let next = self.readc();
            match next {
                't' => return ExprAst::Bool(BoolNode::new(true)),
                'f' => return ExprAst::Bool(BoolNode::new(false)),
                '\\' => return self.read_char(),
                _ => panic!("error")
            }
        } else if UnicodeChar::is_numeric(cur) ||
            (cur == '-' && (UnicodeChar::is_numeric(self.peekc()))) {
                let mut sign = 1i;
                let mut num = 0i;
                if cur == '-' {
                    sign = -1;
                } else {
                    self.unread();
                }
                loop {
                    cur = self.readc();
                    if !UnicodeChar::is_numeric(cur) {
                        break;
                    }
                    num = (num * 10i) + (cur as int - '0' as int);
                }
                num *= sign;
                if self.is_delimiter(cur) {
                    self.unread();
                    return ExprAst::Int(IntNode::new(num));
                } else {
                    panic!("number not followed by delimiter");
                }
            } else if cur == '\"' {
                let mut buf = String::new();
                loop {
                    cur = self.readc();
                    if cur == '\"' {
                        break;
                    }
                    buf.push(cur);
                }
                return ExprAst::Str(StrNode::new(buf.as_slice()));
            } else if cur == '(' {
                return self.read_pair();
            } else if self.is_initial(cur) {
                let mut buf = String::new();
                buf.push(cur);
                loop {
                    cur = self.readc();
                    if !(self.is_initial(cur) || UnicodeChar::is_numeric(cur)) {
                        break;
                    }
                    buf.push(cur);
                }
                if self.is_delimiter(cur) {
                    self.unread();
                    return ExprAst::Symbol(SymbolNode::new(buf.as_slice()));
                }
            } else if cur == '\'' {
                let quote_sym = ExprAst::Symbol(SymbolNode::new("quote"));
                let quote_exp = ExprAst::Pair(PairNode::new(self.read_exp(),
                                                            ExprAst::Nil));
                return ExprAst::Pair(PairNode::new(quote_sym,
                                                   quote_exp));
            }
        ExprAst::Int(IntNode::new(0))
    }

    fn read_pair(&mut self) -> ExprAst {
        self.skip_space();
        let mut cur = self.readc();
        // rust-mode bug here
        if cur != '(' && cur == ')' {
            return ExprAst::Nil;
        }
        self.unread();
        let car_obj = self.read_exp();
        self.skip_space();
        cur = self.readc();
        if cur != '.' {
            self.unread();
            let cdr_obj = self.read_pair();
            return ExprAst::Pair(PairNode::new(car_obj, cdr_obj));
        } else {
            return ExprAst::Nil;
        }
    }

    fn is_delimiter(&self, ch: char) -> bool {
        ch.is_whitespace() ||
            ch == '\"' || ch == '(' || ch == ')' ||  ch == ';' ||
            (ch as u32 == 0)
    }

    fn is_initial(&self, ch: char) -> bool {
        ch.is_alphabetic() ||
            ch == '*' || ch == '/' || ch == '+' || ch == '-' ||
            ch == '>' || ch == '<' || ch == '=' || ch == '?' ||
            ch == '!'
    }

    fn skip_space(&mut self) {
        while self.peekc().is_whitespace() {
            self.readc();
        }
    }

    fn peekc(&self) -> char {
        if self.cur >= self.code.len() {
            panic!("invalid position");
        }
        self.code.char_at(self.cur)
    }

    fn prevc(&self) -> char {
        if self.cur <= 0 {
            panic!("invalid position");
        }
        self.code.char_at(self.cur - 1)
    }

    fn readc(&mut self) -> char {
        if self.cur < self.code.len() {
            let res = self.peekc();
            if res == '\n' {
                self.line += 1;
            }
            self.cur += 1;
            res
        } else {
            0 as char
        }
    }

    fn unread(&mut self) {
        if self.cur == 0 {
            panic!("error current position");
        }
        if self.prevc() == '\n' {
            assert!(self.line > 1);
            self.line -= 1;
        }
        self.cur -= 1;
    }

    fn read_char(&mut self) -> ExprAst {
        ExprAst::Char(CharNode::new('a'))
    }
}

#[test]
fn test_parser() {
    let mut parser = Parser::new();
    let res = parser.load("11".to_string());
    assert!(res.as_int() == 11);
    res.print();

    let res = parser.load("-11".to_string());
    assert!(res.as_int() == -11);
    res.print();

    let res = parser.load(r#""hello""#.to_string());
    assert!(res.as_str() == "hello");
    res.print();

    let res = parser.load("()".to_string());
    assert!(res.is_empty());
    res.print();

    let res = parser.load("(1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().as_int() == 1);
    assert!(res.cdr().car().as_int() == 2);
    assert!(res.cdr().cdr().is_empty());

    let res = parser.load("(+ 1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().is_symbol());
    assert!(res.cdr().car().as_int() == 1);
    assert!(res.cdr().cdr().car().as_int() == 2);
}
<|endoftext|>


<commit_before>use regex::Regex;

#[deriving(Show)]
pub enum Block<'s> {
    Header(Vec<Span<'s>>, uint),
    Break,
    Paragraph(Vec<Span<'s>>)
}

#[deriving(Show)]
pub enum Span<'s> {
    Text(&'s str),
    Emphasis(&'s str),
    Strong(&'s str),
    Code(&'s str),
    Link(&'s str, &'s str, &'s str),
    Image(&'s str, &'s str, &'s str)
}

// Block Patters
static SPLIT           : Regex = regex!(r"\n\n");
static ATX_HEADER      : Regex = regex!(r"(?P<level>#{1,6})\s(?P<text>.*)");
static SETEXT_HEADER_1 : Regex = regex!(r"(?P<text>.+)\n===+");
static SETEXT_HEADER_2 : Regex = regex!(r"(?P<text>.+)\n---+");
static BREAK           : Regex = regex!(r"  ");

// Span Patters
static SPANS : Regex = regex!(r"(!?\[.*\]\([^\(\)]*\))|(\*[^\*].+?\*)|(\*\*.+?\*\*)|(_[^_].+?_)|(__.+?__)|(`[^`].+?`)|(``.+?``)");
static LINK  : Regex = regex!("\\[(?P<text>.*)\\]\\((?P<url>.*?)(?:\\s\"(?P<title>.*?)\")?\\)");
static EMPHASIS_UNDERSCORE  : Regex = regex!(r"^_(?P<text>[^_].+?)_");
static EMPHASIS_STAR  : Regex = regex!(r"^\*(?P<text>[^\*].+?)\*");

pub fn parse (md : &str) -> Vec<Block> {
    let mut split = SPLIT.split(md);
    let mut tokens = vec![];
    for block in split{
        match parse_block(block) {
            Some(e) => tokens.push(e),
            None => {},
        };
    }
    tokens
}

fn parse_spans(text : &str) -> Vec<Span>{
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in SPANS.find_iter(text) {
        tokens.push(Text(text.slice(current, begin)));
        tokens.push(parse_span(text.slice(begin, end)));
        current = end;
    }
    tokens.push(Text(text.slice(current, text.len())));
    tokens
}

#[test]
fn parse_link_test() {
    match parse_span("[an example](example.com \"Title\")"){
      Link("an example", "example.com", "Title") => {},
      _ => fail!()
    }
}

fn parse_span(text : &str) -> Span{
    if LINK.is_match(text){
        let caps = LINK.captures(text).unwrap();
        return Link(
            caps.name("text"),
            caps.name("url"),
            caps.name("title")
            );
    }else if EMPHASIS_UNDERSCORE.is_match(text){
        let caps = EMPHASIS_UNDERSCORE.captures(text).unwrap();
        return Emphasis(caps.name("text"));
    }else if EMPHASIS_STAR.is_match(text){
        let caps = EMPHASIS_STAR.captures(text).unwrap();
        return Emphasis(caps.name("text"));
    }
    return Text(text);
}

fn parse_block (text : &str) -> Option<Block>{
    if text.is_empty(){
        return None;
    }else if ATX_HEADER.is_match(text){
        let caps = ATX_HEADER.captures(text).unwrap();
        return Some(
            Header(
                parse_spans(caps.name("text")),
                caps.name("level").len()
                )
            );
    }else if SETEXT_HEADER_1.is_match(text){
        let caps = SETEXT_HEADER_1.captures(text).unwrap();
        return Some(
            Header(
                parse_spans(caps.name("text")),
                1
                )
            );
    }else if SETEXT_HEADER_2.is_match(text){
        let caps = SETEXT_HEADER_2.captures(text).unwrap();
        return Some(
            Header(
                parse_spans(caps.name("text")),
                2
                )
            );
    }else if BREAK.is_match(text){
        return Some(Break);
    }
    return Some(Paragraph(parse_spans(text)));
}

<commit_msg>moar tests<commit_after>use regex::Regex;

#[deriving(Show)]
pub enum Block<'s> {
    Header(Vec<Span<'s>>, uint),
    Break,
    Paragraph(Vec<Span<'s>>)
}

#[deriving(Show)]
pub enum Span<'s> {
    Text(&'s str),
    Emphasis(&'s str),
    Strong(&'s str),
    Code(&'s str),
    Link(&'s str, &'s str, &'s str),
    Image(&'s str, &'s str, &'s str)
}

// Block Patters
static SPLIT           : Regex = regex!(r"\n\n");
static ATX_HEADER      : Regex = regex!(r"(?P<level>#{1,6})\s(?P<text>.*)");
static SETEXT_HEADER_1 : Regex = regex!(r"(?P<text>.+)\n===+");
static SETEXT_HEADER_2 : Regex = regex!(r"(?P<text>.+)\n---+");
static BREAK           : Regex = regex!(r"  ");

// Span Patters
static SPANS : Regex = regex!(r"(!?\[.*\]\([^\(\)]*\))|(\*[^\*].+?\*)|(\*\*.+?\*\*)|(_[^_].+?_)|(__.+?__)|(`[^`].+?`)|(``.+?``)");
static LINK  : Regex = regex!("\\[(?P<text>.*)\\]\\((?P<url>.*?)(?:\\s\"(?P<title>.*?)\")?\\)");
static EMPHASIS_UNDERSCORE  : Regex = regex!(r"^_(?P<text>[^_].+?)_");
static EMPHASIS_STAR  : Regex = regex!(r"^\*(?P<text>[^\*].+?)\*");

pub fn parse (md : &str) -> Vec<Block> {
    let mut split = SPLIT.split(md);
    let mut tokens = vec![];
    for block in split{
        match parse_block(block) {
            Some(e) => tokens.push(e),
            None => {},
        };
    }
    tokens
}

fn parse_spans(text : &str) -> Vec<Span>{
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in SPANS.find_iter(text) {
        tokens.push(Text(text.slice(current, begin)));
        tokens.push(parse_span(text.slice(begin, end)));
        current = end;
    }
    tokens.push(Text(text.slice(current, text.len())));
    tokens
}

#[test]
fn parse_emphasis_test() {
    match parse_span("_whatever_"){
      Emphasis("whatever") => {},
      _ => fail!()
    }
    match parse_span("__whatever__"){
      Emphasis("whatever") => fail!(),
      _ => {}
    }
}

#[test]
fn parse_link_test() {
    match parse_span("[an example](example.com)"){
      Link("an example", "example.com", "") => {},
      _ => fail!()
    }
    match parse_span("[an example](example.com \"Title\")"){
      Link("an example", "example.com", "Title") => assert!(true),
      _ => fail!()
    }
}

fn parse_span(text : &str) -> Span{
    if LINK.is_match(text){
        let caps = LINK.captures(text).unwrap();
        return Link(
            caps.name("text"),
            caps.name("url"),
            caps.name("title")
            );
    }else if EMPHASIS_UNDERSCORE.is_match(text){
        let caps = EMPHASIS_UNDERSCORE.captures(text).unwrap();
        return Emphasis(caps.name("text"));
    }else if EMPHASIS_STAR.is_match(text){
        let caps = EMPHASIS_STAR.captures(text).unwrap();
        return Emphasis(caps.name("text"));
    }
    return Text(text);
}

fn parse_block (text : &str) -> Option<Block>{
    if text.is_empty(){
        return None;
    }else if ATX_HEADER.is_match(text){
        let caps = ATX_HEADER.captures(text).unwrap();
        return Some(
            Header(
                parse_spans(caps.name("text")),
                caps.name("level").len()
                )
            );
    }else if SETEXT_HEADER_1.is_match(text){
        let caps = SETEXT_HEADER_1.captures(text).unwrap();
        return Some(
            Header(
                parse_spans(caps.name("text")),
                1
                )
            );
    }else if SETEXT_HEADER_2.is_match(text){
        let caps = SETEXT_HEADER_2.captures(text).unwrap();
        return Some(
            Header(
                parse_spans(caps.name("text")),
                2
                )
            );
    }else if BREAK.is_match(text){
        return Some(Break);
    }
    return Some(Paragraph(parse_spans(text)));
}

<|endoftext|>


<commit_before>use std::io::{self, BufRead, Read, BufReader, Error, ErrorKind, Write};
use std::{cmp,mem};

use super::InflateStream;

/// Workaround for lack of copy_from_slice on pre-1.9 rust.
#[inline]
fn copy_from_slice(mut to: &mut [u8], from: &[u8]) {
    assert_eq!(to.len(), from.len());
    to.write_all(from).unwrap();
}

/// A DEFLATE decoder/decompressor.
///
/// This structure implements a `Read` interface and takes a stream
/// of compressed data that implements the `BufRead` trait as input,
/// providing the decompressed data when read from.
///
/// # Example
/// ```
/// use std::io::Read;
/// use inflate::DeflateDecoderBuf;
///
/// const TEST_STRING: &'static str = "Hello, world";
/// let encoded = vec![243, 72, 205, 201, 201, 215, 81, 40, 207, 47, 202, 73, 1, 0];
/// let mut decoder = DeflateDecoderBuf::new(&encoded[..]);
/// let mut output = Vec::new();
/// let status = decoder.read_to_end(&mut output);
/// # let _ = status;
/// assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
/// ```
pub struct DeflateDecoderBuf<R> {
    /// The inner reader instance
    reader: R,
    /// The raw decompressor
    decompressor: InflateStream,
    /// How many bytes of the decompressor's output buffer still need to be output.
    pending_output_bytes: usize,
    /// Total number of bytes read from the underlying reader.
    total_in: u64,
    /// Total number of bytes written in `read` calls.
    total_out: u64,
}

impl<R: BufRead> DeflateDecoderBuf<R> {
    /// Create a new `Deflatedecoderbuf` to read from a raw deflate stream.
    pub fn new(reader: R) -> DeflateDecoderBuf<R> {
        DeflateDecoderBuf {
            reader: reader,
            decompressor: InflateStream::new(),
            pending_output_bytes: 0,
            total_in: 0,
            total_out: 0,
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    pub fn from_zlib(reader: R) -> DeflateDecoderBuf<R> {
        DeflateDecoderBuf {
            reader: reader,
            decompressor: InflateStream::from_zlib(),
            pending_output_bytes: 0,
            total_in: 0,
            total_out: 0,
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    /// without calculating and validating the checksum.
    pub fn from_zlib_no_checksum(reader: R) -> DeflateDecoderBuf<R> {
        DeflateDecoderBuf {
            reader: reader,
            decompressor: InflateStream::from_zlib_no_checksum(),
            pending_output_bytes: 0,
            total_in: 0,
            total_out: 0,
        }
    }
}

impl<R> DeflateDecoderBuf<R> {
    /// Resets the decompressor, and replaces the current inner `BufRead` instance by `r`.
    /// without doing any extra reallocations.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset(&mut self, r: R) -> R {
        self.decompressor.reset();
        mem::replace(&mut self.reader, r)
    }

    /// Resets the decoder, but continue to read from the same reader.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset_data(&mut self) {
        self.decompressor.reset()
    }

    /// Returns a reference to the underlying `BufRead` instance.
    #[inline]
    pub fn get_ref(&self) -> &R {
        &self.reader
    }

    /// Returns a mutable reference to the underlying `BufRead` instance.
    ///
    /// Note that mutation of the reader may cause surprising results if the decoder is going to
    /// keep being used.
    #[inline]
    pub fn get_mut(&mut self) -> &mut R {
        &mut self.reader
    }

    /// Drops the decoder and return the inner `BufRead` instance.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn into_inner(self) -> R {
        self.reader
    }

    /// Returns the total bytes read from the underlying `BufRead` instance.
    #[inline]
    pub fn total_in(&self) -> u64 {
        self.total_in
    }

    /// Returns the total number of bytes output from this decoder.
    #[inline]
    pub fn total_out(&self) -> u64 {
        self.total_out
    }

    /// Returns the calculated checksum value of the currently decoded data.
    ///
    /// Will return 0 for cases where the checksum is not validated.
    #[inline]
    pub fn current_checksum(&self) -> u32 {
        self.decompressor.current_checksum()
    }
}

impl<R: BufRead> Read for DeflateDecoderBuf<R> {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        let mut bytes_out = 0;
        // If there is still data left to ouput from the last call to `update()`, that needs to be
        // output first
        if self.pending_output_bytes != 0 {
            // Get the part of the buffer that has not been output yet.
            // The decompressor sets `pos` to 0 when it reaches the end of it's internal buffer,
            // so we have to check for that.
            let start = if self.decompressor.pos != 0 {
                self.decompressor.pos as usize - self.pending_output_bytes
            } else {
                self.decompressor.buffer.len() - self.pending_output_bytes
            };

            // Copy as much decompressed as possible to buf.
            let bytes_to_copy = cmp::min(buf.len(), self.pending_output_bytes);
            let pending_data =
                &self.decompressor.buffer[start..
                                          start + bytes_to_copy];
            copy_from_slice(&mut buf[..bytes_to_copy],pending_data);
            bytes_out += bytes_to_copy;
            // This won't underflow since `bytes_to_copy` will be at most
            // the same value as `pending_output_bytes`.
            self.pending_output_bytes -= bytes_to_copy;
            if self.pending_output_bytes != 0 {
                self.total_out += bytes_out as u64;
                // If there is still decompressed data left that didn't
                // fit in `buf`, return what we read.
                return Ok(bytes_out);
            }
        }

        // There is space in `buf` for more data, so try to read more.
        let (input_bytes_read, remaining_bytes) = {
            self.pending_output_bytes = 0;
            let input = self.reader.fill_buf()?;
            if input.len() == 0 {
                self.total_out += bytes_out as u64;
                //If there is nothing more to read, return.
                return Ok(bytes_out);
            }
            let (input_bytes_read, data) =
                match self.decompressor.update(&input) {
                    Ok(res) => res,
                    Err(m) => return Err(Error::new(ErrorKind::Other, m))
                };

            // Space left in `buf`
            let space_left = buf.len() - bytes_out;
            let bytes_to_copy = cmp::min(space_left, data.len());

            copy_from_slice(&mut buf[bytes_out..bytes_out + bytes_to_copy], &data[..bytes_to_copy]);

            bytes_out += bytes_to_copy;

            // Can't underflow as bytes_to_copy is bounded by data.len().
            (input_bytes_read, data.len() - bytes_to_copy)

        };

        self.pending_output_bytes += remaining_bytes;
        self.total_in += input_bytes_read as u64;
        self.total_out += bytes_out as u64;
        self.reader.consume(input_bytes_read);

        Ok(bytes_out)
    }
}

/// A DEFLATE decoder/decompressor.
///
/// This structure implements a `Read` interface and takes a stream of compressed data that
/// implements the `Read` trait as input,
/// provoding the decompressed data when read from.
/// # Example
/// ```
/// use std::io::Read;
/// use inflate::DeflateDecoder;
/// const TEST_STRING: &'static str = "Hello, world";
/// let encoded = vec![243, 72, 205, 201, 201, 215, 81, 40, 207, 47, 202, 73, 1, 0];
/// let mut decoder = DeflateDecoder::new(&encoded[..]);
/// let mut output = Vec::new();
/// let status = decoder.read_to_end(&mut output);
/// # let _ = status;
/// assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
/// ```
pub struct DeflateDecoder<R> {
    /// Inner DeflateDecoderBuf, with R wrapped in a `BufReader`.
    inner: DeflateDecoderBuf<BufReader<R>>
}

impl<R: Read> DeflateDecoder<R> {
    /// Create a new `Deflatedecoderbuf` to read from a raw deflate stream.
    pub fn new(reader: R) -> DeflateDecoder<R> {
        DeflateDecoder {
            inner: DeflateDecoderBuf::new(BufReader::new(reader))
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    pub fn from_zlib(reader: R) -> DeflateDecoder<R> {
        DeflateDecoder {
            inner: DeflateDecoderBuf::from_zlib(BufReader::new(reader))
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    /// without calculating and validating the checksum.
    pub fn from_zlib_no_checksum(reader: R) -> DeflateDecoder<R> {
        DeflateDecoder {
            inner: DeflateDecoderBuf::from_zlib_no_checksum(BufReader::new(reader))
        }
    }

    /// Resets the decompressor, and replaces the current inner `BufRead` instance by `r`.
    /// without doing any extra reallocations.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset(&mut self, r: R) -> R {
        self.inner.reset(BufReader::new(r)).into_inner()
    }

    /// Returns a reference to the underlying reader.
    #[inline]
    pub fn get_ref(&self) -> &R {
        self.inner.get_ref().get_ref()
    }

    /// Returns a mutable reference to the underlying reader.
    ///
    /// Note that mutation of the reader may cause surprising results if the decoder is going to
    /// keep being used.
    #[inline]
    pub fn get_mut(&mut self) -> &mut R {
        self.inner.get_mut().get_mut()
    }

    /// Returns the total number of bytes output from this decoder.
    #[inline]
    pub fn into_inner(self) -> R {
        self.inner.into_inner().into_inner()
    }
}

impl<R> DeflateDecoder<R> {
    /// Resets the decoder, but continue to read from the same reader.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset_data(&mut self) {
        self.inner.reset_data()
    }

    /// Returns the total bytes read from the underlying reader.
    #[inline]
    pub fn total_in(&self) -> u64 {
        self.inner.total_in
    }

    /// Returns the total number of bytes output from this decoder.
    #[inline]
    pub fn total_out(&self) -> u64 {
        self.inner.total_out
    }

    /// Returns the calculated checksum value of the currently decoded data.
    ///
    /// Will return 0 for cases where the checksum is not validated.
    #[inline]
    pub fn current_checksum(&self) -> u32 {
        self.inner.current_checksum()
    }
}

impl<R: Read> Read for DeflateDecoder<R> {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        self.inner.read(buf)
    }
}

#[cfg(test)]
mod test {
    use super::{DeflateDecoder};
    use std::io::Read;

    #[test]
    fn deflate_reader() {
        const TEST_STRING: &'static str = "Hello, world";
        let encoded = vec![243, 72, 205, 201, 201, 215, 81, 40, 207, 47, 202, 73, 1, 0];
        let mut decoder = DeflateDecoder::new(&encoded[..]);
        let mut output = Vec::new();
        decoder.read_to_end(&mut output).unwrap();
        assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
        assert_eq!(decoder.total_in(), encoded.len() as u64);
        assert_eq!(decoder.total_out(), TEST_STRING.len() as u64);
    }

    #[test]
    fn zlib_reader() {
        const TEST_STRING: &'static str = "Hello, zlib!";
        let encoded = vec![120, 156, 243, 72, 205, 201, 201, 215, 81, 168, 202, 201,
                       76, 82, 4, 0, 27, 101, 4, 19];
        let mut decoder = DeflateDecoder::from_zlib(&encoded[..]);
        let mut output = Vec::new();
        decoder.read_to_end(&mut output).unwrap();
        assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
        assert_eq!(decoder.total_in(), encoded.len() as u64);
        assert_eq!(decoder.total_out(), TEST_STRING.len() as u64);
    }
}
<commit_msg>drop local copy_from_slice()<commit_after>use std::io::{self, BufRead, Read, BufReader, Error, ErrorKind};
use std::{cmp,mem};

use super::InflateStream;

/// A DEFLATE decoder/decompressor.
///
/// This structure implements a `Read` interface and takes a stream
/// of compressed data that implements the `BufRead` trait as input,
/// providing the decompressed data when read from.
///
/// # Example
/// ```
/// use std::io::Read;
/// use inflate::DeflateDecoderBuf;
///
/// const TEST_STRING: &'static str = "Hello, world";
/// let encoded = vec![243, 72, 205, 201, 201, 215, 81, 40, 207, 47, 202, 73, 1, 0];
/// let mut decoder = DeflateDecoderBuf::new(&encoded[..]);
/// let mut output = Vec::new();
/// let status = decoder.read_to_end(&mut output);
/// # let _ = status;
/// assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
/// ```
pub struct DeflateDecoderBuf<R> {
    /// The inner reader instance
    reader: R,
    /// The raw decompressor
    decompressor: InflateStream,
    /// How many bytes of the decompressor's output buffer still need to be output.
    pending_output_bytes: usize,
    /// Total number of bytes read from the underlying reader.
    total_in: u64,
    /// Total number of bytes written in `read` calls.
    total_out: u64,
}

impl<R: BufRead> DeflateDecoderBuf<R> {
    /// Create a new `Deflatedecoderbuf` to read from a raw deflate stream.
    pub fn new(reader: R) -> DeflateDecoderBuf<R> {
        DeflateDecoderBuf {
            reader: reader,
            decompressor: InflateStream::new(),
            pending_output_bytes: 0,
            total_in: 0,
            total_out: 0,
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    pub fn from_zlib(reader: R) -> DeflateDecoderBuf<R> {
        DeflateDecoderBuf {
            reader: reader,
            decompressor: InflateStream::from_zlib(),
            pending_output_bytes: 0,
            total_in: 0,
            total_out: 0,
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    /// without calculating and validating the checksum.
    pub fn from_zlib_no_checksum(reader: R) -> DeflateDecoderBuf<R> {
        DeflateDecoderBuf {
            reader: reader,
            decompressor: InflateStream::from_zlib_no_checksum(),
            pending_output_bytes: 0,
            total_in: 0,
            total_out: 0,
        }
    }
}

impl<R> DeflateDecoderBuf<R> {
    /// Resets the decompressor, and replaces the current inner `BufRead` instance by `r`.
    /// without doing any extra reallocations.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset(&mut self, r: R) -> R {
        self.decompressor.reset();
        mem::replace(&mut self.reader, r)
    }

    /// Resets the decoder, but continue to read from the same reader.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset_data(&mut self) {
        self.decompressor.reset()
    }

    /// Returns a reference to the underlying `BufRead` instance.
    #[inline]
    pub fn get_ref(&self) -> &R {
        &self.reader
    }

    /// Returns a mutable reference to the underlying `BufRead` instance.
    ///
    /// Note that mutation of the reader may cause surprising results if the decoder is going to
    /// keep being used.
    #[inline]
    pub fn get_mut(&mut self) -> &mut R {
        &mut self.reader
    }

    /// Drops the decoder and return the inner `BufRead` instance.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn into_inner(self) -> R {
        self.reader
    }

    /// Returns the total bytes read from the underlying `BufRead` instance.
    #[inline]
    pub fn total_in(&self) -> u64 {
        self.total_in
    }

    /// Returns the total number of bytes output from this decoder.
    #[inline]
    pub fn total_out(&self) -> u64 {
        self.total_out
    }

    /// Returns the calculated checksum value of the currently decoded data.
    ///
    /// Will return 0 for cases where the checksum is not validated.
    #[inline]
    pub fn current_checksum(&self) -> u32 {
        self.decompressor.current_checksum()
    }
}

impl<R: BufRead> Read for DeflateDecoderBuf<R> {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        let mut bytes_out = 0;
        // If there is still data left to ouput from the last call to `update()`, that needs to be
        // output first
        if self.pending_output_bytes != 0 {
            // Get the part of the buffer that has not been output yet.
            // The decompressor sets `pos` to 0 when it reaches the end of it's internal buffer,
            // so we have to check for that.
            let start = if self.decompressor.pos != 0 {
                self.decompressor.pos as usize - self.pending_output_bytes
            } else {
                self.decompressor.buffer.len() - self.pending_output_bytes
            };

            // Copy as much decompressed as possible to buf.
            let bytes_to_copy = cmp::min(buf.len(), self.pending_output_bytes);
            let pending_data =
                &self.decompressor.buffer[start..
                                          start + bytes_to_copy];
            buf[..bytes_to_copy].copy_from_slice(pending_data);
            bytes_out += bytes_to_copy;
            // This won't underflow since `bytes_to_copy` will be at most
            // the same value as `pending_output_bytes`.
            self.pending_output_bytes -= bytes_to_copy;
            if self.pending_output_bytes != 0 {
                self.total_out += bytes_out as u64;
                // If there is still decompressed data left that didn't
                // fit in `buf`, return what we read.
                return Ok(bytes_out);
            }
        }

        // There is space in `buf` for more data, so try to read more.
        let (input_bytes_read, remaining_bytes) = {
            self.pending_output_bytes = 0;
            let input = self.reader.fill_buf()?;
            if input.len() == 0 {
                self.total_out += bytes_out as u64;
                //If there is nothing more to read, return.
                return Ok(bytes_out);
            }
            let (input_bytes_read, data) =
                match self.decompressor.update(&input) {
                    Ok(res) => res,
                    Err(m) => return Err(Error::new(ErrorKind::Other, m))
                };

            // Space left in `buf`
            let space_left = buf.len() - bytes_out;
            let bytes_to_copy = cmp::min(space_left, data.len());
            buf[bytes_out..bytes_out + bytes_to_copy].copy_from_slice(&data[..bytes_to_copy]);
            bytes_out += bytes_to_copy;

            // Can't underflow as bytes_to_copy is bounded by data.len().
            (input_bytes_read, data.len() - bytes_to_copy)

        };

        self.pending_output_bytes += remaining_bytes;
        self.total_in += input_bytes_read as u64;
        self.total_out += bytes_out as u64;
        self.reader.consume(input_bytes_read);

        Ok(bytes_out)
    }
}

/// A DEFLATE decoder/decompressor.
///
/// This structure implements a `Read` interface and takes a stream of compressed data that
/// implements the `Read` trait as input,
/// provoding the decompressed data when read from.
/// # Example
/// ```
/// use std::io::Read;
/// use inflate::DeflateDecoder;
/// const TEST_STRING: &'static str = "Hello, world";
/// let encoded = vec![243, 72, 205, 201, 201, 215, 81, 40, 207, 47, 202, 73, 1, 0];
/// let mut decoder = DeflateDecoder::new(&encoded[..]);
/// let mut output = Vec::new();
/// let status = decoder.read_to_end(&mut output);
/// # let _ = status;
/// assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
/// ```
pub struct DeflateDecoder<R> {
    /// Inner DeflateDecoderBuf, with R wrapped in a `BufReader`.
    inner: DeflateDecoderBuf<BufReader<R>>
}

impl<R: Read> DeflateDecoder<R> {
    /// Create a new `Deflatedecoderbuf` to read from a raw deflate stream.
    pub fn new(reader: R) -> DeflateDecoder<R> {
        DeflateDecoder {
            inner: DeflateDecoderBuf::new(BufReader::new(reader))
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    pub fn from_zlib(reader: R) -> DeflateDecoder<R> {
        DeflateDecoder {
            inner: DeflateDecoderBuf::from_zlib(BufReader::new(reader))
        }
    }

    /// Create a new `DeflateDecoderbuf` that reads from a zlib wrapped deflate stream.
    /// without calculating and validating the checksum.
    pub fn from_zlib_no_checksum(reader: R) -> DeflateDecoder<R> {
        DeflateDecoder {
            inner: DeflateDecoderBuf::from_zlib_no_checksum(BufReader::new(reader))
        }
    }

    /// Resets the decompressor, and replaces the current inner `BufRead` instance by `r`.
    /// without doing any extra reallocations.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset(&mut self, r: R) -> R {
        self.inner.reset(BufReader::new(r)).into_inner()
    }

    /// Returns a reference to the underlying reader.
    #[inline]
    pub fn get_ref(&self) -> &R {
        self.inner.get_ref().get_ref()
    }

    /// Returns a mutable reference to the underlying reader.
    ///
    /// Note that mutation of the reader may cause surprising results if the decoder is going to
    /// keep being used.
    #[inline]
    pub fn get_mut(&mut self) -> &mut R {
        self.inner.get_mut().get_mut()
    }

    /// Returns the total number of bytes output from this decoder.
    #[inline]
    pub fn into_inner(self) -> R {
        self.inner.into_inner().into_inner()
    }
}

impl<R> DeflateDecoder<R> {
    /// Resets the decoder, but continue to read from the same reader.
    ///
    /// Note that this function doesn't ensure that all data has been output.
    #[inline]
    pub fn reset_data(&mut self) {
        self.inner.reset_data()
    }

    /// Returns the total bytes read from the underlying reader.
    #[inline]
    pub fn total_in(&self) -> u64 {
        self.inner.total_in
    }

    /// Returns the total number of bytes output from this decoder.
    #[inline]
    pub fn total_out(&self) -> u64 {
        self.inner.total_out
    }

    /// Returns the calculated checksum value of the currently decoded data.
    ///
    /// Will return 0 for cases where the checksum is not validated.
    #[inline]
    pub fn current_checksum(&self) -> u32 {
        self.inner.current_checksum()
    }
}

impl<R: Read> Read for DeflateDecoder<R> {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        self.inner.read(buf)
    }
}

#[cfg(test)]
mod test {
    use super::{DeflateDecoder};
    use std::io::Read;

    #[test]
    fn deflate_reader() {
        const TEST_STRING: &'static str = "Hello, world";
        let encoded = vec![243, 72, 205, 201, 201, 215, 81, 40, 207, 47, 202, 73, 1, 0];
        let mut decoder = DeflateDecoder::new(&encoded[..]);
        let mut output = Vec::new();
        decoder.read_to_end(&mut output).unwrap();
        assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
        assert_eq!(decoder.total_in(), encoded.len() as u64);
        assert_eq!(decoder.total_out(), TEST_STRING.len() as u64);
    }

    #[test]
    fn zlib_reader() {
        const TEST_STRING: &'static str = "Hello, zlib!";
        let encoded = vec![120, 156, 243, 72, 205, 201, 201, 215, 81, 168, 202, 201,
                       76, 82, 4, 0, 27, 101, 4, 19];
        let mut decoder = DeflateDecoder::from_zlib(&encoded[..]);
        let mut output = Vec::new();
        decoder.read_to_end(&mut output).unwrap();
        assert_eq!(String::from_utf8(output).unwrap(), TEST_STRING);
        assert_eq!(decoder.total_in(), encoded.len() as u64);
        assert_eq!(decoder.total_out(), TEST_STRING.len() as u64);
    }
}
<|endoftext|>


<commit_before>use std::io;

use token::{Sentence, Token, EMPTY_TOKEN};
use error::Error;

/// A trait for objects that can read CoNLL-X `Sentence`s
pub trait ReadSentence {
    /// Read a `Sentence` from this object.
    ///
    /// # Errors
    ///
    /// A call to `read_sentence` may generate an error to indicate that
    /// the operation could not be completed.
    fn read_sentence(&mut self) -> Result<Option<Sentence>, Error>;
}

/// Reader for CoNLL-X sentences.
pub struct Reader<R> {
    read: R,
}

impl<R: io::BufRead> Reader<R> {
    /// Construct a new reader from an object that implements the
    /// `io::BufRead` trait.
    pub fn new(read: R) -> Reader<R> {
        Reader { read: read }
    }

    /// Get an iterator over the `Sentence`s in this reader.
    pub fn sentences(self) -> Sentences<R> {
        Sentences { reader: self }
    }
}

impl<R: io::BufRead> ReadSentence for Reader<R> {
    fn read_sentence(&mut self) -> Result<Option<Sentence>, Error> {
        let mut line = String::new();
        let mut tokens = Vec::new();

        loop {
            line.clear();

            // End of reader.
            if try!(self.read.read_line(&mut line)) == 0 {
                if tokens.is_empty() {
                    return Ok(None);
                }

                return Ok(Some(Sentence::new(tokens)));
            }

            // The blank line is a sentence separator. We want to be robust
            // in the case a CoNLL file is malformed and has two newlines as
            // a separator.
            if line.trim().is_empty() {
                if tokens.is_empty() {
                    continue;
                }

                return Ok(Some(Sentence::new(tokens)));
            }

            let mut iter = line.trim().split_terminator('\t');

            try!(parse_numeric_field(iter.next()));

            let mut token = Token::new();
            token.set_form(parse_string_field(iter.next()));
            token.set_lemma(parse_string_field(iter.next()));
            token.set_cpos(parse_string_field(iter.next()));
            token.set_pos(parse_string_field(iter.next()));
            token.set_features(parse_string_field(iter.next()));
            token.set_head(try!(parse_numeric_field(iter.next())));
            token.set_head_rel(parse_string_field(iter.next()));
            token.set_p_head(try!(parse_numeric_field(iter.next())));
            token.set_p_head_rel(parse_string_field(iter.next()));

            tokens.push(token);
        }
    }
}

/// Sentences is an iterator over the sentences in a `Reader`.
pub struct Sentences<R> {
    reader: Reader<R>,
}

impl<R: io::BufRead> Iterator for Sentences<R> {
    type Item = Result<Sentence, Error>;

    fn next(&mut self) -> Option<Result<Sentence, Error>> {
        match self.reader.read_sentence() {
            Ok(None) => None,
            Ok(Some(sent)) => Some(Ok(sent)),
            Err(e) => Some(Err(e)),
        }
    }
}

fn parse_string_field(field: Option<&str>) -> Option<String> {
    field.and_then(|s| {
        if s == EMPTY_TOKEN {
            None
        } else {
            Some(s.to_string())
        }
    })
}

fn parse_numeric_field(field: Option<&str>) -> Result<Option<usize>, Error> {
    match field {
        None => Ok(None),
        Some(s) => {
            if s == EMPTY_TOKEN {
                return Ok(None);
            } else {
                Ok(Some(try!(s.parse())))
            }
        }
    }
}
<commit_msg>Implement IntoIter for Reader.<commit_after>use std::io;

use token::{Sentence, Token, EMPTY_TOKEN};
use error::Error;

/// A trait for objects that can read CoNLL-X `Sentence`s
pub trait ReadSentence {
    /// Read a `Sentence` from this object.
    ///
    /// # Errors
    ///
    /// A call to `read_sentence` may generate an error to indicate that
    /// the operation could not be completed.
    fn read_sentence(&mut self) -> Result<Option<Sentence>, Error>;
}

/// Reader for CoNLL-X sentences.
pub struct Reader<R> {
    read: R,
}

impl<R: io::BufRead> Reader<R> {
    /// Construct a new reader from an object that implements the
    /// `io::BufRead` trait.
    pub fn new(read: R) -> Reader<R> {
        Reader { read: read }
    }

    /// Get an iterator over the `Sentence`s in this reader.
    pub fn sentences(self) -> Sentences<R> {
        Sentences { reader: self }
    }
}

impl<R: io::BufRead> IntoIterator for Reader<R> {
    type Item = Result<Sentence, Error>;
    type IntoIter = Sentences<R>;

    fn into_iter(self) -> Self::IntoIter {
        self.sentences()
    }
}

impl<R: io::BufRead> ReadSentence for Reader<R> {
    fn read_sentence(&mut self) -> Result<Option<Sentence>, Error> {
        let mut line = String::new();
        let mut tokens = Vec::new();

        loop {
            line.clear();

            // End of reader.
            if try!(self.read.read_line(&mut line)) == 0 {
                if tokens.is_empty() {
                    return Ok(None);
                }

                return Ok(Some(Sentence::new(tokens)));
            }

            // The blank line is a sentence separator. We want to be robust
            // in the case a CoNLL file is malformed and has two newlines as
            // a separator.
            if line.trim().is_empty() {
                if tokens.is_empty() {
                    continue;
                }

                return Ok(Some(Sentence::new(tokens)));
            }

            let mut iter = line.trim().split_terminator('\t');

            try!(parse_numeric_field(iter.next()));

            let mut token = Token::new();
            token.set_form(parse_string_field(iter.next()));
            token.set_lemma(parse_string_field(iter.next()));
            token.set_cpos(parse_string_field(iter.next()));
            token.set_pos(parse_string_field(iter.next()));
            token.set_features(parse_string_field(iter.next()));
            token.set_head(try!(parse_numeric_field(iter.next())));
            token.set_head_rel(parse_string_field(iter.next()));
            token.set_p_head(try!(parse_numeric_field(iter.next())));
            token.set_p_head_rel(parse_string_field(iter.next()));

            tokens.push(token);
        }
    }
}

/// Sentences is an iterator over the sentences in a `Reader`.
pub struct Sentences<R> {
    reader: Reader<R>,
}

impl<R: io::BufRead> Iterator for Sentences<R> {
    type Item = Result<Sentence, Error>;

    fn next(&mut self) -> Option<Result<Sentence, Error>> {
        match self.reader.read_sentence() {
            Ok(None) => None,
            Ok(Some(sent)) => Some(Ok(sent)),
            Err(e) => Some(Err(e)),
        }
    }
}

fn parse_string_field(field: Option<&str>) -> Option<String> {
    field.and_then(|s| {
        if s == EMPTY_TOKEN {
            None
        } else {
            Some(s.to_string())
        }
    })
}

fn parse_numeric_field(field: Option<&str>) -> Result<Option<usize>, Error> {
    match field {
        None => Ok(None),
        Some(s) => {
            if s == EMPTY_TOKEN {
                return Ok(None);
            } else {
                Ok(Some(try!(s.parse())))
            }
        }
    }
}
<|endoftext|>


<commit_before>//! Error types that can be emitted from this library

use std::io;
use std::error;
use std::fmt;

/// Generic result type with ZipError as its error variant
pub type ZipResult<T> = Result<T, ZipError>;

/// Error type for Zip
#[derive(Debug)]
pub enum ZipError
{
    /// An Error caused by I/O
    Io(io::Error),

    /// This file is probably not a zip archive
    InvalidArchive(&'static str),

    /// This archive is not supported
    UnsupportedArchive(&'static str),

    /// The requested file could not be found in the archive
    FileNotFound,
}

impl ZipError
{
    fn detail(&self) -> ::std::borrow::Cow<str>
    {
        use ::std::error::Error;
        use ::std::borrow::IntoCow;

        match *self
        {
            ZipError::Io(ref io_err) => {
                ("Io Error: ".to_string() + io_err.description()).into_cow()
            },
            ZipError::InvalidArchive(msg) | ZipError::UnsupportedArchive(msg) => {
                (self.description().to_string() + ": " + msg).into_cow()
            },
            ZipError::FileNotFound => {
                self.description().into_cow()
            },
        }
    }
}

impl error::FromError<io::Error> for ZipError
{
    fn from_error(err: io::Error) -> ZipError
    {
        ZipError::Io(err)
    }
}

impl fmt::Display for ZipError
{
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error>
    {
        fmt.write_str(&*self.detail())
    }
}

impl error::Error for ZipError
{
    fn description(&self) -> &str
    {
        match *self
        {
            ZipError::Io(ref io_err) => io_err.description(),
            ZipError::InvalidArchive(..) => "Invalid Zip archive",
            ZipError::UnsupportedArchive(..) => "Unsupported Zip archive",
            ZipError::FileNotFound => "Specified file not found in archive",
        }
    }

    fn cause(&self) -> Option<&error::Error>
    {
        match *self
        {
            ZipError::Io(ref io_err) => Some(io_err as &error::Error),
            _ => None,
        }
    }
}
<commit_msg>Silence deprecation warnings by casting<commit_after>//! Error types that can be emitted from this library

use std::io;
use std::error;
use std::fmt;

/// Generic result type with ZipError as its error variant
pub type ZipResult<T> = Result<T, ZipError>;

/// Error type for Zip
#[derive(Debug)]
pub enum ZipError
{
    /// An Error caused by I/O
    Io(io::Error),

    /// This file is probably not a zip archive
    InvalidArchive(&'static str),

    /// This archive is not supported
    UnsupportedArchive(&'static str),

    /// The requested file could not be found in the archive
    FileNotFound,
}

impl ZipError
{
    fn detail(&self) -> ::std::borrow::Cow<str>
    {
        use ::std::error::Error;
        use ::std::borrow::IntoCow;

        match *self
        {
            ZipError::Io(ref io_err) => {
                ("Io Error: ".to_string() + (io_err as &error::Error).description()).into_cow()
            },
            ZipError::InvalidArchive(msg) | ZipError::UnsupportedArchive(msg) => {
                (self.description().to_string() + ": " + msg).into_cow()
            },
            ZipError::FileNotFound => {
                self.description().into_cow()
            },
        }
    }
}

impl error::FromError<io::Error> for ZipError
{
    fn from_error(err: io::Error) -> ZipError
    {
        ZipError::Io(err)
    }
}

impl fmt::Display for ZipError
{
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error>
    {
        fmt.write_str(&*self.detail())
    }
}

impl error::Error for ZipError
{
    fn description(&self) -> &str
    {
        match *self
        {
            ZipError::Io(ref io_err) => (io_err as &error::Error).description(),
            ZipError::InvalidArchive(..) => "Invalid Zip archive",
            ZipError::UnsupportedArchive(..) => "Unsupported Zip archive",
            ZipError::FileNotFound => "Specified file not found in archive",
        }
    }

    fn cause(&self) -> Option<&error::Error>
    {
        match *self
        {
            ZipError::Io(ref io_err) => Some(io_err as &error::Error),
            _ => None,
        }
    }
}
<|endoftext|>