<commit_before>extern crate rocksdb;
extern crate rustc_serialize;

use std::collections::HashMap;

use self::rustc_serialize::json::{JsonEvent, Parser, StackElement};
// Needed for a trait in order to `put()` into a `rocksdb::WriteBatch`
use self::rocksdb::Writable;

use key_builder::{KeyBuilder, SegmentType};
use records_capnp::payload;
use stems::Stems;
use query::{Error};

// Good example of using rustc_serialize: https://github.com/ajroetker/beautician/blob/master/src/lib.rs
// Callback based JSON streaming parser: https://github.com/gyscos/json-streamer.rs
// Another parser pased on rustc_serializ: https://github.com/isagalaev/ijson-rust/blob/master/src/test.rs#L11

#[derive(Debug, PartialEq)]
struct WordInfo {
    //offset in the text field where the stemmed text starts
    stemmed_offset: u64,

    // the suffix of the stemmed text. When applied over stemmed, the original
    // text is returned.
    suffix_text: String,

    // the start of the suffixText
    suffix_offset: u64,
}

type ArrayOffsets = Vec<u64>;
type ArrayOffsetsToWordInfo = HashMap<ArrayOffsets, Vec<WordInfo>>;
type WordPathInfoMap = HashMap<String, ArrayOffsetsToWordInfo>;

enum ObjectKeyTypes {
    /// _id field
    Id,
    /// Normal key
    Key(String),
    /// Reserved key starting with underscore
    Ignore,
    /// No key found
    NoKey,
}

#[derive(Debug)]
pub struct Shredder {
    keybuilder: KeyBuilder,
    map: WordPathInfoMap,
    path_array_offsets: ArrayOffsets,
    // Top-level fields prefixed with an underscore are ignored
    ignore_children: u64,
    doc_id: String,
}

impl Shredder {
    pub fn new() -> Shredder {
        Shredder{
            keybuilder: KeyBuilder::new(),
            map: WordPathInfoMap::new(),
            path_array_offsets: Vec::new(),
            ignore_children: 0,
            doc_id: String::new(),
        }
    }
    fn add_entries(&mut self, text: &String, docseq: u64) {
        let stems = Stems::new(text.as_str());
        for stem in stems {
            self.keybuilder.push_word(&stem.stemmed);
            self.keybuilder.push_doc_seq(docseq);
            let map_path_array_offsets = self.map.entry(self.keybuilder.key())
                                                        .or_insert(ArrayOffsetsToWordInfo::new());
            let map_word_infos = map_path_array_offsets.entry(self.path_array_offsets.clone())
                .or_insert(Vec::new());
            map_word_infos.push(WordInfo{
                stemmed_offset: stem.stemmed_offset as u64,
                suffix_text: stem.suffix.to_string(),
                suffix_offset: stem.suffix_offset as u64,
            });
            self.keybuilder.pop_doc_seq();
            self.keybuilder.pop_word();
        }
        println!("add_entries: map: {:?}", self.map);
    }

    fn inc_top_array_offset(&mut self) {
        // we encounter a new element. if we are a child element of an array
        // increment the offset. If we aren't (we are the root value or a map
        // value) we don't increment
        if let Some(SegmentType::Array) = self.keybuilder.last_pushed_segment_type() {
            if let Some(last) = self.path_array_offsets.last_mut() {
                *last += 1;
            }
        }
    }

    // Extract key if it exists and indicates if it's a special type of key
    fn extract_key(&mut self, stack_element: Option<StackElement>) -> ObjectKeyTypes {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey {
            match stack_element {
                Some(StackElement::Key(key)) => {
                    if self.keybuilder.segments.len() == 1 && key.starts_with("_") {
                        if key == "_id" {
                            ObjectKeyTypes::Id
                        } else {
                            ObjectKeyTypes::Ignore
                        }
                    } else {
                        ObjectKeyTypes::Key(key.to_string())
                    }
                },
                _ => ObjectKeyTypes::NoKey,
            }
        } else {
            ObjectKeyTypes::NoKey
        }
    }

    // If we are inside an object we need to push the key to the key builder
    // Don't push them if they are reserved fields (starting with underscore)
    fn maybe_push_key(&mut self, stack_element: Option<StackElement>) -> Result<(), Error> {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey
                && self.keybuilder.segments.len() == 1 {
            if let Some(StackElement::Key(key)) = stack_element {
                if key.starts_with("_") {
                    if key == "_id" {
                        return Err(Error::Shred(
                            "Expected string for `_id` field, got another type".to_string()));
                    } else {
                        self.ignore_children = 1;
                    }
                } else {
                    // Pop the dummy object that makes ObjectEnd happy
                    // or the previous object key
                    self.keybuilder.pop_object_key();
                    self.keybuilder.push_object_key(key.to_string());
                }
            }
        }
        Ok(())
    }

   pub fn shred(&mut self, json: &str, docseq: u64) -> Result<String, Error> {
        let mut parser = Parser::new(json.chars());
        let mut token = parser.next();

        loop {
            // Get the next token, so that in case of an `ObjectStart` the key is already
            // on the stack.
            match token.take() {
                Some(JsonEvent::ObjectStart) => {
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    }
                    else {
                        // Just push something to make `ObjectEnd` happy
                        self.keybuilder.push_object_key("".to_string());
                    }
                },
                Some(JsonEvent::ObjectEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {//if !self.keybuilder.segments.is_empty() {
                        self.keybuilder.pop_object_key();
                    }
                },
                Some(JsonEvent::ArrayStart) => {
                    try!(self.maybe_push_key(parser.stack().top()));
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    } else {
                        self.keybuilder.push_array();
                        self.path_array_offsets.push(0);
                    }
                },
                Some(JsonEvent::ArrayEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {
                        self.path_array_offsets.pop();
                        self.keybuilder.pop_array();
                    }
                },
                Some(JsonEvent::StringValue(value)) => {
                    // No children to ignore
                    if self.ignore_children == 0 {
                        println!("stringvalue: {:?}", value);
                        match self.extract_key(parser.stack().top()) {
                            ObjectKeyTypes::Id => self.doc_id = value,
                            ObjectKeyTypes::Key(key) => {
                                // Pop the dummy object that makes ObjectEnd happy
                                // or the previous object key
                                self.keybuilder.pop_object_key();
                                self.keybuilder.push_object_key(key.to_string());

                                self.add_entries(&value, docseq);
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::NoKey => {
                                self.add_entries(&value, docseq);
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::Ignore => {
                                self.ignore_children = 1;
                            },
                        }
                    }
                },
                not_implemented => {
                    panic!("Not yet implemented other JSON types! {:?}", not_implemented);
                }
            };

            token = parser.next();
            if token == None {
                break;
            }
        }
        println!("keybuilder: {}", self.keybuilder.key());
        println!("shredder: keys:");
        for key in self.map.keys() {
            println!("  {}", key);
        }
        Ok(self.doc_id.clone())
    }

    pub fn add_to_batch(&self, batch: &rocksdb::WriteBatch) -> Result<(), Error> {
        for (key_path, word_path_infos) in &self.map {
            let mut message = ::capnp::message::Builder::new_default();
            {
                let capn_payload = message.init_root::<payload::Builder>();
                let mut capn_arrayoffsets_to_wordinfo = capn_payload.init_arrayoffsets_to_wordinfos(
                    word_path_infos.len() as u32);
                for (infos_pos, (arrayoffsets, wordinfos)) in word_path_infos.iter().enumerate() {

                    let mut capn_a2w = capn_arrayoffsets_to_wordinfo.borrow().get(infos_pos as u32);
                    {
                        let mut capn_arrayoffsets =
                            capn_a2w.borrow().init_arrayoffsets(arrayoffsets.len() as u32);
                        for (pos, arrayoffset) in arrayoffsets.iter().enumerate() {
                            capn_arrayoffsets.set(pos as u32, arrayoffset.clone());
                        }
                    }
                    {
                        let mut capn_wordinfos = capn_a2w.init_wordinfos(wordinfos.len() as u32);
                        for (pos, wordinfo) in wordinfos.iter().enumerate() {
                            let mut capn_wordinfo = capn_wordinfos.borrow().get(pos as u32);
                            capn_wordinfo.set_stemmed_offset(wordinfo.stemmed_offset);
                            capn_wordinfo.set_suffix_text(&wordinfo.suffix_text);
                            capn_wordinfo.set_suffix_offset(wordinfo.suffix_offset);
                        }
                    }
                }
            }
            let mut bytes = Vec::new();
            ::capnp::serialize_packed::write_message(&mut bytes, &message).unwrap();
            try!(batch.put(&key_path.clone().into_bytes(), &bytes));
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::{ArrayOffsetsToWordInfo, WordInfo, WordPathInfoMap};

    #[test]
    fn test_shred_nested() {
        let mut shredder = super::Shredder::new();
        //let json = r#"{"hello": {"my": "world!"}, "anumber": 2}"#;
        //let json = r#"{"A":[{"B":"B2VMX two three","C":"C2"},{"B": "b1","C":"C2"}]}"#;
        //let json = r#"{"A":[[[{"B": "string within deeply nested array should be stemmed"}]]]}"#;
        //let json = r#"[{"A": 1, "B": 2, "C": 3}]"#;
        //let json = r#"{"foo": {"bar": 1}}"#;
        let json = r#"{"some": ["array", "data", ["also", "nested"]]}"#;
        let docseq = 123;
        shredder.shred(json, docseq).unwrap();
        let expected = vec![
            ("W.some$!array#123", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 5 }])]),
            ("W.some$!data#123", vec![
                (vec![1], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ("W.some$$!also#123", vec![
                (vec![2, 0], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ("W.some$$!nest#123", vec![
                (vec![2, 1], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "ed".to_string(), suffix_offset: 4 }])]),
            ];
        compare_shredded(&shredder.map, &expected);
    }

    #[test]
    fn test_shred_objects() {
        let mut shredder = super::Shredder::new();
        let json = r#"{"A":[{"B":"B2VMX two three","C":"..C2"},{"B": "b1","C":"..C2"}]}"#;
        let docseq = 1234;
        shredder.shred(json, docseq).unwrap();
        let expected = vec![
            ("W.A$.B!b1#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }])]),
            ("W.A$.B!b2vmx#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 5 }])]),
            ("W.A$.B!three#1234", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 10, suffix_text: "".to_string(), suffix_offset: 15 }])]),
            ("W.A$.B!two#1234", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 6, suffix_text: "".to_string(), suffix_offset: 9 }])]),
            ("W.A$.C!c2#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 2, suffix_text: "".to_string(), suffix_offset: 4 },
                    WordInfo {
                        stemmed_offset: 2, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ];
        compare_shredded(&shredder.map, &expected);
    }

    fn compare_shredded(result_map: &WordPathInfoMap,
                        expected: &Vec<(&str, Vec<(Vec<u64>, Vec<WordInfo>)>)>) {
        // HashMap have an arbitraty order of the elements
        let mut result: Vec<(&String, &ArrayOffsetsToWordInfo)> = result_map.into_iter().collect();
        result.sort_by(|a, b| Ord::cmp(&a.0, &b.0));
        for (ii, &(key, values)) in result.iter().enumerate() {
            assert_eq!(key, expected[ii].0);
            let mut wordinfos: Vec<(&Vec<u64>, &Vec<WordInfo>)> = values.iter().collect();
            wordinfos.sort_by_key(|item| item.0);
            for (jj, wordinfo) in wordinfos.iter().enumerate() {
                assert_eq!(wordinfo.0, &expected[ii].1[jj].0);
                assert_eq!(wordinfo.1, &expected[ii].1[jj].1);
            }
        }
    }

    #[test]
    fn test_shred_empty_object() {
        let mut shredder = super::Shredder::new();
        let json = r#"{}"#;
        let docseq = 123;
        shredder.shred(json, docseq).unwrap();
        // TODO vmx 2016-11-08: Finish this test
    }
}
<commit_msg>Finish simple empty object test<commit_after>extern crate rocksdb;
extern crate rustc_serialize;

use std::collections::HashMap;

use self::rustc_serialize::json::{JsonEvent, Parser, StackElement};
// Needed for a trait in order to `put()` into a `rocksdb::WriteBatch`
use self::rocksdb::Writable;

use key_builder::{KeyBuilder, SegmentType};
use records_capnp::payload;
use stems::Stems;
use query::{Error};

// Good example of using rustc_serialize: https://github.com/ajroetker/beautician/blob/master/src/lib.rs
// Callback based JSON streaming parser: https://github.com/gyscos/json-streamer.rs
// Another parser pased on rustc_serializ: https://github.com/isagalaev/ijson-rust/blob/master/src/test.rs#L11

#[derive(Debug, PartialEq)]
struct WordInfo {
    //offset in the text field where the stemmed text starts
    stemmed_offset: u64,

    // the suffix of the stemmed text. When applied over stemmed, the original
    // text is returned.
    suffix_text: String,

    // the start of the suffixText
    suffix_offset: u64,
}

type ArrayOffsets = Vec<u64>;
type ArrayOffsetsToWordInfo = HashMap<ArrayOffsets, Vec<WordInfo>>;
type WordPathInfoMap = HashMap<String, ArrayOffsetsToWordInfo>;

enum ObjectKeyTypes {
    /// _id field
    Id,
    /// Normal key
    Key(String),
    /// Reserved key starting with underscore
    Ignore,
    /// No key found
    NoKey,
}

#[derive(Debug)]
pub struct Shredder {
    keybuilder: KeyBuilder,
    map: WordPathInfoMap,
    path_array_offsets: ArrayOffsets,
    // Top-level fields prefixed with an underscore are ignored
    ignore_children: u64,
    doc_id: String,
}

impl Shredder {
    pub fn new() -> Shredder {
        Shredder{
            keybuilder: KeyBuilder::new(),
            map: WordPathInfoMap::new(),
            path_array_offsets: Vec::new(),
            ignore_children: 0,
            doc_id: String::new(),
        }
    }
    fn add_entries(&mut self, text: &String, docseq: u64) {
        let stems = Stems::new(text.as_str());
        for stem in stems {
            self.keybuilder.push_word(&stem.stemmed);
            self.keybuilder.push_doc_seq(docseq);
            let map_path_array_offsets = self.map.entry(self.keybuilder.key())
                                                        .or_insert(ArrayOffsetsToWordInfo::new());
            let map_word_infos = map_path_array_offsets.entry(self.path_array_offsets.clone())
                .or_insert(Vec::new());
            map_word_infos.push(WordInfo{
                stemmed_offset: stem.stemmed_offset as u64,
                suffix_text: stem.suffix.to_string(),
                suffix_offset: stem.suffix_offset as u64,
            });
            self.keybuilder.pop_doc_seq();
            self.keybuilder.pop_word();
        }
        println!("add_entries: map: {:?}", self.map);
    }

    fn inc_top_array_offset(&mut self) {
        // we encounter a new element. if we are a child element of an array
        // increment the offset. If we aren't (we are the root value or a map
        // value) we don't increment
        if let Some(SegmentType::Array) = self.keybuilder.last_pushed_segment_type() {
            if let Some(last) = self.path_array_offsets.last_mut() {
                *last += 1;
            }
        }
    }

    // Extract key if it exists and indicates if it's a special type of key
    fn extract_key(&mut self, stack_element: Option<StackElement>) -> ObjectKeyTypes {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey {
            match stack_element {
                Some(StackElement::Key(key)) => {
                    if self.keybuilder.segments.len() == 1 && key.starts_with("_") {
                        if key == "_id" {
                            ObjectKeyTypes::Id
                        } else {
                            ObjectKeyTypes::Ignore
                        }
                    } else {
                        ObjectKeyTypes::Key(key.to_string())
                    }
                },
                _ => ObjectKeyTypes::NoKey,
            }
        } else {
            ObjectKeyTypes::NoKey
        }
    }

    // If we are inside an object we need to push the key to the key builder
    // Don't push them if they are reserved fields (starting with underscore)
    fn maybe_push_key(&mut self, stack_element: Option<StackElement>) -> Result<(), Error> {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey
                && self.keybuilder.segments.len() == 1 {
            if let Some(StackElement::Key(key)) = stack_element {
                if key.starts_with("_") {
                    if key == "_id" {
                        return Err(Error::Shred(
                            "Expected string for `_id` field, got another type".to_string()));
                    } else {
                        self.ignore_children = 1;
                    }
                } else {
                    // Pop the dummy object that makes ObjectEnd happy
                    // or the previous object key
                    self.keybuilder.pop_object_key();
                    self.keybuilder.push_object_key(key.to_string());
                }
            }
        }
        Ok(())
    }

   pub fn shred(&mut self, json: &str, docseq: u64) -> Result<String, Error> {
        let mut parser = Parser::new(json.chars());
        let mut token = parser.next();

        loop {
            // Get the next token, so that in case of an `ObjectStart` the key is already
            // on the stack.
            match token.take() {
                Some(JsonEvent::ObjectStart) => {
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    }
                    else {
                        // Just push something to make `ObjectEnd` happy
                        self.keybuilder.push_object_key("".to_string());
                    }
                },
                Some(JsonEvent::ObjectEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {//if !self.keybuilder.segments.is_empty() {
                        self.keybuilder.pop_object_key();
                    }
                },
                Some(JsonEvent::ArrayStart) => {
                    try!(self.maybe_push_key(parser.stack().top()));
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    } else {
                        self.keybuilder.push_array();
                        self.path_array_offsets.push(0);
                    }
                },
                Some(JsonEvent::ArrayEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {
                        self.path_array_offsets.pop();
                        self.keybuilder.pop_array();
                    }
                },
                Some(JsonEvent::StringValue(value)) => {
                    // No children to ignore
                    if self.ignore_children == 0 {
                        println!("stringvalue: {:?}", value);
                        match self.extract_key(parser.stack().top()) {
                            ObjectKeyTypes::Id => self.doc_id = value,
                            ObjectKeyTypes::Key(key) => {
                                // Pop the dummy object that makes ObjectEnd happy
                                // or the previous object key
                                self.keybuilder.pop_object_key();
                                self.keybuilder.push_object_key(key.to_string());

                                self.add_entries(&value, docseq);
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::NoKey => {
                                self.add_entries(&value, docseq);
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::Ignore => {
                                self.ignore_children = 1;
                            },
                        }
                    }
                },
                not_implemented => {
                    panic!("Not yet implemented other JSON types! {:?}", not_implemented);
                }
            };

            token = parser.next();
            if token == None {
                break;
            }
        }
        println!("keybuilder: {}", self.keybuilder.key());
        println!("shredder: keys:");
        for key in self.map.keys() {
            println!("  {}", key);
        }
        Ok(self.doc_id.clone())
    }

    pub fn add_to_batch(&self, batch: &rocksdb::WriteBatch) -> Result<(), Error> {
        for (key_path, word_path_infos) in &self.map {
            let mut message = ::capnp::message::Builder::new_default();
            {
                let capn_payload = message.init_root::<payload::Builder>();
                let mut capn_arrayoffsets_to_wordinfo = capn_payload.init_arrayoffsets_to_wordinfos(
                    word_path_infos.len() as u32);
                for (infos_pos, (arrayoffsets, wordinfos)) in word_path_infos.iter().enumerate() {

                    let mut capn_a2w = capn_arrayoffsets_to_wordinfo.borrow().get(infos_pos as u32);
                    {
                        let mut capn_arrayoffsets =
                            capn_a2w.borrow().init_arrayoffsets(arrayoffsets.len() as u32);
                        for (pos, arrayoffset) in arrayoffsets.iter().enumerate() {
                            capn_arrayoffsets.set(pos as u32, arrayoffset.clone());
                        }
                    }
                    {
                        let mut capn_wordinfos = capn_a2w.init_wordinfos(wordinfos.len() as u32);
                        for (pos, wordinfo) in wordinfos.iter().enumerate() {
                            let mut capn_wordinfo = capn_wordinfos.borrow().get(pos as u32);
                            capn_wordinfo.set_stemmed_offset(wordinfo.stemmed_offset);
                            capn_wordinfo.set_suffix_text(&wordinfo.suffix_text);
                            capn_wordinfo.set_suffix_offset(wordinfo.suffix_offset);
                        }
                    }
                }
            }
            let mut bytes = Vec::new();
            ::capnp::serialize_packed::write_message(&mut bytes, &message).unwrap();
            try!(batch.put(&key_path.clone().into_bytes(), &bytes));
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::{ArrayOffsetsToWordInfo, WordInfo, WordPathInfoMap};

    #[test]
    fn test_shred_nested() {
        let mut shredder = super::Shredder::new();
        //let json = r#"{"hello": {"my": "world!"}, "anumber": 2}"#;
        //let json = r#"{"A":[{"B":"B2VMX two three","C":"C2"},{"B": "b1","C":"C2"}]}"#;
        //let json = r#"{"A":[[[{"B": "string within deeply nested array should be stemmed"}]]]}"#;
        //let json = r#"[{"A": 1, "B": 2, "C": 3}]"#;
        //let json = r#"{"foo": {"bar": 1}}"#;
        let json = r#"{"some": ["array", "data", ["also", "nested"]]}"#;
        let docseq = 123;
        shredder.shred(json, docseq).unwrap();
        let expected = vec![
            ("W.some$!array#123", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 5 }])]),
            ("W.some$!data#123", vec![
                (vec![1], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ("W.some$$!also#123", vec![
                (vec![2, 0], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ("W.some$$!nest#123", vec![
                (vec![2, 1], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "ed".to_string(), suffix_offset: 4 }])]),
            ];
        compare_shredded(&shredder.map, &expected);
    }

    #[test]
    fn test_shred_objects() {
        let mut shredder = super::Shredder::new();
        let json = r#"{"A":[{"B":"B2VMX two three","C":"..C2"},{"B": "b1","C":"..C2"}]}"#;
        let docseq = 1234;
        shredder.shred(json, docseq).unwrap();
        let expected = vec![
            ("W.A$.B!b1#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }])]),
            ("W.A$.B!b2vmx#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 5 }])]),
            ("W.A$.B!three#1234", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 10, suffix_text: "".to_string(), suffix_offset: 15 }])]),
            ("W.A$.B!two#1234", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 6, suffix_text: "".to_string(), suffix_offset: 9 }])]),
            ("W.A$.C!c2#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 2, suffix_text: "".to_string(), suffix_offset: 4 },
                    WordInfo {
                        stemmed_offset: 2, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ];
        compare_shredded(&shredder.map, &expected);
    }

    fn compare_shredded(result_map: &WordPathInfoMap,
                        expected: &Vec<(&str, Vec<(Vec<u64>, Vec<WordInfo>)>)>) {
        // HashMap have an arbitraty order of the elements
        let mut result: Vec<(&String, &ArrayOffsetsToWordInfo)> = result_map.into_iter().collect();
        result.sort_by(|a, b| Ord::cmp(&a.0, &b.0));
        for (ii, &(key, values)) in result.iter().enumerate() {
            assert_eq!(key, expected[ii].0);
            let mut wordinfos: Vec<(&Vec<u64>, &Vec<WordInfo>)> = values.iter().collect();
            wordinfos.sort_by_key(|item| item.0);
            for (jj, wordinfo) in wordinfos.iter().enumerate() {
                assert_eq!(wordinfo.0, &expected[ii].1[jj].0);
                assert_eq!(wordinfo.1, &expected[ii].1[jj].1);
            }
        }
    }

    #[test]
    fn test_shred_empty_object() {
        let mut shredder = super::Shredder::new();
        let json = r#"{}"#;
        let docseq = 123;
        shredder.shred(json, docseq).unwrap();
        assert!(shredder.map.is_empty());
    }
}
<|endoftext|>


<commit_before>mod test_common;
mod partition;
mod bounds;
mod distance;

use self::bounds::*;
use self::distance::*;

pub trait KdtreePointTrait {
    fn dims(&self) -> &[f64];
}

pub struct Kdtree<T> {
    nodes: Vec<KdtreeNode<T>>,
}

impl<T: KdtreePointTrait + Copy> Kdtree<T> {
    pub fn new(mut points: Vec<T>) -> Kdtree<T> {
        if points.len() == 0 {
            panic!("empty vector point not allowed");
        }

        let rect = Bounds::new_from_points(&points);

        let mut tree = Kdtree {
            nodes: vec![],
        };

        tree.build_tree(&mut points, &rect);

        tree
    }

    pub fn nearest_search(&self, node : &T) -> T
    {
        let mut nearest_neighbor = 0usize;
        let mut best_distance = squared_euclidean(node.dims(), &self.nodes[0].point.dims());
        self.nearest_search_impl(node, 0usize, &mut best_distance , &mut nearest_neighbor);

        self.nodes[nearest_neighbor].point
    }

    fn nearest_search_impl(&self, p : &T, searched_index: usize, best_distance_squared : &mut f64, best_leaf_found : &mut usize) {
        let node = &self.nodes[searched_index];

        let dimension = node.dimension;
        let splitting_value = node.split_on;
        let point_splitting_dim_value = p.dims()[dimension];

        let (closer_node, farther_node) =
        if point_splitting_dim_value <= splitting_value {
            (node.left_node, node.right_node)
        } else {
            (node.right_node, node.left_node)
        };

        if let Some(closer_node) = closer_node {
            self.nearest_search_impl(p, closer_node, best_distance_squared, best_leaf_found);
        }

        let distance = squared_euclidean(p.dims(), node.point.dims());
        if distance < *best_distance_squared {
            *best_distance_squared = distance;
            *best_leaf_found = searched_index;
        }

        if let Some(farther_node) = farther_node {
            let distance_on_single_dimension = squared_euclidean(&[splitting_value],&[point_splitting_dim_value]);

            if distance_on_single_dimension <= *best_distance_squared {
                self.nearest_search_impl(p, farther_node, best_distance_squared, best_leaf_found);
            }
        }
    }

    fn add_node(&mut self, p: T, dimension : usize, split_on : f64) -> usize {
        let node = KdtreeNode::new(p, dimension, split_on );

        self.nodes.push(node);
        self.nodes.len() - 1
    }

    fn build_tree(&mut self, nodes: &mut [T], bounds: &Bounds) -> usize {
        let (splitting_index, pivot_value) = partition::partition_sliding_midpoint(nodes, bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());

        let node_id = self.add_node(nodes[splitting_index], bounds.get_widest_dim(), bounds.get_midvalue_of_widest_dim());
        let nodes_len = nodes.len();

        if splitting_index > 0 {
            let left_rect = bounds.clone_moving_max(bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());
            let left_child_id = self.build_tree(&mut nodes[0..splitting_index], &left_rect);
            self.nodes[node_id].left_node = Some(left_child_id);
        }

        if splitting_index < nodes.len() - 1 {
            let right_rect = bounds.clone_moving_min(bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());

            let right_child_id = self.build_tree(&mut nodes[splitting_index + 1..nodes_len], &right_rect);
            self.nodes[node_id].right_node = Some(right_child_id);
        }

        node_id
    }
}

pub struct KdtreeNode<T> {
    left_node: Option<usize>,
    right_node: Option<usize>,

    point: T,
    dimension: usize,
    split_on: f64
}

impl<T: KdtreePointTrait> KdtreeNode<T> {
    fn new(p: T, splitting_dimension: usize, split_on_value : f64) -> KdtreeNode<T> {
        KdtreeNode {
            left_node: None,
            right_node: None,

            point: p,
            dimension : splitting_dimension,
            split_on : split_on_value
        }
    }
}

#[cfg(test)]
mod tests {
    use ::kdtree::test_common::tests_utils::Point2WithId;

    use super::*;

    #[test]
    #[should_panic(expected = "empty vector point not allowed")]
    fn should_panic_given_empty_vector() {
        let empty_vec: Vec<Point2WithId> = vec![];

        Kdtree::new(empty_vec);
    }

    quickcheck! {
        fn tree_build_creates_tree_with_as_many_leafs_as_there_is_points(xs : Vec<f64>) -> bool {
            if(xs.len() == 0) {
                return true;
            }
            let mut vec : Vec<Point2WithId> = vec![];
            for i in 0 .. xs.len() {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);

                vec.push(p);
            }

            let tree = Kdtree::new(qc_value_vec_to_2d_points_vec(&xs));

            let mut to_iterate : Vec<usize> = vec![];
            to_iterate.push(0);

            while to_iterate.len() > 0 {
                let last_index = to_iterate.last().unwrap().clone();
                let ref x = tree.nodes.get(last_index).unwrap();
                to_iterate.pop();
                if x.left_node.is_some() {
                    to_iterate.push(x.left_node.unwrap());
                }
                if x.right_node.is_some() {
                    to_iterate.push(x.right_node.unwrap());
                }
            }
            xs.len() == tree.nodes.len()
        }
    }

    quickcheck! {
        fn nearest_neighbor_search_using_qc(xs : Vec<f64>) -> bool {
            if(xs.len() == 0) {
                return true;
            }

            let point_vec = qc_value_vec_to_2d_points_vec(&xs);
            let tree = Kdtree::new(point_vec.clone());

            for p in &point_vec {
                let found_nn = tree.nearest_search(p);

                assert_eq!(p.id,found_nn.id);
            }

            true
        }
    }

    fn qc_value_vec_to_2d_points_vec(xs : &Vec<f64>) -> Vec<Point2WithId> {
        let mut vec : Vec<Point2WithId> = vec![];
        for i in 0 .. xs.len() {
            let mut is_duplicated_value = false;
            for j in 0 .. i {
                if xs[i] == xs[j] {
                    is_duplicated_value = true;
                    break;
                }
            }
            if !is_duplicated_value {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);
                vec.push(p);
            }
        }

        vec
    }
}<commit_msg>Fixed cloning, invalid arguments<commit_after>mod test_common;
mod partition;
mod bounds;
mod distance;

use self::bounds::*;
use self::distance::*;

pub trait KdtreePointTrait {
    fn dims(&self) -> &[f64];
}

pub struct Kdtree<T> {
    nodes: Vec<KdtreeNode<T>>,
}

impl<T: KdtreePointTrait + Copy> Kdtree<T> {
    pub fn new(mut points: Vec<T>) -> Kdtree<T> {
        if points.len() == 0 {
            panic!("empty vector point not allowed");
        }

        let rect = Bounds::new_from_points(&points);

        let mut tree = Kdtree {
            nodes: vec![],
        };

        tree.build_tree(&mut points, &rect);

        tree
    }

    pub fn nearest_search(&self, node : &T) -> T
    {
        let mut nearest_neighbor = 0usize;
        let mut best_distance = squared_euclidean(node.dims(), &self.nodes[0].point.dims());
        self.nearest_search_impl(node, 0usize, &mut best_distance , &mut nearest_neighbor);

        self.nodes[nearest_neighbor].point
    }

    fn nearest_search_impl(&self, p : &T, searched_index: usize, best_distance_squared : &mut f64, best_leaf_found : &mut usize) {
        let node = &self.nodes[searched_index];

        let dimension = node.dimension;
        let splitting_value = node.split_on;
        let point_splitting_dim_value = p.dims()[dimension];

        let (closer_node, farther_node) =
        if point_splitting_dim_value <= splitting_value {
            (node.left_node, node.right_node)
        } else {
            (node.right_node, node.left_node)
        };

        if let Some(closer_node) = closer_node {
            self.nearest_search_impl(p, closer_node, best_distance_squared, best_leaf_found);
        }

        let distance = squared_euclidean(p.dims(), node.point.dims());
        if distance < *best_distance_squared {
            *best_distance_squared = distance;
            *best_leaf_found = searched_index;
        }

        if let Some(farther_node) = farther_node {
            let distance_on_single_dimension = squared_euclidean(&[splitting_value],&[point_splitting_dim_value]);

            if distance_on_single_dimension <= *best_distance_squared {
                self.nearest_search_impl(p, farther_node, best_distance_squared, best_leaf_found);
            }
        }
    }

    fn add_node(&mut self, p: T, dimension : usize, split_on : f64) -> usize {
        let node = KdtreeNode::new(p, dimension, split_on );

        self.nodes.push(node);
        self.nodes.len() - 1
    }

    fn build_tree(&mut self, nodes: &mut [T], bounds: &Bounds) -> usize {
        let (splitting_index, pivot_value) = partition::partition_sliding_midpoint(nodes, bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());

        let node_id = self.add_node(nodes[splitting_index], bounds.get_widest_dim(), bounds.get_midvalue_of_widest_dim());
        let nodes_len = nodes.len();

        if splitting_index > 0 {
            let left_rect = bounds.clone_moving_max(pivot_value, bounds.get_widest_dim());
            let left_child_id = self.build_tree(&mut nodes[0..splitting_index], &left_rect);
            self.nodes[node_id].left_node = Some(left_child_id);
        }

        if splitting_index < nodes.len() - 1 {
            let right_rect = bounds.clone_moving_min(pivot_value, bounds.get_widest_dim());

            let right_child_id = self.build_tree(&mut nodes[splitting_index + 1..nodes_len], &right_rect);
            self.nodes[node_id].right_node = Some(right_child_id);
        }

        node_id
    }
}

pub struct KdtreeNode<T> {
    left_node: Option<usize>,
    right_node: Option<usize>,

    point: T,
    dimension: usize,
    split_on: f64
}

impl<T: KdtreePointTrait> KdtreeNode<T> {
    fn new(p: T, splitting_dimension: usize, split_on_value : f64) -> KdtreeNode<T> {
        KdtreeNode {
            left_node: None,
            right_node: None,

            point: p,
            dimension : splitting_dimension,
            split_on : split_on_value
        }
    }
}

#[cfg(test)]
mod tests {
    use ::kdtree::test_common::tests_utils::Point2WithId;

    use super::*;

    #[test]
    #[should_panic(expected = "empty vector point not allowed")]
    fn should_panic_given_empty_vector() {
        let empty_vec: Vec<Point2WithId> = vec![];

        Kdtree::new(empty_vec);
    }

    quickcheck! {
        fn tree_build_creates_tree_with_as_many_leafs_as_there_is_points(xs : Vec<f64>) -> bool {
            if(xs.len() == 0) {
                return true;
            }
            let mut vec : Vec<Point2WithId> = vec![];
            for i in 0 .. xs.len() {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);

                vec.push(p);
            }

            let tree = Kdtree::new(qc_value_vec_to_2d_points_vec(&xs));

            let mut to_iterate : Vec<usize> = vec![];
            to_iterate.push(0);

            while to_iterate.len() > 0 {
                let last_index = to_iterate.last().unwrap().clone();
                let ref x = tree.nodes.get(last_index).unwrap();
                to_iterate.pop();
                if x.left_node.is_some() {
                    to_iterate.push(x.left_node.unwrap());
                }
                if x.right_node.is_some() {
                    to_iterate.push(x.right_node.unwrap());
                }
            }
            xs.len() == tree.nodes.len()
        }
    }

    quickcheck! {
        fn nearest_neighbor_search_using_qc(xs : Vec<f64>) -> bool {
            if(xs.len() == 0) {
                return true;
            }

            let point_vec = qc_value_vec_to_2d_points_vec(&xs);
            let tree = Kdtree::new(point_vec.clone());

            for p in &point_vec {
                let found_nn = tree.nearest_search(p);

                assert_eq!(p.id,found_nn.id);
            }

            true
        }
    }

    fn qc_value_vec_to_2d_points_vec(xs : &Vec<f64>) -> Vec<Point2WithId> {
        let mut vec : Vec<Point2WithId> = vec![];
        for i in 0 .. xs.len() {
            let mut is_duplicated_value = false;
            for j in 0 .. i {
                if xs[i] == xs[j] {
                    is_duplicated_value = true;
                    break;
                }
            }
            if !is_duplicated_value {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);
                vec.push(p);
            }
        }

        vec
    }
}<|endoftext|>


<commit_before>use l2::ast::*;
use l2::stack::*;

use std::collections::HashMap;
use std::ops::{Index, IndexMut};
use std::vec::Vec;

#[derive(Clone, Copy)]
enum MachineMode {
    Read,
    Write
}

struct MachineState {
    h: usize,
    s: usize,
    p: CodePtr,
    cp: CodePtr,
    fail: bool,
    heap: Heap,
    mode: MachineMode,
    stack: Stack,
    registers: Registers
}

type CodeDir = HashMap<(Atom, usize), usize>;

pub struct Machine {
    ms: MachineState,
    code: Code,
    code_dir: CodeDir
}

impl Index<Addr> for MachineState {
    type Output = HeapCellValue;

    fn index(&self, index: Addr) -> &Self::Output {
        match index {
            Addr::HeapCell(hc)  => &self.heap[hc],
            Addr::RegNum(reg)   => &self.registers[reg],
            Addr::StackCell(sc) => &self.stack[sc]
        }
    }
}

impl IndexMut<Addr> for MachineState {
    fn index_mut(&mut self, index: Addr) -> &mut Self::Output {
        match index {
            Addr::HeapCell(hc)  => &mut self.heap[hc],
            Addr::RegNum(reg)   => &mut self.registers[reg],
            Addr::StackCell(sc) => &mut self.stack[sc]
        }
    }
}

impl Machine {
    pub fn new() -> Self {
        Machine {
            ms: MachineState::new(),
            code: Vec::new(),
            code_dir: HashMap::new()
        }
    }

    fn failed(&self) -> bool {
        self.ms.fail
    }

    pub fn add_fact(&mut self, fact: &Term, mut code: Code) {
        let p = self.code.len();
        let name  = fact.name().clone();
        let arity = fact.arity();

        self.code.append(&mut code);
        self.code_dir.insert((name, arity), p);
    }

    pub fn add_rule(&mut self, rule: &Rule, mut code: Code) {
        let p = self.code.len();
        let name  = rule.head.0.name().clone();
        let arity = rule.head.0.arity();

        self.code.append(&mut code);
        self.code_dir.insert((name, arity), p);
    }

    fn execute_instr(&mut self, instr: &Line) -> bool {
        let mut instr = instr;

        loop {
            match instr {
                &Line::Fact(ref fact) => {
                    for fact_instr in fact {
                        self.ms.execute_fact_instr(&fact_instr);
                    }
                    self.ms.p += 1;
                },
                &Line::Query(ref query) => {
                    for query_instr in query {
                        self.ms.execute_query_instr(&query_instr);
                    }
                    self.ms.p += 1;
                },
                &Line::Control(ref control_instr) =>
                    self.ms.execute_ctrl_instr(&self.code_dir, control_instr),
            }

            if self.failed() {
                return false;
            }

            match self.ms.p {
                CodePtr::DirEntry(p) if p < self.code.len() =>
                    instr = &self.code[p],
                _ => break
            }
        }

        true
    }

    pub fn execute_query(&mut self, query: Code) -> bool {
        let mut succeeded = true;

        for instr in query {
            succeeded = self.execute_instr(&instr);
            if !succeeded {
                break;
            }
        }

        self.ms.reset();
        succeeded
    }
}

impl MachineState {
    fn new() -> MachineState {
        MachineState { h: 0,
                       s: 0,
                       p: CodePtr::TopLevel,
                       cp: CodePtr::TopLevel,
                       fail: false,
                       heap: Vec::with_capacity(256),
                       mode: MachineMode::Write,
                       stack: Stack::new(),
                       registers: vec![HeapCellValue::Ref(0); 32] }
    }

    fn lookup(&self, a: Addr) -> &HeapCellValue {
        match a {
            Addr::HeapCell(hc)  => &self.heap[hc],
            Addr::RegNum(reg)   => &self.registers[reg],
            Addr::StackCell(sc) => &self.stack[sc]
        }
    }

    fn deref(&self, a: Addr) -> Addr {
        let mut a = a;

        loop {
            if let &HeapCellValue::Ref(value) = self.lookup(a) {
                if let Addr::HeapCell(av) = a {
                    if value != av {
                        a = Addr::HeapCell(value);
                        continue;
                    }
                } else {
                    a = Addr::HeapCell(value);
                    continue;
                }
            }

            return a;
        };
    }

    fn is_unbound(hc: &HeapCellValue, index: usize) -> bool {
        match hc {
            &HeapCellValue::Ref(r) => r == index,
            _ => false
        }
    }

    fn bind(&mut self, a: Addr, val: usize) {
        let mut a = a;

        loop {
            match a {
                addr @ Addr::RegNum(_) | addr @ Addr::StackCell(_) => {
                    if let HeapCellValue::Ref(hc) = self[addr] {
                        a = Addr::HeapCell(hc);
                    } else if Self::is_unbound(&self.heap[val], val) {
                        self.heap[val] = self[addr].clone();
                        break;
                    } else {
                        self.fail = true;
                        break;
                    }
                },
                Addr::HeapCell(hc) => {
                    if Self::is_unbound(&self.heap[hc], hc) {
                        self.heap[hc] = HeapCellValue::Ref(val);
                        break;
                    } else if Self::is_unbound(&self.heap[val], val) {
                        self.heap[val] = HeapCellValue::Ref(hc);
                        break;
                    } else {
                        self.fail = true;
                        break;
                    }
                }
            };
        }
    }

    fn unify(&mut self, a1: Addr, a2: Addr) {
        let mut pdl = vec![a1, a2];

        self.fail = false;

        while !(pdl.is_empty() || self.fail) {
            let d1 = self.deref(pdl.pop().unwrap());
            let d2 = self.deref(pdl.pop().unwrap());

            if d1 != d2 {
                match (self.lookup(d1), self.lookup(d2)) {
                    (&HeapCellValue::Ref(hc), _) =>
                        self.bind(d2, hc),
                    (_, &HeapCellValue::Ref(hc)) =>
                        self.bind(d1, hc),
                    (&HeapCellValue::Str(a1), &HeapCellValue::Str(a2)) => {
                        let r1 = &self.heap[a1];
                        let r2 = &self.heap[a2];

                        if let &HeapCellValue::NamedStr(n1, ref f1) = r1 {
                            if let &HeapCellValue::NamedStr(n2, ref f2) = r2 {
                                if n1 == n2 && *f1 == *f2 {
                                    for i in 1 .. n1 {
                                        pdl.push(Addr::HeapCell(a1 + i));
                                        pdl.push(Addr::HeapCell(a2 + i));
                                    }

                                    continue;
                                }
                            }
                        }

                        self.fail = true;
                    },
                    _ => self.fail = true,
                };
            }
        }
    }

    fn execute_query_instr(&mut self, instr: &QueryInstruction) {
        match instr {
            &QueryInstruction::PutStructure(_, ref name, arity, reg) => {
                self.heap.push(HeapCellValue::Str(self.h + 1));
                self.heap.push(HeapCellValue::NamedStr(arity, name.clone()));

                self[Addr::from(reg)] = self.heap[self.h].clone();

                self.h += 2;
            },
            &QueryInstruction::PutValue(norm, arg) =>
                self.registers[arg] = self[Addr::from(norm)].clone(),
            &QueryInstruction::PutVariable(norm, arg) => {
                self.heap.push(HeapCellValue::Ref(self.h));

                self[Addr::from(norm)] = self.heap[self.h].clone();
                self.registers[arg] = self.heap[self.h].clone();

                self.h += 1;
            },
            &QueryInstruction::SetVariable(reg) => {
                self.heap.push(HeapCellValue::Ref(self.h));
                self[Addr::from(reg)] = self.heap[self.h].clone();

                self.h += 1;
            },
            &QueryInstruction::SetValue(reg) => {
                let heap_val = self[Addr::from(reg)].clone();
                self.heap.push(heap_val);
                self.h += 1;
            },
        }
    }

    fn execute_fact_instr(&mut self, instr: &FactInstruction) {
        match instr {
            &FactInstruction::GetStructure(_, ref name, arity, reg) => {
                let addr = self.deref(Addr::from(reg));

                match self.lookup(addr) {
                    &HeapCellValue::Str(a) => {
                        let result = &self.heap[a];

                        if let &HeapCellValue::NamedStr(narity, ref str) = result {
                            if narity == arity && *name == *str {
                                self.s = a + 1;
                                self.mode = MachineMode::Read;
                            } else {
                                self.fail = true;
                            }
                        }
                    },
                    &HeapCellValue::Ref(r) => {
                        self.heap.push(HeapCellValue::Str(self.h + 1));
                        self.heap.push(HeapCellValue::NamedStr(arity, name.clone()));

                        let h = self.h;

                        self.bind(Addr::HeapCell(r), h);

                        self.h += 2;
                        self.mode = MachineMode::Write;
                    },
                    _ => self.fail = true,
                };
            },
            &FactInstruction::GetVariable(norm, arg) =>
                self[Addr::from(norm)] = self.registers[arg].clone(),
            &FactInstruction::GetValue(norm, arg) =>
                self.unify(Addr::from(norm), Addr::RegNum(arg)),
            &FactInstruction::UnifyVariable(reg) => {
                match self.mode {
                    MachineMode::Read  =>
                        self[Addr::from(reg)] = self.heap[self.s].clone(),
                    MachineMode::Write => {
                        self.heap.push(HeapCellValue::Ref(self.h));

                        self[Addr::from(reg)] = self.heap[self.h].clone();
                        self.h += 1;
                    }
                };

                self.s += 1;
            },
            &FactInstruction::UnifyValue(reg) => {
                let s = self.s;

                match self.mode {
                    MachineMode::Read  =>
                        self.unify(Addr::from(reg), Addr::HeapCell(s)),
                    MachineMode::Write => {
                        let heap_val = self[Addr::from(reg)].clone();
                        self.heap.push(heap_val);
                        self.h += 1;
                    }
                };

                self.s += 1;
            }
        }
    }

    fn execute_ctrl_instr(&mut self, code_dir: &CodeDir, instr: &ControlInstruction)
    {
        match instr {
            &ControlInstruction::Allocate(num_cells) => {
                self.stack.push(self.cp, num_cells);
                self.p += 1;
            },
            &ControlInstruction::Call(ref name, arity) => {
                let compiled_tl_index = code_dir.get(&(name.clone(), arity))
                                                .map(|index| *index);

                match compiled_tl_index {
                    Some(compiled_tl_index) => {
                        self.cp = self.p + 1;
                        self.p  = CodePtr::DirEntry(compiled_tl_index);
                    },
                    None => self.fail = true
                };
            },
            &ControlInstruction::Deallocate => {
                self.p = self.stack.get_cp();
                self.stack.pop();
            },
            &ControlInstruction::Proceed => {
                self.p = self.cp;
            }
        };
    }

    fn reset(&mut self) {
        self.h = 0;
        self.s = 0;
        self.p = CodePtr::TopLevel;
        self.cp = CodePtr::TopLevel;

        self.fail = false;
        self.heap.clear();
        self.mode = MachineMode::Write;
        self.stack = Stack::new();
        self.registers = vec![HeapCellValue::Ref(0); 32];
    }
}
<commit_msg>removed unnecessary lookup function<commit_after>use l2::ast::*;
use l2::stack::*;

use std::collections::HashMap;
use std::ops::{Index, IndexMut};
use std::vec::Vec;

#[derive(Clone, Copy)]
enum MachineMode {
    Read,
    Write
}

struct MachineState {
    h: usize,
    s: usize,
    p: CodePtr,
    cp: CodePtr,
    fail: bool,
    heap: Heap,
    mode: MachineMode,
    stack: Stack,
    registers: Registers
}

type CodeDir = HashMap<(Atom, usize), usize>;

pub struct Machine {
    ms: MachineState,
    code: Code,
    code_dir: CodeDir
}

impl Index<Addr> for MachineState {
    type Output = HeapCellValue;

    fn index(&self, index: Addr) -> &Self::Output {
        match index {
            Addr::HeapCell(hc)  => &self.heap[hc],
            Addr::RegNum(reg)   => &self.registers[reg],
            Addr::StackCell(sc) => &self.stack[sc]
        }
    }
}

impl IndexMut<Addr> for MachineState {
    fn index_mut(&mut self, index: Addr) -> &mut Self::Output {
        match index {
            Addr::HeapCell(hc)  => &mut self.heap[hc],
            Addr::RegNum(reg)   => &mut self.registers[reg],
            Addr::StackCell(sc) => &mut self.stack[sc]
        }
    }
}

impl Machine {
    pub fn new() -> Self {
        Machine {
            ms: MachineState::new(),
            code: Vec::new(),
            code_dir: HashMap::new()
        }
    }

    fn failed(&self) -> bool {
        self.ms.fail
    }

    pub fn add_fact(&mut self, fact: &Term, mut code: Code) {
        let p = self.code.len();
        let name  = fact.name().clone();
        let arity = fact.arity();

        self.code.append(&mut code);
        self.code_dir.insert((name, arity), p);
    }

    pub fn add_rule(&mut self, rule: &Rule, mut code: Code) {
        let p = self.code.len();
        let name  = rule.head.0.name().clone();
        let arity = rule.head.0.arity();

        self.code.append(&mut code);
        self.code_dir.insert((name, arity), p);
    }

    fn execute_instr(&mut self, instr: &Line) -> bool {
        let mut instr = instr;

        loop {
            match instr {
                &Line::Fact(ref fact) => {
                    for fact_instr in fact {
                        self.ms.execute_fact_instr(&fact_instr);
                    }
                    self.ms.p += 1;
                },
                &Line::Query(ref query) => {
                    for query_instr in query {
                        self.ms.execute_query_instr(&query_instr);
                    }
                    self.ms.p += 1;
                },
                &Line::Control(ref control_instr) =>
                    self.ms.execute_ctrl_instr(&self.code_dir, control_instr),
            }

            if self.failed() {
                return false;
            }

            match self.ms.p {
                CodePtr::DirEntry(p) if p < self.code.len() =>
                    instr = &self.code[p],
                _ => break
            }
        }

        true
    }

    pub fn execute_query(&mut self, query: Code) -> bool {
        let mut succeeded = true;

        for instr in query {
            succeeded = self.execute_instr(&instr);
            if !succeeded {
                break;
            }
        }

        self.ms.reset();
        succeeded
    }
}

impl MachineState {
    fn new() -> MachineState {
        MachineState { h: 0,
                       s: 0,
                       p: CodePtr::TopLevel,
                       cp: CodePtr::TopLevel,
                       fail: false,
                       heap: Vec::with_capacity(256),
                       mode: MachineMode::Write,
                       stack: Stack::new(),
                       registers: vec![HeapCellValue::Ref(0); 32] }
    }

    fn deref(&self, a: Addr) -> Addr {
        let mut a = a;

        loop {
            if let &HeapCellValue::Ref(value) = &self[a] {
                if let Addr::HeapCell(av) = a {
                    if value != av {
                        a = Addr::HeapCell(value);
                        continue;
                    }
                } else {
                    a = Addr::HeapCell(value);
                    continue;
                }
            }

            return a;
        };
    }

    fn is_unbound(hc: &HeapCellValue, index: usize) -> bool {
        match hc {
            &HeapCellValue::Ref(r) => r == index,
            _ => false
        }
    }

    fn bind(&mut self, a: Addr, val: usize) {
        let mut a = a;

        loop {
            match a {
                addr @ Addr::RegNum(_) | addr @ Addr::StackCell(_) => {
                    if let HeapCellValue::Ref(hc) = self[addr] {
                        a = Addr::HeapCell(hc);
                    } else if Self::is_unbound(&self.heap[val], val) {
                        self.heap[val] = self[addr].clone();
                        break;
                    } else {
                        self.fail = true;
                        break;
                    }
                },
                Addr::HeapCell(hc) => {
                    if Self::is_unbound(&self.heap[hc], hc) {
                        self.heap[hc] = HeapCellValue::Ref(val);
                        break;
                    } else if Self::is_unbound(&self.heap[val], val) {
                        self.heap[val] = HeapCellValue::Ref(hc);
                        break;
                    } else {
                        self.fail = true;
                        break;
                    }
                }
            };
        }
    }

    fn unify(&mut self, a1: Addr, a2: Addr) {
        let mut pdl = vec![a1, a2];

        self.fail = false;

        while !(pdl.is_empty() || self.fail) {
            let d1 = self.deref(pdl.pop().unwrap());
            let d2 = self.deref(pdl.pop().unwrap());

            if d1 != d2 {
                match (&self[d1], &self[d2]) {
                    (&HeapCellValue::Ref(hc), _) =>
                        self.bind(d2, hc),
                    (_, &HeapCellValue::Ref(hc)) =>
                        self.bind(d1, hc),
                    (&HeapCellValue::Str(a1), &HeapCellValue::Str(a2)) => {
                        let r1 = &self.heap[a1];
                        let r2 = &self.heap[a2];

                        if let &HeapCellValue::NamedStr(n1, ref f1) = r1 {
                            if let &HeapCellValue::NamedStr(n2, ref f2) = r2 {
                                if n1 == n2 && *f1 == *f2 {
                                    for i in 1 .. n1 {
                                        pdl.push(Addr::HeapCell(a1 + i));
                                        pdl.push(Addr::HeapCell(a2 + i));
                                    }

                                    continue;
                                }
                            }
                        }

                        self.fail = true;
                    },
                    _ => self.fail = true,
                };
            }
        }
    }

    fn execute_query_instr(&mut self, instr: &QueryInstruction) {
        match instr {
            &QueryInstruction::PutStructure(_, ref name, arity, reg) => {
                self.heap.push(HeapCellValue::Str(self.h + 1));
                self.heap.push(HeapCellValue::NamedStr(arity, name.clone()));

                self[Addr::from(reg)] = self.heap[self.h].clone();

                self.h += 2;
            },
            &QueryInstruction::PutValue(norm, arg) =>
                self.registers[arg] = self[Addr::from(norm)].clone(),
            &QueryInstruction::PutVariable(norm, arg) => {
                self.heap.push(HeapCellValue::Ref(self.h));

                self[Addr::from(norm)] = self.heap[self.h].clone();
                self.registers[arg] = self.heap[self.h].clone();

                self.h += 1;
            },
            &QueryInstruction::SetVariable(reg) => {
                self.heap.push(HeapCellValue::Ref(self.h));
                self[Addr::from(reg)] = self.heap[self.h].clone();

                self.h += 1;
            },
            &QueryInstruction::SetValue(reg) => {
                let heap_val = self[Addr::from(reg)].clone();
                self.heap.push(heap_val);
                self.h += 1;
            },
        }
    }

    fn execute_fact_instr(&mut self, instr: &FactInstruction) {
        match instr {
            &FactInstruction::GetStructure(_, ref name, arity, reg) => {
                let addr = self.deref(Addr::from(reg));

                match &self[addr] {
                    &HeapCellValue::Str(a) => {
                        let result = &self.heap[a];

                        if let &HeapCellValue::NamedStr(narity, ref str) = result {
                            if narity == arity && *name == *str {
                                self.s = a + 1;
                                self.mode = MachineMode::Read;
                            } else {
                                self.fail = true;
                            }
                        }
                    },
                    &HeapCellValue::Ref(r) => {
                        self.heap.push(HeapCellValue::Str(self.h + 1));
                        self.heap.push(HeapCellValue::NamedStr(arity, name.clone()));

                        let h = self.h;

                        self.bind(Addr::HeapCell(r), h);

                        self.h += 2;
                        self.mode = MachineMode::Write;
                    },
                    _ => self.fail = true,
                };
            },
            &FactInstruction::GetVariable(norm, arg) =>
                self[Addr::from(norm)] = self.registers[arg].clone(),
            &FactInstruction::GetValue(norm, arg) =>
                self.unify(Addr::from(norm), Addr::RegNum(arg)),
            &FactInstruction::UnifyVariable(reg) => {
                match self.mode {
                    MachineMode::Read  =>
                        self[Addr::from(reg)] = self.heap[self.s].clone(),
                    MachineMode::Write => {
                        self.heap.push(HeapCellValue::Ref(self.h));

                        self[Addr::from(reg)] = self.heap[self.h].clone();
                        self.h += 1;
                    }
                };

                self.s += 1;
            },
            &FactInstruction::UnifyValue(reg) => {
                let s = self.s;

                match self.mode {
                    MachineMode::Read  =>
                        self.unify(Addr::from(reg), Addr::HeapCell(s)),
                    MachineMode::Write => {
                        let heap_val = self[Addr::from(reg)].clone();
                        self.heap.push(heap_val);
                        self.h += 1;
                    }
                };

                self.s += 1;
            }
        }
    }

    fn execute_ctrl_instr(&mut self, code_dir: &CodeDir, instr: &ControlInstruction)
    {
        match instr {
            &ControlInstruction::Allocate(num_cells) => {
                self.stack.push(self.cp, num_cells);
                self.p += 1;
            },
            &ControlInstruction::Call(ref name, arity) => {
                let compiled_tl_index = code_dir.get(&(name.clone(), arity))
                                                .map(|index| *index);

                match compiled_tl_index {
                    Some(compiled_tl_index) => {
                        self.cp = self.p + 1;
                        self.p  = CodePtr::DirEntry(compiled_tl_index);
                    },
                    None => self.fail = true
                };
            },
            &ControlInstruction::Deallocate => {
                self.p = self.stack.get_cp();
                self.stack.pop();
            },
            &ControlInstruction::Proceed => {
                self.p = self.cp;
            }
        };
    }

    fn reset(&mut self) {
        self.h = 0;
        self.s = 0;
        self.p = CodePtr::TopLevel;
        self.cp = CodePtr::TopLevel;

        self.fail = false;
        self.heap.clear();
        self.mode = MachineMode::Write;
        self.stack = Stack::new();
        self.registers = vec![HeapCellValue::Ref(0); 32];
    }
}
<|endoftext|>


<commit_before><commit_msg>FIXUP Correct definitions to match musl<commit_after><|endoftext|>


<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use error::{Result,Error, Source};

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = try!(File::open(path).map_err(|_| Error::file_not_found(&self.source,
                                                                                "markdown file",
                                                                                format!("{}", path.display()))));
        let mut s = String::new();

        try!(f.read_to_string(&mut s).map_err(|_| Error::parser(&self.source,
                                                                    format!("file {} contains invalid UTF-8, could not parse it", path.display()))));
        self.parse(&s)
    }

    /// Parse a string and returns an AST, or an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec!();
        try!(self.parse_events(&mut p, &mut res, None));

        try!(self.parse_footnotes(&mut res));

        find_standalone(&mut res);
        Ok(res)
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) ->Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                     format!("footnote reference {} does not have a matching definition", &reference)));
                    }
                },
                Token::Paragraph(ref mut vec) | Token::Header(_, ref mut vec) | Token::Emphasis(ref mut vec)
                    | Token::Strong(ref mut vec) | Token::Code(ref mut vec) | Token::BlockQuote(ref mut vec)
                    | Token::CodeBlock(_, ref mut vec) | Token::List(ref mut vec) | Token::OrderedList(_, ref mut vec)
                    | Token::Item(ref mut vec) | Token::Table(_, ref mut vec) | Token::TableHead(ref mut vec)
                    | Token::TableRow(ref mut vec) | Token::TableCell(ref mut vec) | Token::Link(_, _, ref mut vec)
                    | Token::Image(_, _, ref mut vec) => try!(self.parse_footnotes(vec)),
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, current_tag: Option<&Tag>) -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) | Event::InlineHtml(text) | Event::Text(text) => {
                    if let Some(&Token::Str(_)) = v.last() {
                        if let Token::Str(ref mut s) = *v.last_mut().unwrap() {
                            s.push_str(text.as_ref());
                        } else {
                            unreachable!();
                        }
                    } else {
                        v.push(Token::Str(text.into_owned()));
                    }
                },
                Event::Start(tag) => try!(self.parse_tag(p, v, tag)),
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ {:?}",
                                          tag,
                                          current_tag));
                    break;
                },
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => v.push(Token::Footnote(vec!(Token::Str(text.into_owned())))),
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, tag: Tag<'a>) -> Result<()> {
        let mut res = vec!();

        try!(self.parse_events(p, &mut res, Some(&tag)));

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url.into_owned(), title.into_owned(), res),
            Tag::Image(url, title) => Token::Image(url.into_owned(), title.into_owned(), res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }},
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language.into_owned(), res),
            Tag::Table(n) => Token::Table(n, res),
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    println!("Warning: footnote definition for {} but previous definition already exist, overriding it", reference);
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            },
        };
        v.push(token);
        Ok(())
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0], Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    continue;
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<commit_msg>Fix formatting<commit_after>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use error::{Result,Error, Source};

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = try!(File::open(path).map_err(|_| Error::file_not_found(&self.source,
                                                                                "markdown file",
                                                                                format!("{}", path.display()))));
        let mut s = String::new();

        try!(f.read_to_string(&mut s).map_err(|_| Error::parser(&self.source,
                                                                    format!("file {} contains invalid UTF-8, could not parse it", path.display()))));
        self.parse(&s)
    }

    /// Parse a string and returns an AST, or an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec!();
        try!(self.parse_events(&mut p, &mut res, None));

        try!(self.parse_footnotes(&mut res));

        find_standalone(&mut res);
        Ok(res)
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) ->Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 format!("footnote reference {} does not have a matching definition", &reference)));
                    }
                },
                Token::Paragraph(ref mut vec) | Token::Header(_, ref mut vec) | Token::Emphasis(ref mut vec)
                    | Token::Strong(ref mut vec) | Token::Code(ref mut vec) | Token::BlockQuote(ref mut vec)
                    | Token::CodeBlock(_, ref mut vec) | Token::List(ref mut vec) | Token::OrderedList(_, ref mut vec)
                    | Token::Item(ref mut vec) | Token::Table(_, ref mut vec) | Token::TableHead(ref mut vec)
                    | Token::TableRow(ref mut vec) | Token::TableCell(ref mut vec) | Token::Link(_, _, ref mut vec)
                    | Token::Image(_, _, ref mut vec) => try!(self.parse_footnotes(vec)),
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, current_tag: Option<&Tag>) -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) | Event::InlineHtml(text) | Event::Text(text) => {
                    if let Some(&Token::Str(_)) = v.last() {
                        if let Token::Str(ref mut s) = *v.last_mut().unwrap() {
                            s.push_str(text.as_ref());
                        } else {
                            unreachable!();
                        }
                    } else {
                        v.push(Token::Str(text.into_owned()));
                    }
                },
                Event::Start(tag) => try!(self.parse_tag(p, v, tag)),
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ {:?}",
                                          tag,
                                          current_tag));
                    break;
                },
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => v.push(Token::Footnote(vec!(Token::Str(text.into_owned())))),
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, tag: Tag<'a>) -> Result<()> {
        let mut res = vec!();

        try!(self.parse_events(p, &mut res, Some(&tag)));

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url.into_owned(), title.into_owned(), res),
            Tag::Image(url, title) => Token::Image(url.into_owned(), title.into_owned(), res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }},
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language.into_owned(), res),
            Tag::Table(n) => Token::Table(n, res),
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    println!("Warning: footnote definition for {} but previous definition already exist, overriding it", reference);
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            },
        };
        v.push(token);
        Ok(())
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0], Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    continue;
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<|endoftext|>


<commit_before>//! Implements Dominance frontier and dominance tree computation.
//!
//! Note: This is completely unstable and maybe merged with someother module in the future.

#![allow(dead_code, unused_variables, unused_imports, unused_mut)]

use std::collections::{BTreeSet, BTreeMap};
use petgraph::graph::{Graph, NodeIndex, EdgeIndex};
use petgraph::EdgeDirection;
use petgraph::{Dfs};

// Design ideas:
//  - We can compute a dominator tree and further the dominance frontier by using the algorithms
//    outlined in:
//        * http://www.cs.rice.edu/~keith/Embed/dom.pdf
//
//  - We can use the same graph as the CFG and overlay the dominator tree information by indicating
//  edges that belong to the dominator tree differently.

pub type DomTree = Graph<NodeIndex, u64>;

#[derive(Clone, Debug)]
pub struct DomResult {
    pub g:    DomTree,
    pub doms: BTreeMap<NodeIndex, BTreeSet<NodeIndex>>,
}

impl DomResult {
    fn new() -> DomResult {
        DomResult {
            g: DomTree::new(),
            doms: BTreeMap::new(),
        }
    }
}

pub fn build_dom_tree(g: &Graph<NodeIndex, u64>) -> DomResult {
    let mut res: DomResult = DomResult::new();

    {
        let mut d = &mut (res.g);
        let mut doms = &mut (res.doms);

        // Make a set of all nodes except the start.
        let mut g_iter = g.raw_nodes().iter();
        let mut tmp_iter;
        let start_n = g_iter.next().unwrap().weight.clone();

        tmp_iter = g_iter.clone();
        let mut node_set = BTreeSet::new();

        while let Some(ref node) = tmp_iter.next() {
            node_set.insert(node.weight.clone());
        }

        d.add_node(start_n.clone());

        {
            let mut tmp_set = BTreeSet::new();
            tmp_set.insert(start_n.clone());
            doms.insert(start_n.clone(), tmp_set);
        }

        // Add all the nodes to the DomTree.
        tmp_iter = g_iter.clone();
        while let Some(ref node) = tmp_iter.next() {
            d.add_node(node.weight.clone());
            doms.insert(node.weight.clone(), node_set.clone());
        }

        let mut change = true;

        while change {
            change = false;
            while let Some(ref node) = g_iter.next() {
                let mut preds = g.neighbors_directed((node.clone()).weight, EdgeDirection::Incoming);
                let mut tmp_set = BTreeSet::new();

                if let Some(ref n) = preds.next() {
                    tmp_set = doms.get(&(g.node_weight(*n).unwrap())).unwrap().clone();
                }

                while let Some(ref n) = preds.next() {
                    let other_set = doms.get(&(g.node_weight(*n).unwrap())).unwrap();
                    let intersect: Vec<NodeIndex> = tmp_set.intersection(other_set).cloned().collect();
                    tmp_set.clear();
                    for e in intersect.iter(){
                        tmp_set.insert(e.clone());
                    }
                }

                tmp_set.insert(node.weight);

                let set = doms.get_mut(&(node.weight)).unwrap();
                let tmp: Vec<NodeIndex> = tmp_set.difference(set).cloned().collect();

                if !(tmp.is_empty()) {
                    change = true;
                    *set = tmp_set.clone();
                }
            }
        }

        for (key, value) in doms.iter() {
            for v in value.iter() {
                d.add_edge(*key, *v, 0);
            }
        }
    }

    res
}

#[cfg(test)]
mod test {
    use super::*;
    use petgraph::graph::{Graph, NodeIndex, EdgeIndex};
    use petgraph::{Dfs};

    #[test]
    fn dummy() {
        let mut g = Graph::<NodeIndex, u64>::new();
        let n0 = g.add_node(NodeIndex::new(0));
        let n1 = g.add_node(NodeIndex::new(1));
        g.add_edge(n0, n1, 0);

        build_dom_tree(&mut g);
    }
}
<commit_msg>[WIP] DomTree. Basic + Dot generation complete<commit_after>//! Implements Dominance frontier and dominance tree computation.
//!
//! Note: This is completely unstable and maybe merged with someother module in the future.

#![allow(dead_code, unused_variables, unused_imports, unused_mut)]

use std::collections::{BTreeSet, BTreeMap};
use petgraph::graph::{Graph, NodeIndex, EdgeIndex, Edge};
use petgraph::EdgeDirection;
use petgraph::{Dfs};

use super::dot::{GraphDot, EdgeInfo, Label};

// Design ideas:
//  - We can compute a dominator tree and further the dominance frontier by using the algorithms
//    outlined in:
//        * http://www.cs.rice.edu/~keith/Embed/dom.pdf
//
//  - We can use the same graph as the CFG and overlay the dominator tree information by indicating
//  edges that belong to the dominator tree differently.

pub type DomTree = Graph<NodeIndex, u64>;

#[derive(Clone, Debug)]
pub struct DomResult {
    pub g:    DomTree,
    pub doms: BTreeMap<NodeIndex, BTreeSet<NodeIndex>>,
}

impl DomResult {
    fn new() -> DomResult {
        DomResult {
            g: DomTree::new(),
            doms: BTreeMap::new(),
        }
    }
}

pub fn build_dom_tree(g: &Graph<NodeIndex, u64>) -> DomResult {
    let mut res: DomResult = DomResult::new();

    {
        let mut d = &mut (res.g);
        let mut doms = &mut (res.doms);

        // Make a set of all nodes except the start.
        let mut g_iter = g.raw_nodes().iter();
        let mut tmp_iter;
        let start_n = g_iter.next().unwrap().weight.clone();

        tmp_iter = g_iter.clone();
        let mut node_set = BTreeSet::new();

        while let Some(ref node) = tmp_iter.next() {
            node_set.insert(node.weight.clone());
        }

        d.add_node(start_n.clone());

        {
            let mut tmp_set = BTreeSet::new();
            tmp_set.insert(start_n.clone());
            doms.insert(start_n.clone(), tmp_set);
        }

        // Add all the nodes to the DomTree.
        tmp_iter = g_iter.clone();
        while let Some(ref node) = tmp_iter.next() {
            d.add_node(node.weight.clone());
            doms.insert(node.weight.clone(), node_set.clone());
        }

        let mut change = true;

        while change {
            change = false;
            while let Some(ref node) = g_iter.next() {
                let mut preds = g.neighbors_directed((node.clone()).weight, EdgeDirection::Incoming);
                let mut tmp_set = BTreeSet::new();

                if let Some(ref n) = preds.next() {
                    tmp_set = doms.get(&(g.node_weight(*n).unwrap())).unwrap().clone();
                }

                while let Some(ref n) = preds.next() {
                    let other_set = doms.get(&(g.node_weight(*n).unwrap())).unwrap();
                    let intersect: Vec<NodeIndex> = tmp_set.intersection(other_set).cloned().collect();
                    tmp_set.clear();
                    for e in intersect.iter(){
                        tmp_set.insert(e.clone());
                    }
                }

                tmp_set.insert(node.weight);

                let set = doms.get_mut(&(node.weight)).unwrap();
                let tmp: Vec<NodeIndex> = tmp_set.difference(set).cloned().collect();

                if !(tmp.is_empty()) {
                    change = true;
                    *set = tmp_set.clone();
                }
            }
        }

        for (key, value) in doms.iter() {
            let list: Vec<NodeIndex>  = g.neighbors_directed(key.clone(), EdgeDirection::Incoming).collect();
            for v in value.iter() {
                // If v is in the adjacency list of key, add an edge
                // (As we need only immediate dominators for the dom tree).
                if list.contains(v) {
                    d.add_edge(*v, *key, 0);
                }
            }
        }
    }

    res
}

///////////////////////////////////////////////////////////////////////////////
//// Implementation of Traits to emit dot for dom.
///////////////////////////////////////////////////////////////////////////////

impl GraphDot for DomResult {
    type NodeType = NodeIndex;
    type EdgeType = Edge<u64>;

    fn configure(&self) -> String {
        format!("digraph cfg {{\nsplines=\"true\";\n")
    }

    fn nodes(&self) -> Vec<Self::NodeType> {
        let res = self.g.raw_nodes().iter().map(|e| e.weight.clone()).collect();
        res
    }

    fn edges(&self) -> Vec<Self::EdgeType> {
        let res = self.g.raw_edges().to_vec();
        res
    }

    fn get_node(&self, n: usize) -> Option<&Self::NodeType> {
        self.g.node_weight(NodeIndex::new(n))
    }
}

impl EdgeInfo for Edge<u64> {
    fn source(&self) -> usize {
        self.source().index()
    }

    fn target(&self) -> usize {
        self.target().index()
    }
}

impl Label for Edge<u64> {
    fn label(&self) -> String {
        ";\n".to_string()
    }

    fn name(&self) -> Option<String> {
        None
    }
}

impl Label for NodeIndex {
    fn label(&self) -> String {
        let tmp = format!("n{}", self.index());
        format!("{} [label={}];\n", tmp, tmp)
    }

    fn name(&self) -> Option<String> {
        Some(format!("n{}", self.index()))
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use petgraph::graph::{Graph, NodeIndex, EdgeIndex};
    use petgraph::{Dfs};
    use super::super::dot;
    use std::io::prelude::*;
    use std::fs::File;

    #[test]
    fn dummy() {
        let mut g = Graph::<NodeIndex, u64>::new();
        let a = g.add_node(NodeIndex::new(0));
        let b = g.add_node(NodeIndex::new(1));
        let c = g.add_node(NodeIndex::new(2));
        let d = g.add_node(NodeIndex::new(3));
        let e = g.add_node(NodeIndex::new(4));
        let f = g.add_node(NodeIndex::new(5));
        let _g = g.add_node(NodeIndex::new(6));

        g.add_edge(a, b, 0);
        g.add_edge(a, _g, 0);
        g.add_edge(b, c, 0);
        g.add_edge(b, e, 0);
        g.add_edge(c, d, 0);
        g.add_edge(d, e, 0);
        g.add_edge(e, f, 0);
        g.add_edge(f, _g, 0);

        let dom = build_dom_tree(&mut g);
        let _dot = dot::emit_dot(&dom);
        let mut dot_file = File::create("dom.dot").ok().expect("Error. Cannot create file!\n");
        dot_file.write_all(_dot.as_bytes()).ok().expect("Error. Cannot write file!\n");

    }
}
<|endoftext|>