<commit_before>extern crate rocksdb;
extern crate rustc_serialize;

use std::collections::HashMap;

use self::rustc_serialize::json::{JsonEvent, Parser, StackElement};
// Needed for a trait in order to `put()` into a `rocksdb::WriteBatch`
use self::rocksdb::Writable;

use error::Error;
use key_builder::{KeyBuilder, SegmentType};
use records_capnp::payload;
use stems::Stems;

// Good example of using rustc_serialize: https://github.com/ajroetker/beautician/blob/master/src/lib.rs
// Callback based JSON streaming parser: https://github.com/gyscos/json-streamer.rs
// Another parser pased on rustc_serializ: https://github.com/isagalaev/ijson-rust/blob/master/src/test.rs#L11

#[derive(Debug, PartialEq)]
struct WordInfo {
    //offset in the text field where the stemmed text starts
    stemmed_offset: u64,

    // the suffix of the stemmed text. When applied over stemmed, the original
    // text is returned.
    suffix_text: String,

    // the start of the suffixText
    suffix_offset: u64,
}

type ArrayOffsets = Vec<u64>;

enum ObjectKeyTypes {
    /// _id field
    Id,
    /// Normal key
    Key(String),
    /// Reserved key starting with underscore
    Ignore,
    /// No key found
    NoKey,
}

#[derive(Debug)]
pub struct Shredder {
    keybuilder: KeyBuilder,
    path_array_offsets: ArrayOffsets,
    // Top-level fields prefixed with an underscore are ignored
    ignore_children: u64,
    doc_id: String,
}

impl Shredder {
    pub fn new() -> Shredder {
        Shredder{
            keybuilder: KeyBuilder::new(),
            path_array_offsets: Vec::new(),
            ignore_children: 0,
            doc_id: String::new(),
        }
    }

    fn add_entries(&mut self, text: &String, docseq: u64, batch: &rocksdb::WriteBatch) -> Result<(), Error> {
        let stems = Stems::new(text.as_str());
        let mut word_to_word_infos = HashMap::new();

        for stem in stems {
            let word_infos = word_to_word_infos.entry(stem.stemmed).or_insert(Vec::new());
            word_infos.push(WordInfo{
                stemmed_offset: stem.stemmed_offset as u64,
                suffix_text: stem.suffix.to_string(),
                suffix_offset: stem.suffix_offset as u64,
            });
        }
        for (stemmed, word_infos) in word_to_word_infos {
            let mut message = ::capnp::message::Builder::new_default();
            {
                let capn_payload = message.init_root::<payload::Builder>();
                let mut capn_wordinfos = capn_payload.init_wordinfos(word_infos.len() as u32);
                for (pos, word_info) in word_infos.iter().enumerate() {
                    let mut capn_wordinfo = capn_wordinfos.borrow().get(pos as u32);
                    capn_wordinfo.set_stemmed_offset(word_info.stemmed_offset);
                    capn_wordinfo.set_suffix_text(&word_info.suffix_text);
                    capn_wordinfo.set_suffix_offset(word_info.suffix_offset);
                }
            }
            self.keybuilder.push_word(&stemmed);
            self.keybuilder.push_doc_seq(docseq);
            self.keybuilder.push_array_path(&self.path_array_offsets);

            let mut bytes = Vec::new();
            ::capnp::serialize_packed::write_message(&mut bytes, &message).unwrap();
            try!(batch.put(&self.keybuilder.key().into_bytes(), &bytes));

            self.keybuilder.pop_array_path();
            self.keybuilder.pop_doc_seq();
            self.keybuilder.pop_word();
        }
        Ok(())
    }

    fn inc_top_array_offset(&mut self) {
        // we encounter a new element. if we are a child element of an array
        // increment the offset. If we aren't (we are the root value or a map
        // value) we don't increment
        if let Some(SegmentType::Array) = self.keybuilder.last_pushed_segment_type() {
            if let Some(last) = self.path_array_offsets.last_mut() {
                *last += 1;
            }
        }
    }

    // Extract key if it exists and indicates if it's a special type of key
    fn extract_key(&mut self, stack_element: Option<StackElement>) -> ObjectKeyTypes {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey {
            match stack_element {
                Some(StackElement::Key(key)) => {
                    if self.keybuilder.segments.len() == 1 && key.starts_with("_") {
                        if key == "_id" {
                            ObjectKeyTypes::Id
                        } else {
                            ObjectKeyTypes::Ignore
                        }
                    } else {
                        ObjectKeyTypes::Key(key.to_string())
                    }
                },
                _ => ObjectKeyTypes::NoKey,
            }
        } else {
            ObjectKeyTypes::NoKey
        }
    }

    // If we are inside an object we need to push the key to the key builder
    // Don't push them if they are reserved fields (starting with underscore)
    fn maybe_push_key(&mut self, stack_element: Option<StackElement>) -> Result<(), Error> {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey {
            if let Some(StackElement::Key(key)) = stack_element {
                if self.keybuilder.segments.len() == 1 && key.starts_with("_") {
                    if key == "_id" {
                        return Err(Error::Shred(
                            "Expected string for `_id` field, got another type".to_string()));
                    } else {
                        self.ignore_children = 1;
                    }
                } else {
                    // Pop the dummy object that makes ObjectEnd happy
                    // or the previous object key
                    self.keybuilder.pop_object_key();
                    self.keybuilder.push_object_key(key.to_string());
                }
            }
        }
        Ok(())
    }

   pub fn shred(&mut self, json: &str, docseq: u64, batch: &rocksdb::WriteBatch) -> Result<String, Error> {
        let mut parser = Parser::new(json.chars());
        let mut token = parser.next();

        loop {
            // Get the next token, so that in case of an `ObjectStart` the key is already
            // on the stack.
            match token.take() {
                Some(JsonEvent::ObjectStart) => {
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    }
                    else {
                        // Just push something to make `ObjectEnd` happy
                        self.keybuilder.push_object_key("".to_string());
                    }
                },
                Some(JsonEvent::ObjectEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {//if !self.keybuilder.segments.is_empty() {
                        self.keybuilder.pop_object_key();
                        self.inc_top_array_offset();
                    }
                },
                Some(JsonEvent::ArrayStart) => {
                    try!(self.maybe_push_key(parser.stack().top()));
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    } else {
                        self.keybuilder.push_array();
                        self.path_array_offsets.push(0);
                    }
                },
                Some(JsonEvent::ArrayEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {
                        self.path_array_offsets.pop();
                        self.keybuilder.pop_array();
                        self.inc_top_array_offset();
                    }
                },
                Some(JsonEvent::StringValue(value)) => {
                    // No children to ignore
                    if self.ignore_children == 0 {
                        println!("stringvalue: {:?}", value);
                        match self.extract_key(parser.stack().top()) {
                            ObjectKeyTypes::Id => self.doc_id = value,
                            ObjectKeyTypes::Key(key) => {
                                // Pop the dummy object that makes ObjectEnd happy
                                // or the previous object key
                                self.keybuilder.pop_object_key();
                                self.keybuilder.push_object_key(key.to_string());

                                try!(self.add_entries(&value, docseq, &batch));
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::NoKey => {
                                try!(self.add_entries(&value, docseq, &batch));
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::Ignore => {
                                self.ignore_children = 1;
                            },
                        }
                    }
                },
                not_implemented => {
                    panic!("Not yet implemented other JSON types! {:?}", not_implemented);
                }
            };

            token = parser.next();
            if token == None {
                break;
            }
        }
        Ok(self.doc_id.clone())
   }
}

/*

#[cfg(test)]
mod tests {

    use super::{WordInfo};
    #[test]

    fn test_shred_nested() {
        let mut shredder = super::Shredder::new();
        //let json = r#"{"hello": {"my": "world!"}, "anumber": 2}"#;
        //let json = r#"{"A":[{"B":"B2VMX two three","C":"C2"},{"B": "b1","C":"C2"}]}"#;
        //let json = r#"{"A":[[[{"B": "string within deeply nested array should be stemmed"}]]]}"#;
        //let json = r#"[{"A": 1, "B": 2, "C": 3}]"#;
        //let json = r#"{"foo": {"bar": 1}}"#;
        let json = r#"{"some": ["array", "data", ["also", "nested"]]}"#;
        let docseq = 123;
        shredder.shred(json, docseq).unwrap();
        let expected = vec![
            ("W.some$!array#123,0", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 5 }])]),
            ("W.some$!data#123,1", vec![
                (vec![1], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ("W.some$$!also#123,2,0", vec![
                (vec![2, 0], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }])]),
            ("W.some$$!nest#1232,1", vec![
                (vec![2, 1], vec![WordInfo {
                    stemmed_offset: 0, suffix_text: "ed".to_string(), suffix_offset: 4 }])]),
            ];
        compare_shredded(&shredder.map, &expected);
    }

    #[test]
    fn test_shred_objects() {
        let mut shredder = super::Shredder::new();
        let json = r#"{"A":[{"B":"B2VMX two three","C":"..C2"},{"B": "b1","C":"..C2"}]}"#;
        let docseq = 1234;
        shredder.shred(json, docseq).unwrap();
        let expected = vec![
            ("W.A$.B!b1#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }])]),
            ("W.A$.B!b2vmx#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "B2VMX ".to_string(), suffix_offset: 0 }])]),
            ("W.A$.B!three#1234", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 10, suffix_text: "".to_string(), suffix_offset: 15 }])]),
            ("W.A$.B!two#1234", vec![
                (vec![0], vec![WordInfo {
                    stemmed_offset: 6, suffix_text: " ".to_string(), suffix_offset: 9 }])]),
            ("W.A$.C!..#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 },
                    WordInfo {
                        stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }])]),
            ("W.A$.C!c2#1234", vec![
                (vec![0], vec![
                    WordInfo {
                        stemmed_offset: 2, suffix_text: "C2".to_string(), suffix_offset: 2 },
                    WordInfo {
                        stemmed_offset: 2, suffix_text: "C2".to_string(), suffix_offset: 2 }])]),
            ];
        compare_shredded(&shredder.map, &expected);
    }

    fn compare_shredded(result_map: &WordPathInfoMap,
                        expected: &Vec<(&str, Vec<(Vec<u64>, Vec<WordInfo>)>)>) {
        // HashMap have an arbitrary order of the elements
        let mut result: Vec<(&String, &ArrayOffsetsToWordInfo)> = result_map.into_iter().collect();
        result.sort_by(|a, b| Ord::cmp(&a.0, &b.0));
        for (ii, &(key, values)) in result.iter().enumerate() {
            assert_eq!(key, expected[ii].0);
            let mut wordinfos: Vec<(&Vec<u64>, &Vec<WordInfo>)> = values.iter().collect();
            wordinfos.sort_by_key(|item| item.0);
            for (jj, wordinfo) in wordinfos.iter().enumerate() {
                assert_eq!(wordinfo.0, &expected[ii].1[jj].0);
                assert_eq!(wordinfo.1, &expected[ii].1[jj].1);
            }
        }
    }

    #[test]
    fn test_shred_empty_object() {
        let mut shredder = super::Shredder::new();
        let json = r#"{}"#;
        let docseq = 123;
        shredder.shred(json, docseq).unwrap();
        assert!(shredder.map.is_empty());
    }
}
*/
<commit_msg>Fix JSON shredder tests<commit_after>extern crate rocksdb;
extern crate rustc_serialize;

use std::collections::HashMap;

use self::rustc_serialize::json::{JsonEvent, Parser, StackElement};
// Needed for a trait in order to `put()` into a `rocksdb::WriteBatch`
use self::rocksdb::Writable;

use error::Error;
use key_builder::{KeyBuilder, SegmentType};
use records_capnp::payload;
use stems::Stems;

// Good example of using rustc_serialize: https://github.com/ajroetker/beautician/blob/master/src/lib.rs
// Callback based JSON streaming parser: https://github.com/gyscos/json-streamer.rs
// Another parser pased on rustc_serializ: https://github.com/isagalaev/ijson-rust/blob/master/src/test.rs#L11

#[derive(Debug, PartialEq)]
struct WordInfo {
    //offset in the text field where the stemmed text starts
    stemmed_offset: u64,

    // the suffix of the stemmed text. When applied over stemmed, the original
    // text is returned.
    suffix_text: String,

    // the start of the suffixText
    suffix_offset: u64,
}

type ArrayOffsets = Vec<u64>;

enum ObjectKeyTypes {
    /// _id field
    Id,
    /// Normal key
    Key(String),
    /// Reserved key starting with underscore
    Ignore,
    /// No key found
    NoKey,
}

#[derive(Debug)]
pub struct Shredder {
    keybuilder: KeyBuilder,
    path_array_offsets: ArrayOffsets,
    // Top-level fields prefixed with an underscore are ignored
    ignore_children: u64,
    doc_id: String,
}

impl Shredder {
    pub fn new() -> Shredder {
        Shredder{
            keybuilder: KeyBuilder::new(),
            path_array_offsets: Vec::new(),
            ignore_children: 0,
            doc_id: String::new(),
        }
    }

    fn add_entries(&mut self, text: &String, docseq: u64, batch: &rocksdb::WriteBatch) -> Result<(), Error> {
        let stems = Stems::new(text.as_str());
        let mut word_to_word_infos = HashMap::new();

        for stem in stems {
            let word_infos = word_to_word_infos.entry(stem.stemmed).or_insert(Vec::new());
            word_infos.push(WordInfo{
                stemmed_offset: stem.stemmed_offset as u64,
                suffix_text: stem.suffix.to_string(),
                suffix_offset: stem.suffix_offset as u64,
            });
        }
        for (stemmed, word_infos) in word_to_word_infos {
            let mut message = ::capnp::message::Builder::new_default();
            {
                let capn_payload = message.init_root::<payload::Builder>();
                let mut capn_wordinfos = capn_payload.init_wordinfos(word_infos.len() as u32);
                for (pos, word_info) in word_infos.iter().enumerate() {
                    let mut capn_wordinfo = capn_wordinfos.borrow().get(pos as u32);
                    capn_wordinfo.set_stemmed_offset(word_info.stemmed_offset);
                    capn_wordinfo.set_suffix_text(&word_info.suffix_text);
                    capn_wordinfo.set_suffix_offset(word_info.suffix_offset);
                }
            }
            self.keybuilder.push_word(&stemmed);
            self.keybuilder.push_doc_seq(docseq);
            self.keybuilder.push_array_path(&self.path_array_offsets);

            let mut bytes = Vec::new();
            ::capnp::serialize_packed::write_message(&mut bytes, &message).unwrap();
            try!(batch.put(&self.keybuilder.key().into_bytes(), &bytes));

            self.keybuilder.pop_array_path();
            self.keybuilder.pop_doc_seq();
            self.keybuilder.pop_word();
        }
        Ok(())
    }

    fn inc_top_array_offset(&mut self) {
        // we encounter a new element. if we are a child element of an array
        // increment the offset. If we aren't (we are the root value or a map
        // value) we don't increment
        if let Some(SegmentType::Array) = self.keybuilder.last_pushed_segment_type() {
            if let Some(last) = self.path_array_offsets.last_mut() {
                *last += 1;
            }
        }
    }

    // Extract key if it exists and indicates if it's a special type of key
    fn extract_key(&mut self, stack_element: Option<StackElement>) -> ObjectKeyTypes {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey {
            match stack_element {
                Some(StackElement::Key(key)) => {
                    if self.keybuilder.segments.len() == 1 && key.starts_with("_") {
                        if key == "_id" {
                            ObjectKeyTypes::Id
                        } else {
                            ObjectKeyTypes::Ignore
                        }
                    } else {
                        ObjectKeyTypes::Key(key.to_string())
                    }
                },
                _ => ObjectKeyTypes::NoKey,
            }
        } else {
            ObjectKeyTypes::NoKey
        }
    }

    // If we are inside an object we need to push the key to the key builder
    // Don't push them if they are reserved fields (starting with underscore)
    fn maybe_push_key(&mut self, stack_element: Option<StackElement>) -> Result<(), Error> {
        if self.keybuilder.last_pushed_segment_type().unwrap() == SegmentType::ObjectKey {
            if let Some(StackElement::Key(key)) = stack_element {
                if self.keybuilder.segments.len() == 1 && key.starts_with("_") {
                    if key == "_id" {
                        return Err(Error::Shred(
                            "Expected string for `_id` field, got another type".to_string()));
                    } else {
                        self.ignore_children = 1;
                    }
                } else {
                    // Pop the dummy object that makes ObjectEnd happy
                    // or the previous object key
                    self.keybuilder.pop_object_key();
                    self.keybuilder.push_object_key(key.to_string());
                }
            }
        }
        Ok(())
    }

   pub fn shred(&mut self, json: &str, docseq: u64, batch: &rocksdb::WriteBatch) -> Result<String, Error> {
        let mut parser = Parser::new(json.chars());
        let mut token = parser.next();

        loop {
            // Get the next token, so that in case of an `ObjectStart` the key is already
            // on the stack.
            match token.take() {
                Some(JsonEvent::ObjectStart) => {
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    }
                    else {
                        // Just push something to make `ObjectEnd` happy
                        self.keybuilder.push_object_key("".to_string());
                    }
                },
                Some(JsonEvent::ObjectEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {//if !self.keybuilder.segments.is_empty() {
                        self.keybuilder.pop_object_key();
                        self.inc_top_array_offset();
                    }
                },
                Some(JsonEvent::ArrayStart) => {
                    try!(self.maybe_push_key(parser.stack().top()));
                    if self.ignore_children > 0 {
                        self.ignore_children += 1;
                    } else {
                        self.keybuilder.push_array();
                        self.path_array_offsets.push(0);
                    }
                },
                Some(JsonEvent::ArrayEnd) => {
                    if self.ignore_children > 0 {
                        self.ignore_children -= 1;
                    } else {
                        self.path_array_offsets.pop();
                        self.keybuilder.pop_array();
                        self.inc_top_array_offset();
                    }
                },
                Some(JsonEvent::StringValue(value)) => {
                    // No children to ignore
                    if self.ignore_children == 0 {
                        println!("stringvalue: {:?}", value);
                        match self.extract_key(parser.stack().top()) {
                            ObjectKeyTypes::Id => self.doc_id = value,
                            ObjectKeyTypes::Key(key) => {
                                // Pop the dummy object that makes ObjectEnd happy
                                // or the previous object key
                                self.keybuilder.pop_object_key();
                                self.keybuilder.push_object_key(key.to_string());

                                try!(self.add_entries(&value, docseq, &batch));
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::NoKey => {
                                try!(self.add_entries(&value, docseq, &batch));
                                self.inc_top_array_offset();
                            },
                            ObjectKeyTypes::Ignore => {
                                self.ignore_children = 1;
                            },
                        }
                    }
                },
                not_implemented => {
                    panic!("Not yet implemented other JSON types! {:?}", not_implemented);
                }
            };

            token = parser.next();
            if token == None {
                break;
            }
        }
        Ok(self.doc_id.clone())
   }
}

#[cfg(test)]
mod tests {
    extern crate rocksdb;
    use std::str;
    use records_capnp;
    use super::{WordInfo};

    fn wordinfos_from_rocks(rocks: rocksdb::DB) -> Vec<(String, Vec<WordInfo>)> {
        let mut result = Vec::new();
        for (key, value) in rocks.iterator(rocksdb::IteratorMode::Start) {
            let mut ref_value = &*value;
            let message_reader = ::capnp::serialize_packed::read_message(
                &mut ref_value, ::capnp::message::ReaderOptions::new()).unwrap();
            let payload = message_reader.get_root::<records_capnp::payload::Reader>().unwrap();

            let mut wordinfos = Vec::new();
            for wi in payload.get_wordinfos().unwrap().iter() {
                wordinfos.push(WordInfo{
                    stemmed_offset: wi.get_stemmed_offset(),
                    suffix_text: wi.get_suffix_text().unwrap().to_string(),
                    suffix_offset: wi.get_suffix_offset(),
                });
            }
            let key_string = unsafe { str::from_utf8_unchecked((&key)) }.to_string();
            result.push((key_string, wordinfos));
        }
        result
    }

    #[test]
    fn test_shred_nested() {
        let mut shredder = super::Shredder::new();
        let json = r#"{"some": ["array", "data", ["also", "nested"]]}"#;
        let docseq = 123;
        let batch = rocksdb::WriteBatch::default();
        shredder.shred(json, docseq, &batch).unwrap();

        let rocks = rocksdb::DB::open_default("target/tests/test_shred_netsted").unwrap();
        rocks.write(batch).unwrap();
        let result = wordinfos_from_rocks(rocks);

        let expected = vec![
            ("W.some$!array#123,0".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 5 }]),
            ("W.some$!data#123,1".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }]),
            ("W.some$$!also#123,2,0".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 4 }]),
            ("W.some$$!nest#123,2,1".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "ed".to_string(), suffix_offset: 4 }]),
            ];
        assert_eq!(result, expected);
    }

    #[test]
    // NOTE vmx 2016-12-06: This test is intentionally made to fail (hence ignored) as the current
    // current tokenizer does the wrong thing when it comes to numbers within words. It's left
    // here as a reminder to fix that
    #[ignore]
    fn test_shred_objects() {
        let mut shredder = super::Shredder::new();
        let json = r#"{"A":[{"B":"B2VMX two three","C":"..C2"},{"B": "b1","C":"..C2"}]}"#;
        let docseq = 1234;
        let batch = rocksdb::WriteBatch::default();
        shredder.shred(json, docseq, &batch).unwrap();

        let rocks = rocksdb::DB::open_default("target/tests/test_shred_objects").unwrap();
        rocks.write(batch).unwrap();
        let result = wordinfos_from_rocks(rocks);
        println!("result: {:?}", result);
        let expected = vec![
            ("W.A$.B!b1#1234,1".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }]),
            ("W.A$.B!b2vmx#1234,0".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "B2 VMX ".to_string(),
                           suffix_offset: 0 }]),
            ("W.A$.B!three#1234,0".to_string(), vec![
                WordInfo { stemmed_offset: 10, suffix_text: "".to_string(), suffix_offset: 15 }]),
            ("W.A$.B!two#1234,0".to_string(), vec![
                WordInfo { stemmed_offset: 6, suffix_text: " ".to_string(), suffix_offset: 9 }]),
            ("W.A$.C!..#1234,0".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }]),
            ("W.A$.C!..#1234,1".to_string(), vec![
                WordInfo { stemmed_offset: 0, suffix_text: "".to_string(), suffix_offset: 2 }]),
            ("W.A$.C!c2#1234,0".to_string(), vec![
                WordInfo { stemmed_offset: 2, suffix_text: "C2".to_string(), suffix_offset: 2 }]),
            ("W.A$.C!c2#1234,1".to_string(), vec![
                WordInfo { stemmed_offset: 2, suffix_text: "C2".to_string(), suffix_offset: 2 }]),
            ];
        assert_eq!(result, expected);
    }

    #[test]
    fn test_shred_empty_object() {
        let mut shredder = super::Shredder::new();
        let json = r#"{}"#;
        let docseq = 123;
        let batch = rocksdb::WriteBatch::default();
        shredder.shred(json, docseq, &batch).unwrap();

        let rocks = rocksdb::DB::open_default("target/tests/test_shred_empty_object").unwrap();
        rocks.write(batch).unwrap();
        let result = wordinfos_from_rocks(rocks);
        assert!(result.is_empty());
    }
}
<|endoftext|>


<commit_before>mod actions;
mod core;
pub mod commands;

#[cfg(test)]
mod unit_tests;

pub use self::actions::movement::MovementError;
pub use self::actions::focus::FocusError;
pub use self::actions::resize::ResizeErr;
pub use self::core::GraphError;

pub use self::core::action::{Action, ActionErr};
pub use self::core::container::{Container, ContainerType, Handle, Layout};
pub use self::core::tree::{Direction, TreeError};
pub use self::core::bar::Bar;
pub use self::core::{InnerTree, MIN_SIZE};

use petgraph::graph::NodeIndex;
use rustc_serialize::json::{Json, ToJson};

use std::sync::{Mutex, MutexGuard, TryLockError, PoisonError};

/// A wrapper around tree, to hide its methods
#[derive(Debug)]
pub struct Tree(LayoutTree);
/// Mutex guard around the tree
pub type TreeGuard = MutexGuard<'static, Tree>;
/// Error for trying to lock the tree
pub type TreeErr = TryLockError<TreeGuard>;

impl Tree {
    /// Constructs a new tree.
    pub fn new() -> Self {
        Tree(LayoutTree {
            tree: InnerTree::new(),
            active_container: None
        })
    }
}

#[derive(Debug)]
pub struct LayoutTree {
    tree: InnerTree,
    active_container: Option<NodeIndex>
}

lazy_static! {
    static ref TREE: Mutex<Tree> = {
        Mutex::new(Tree::new())
    };
    static ref PREV_ACTION: Mutex<Option<Action>> = Mutex::new(None);
}

impl ToJson for LayoutTree {
    fn to_json(&self) -> Json {
        use std::collections::BTreeMap;
        fn node_to_json(node_ix: NodeIndex, tree: &LayoutTree) -> Json {
            match &tree.tree[node_ix] {
                &Container::Workspace { ref name, .. } => {
                    let mut inner_map = BTreeMap::new();
                    let children = tree.tree.children_of(node_ix).iter()
                        .map(|node| node_to_json(*node, tree)).collect();
                    inner_map.insert(format!("Workspace {}", name), Json::Array(children));
                    return Json::Object(inner_map);
                }
                &Container::Container { ref layout, id, .. } => {
                    let mut inner_map = BTreeMap::new();
                    let children = tree.tree.children_of(node_ix).iter()
                        .map(|node| node_to_json(*node, tree)).collect();
                    inner_map.insert(format!("Container w/ layout {:?} and id {:?}", layout, id),
                                     Json::Array(children));
                    return Json::Object(inner_map);
                }
                &Container::View { ref handle, id, .. } => {
                    return Json::String(format!("View: title: \"{:?}\", class: \"{:?}\", id: {}",
                                                handle.get_title(), handle.get_class(), id));
                },
                ref container => {
                    let mut inner_map = BTreeMap::new();
                    let children = tree.tree.children_of(node_ix).iter()
                        .map(|node| node_to_json(*node, tree)).collect();
                    inner_map.insert(format!("{:?}", container.get_type()),
                                     Json::Array(children));
                    return Json::Object(inner_map)
                }
            }
        }
        return node_to_json(self.tree.root_ix(), self);
    }
}

/// Attempts to lock the tree. If the Result is Err, then the lock could
/// not be returned at this time, already locked.
pub fn try_lock_tree() -> Result<TreeGuard, TreeErr> {
    let tree = try!(TREE.try_lock());
    Ok(tree)
}

pub fn lock_tree() -> Result<TreeGuard, PoisonError<TreeGuard>> {
    let tree = try!(TREE.lock());
    Ok(tree)
}

/// Attempts to lock the action mutex. If the Result is Err, then the lock could
/// not be returned at this time, already locked.
pub fn try_lock_action() -> Result<MutexGuard<'static, Option<Action>>,
                                 TryLockError<MutexGuard<'static,
                                                         Option<Action>>>> {
    PREV_ACTION.try_lock()
}
<commit_msg>Don't make InnerTree public<commit_after>mod actions;
mod core;
pub mod commands;

#[cfg(test)]
mod unit_tests;

pub use self::actions::movement::MovementError;
pub use self::actions::focus::FocusError;
pub use self::actions::resize::ResizeErr;
pub use self::core::GraphError;

pub use self::core::action::{Action, ActionErr};
pub use self::core::container::{Container, ContainerType, Handle, Layout};
pub use self::core::tree::{Direction, TreeError};
pub use self::core::bar::Bar;
use self::core::InnerTree;
pub use self::core::MIN_SIZE;

use petgraph::graph::NodeIndex;
use rustc_serialize::json::{Json, ToJson};

use std::sync::{Mutex, MutexGuard, TryLockError, PoisonError};

/// A wrapper around tree, to hide its methods
#[derive(Debug)]
pub struct Tree(LayoutTree);
/// Mutex guard around the tree
pub type TreeGuard = MutexGuard<'static, Tree>;
/// Error for trying to lock the tree
pub type TreeErr = TryLockError<TreeGuard>;

impl Tree {
    /// Constructs a new tree.
    pub fn new() -> Self {
        Tree(LayoutTree {
            tree: InnerTree::new(),
            active_container: None
        })
    }
}

#[derive(Debug)]
pub struct LayoutTree {
    tree: InnerTree,
    active_container: Option<NodeIndex>
}

lazy_static! {
    static ref TREE: Mutex<Tree> = {
        Mutex::new(Tree::new())
    };
    static ref PREV_ACTION: Mutex<Option<Action>> = Mutex::new(None);
}

impl ToJson for LayoutTree {
    fn to_json(&self) -> Json {
        use std::collections::BTreeMap;
        fn node_to_json(node_ix: NodeIndex, tree: &LayoutTree) -> Json {
            match &tree.tree[node_ix] {
                &Container::Workspace { ref name, .. } => {
                    let mut inner_map = BTreeMap::new();
                    let children = tree.tree.children_of(node_ix).iter()
                        .map(|node| node_to_json(*node, tree)).collect();
                    inner_map.insert(format!("Workspace {}", name), Json::Array(children));
                    return Json::Object(inner_map);
                }
                &Container::Container { ref layout, id, .. } => {
                    let mut inner_map = BTreeMap::new();
                    let children = tree.tree.children_of(node_ix).iter()
                        .map(|node| node_to_json(*node, tree)).collect();
                    inner_map.insert(format!("Container w/ layout {:?} and id {:?}", layout, id),
                                     Json::Array(children));
                    return Json::Object(inner_map);
                }
                &Container::View { ref handle, id, .. } => {
                    return Json::String(format!("View: title: \"{:?}\", class: \"{:?}\", id: {}",
                                                handle.get_title(), handle.get_class(), id));
                },
                ref container => {
                    let mut inner_map = BTreeMap::new();
                    let children = tree.tree.children_of(node_ix).iter()
                        .map(|node| node_to_json(*node, tree)).collect();
                    inner_map.insert(format!("{:?}", container.get_type()),
                                     Json::Array(children));
                    return Json::Object(inner_map)
                }
            }
        }
        return node_to_json(self.tree.root_ix(), self);
    }
}

/// Attempts to lock the tree. If the Result is Err, then the lock could
/// not be returned at this time, already locked.
pub fn try_lock_tree() -> Result<TreeGuard, TreeErr> {
    let tree = try!(TREE.try_lock());
    Ok(tree)
}

pub fn lock_tree() -> Result<TreeGuard, PoisonError<TreeGuard>> {
    let tree = try!(TREE.lock());
    Ok(tree)
}

/// Attempts to lock the action mutex. If the Result is Err, then the lock could
/// not be returned at this time, already locked.
pub fn try_lock_action() -> Result<MutexGuard<'static, Option<Action>>,
                                 TryLockError<MutexGuard<'static,
                                                         Option<Action>>>> {
    PREV_ACTION.try_lock()
}
<|endoftext|>


<commit_before>extern crate toml;

use self::toml::Value as Toml;

use error::{CannotHappenError, CliError, ConfigError, ConfigErrorKind};
use constant::CONFIG_FILE_NAME;
use lib::io::*;
use lib::setting;
use std::fs::File;
use std::io::Read;

pub enum ParamKind {
    BurnAfter,
}

pub struct Config;

impl Config {
    pub fn default() -> String {
        r#"
[burn]
  after = "2 weeks"

[sweep]
  period = "daily"
  time   = "00:00"
"#
            .to_string()
    }

    pub fn read() -> Result<Toml, CliError> {
        print_with_tag(0, Tag::Execution, format!("Read \"{}\" file as TOML", CONFIG_FILE_NAME));

        let mut f = try!(File::open(setting::config_file()));

        let mut contents = String::new();
        try!(f.read_to_string(&mut contents));

        let result = contents
            .parse()
            .map_err(|e: Vec<_>| e.into_iter().find(|_| true));
        let toml = match result {
            Ok(toml)    => toml,
            Err(option) => match option {
                Some(e) => return Err(From::from(e)),
                None    => return Err(From::from(CannotHappenError)),
            },
        };

        print_with_okay(1);
        Ok(toml)
    }

    pub fn extract(kind: ParamKind) -> Result<String, CliError> {
        let key = match kind {
            ParamKind::BurnAfter => "burn.after",
        };
        print_with_tag(0, Tag::Execution, format!("Extract \"{}\" parameter", key));

        let config = try!(Self::read());
        let result = config
            .lookup(key)
            .ok_or(ConfigError::new(match kind {
                ParamKind::BurnAfter => ConfigErrorKind::NotFoundBurnAfter,
            }));
        let value = try!(result).to_string(); // This to_string() is not documented.

        print_with_okay(1);
        Ok(value)
    }
}
<commit_msg>[Remedy] Create ParamKind::to_str()<commit_after>extern crate toml;

use self::toml::Value as Toml;

use error::{CannotHappenError, CliError, ConfigError, ConfigErrorKind};
use constant::CONFIG_FILE_NAME;
use lib::io::*;
use lib::setting;
use std::fs::File;
use std::io::Read;

pub enum ParamKind {
    BurnAfter,
}
impl ParamKind {
    fn to_str(&self) -> &str {
        match *self {
            ParamKind::BurnAfter => "burn.after",
        }
    }
}

pub struct Config;

impl Config {
    pub fn default() -> String {
        r#"
[burn]
  after = "2 weeks"

[sweep]
  period = "daily"
  time   = "00:00"
"#
            .to_string()
    }

    pub fn read() -> Result<Toml, CliError> {
        print_with_tag(0, Tag::Execution, format!("Read \"{}\" file as TOML", CONFIG_FILE_NAME));

        let mut f = try!(File::open(setting::config_file()));

        let mut contents = String::new();
        try!(f.read_to_string(&mut contents));

        let result = contents
            .parse()
            .map_err(|e: Vec<_>| e.into_iter().find(|_| true));
        let toml = match result {
            Ok(toml)    => toml,
            Err(option) => match option {
                Some(e) => return Err(From::from(e)),
                None    => return Err(From::from(CannotHappenError)),
            },
        };

        print_with_okay(1);
        Ok(toml)
    }

    pub fn extract(kind: ParamKind) -> Result<String, CliError> {
        let key = kind.to_str();
        print_with_tag(0, Tag::Execution, format!("Extract \"{}\" parameter", key));

        let config = try!(Self::read());
        let result = config
            .lookup(key)
            .ok_or(ConfigError::new(match kind {
                ParamKind::BurnAfter => ConfigErrorKind::NotFoundBurnAfter,
            }));
        let value = try!(result).to_string(); // This to_string() is not documented.

        print_with_okay(1);
        Ok(value)
    }
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Peter Nelson
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#![crate_id = "jit#0.1.2"]
#![comment = "LibJIT Bindings"]
#![crate_type = "dylib"]
#![crate_type = "rlib"]
#![allow(raw_pointer_deriving, dead_code, non_camel_case_types)]
#![deny(unnecessary_parens, unrecognized_lint, unreachable_code, unnecessary_allocation, unnecessary_typecast, unnecessary_allocation, uppercase_variables, unused_must_use)]
#![feature(globs, macro_rules)]
#![stable]
//! This crate wraps LibJIT in an idiomatic style.
//! For example, here's a quick example which makes a multiply function using LibJIT:
//!
//! ```rust
//! extern crate jit;
//! use jit::{C_DECL, Context, Function, Type, Types, get_type};
//! fn main() {
//!     let cx = Context::new();
//!     cx.build(|| {
//!         // build the IR
//!         let sig = get_type(fn(int, int) -> int);
//!         let func = Function::new(cx, sig);
//!         let x = func.get_param(0);
//!         let y = func.get_param(1);
//!         let result = func.insn_mul(&x, &y);
//!         func.insn_return(&result);
//!         /// run the IR
//!         func.compile();
//!         let rfunc:fn(int, int) -> int = func.closure2();
//!         assert_eq(rfunc(4, 5), 20)
//!     });
//! }
//! ```

extern crate libc;
use bindings::{
    jit_init,
    jit_uses_interpreter,
    jit_supports_threads,
    jit_supports_virtual_memory
};
pub use bindings::{
    jit_nint,
    jit_nuint
};
pub use compile::Compile;
pub use context::{
    Context,
    InContext,
    Functions
};
pub use function::{
    ABI,
    CDECL,
    Function
};
pub use label::Label;
pub use types::*;
pub use get_type = types::get;
pub use util::NativeRef;
pub use value::Value;

/// Initialise the library and prepare for operations
#[inline]
pub fn init() -> () {
    unsafe {
        jit_init()
    }
}
/// Check if the JIT is using a fallback interpreter
#[inline]
pub fn uses_interpreter() -> bool {
    unsafe {
        jit_uses_interpreter() != 0
    }
}
/// Check if the JIT supports theads
#[inline]
pub fn supports_threads() -> bool {
    unsafe {
        jit_supports_threads() != 0
    }
}
/// Check if the JIT supports virtual memory
#[inline]
pub fn supports_virtual_memory() -> bool {
    unsafe {
        jit_supports_virtual_memory() != 0
    }
}
macro_rules! native_ref(
    ($name:ident, $field:ident, $pointer_ty:ty) => (
        #[deriving(PartialEq)]
        pub struct $name {
            $field: $pointer_ty
        }
        impl NativeRef for $name {
            #[inline]
            /// Convert to a native pointer
            unsafe fn as_ptr(&self) -> $pointer_ty {
                self.$field
            }
            #[inline]
            /// Convert from a native pointer
            unsafe fn from_ptr(ptr:$pointer_ty) -> $name {
                $name {
                    $field: ptr
                }
            }
        }
    );
    ($name:ident, $field:ident, $pointer_ty:ty, $lifetime:ident) => (
        #[deriving(PartialEq)]
        pub struct $name<'a> {
            $field: $pointer_ty,
            marker: $lifetime<'a>
        }
        impl<'a> NativeRef for $name<'a> {
            #[inline]
            /// Convert to a native pointer
            unsafe fn as_ptr(&self) -> $pointer_ty {
                self.$field
            }
            #[inline]
            /// Convert from a native pointer
            unsafe fn from_ptr(ptr:$pointer_ty) -> $name {
                $name {
                    $field: ptr,
                    marker: $lifetime::<'a>
                }
            }
        }
    )
)
mod bindings;
mod context;
mod compile;
mod elf;
mod function;
mod label;
#[cfg(test)]
mod tests;
mod types;
mod util;
mod value;<commit_msg>Add `unsafe_destructor` as feature<commit_after>/* Copyright (c) 2014, Peter Nelson
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#![crate_id = "jit#0.1.2"]
#![comment = "LibJIT Bindings"]
#![crate_type = "dylib"]
#![crate_type = "rlib"]
#![allow(raw_pointer_deriving, dead_code, non_camel_case_types)]
#![deny(unnecessary_parens, unrecognized_lint, unreachable_code, unnecessary_allocation, unnecessary_typecast, unnecessary_allocation, uppercase_variables, unused_must_use)]
#![feature(unsafe_destructor, globs, macro_rules)]
#![stable]
//! This crate wraps LibJIT in an idiomatic style.
//! For example, here's a quick example which makes a multiply function using LibJIT:
//!
//! ```rust
//! extern crate jit;
//! use jit::{C_DECL, Context, Function, Type, Types, get_type};
//! fn main() {
//!     let cx = Context::new();
//!     cx.build(|| {
//!         // build the IR
//!         let sig = get_type(fn(int, int) -> int);
//!         let func = Function::new(cx, sig);
//!         let x = func.get_param(0);
//!         let y = func.get_param(1);
//!         let result = func.insn_mul(&x, &y);
//!         func.insn_return(&result);
//!         /// run the IR
//!         func.compile();
//!         let rfunc:fn(int, int) -> int = func.closure2();
//!         assert_eq(rfunc(4, 5), 20)
//!     });
//! }
//! ```

extern crate libc;
use bindings::{
    jit_init,
    jit_uses_interpreter,
    jit_supports_threads,
    jit_supports_virtual_memory
};
pub use bindings::{
    jit_nint,
    jit_nuint
};
pub use compile::Compile;
pub use context::{
    Context,
    InContext,
    Functions
};
pub use function::{
    ABI,
    CDECL,
    Function
};
pub use label::Label;
pub use types::*;
pub use get_type = types::get;
pub use util::NativeRef;
pub use value::Value;

/// Initialise the library and prepare for operations
#[inline]
pub fn init() -> () {
    unsafe {
        jit_init()
    }
}
/// Check if the JIT is using a fallback interpreter
#[inline]
pub fn uses_interpreter() -> bool {
    unsafe {
        jit_uses_interpreter() != 0
    }
}
/// Check if the JIT supports theads
#[inline]
pub fn supports_threads() -> bool {
    unsafe {
        jit_supports_threads() != 0
    }
}
/// Check if the JIT supports virtual memory
#[inline]
pub fn supports_virtual_memory() -> bool {
    unsafe {
        jit_supports_virtual_memory() != 0
    }
}
macro_rules! native_ref(
    ($name:ident, $field:ident, $pointer_ty:ty) => (
        #[deriving(PartialEq)]
        pub struct $name {
            $field: $pointer_ty
        }
        impl NativeRef for $name {
            #[inline]
            /// Convert to a native pointer
            unsafe fn as_ptr(&self) -> $pointer_ty {
                self.$field
            }
            #[inline]
            /// Convert from a native pointer
            unsafe fn from_ptr(ptr:$pointer_ty) -> $name {
                $name {
                    $field: ptr
                }
            }
        }
    );
    ($name:ident, $field:ident, $pointer_ty:ty, $lifetime:ident) => (
        #[deriving(PartialEq)]
        pub struct $name<'a> {
            $field: $pointer_ty,
            marker: $lifetime<'a>
        }
        impl<'a> NativeRef for $name<'a> {
            #[inline]
            /// Convert to a native pointer
            unsafe fn as_ptr(&self) -> $pointer_ty {
                self.$field
            }
            #[inline]
            /// Convert from a native pointer
            unsafe fn from_ptr(ptr:$pointer_ty) -> $name {
                $name {
                    $field: ptr,
                    marker: $lifetime::<'a>
                }
            }
        }
    )
)
mod bindings;
mod context;
mod compile;
mod elf;
mod function;
mod label;
#[cfg(test)]
mod tests;
mod types;
mod util;
mod value;<|endoftext|>


<commit_before>
include!(concat!(env!("OUT_DIR"), "/exceptional-grammar.rs"));

#[cfg(test)]
mod test_helpers {
    use super::*;
    use ast::*;
    use num::rational::{Ratio};
    use num::{BigInt};

    pub fn l_string(name: &str) -> Literal {
        Literal::CharString(
           name.to_owned()
        )
    }

    pub fn l_number(num: i64, denom: i64) -> Literal {
        Literal::Number(
            Ratio::new(
                BigInt::from(num),
                BigInt::from(denom)
            )
        )
    }

    pub fn l_bool(b: bool) -> Literal {
        match b {
            true => Literal::Boolean(true),
            false => Literal::Boolean(false)
        }
    }

    pub fn l_function(args: Vec<String>, statements: Vec<Statement>) -> Literal {
        Literal::Fn(Box::new((args, statements)))
    }

    pub fn s_assign(name: &str, literal: Literal) -> Statement {
        Statement::Assign(
            true,
            name.to_owned(),
            Box::new(
                e_literal(
                    literal
                )
            )
        )
    }

    pub fn s_call(name: &str, args: Vec<Expression>) -> Statement {
        Statement::Call(name.to_owned(), args)
    }

    pub fn e_literal(literal: Literal) -> Expression {
        Expression::Literal(literal)
    }

    pub fn e_identifier(name: &str) -> Expression {
        Expression::Identifier(name.to_owned())
    }

    pub fn e_binop(op: &str, left: Expression, right: Expression) -> Expression {
        Expression::BinOp(op.to_owned(), Box::new(left), Box::new(right))
    }

    pub fn parse_expression(input: &str) -> Expression {
        expression(input).unwrap()
    }

    pub fn parse_statements(input: &str) -> Vec<Statement> {
        statements(input).unwrap()
    }

    pub fn parse_literal(input: &str) -> Literal {
        literal(input).unwrap()
    }
}

#[cfg(test)]
mod test_literals {
    use super::test_helpers::*;

    #[test]
    fn parses_number() {
        assert_eq!(
            parse_literal(&"1234"),
            l_number(1234, 1)
        );

        assert_eq!(
            parse_literal(&"0011"),
            l_number(11, 1)
        );
    }

    #[test]
    fn parses_strings() {
        assert_eq!(
            parse_literal(&"\"\""),
            l_string(&"")
        );

        assert_eq!(
            parse_literal(&"\"string with more words\""),
            l_string(&"string with more words")
        );
    }

    #[test]
    fn parses_booleans() {
        assert_eq!(
            parse_literal(&"true"),
            l_bool(true)
        );

        assert_eq!(
            parse_literal(&"false"),
            l_bool(false)
        );
    }
}

#[cfg(test)]
mod test_expressions {
    use super::test_helpers::*;

    #[test]
    fn parses_simple_literal_expressions() {
        assert_eq!(
            parse_expression(&"1"),
            e_literal(l_number(1, 1))
        );

        assert_eq!(
            parse_expression(&"\"\""),
            e_literal(l_string(&""))
        )
    }

    #[test]
    fn parses_math() {
        assert_eq!(
            parse_expression(&"1 + 2 * 3 / 5 - d"),
            e_binop(
                "+",
                e_literal(l_number(1, 1)),
                e_binop(
                    "-",
                    e_binop(
                        "*",
                        e_literal(l_number(2, 1)),
                        e_binop("/", e_literal(l_number(3, 1)), e_literal(l_number(5, 1)))
                    ),
                    e_identifier(&"d")
                )
            )
        );
    }

    #[test]
    fn parses_identifiers() {
        assert_eq!(
            parse_expression(&"toto"),
            e_identifier(&"toto")
        )
    }

    #[test]
    fn parses_simple_functions() {
        assert_eq!(
            parse_expression(&"def() do end"),
            e_literal(
                l_function(vec![], vec![])
            )
        )
    }

    #[test]
    fn parses_functions_with_args() {
        assert_eq!(
            parse_expression(&"def(a, b) do end"),
            e_literal(
                l_function(vec!["a".to_owned(), "b".to_owned()], vec![])
            )
        )
    }

    #[test]
    fn prases_functions_with_bodies() {
        assert_eq!(
            parse_expression(&"def(a, b) do\nlet c = 1\nend"),
            e_literal(
                l_function(
                    vec!["a".to_owned(), "b".to_owned()],
                    vec![s_assign(&"c", l_number(1, 1))]
                )
            )
        )
    }
}

#[cfg(test)]
mod test_statements {
    use super::test_helpers::*;

    #[test]
    fn parses_assigns() {
        assert_eq!(
            parse_statements(&"let a = 1"),
            [s_assign(&"a", l_number(1, 1))]
        )
    }

    #[test]
    fn parses_calls() {
        assert_eq!(
            parse_statements("a()"),
            [s_call(&"a", vec![])]
        )
    }

    #[test]
    fn parses_calls_with_simple_args() {
        assert_eq!(
            parse_statements("a(1, b)"),
            [s_call(&"a", vec![e_literal(l_number(1, 1)), e_identifier(&"b")])]
        )
    }

    #[test]
    fn parses_calls_with_expressive_args() {
        let args = vec![
            e_binop(
                "+",
                e_literal(l_number(1, 1)),
                e_literal(l_number(2, 1)),
            ),
            e_literal(l_function(vec!["x".to_owned()], vec![]))
        ];
        assert_eq!(
            parse_statements("a(1 + 2, def(x) do end)"),
            [s_call(&"a", args)]
        )
    }
}
<commit_msg>Add tests for maps<commit_after>
include!(concat!(env!("OUT_DIR"), "/exceptional-grammar.rs"));

#[cfg(test)]
mod test_helpers {
    use super::*;
    use ast::*;
    use num::rational::{Ratio};
    use num::{BigInt};

    pub fn l_string(name: &str) -> Literal {
        Literal::CharString(
           name.to_owned()
        )
    }

    pub fn l_number(num: i64, denom: i64) -> Literal {
        Literal::Number(
            Ratio::new(
                BigInt::from(num),
                BigInt::from(denom)
            )
        )
    }

    pub fn l_bool(b: bool) -> Literal {
        match b {
            true => Literal::Boolean(true),
            false => Literal::Boolean(false)
        }
    }

    pub fn l_function(args: Vec<String>, statements: Vec<Statement>) -> Literal {
        Literal::Fn(Box::new((args, statements)))
    }

    pub fn l_map(pairs: Vec<(Expression, Expression)>) -> Literal {
        Literal::Map(pairs)
    }

    pub fn s_assign(name: &str, literal: Literal) -> Statement {
        Statement::Assign(
            true,
            name.to_owned(),
            Box::new(
                e_literal(
                    literal
                )
            )
        )
    }

    pub fn s_call(name: &str, args: Vec<Expression>) -> Statement {
        Statement::Call(name.to_owned(), args)
    }

    pub fn e_literal(literal: Literal) -> Expression {
        Expression::Literal(literal)
    }

    pub fn e_identifier(name: &str) -> Expression {
        Expression::Identifier(name.to_owned())
    }

    pub fn e_binop(op: &str, left: Expression, right: Expression) -> Expression {
        Expression::BinOp(op.to_owned(), Box::new(left), Box::new(right))
    }

    pub fn parse_expression(input: &str) -> Expression {
        expression(input).unwrap()
    }

    pub fn parse_statements(input: &str) -> Vec<Statement> {
        statements(input).unwrap()
    }

    pub fn parse_literal(input: &str) -> Literal {
        literal(input).unwrap()
    }
}

#[cfg(test)]
mod test_literals {
    use super::test_helpers::*;

    #[test]
    fn parses_number() {
        assert_eq!(
            parse_literal(&"1234"),
            l_number(1234, 1)
        );

        assert_eq!(
            parse_literal(&"0011"),
            l_number(11, 1)
        );
    }

    #[test]
    fn parses_strings() {
        assert_eq!(
            parse_literal(&"\"\""),
            l_string(&"")
        );

        assert_eq!(
            parse_literal(&"\"string with more words\""),
            l_string(&"string with more words")
        );
    }

    #[test]
    fn parses_booleans() {
        assert_eq!(
            parse_literal(&"true"),
            l_bool(true)
        );

        assert_eq!(
            parse_literal(&"false"),
            l_bool(false)
        );
    }

    #[test]
    fn parses_maps() {
        assert_eq!(
            parse_literal(&"{ a => b }"),
            l_map(vec![(e_identifier(&"a"), e_identifier(&"b"))])
        )
    }
}

#[cfg(test)]
mod test_expressions {
    use super::test_helpers::*;

    #[test]
    fn parses_simple_literal_expressions() {
        assert_eq!(
            parse_expression(&"1"),
            e_literal(l_number(1, 1))
        );

        assert_eq!(
            parse_expression(&"\"\""),
            e_literal(l_string(&""))
        )
    }

    #[test]
    fn parses_math() {
        assert_eq!(
            parse_expression(&"1 + 2 * 3 / 5 - d"),
            e_binop(
                "+",
                e_literal(l_number(1, 1)),
                e_binop(
                    "-",
                    e_binop(
                        "*",
                        e_literal(l_number(2, 1)),
                        e_binop("/", e_literal(l_number(3, 1)), e_literal(l_number(5, 1)))
                    ),
                    e_identifier(&"d")
                )
            )
        );
    }

    #[test]
    fn parses_identifiers() {
        assert_eq!(
            parse_expression(&"toto"),
            e_identifier(&"toto")
        )
    }

    #[test]
    fn parses_simple_functions() {
        assert_eq!(
            parse_expression(&"def() do end"),
            e_literal(
                l_function(vec![], vec![])
            )
        )
    }

    #[test]
    fn parses_functions_with_args() {
        assert_eq!(
            parse_expression(&"def(a, b) do end"),
            e_literal(
                l_function(vec!["a".to_owned(), "b".to_owned()], vec![])
            )
        )
    }

    #[test]
    fn prases_functions_with_bodies() {
        assert_eq!(
            parse_expression(&"def(a, b) do\nlet c = 1\nend"),
            e_literal(
                l_function(
                    vec!["a".to_owned(), "b".to_owned()],
                    vec![s_assign(&"c", l_number(1, 1))]
                )
            )
        )
    }
}

#[cfg(test)]
mod test_statements {
    use super::test_helpers::*;

    #[test]
    fn parses_assigns() {
        assert_eq!(
            parse_statements(&"let a = 1"),
            [s_assign(&"a", l_number(1, 1))]
        )
    }

    #[test]
    fn parses_calls() {
        assert_eq!(
            parse_statements("a()"),
            [s_call(&"a", vec![])]
        )
    }

    #[test]
    fn parses_calls_with_simple_args() {
        assert_eq!(
            parse_statements("a(1, b)"),
            [s_call(&"a", vec![e_literal(l_number(1, 1)), e_identifier(&"b")])]
        )
    }

    #[test]
    fn parses_calls_with_expressive_args() {
        let args = vec![
            e_binop(
                "+",
                e_literal(l_number(1, 1)),
                e_literal(l_number(2, 1)),
            ),
            e_literal(l_function(vec!["x".to_owned()], vec![]))
        ];
        assert_eq!(
            parse_statements("a(1 + 2, def(x) do end)"),
            [s_call(&"a", args)]
        )
    }
}
<|endoftext|>


<commit_before>#![feature(plugin_registrar, rustc_private)]

extern crate rustc;
extern crate rustc_plugin;
extern crate syntax;
extern crate bifrost_hasher;

use syntax::tokenstream::TokenTree;
use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
use syntax::ext::build::AstBuilder;  // trait for expr_usize
use syntax::ext::quote::rt::Span;
use syntax::parse::token;
use rustc_plugin::Registry;
use bifrost_hasher::hash_str;

fn hash_ident (cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) -> Box<MacResult + 'static> {
    if args.len() != 1 {
        cx.span_err(
            sp,
            &format!("argument should be a single identifier, but got {} arguments {:?}",
                     args.len(), args));
        return DummyResult::any(sp);
    }

    let text = match args[0] {
        TokenTree::Token(_, token::Ident(s)) => s.to_string(),
        _ => {
            cx.span_err(sp, "argument should be a single identifier");
            return DummyResult::any(sp);
        }
    };
    let text = &*text;
    let str = String::from(text);
    MacEager::expr(cx.expr_usize(sp, hash_str(&str) as usize))
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("hash_ident", hash_ident);
}<commit_msg>update with up stream nightly rust<commit_after>#![feature(plugin_registrar, rustc_private)]

extern crate rustc;
extern crate rustc_plugin;
extern crate syntax;
extern crate bifrost_hasher;

use syntax::tokenstream::TokenTree;
use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
use syntax::ext::build::AstBuilder;  // trait for expr_usize
use syntax::ext::quote::rt::Span;
use syntax::parse::token;
use rustc_plugin::Registry;
use bifrost_hasher::hash_str;

fn hash_ident (cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) -> Box<MacResult + 'static> {
    if args.len() != 1 {
        cx.span_err(
            sp,
            &format!("argument should be a single identifier, but got {} arguments {:?}",
                     args.len(), args));
        return DummyResult::any(sp);
    }

    let text = match args[0] {
        TokenTree::Token(_, token::Ident(s, _)) => s.to_string(),
        _ => {
            cx.span_err(sp, "argument should be a single identifier");
            return DummyResult::any(sp);
        }
    };
    let text = &*text;
    let str = String::from(text);
    MacEager::expr(cx.expr_usize(sp, hash_str(&str) as usize))
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("hash_ident", hash_ident);
}<|endoftext|>


<commit_before>#[macro_use]
extern crate validator_derive;
extern crate validator;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
extern crate regex;
#[macro_use]
extern crate lazy_static;

use regex::Regex;
use validator::{Validate, ValidationError, ValidationErrors};

fn validate_unique_username(username: &str) -> Result<(), ValidationError> {
    if username == "xXxShad0wxXx" {
        return Err(ValidationError::new("terrible_username"));
    }

    Ok(())
}

fn validate_signup(data: &SignupData) -> Result<(), ValidationError> {
    if data.mail.ends_with("gmail.com") && data.age == 18 {
        return Err(ValidationError::new("stupid_rule"));
    }

    Ok(())
}

#[derive(Debug, Validate, Deserialize)]
#[validate(schema(function = "validate_signup", skip_on_field_errors = "false"))]
struct SignupData {
    #[validate(email)]
    mail: String,
    #[validate(url)]
    site: String,
    #[validate(length(min = "1"), custom = "validate_unique_username")]
    #[serde(rename = "firstName")]
    first_name: String,
    #[validate(range(min = "18", max = "20"))]
    age: u32,
}

#[test]
fn is_fine_with_many_valid_validations() {
    let signup = SignupData {
        mail: "bob@bob.com".to_string(),
        site: "http://hello.com".to_string(),
        first_name: "Bob".to_string(),
        age: 18,
    };

    assert!(signup.validate().is_ok());
}

#[test]
fn failed_validation_points_to_original_field_name() {
    let signup = SignupData {
        mail: "bob@bob.com".to_string(),
        site: "http://hello.com".to_string(),
        first_name: "".to_string(),
        age: 18,
    };
    let res = signup.validate();
    assert!(res.is_err());
    let errs = res.unwrap_err().inner();
    assert!(errs.contains_key("firstName"));
    assert_eq!(errs["firstName"].len(), 1);
    assert_eq!(errs["firstName"][0].code, "length");
}

#[test]
fn test_can_validate_option_fields_with_lifetime() {
    lazy_static! {
        static ref RE2: Regex = Regex::new(r"[a-z]{2}").unwrap();
    }

    #[derive(Debug, Validate)]
    struct PutStruct<'a> {
        #[validate(length(min = "1", max = "10"))]
        name: Option<&'a str>,
        #[validate(length(min = "1", max = "10"))]
        address: Option<Option<&'a str>>,
        #[validate(range(min = "1", max = "100"))]
        age: Option<Option<usize>>,
        #[validate(range(min = "1", max = "10"))]
        range: Option<usize>,
        #[validate(email)]
        email: Option<&'a str>,
        #[validate(url)]
        url: Option<&'a str>,
        #[validate(contains = "@")]
        text: Option<&'a str>,
        #[validate(regex = "RE2")]
        re: Option<&'a str>,
        #[validate(custom = "check_str")]
        custom: Option<&'a str>,
    }

    fn check_str(_: &str) -> Result<(), ValidationError> {
        Ok(())
    }

    let s = PutStruct {
        name: Some("al"),
        address: Some(Some("gol")),
        age: Some(Some(20)),
        range: Some(2),
        email: Some("hi@gmail.com"),
        url: Some("http://google.com"),
        text: Some("@someone"),
        re: Some("hi"),
        custom: Some("hey"),
    };
    assert!(s.validate().is_ok());
}

#[test]
fn test_can_validate_option_fields_without_lifetime() {
    lazy_static! {
        static ref RE2: Regex = Regex::new(r"[a-z]{2}").unwrap();
    }

    #[derive(Debug, Validate)]
    struct PutStruct {
        #[validate(length(min = "1", max = "10"))]
        name: Option<String>,
        #[validate(length(min = "1", max = "10"))]
        address: Option<Option<String>>,
        #[validate(length(min = "1", max = "10"))]
        ids: Option<Vec<usize>>,
        #[validate(length(min = "1", max = "10"))]
        opt_ids: Option<Option<Vec<usize>>>,
        #[validate(range(min = "1", max = "100"))]
        age: Option<Option<usize>>,
        #[validate(range(min = "1", max = "10"))]
        range: Option<usize>,
        #[validate(email)]
        email: Option<String>,
        #[validate(url)]
        url: Option<String>,
        #[validate(contains = "@")]
        text: Option<String>,
        #[validate(regex = "RE2")]
        re: Option<String>,
        #[validate(custom = "check_str")]
        custom: Option<String>,
    }

    fn check_str(_: &str) -> Result<(), ValidationError> {
        Ok(())
    }

    let s = PutStruct {
        name: Some("al".to_string()),
        address: Some(Some("gol".to_string())),
        ids: Some(vec![1, 2, 3]),
        opt_ids: Some(Some(vec![1, 2, 3])),
        age: Some(Some(20)),
        range: Some(2),
        email: Some("hi@gmail.com".to_string()),
        url: Some("http://google.com".to_string()),
        text: Some("@someone".to_string()),
        re: Some("hi".to_string()),
        custom: Some("hey".to_string()),
    };
    assert!(s.validate().is_ok());
}

#[test]
fn test_works_with_question_mark_operator() {
    fn some_fn() -> Result<(), ValidationErrors> {
        let signup = SignupData {
            mail: "invalid_email".to_string(),
            site: "http://hello.com".to_string(),
            first_name: "Bob".to_string(),
            age: 18,
        };

        signup.validate()?;
        Ok(())
    }

    assert!(some_fn().is_err());
}
<commit_msg>Added test to validate None values<commit_after>#[macro_use]
extern crate validator_derive;
extern crate validator;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
extern crate regex;
#[macro_use]
extern crate lazy_static;

use regex::Regex;
use validator::{Validate, ValidationError, ValidationErrors};

fn validate_unique_username(username: &str) -> Result<(), ValidationError> {
    if username == "xXxShad0wxXx" {
        return Err(ValidationError::new("terrible_username"));
    }

    Ok(())
}

fn validate_signup(data: &SignupData) -> Result<(), ValidationError> {
    if data.mail.ends_with("gmail.com") && data.age == 18 {
        return Err(ValidationError::new("stupid_rule"));
    }

    Ok(())
}

#[derive(Debug, Validate, Deserialize)]
#[validate(schema(function = "validate_signup", skip_on_field_errors = "false"))]
struct SignupData {
    #[validate(email)]
    mail: String,
    #[validate(url)]
    site: String,
    #[validate(length(min = "1"), custom = "validate_unique_username")]
    #[serde(rename = "firstName")]
    first_name: String,
    #[validate(range(min = "18", max = "20"))]
    age: u32,
}

#[test]
fn is_fine_with_many_valid_validations() {
    let signup = SignupData {
        mail: "bob@bob.com".to_string(),
        site: "http://hello.com".to_string(),
        first_name: "Bob".to_string(),
        age: 18,
    };

    assert!(signup.validate().is_ok());
}

#[test]
fn failed_validation_points_to_original_field_name() {
    let signup = SignupData {
        mail: "bob@bob.com".to_string(),
        site: "http://hello.com".to_string(),
        first_name: "".to_string(),
        age: 18,
    };
    let res = signup.validate();
    assert!(res.is_err());
    let errs = res.unwrap_err().inner();
    assert!(errs.contains_key("firstName"));
    assert_eq!(errs["firstName"].len(), 1);
    assert_eq!(errs["firstName"][0].code, "length");
}

#[test]
fn test_can_validate_option_fields_with_lifetime() {
    lazy_static! {
        static ref RE2: Regex = Regex::new(r"[a-z]{2}").unwrap();
    }

    #[derive(Debug, Validate)]
    struct PutStruct<'a> {
        #[validate(length(min = "1", max = "10"))]
        name: Option<&'a str>,
        #[validate(length(min = "1", max = "10"))]
        address: Option<Option<&'a str>>,
        #[validate(range(min = "1", max = "100"))]
        age: Option<Option<usize>>,
        #[validate(range(min = "1", max = "10"))]
        range: Option<usize>,
        #[validate(email)]
        email: Option<&'a str>,
        #[validate(url)]
        url: Option<&'a str>,
        #[validate(contains = "@")]
        text: Option<&'a str>,
        #[validate(regex = "RE2")]
        re: Option<&'a str>,
        #[validate(custom = "check_str")]
        custom: Option<&'a str>,
    }

    fn check_str(_: &str) -> Result<(), ValidationError> {
        Ok(())
    }

    let s = PutStruct {
        name: Some("al"),
        address: Some(Some("gol")),
        age: Some(Some(20)),
        range: Some(2),
        email: Some("hi@gmail.com"),
        url: Some("http://google.com"),
        text: Some("@someone"),
        re: Some("hi"),
        custom: Some("hey"),
    };
    assert!(s.validate().is_ok());
}

#[test]
fn test_can_validate_option_fields_without_lifetime() {
    lazy_static! {
        static ref RE2: Regex = Regex::new(r"[a-z]{2}").unwrap();
    }

    #[derive(Debug, Validate)]
    struct PutStruct {
        #[validate(length(min = "1", max = "10"))]
        name: Option<String>,
        #[validate(length(min = "1", max = "10"))]
        address: Option<Option<String>>,
        #[validate(length(min = "1", max = "10"))]
        ids: Option<Vec<usize>>,
        #[validate(length(min = "1", max = "10"))]
        opt_ids: Option<Option<Vec<usize>>>,
        #[validate(range(min = "1", max = "100"))]
        age: Option<Option<usize>>,
        #[validate(range(min = "1", max = "10"))]
        range: Option<usize>,
        #[validate(email)]
        email: Option<String>,
        #[validate(url)]
        url: Option<String>,
        #[validate(contains = "@")]
        text: Option<String>,
        #[validate(regex = "RE2")]
        re: Option<String>,
        #[validate(custom = "check_str")]
        custom: Option<String>,
    }

    fn check_str(_: &str) -> Result<(), ValidationError> {
        Ok(())
    }

    let s = PutStruct {
        name: Some("al".to_string()),
        address: Some(Some("gol".to_string())),
        ids: Some(vec![1, 2, 3]),
        opt_ids: Some(Some(vec![1, 2, 3])),
        age: Some(Some(20)),
        range: Some(2),
        email: Some("hi@gmail.com".to_string()),
        url: Some("http://google.com".to_string()),
        text: Some("@someone".to_string()),
        re: Some("hi".to_string()),
        custom: Some("hey".to_string()),
    };
    assert!(s.validate().is_ok());
}

#[test]
fn test_works_with_question_mark_operator() {
    fn some_fn() -> Result<(), ValidationErrors> {
        let signup = SignupData {
            mail: "invalid_email".to_string(),
            site: "http://hello.com".to_string(),
            first_name: "Bob".to_string(),
            age: 18,
        };

        signup.validate()?;
        Ok(())
    }

    assert!(some_fn().is_err());
}

#[test]
fn test_works_with_none_values() {
    #[derive(Debug, Validate)]
    struct PutStruct {
        #[validate(length(min = "1", max = "10"))]
        name: Option<String>,
        #[validate(length(min = "1", max = "10"))]
        address: Option<Option<String>>,
        #[validate(range(min = "1", max = "100"))]
        age: Option<Option<usize>>,
        #[validate(range(min = "1", max = "10"))]
        range: Option<usize>,
    }

    let p = PutStruct {
        name: None,
        address: None,
        age: None,
        range: None,
    };

    let q = PutStruct {
        name: None,
        address: Some(None),
        age: Some(None),
        range: None,
    };

    assert!(p.validate().is_ok());
    assert!(q.validate().is_ok());
}<|endoftext|>


<commit_before>use main::{WINDOW_WIDTH, WINDOW_HEIGHT, RGB};
use opencl;
use opencl::hl::{Program, Kernel};
use opencl::mem::CLBuffer;
use opencl_context::CL;
use std::borrow::Borrow;

pub struct Mandelbrot {
  pub low_x: f64,
  pub low_y: f64,
  pub width: f64,
  pub height: f64,
  pub max_iter: u32,
  pub radius: f64,

  output_buffer: CLBuffer<RGB>,
  len: usize,
  // TODO: Does this actually need to be kept around?
  program: Program,
  kernel: Kernel,
}

impl Mandelbrot {
  pub fn new(cl: &CL) -> Mandelbrot {
    let len = WINDOW_WIDTH as usize * WINDOW_HEIGHT as usize;

    let program = {
      let ker = format!("
          __kernel void color(
            const double low_x,
            const double low_y,
            const double width,
            const double height,
            const int max_iter,
            const double radius,
            __global float * output)
          {{
            int W = {};
            int H = {};

            int i = get_global_id(0);

            double c_x = i % W;
            double c_y = i / W;
            c_x = c_x * width / W + low_x;
            c_y = c_y * height / H + low_y;

            double x = 0;
            double y = 0;
            int it;
            for (it = 0; it < max_iter; ++it)
            {{
              double x2 = x * x;
              double y2 = y * y;
              if (x2 + y2 > radius * radius)
                break;
              // Ordering is important here.
              y = 2*x*y + c_y;
              x = x2 - y2 + c_x;
            }}

            i = i * 3;

            if (it < max_iter) {{
              float progress = (float)it / (float)max_iter;
              output[i] = progress;
              output[i + 1] = 1 - 2 * fabs(0.5 - progress);
              output[i + 2] = 0.5 * (1 - progress);
            }} else {{
              output[i] = 0;
              output[i + 1] = 0;
              output[i + 2] = 0;
            }}
          }}
        ", WINDOW_WIDTH, WINDOW_HEIGHT);
      cl.context.create_program_from_source(ker.borrow())
    };
    program.build(&cl.device).unwrap();

    let kernel = program.create_kernel("color");

    Mandelbrot {
      low_x: 0.0,
      low_y: 0.0,
      width: 0.0,
      height: 0.0,
      max_iter: 0,
      radius: 0.0,

      output_buffer: cl.context.create_buffer(len, opencl::cl::CL_MEM_WRITE_ONLY),
      program: program,
      kernel: kernel,
      len: len,
    }
  }

  pub fn render(&self, cl: &CL) -> Vec<RGB> {
    self.kernel.set_arg(0, &self.low_x);
    self.kernel.set_arg(1, &self.low_y);
    self.kernel.set_arg(2, &self.width);
    self.kernel.set_arg(3, &self.height);
    self.kernel.set_arg(4, &self.max_iter);
    self.kernel.set_arg(5, &self.radius);

    // This is sketchy; we "implicitly cast" output_buffer from a CLBuffer<RGB> to a CLBuffer<f32>.
    self.kernel.set_arg(6, &self.output_buffer);

    let event = cl.queue.enqueue_async_kernel(&self.kernel, self.len, None, ());
    cl.queue.get(&self.output_buffer, &event)
  }
}
<commit_msg>Don't keep around the unused Program<commit_after>use main::{WINDOW_WIDTH, WINDOW_HEIGHT, RGB};
use opencl;
use opencl::hl::{Kernel};
use opencl::mem::CLBuffer;
use opencl_context::CL;
use std::borrow::Borrow;

pub struct Mandelbrot {
  pub low_x: f64,
  pub low_y: f64,
  pub width: f64,
  pub height: f64,
  pub max_iter: u32,
  pub radius: f64,

  output_buffer: CLBuffer<RGB>,
  len: usize,
  kernel: Kernel,
}

impl Mandelbrot {
  pub fn new(cl: &CL) -> Mandelbrot {
    let len = WINDOW_WIDTH as usize * WINDOW_HEIGHT as usize;

    let program = {
      let ker = format!("
          __kernel void color(
            const double low_x,
            const double low_y,
            const double width,
            const double height,
            const int max_iter,
            const double radius,
            __global float * output)
          {{
            int W = {};
            int H = {};

            int i = get_global_id(0);

            double c_x = i % W;
            double c_y = i / W;
            c_x = c_x * width / W + low_x;
            c_y = c_y * height / H + low_y;

            double x = 0;
            double y = 0;
            int it;
            for (it = 0; it < max_iter; ++it)
            {{
              double x2 = x * x;
              double y2 = y * y;
              if (x2 + y2 > radius * radius)
                break;
              // Ordering is important here.
              y = 2*x*y + c_y;
              x = x2 - y2 + c_x;
            }}

            i = i * 3;

            if (it < max_iter) {{
              float progress = (float)it / (float)max_iter;
              output[i] = progress;
              output[i + 1] = 1 - 2 * fabs(0.5 - progress);
              output[i + 2] = 0.5 * (1 - progress);
            }} else {{
              output[i] = 0;
              output[i + 1] = 0;
              output[i + 2] = 0;
            }}
          }}
        ", WINDOW_WIDTH, WINDOW_HEIGHT);
      cl.context.create_program_from_source(ker.borrow())
    };
    program.build(&cl.device).unwrap();

    let kernel = program.create_kernel("color");

    Mandelbrot {
      low_x: 0.0,
      low_y: 0.0,
      width: 0.0,
      height: 0.0,
      max_iter: 0,
      radius: 0.0,

      output_buffer: cl.context.create_buffer(len, opencl::cl::CL_MEM_WRITE_ONLY),
      kernel: kernel,
      len: len,
    }
  }

  pub fn render(&self, cl: &CL) -> Vec<RGB> {
    self.kernel.set_arg(0, &self.low_x);
    self.kernel.set_arg(1, &self.low_y);
    self.kernel.set_arg(2, &self.width);
    self.kernel.set_arg(3, &self.height);
    self.kernel.set_arg(4, &self.max_iter);
    self.kernel.set_arg(5, &self.radius);

    // This is sketchy; we "implicitly cast" output_buffer from a CLBuffer<RGB> to a CLBuffer<f32>.
    self.kernel.set_arg(6, &self.output_buffer);

    let event = cl.queue.enqueue_async_kernel(&self.kernel, self.len, None, ());
    cl.queue.get(&self.output_buffer, &event)
  }
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::cmp;
use std::rc::Rc;
use std::cell::Cell;
use arena::TypedArena;

use errors::Result;

#[derive(Clone, Debug, PartialEq)]
/// An representation of a slice on a reference-counting `Vec<u8>`.
pub struct BytePtr {
  data: Rc<Vec<u8>>,
  start: usize,
  len: usize
}

impl BytePtr {
  pub fn new(v: Vec<u8>) -> Self {
    let len = v.len();
    Self { data: Rc::new(v), start: 0, len: len }
  }

  pub fn start(&self) -> usize {
    self.start
  }

  pub fn len(&self) -> usize {
    self.len
  }

  pub fn all(&self) -> BytePtr {
    BytePtr { data: self.data.clone(), start: self.start, len: self.len }
  }

  pub fn start_from(&self, start: usize) -> BytePtr {
    assert!(start <= self.len);
    BytePtr { data: self.data.clone(), start: self.start + start, len: self.len - start }
  }

  pub fn range(&self, start: usize, len: usize) -> BytePtr {
    assert!(start + len <= self.len);
    BytePtr { data: self.data.clone(), start: self.start + start, len: len }
  }

  pub fn slice_all(&self) -> &[u8] {
    &self.data[self.start..self.start + self.len]
  }

  pub fn slice_start_from(&self, start: usize) -> &[u8] {
    assert!(start <= self.len);
    let new_start = self.start + start;
    let new_end = self.start + self.len;
    &self.data[new_start..new_end]
  }

  pub fn slice_range(&self, start: usize, len: usize) -> &[u8] {
    assert!(start + len <= self.len);
    let new_start = self.start + start;
    &self.data[new_start..new_start + len]
  }
}

// ----------------------------------------------------------------------
// Buffer classes

/// Basic APIs for byte buffers. A byte buffer has two attributes:
/// `capacity` and `size`: the former is the total bytes allocated for
/// the buffer, while the latter is the actual bytes that have valid data.
/// Invariant: `capacity` >= `size`.
///
/// A `Buffer` is immutable, meaning that one can only obtain the
/// underlying data for read only
pub trait Buffer {
  /// Get a shared reference to the underlying data
  fn data(&self) -> &[u8];

  /// Get the capacity of this buffer
  fn capacity(&self) -> usize;

  /// Get the size for this buffer
  fn size(&self) -> usize;
}

/// A byte buffer where client can obtain a unique reference to
/// the underlying data for both read and write
pub trait MutableBuffer: Buffer {
  /// Get a unique reference to the underlying data
  fn mut_data(&mut self) -> &mut [u8];

  /// Set the internal buffer to be `new_data`, discarding the old buffer.
  fn set_data(&mut self, new_data: Vec<u8>);

  /// Adjust the internal buffer's capacity to be `new_cap`.
  /// If the current size of the buffer is smaller than `new_cap`, data
  /// will be truncated.
  fn resize(&mut self, new_cap: usize) -> Result<()>;
}

// A mutable byte buffer struct

pub struct ByteBuffer {
  data: Vec<u8>
}

impl ByteBuffer {
  pub fn new(size: usize) -> Self {
    let data = vec![0; size];
    ByteBuffer { data: data }
  }

  pub fn to_immutable(self) -> ImmutableByteBuffer {
    ImmutableByteBuffer::new(BytePtr::new(self.data))
  }
}

impl Buffer for ByteBuffer {
  fn data(&self) -> &[u8] {
    self.data.as_slice()
  }

  fn capacity(&self) -> usize {
    self.data.capacity()
  }

  fn size(&self) -> usize {
    self.data.len()
  }
}

impl MutableBuffer for ByteBuffer {
  fn mut_data(&mut self) -> &mut [u8] {
    self.data.as_mut_slice()
  }

  fn set_data(&mut self, new_data: Vec<u8>) {
    self.data = new_data;
  }

  fn resize(&mut self, new_cap: usize) -> Result<()> {
    self.data.resize(new_cap, 0);
    Ok(())
  }
}

// A immutable byte buffer struct

pub struct ImmutableByteBuffer {
  data: BytePtr
}

impl ImmutableByteBuffer {
  pub fn new(data: BytePtr) -> Self {
    Self { data: data }
  }
}

impl Buffer for ImmutableByteBuffer {
  fn data(&self) -> &[u8] {
    self.data.slice_all()
  }

  fn capacity(&self) -> usize {
    self.data.len()
  }

  fn size(&self) -> usize {
    self.data.len()
  }
}

// ----------------------------------------------------------------------
// MemoryPool classes

/// A central place for managing memory.
/// NOTE: client can only acquire bytes through this API, but not releasing.
/// All the memory will be released once the instance of this trait goes out of scope.
pub struct MemoryPool {
  arena: TypedArena<Vec<u8>>,

  // NOTE: these need to be in `Cell` since all public APIs of
  // this struct take `&self`, instead of `&mut self`. Otherwise, we cannot make the
  // lifetime of outputs to be the same as this memory pool.
  cur_bytes_allocated: Cell<i64>,
  max_bytes_allocated: Cell<i64>
}

impl MemoryPool {
  pub fn new() -> Self {
    let arena = TypedArena::new();
    Self { arena: arena, cur_bytes_allocated: Cell::new(0), max_bytes_allocated: Cell::new(0) }
  }

  /// Acquire a new byte buffer of at least `size` bytes
  /// Return a unique reference to the buffer
  pub fn acquire(&self, size: usize) -> &mut [u8] {
    let buf = vec![0; size];
    self.consume(buf)
  }

  /// Consume `buf` and add it to this memory pool
  /// After the call, `buf` has the same lifetime as the pool.
  /// Return a unique reference to the consumed buffer.
  pub fn consume(&self, data: Vec<u8>) -> &mut [u8] {
    let bytes_allocated = data.capacity();
    let result = self.arena.alloc(data);
    self.cur_bytes_allocated.set(self.cur_bytes_allocated.get() + bytes_allocated as i64);
    self.max_bytes_allocated.set(
      cmp::max(self.max_bytes_allocated.get(), self.cur_bytes_allocated.get()));
    result
  }

  /// Return the total number of bytes allocated so far
  fn cur_allocated(&self) -> i64 {
    self.cur_bytes_allocated.get()
  }

  /// Return the maximum number of bytes allocated so far
  fn max_allocated(&self) -> i64 {
    self.max_bytes_allocated.get()
  }
}
<commit_msg>Implement Display trait for BytePtr<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::cmp;
use std::fmt::{Display, Result as FmtResult, Formatter};
use std::rc::Rc;
use std::cell::Cell;
use arena::TypedArena;

use errors::Result;

#[derive(Clone, Debug, PartialEq)]
/// An representation of a slice on a reference-counting `Vec<u8>`.
pub struct BytePtr {
  data: Rc<Vec<u8>>,
  start: usize,
  len: usize
}

impl BytePtr {
  pub fn new(v: Vec<u8>) -> Self {
    let len = v.len();
    Self { data: Rc::new(v), start: 0, len: len }
  }

  pub fn start(&self) -> usize {
    self.start
  }

  pub fn len(&self) -> usize {
    self.len
  }

  pub fn all(&self) -> BytePtr {
    BytePtr { data: self.data.clone(), start: self.start, len: self.len }
  }

  pub fn start_from(&self, start: usize) -> BytePtr {
    assert!(start <= self.len);
    BytePtr { data: self.data.clone(), start: self.start + start, len: self.len - start }
  }

  pub fn range(&self, start: usize, len: usize) -> BytePtr {
    assert!(start + len <= self.len);
    BytePtr { data: self.data.clone(), start: self.start + start, len: len }
  }

  pub fn slice_all(&self) -> &[u8] {
    &self.data[self.start..self.start + self.len]
  }

  pub fn slice_start_from(&self, start: usize) -> &[u8] {
    assert!(start <= self.len);
    let new_start = self.start + start;
    let new_end = self.start + self.len;
    &self.data[new_start..new_end]
  }

  pub fn slice_range(&self, start: usize, len: usize) -> &[u8] {
    assert!(start + len <= self.len);
    let new_start = self.start + start;
    &self.data[new_start..new_start + len]
  }
}

impl Display for BytePtr {
  fn fmt(&self, f: &mut Formatter) -> FmtResult {
    write!(f, "{:?}", self.data)
  }
}

// ----------------------------------------------------------------------
// Buffer classes

/// Basic APIs for byte buffers. A byte buffer has two attributes:
/// `capacity` and `size`: the former is the total bytes allocated for
/// the buffer, while the latter is the actual bytes that have valid data.
/// Invariant: `capacity` >= `size`.
///
/// A `Buffer` is immutable, meaning that one can only obtain the
/// underlying data for read only
pub trait Buffer {
  /// Get a shared reference to the underlying data
  fn data(&self) -> &[u8];

  /// Get the capacity of this buffer
  fn capacity(&self) -> usize;

  /// Get the size for this buffer
  fn size(&self) -> usize;
}

/// A byte buffer where client can obtain a unique reference to
/// the underlying data for both read and write
pub trait MutableBuffer: Buffer {
  /// Get a unique reference to the underlying data
  fn mut_data(&mut self) -> &mut [u8];

  /// Set the internal buffer to be `new_data`, discarding the old buffer.
  fn set_data(&mut self, new_data: Vec<u8>);

  /// Adjust the internal buffer's capacity to be `new_cap`.
  /// If the current size of the buffer is smaller than `new_cap`, data
  /// will be truncated.
  fn resize(&mut self, new_cap: usize) -> Result<()>;
}

// A mutable byte buffer struct

pub struct ByteBuffer {
  data: Vec<u8>
}

impl ByteBuffer {
  pub fn new(size: usize) -> Self {
    let data = vec![0; size];
    ByteBuffer { data: data }
  }

  pub fn to_immutable(self) -> ImmutableByteBuffer {
    ImmutableByteBuffer::new(BytePtr::new(self.data))
  }
}

impl Buffer for ByteBuffer {
  fn data(&self) -> &[u8] {
    self.data.as_slice()
  }

  fn capacity(&self) -> usize {
    self.data.capacity()
  }

  fn size(&self) -> usize {
    self.data.len()
  }
}

impl MutableBuffer for ByteBuffer {
  fn mut_data(&mut self) -> &mut [u8] {
    self.data.as_mut_slice()
  }

  fn set_data(&mut self, new_data: Vec<u8>) {
    self.data = new_data;
  }

  fn resize(&mut self, new_cap: usize) -> Result<()> {
    self.data.resize(new_cap, 0);
    Ok(())
  }
}

// A immutable byte buffer struct

pub struct ImmutableByteBuffer {
  data: BytePtr
}

impl ImmutableByteBuffer {
  pub fn new(data: BytePtr) -> Self {
    Self { data: data }
  }
}

impl Buffer for ImmutableByteBuffer {
  fn data(&self) -> &[u8] {
    self.data.slice_all()
  }

  fn capacity(&self) -> usize {
    self.data.len()
  }

  fn size(&self) -> usize {
    self.data.len()
  }
}

// ----------------------------------------------------------------------
// MemoryPool classes

/// A central place for managing memory.
/// NOTE: client can only acquire bytes through this API, but not releasing.
/// All the memory will be released once the instance of this trait goes out of scope.
pub struct MemoryPool {
  arena: TypedArena<Vec<u8>>,

  // NOTE: these need to be in `Cell` since all public APIs of
  // this struct take `&self`, instead of `&mut self`. Otherwise, we cannot make the
  // lifetime of outputs to be the same as this memory pool.
  cur_bytes_allocated: Cell<i64>,
  max_bytes_allocated: Cell<i64>
}

impl MemoryPool {
  pub fn new() -> Self {
    let arena = TypedArena::new();
    Self { arena: arena, cur_bytes_allocated: Cell::new(0), max_bytes_allocated: Cell::new(0) }
  }

  /// Acquire a new byte buffer of at least `size` bytes
  /// Return a unique reference to the buffer
  pub fn acquire(&self, size: usize) -> &mut [u8] {
    let buf = vec![0; size];
    self.consume(buf)
  }

  /// Consume `buf` and add it to this memory pool
  /// After the call, `buf` has the same lifetime as the pool.
  /// Return a unique reference to the consumed buffer.
  pub fn consume(&self, data: Vec<u8>) -> &mut [u8] {
    let bytes_allocated = data.capacity();
    let result = self.arena.alloc(data);
    self.cur_bytes_allocated.set(self.cur_bytes_allocated.get() + bytes_allocated as i64);
    self.max_bytes_allocated.set(
      cmp::max(self.max_bytes_allocated.get(), self.cur_bytes_allocated.get()));
    result
  }

  /// Return the total number of bytes allocated so far
  fn cur_allocated(&self) -> i64 {
    self.cur_bytes_allocated.get()
  }

  /// Return the maximum number of bytes allocated so far
  fn max_allocated(&self) -> i64 {
    self.max_bytes_allocated.get()
  }
}
<|endoftext|>