<commit_before>#![feature(io,test)]

//! jed is creates Json iterators over instances of io.Read

extern crate rustc_serialize;
extern crate test;

use std::iter::Iterator;
use std::io::Chars;
use rustc_serialize::json::{ Json, Builder };
use std::io::Read;

/// An iterator over the Json elements of an io::Read stream
pub struct Iter<R> {
  inner: R
}

impl<R: Read> Iter<R> {
  pub fn new(inner: R) -> Iter<R> {
    Iter { inner: inner }
  }
}

impl<R: Read> Iterator for Iter<R> {
  type Item = Json;

  fn next(&mut self) -> Option<Json> {

    fn chomp<R: Read>(mut chars: Chars<R>, buf: &mut String) -> Option<Json> {
      match chars.next() {
        Some(Ok(c)) => {
          buf.push(c);
          match c {
            '}' | ']' => {
              let cbuf = buf.clone();
              match Builder::new(cbuf.chars()).build() {
                Ok(j) => Some(j),
                _ => chomp(chars, buf)
              }
            }, _ =>
              chomp(chars, buf)
          }
        }, _ => None
      }
    }
    let ref mut inner = self.inner;
    chomp(inner.chars(), &mut String::new())
  }
}

#[cfg(test)]
mod tests {
  use super::Iter;
  use test::Bencher;
  use std::io::{ empty, BufReader };

  #[test]
  fn test_not_json_iter() {
    let reader = BufReader::new("bogus".as_bytes());
    assert_eq!(Iter::new(reader).count(), 0);
  }

  #[test]
  fn test_empty_iter() {
    assert_eq!(Iter::new(empty()).count(), 0);
  }

  #[test]
  fn test_ary_iter() {
    let reader = BufReader::new("[][]".as_bytes());
    assert_eq!(Iter::new(reader).count(), 2)
  }

  #[test]
  fn test_obj_iter() {
    let reader = BufReader::new("{}{}".as_bytes());
    assert_eq!(Iter::new(reader).count(), 2)
  }

  #[bench]
  fn bench_iter(b: &mut Bencher) {
    b.iter(|| Iter::new(BufReader::new("{'foo':'bar'}{'foo':'baz'}".as_bytes())).count())
  }
}
<commit_msg>doc typo<commit_after>#![feature(io,test)]

//! jed creates Json iterators over instances of io.Read

extern crate rustc_serialize;
extern crate test;

use std::iter::Iterator;
use std::io::Chars;
use rustc_serialize::json::{ Json, Builder };
use std::io::Read;

/// An iterator over the Json elements of an io::Read stream
pub struct Iter<R> {
  inner: R
}

impl<R: Read> Iter<R> {
  pub fn new(inner: R) -> Iter<R> {
    Iter { inner: inner }
  }
}

impl<R: Read> Iterator for Iter<R> {
  type Item = Json;

  fn next(&mut self) -> Option<Json> {

    fn chomp<R: Read>(mut chars: Chars<R>, buf: &mut String) -> Option<Json> {
      match chars.next() {
        Some(Ok(c)) => {
          buf.push(c);
          match c {
            '}' | ']' => {
              let cbuf = buf.clone();
              match Builder::new(cbuf.chars()).build() {
                Ok(j) => Some(j),
                _ => chomp(chars, buf)
              }
            }, _ =>
              chomp(chars, buf)
          }
        }, _ => None
      }
    }
    let ref mut inner = self.inner;
    chomp(inner.chars(), &mut String::new())
  }
}

#[cfg(test)]
mod tests {
  use super::Iter;
  use test::Bencher;
  use std::io::{ empty, BufReader };

  #[test]
  fn test_not_json_iter() {
    let reader = BufReader::new("bogus".as_bytes());
    assert_eq!(Iter::new(reader).count(), 0);
  }

  #[test]
  fn test_empty_iter() {
    assert_eq!(Iter::new(empty()).count(), 0);
  }

  #[test]
  fn test_ary_iter() {
    let reader = BufReader::new("[][]".as_bytes());
    assert_eq!(Iter::new(reader).count(), 2)
  }

  #[test]
  fn test_obj_iter() {
    let reader = BufReader::new("{}{}".as_bytes());
    assert_eq!(Iter::new(reader).count(), 2)
  }

  #[bench]
  fn bench_iter(b: &mut Bencher) {
    b.iter(|| Iter::new(BufReader::new("{'foo':'bar'}{'foo':'baz'}".as_bytes())).count())
  }
}
<|endoftext|>


<commit_before>extern crate itertools;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::rc::Rc;

use itertools::Itertools;

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(self.id.as_bytes());
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Rc<Document>,
    pub highlighted: (usize, usize),
}

/// An Index is anything that can store Documents and return matching Documents for a query.
pub trait Index {
    /// Given a Document, stores it in the index.
    fn index(&mut self, doc: Document);
    /// Given a query, return the set of Documents that match the query.
    /// What constitutes a "match" is implementation-specific; loosely, a Document should
    /// match a query if they are somehow related. A naive implementation might simply
    /// return the set of Documents for which `query` is a substring of their content.
    fn search(&self, query: &str) -> HashSet<SearchResult>;
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
pub type InvertedIndex = BTreeMap<String, HashSet<IndexedDocument>>;

impl Index for InvertedIndex {
    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    fn index(&mut self, doc: Document) {
        let doc = Rc::new(doc);
        let analyzed = doc.content()
            .char_indices()
            .group_by(|&(_, c)| c.is_whitespace())
            .filter(|&(is_whitespace, _)| !is_whitespace)
            .flat_map(|(_, chars)| (1..chars.len() + 1).map(move |to| {
                let word: String = chars[..to].iter().map(|&(_, c)| c).collect();
                let start = chars[0].0;
                let (last_idx, last_char) = chars[to - 1];
                let finish = last_idx + last_char.len_utf8();
                (word, (start, finish))
            }));

        for (ngram, highlighted) in analyzed {
            self.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    fn search(&self, query: &str) -> HashSet<SearchResult> {
        let map = query.split_whitespace()
            .flat_map(|word| self.get(word))
            .flat_map(|docs| docs)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        map.into_iter()
            .map(|(doc, mut highlights)| {
                highlights.sort();
                SearchResult { doc: doc, highlights: highlights }
            })
        .collect()
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
pub struct SearchResult {
    doc: Rc<Document>,
    highlights: Vec<(usize, usize)>,
}

impl SearchResult {
    #[cfg(test)]
    fn new(doc: Rc<Document>, highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            doc: doc,
            highlights: highlights,
        }
    }

    pub fn doc(&self) -> &Rc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted_content(&self) -> String {
        let &SearchResult {ref doc, ref highlights} = self;
        let content = doc.content();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &(begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("today");
    let expected = [
        SearchResult::new(Rc::new(doc1), vec![(25, 30)]),
        SearchResult::new(Rc::new(doc2), vec![(13, 18)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Rc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Rc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Rc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Rc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!(expected[0].highlighted_content(), "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}
<commit_msg>Use Arc instead of Rc so it's thread safe.<commit_after>extern crate itertools;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;

use itertools::Itertools;

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(self.id.as_bytes());
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

/// An Index is anything that can store Documents and return matching Documents for a query.
pub trait Index {
    /// Given a Document, stores it in the index.
    fn index(&mut self, doc: Document);
    /// Given a query, return the set of Documents that match the query.
    /// What constitutes a "match" is implementation-specific; loosely, a Document should
    /// match a query if they are somehow related. A naive implementation might simply
    /// return the set of Documents for which `query` is a substring of their content.
    fn search(&self, query: &str) -> HashSet<SearchResult>;
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
pub type InvertedIndex = BTreeMap<String, HashSet<IndexedDocument>>;

impl Index for InvertedIndex {
    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed = doc.content()
            .char_indices()
            .group_by(|&(_, c)| c.is_whitespace())
            .filter(|&(is_whitespace, _)| !is_whitespace)
            .flat_map(|(_, chars)| (1..chars.len() + 1).map(move |to| {
                let word: String = chars[..to].iter().map(|&(_, c)| c).collect();
                let start = chars[0].0;
                let (last_idx, last_char) = chars[to - 1];
                let finish = last_idx + last_char.len_utf8();
                (word, (start, finish))
            }));

        for (ngram, highlighted) in analyzed {
            self.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    fn search(&self, query: &str) -> HashSet<SearchResult> {
        let map = query.split_whitespace()
            .flat_map(|word| self.get(word))
            .flat_map(|docs| docs)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        map.into_iter()
            .map(|(doc, mut highlights)| {
                highlights.sort();
                SearchResult { doc: doc, highlights: highlights }
            })
        .collect()
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
}

impl SearchResult {
    #[cfg(test)]
    fn new(doc: Arc<Document>, highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            doc: doc,
            highlights: highlights,
        }
    }

    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted_content(&self) -> String {
        let &SearchResult {ref doc, ref highlights} = self;
        let content = doc.content();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &(begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("today");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(25, 30)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 18)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!(expected[0].highlighted_content(), "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}
<|endoftext|>


<commit_before>//! An image processing library, based on the
//! [image](https://github.com/PistonDevelopers/image) crate.
#![deny(missing_docs)]
#![cfg_attr(test, feature(test))]

#[cfg(test)]
extern crate test;
extern crate conv;
extern crate image;
extern crate itertools;
extern crate num;
extern crate quickcheck;
extern crate rand;
extern crate rusttype;
extern crate rayon;
#[cfg(test)]
#[macro_use]
extern crate assert_approx_eq;

#[macro_use]
pub mod utils;
pub mod affine;
pub mod contrast;
pub mod corners;
pub mod definitions;
pub mod distance_transform;
pub mod drawing;
pub mod edges;
pub mod filter;
pub mod gradients;
pub mod haar;
pub mod hog;
pub mod hough;
pub mod integral_image;
pub mod local_binary_patterns;
pub mod map;
pub mod math;
pub mod morphology;
pub mod noise;
pub mod pixelops;
pub mod property_testing;
pub mod rect;
pub mod region_labelling;
pub mod seam_carving;
pub mod stats;
pub mod suppress;
pub mod template_matching;
pub mod union_find;
<commit_msg>Mention imageops module in image crate (#292)<commit_after>//! An image processing library based on the
//! [image] crate.
//!
//! Note that the image crate contains some image
//! processing functions (including image resizing) in its
//! `imageops` module, so check there if you cannot find
//! a standard image processing function in this crate.
//!
//! [image]: https://github.com/PistonDevelopers/image
#![deny(missing_docs)]
#![cfg_attr(test, feature(test))]

#[cfg(test)]
extern crate test;
extern crate conv;
extern crate image;
extern crate itertools;
extern crate num;
extern crate quickcheck;
extern crate rand;
extern crate rusttype;
extern crate rayon;
#[cfg(test)]
#[macro_use]
extern crate assert_approx_eq;

#[macro_use]
pub mod utils;
pub mod affine;
pub mod contrast;
pub mod corners;
pub mod definitions;
pub mod distance_transform;
pub mod drawing;
pub mod edges;
pub mod filter;
pub mod gradients;
pub mod haar;
pub mod hog;
pub mod hough;
pub mod integral_image;
pub mod local_binary_patterns;
pub mod map;
pub mod math;
pub mod morphology;
pub mod noise;
pub mod pixelops;
pub mod property_testing;
pub mod rect;
pub mod region_labelling;
pub mod seam_carving;
pub mod stats;
pub mod suppress;
pub mod template_matching;
pub mod union_find;
<|endoftext|>


<commit_before>//! Clustering algorithms.
//!
//! ![A cluster](http://upload.wikimedia.org/wikipedia/commons/e/e1/Cassetteandfreehub.jpg)
//!
//! This crate provides generic implementations of clustering
//! algorithms, allowing them to work with any back-end "point
//! database" that implements the required operations, e.g. one might
//! be happy with using the naive collection `BruteScan` from this
//! crate, or go all out and implement a specialised R*-tree for
//! optimised performance.
//!
//! Density-based clustering algorithms:
//!
//! - DBSCAN (`Dbscan`)
//! - OPTICS (`Optics`)
//!
//! Others:
//!
//! - *k*-means (`Kmeans`)
//!
//! [Source](https://github.com/huonw/cogset).
//!
//! # Installation
//!
//! Add the following to your `Cargo.toml` file:
//!
//! ```toml
//! [dependencies]
//! cogset = "0.1"
//! ```

#![cfg_attr(all(test, feature = "unstable"), feature(test))]
#[cfg(all(test, feature = "unstable"))] extern crate test;
#[cfg(test)] extern crate rand;

extern crate order_stat;

#[cfg(all(test, feature = "unstable"))]
#[macro_use]
mod benches;
#[cfg(not(all(test, feature = "unstable")))]
macro_rules! make_benches {
    ($($_x: tt)*) => {}
}

mod dbscan;
pub use dbscan::Dbscan;

mod optics;
pub use optics::{Optics, OpticsDbscanClustering};

mod point;
pub use point::{Point, RegionQuery, Points, ListPoints, BruteScan, BruteScanNeighbours, Euclid};

mod kmeans;

pub use kmeans::{Kmeans, KmeansBuilder};
<commit_msg>Missed version number in docs.<commit_after>//! Clustering algorithms.
//!
//! ![A cluster](http://upload.wikimedia.org/wikipedia/commons/e/e1/Cassetteandfreehub.jpg)
//!
//! This crate provides generic implementations of clustering
//! algorithms, allowing them to work with any back-end "point
//! database" that implements the required operations, e.g. one might
//! be happy with using the naive collection `BruteScan` from this
//! crate, or go all out and implement a specialised R*-tree for
//! optimised performance.
//!
//! Density-based clustering algorithms:
//!
//! - DBSCAN (`Dbscan`)
//! - OPTICS (`Optics`)
//!
//! Others:
//!
//! - *k*-means (`Kmeans`)
//!
//! [Source](https://github.com/huonw/cogset).
//!
//! # Installation
//!
//! Add the following to your `Cargo.toml` file:
//!
//! ```toml
//! [dependencies]
//! cogset = "0.2"
//! ```

#![cfg_attr(all(test, feature = "unstable"), feature(test))]
#[cfg(all(test, feature = "unstable"))] extern crate test;
#[cfg(test)] extern crate rand;

extern crate order_stat;

#[cfg(all(test, feature = "unstable"))]
#[macro_use]
mod benches;
#[cfg(not(all(test, feature = "unstable")))]
macro_rules! make_benches {
    ($($_x: tt)*) => {}
}

mod dbscan;
pub use dbscan::Dbscan;

mod optics;
pub use optics::{Optics, OpticsDbscanClustering};

mod point;
pub use point::{Point, RegionQuery, Points, ListPoints, BruteScan, BruteScanNeighbours, Euclid};

mod kmeans;

pub use kmeans::{Kmeans, KmeansBuilder};
<|endoftext|>


<commit_before>// Future TimeSteward API goals:
//
// – groups
// I made up API functions for this before, but I'm not sure they were perfect.
// We can use RowId's for group ids, but what kinds of things can be stored IN a group? Previously I said only RowId's could, which doesn't seem ideal. I think it would work to allow anything hashable. That would basically make a group behave like a HashSet. But then, why not make it behave like a HashMap instead? But accessor itself already behaves like a HashMap over RowId's – the essential thing groups do is to allow iterating particular subsets of that HashMap.
//
// – conveniences for serializing snapshots (not sure what)
//
// – be able to construct a new TimeSteward from any snapshot
// Issues: should snapshots expose the predictors? Also, the predictors can't be serialized. So you'd probably actually have to construct a TimeSteward from snapshot + predictors. This requires a way to iterate all fields (and group data if we add groups) in a snapshot, which is similar to the previous 2 items (iterating the whole set, which is a special case of iterating a subset; and serializing requires you to iterate everything as well)
//
// – parallelism support for predictors and events
// When an event or predictor gets invalidated while it is still running, it would be nice for it to save time by exiting early.
// Moreover, it would probably be more efficient to discard invalidated fields than to preserve them for predictors/events that are in process. The natural way for the accessors to handle this is to have get() return None, which would mean that you can never safely unwrap() the result. We could provide a "unwrap or return" macro.
//
// – Optimization features
// one possibility: user can provide a function FieldId->[(PredictorId, RowId)] that lists predictors you KNOW will be invalidated by a change to that field, then have that predictor run its get() calls with an input called "promise_inferred" or something so that we don't spend time and memory recording the dependency
// another: a predictor might have a costly computation to find the exact time of a future event, which it won't need to do if it gets invalidated long before that time comes. For that, we can provide a defer_until(time) method
//
//
//

#![feature(unboxed_closures, fn_traits, specialization, never_type, shared)]
// #![feature (plugin, custom_derive)]
// #![plugin (serde_macros)]
// #![plugin (quickcheck_macros)]

extern crate rand;
extern crate nalgebra;
extern crate siphasher;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate bincode;
extern crate crossbeam;
#[macro_use]
extern crate quickcheck;
#[macro_use]
extern crate derivative;

macro_rules! printlnerr(
    ($($arg:tt)*) => { {use std::io::Write;
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

/// Used by TimeSteward implementations.
///
/// These are public in case someone wants to try implementing a TimeSteward
/// outside of this crate, sharing some of the details with my own implementations.
/// However, anything inside this module should be
/// considered very unstable at present.
#[macro_use]
pub mod implementation_support {
  #[macro_use]
  pub mod list_of_types;
  pub mod insert_only;
  pub mod data_structures;
  #[macro_use]
  pub mod common;
}

/*#[macro_use]
/// Allows dynamic dispatch to object-unsafe traits, with some caveats.
pub mod dynamic {
  #[macro_use]
  pub mod list_of_types;
  pub mod thin_arc;
}*/

mod deterministic_random_id;
pub use deterministic_random_id::*;
#[macro_use]
mod api;
#[macro_use]
mod api_macros;
pub use api::*;
pub use implementation_support::list_of_types::{ColumnType, EventType, PredictorType};

#[macro_use]
pub mod stewards {
  pub mod inefficient_flat;
  pub mod memoized_flat;
  pub mod amortized;
  //pub mod optimized;

  pub mod flat_to_inefficient_full;
  pub mod crossverified;
  pub mod simply_synchronized;
}

#[macro_use]
pub mod support {
  pub mod rounding_error_tolerant_math;
  pub mod time_functions;
  pub mod collision_detection;
}

pub mod rowless {
  #[macro_use]
  pub mod implementation_support { #[macro_use] pub mod common; }
  #[macro_use]
  pub mod serialization;
  #[macro_use]
  pub mod api;
  pub mod api_impls;
  #[macro_use]
  pub mod support { #[macro_use] pub mod simple_timeline; }
  pub mod stewards { pub mod simple_flat; }
}

<commit_msg>Necessary features I forgot to commit before<commit_after>// Future TimeSteward API goals:
//
// – groups
// I made up API functions for this before, but I'm not sure they were perfect.
// We can use RowId's for group ids, but what kinds of things can be stored IN a group? Previously I said only RowId's could, which doesn't seem ideal. I think it would work to allow anything hashable. That would basically make a group behave like a HashSet. But then, why not make it behave like a HashMap instead? But accessor itself already behaves like a HashMap over RowId's – the essential thing groups do is to allow iterating particular subsets of that HashMap.
//
// – conveniences for serializing snapshots (not sure what)
//
// – be able to construct a new TimeSteward from any snapshot
// Issues: should snapshots expose the predictors? Also, the predictors can't be serialized. So you'd probably actually have to construct a TimeSteward from snapshot + predictors. This requires a way to iterate all fields (and group data if we add groups) in a snapshot, which is similar to the previous 2 items (iterating the whole set, which is a special case of iterating a subset; and serializing requires you to iterate everything as well)
//
// – parallelism support for predictors and events
// When an event or predictor gets invalidated while it is still running, it would be nice for it to save time by exiting early.
// Moreover, it would probably be more efficient to discard invalidated fields than to preserve them for predictors/events that are in process. The natural way for the accessors to handle this is to have get() return None, which would mean that you can never safely unwrap() the result. We could provide a "unwrap or return" macro.
//
// – Optimization features
// one possibility: user can provide a function FieldId->[(PredictorId, RowId)] that lists predictors you KNOW will be invalidated by a change to that field, then have that predictor run its get() calls with an input called "promise_inferred" or something so that we don't spend time and memory recording the dependency
// another: a predictor might have a costly computation to find the exact time of a future event, which it won't need to do if it gets invalidated long before that time comes. For that, we can provide a defer_until(time) method
//
//
//

#![feature(unboxed_closures, fn_traits, specialization, never_type, shared, raw, get_type_id, core_intrinsics)]
// #![feature (plugin, custom_derive)]
// #![plugin (serde_macros)]
// #![plugin (quickcheck_macros)]

extern crate rand;
extern crate nalgebra;
extern crate siphasher;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate bincode;
extern crate crossbeam;
#[macro_use]
extern crate quickcheck;
#[macro_use]
extern crate derivative;

macro_rules! printlnerr(
    ($($arg:tt)*) => { {use std::io::Write;
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

/// Used by TimeSteward implementations.
///
/// These are public in case someone wants to try implementing a TimeSteward
/// outside of this crate, sharing some of the details with my own implementations.
/// However, anything inside this module should be
/// considered very unstable at present.
#[macro_use]
pub mod implementation_support {
  #[macro_use]
  pub mod list_of_types;
  pub mod insert_only;
  pub mod data_structures;
  #[macro_use]
  pub mod common;
}

/*#[macro_use]
/// Allows dynamic dispatch to object-unsafe traits, with some caveats.
pub mod dynamic {
  #[macro_use]
  pub mod list_of_types;
  pub mod thin_arc;
}*/

mod deterministic_random_id;
pub use deterministic_random_id::*;
#[macro_use]
mod api;
#[macro_use]
mod api_macros;
pub use api::*;
pub use implementation_support::list_of_types::{ColumnType, EventType, PredictorType};

#[macro_use]
pub mod stewards {
  pub mod inefficient_flat;
  pub mod memoized_flat;
  pub mod amortized;
  //pub mod optimized;

  pub mod flat_to_inefficient_full;
  pub mod crossverified;
  pub mod simply_synchronized;
}

#[macro_use]
pub mod support {
  pub mod rounding_error_tolerant_math;
  pub mod time_functions;
  pub mod collision_detection;
}

pub mod rowless {
  #[macro_use]
  pub mod implementation_support { #[macro_use] pub mod common; }
  #[macro_use]
  pub mod serialization;
  #[macro_use]
  pub mod api;
  pub mod api_impls;
  #[macro_use]
  pub mod support { #[macro_use] pub mod simple_timeline; }
  pub mod stewards { pub mod simple_flat; }
}

<|endoftext|>


<commit_before>//! What all the kool kidz are playing these days
//!
//! # Library doc
//!
//! This library is used by `poke-a-mango` itself for all its function and is therefore contains all necessary functions.
//!
//! ## Data flow
//!
//! ```plaintext
//! Options::parse()
//! |> create_window()
//! |> Widgets::new()
//! ```
//!
//! Then, each update event:
//!
//! ```plaintext
//! |> Widgets::update()
//! |> [check for states requiring usercode handling and act accordingly]
//! ```
//!
//! ### Prose explanation
//!
//! First, get an `Options` instance, be it via a struct-literal or `Options::parse()`;
//! or don't and just create the individual arguments manually.
//!
//! Then, use `ops::load_image()`. If you know your image's format, great. If you don't, get it via `ops::guess_format()`.
//!
//! After that resize the image to an output-ready size provided by `ops::image_resized_size()` with `resize_image()`.
//! `ops::image_resized_size()` takes into consideration using two pixels per cell in the output functions,
//! so the size it returns is twice as tall as the terminal output size passed to it.
//!
//! Finally, call `ops::write_ansi()`/`ops::write_ansi_truecolor()`/`ops::write_no_ansi()` depending on your liking with the
//! resulting image.
//!
//! Or, if you want to display images manually, use `ops::create_colourtable()` to create an approximate colours table and
//! display it, for example, with `ncurses`.
//!
//! ### Example
//!
//! This is a semi-complete example, the only thing it lacks is event handling and rendering.
//!
//! ```no_run
//! # extern crate conrod;
//! # extern crate poke_a_mango;
//! # extern crate piston_window;
//! # use piston_window::{PistonWindow, UpdateEvent, Window};
//! # use poke_a_mango::*;
//! # fn main() {
//! #   not_main();
//! # }
//! # fn not_main() -> Result<(), Error> {
//! let opts = Options::parse();
//!
//! let mut window: PistonWindow = try!(ops::create_window(opts.desktop_size));
//! let mut ui = conrod::UiBuilder::new().build();
//! // Set up the UI like normal
//!
//! let mut game_state = ops::GameState::MainMenu;
//! let widgets = ops::Widgets::new(ui.widget_id_generator());
//!
//! while let Some(event) = window.next() {
//!     event.update(|_| {
//!         widgets.update(ui.set_widgets(), &mut game_state);
//!
//!         if game_state.should_exit() {
//!             window.set_should_close(true);
//!         } else if game_state.should_load_leaderboard() {
//!             game_state =
//!                 ops::GameState::DisplayLeaderboard(ops::Leader::read(&opts.config_dir.1.join("leaderboard.toml")).unwrap());
//!         } else if let ops::GameState::GameEnded { .. } = game_state {
//!             if let ops::GameState::GameEnded { ref name, score } = game_state {
//!                 ops::Leader::append(ops::Leader::now(name.clone(), score), &opts.config_dir.1.join("leaderboard.toml")).unwrap();
//!             }
//!             game_state = ops::GameState::MainMenu;
//!         }
//!     });
//!
//!     // Handle events and draw like you'd normally do
//! }
//! # Ok(())
//! # }
//! ```
//!
//! # Executable manpage
//!
//! Played with your friends or alone, this game provides great entertainment to
//! everyone all over the globe, requiring great eye-hand coordination and keeping
//! the atmosphere tense.
//!
//! Exit values and possible errors:
//!
//! ```text
//! 1 - Failed to parse a file
//! 2 - An I/O error occured
//! 3 - UI failed to cooperate
//! ```
//!
//! ## OPTIONS
//!
//! -c --config-dir <<config_dir>>
//!
//! ```text
//! Directory with the configuration.
//!
//! The configuration directory contains all of poke-a-mango's data.
//!
//! Default: $HOME/.poke-a-mango
//! ```
//!
//! -d --desktop-size <<size>>
//!
//! ```text
//! The target desktop's resolution.
//!
//! By default this is autodetected to match the primary monitor's resolution,
//! but can be overriden to scale the game window better.
//!
//! Format: NxM
//! ```
//!
//! ## EXAMPLES
//!
//! `poke-a-mango` `-d` *1280x720*
//!
//! ```text
//! Run the game as on an HD monitor.
//! ```
//!
//! `poke-a-mango` `-c` *pkmngo*
//!
//! ```text
//! Save the game data in "pkmngo" directory instead of the default one.
//! ```

extern crate rustc_serialize;
extern crate piston_window;
#[macro_use]
extern crate lazy_static;
extern crate chrono;
extern crate conrod;
extern crate glutin;
extern crate regex;
#[macro_use]
extern crate clap;
extern crate toml;
extern crate rand;

mod error;
mod options;

pub mod ops;
pub mod util;

pub use error::Error;
pub use options::Options;
<commit_msg>>html<commit_after>//! What all the kool kidz are playing these days
//!
//! # Library doc
//!
//! This library is used by `poke-a-mango` itself for all its function and is therefore contains all necessary functions.
//!
//! ## Data flow
//!
//! ```plaintext
//! Options::parse()
//! |> create_window()
//! |> Widgets::new()
//! ```
//!
//! Then, each update event:
//!
//! ```plaintext
//! |> Widgets::update()
//! |> [check for states requiring usercode handling and act accordingly]
//! ```
//!
//! ### Prose explanation
//!
//! First, get an `Options` instance, be it via a struct-literal or `Options::parse()`;
//! or don't and just create the individual arguments manually.
//!
//! Then, use `ops::load_image()`. If you know your image's format, great. If you don't, get it via `ops::guess_format()`.
//!
//! After that resize the image to an output-ready size provided by `ops::image_resized_size()` with `resize_image()`.
//! `ops::image_resized_size()` takes into consideration using two pixels per cell in the output functions,
//! so the size it returns is twice as tall as the terminal output size passed to it.
//!
//! Finally, call `ops::write_ansi()`/`ops::write_ansi_truecolor()`/`ops::write_no_ansi()` depending on your liking with the
//! resulting image.
//!
//! Or, if you want to display images manually, use `ops::create_colourtable()` to create an approximate colours table and
//! display it, for example, with `ncurses`.
//!
//! ### Example
//!
//! This is a semi-complete example, the only thing it lacks is event handling and rendering.
//!
//! ```no_run
//! # extern crate conrod;
//! # extern crate poke_a_mango;
//! # extern crate piston_window;
//! # use piston_window::{PistonWindow, UpdateEvent, Window};
//! # use poke_a_mango::*;
//! # fn main() {
//! #   not_main();
//! # }
//! # fn not_main() -> Result<(), Error> {
//! let opts = Options::parse();
//!
//! let mut window: PistonWindow = try!(ops::create_window(opts.desktop_size));
//! let mut ui = conrod::UiBuilder::new().build();
//! // Set up the UI like normal
//!
//! let mut game_state = ops::GameState::MainMenu;
//! let widgets = ops::Widgets::new(ui.widget_id_generator());
//!
//! while let Some(event) = window.next() {
//!     event.update(|_| {
//!         widgets.update(ui.set_widgets(), &mut game_state);
//!
//!         if game_state.should_exit() {
//!             window.set_should_close(true);
//!         } else if game_state.should_load_leaderboard() {
//!             game_state =
//!                 ops::GameState::DisplayLeaderboard(ops::Leader::read(&opts.config_dir.1.join("leaderboard.toml")).unwrap());
//!         } else if let ops::GameState::GameEnded { .. } = game_state {
//!             if let ops::GameState::GameEnded { ref name, score } = game_state {
//!                 ops::Leader::append(ops::Leader::now(name.clone(), score), &opts.config_dir.1.join("leaderboard.toml")).unwrap();
//!             }
//!             game_state = ops::GameState::MainMenu;
//!         }
//!     });
//!
//!     // Handle events and draw like you'd normally do
//! }
//! # Ok(())
//! # }
//! ```
//!
//! # Executable manpage
//!
//! Played with your friends or alone, this game provides great entertainment to
//! everyone all over the globe, requiring great eye-hand coordination and keeping
//! the atmosphere tense.
//!
//! Exit values and possible errors:
//!
//! ```text
//! 1 - Failed to parse a file
//! 2 - An I/O error occured
//! 3 - UI failed to cooperate
//! ```
//!
//! ## OPTIONS
//!
//! -c --config-dir <config_dir>
//!
//! ```text
//! Directory with the configuration.
//!
//! The configuration directory contains all of poke-a-mango's data.
//!
//! Default: $HOME/.poke-a-mango
//! ```
//!
//! -d --desktop-size <size>
//!
//! ```text
//! The target desktop's resolution.
//!
//! By default this is autodetected to match the primary monitor's resolution,
//! but can be overriden to scale the game window better.
//!
//! Format: NxM
//! ```
//!
//! ## EXAMPLES
//!
//! `poke-a-mango` `-d` *1280x720*
//!
//! ```text
//! Run the game as on an HD monitor.
//! ```
//!
//! `poke-a-mango` `-c` *pkmngo*
//!
//! ```text
//! Save the game data in "pkmngo" directory instead of the default one.
//! ```

extern crate rustc_serialize;
extern crate piston_window;
#[macro_use]
extern crate lazy_static;
extern crate chrono;
extern crate conrod;
extern crate glutin;
extern crate regex;
#[macro_use]
extern crate clap;
extern crate toml;
extern crate rand;

mod error;
mod options;

pub mod ops;
pub mod util;

pub use error::Error;
pub use options::Options;
<|endoftext|>


<commit_before>//! Lorem ipsum generator.
//!
//! This crate contains functions for generating pseudo-Latin lorem
//! ipsum placeholder text. The traditional lorem ipsum text start
//! like this:
//!
//! > Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
//! > eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
//! > enim ad minim veniam, quis nostrud exercitation ullamco laboris
//! > nisi ut aliquip ex ea commodo consequat. [...]
//!
//! This text is in the [`LOREM_IPSUM`] constant. Random text looking
//! like the above can be generated using the [`lipsum`] function.
//! This function allows you to generate as much text as desired and
//! each invocation will generate different text. This is done using a
//! [Markov chain] based on both the [`LOREM_IPSUM`] and
//! [`LIBER_PRIMUS`] texts. The latter constant holds the full text of
//! the first book of a work by Cicero, of which the lorem ipsum text
//! is a scrambled subset.
//!
//! The random looking text is generatd using a Markov chain of order
//! two, which simply means that the next word is based on the
//! previous two words in the input texts. The Markov chain can be
//! used with other input texts by creating an instance of
//! [`MarkovChain`] and calling its [`learn`] method.
//!
//! [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
//! [`LIBER_PRIMUS`]: constant.LIBER_PRIMUS.html
//! [`lipsum`]: fn.lipsum.html
//! [`MarkovChain`]: struct.MarkovChain.html
//! [`learn`]: struct.MarkovChain.html#method.learn
//! [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain

extern crate rand;

use std::cell::RefCell;
use std::collections::HashMap;
use rand::Rng;

/// A bigram is simply two consecutive words.
pub type Bigram<'a> = (&'a str, &'a str);

/// Simple order two Markov chain implementation.
///
/// The [Markov chain] is a chain of order two, which means that it
/// will use the previous two words (a bigram) when predicting the
/// next word. This is normally enough to generate random text that
/// looks somewhat plausible. The implementation is based on
/// [Generating arbitrary text with Markov chains in Rust][blog post].
///
/// [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain
/// [blog post]: https://blakewilliams.me/posts/generating-arbitrary-text-with-markov-chains-in-rust
pub struct MarkovChain<'a, R: Rng> {
    map: HashMap<Bigram<'a>, Vec<&'a str>>,
    keys: Vec<Bigram<'a>>,
    rng: R,
}

impl<'a> MarkovChain<'a, rand::ThreadRng> {
    /// Create a new Markov chain. It will use a default thread-local
    /// random number generator.
    pub fn new() -> MarkovChain<'a, rand::ThreadRng> {
        MarkovChain {
            map: HashMap::new(),
            keys: Vec::new(),
            rng: rand::thread_rng(),
        }
    }
}

impl<'a, R: Rng> MarkovChain<'a, R> {
    /// Create a new Markov chain that uses the given random number
    /// generator.
    pub fn new_with_rng(rng: R) -> MarkovChain<'a, R> {
        MarkovChain {
            map: HashMap::new(),
            keys: Vec::new(),
            rng: rng,
        }
    }

    /// Add new text to the Markov chain. This can be called several
    /// times to build up the chain.
    ///
    /// # Examples
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("red green blue");
    /// assert_eq!(chain.words(("red", "green")), Some(&vec!["blue"]));
    ///
    /// chain.learn("red green yellow");
    /// assert_eq!(chain.words(("red", "green")), Some(&vec!["blue", "yellow"]));
    /// ```
    pub fn learn(&mut self, sentence: &'a str) {
        let words = sentence.split_whitespace().collect::<Vec<&str>>();
        for window in words.windows(3) {
            let (a, b, c) = (window[0], window[1], window[2]);
            self.map.entry((a, b)).or_insert_with(Vec::new).push(c);
        }
        // Sync the keys with the current map.
        self.keys = self.map.keys().cloned().collect();
        self.keys.sort();
    }

    /// Get the possible words following the given bigram, or `None`
    /// if the state is invalid.
    ///
    /// # Examples
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("red green blue");
    /// assert_eq!(chain.words(("red", "green")), Some(&vec!["blue"]));
    /// assert_eq!(chain.words(("foo", "bar")), None);
    /// ```
    pub fn words(&self, state: Bigram<'a>) -> Option<&Vec<&str>> {
        self.map.get(&state)
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from a random point in the Markov chain.
    ///
    /// See [`generate_from`] if you want to control the starting
    /// point for the generated text.
    ///
    /// # Examples
    ///
    /// Generating the sounds of a grandfather clock:
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("Tick, Tock, Tick, Tock, Ding! Tick, Tock, Ding! Ding!");
    /// println!("{}", chain.generate(15));
    /// ```
    ///
    /// The output looks like this:
    ///
    /// > Ding! Tick, Tock, Tick, Tock, Ding! Ding! Tock, Ding! Tick,
    /// > Tock, Tick, Tock, Tick, Tock
    ///
    /// [`generate_from`]: struct.MarkovChain.html#method.generate_from
    pub fn generate(&mut self, n: usize) -> String {
        join_words(self.iter().take(n))
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from the given bigram.
    ///
    /// Use [`generate`] if the starting point is not important.
    ///
    /// [`generate`]: struct.MarkovChain.html#method.generate
    pub fn generate_from(&mut self, n: usize, from: Bigram<'a>) -> String {
        join_words(self.iter_from(from).take(n))
    }

    /// Make a never-ending iterator over the words in the Markov
    /// chain. The iterator starts at a random point in the chain.
    pub fn iter(&mut self) -> Words {
        let state = if self.keys.is_empty() {
            ("", "")
        } else {
            *choose(&mut self.rng, &self.keys).unwrap()
        };
        Words {
            map: &self.map,
            rng: &mut self.rng,
            keys: &self.keys,
            state: state,
        }
    }

    /// Make a never-ending iterator over the words in the Markov
    /// chain. The iterator starts at the given bigram.
    pub fn iter_from(&mut self, from: Bigram<'a>) -> Words {
        Words {
            map: &self.map,
            rng: &mut self.rng,
            keys: &self.keys,
            state: from,
        }
    }
}

pub struct Words<'a> {
    map: &'a HashMap<Bigram<'a>, Vec<&'a str>>,
    rng: &'a mut rand::Rng,
    keys: &'a Vec<Bigram<'a>>,
    state: Bigram<'a>,
}

impl<'a> Iterator for Words<'a> {
    type Item = &'a str;

    fn next(&mut self) -> Option<&'a str> {
        if self.map.is_empty() {
            return None;
        }

        let result = Some(self.state.0);

        while !self.map.contains_key(&self.state) {
            self.state = *choose(self.rng, self.keys).unwrap();
        }
        let next_words = &self.map[&self.state];
        let next = choose(self.rng, next_words).unwrap();
        self.state = (self.state.1, next);
        result
    }
}

/// Choose a random element from a slice.
///
/// Unlike `Rng::choose`, this function does not require the RNG to be
/// Sized and thus works on any random number generator.
fn choose<'a, T>(rng: &mut Rng, values: &'a [T]) -> Option<&'a T> {
    if values.is_empty() {
        None
    } else {
        let idx = (values.len() as f32 * rng.next_f32()) as usize;
        Some(&values[idx])
    }
}

fn join_words<'a, I: Iterator<Item = &'a str>>(mut words: I) -> String {
    match words.next() {
        None => String::new(),
        Some(word) => {
            let mut sentence = String::from(word);
            for word in words {
                sentence.push(' ');
                sentence.push_str(word);
            }
            sentence
        }
    }
}

/// The traditional lorem ipsum text as given in [Wikipedia]. Using
/// this text alone for a Markov chain of order two doesn't work very
/// well since each bigram (two consequtive words) is followed by just
/// one other word. In other words, the Markov chain will always
/// produce the same output and recreate the lorem ipsum text
/// precisely. However, combining it with the full text in
/// [`LIBER_PRIMUS`] works well.
///
/// [Wikipedia]: https://en.wikipedia.org/wiki/Lorem_ipsum
/// [`LIBER_PRIMUS`]: constant.LIBER_PRIMUS.html
pub const LOREM_IPSUM: &'static str = include_str!("lorem-ipsum.txt");

/// The first book in Cicero's work De finibus bonorum et malorum ("On
/// the ends of good and evil"). The lorem ipsum text in
/// [`LOREM_IPSUM`] is derived from part of this text.
///
/// [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
pub const LIBER_PRIMUS: &'static str = include_str!("liber-primus.txt");

thread_local! {
    // Markov chain generating lorem ipsum text.
    static LOREM_IPSUM_CHAIN: RefCell<MarkovChain<'static, rand::ThreadRng>> = {
        let mut chain = MarkovChain::new();
        chain.learn(LOREM_IPSUM);
        chain.learn(LIBER_PRIMUS);
        RefCell::new(chain)
    }
}

/// Generate `n` words of lorem ipsum text. The output starts with
/// "Lorem ipsum" and continues with the standard lorem ipsum text
/// from [`LOREM_IPSUM`]. The text will become random if sufficiently
/// long output is requested.
///
/// [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
pub fn lipsum(n: usize) -> String {
    LOREM_IPSUM_CHAIN.with(|cell| {
                               let mut chain = cell.borrow_mut();
                               chain.generate_from(n, ("Lorem", "ipsum"))
                           })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn starts_with_lorem_ipsum() {
        assert_eq!(&lipsum(10)[..11], "Lorem ipsum");
    }

    #[test]
    fn generate_zero_words() {
        assert_eq!(lipsum(0).split_whitespace().count(), 0);
    }

    #[test]
    fn generate_one_word() {
        assert_eq!(lipsum(1).split_whitespace().count(), 1);
    }

    #[test]
    fn generate_two_words() {
        assert_eq!(lipsum(2).split_whitespace().count(), 2);
    }

    #[test]
    fn empty_chain() {
        let mut chain = MarkovChain::new();
        assert_eq!(chain.generate(10), "");
    }

    #[test]
    fn generate_from() {
        let mut chain = MarkovChain::new();
        chain.learn("red orange yellow green blue indigo violet");
        assert_eq!(chain.generate_from(5, ("orange", "yellow")),
                   "orange yellow green blue indigo");
    }

    #[test]
    fn generate_last_bigram() {
        // The bigram "yyy zzz" will not be present in the Markov
        // chain's map, and so we will not generate "xxx yyy zzz" as
        // one would expect. The chain moves from state "xxx yyy" to
        // "yyy zzz", but sees that as invalid state and resets itsel
        // back to "xxx yyy".
        let mut chain = MarkovChain::new();
        chain.learn("xxx yyy zzz");
        // We use assert! instead of assert_ne! to support early
        // versions of Rust.
        assert!(chain.generate_from(3, ("xxx", "yyy")) != "xxx yyy zzz");
    }

    #[test]
    fn generate_from_no_panic() {
        // No panic when asked to generate a chain from a starting
        // point that doesn't exist in the chain.
        let mut chain = MarkovChain::new();
        chain.learn("foo bar baz");
        chain.generate_from(3, ("xxx", "yyy"));
    }

    #[test]
    fn chain_map() {
        let mut chain = MarkovChain::new();
        chain.learn("foo bar baz quuz");
        let map = &chain.map;

        assert_eq!(map.len(), 2);
        assert_eq!(map[&("foo", "bar")], vec!["baz"]);
        assert_eq!(map[&("bar", "baz")], vec!["quuz"]);
    }

    #[test]
    fn new_with_rng() {
        extern crate rand;
        use rand::SeedableRng;

        let rng = rand::XorShiftRng::from_seed([1, 2, 3, 4]);
        let mut chain = MarkovChain::new_with_rng(rng);
        chain.learn("foo bar x y z");
        chain.learn("foo bar a b c");

        assert_eq!(chain.generate(15), "a b b x y b x y x y x y bar x y");
    }
}
<commit_msg>LOREM_IPSUM_CHAIN: explain why we should start with LOREM_IPSUM<commit_after>//! Lorem ipsum generator.
//!
//! This crate contains functions for generating pseudo-Latin lorem
//! ipsum placeholder text. The traditional lorem ipsum text start
//! like this:
//!
//! > Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
//! > eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
//! > enim ad minim veniam, quis nostrud exercitation ullamco laboris
//! > nisi ut aliquip ex ea commodo consequat. [...]
//!
//! This text is in the [`LOREM_IPSUM`] constant. Random text looking
//! like the above can be generated using the [`lipsum`] function.
//! This function allows you to generate as much text as desired and
//! each invocation will generate different text. This is done using a
//! [Markov chain] based on both the [`LOREM_IPSUM`] and
//! [`LIBER_PRIMUS`] texts. The latter constant holds the full text of
//! the first book of a work by Cicero, of which the lorem ipsum text
//! is a scrambled subset.
//!
//! The random looking text is generatd using a Markov chain of order
//! two, which simply means that the next word is based on the
//! previous two words in the input texts. The Markov chain can be
//! used with other input texts by creating an instance of
//! [`MarkovChain`] and calling its [`learn`] method.
//!
//! [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
//! [`LIBER_PRIMUS`]: constant.LIBER_PRIMUS.html
//! [`lipsum`]: fn.lipsum.html
//! [`MarkovChain`]: struct.MarkovChain.html
//! [`learn`]: struct.MarkovChain.html#method.learn
//! [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain

extern crate rand;

use std::cell::RefCell;
use std::collections::HashMap;
use rand::Rng;

/// A bigram is simply two consecutive words.
pub type Bigram<'a> = (&'a str, &'a str);

/// Simple order two Markov chain implementation.
///
/// The [Markov chain] is a chain of order two, which means that it
/// will use the previous two words (a bigram) when predicting the
/// next word. This is normally enough to generate random text that
/// looks somewhat plausible. The implementation is based on
/// [Generating arbitrary text with Markov chains in Rust][blog post].
///
/// [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain
/// [blog post]: https://blakewilliams.me/posts/generating-arbitrary-text-with-markov-chains-in-rust
pub struct MarkovChain<'a, R: Rng> {
    map: HashMap<Bigram<'a>, Vec<&'a str>>,
    keys: Vec<Bigram<'a>>,
    rng: R,
}

impl<'a> MarkovChain<'a, rand::ThreadRng> {
    /// Create a new Markov chain. It will use a default thread-local
    /// random number generator.
    pub fn new() -> MarkovChain<'a, rand::ThreadRng> {
        MarkovChain {
            map: HashMap::new(),
            keys: Vec::new(),
            rng: rand::thread_rng(),
        }
    }
}

impl<'a, R: Rng> MarkovChain<'a, R> {
    /// Create a new Markov chain that uses the given random number
    /// generator.
    pub fn new_with_rng(rng: R) -> MarkovChain<'a, R> {
        MarkovChain {
            map: HashMap::new(),
            keys: Vec::new(),
            rng: rng,
        }
    }

    /// Add new text to the Markov chain. This can be called several
    /// times to build up the chain.
    ///
    /// # Examples
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("red green blue");
    /// assert_eq!(chain.words(("red", "green")), Some(&vec!["blue"]));
    ///
    /// chain.learn("red green yellow");
    /// assert_eq!(chain.words(("red", "green")), Some(&vec!["blue", "yellow"]));
    /// ```
    pub fn learn(&mut self, sentence: &'a str) {
        let words = sentence.split_whitespace().collect::<Vec<&str>>();
        for window in words.windows(3) {
            let (a, b, c) = (window[0], window[1], window[2]);
            self.map.entry((a, b)).or_insert_with(Vec::new).push(c);
        }
        // Sync the keys with the current map.
        self.keys = self.map.keys().cloned().collect();
        self.keys.sort();
    }

    /// Get the possible words following the given bigram, or `None`
    /// if the state is invalid.
    ///
    /// # Examples
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("red green blue");
    /// assert_eq!(chain.words(("red", "green")), Some(&vec!["blue"]));
    /// assert_eq!(chain.words(("foo", "bar")), None);
    /// ```
    pub fn words(&self, state: Bigram<'a>) -> Option<&Vec<&str>> {
        self.map.get(&state)
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from a random point in the Markov chain.
    ///
    /// See [`generate_from`] if you want to control the starting
    /// point for the generated text.
    ///
    /// # Examples
    ///
    /// Generating the sounds of a grandfather clock:
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("Tick, Tock, Tick, Tock, Ding! Tick, Tock, Ding! Ding!");
    /// println!("{}", chain.generate(15));
    /// ```
    ///
    /// The output looks like this:
    ///
    /// > Ding! Tick, Tock, Tick, Tock, Ding! Ding! Tock, Ding! Tick,
    /// > Tock, Tick, Tock, Tick, Tock
    ///
    /// [`generate_from`]: struct.MarkovChain.html#method.generate_from
    pub fn generate(&mut self, n: usize) -> String {
        join_words(self.iter().take(n))
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from the given bigram.
    ///
    /// Use [`generate`] if the starting point is not important.
    ///
    /// [`generate`]: struct.MarkovChain.html#method.generate
    pub fn generate_from(&mut self, n: usize, from: Bigram<'a>) -> String {
        join_words(self.iter_from(from).take(n))
    }

    /// Make a never-ending iterator over the words in the Markov
    /// chain. The iterator starts at a random point in the chain.
    pub fn iter(&mut self) -> Words {
        let state = if self.keys.is_empty() {
            ("", "")
        } else {
            *choose(&mut self.rng, &self.keys).unwrap()
        };
        Words {
            map: &self.map,
            rng: &mut self.rng,
            keys: &self.keys,
            state: state,
        }
    }

    /// Make a never-ending iterator over the words in the Markov
    /// chain. The iterator starts at the given bigram.
    pub fn iter_from(&mut self, from: Bigram<'a>) -> Words {
        Words {
            map: &self.map,
            rng: &mut self.rng,
            keys: &self.keys,
            state: from,
        }
    }
}

pub struct Words<'a> {
    map: &'a HashMap<Bigram<'a>, Vec<&'a str>>,
    rng: &'a mut rand::Rng,
    keys: &'a Vec<Bigram<'a>>,
    state: Bigram<'a>,
}

impl<'a> Iterator for Words<'a> {
    type Item = &'a str;

    fn next(&mut self) -> Option<&'a str> {
        if self.map.is_empty() {
            return None;
        }

        let result = Some(self.state.0);

        while !self.map.contains_key(&self.state) {
            self.state = *choose(self.rng, self.keys).unwrap();
        }
        let next_words = &self.map[&self.state];
        let next = choose(self.rng, next_words).unwrap();
        self.state = (self.state.1, next);
        result
    }
}

/// Choose a random element from a slice.
///
/// Unlike `Rng::choose`, this function does not require the RNG to be
/// Sized and thus works on any random number generator.
fn choose<'a, T>(rng: &mut Rng, values: &'a [T]) -> Option<&'a T> {
    if values.is_empty() {
        None
    } else {
        let idx = (values.len() as f32 * rng.next_f32()) as usize;
        Some(&values[idx])
    }
}

fn join_words<'a, I: Iterator<Item = &'a str>>(mut words: I) -> String {
    match words.next() {
        None => String::new(),
        Some(word) => {
            let mut sentence = String::from(word);
            for word in words {
                sentence.push(' ');
                sentence.push_str(word);
            }
            sentence
        }
    }
}

/// The traditional lorem ipsum text as given in [Wikipedia]. Using
/// this text alone for a Markov chain of order two doesn't work very
/// well since each bigram (two consequtive words) is followed by just
/// one other word. In other words, the Markov chain will always
/// produce the same output and recreate the lorem ipsum text
/// precisely. However, combining it with the full text in
/// [`LIBER_PRIMUS`] works well.
///
/// [Wikipedia]: https://en.wikipedia.org/wiki/Lorem_ipsum
/// [`LIBER_PRIMUS`]: constant.LIBER_PRIMUS.html
pub const LOREM_IPSUM: &'static str = include_str!("lorem-ipsum.txt");

/// The first book in Cicero's work De finibus bonorum et malorum ("On
/// the ends of good and evil"). The lorem ipsum text in
/// [`LOREM_IPSUM`] is derived from part of this text.
///
/// [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
pub const LIBER_PRIMUS: &'static str = include_str!("liber-primus.txt");

thread_local! {
    // Markov chain generating lorem ipsum text.
    static LOREM_IPSUM_CHAIN: RefCell<MarkovChain<'static, rand::ThreadRng>> = {
        let mut chain = MarkovChain::new();
        // The cost of learning increases as more and more text is
        // added, so we start with the smallest text.
        chain.learn(LOREM_IPSUM);
        chain.learn(LIBER_PRIMUS);
        RefCell::new(chain)
    }
}

/// Generate `n` words of lorem ipsum text. The output starts with
/// "Lorem ipsum" and continues with the standard lorem ipsum text
/// from [`LOREM_IPSUM`]. The text will become random if sufficiently
/// long output is requested.
///
/// [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
pub fn lipsum(n: usize) -> String {
    LOREM_IPSUM_CHAIN.with(|cell| {
                               let mut chain = cell.borrow_mut();
                               chain.generate_from(n, ("Lorem", "ipsum"))
                           })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn starts_with_lorem_ipsum() {
        assert_eq!(&lipsum(10)[..11], "Lorem ipsum");
    }

    #[test]
    fn generate_zero_words() {
        assert_eq!(lipsum(0).split_whitespace().count(), 0);
    }

    #[test]
    fn generate_one_word() {
        assert_eq!(lipsum(1).split_whitespace().count(), 1);
    }

    #[test]
    fn generate_two_words() {
        assert_eq!(lipsum(2).split_whitespace().count(), 2);
    }

    #[test]
    fn empty_chain() {
        let mut chain = MarkovChain::new();
        assert_eq!(chain.generate(10), "");
    }

    #[test]
    fn generate_from() {
        let mut chain = MarkovChain::new();
        chain.learn("red orange yellow green blue indigo violet");
        assert_eq!(chain.generate_from(5, ("orange", "yellow")),
                   "orange yellow green blue indigo");
    }

    #[test]
    fn generate_last_bigram() {
        // The bigram "yyy zzz" will not be present in the Markov
        // chain's map, and so we will not generate "xxx yyy zzz" as
        // one would expect. The chain moves from state "xxx yyy" to
        // "yyy zzz", but sees that as invalid state and resets itsel
        // back to "xxx yyy".
        let mut chain = MarkovChain::new();
        chain.learn("xxx yyy zzz");
        // We use assert! instead of assert_ne! to support early
        // versions of Rust.
        assert!(chain.generate_from(3, ("xxx", "yyy")) != "xxx yyy zzz");
    }

    #[test]
    fn generate_from_no_panic() {
        // No panic when asked to generate a chain from a starting
        // point that doesn't exist in the chain.
        let mut chain = MarkovChain::new();
        chain.learn("foo bar baz");
        chain.generate_from(3, ("xxx", "yyy"));
    }

    #[test]
    fn chain_map() {
        let mut chain = MarkovChain::new();
        chain.learn("foo bar baz quuz");
        let map = &chain.map;

        assert_eq!(map.len(), 2);
        assert_eq!(map[&("foo", "bar")], vec!["baz"]);
        assert_eq!(map[&("bar", "baz")], vec!["quuz"]);
    }

    #[test]
    fn new_with_rng() {
        extern crate rand;
        use rand::SeedableRng;

        let rng = rand::XorShiftRng::from_seed([1, 2, 3, 4]);
        let mut chain = MarkovChain::new_with_rng(rng);
        chain.learn("foo bar x y z");
        chain.learn("foo bar a b c");

        assert_eq!(chain.generate(15), "a b b x y b x y x y x y bar x y");
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 Ilkka Rauta
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! BitReader is a helper type to extract strings of bits from a slice of bytes.
//!
//! Here is how you read first a single bit, then three bits and finally four bits from a byte
//! buffer:
//!
//! ```
//! use bitreader::BitReader;
//!
//! let slice_of_u8 = &[0b1000_1111];
//! let mut reader = BitReader::new(slice_of_u8);
//!
//! // You probably should use try! or some other error handling mechanism in real code if the
//! // length of the input is not known in advance.
//! let a_single_bit = reader.read_u8(1).unwrap(); // 1
//! let more_bits = reader.read_u8(3).unwrap(); // 0
//! let last_bits_of_byte = reader.read_u8(4).unwrap(); // 0b1111
//! ```
//! You can naturally read bits from longer buffer of data than just a single byte.
//!
//! As you read bits, the internal cursor of BitReader moves on along the stream of bits. Little
//! endian format is assumed when reading the multi-byte values. BitReader supports reading maximum
//! of 64 bits at a time (with read_u64). Reading signed values directly is not supported at the
//! moment.
//!
//! The reads do not need to be aligned in any particular way.
//!
//! Reading zero bits is a no-op.
//!
//! You can also skip over a number of bits, in which case there is no arbitrary small limits like
//! when reading the values to a variable. However, you can not read past the end of the slice.
//!
//! Note that the code will likely not work correctly if the slice is longer than 2^61 bytes, but
//! exceeding that should be pretty unlikely.

use std::fmt;
use std::error::Error;
use std::result;

/// BitReader reads data from a byte slice at the granularity of a single bit.
pub struct BitReader<'a> {
    bytes: &'a [u8],
    /// Position from the start of the slice, counted as bits instead of bytes
    position: u64,
}

impl<'a> BitReader<'a> {
    /// Construct a new BitReader from a byte slice. The returned reader lives at most as long as
    /// the slice given to is valid.
    pub fn new(bytes: &'a [u8]) -> BitReader<'a> {
        BitReader {
            bytes: bytes,
            position: 0
        }
    }

    /// Read at most 8 bits into a u8.
    pub fn read_u8(&mut self, bit_count: u8) -> Result<u8> {
        let value = try!(self.read_value(bit_count, 8));
        Ok((value & 0xff) as u8)
    }

    /// Read at most 16 bits into a u16.
    pub fn read_u16(&mut self, bit_count: u8) -> Result<u16> {
        let value = try!(self.read_value(bit_count, 16));
        Ok((value & 0xffff) as u16)
    }

    /// Read at most 32 bits into a u32.
    pub fn read_u32(&mut self, bit_count: u8) -> Result<u32> {
        let value = try!(self.read_value(bit_count, 32));
        Ok((value & 0xffffffff) as u32)
    }

    /// Read at most 64 bits into a u64.
    pub fn read_u64(&mut self, bit_count: u8) -> Result<u64> {
        let value = try!(self.read_value(bit_count, 64));
        Ok(value)
    }

    /// Skip arbitrary number of bits. However, you can skip at most to the end of the byte slice.
    pub fn skip(&mut self, bit_count: u32) -> Result<()> {
        let end_position = self.position + bit_count as u64;
        if end_position > self.bytes.len() as u64 * 8 {
            return Err(BitReaderError::NotEnoughData);
        }
        self.position = end_position;
        Ok(())
    }

    fn read_value(&mut self, bit_count: u8, maximum_count: u8) -> Result<u64> {
        if bit_count == 0 {
            return Ok(0);
        }
        if bit_count > maximum_count {
            return Err(BitReaderError::TooManyBitsForType);
        }
        let start_position = self.position;
        let end_position = self.position + bit_count as u64;
        if end_position > self.bytes.len() as u64 * 8 {
            return Err(BitReaderError::NotEnoughData);
        }

        let mut value: u64 = 0;

        for i in start_position..end_position {
            let byte_index = (i / 8) as usize;
            let byte = self.bytes[byte_index];
            let shift = 7 - (i % 8);
            let bit = (byte >> shift) as u64 & 1;
            value = (value << 1) | bit;
        }

        self.position = end_position;
        Ok(value)
    }
}

/// Result type for those BitReader operations that can fail.
pub type Result<T> = result::Result<T, BitReaderError>;

/// Error enumeration of BitReader errors.
#[derive(Debug,PartialEq)]
pub enum BitReaderError {
    /// Requested more bits than there are left in the byte slice at the current position.
    NotEnoughData,
    /// Requested more bits than the returned variable can hold, for example more than 8 bits when
    /// reading into a u8.
    TooManyBitsForType
}

impl Error for BitReaderError {
    fn description(&self) -> &str {
        match *self {
            BitReaderError::NotEnoughData => "Requested more bits than the byte slice has left",
            BitReaderError::TooManyBitsForType => "Requested more bits than the requested integer type can hold",
        }
    }
}

impl fmt::Display for BitReaderError {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        self.description().fmt(fmt)
    }
}

#[test]
fn read_buffer() {
    let bytes = &[
        0b1011_0101, 0b0110_1010, 0b1010_1100, 0b1001_1001,
        0b1001_1001, 0b1001_1001, 0b1001_1001, 0b1110_0111,
    ];

    let mut reader = BitReader::new(bytes);

    assert_eq!(reader.read_u8(1).unwrap(), 0b1);
    assert_eq!(reader.read_u8(1).unwrap(), 0b0);
    assert_eq!(reader.read_u8(2).unwrap(), 0b11);

    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);

    assert_eq!(reader.read_u8(3).unwrap(), 0b11);
    assert_eq!(reader.read_u16(10).unwrap(), 0b01_0101_0101);
    assert_eq!(reader.read_u8(3).unwrap(), 0b100);

    assert_eq!(reader.read_u32(32).unwrap(), 0b1001_1001_1001_1001_1001_1001_1001_1001);

    assert_eq!(reader.read_u8(4).unwrap(), 0b1110);
    assert_eq!(reader.read_u8(3).unwrap(), 0b011);
    assert_eq!(reader.read_u8(1).unwrap(), 0b1);
}

#[test]
fn try_all_sizes() {
    let bytes = &[
        0x4a, 0x1e, 0x39, 0xbb, 0xd0, 0x07, 0xca, 0x9a,
        0xa6, 0xba, 0x25, 0x52, 0x6f, 0x0a, 0x6a, 0xba,
    ];

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u64(64).unwrap(), 0x4a1e39bbd007ca9a);
    assert_eq!(reader.read_u64(64).unwrap(), 0xa6ba25526f0a6aba);

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u32(32).unwrap(), 0x4a1e39bb);
    assert_eq!(reader.read_u32(32).unwrap(), 0xd007ca9a);
    assert_eq!(reader.read_u32(32).unwrap(), 0xa6ba2552);
    assert_eq!(reader.read_u32(32).unwrap(), 0x6f0a6aba);

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u16(16).unwrap(), 0x4a1e);
    assert_eq!(reader.read_u16(16).unwrap(), 0x39bb);
    assert_eq!(reader.read_u16(16).unwrap(), 0xd007);
    assert_eq!(reader.read_u16(16).unwrap(), 0xca9a);
    assert_eq!(reader.read_u16(16).unwrap(), 0xa6ba);
    assert_eq!(reader.read_u16(16).unwrap(), 0x2552);
    assert_eq!(reader.read_u16(16).unwrap(), 0x6f0a);
    assert_eq!(reader.read_u16(16).unwrap(), 0x6aba);

    let mut reader = BitReader::new(&bytes[..]);
    for byte in bytes {
        assert_eq!(reader.read_u8(8).unwrap(), *byte);
    }
}

#[test]
fn skipping_and_zero_reads() {
    let bytes = &[
        0b1011_0101, 0b1110_1010, 0b1010_1100,
    ];

    let mut reader = BitReader::new(bytes);

    assert_eq!(reader.read_u8(4).unwrap(), 0b1011);
    // Reading zero bits should be a no-op
    assert_eq!(reader.read_u8(0).unwrap(), 0b0);
    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);
    reader.skip(3).unwrap(); // 0b111
    assert_eq!(reader.read_u16(10).unwrap(), 0b0101010101);
    assert_eq!(reader.read_u8(3).unwrap(), 0b100);
}

#[test]
fn errors() {
    let bytes = &[
        0b1011_0101, 0b1110_1010, 0b1010_1100,
    ];

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u8(4).unwrap(), 0b1011);
    assert_eq!(reader.read_u8(9).unwrap_err(), BitReaderError::TooManyBitsForType);
    // If an error happens, it should be possible to resume as if nothing had happened
    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u8(4).unwrap(), 0b1011);
    // Same with this error
    assert_eq!(reader.read_u32(21).unwrap_err(), BitReaderError::NotEnoughData);
    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);
}
<commit_msg>Update documentation.<commit_after>// Copyright 2015 Ilkka Rauta
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! BitReader is a helper type to extract strings of bits from a slice of bytes.
//!
//! Here is how you read first a single bit, then three bits and finally four bits from a byte
//! buffer:
//!
//! ```
//! use bitreader::BitReader;
//!
//! let slice_of_u8 = &[0b1000_1111];
//! let mut reader = BitReader::new(slice_of_u8);
//!
//! // You probably should use try! or some other error handling mechanism in real code if the
//! // length of the input is not known in advance.
//! let a_single_bit = reader.read_u8(1).unwrap(); // 1
//! let more_bits = reader.read_u8(3).unwrap(); // 0
//! let last_bits_of_byte = reader.read_u8(4).unwrap(); // 0b1111
//! ```
//! You can naturally read bits from longer buffer of data than just a single byte.
//!
//! As you read bits, the internal cursor of BitReader moves on along the stream of bits. Little
//! endian format is assumed when reading the multi-byte values. BitReader supports reading maximum
//! of 64 bits at a time (with read_u64). Reading signed values directly is not supported at the
//! moment.
//!
//! The reads do not need to be aligned in any particular way.
//!
//! Reading zero bits is a no-op.
//!
//! You can also skip over a number of bits, in which case there is no arbitrary small limits like
//! when reading the values to a variable. However, you can not seek past the end of the slice,
//! either when reading or when skipping bits.
//!
//! Note that the code will likely not work correctly if the slice is longer than 2^61 bytes, but
//! exceeding that should be pretty unlikely. Let's get back to this when people read exabytes of
//! information one bit at a time.

use std::fmt;
use std::error::Error;
use std::result;

/// BitReader reads data from a byte slice at the granularity of a single bit.
pub struct BitReader<'a> {
    bytes: &'a [u8],
    /// Position from the start of the slice, counted as bits instead of bytes
    position: u64,
}

impl<'a> BitReader<'a> {
    /// Construct a new BitReader from a byte slice. The returned reader lives at most as long as
    /// the slice given to is valid.
    pub fn new(bytes: &'a [u8]) -> BitReader<'a> {
        BitReader {
            bytes: bytes,
            position: 0
        }
    }

    /// Read at most 8 bits into a u8.
    pub fn read_u8(&mut self, bit_count: u8) -> Result<u8> {
        let value = try!(self.read_value(bit_count, 8));
        Ok((value & 0xff) as u8)
    }

    /// Read at most 16 bits into a u16.
    pub fn read_u16(&mut self, bit_count: u8) -> Result<u16> {
        let value = try!(self.read_value(bit_count, 16));
        Ok((value & 0xffff) as u16)
    }

    /// Read at most 32 bits into a u32.
    pub fn read_u32(&mut self, bit_count: u8) -> Result<u32> {
        let value = try!(self.read_value(bit_count, 32));
        Ok((value & 0xffffffff) as u32)
    }

    /// Read at most 64 bits into a u64.
    pub fn read_u64(&mut self, bit_count: u8) -> Result<u64> {
        let value = try!(self.read_value(bit_count, 64));
        Ok(value)
    }

    /// Skip arbitrary number of bits. However, you can skip at most to the end of the byte slice.
    pub fn skip(&mut self, bit_count: u32) -> Result<()> {
        let end_position = self.position + bit_count as u64;
        if end_position > self.bytes.len() as u64 * 8 {
            return Err(BitReaderError::NotEnoughData);
        }
        self.position = end_position;
        Ok(())
    }

    fn read_value(&mut self, bit_count: u8, maximum_count: u8) -> Result<u64> {
        if bit_count == 0 {
            return Ok(0);
        }
        if bit_count > maximum_count {
            return Err(BitReaderError::TooManyBitsForType);
        }
        let start_position = self.position;
        let end_position = self.position + bit_count as u64;
        if end_position > self.bytes.len() as u64 * 8 {
            return Err(BitReaderError::NotEnoughData);
        }

        let mut value: u64 = 0;

        for i in start_position..end_position {
            let byte_index = (i / 8) as usize;
            let byte = self.bytes[byte_index];
            let shift = 7 - (i % 8);
            let bit = (byte >> shift) as u64 & 1;
            value = (value << 1) | bit;
        }

        self.position = end_position;
        Ok(value)
    }
}

/// Result type for those BitReader operations that can fail.
pub type Result<T> = result::Result<T, BitReaderError>;

/// Error enumeration of BitReader errors.
#[derive(Debug,PartialEq)]
pub enum BitReaderError {
    /// Requested more bits than there are left in the byte slice at the current position.
    NotEnoughData,
    /// Requested more bits than the returned variable can hold, for example more than 8 bits when
    /// reading into a u8.
    TooManyBitsForType
}

impl Error for BitReaderError {
    fn description(&self) -> &str {
        match *self {
            BitReaderError::NotEnoughData => "Requested more bits than the byte slice has left",
            BitReaderError::TooManyBitsForType => "Requested more bits than the requested integer type can hold",
        }
    }
}

impl fmt::Display for BitReaderError {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        self.description().fmt(fmt)
    }
}

#[test]
fn read_buffer() {
    let bytes = &[
        0b1011_0101, 0b0110_1010, 0b1010_1100, 0b1001_1001,
        0b1001_1001, 0b1001_1001, 0b1001_1001, 0b1110_0111,
    ];

    let mut reader = BitReader::new(bytes);

    assert_eq!(reader.read_u8(1).unwrap(), 0b1);
    assert_eq!(reader.read_u8(1).unwrap(), 0b0);
    assert_eq!(reader.read_u8(2).unwrap(), 0b11);

    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);

    assert_eq!(reader.read_u8(3).unwrap(), 0b11);
    assert_eq!(reader.read_u16(10).unwrap(), 0b01_0101_0101);
    assert_eq!(reader.read_u8(3).unwrap(), 0b100);

    assert_eq!(reader.read_u32(32).unwrap(), 0b1001_1001_1001_1001_1001_1001_1001_1001);

    assert_eq!(reader.read_u8(4).unwrap(), 0b1110);
    assert_eq!(reader.read_u8(3).unwrap(), 0b011);
    assert_eq!(reader.read_u8(1).unwrap(), 0b1);
}

#[test]
fn try_all_sizes() {
    let bytes = &[
        0x4a, 0x1e, 0x39, 0xbb, 0xd0, 0x07, 0xca, 0x9a,
        0xa6, 0xba, 0x25, 0x52, 0x6f, 0x0a, 0x6a, 0xba,
    ];

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u64(64).unwrap(), 0x4a1e39bbd007ca9a);
    assert_eq!(reader.read_u64(64).unwrap(), 0xa6ba25526f0a6aba);

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u32(32).unwrap(), 0x4a1e39bb);
    assert_eq!(reader.read_u32(32).unwrap(), 0xd007ca9a);
    assert_eq!(reader.read_u32(32).unwrap(), 0xa6ba2552);
    assert_eq!(reader.read_u32(32).unwrap(), 0x6f0a6aba);

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u16(16).unwrap(), 0x4a1e);
    assert_eq!(reader.read_u16(16).unwrap(), 0x39bb);
    assert_eq!(reader.read_u16(16).unwrap(), 0xd007);
    assert_eq!(reader.read_u16(16).unwrap(), 0xca9a);
    assert_eq!(reader.read_u16(16).unwrap(), 0xa6ba);
    assert_eq!(reader.read_u16(16).unwrap(), 0x2552);
    assert_eq!(reader.read_u16(16).unwrap(), 0x6f0a);
    assert_eq!(reader.read_u16(16).unwrap(), 0x6aba);

    let mut reader = BitReader::new(&bytes[..]);
    for byte in bytes {
        assert_eq!(reader.read_u8(8).unwrap(), *byte);
    }
}

#[test]
fn skipping_and_zero_reads() {
    let bytes = &[
        0b1011_0101, 0b1110_1010, 0b1010_1100,
    ];

    let mut reader = BitReader::new(bytes);

    assert_eq!(reader.read_u8(4).unwrap(), 0b1011);
    // Reading zero bits should be a no-op
    assert_eq!(reader.read_u8(0).unwrap(), 0b0);
    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);
    reader.skip(3).unwrap(); // 0b111
    assert_eq!(reader.read_u16(10).unwrap(), 0b0101010101);
    assert_eq!(reader.read_u8(3).unwrap(), 0b100);
}

#[test]
fn errors() {
    let bytes = &[
        0b1011_0101, 0b1110_1010, 0b1010_1100,
    ];

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u8(4).unwrap(), 0b1011);
    assert_eq!(reader.read_u8(9).unwrap_err(), BitReaderError::TooManyBitsForType);
    // If an error happens, it should be possible to resume as if nothing had happened
    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);

    let mut reader = BitReader::new(bytes);
    assert_eq!(reader.read_u8(4).unwrap(), 0b1011);
    // Same with this error
    assert_eq!(reader.read_u32(21).unwrap_err(), BitReaderError::NotEnoughData);
    assert_eq!(reader.read_u8(4).unwrap(), 0b0101);
}
<|endoftext|>