<commit_before>use regex::Regex;
use parser::{Span, Text, Emphasis, Strong, Code, Link, Image};

static SPANS : Regex = regex!(r"(!?\[.*\]\([^\(\)]*\))|(\*[^\*].+?\*)|(\*\*.+?\*\*)|(_[^_].+?_)|(__.+?__)|(`[^`].+?`)|(``.+?``)");
static LINK  : Regex = regex!("\\[(?P<text>.*)\\]\\((?P<url>.*?)(?:\\s\"(?P<title>.*?)\")?\\)");
static IMAGE  : Regex = regex!("!\\[(?P<text>.*)\\]\\((?P<url>.*?)(?:\\s\"(?P<title>.*?)\")?\\)");
static EMPHASIS_UNDERSCORE  : Regex = regex!(r"^_(?P<text>[^_].+?)_");
static EMPHASIS_STAR  : Regex = regex!(r"^\*(?P<text>[^\*].+?)\*");

pub fn parse_spans(text : &str) -> Vec<Span>{
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in SPANS.find_iter(text) {
        tokens.push(Text(text.slice(current, begin)));
        tokens.push(parse_span(text.slice(begin, end)));
        current = end;
    }
    tokens.push(Text(text.slice(current, text.len())));
    tokens
}

fn parse_span(text : &str) -> Span{
    if IMAGE.is_match(text){
        let caps = IMAGE.captures(text).unwrap();
        return Image(
            caps.name("text"),
            caps.name("url"),
            caps.name("title")
            );
    }else if LINK.is_match(text){
        let caps = LINK.captures(text).unwrap();
        return Link(
            caps.name("text"),
            caps.name("url"),
            caps.name("title")
            );
    }else if EMPHASIS_UNDERSCORE.is_match(text){
        let caps = EMPHASIS_UNDERSCORE.captures(text).unwrap();
        return Emphasis(parse_spans(caps.name("text")));
    }else if EMPHASIS_STAR.is_match(text){
        let caps = EMPHASIS_STAR.captures(text).unwrap();
        return Emphasis(parse_spans(caps.name("text")));
    }
    return Text(text);
}

// HERE BE TEST

#[test]
fn parse_link_test() {
    assert_eq!(parse_span("[an example](example.com)"),
                Link("an example", "example.com", ""));

    assert_eq!(parse_span("[an example](example.com \"Title\")"),
                Link("an example", "example.com", "Title"));
}

#[test]
fn parse_image_test() {
    assert_eq!(parse_span("![an example](example.com)"),
                Image("an example", "example.com", ""));
    assert_eq!(parse_span("![an example](example.com \"Title\")"),
                Image("an example", "example.com", "Title"));
}

#[test]
fn parse_emphasis_test() {

    // simple text behaviour
    assert_eq!(parse_span("_whatever_"), Emphasis(vec![Text("whatever")]));
    assert_eq!(parse_span("*whatever*"), Emphasis(vec![Text("whatever")]));

    // multiple words
    assert_eq!(
        parse_span("_markdown is better than nothing_"),
        Emphasis(vec![Text("markdown is better than nothing")])
              );
    assert_eq!(
        parse_span("*markdown is better than nothing*"),
        Emphasis(vec![Text("markdown is better than nothing")])
              );

    // multiple words
    assert_eq!(
        parse_span("_[an example](example.com) is better than nothing_"),
        Emphasis(vec![
                 Link("an example", "example.com", ""),
                 Text(" is better than nothing")
                 ])
              );

    assert_eq!(
        parse_span("*[an example](example.com) is better than nothing*"),
        Emphasis(vec![
                 Link("an example", "example.com", ""),
                 Text(" is better than nothing")
                 ])
              );

    // does not compile strong
    match parse_span("__whatever__"){
      Emphasis(_) => fail!(),
      _ => {}
    }
    match parse_span("**whatever**"){
      Emphasis(_) => fail!(),
      _ => {}
    }
}

//#[test]
//#[ignore]
//fn parse_strong_test() {
    //match parse_span("__whatever__"){
      //Strong("whatever") => {},
      //_ => fail!()
    //}
    //match parse_span("__what_ever__"){
      //Strong("what_ever") => {},
      //_ => fail!()
    //}
    //match parse_span("_whatever_"){
      //Strong("whatever") => fail!(),
      //_ => {}
    //}
    //match parse_span("**whatever**"){
      //Strong("whatever") => {},
      //_ => fail!()
    //}
    //match parse_span("**what*ever**"){
      //Strong("what*ever") => {},
      //_ => fail!()
    //}
    //match parse_span("*whatever*"){
      //Strong("whatever") => fail!(),
      //_ => {}
    //}
//}

<commit_msg>fix emphasis parsing<commit_after>use regex::Regex;
use parser::{Span, Text, Emphasis, Strong, Code, Link, Image};

static SPANS : Regex = regex!(r"(!?\[.*\]\([^\(\)]*\))|(\*[^\*].+?\*)|(\*\*.+?\*\*)|(_[^_].+?_)|(__.+?__)|(`[^`].+?`)|(``.+?``)");
static LINK  : Regex = regex!("\\[(?P<text>.*)\\]\\((?P<url>.*?)(?:\\s\"(?P<title>.*?)\")?\\)");
static IMAGE  : Regex = regex!("!\\[(?P<text>.*)\\]\\((?P<url>.*?)(?:\\s\"(?P<title>.*?)\")?\\)");
static EMPHASIS_UNDERSCORE  : Regex = regex!(r"^_(?P<text>[^_].+?)_");
static EMPHASIS_STAR  : Regex = regex!(r"^\*(?P<text>[^\*].+?)\*");

pub fn parse_spans(text : &str) -> Vec<Span>{
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in SPANS.find_iter(text) {
        match text.slice(current, begin){
            "" => {}
            t => tokens.push(Text(t))
        };
        tokens.push(parse_span(text.slice(begin, end)));
        current = end;
    }
    tokens.push(Text(text.slice(current, text.len())));
    tokens
}

fn parse_span(text : &str) -> Span{
    if EMPHASIS_UNDERSCORE.is_match(text){
        let caps = EMPHASIS_UNDERSCORE.captures(text).unwrap();
        return Emphasis(parse_spans(caps.name("text")));
    }else if EMPHASIS_STAR.is_match(text){
        let caps = EMPHASIS_STAR.captures(text).unwrap();
        return Emphasis(parse_spans(caps.name("text")));
    }else if IMAGE.is_match(text){
        let caps = IMAGE.captures(text).unwrap();
        return Image(
            caps.name("text"),
            caps.name("url"),
            caps.name("title")
            );
    }else if LINK.is_match(text){
        let caps = LINK.captures(text).unwrap();
        return Link(
            caps.name("text"),
            caps.name("url"),
            caps.name("title")
            );
    }
    return Text(text);
}

// HERE BE TEST

#[test]
fn parse_link_test() {
    assert_eq!(parse_span("[an example](example.com)"),
                Link("an example", "example.com", ""));

    assert_eq!(parse_span("[an example](example.com \"Title\")"),
                Link("an example", "example.com", "Title"));
}

#[test]
fn parse_image_test() {
    assert_eq!(parse_span("![an example](example.com)"),
                Image("an example", "example.com", ""));
    assert_eq!(parse_span("![an example](example.com \"Title\")"),
                Image("an example", "example.com", "Title"));
}

#[test]
fn parse_emphasis_test() {

    assert_eq!(parse_span("_whatever_"), Emphasis(vec![Text("whatever")]));
    assert_eq!(parse_span("*whatever*"), Emphasis(vec![Text("whatever")]));

    assert_eq!(
        parse_span("_markdown is better than nothing_"),
        Emphasis(vec![Text("markdown is better than nothing")])
              );
    assert_eq!(
        parse_span("*markdown is better than nothing*"),
        Emphasis(vec![Text("markdown is better than nothing")])
              );

    assert_eq!(
        parse_span("_[an example](example.com) is better than nothing_"),
        Emphasis(vec![
                 Link("an example", "example.com", ""),
                 Text(" is better than nothing")
                 ])
              );

    assert_eq!(
        parse_span("*[an example](example.com) is better than nothing*"),
        Emphasis(vec![
                 Link("an example", "example.com", ""),
                 Text(" is better than nothing")
                 ])
              );

    // does not compile strong
    match parse_span("__whatever__"){
      Emphasis(_) => fail!(),
      _ => {}
    }
    match parse_span("**whatever**"){
      Emphasis(_) => fail!(),
      _ => {}
    }
}
<|endoftext|>


<commit_before>extern crate draw_queue;

use std::cmp::Ordering;
use std::collections::HashMap;
use gfx;
use mem;

pub type FlushError = gfx::DrawError<gfx::batch::OutOfBounds>;

/// An abstract phase. Needs to be object-safe as phases should be
/// allowed to be stored in boxed form in containers.
pub trait AbstractPhase<D: gfx::Device, E, V: ::ToDepth> {
    /// Check if it makes sense to draw this entity
    fn test(&self, &E) -> bool;
    /// Add an entity to the queue
    fn enqueue(&mut self, &E, V, &mut gfx::batch::Context<D::Resources>)
               -> Result<(), gfx::batch::Error>;
    /// Flush the queue into a given renderer
    fn flush(&mut self, &gfx::Frame<D::Resources>,
             &gfx::batch::Context<D::Resources>,
             &mut gfx::Renderer<D::Resources, D::CommandBuffer>)
             -> Result<(), FlushError>;
}

struct Object<S, P: gfx::shade::ShaderParam> {
    batch: gfx::batch::CoreBatch<P>,
    params: P,
    slice: gfx::Slice<P::Resources>,
    depth: S,
}

impl<S: Copy, P: gfx::shade::ShaderParam + Clone> Clone
for Object<S, P> where P::Link: Copy
{
    fn clone(&self) -> Object<S, P> {
        Object {
            batch: self.batch,
            params: self.params.clone(),
            slice: self.slice.clone(),
            depth: self.depth,
        }
    }
}

impl<S: PartialOrd, P: gfx::shade::ShaderParam> Object<S, P> {
    fn cmp_depth(&self, other: &Object<S, P>) -> Ordering {
        self.depth.partial_cmp(&other.depth)
            .unwrap_or(Ordering::Equal)
    }
}

pub enum Sort {
    FrontToBack,
    BackToFront,
    Program,
    Mesh,
    DrawState,
}

/// Phase is doing draw call accumulating and sorting,
/// based a given technique.
pub struct Phase<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
    Y,  // Memory
>{
    pub name: String,
    pub technique: T,
    memory: Y,
    pub sort: Vec<Sort>,
    queue: draw_queue::Queue<Object<V::Depth, T::Params>>,
}

impl<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
> Phase<R, M, V, T, ()> {
    pub fn new(name: &str, tech: T) -> Phase<R, M, V, T, ()> {
        Phase {
            name: name.to_string(),
            technique: tech,
            memory: (),
            sort: Vec::new(),
            queue: draw_queue::Queue::new(),
        }
    }
}

pub type CacheMap<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
> = HashMap<T::Essense, mem::MemResult<Object<V::Depth, T::Params>>>;

impl<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
> Phase<R, M, V, T, CacheMap<R, M, V, T>> {
    pub fn new_cached(name: &str, tech: T) -> Phase<R, M, V, T, CacheMap<R, M, V, T>> {
        Phase {
            name: name.to_string(),
            technique: tech,
            memory: HashMap::new(),
            sort: Vec::new(),
            queue: draw_queue::Queue::new(),
        }
    }
}

impl<
    D: gfx::Device,
    M: ::Material,
    V: ::ToDepth + Copy,
    E: ::Entity<D::Resources, M>,
    T: ::Technique<D::Resources, M, V>,
    Y: mem::Memory<T::Essense, Object<V::Depth, T::Params>>,
>AbstractPhase<D, E, V> for Phase<D::Resources, M, V, T, Y> where
    V::Depth: Copy,
    T::Params: Clone,
    <T::Params as gfx::shade::ShaderParam>::Link: Copy,
{
    fn test(&self, entity: &E) -> bool {
        self.technique.test(entity.get_mesh().0, entity.get_material())
                      .is_some()
    }

    fn enqueue(&mut self, entity: &E, view_info: V,
               context: &mut gfx::batch::Context<D::Resources>)
               -> Result<(), gfx::batch::Error> {
        let essense = self.technique.test(
            entity.get_mesh().0, entity.get_material())
            .unwrap(); //TODO?
        let (orig_mesh, slice) = entity.get_mesh();
        // Try recalling from memory
        match self.memory.lookup(essense) {
            Some(Ok(mut o)) => {
                o.slice = slice.clone();
                self.technique.fix_params(entity.get_material(),
                                          &view_info, &mut o.params);
                self.queue.objects.push(o);
                return Ok(())
            },
            Some(Err(e)) => return Err(e),
            None => ()
        }
        // Compile with the technique
        let depth = view_info.to_depth();
        let (program, params, inst_mesh, state) =
            self.technique.compile(essense, view_info);
        // this would be useful, but requires a ton of new constraints on Params
        //debug_assert_eq!({
        //    let mut p2 = params;
        //    self.technique.fix_params(entity.get_material(), &data, &mut p2);
        //    p2}, params);
        let mut temp_mesh = gfx::Mesh::new(orig_mesh.num_vertices);
        let mesh = match inst_mesh {
            Some(m) => {
                temp_mesh.attributes.extend(orig_mesh.attributes.iter()
                    .chain(m.attributes.iter()).map(|a| a.clone()));
                &temp_mesh
            },
            None => orig_mesh,
        };
        // Create queue object
        let object = context.make_core(program, mesh, state)
                            .map(|b| Object {
                                batch: b,
                                params: params,
                                slice: slice.clone(),
                                depth: depth,
                            });
        // Remember and return
        self.memory.store(essense, object.clone());
        match object {
            Ok(o) => Ok(self.queue.objects.push(o)),
            Err(e) => Err(e),
        }
    }

    fn flush(&mut self, frame: &gfx::Frame<D::Resources>,
             context: &gfx::batch::Context<D::Resources>,
             renderer: &mut gfx::Renderer<D::Resources, D::CommandBuffer>)
             -> Result<(), FlushError> {
        // sort the queue
        match self.sort.first() {
            Some(&Sort::FrontToBack) =>
                self.queue.sort(|a, b| a.cmp_depth(&b)),
            Some(&Sort::BackToFront) =>
                self.queue.sort(|a, b| b.cmp_depth(&a)),
            Some(&Sort::Program) =>
                self.queue.sort(|a, b| a.batch.cmp_program(&b.batch)),
            Some(&Sort::Mesh) =>
                self.queue.sort(|a, b| a.batch.cmp_mesh(&b.batch)),
            Some(&Sort::DrawState) =>
                self.queue.sort(|a, b| a.batch.cmp_state(&b.batch)),
            None => (),
        }
        // call the draws
        for o in self.queue.iter() {
            match renderer.draw(&context.bind(&o.batch, &o.slice, &o.params), frame) {
                Ok(_) => (),
                e => return e,
            }
        }
        // done
        self.queue.objects.clear();
        Ok(())
    }
}
<commit_msg>Using fix_params for new batches<commit_after>extern crate draw_queue;

use std::cmp::Ordering;
use std::collections::HashMap;
use gfx;
use mem;

pub type FlushError = gfx::DrawError<gfx::batch::OutOfBounds>;

/// An abstract phase. Needs to be object-safe as phases should be
/// allowed to be stored in boxed form in containers.
pub trait AbstractPhase<D: gfx::Device, E, V: ::ToDepth> {
    /// Check if it makes sense to draw this entity
    fn test(&self, &E) -> bool;
    /// Add an entity to the queue
    fn enqueue(&mut self, &E, V, &mut gfx::batch::Context<D::Resources>)
               -> Result<(), gfx::batch::Error>;
    /// Flush the queue into a given renderer
    fn flush(&mut self, &gfx::Frame<D::Resources>,
             &gfx::batch::Context<D::Resources>,
             &mut gfx::Renderer<D::Resources, D::CommandBuffer>)
             -> Result<(), FlushError>;
}

struct Object<S, P: gfx::shade::ShaderParam> {
    batch: gfx::batch::CoreBatch<P>,
    params: P,
    slice: gfx::Slice<P::Resources>,
    depth: S,
}

impl<S: Copy, P: gfx::shade::ShaderParam + Clone> Clone
for Object<S, P> where P::Link: Copy
{
    fn clone(&self) -> Object<S, P> {
        Object {
            batch: self.batch,
            params: self.params.clone(),
            slice: self.slice.clone(),
            depth: self.depth,
        }
    }
}

impl<S: PartialOrd, P: gfx::shade::ShaderParam> Object<S, P> {
    fn cmp_depth(&self, other: &Object<S, P>) -> Ordering {
        self.depth.partial_cmp(&other.depth)
            .unwrap_or(Ordering::Equal)
    }
}

pub enum Sort {
    FrontToBack,
    BackToFront,
    Program,
    Mesh,
    DrawState,
}

/// Phase is doing draw call accumulating and sorting,
/// based a given technique.
pub struct Phase<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
    Y,  // Memory
>{
    pub name: String,
    pub technique: T,
    memory: Y,
    pub sort: Vec<Sort>,
    queue: draw_queue::Queue<Object<V::Depth, T::Params>>,
}

impl<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
> Phase<R, M, V, T, ()> {
    pub fn new(name: &str, tech: T) -> Phase<R, M, V, T, ()> {
        Phase {
            name: name.to_string(),
            technique: tech,
            memory: (),
            sort: Vec::new(),
            queue: draw_queue::Queue::new(),
        }
    }
}

pub type CacheMap<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
> = HashMap<T::Essense, mem::MemResult<Object<V::Depth, T::Params>>>;

impl<
    R: gfx::Resources,
    M: ::Material,
    V: ::ToDepth,
    T: ::Technique<R, M, V>,
> Phase<R, M, V, T, CacheMap<R, M, V, T>> {
    pub fn new_cached(name: &str, tech: T) -> Phase<R, M, V, T, CacheMap<R, M, V, T>> {
        Phase {
            name: name.to_string(),
            technique: tech,
            memory: HashMap::new(),
            sort: Vec::new(),
            queue: draw_queue::Queue::new(),
        }
    }
}

impl<
    D: gfx::Device,
    M: ::Material,
    V: ::ToDepth + Copy,
    E: ::Entity<D::Resources, M>,
    T: ::Technique<D::Resources, M, V>,
    Y: mem::Memory<T::Essense, Object<V::Depth, T::Params>>,
>AbstractPhase<D, E, V> for Phase<D::Resources, M, V, T, Y> where
    V::Depth: Copy,
    T::Params: Clone,
    <T::Params as gfx::shade::ShaderParam>::Link: Copy,
{
    fn test(&self, entity: &E) -> bool {
        self.technique.test(entity.get_mesh().0, entity.get_material())
                      .is_some()
    }

    fn enqueue(&mut self, entity: &E, view_info: V,
               context: &mut gfx::batch::Context<D::Resources>)
               -> Result<(), gfx::batch::Error> {
        let essense = self.technique.test(
            entity.get_mesh().0, entity.get_material())
            .unwrap(); //TODO?
        let (orig_mesh, slice) = entity.get_mesh();
        // Try recalling from memory
        match self.memory.lookup(essense) {
            Some(Ok(mut o)) => {
                o.slice = slice.clone();
                self.technique.fix_params(entity.get_material(),
                                          &view_info, &mut o.params);
                self.queue.objects.push(o);
                return Ok(())
            },
            Some(Err(e)) => return Err(e),
            None => ()
        }
        // Compile with the technique
        let depth = view_info.to_depth();
        let (program, mut params, inst_mesh, state) =
            self.technique.compile(essense, view_info);
        self.technique.fix_params(entity.get_material(),
                                  &view_info, &mut params);
        let mut temp_mesh = gfx::Mesh::new(orig_mesh.num_vertices);
        let mesh = match inst_mesh {
            Some(m) => {
                temp_mesh.attributes.extend(orig_mesh.attributes.iter()
                    .chain(m.attributes.iter()).map(|a| a.clone()));
                &temp_mesh
            },
            None => orig_mesh,
        };
        // Create queue object
        let object = context.make_core(program, mesh, state)
                            .map(|b| Object {
                                batch: b,
                                params: params,
                                slice: slice.clone(),
                                depth: depth,
                            });
        // Remember and return
        self.memory.store(essense, object.clone());
        match object {
            Ok(o) => Ok(self.queue.objects.push(o)),
            Err(e) => Err(e),
        }
    }

    fn flush(&mut self, frame: &gfx::Frame<D::Resources>,
             context: &gfx::batch::Context<D::Resources>,
             renderer: &mut gfx::Renderer<D::Resources, D::CommandBuffer>)
             -> Result<(), FlushError> {
        // sort the queue
        match self.sort.first() {
            Some(&Sort::FrontToBack) =>
                self.queue.sort(|a, b| a.cmp_depth(&b)),
            Some(&Sort::BackToFront) =>
                self.queue.sort(|a, b| b.cmp_depth(&a)),
            Some(&Sort::Program) =>
                self.queue.sort(|a, b| a.batch.cmp_program(&b.batch)),
            Some(&Sort::Mesh) =>
                self.queue.sort(|a, b| a.batch.cmp_mesh(&b.batch)),
            Some(&Sort::DrawState) =>
                self.queue.sort(|a, b| a.batch.cmp_state(&b.batch)),
            None => (),
        }
        // call the draws
        for o in self.queue.iter() {
            match renderer.draw(&context.bind(&o.batch, &o.slice, &o.params), frame) {
                Ok(_) => (),
                e => return e,
            }
        }
        // done
        self.queue.objects.clear();
        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::path::Path;

use env::Env;
use config::WorkMode;
use file_saver::*;

mod doc;
mod function;
mod function_body_chunk;
mod general;
mod object;
mod objects;
mod parameter;
mod record;
mod records;
mod return_value;
mod sys;
mod trait_impls;
pub mod translate_from_glib;
pub mod translate_to_glib;

pub fn generate(env: &Env) {
    match env.config.work_mode {
        WorkMode::Normal => normal_generate(env),
        WorkMode::Sys => sys::generate(env),
        WorkMode::Doc => doc::generate(env),
    }
}

fn normal_generate(env: &Env) {
    let mut mod_rs: Vec<String> = Vec::new();
    let mut traits: Vec<String> = Vec::new();
    let root_path = env.config.target_path.join("src").join("auto");

    objects::generate(env, &root_path, &mut mod_rs, &mut traits);
    records::generate(env, &root_path, &mut mod_rs);

    generate_mod_rs(env, &root_path, mod_rs, traits);
}

pub fn generate_mod_rs(env: &Env, root_path: &Path, mod_rs: Vec<String>, traits: Vec<String>) {
    let path = root_path.join("mod.rs");
    save_to_file(path, env.config.make_backup, |w| {
        try!(general::start_comments(w, &env.config));
        try!(general::write_vec(w, &mod_rs));
        try!(writeln!(w, ""));
        try!(writeln!(w, "pub mod traits {{"));
        try!(general::write_vec(w, &traits));
        writeln!(w, "}}")
    });
}
<commit_msg>Hide `mod traits` from rustdoc<commit_after>use std::path::Path;

use env::Env;
use config::WorkMode;
use file_saver::*;

mod doc;
mod function;
mod function_body_chunk;
mod general;
mod object;
mod objects;
mod parameter;
mod record;
mod records;
mod return_value;
mod sys;
mod trait_impls;
pub mod translate_from_glib;
pub mod translate_to_glib;

pub fn generate(env: &Env) {
    match env.config.work_mode {
        WorkMode::Normal => normal_generate(env),
        WorkMode::Sys => sys::generate(env),
        WorkMode::Doc => doc::generate(env),
    }
}

fn normal_generate(env: &Env) {
    let mut mod_rs: Vec<String> = Vec::new();
    let mut traits: Vec<String> = Vec::new();
    let root_path = env.config.target_path.join("src").join("auto");

    objects::generate(env, &root_path, &mut mod_rs, &mut traits);
    records::generate(env, &root_path, &mut mod_rs);

    generate_mod_rs(env, &root_path, mod_rs, traits);
}

pub fn generate_mod_rs(env: &Env, root_path: &Path, mod_rs: Vec<String>, traits: Vec<String>) {
    let path = root_path.join("mod.rs");
    save_to_file(path, env.config.make_backup, |w| {
        try!(general::start_comments(w, &env.config));
        try!(general::write_vec(w, &mod_rs));
        try!(writeln!(w, ""));
        try!(writeln!(w, "#[doc(hidden)]"));
        try!(writeln!(w, "pub mod traits {{"));
        try!(general::write_vec(w, &traits));
        writeln!(w, "}}")
    });
}
<|endoftext|>


<commit_before>use super::interface;
use super::super::alloc;
use super::super::alloc::{SliceWrapper, SliceWrapperMut};
use core::mem;
use core::ops::{Index,IndexMut};
use super::bit_cost::BitsEntropy;
use super::command::{Command, GetCopyLengthCode, GetInsertLengthCode, CommandDistanceIndexAndOffset};
struct EntropyBucketPopulation<AllocU32: alloc::Allocator<u32> > {
    pub bucket_populations: AllocU32::AllocatedMemory,
}
impl<AllocU32:alloc::Allocator<u32>> EntropyBucketPopulation<AllocU32> {
   fn add_assign(&mut self, other: &EntropyBucketPopulation<AllocU32>) {
       assert_eq!(self.bucket_populations.slice().len(), other.bucket_populations.slice().len());
       for (item, other_item) in self.bucket_populations.slice_mut().iter_mut().zip(other.bucket_populations.slice().iter()) {
           *item += *other_item;
       }
   }
}

const NUM_STRIDES:usize = 8;
#[derive(Copy,Clone)]
pub struct BucketPopIndex {
    pub val: u8,
    pub six_bits: u8,
    pub stride: u8,
}

impl <AllocU32: alloc::Allocator<u32> > Index<BucketPopIndex> for EntropyBucketPopulation<AllocU32> {
    type Output = u32;
    fn index<'a>(&'a self, index: BucketPopIndex) -> &'a u32 {
        &self.bucket_populations.slice()[index.val as usize + index.six_bits as usize * 256 + index.stride as usize * 256 * 64]
    }
}
impl <AllocU32: alloc::Allocator<u32> > IndexMut<BucketPopIndex> for EntropyBucketPopulation<AllocU32> {
    fn index_mut<'a>(&'a mut self, index: BucketPopIndex) -> &'a mut u32 {
        &mut self.bucket_populations.slice_mut()[index.val as usize + index.six_bits as usize * 256 + index.stride as usize * 256 * 64]
    }
}

pub struct EntropyTally<AllocU32: alloc::Allocator<u32> > {
    pop:[EntropyBucketPopulation<AllocU32>;NUM_STRIDES],
}

impl<AllocU32:alloc::Allocator<u32> > EntropyTally<AllocU32> {
    pub fn new_pair(m32: &mut AllocU32) -> (EntropyTally<AllocU32>, EntropyTally<AllocU32>) {
        let size = 256 * 256;
        (EntropyTally::<AllocU32> {
            pop:[
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
            ]},
            EntropyTally::<AllocU32> {
            pop:[
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    bucket_populations:m32.alloc_cell(size),
                },
                ]
            })
    }
    fn identify_best_population_from_scratch(&self, scratch:&mut EntropyTally<AllocU32>) -> u8 {
        let mut entropies = [0.0f64;8];
        for index in 0..scratch.pop.len() {
           scratch.pop[index].add_assign(&self.pop[index]);
           entropies[index] = BitsEntropy(scratch.pop[index].bucket_populations.slice(), scratch.pop[index].bucket_populations.slice().len()) as f64;
        }
        let mut best = entropies[0];
        let mut best_index = 0usize;
        for index in 1..entropies.len() {
            if entropies[index] < best {
                 best_index=index;
                 best = entropies[index];
            }
        }
        best_index as u8
    }
    fn get_previous_bytes(&self, input0:&[u8], input1:&[u8], bytes_processed: usize) -> [u8; 8] {
        let mut retval = [0u8; 8];
        for index in 0..8 {
            let bp_offset = index + 1;
            if bp_offset <= bytes_processed {
                 let offset = bytes_processed - bp_offset;
                 if offset >= input0.len() {
		    retval[index] = input1[offset - input0.len()];
                 } else {
                    retval[index] = input0[offset];
                 }
            }
	}
	retval
    }
    pub fn pick_best_stride<InputReference:SliceWrapper<u8>>(&mut self, commands: &[interface::Command<InputReference>], scratch: &mut EntropyTally<AllocU32>, input0: &[u8], input1: &[u8], bytes_processed: &mut usize) -> u8 {
        for cmd in commands.iter() {
            match cmd {
                &interface::Command::Copy(ref copy) => {
		    *bytes_processed += copy.num_bytes as usize;
                },
                &interface::Command::Dict(ref dict) => {
		    *bytes_processed += dict.final_size as usize;
                },
                &interface::Command::Literal(ref lit) => {
                    let mut priors = self.get_previous_bytes(input0, input1, *bytes_processed);
                    for val in lit.data.slice().iter() {
                         for (index, prior) in priors.iter().enumerate() {
                             scratch.pop[index].bucket_populations.slice_mut()[256 * (*prior as usize) + *val as usize] += 1;
                             // increment the population value of this literal
                             // for the respective prior for the stride index
                         }
                         { //reset prior values for the next item
                             let mut tmp = [0u8;7];
                             tmp.clone_from_slice(&priors[..7]);
                             priors[1..].clone_from_slice(&tmp[..]);
                             priors[0] = *val;
                         }
                    }
                    *bytes_processed += lit.data.slice().len();
                },
                _ => {},
            }
        }
        let retval = self.identify_best_population_from_scratch(scratch);
        self.pop[retval as usize].bucket_populations.slice_mut().clone_from_slice(scratch.pop[retval as usize].bucket_populations.slice());
        retval
    }
    pub fn free(&mut self, m32: &mut AllocU32) {
        for item in self.pop.iter_mut() {
            m32.free_cell(mem::replace(&mut item.bucket_populations, AllocU32::AllocatedMemory::default()))
        }
    }
    pub fn is_free(&mut self) -> bool {
        self.pop[0].bucket_populations.slice().len() == 0
    }
}
<commit_msg>Use the delta of the new population map as compared with the old pop map to compute the entropy of the new set of literals<commit_after>use super::interface;
use super::super::alloc;
use super::super::alloc::{SliceWrapper, SliceWrapperMut};
use core::mem;
use core::ops::{Index,IndexMut};
use super::bit_cost::BitsEntropy;
use super::command::{Command, GetCopyLengthCode, GetInsertLengthCode, CommandDistanceIndexAndOffset};
struct EntropyBucketPopulation<AllocU32: alloc::Allocator<u32> > {
    pub bucket_populations: AllocU32::AllocatedMemory,
    pub cached_bit_entropy: f64,
}
impl<AllocU32:alloc::Allocator<u32>> EntropyBucketPopulation<AllocU32> {
   fn add_assign(&mut self, other: &EntropyBucketPopulation<AllocU32>) {
       assert_eq!(self.bucket_populations.slice().len(), other.bucket_populations.slice().len());
       for (item, other_item) in self.bucket_populations.slice_mut().iter_mut().zip(other.bucket_populations.slice().iter()) {
           *item += *other_item;
       }
       self.cached_bit_entropy = BitsEntropy(self.bucket_populations.slice(), self.bucket_populations.slice().len()) as f64;
   }
}

const NUM_STRIDES:usize = 8;
#[derive(Copy,Clone)]
pub struct BucketPopIndex {
    pub val: u8,
    pub six_bits: u8,
    pub stride: u8,
}

impl <AllocU32: alloc::Allocator<u32> > Index<BucketPopIndex> for EntropyBucketPopulation<AllocU32> {
    type Output = u32;
    fn index<'a>(&'a self, index: BucketPopIndex) -> &'a u32 {
        &self.bucket_populations.slice()[index.val as usize + index.six_bits as usize * 256 + index.stride as usize * 256 * 64]
    }
}
impl <AllocU32: alloc::Allocator<u32> > IndexMut<BucketPopIndex> for EntropyBucketPopulation<AllocU32> {
    fn index_mut<'a>(&'a mut self, index: BucketPopIndex) -> &'a mut u32 {
        &mut self.bucket_populations.slice_mut()[index.val as usize + index.six_bits as usize * 256 + index.stride as usize * 256 * 64]
    }
}

pub struct EntropyTally<AllocU32: alloc::Allocator<u32> > {
    pop:[EntropyBucketPopulation<AllocU32>;NUM_STRIDES],
}

impl<AllocU32:alloc::Allocator<u32> > EntropyTally<AllocU32> {
    pub fn new_pair(m32: &mut AllocU32) -> (EntropyTally<AllocU32>, EntropyTally<AllocU32>) {
        let size = 256 * 256;
        (EntropyTally::<AllocU32> {
            pop:[
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
            ]},
            EntropyTally::<AllocU32> {
            pop:[
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                EntropyBucketPopulation::<AllocU32>{
                    cached_bit_entropy:0.0f64,
                    bucket_populations:m32.alloc_cell(size),
                },
                ]
            })
    }
    fn identify_best_population_from_scratch(&self, scratch:&mut EntropyTally<AllocU32>) -> u8 {
        for index in 0..scratch.pop.len() {
           scratch.pop[index].add_assign(&self.pop[index]);

        }
        let mut best_index = scratch.pop.len() - 1;
        let mut best = scratch.pop[best_index].cached_bit_entropy - self.pop[best_index].cached_bit_entropy;
        for index in 0..(scratch.pop.len() - 1){
            if scratch.pop[index].cached_bit_entropy - self.pop[index].cached_bit_entropy < best {
                 best_index=index;
                 best = scratch.pop[index].cached_bit_entropy - self.pop[index].cached_bit_entropy;
            }
        }
        best_index as u8
    }
    fn get_previous_bytes(&self, input0:&[u8], input1:&[u8], bytes_processed: usize) -> [u8; 8] {
        let mut retval = [0u8; 8];
        for index in 0..8 {
            let bp_offset = index + 1;
            if bp_offset <= bytes_processed {
                 let offset = bytes_processed - bp_offset;
                 if offset >= input0.len() {
		    retval[index] = input1[offset - input0.len()];
                 } else {
                    retval[index] = input0[offset];
                 }
            }
	}
	retval
    }
    pub fn pick_best_stride<InputReference:SliceWrapper<u8>>(&mut self, commands: &[interface::Command<InputReference>], scratch: &mut EntropyTally<AllocU32>, input0: &[u8], input1: &[u8], bytes_processed: &mut usize) -> u8 {
        for cmd in commands.iter() {
            match cmd {
                &interface::Command::Copy(ref copy) => {
		    *bytes_processed += copy.num_bytes as usize;
                },
                &interface::Command::Dict(ref dict) => {
		    *bytes_processed += dict.final_size as usize;
                },
                &interface::Command::Literal(ref lit) => {
                    let mut priors = self.get_previous_bytes(input0, input1, *bytes_processed);
                    for (lindex, val) in lit.data.slice().iter().enumerate() {
			 if lindex == 8 {
			    let vpriors = self.get_previous_bytes(input0, input1, 8+*bytes_processed);
			    assert_eq!(vpriors, priors);
			 }
                         for (index, prior) in priors.iter().enumerate() {
                             scratch.pop[index].bucket_populations.slice_mut()[256 * (*prior as usize) + *val as usize] += 1;
                             // increment the population value of this literal
                             // for the respective prior for the stride index
                         }
                         { //reset prior values for the next item
                             let mut tmp = [0u8;7];
                             tmp.clone_from_slice(&priors[..7]);
                             priors[1..].clone_from_slice(&tmp[..]);
                             priors[0] = *val;
                         }
                    }
                    *bytes_processed += lit.data.slice().len();
                },
                _ => {},
            }
        }
        let retval = self.identify_best_population_from_scratch(scratch);
        self.pop[retval as usize].bucket_populations.slice_mut().clone_from_slice(scratch.pop[retval as usize].bucket_populations.slice());
	self.pop[retval as usize].cached_bit_entropy = scratch.pop[retval as usize].cached_bit_entropy;
        retval + 1
    }
    pub fn free(&mut self, m32: &mut AllocU32) {
        for item in self.pop.iter_mut() {
            m32.free_cell(mem::replace(&mut item.bucket_populations, AllocU32::AllocatedMemory::default()))
        }
    }
    pub fn is_free(&mut self) -> bool {
        self.pop[0].bucket_populations.slice().len() == 0
    }
}
<|endoftext|>


<commit_before>/*!

Arithmetic encoder/decoder using the Range encoder underneath. Requires `entropy` feature, enabled by default
Can be used in a general case of entropy coding stage. Supposed to be fast.

# Links

http://en.wikipedia.org/wiki/Arithmetic_coding
http://en.wikipedia.org/wiki/Range_encoding

# Example
```rust
# #[allow(unused_must_use)];
use std::io::{MemWriter, MemReader};
use compress::entropy::ari;

// Encode some text
let text = "some text";
let mut e = ari::ByteEncoder::new(MemWriter::new());
e.write_str(text);
let (encoded, _) = e.finish();

// Decode the encoded text
let mut d = ari::ByteDecoder::new(MemReader::new(encoded.unwrap()));
let decoded = d.read_to_end().unwrap();
```
# Credit

This is an original implementation.

*/

#![allow(missing_docs)]

use std::fmt::Show;
use std::io::IoResult;

pub use self::table::{ByteDecoder, ByteEncoder};

pub mod apm;
pub mod bin;
pub mod table;
#[cfg(test)]
mod test;

pub type Symbol = u8;
const SYMBOL_BITS: uint = 8;
const SYMBOL_TOTAL: uint = 1<<SYMBOL_BITS;

pub type Border = u32;
const BORDER_BYTES: uint = 4;
const BORDER_BITS: uint = BORDER_BYTES * 8;
const BORDER_EXCESS: uint = BORDER_BITS-SYMBOL_BITS;
const BORDER_SYMBOL_MASK: u32 = ((SYMBOL_TOTAL-1) << BORDER_EXCESS) as u32;

pub const RANGE_DEFAULT_THRESHOLD: Border = 1<<14;

/// Range Encoder basic primitive
/// Gets probability ranges on the input, produces whole bytes of code on the output,
/// where the code is an arbitrary fixed-ppoint value inside the resulting probability range.
pub struct RangeEncoder {
    low: Border,
    hai: Border,
    /// The minimum distance between low and hai to keep at all times,
    /// has to be at least the largest incoming 'total',
    /// and optimally many times larger
    pub threshold: Border,
    /// Tuning parameters
    bits_lost_on_threshold_cut: f32,
    bits_lost_on_division: f32,
}

impl RangeEncoder {
    /// Create a new instance
    /// will keep the active range below 'max_range'
    pub fn new(max_range: Border) -> RangeEncoder {
        debug_assert!(max_range > (SYMBOL_TOTAL as Border));
        RangeEncoder {
            low: 0,
            hai: -1,
            threshold: max_range,
            bits_lost_on_threshold_cut: 0.0,
            bits_lost_on_division: 0.0,
        }
    }

    /// Reset the current range
    pub fn reset(&mut self) {
        self.low = 0;
        self.hai = -1;
    }

    #[cfg(tune)]
    fn count_bits(range: Border, total: Border) -> f32 {
        -((range as f32) / (total as f32)).log2()
    }

    #[cfg(not(tune))]
    fn count_bits(_range: Border, _total: Border) -> f32 {
        0.0
    }

    /// Return the number of bits lost due to threshold cuts and integer operations
    #[cfg(tune)]
    pub fn get_bits_lost(&self) -> (f32, f32) {
        (self.bits_lost_on_threshold_cut, self.bits_lost_on_division)
    }

    /// Process a given interval [from/total,to/total) into the current range
    /// write into the output slice, and return the number of symbols produced
    pub fn process(&mut self, total: Border, from: Border, to: Border, output: &mut [Symbol]) -> uint {
        debug_assert!(from<to && to<=total);
        let old_range = self.hai - self.low;
        let range = old_range / total;
        debug_assert!(range>0, "RangeCoder range is too narrow [{}-{}) for the total {}",
            self.low, self.hai, total);
        debug!("\t\tProcessing [{}-{})/{} with range {}", from, to, total, range);
        let mut lo = self.low + range*from;
        let mut hi = self.low + range*to;
        self.bits_lost_on_division += RangeEncoder::count_bits(range*total, old_range);
        let mut num_shift = 0u;
        loop {
            if (lo^hi) & BORDER_SYMBOL_MASK != 0 {
                if hi-lo > self.threshold {
                    break
                }
                let old_range = hi-lo;
                let lim = hi & BORDER_SYMBOL_MASK;
                if hi-lim >= lim-lo {lo=lim}
                else {hi=lim-1};
                debug_assert!(lo < hi);
                self.bits_lost_on_threshold_cut += RangeEncoder::count_bits(hi-lo, old_range);
            }

            debug!("\t\tShifting on [{}-{}) to symbol {}", lo, hi, lo>>BORDER_EXCESS);
            output[num_shift] = (lo>>BORDER_EXCESS) as Symbol;
            num_shift += 1;
            lo<<=SYMBOL_BITS; hi<<=SYMBOL_BITS;
            debug_assert!(lo < hi);
        }
        self.low = lo;
        self.hai = hi;
        num_shift
    }

    /// Query the value encoded by 'code' in range [0,total)
    pub fn query(&self, total: Border, code: Border) -> Border {
        debug!("\t\tQuerying code {} of total {} under range [{}-{})",
            code, total, self.low, self.hai);
        debug_assert!(self.low <= code && code < self.hai)
        let range = (self.hai - self.low) / total;
        (code - self.low) / range
    }

    /// Get the code tail and close the range
    /// used at the end of encoding
    pub fn get_code_tail(&mut self) -> Border {
        let tail = self.low;
        self.low = 0;
        self.hai = 0;
        tail
    }
}

/// An abstract model to produce probability ranges
/// Can be a table, a mix of tables, or just a smart function.
pub trait Model<V: Copy + Show> {
    /// Get the probability range of a value
    fn get_range(&self, value: V) -> (Border,Border);
    /// Find the value by a given probability offset, return with the range
    fn find_value(&self, offset: Border) -> (V,Border,Border);
    /// Get the sum of all probabilities
    fn get_denominator(&self) -> Border;

    /// Encode a value using a range encoder
    /// return the number of symbols written
    fn encode(&self, value: V, re: &mut RangeEncoder, out: &mut [Symbol]) -> uint {
        let (lo, hi) = self.get_range(value);
        let total = self.get_denominator();
        debug!("\tEncoding value {} of range [{}-{}) with total {}", value, lo, hi, total);
        re.process(total, lo, hi, out)
    }

    /// Decode a value using given 'code' on the range encoder
    /// return a (value, num_symbols_to_shift) pair
    fn decode(&self, code: Border, re: &mut RangeEncoder) -> (V, uint) {
        let total = self.get_denominator();
        let offset = re.query(total, code);
        let (value, lo, hi) = self.find_value(offset);
        debug!("\tDecoding value {} of offset {} with total {}", value, offset, total);
        let mut out = [0 as Symbol, ..BORDER_BYTES];
        let shift = re.process(total, lo, hi, out.as_mut_slice());
        debug_assert_eq!(if shift==0 {0} else {code>>(BORDER_BITS - shift*8)},
            out.slice_to(shift).iter().fold(0 as Border, |u,&b| (u<<8)+(b as Border)));
        (value, shift)
    }
}

/// An arithmetic encoder helper
pub struct Encoder<W> {
    stream: W,
    range: RangeEncoder,
}

impl<W: Writer> Encoder<W> {
    /// Create a new encoder on top of a given Writer
    pub fn new(w: W) -> Encoder<W> {
        Encoder {
            stream: w,
            range: RangeEncoder::new(RANGE_DEFAULT_THRESHOLD),
        }
    }

    /// Encode an abstract value under the given Model
    pub fn encode<V: Copy + Show, M: Model<V>>(&mut self, value: V, model: &M) -> IoResult<()> {
        let mut buf = [0 as Symbol, ..BORDER_BYTES];
        let num = model.encode(value, &mut self.range, buf.as_mut_slice());
        self.stream.write(buf.slice_to(num))
    }

    /// Finish encoding by writing the code tail word
    pub fn finish(mut self) -> (W, IoResult<()>) {
        debug_assert!(BORDER_BITS == 32);
        let code = self.range.get_code_tail();
        let result = self.stream.write_be_u32(code);
        let result = result.and(self.stream.flush());
        (self.stream, result)
    }

    /// Flush the output stream
    pub fn flush(&mut self) -> IoResult<()> {
        self.stream.flush()
    }

    /// Return the number of bytes lost due to threshold cuts and integer operations
    #[cfg(tune)]
    pub fn get_bytes_lost(&self) -> (f32, f32) {
        let (a,b) = self.range.get_bits_lost();
        (a/8.0, b/8.0)
    }
}

/// An arithmetic decoder helper
pub struct Decoder<R> {
    stream: R,
    range: RangeEncoder,
    code: Border,
    bytes_pending: uint,
}

impl<R: Reader> Decoder<R> {
    /// Create a decoder on top of a given Reader
    pub fn new(r: R) -> Decoder<R> {
        Decoder {
            stream: r,
            range: RangeEncoder::new(RANGE_DEFAULT_THRESHOLD),
            code: 0,
            bytes_pending: BORDER_BYTES,
        }
    }

    fn feed(&mut self) -> IoResult<()> {
        while self.bytes_pending != 0 {
            let b = try!(self.stream.read_u8());
            self.code = (self.code<<8) + (b as Border);
            self.bytes_pending -= 1;
        }
        Ok(())
    }

    /// Decode an abstract value based on the given Model
    pub fn decode<V: Copy + Show, M: Model<V>>(&mut self, model: &M) -> IoResult<V> {
        self.feed().unwrap();
        let (value, shift) = model.decode(self.code, &mut self.range);
        self.bytes_pending = shift;
        Ok(value)
    }

    /// Finish decoding
    pub fn finish(mut self) -> (R, IoResult<()>)  {
        let err = self.feed();
        (self.stream, err)
    }
}
<commit_msg>Update to rust master<commit_after>/*!

Arithmetic encoder/decoder using the Range encoder underneath. Requires `entropy` feature, enabled by default
Can be used in a general case of entropy coding stage. Supposed to be fast.

# Links

http://en.wikipedia.org/wiki/Arithmetic_coding
http://en.wikipedia.org/wiki/Range_encoding

# Example
```rust
# #[allow(unused_must_use)];
use std::io::{MemWriter, MemReader};
use compress::entropy::ari;

// Encode some text
let text = "some text";
let mut e = ari::ByteEncoder::new(MemWriter::new());
e.write_str(text);
let (encoded, _) = e.finish();

// Decode the encoded text
let mut d = ari::ByteDecoder::new(MemReader::new(encoded.unwrap()));
let decoded = d.read_to_end().unwrap();
```
# Credit

This is an original implementation.

*/

#![allow(missing_docs)]

use std::fmt::Show;
use std::io::IoResult;

pub use self::table::{ByteDecoder, ByteEncoder};

pub mod apm;
pub mod bin;
pub mod table;
#[cfg(test)]
mod test;

pub type Symbol = u8;
const SYMBOL_BITS: uint = 8;
const SYMBOL_TOTAL: uint = 1<<SYMBOL_BITS;

pub type Border = u32;
const BORDER_BYTES: uint = 4;
const BORDER_BITS: uint = BORDER_BYTES * 8;
const BORDER_EXCESS: uint = BORDER_BITS-SYMBOL_BITS;
const BORDER_SYMBOL_MASK: u32 = ((SYMBOL_TOTAL-1) << BORDER_EXCESS) as u32;

pub const RANGE_DEFAULT_THRESHOLD: Border = 1<<14;

/// Range Encoder basic primitive
/// Gets probability ranges on the input, produces whole bytes of code on the output,
/// where the code is an arbitrary fixed-ppoint value inside the resulting probability range.
pub struct RangeEncoder {
    low: Border,
    hai: Border,
    /// The minimum distance between low and hai to keep at all times,
    /// has to be at least the largest incoming 'total',
    /// and optimally many times larger
    pub threshold: Border,
    /// Tuning parameters
    bits_lost_on_threshold_cut: f32,
    bits_lost_on_division: f32,
}

impl RangeEncoder {
    /// Create a new instance
    /// will keep the active range below 'max_range'
    pub fn new(max_range: Border) -> RangeEncoder {
        debug_assert!(max_range > (SYMBOL_TOTAL as Border));
        RangeEncoder {
            low: 0,
            hai: -1,
            threshold: max_range,
            bits_lost_on_threshold_cut: 0.0,
            bits_lost_on_division: 0.0,
        }
    }

    /// Reset the current range
    pub fn reset(&mut self) {
        self.low = 0;
        self.hai = -1;
    }

    #[cfg(tune)]
    fn count_bits(range: Border, total: Border) -> f32 {
        -((range as f32) / (total as f32)).log2()
    }

    #[cfg(not(tune))]
    fn count_bits(_range: Border, _total: Border) -> f32 {
        0.0
    }

    /// Return the number of bits lost due to threshold cuts and integer operations
    #[cfg(tune)]
    pub fn get_bits_lost(&self) -> (f32, f32) {
        (self.bits_lost_on_threshold_cut, self.bits_lost_on_division)
    }

    /// Process a given interval [from/total,to/total) into the current range
    /// write into the output slice, and return the number of symbols produced
    pub fn process(&mut self, total: Border, from: Border, to: Border, output: &mut [Symbol]) -> uint {
        debug_assert!(from<to && to<=total);
        let old_range = self.hai - self.low;
        let range = old_range / total;
        debug_assert!(range>0, "RangeCoder range is too narrow [{}-{}) for the total {}",
            self.low, self.hai, total);
        debug!("\t\tProcessing [{}-{})/{} with range {}", from, to, total, range);
        let mut lo = self.low + range*from;
        let mut hi = self.low + range*to;
        self.bits_lost_on_division += RangeEncoder::count_bits(range*total, old_range);
        let mut num_shift = 0u;
        loop {
            if (lo^hi) & BORDER_SYMBOL_MASK != 0 {
                if hi-lo > self.threshold {
                    break
                }
                let old_range = hi-lo;
                let lim = hi & BORDER_SYMBOL_MASK;
                if hi-lim >= lim-lo {lo=lim}
                else {hi=lim-1};
                debug_assert!(lo < hi);
                self.bits_lost_on_threshold_cut += RangeEncoder::count_bits(hi-lo, old_range);
            }

            debug!("\t\tShifting on [{}-{}) to symbol {}", lo, hi, lo>>BORDER_EXCESS);
            output[num_shift] = (lo>>BORDER_EXCESS) as Symbol;
            num_shift += 1;
            lo<<=SYMBOL_BITS; hi<<=SYMBOL_BITS;
            debug_assert!(lo < hi);
        }
        self.low = lo;
        self.hai = hi;
        num_shift
    }

    /// Query the value encoded by 'code' in range [0,total)
    pub fn query(&self, total: Border, code: Border) -> Border {
        debug!("\t\tQuerying code {} of total {} under range [{}-{})",
            code, total, self.low, self.hai);
        debug_assert!(self.low <= code && code < self.hai);
        let range = (self.hai - self.low) / total;
        (code - self.low) / range
    }

    /// Get the code tail and close the range
    /// used at the end of encoding
    pub fn get_code_tail(&mut self) -> Border {
        let tail = self.low;
        self.low = 0;
        self.hai = 0;
        tail
    }
}

/// An abstract model to produce probability ranges
/// Can be a table, a mix of tables, or just a smart function.
pub trait Model<V: Copy + Show> {
    /// Get the probability range of a value
    fn get_range(&self, value: V) -> (Border,Border);
    /// Find the value by a given probability offset, return with the range
    fn find_value(&self, offset: Border) -> (V,Border,Border);
    /// Get the sum of all probabilities
    fn get_denominator(&self) -> Border;

    /// Encode a value using a range encoder
    /// return the number of symbols written
    fn encode(&self, value: V, re: &mut RangeEncoder, out: &mut [Symbol]) -> uint {
        let (lo, hi) = self.get_range(value);
        let total = self.get_denominator();
        debug!("\tEncoding value {} of range [{}-{}) with total {}", value, lo, hi, total);
        re.process(total, lo, hi, out)
    }

    /// Decode a value using given 'code' on the range encoder
    /// return a (value, num_symbols_to_shift) pair
    fn decode(&self, code: Border, re: &mut RangeEncoder) -> (V, uint) {
        let total = self.get_denominator();
        let offset = re.query(total, code);
        let (value, lo, hi) = self.find_value(offset);
        debug!("\tDecoding value {} of offset {} with total {}", value, offset, total);
        let mut out = [0 as Symbol, ..BORDER_BYTES];
        let shift = re.process(total, lo, hi, out.as_mut_slice());
        debug_assert_eq!(if shift==0 {0} else {code>>(BORDER_BITS - shift*8)},
            out.slice_to(shift).iter().fold(0 as Border, |u,&b| (u<<8)+(b as Border)));
        (value, shift)
    }
}

/// An arithmetic encoder helper
pub struct Encoder<W> {
    stream: W,
    range: RangeEncoder,
}

impl<W: Writer> Encoder<W> {
    /// Create a new encoder on top of a given Writer
    pub fn new(w: W) -> Encoder<W> {
        Encoder {
            stream: w,
            range: RangeEncoder::new(RANGE_DEFAULT_THRESHOLD),
        }
    }

    /// Encode an abstract value under the given Model
    pub fn encode<V: Copy + Show, M: Model<V>>(&mut self, value: V, model: &M) -> IoResult<()> {
        let mut buf = [0 as Symbol, ..BORDER_BYTES];
        let num = model.encode(value, &mut self.range, buf.as_mut_slice());
        self.stream.write(buf.slice_to(num))
    }

    /// Finish encoding by writing the code tail word
    pub fn finish(mut self) -> (W, IoResult<()>) {
        debug_assert!(BORDER_BITS == 32);
        let code = self.range.get_code_tail();
        let result = self.stream.write_be_u32(code);
        let result = result.and(self.stream.flush());
        (self.stream, result)
    }

    /// Flush the output stream
    pub fn flush(&mut self) -> IoResult<()> {
        self.stream.flush()
    }

    /// Return the number of bytes lost due to threshold cuts and integer operations
    #[cfg(tune)]
    pub fn get_bytes_lost(&self) -> (f32, f32) {
        let (a,b) = self.range.get_bits_lost();
        (a/8.0, b/8.0)
    }
}

/// An arithmetic decoder helper
pub struct Decoder<R> {
    stream: R,
    range: RangeEncoder,
    code: Border,
    bytes_pending: uint,
}

impl<R: Reader> Decoder<R> {
    /// Create a decoder on top of a given Reader
    pub fn new(r: R) -> Decoder<R> {
        Decoder {
            stream: r,
            range: RangeEncoder::new(RANGE_DEFAULT_THRESHOLD),
            code: 0,
            bytes_pending: BORDER_BYTES,
        }
    }

    fn feed(&mut self) -> IoResult<()> {
        while self.bytes_pending != 0 {
            let b = try!(self.stream.read_u8());
            self.code = (self.code<<8) + (b as Border);
            self.bytes_pending -= 1;
        }
        Ok(())
    }

    /// Decode an abstract value based on the given Model
    pub fn decode<V: Copy + Show, M: Model<V>>(&mut self, model: &M) -> IoResult<V> {
        self.feed().unwrap();
        let (value, shift) = model.decode(self.code, &mut self.range);
        self.bytes_pending = shift;
        Ok(value)
    }

    /// Finish decoding
    pub fn finish(mut self) -> (R, IoResult<()>)  {
        let err = self.feed();
        (self.stream, err)
    }
}
<|endoftext|>


<commit_before>use std;
import std::io;
import std::io::{writer_util, reader_util};
import std::map;
import result;

export rowreader, rowaccess, rowiter,
       new_reader, new_reader_readlen,
       hashmap_iter, hashmap_iter_full;

enum state {
    fieldstart(bool),
    infield(uint, uint),
    inescapedfield(uint, uint),
    inquote(uint, uint)
}

type rowreader = {
    readlen: uint,
    delim: char,
    quote: char,
    f : io::reader,
    mutable offset : uint,
    mutable buffers : [@[char]],
    mutable state : state
};

type row = {
    fields : [ fieldtype ]
};

type bufferdescr = {
    escaped: bool,
    buffers: [@[char]],
    start: uint,
    end: uint
};

enum fieldtype {
    emptyfield(),
    bufferfield(bufferdescr)
}

iface rowiter {
    fn readrow() -> result::t<row, str>;
}

iface rowaccess {
    fn len() -> uint;
    fn getchars(uint) -> [char];
    fn getstr(uint) -> str;
}

fn new_reader(+f: io::reader, +delim: char, +quote: char) -> rowreader {
    {
        readlen: 1024u,
        delim: delim,
        quote: quote,
        f: f,
        mutable offset : 0u,
        mutable buffers : [],
        mutable state : fieldstart(false)
    }
}

fn new_reader_readlen(+f: io::reader, +delim: char, +quote: char, rl: uint) -> rowreader {
    {
        readlen: rl,
        delim: delim,
        quote: quote,
        f: f,
        mutable offset : 0u,
        mutable buffers : [],
        mutable state : fieldstart(false)
    }
}

impl of rowaccess for row {
    fn len() -> uint {
        vec::len(self.fields)
    }
    fn getchars(field: uint) -> [char] {
        fn unescape(escaped: [char]) -> [char] {
            let r : [char] = [];
            vec::reserve(r, vec::len(escaped));
            let in_q = false;
            for c in escaped {
                if in_q {
                    assert(c == '"');
                    in_q = false;
                } else {
                    in_q = c == '"';
                    r += [c];
                }
            }
            ret r;
        }
        alt self.fields[field] {
            emptyfield() { ret []; }
            bufferfield(desc) {
                let buf = [];
                {
                    let i = 0u;
                    while i < vec::len(desc.buffers) {
                        let from = if (i == 0u)
                            { desc.start } else { 0u };
                        let to = if (i == vec::len(desc.buffers) - 1u)
                            { desc.end } else { vec::len(*desc.buffers[i]) };
                        buf += vec::slice(*desc.buffers[i], from, to);
                        i = i + 1u;
                    }
                }
                if field == self.len() - 1u {
                    // there may be a trailing \r on the last field; we should strip it
                    // if so. bodgy here but seems the most efficient place to deal with this
                    if vec::len(buf) > 0u {
                        if buf[vec::len(buf)-1u] == '\r' {
                            buf = vec::slice(buf, 0u, vec::len(buf)-1u);
                        }
                    }
                }
                if desc.escaped {
                    buf = unescape(buf);
                }
                ret buf;
            }
        };
    }
    fn getstr(field: uint) -> str {
        ret str::from_chars(self.getchars(field));
    }
    fn getall() -> [str] {
        let a = [];
        self.map() { |s|
            a += [s];
        }
        ret a;
    }
    fn map(f: fn(s: str)) {
        let i = 0u;
        let len = self.len();
        while i < len {
            f(self.getstr(i));
            i += 1u;
        }
    }
}

impl of rowiter for rowreader {
    fn readrow() -> result::t<row, str> {
        fn row_from_buf(self: rowreader, &fields: [fieldtype]) -> bool {
            fn new_bufferfield(self: rowreader, escaped: bool, sb: uint, so: uint, eo: uint) -> fieldtype {
                let eb = vec::len(self.buffers);
                let sb = sb, so = so, eo = eo;
                if escaped {
                    so += 1u;
                    if so > vec::len(*self.buffers[sb]) {
                        sb += 1u;
                        so = vec::len(*self.buffers[sb]) - 1u;
                    }
                    if eo > 0u {
                        eo -= 1u;
                    } else {
                        eb -= 1u;
                        eo = vec::len(*self.buffers[sb]) - 1u;
                    }
                }
                bufferfield({ escaped: escaped, buffers: vec::slice(self.buffers, sb, eb), start: so, end: eo })
            }
            let cbuffer = vec::len(self.buffers) - 1u;
            let buf: @[char] = self.buffers[cbuffer];
            while self.offset < vec::len(*buf) {
                let coffset = self.offset;
                let c : char = buf[coffset];
                self.offset += 1u;
                alt self.state {
                    fieldstart(after_delim) {
                        if c == self.quote {
                            self.state = inescapedfield(cbuffer, coffset);
                        } else if c == '\n' {
                            if after_delim {
                                fields += [emptyfield];
                            }
                            ret true;
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [emptyfield];
                        } else {
                            self.state = infield(cbuffer, coffset);
                        }
                    }
                    infield(b,o) {
                        if c == '\n' {
                            fields += [new_bufferfield(self, false, b, o, coffset)];
                            ret true;
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [new_bufferfield(self, false, b, o, coffset)];
                        }
                    }
                    inescapedfield(b, o) {
                        if c == self.quote {
                            self.state = inquote(b, o);
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [new_bufferfield(self, true, b, o, coffset)];
                        }
                    }
                    inquote(b, o) {
                        if c == '\n' {
                            fields += [new_bufferfield(self, true, b, o, coffset)];
                            ret true;
                        } else if c == self.quote {
                            self.state = inescapedfield(b, o);
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [new_bufferfield(self, true, b, o, coffset)];
                        }
                        // swallow odd chars, eg. space between field and "
                    }
                }
            }
            ret false;
        }

        self.state = fieldstart(false);
        let do_read = vec::len(self.buffers) == 0u;
        let fields = [];
        while true {
            if do_read {
                let data: @[char] = @self.f.read_chars(self.readlen);
                if vec::len(*data) == 0u {
                    ret result::err("EOF");
                }
                self.buffers += [data];
                self.offset = 0u;
            }

            if row_from_buf(self, fields) {
                let r: row = { fields: fields };
                fields = [];
                if vec::len(self.buffers) > 1u {
                    self.buffers = vec::slice(self.buffers, vec::len(self.buffers) - 1u, vec::len(self.buffers));
                }
                ret result::ok(r);
            }
            do_read = true;
        }
        ret result::err("unreachable");
    }
}

fn hashmap_iter_cols(r: rowreader, cols: [str], f: fn(map::hashmap<str, str>)) {
    while true {
        let res = r.readrow();
        if result::failure(res) {
            break;
        }
        let m : map::hashmap<str, str> = map::new_str_hash();
        let col = 0u;
        let row = result::get(res);
        if row.len() != vec::len(cols) {
            cont; // FIXME: how to flag that we dropped a crazy row?
        }
        result::get(res).map() { |s|
            m.insert(cols[col], s);
            col += 1u;
        };
        f(m);
    }
}

// reads the first row as a header, to derive keys for a hashmap
// emitted for each subsequent row
fn hashmap_iter(r: rowreader, f: fn(map::hashmap<str, str>)) {
    let res = r.readrow();
    alt res {
        result::ok(row) {
            hashmap_iter_cols(r, result::get(res).getall(), f);
        }
        result::err(_) { }
    }
}

// as hashmap_iter, but first apply 'hc' to each header; allows
// cleaning up headers; also allows verification that heads are
// satisfactory
fn hashmap_iter_full(r: rowreader, hmap: fn(&&h: str) -> str, hver: fn(cols: [str]) -> bool, f: fn(map::hashmap<str, str>)) {
    let res = r.readrow();
    alt res {
        result::ok(row) {
            let cols : [str] = vec::map(result::get(res).getall(), hmap);
            if !hver(cols) {
                ret;
            }
            hashmap_iter_cols(r, cols, f);
        }
        result::err(_) { }
    }
}

#[cfg(test)]
mod test {
    fn rowmatch(testdata: str, expected: [[str]]) {
        let chk = fn@(mk: fn(io::reader) -> rowreader) {
            let f = io::string_reader(testdata);
            let r = mk(f);
            let i = 0u;
            while true {
                let res = r.readrow();
                if result::failure(res) {
                    break;
                }
                let row = result::get(res);
                let expect = expected[i];

                assert(row.len() == vec::len(expect));
                let j = 0u;
                while j < row.len() {
                    assert(row.getstr(j) == expect[j]);
                    j += 1u;
                }
                i += 1u;
            }
            assert(i == vec::len(expected));
        };
        let runchecks = fn@(testdata: str) {
            // test default reader params
            chk() { |inp|
                new_reader_readlen(inp, ',', '"', 2u)
            };
            // test continuations over read buffers
            let j = 1u;
            while j < str::len(testdata) {
                chk() { |inp|
                    new_reader_readlen(inp, ',', '"', j)
                };
                j += 1u;
            }
        };
        runchecks(testdata);
        runchecks(str::replace(testdata, "\n", "\r\n"));
    }

    #[test]
    fn test_simple() {
        rowmatch("a,b,c,d\n1,2,3,4\n",
                 [["a", "b", "c", "d"], ["1", "2", "3", "4"]]);
    }

    #[test]
    fn test_trailing_comma() {
        rowmatch("a,b,c,d\n1,2,3,4,\n",
                 [["a", "b", "c", "d"], ["1", "2", "3", "4", ""]]);
    }

    #[test]
    fn test_leading_comma() {
        rowmatch("a,b,c,d\n,1,2,3,4\n",
                 [["a", "b", "c", "d"], ["", "1", "2", "3", "4"]]);
    }

    #[test]
    fn test_quote_simple() {
        rowmatch("\"Hello\",\"There\"\na,b,\"c\",d\n",
                 [["Hello", "There"], ["a", "b", "c", "d"]]);
    }

    #[test]
    fn test_quote_nested() {
        rowmatch("\"Hello\",\"There is a \"\"fly\"\" in my soup\"\na,b,\"c\",d\n",
                 [["Hello", "There is a \"fly\" in my soup"], ["a", "b", "c", "d"]]);
    }

    #[test]
    fn test_quote_with_comma() {
        rowmatch("1,2,3,\"a,b,c\"",
                 [["1", "2", "3", "a,b,c"]])
    }

    #[test]
    fn test_blank_line() {
        rowmatch("\n\n", [[], []]);
    }
}

<commit_msg>fix for latest rust<commit_after>use std;
import std::io;
import std::io::{writer_util, reader_util};
import std::map;
import result;

export rowreader, rowaccess, rowiter,
       new_reader, new_reader_readlen,
       hashmap_iter, hashmap_iter_full;

enum state {
    fieldstart(bool),
    infield(uint, uint),
    inescapedfield(uint, uint),
    inquote(uint, uint)
}

type rowreader = {
    readlen: uint,
    delim: char,
    quote: char,
    f : io::reader,
    mutable offset : uint,
    mutable buffers : [@[char]],
    mutable state : state
};

type row = {
    fields : [ fieldtype ]
};

type bufferdescr = {
    escaped: bool,
    buffers: [@[char]],
    start: uint,
    end: uint
};

enum fieldtype {
    emptyfield(),
    bufferfield(bufferdescr)
}

iface rowiter {
    fn readrow() -> result::t<row, str>;
}

iface rowaccess {
    fn len() -> uint;
    fn getchars(uint) -> [char];
    fn getstr(uint) -> str;
}

fn new_reader(+f: io::reader, +delim: char, +quote: char) -> rowreader {
    {
        readlen: 1024u,
        delim: delim,
        quote: quote,
        f: f,
        mutable offset : 0u,
        mutable buffers : [],
        mutable state : fieldstart(false)
    }
}

fn new_reader_readlen(+f: io::reader, +delim: char, +quote: char, rl: uint) -> rowreader {
    {
        readlen: rl,
        delim: delim,
        quote: quote,
        f: f,
        mutable offset : 0u,
        mutable buffers : [],
        mutable state : fieldstart(false)
    }
}

impl of rowaccess for row {
    fn len() -> uint {
        vec::len(self.fields)
    }
    fn getchars(field: uint) -> [char] {
        fn unescape(escaped: [char]) -> [char] {
            let r : [char] = [];
            vec::reserve(r, vec::len(escaped));
            let in_q = false;
            for c in escaped {
                if in_q {
                    assert(c == '"');
                    in_q = false;
                } else {
                    in_q = c == '"';
                    r += [c];
                }
            }
            ret r;
        }
        alt self.fields[field] {
            emptyfield() { ret []; }
            bufferfield(desc) {
                let buf = [];
                {
                    let i = 0u;
                    while i < vec::len(desc.buffers) {
                        let from = if (i == 0u)
                            { desc.start } else { 0u };
                        let to = if (i == vec::len(desc.buffers) - 1u)
                            { desc.end } else { vec::len(*desc.buffers[i]) };
                        buf += vec::slice(*desc.buffers[i], from, to);
                        i = i + 1u;
                    }
                }
                if field == self.len() - 1u {
                    // there may be a trailing \r on the last field; we should strip it
                    // if so. bodgy here but seems the most efficient place to deal with this
                    if vec::len(buf) > 0u {
                        if buf[vec::len(buf)-1u] == '\r' {
                            buf = vec::slice(buf, 0u, vec::len(buf)-1u);
                        }
                    }
                }
                if desc.escaped {
                    buf = unescape(buf);
                }
                ret buf;
            }
        };
    }
    fn getstr(field: uint) -> str {
        ret str::from_chars(self.getchars(field));
    }
    fn getall() -> [str] {
        let a = [];
        self.map() { |s|
            a += [s];
        }
        ret a;
    }
    fn map(f: fn(s: str)) {
        let i = 0u;
        let len = self.len();
        while i < len {
            f(self.getstr(i));
            i += 1u;
        }
    }
}

impl of rowiter for rowreader {
    fn readrow() -> result::t<row, str> {
        fn row_from_buf(self: rowreader, &fields: [fieldtype]) -> bool {
            fn new_bufferfield(self: rowreader, escaped: bool, sb: uint, so: uint, eo: uint) -> fieldtype {
                let eb = vec::len(self.buffers);
                let sb = sb, so = so, eo = eo;
                if escaped {
                    so += 1u;
                    if so > vec::len(*self.buffers[sb]) {
                        sb += 1u;
                        so = vec::len(*self.buffers[sb]) - 1u;
                    }
                    if eo > 0u {
                        eo -= 1u;
                    } else {
                        eb -= 1u;
                        eo = vec::len(*self.buffers[sb]) - 1u;
                    }
                }
                bufferfield({ escaped: escaped, buffers: vec::slice(self.buffers, sb, eb), start: so, end: eo })
            }
            fn statestr(state: state) -> str {
                alt state {
                    fieldstart(after_delim) {
                        #fmt("fieldstart : after_delim %b", after_delim)
                    }
                    infield(b,o) {
                        #fmt("field : %u %u", b, o)
                    }
                    inescapedfield(b, o) {
                        #fmt("inescapedfield : %u %u", b, o)
                    }
                    inquote(b, o) {
                        #fmt("inquote : %u %u", b, o)
                    }
                }
            }
            let cbuffer = vec::len(self.buffers) - 1u;
            let buf: @[char] = self.buffers[cbuffer];
            while self.offset < vec::len(*buf) {
                let coffset = self.offset;
                let c : char = buf[coffset];
                //io::println(#fmt("-> %c | %s", c, statestr(self.state)));
                self.offset += 1u;
                alt self.state {
                    fieldstart(after_delim) {
                        if c == self.quote {
                            self.state = inescapedfield(cbuffer, coffset);
                        } else if c == '\n' {
                            if after_delim {
                                fields += [emptyfield];
                            }
                            ret true;
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [emptyfield];
                        } else {
                            self.state = infield(cbuffer, coffset);
                        }
                    }
                    infield(b,o) {
                        if c == '\n' {
                            fields += [new_bufferfield(self, false, b, o, coffset)];
                            ret true;
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [new_bufferfield(self, false, b, o, coffset)];
                        }
                    }
                    inescapedfield(b, o) {
                        if c == self.quote {
                            self.state = inquote(b, o);
                        }
                        else if c == self.delim {
                         self.state = fieldstart(true);
                         fields += [new_bufferfield(self, true, b, o, coffset)];
                        }
                    }
                    inquote(b, o) {
                        //io::println(#fmt("inquote : %u %u", b, o));
                        if c == '\n' {
                            fields += [new_bufferfield(self, true, b, o, coffset)];
                            ret true;
                        } else if c == self.quote {
                            self.state = inescapedfield(b, o);
                        } else if c == self.delim {
                            self.state = fieldstart(true);
                            fields += [new_bufferfield(self, true, b, o, coffset)];
                        }
                        // swallow odd chars, eg. space between field and "
                    }
                }
                //io::println(#fmt(".... %s", statestr(self.state)));
            }
            ret false;
        }

        self.state = fieldstart(false);
        let do_read = vec::len(self.buffers) == 0u;
        let fields = [];
        while true {
            if do_read {
                let data: @[char] = @self.f.read_chars(self.readlen);
                if vec::len(*data) == 0u {
                    ret result::err("EOF");
                }
                self.buffers += [data];
                self.offset = 0u;
            }

            if row_from_buf(self, fields) {
                let r: row = { fields: fields };
                fields = [];
                if vec::len(self.buffers) > 1u {
                    self.buffers = vec::slice(self.buffers, vec::len(self.buffers) - 1u, vec::len(self.buffers));
                }
                ret result::ok(r);
            }
            do_read = true;
        }
        ret result::err("unreachable");
    }
}

fn hashmap_iter_cols(r: rowreader, cols: [str], f: fn(map::hashmap<str, str>)) {
    while true {
        let res = r.readrow();
        if result::failure(res) {
            break;
        }
        let m : map::hashmap<str, str> = map::new_str_hash();
        let col = 0u;
        let row = result::get(res);
        if row.len() != vec::len(cols) {
            cont; // FIXME: how to flag that we dropped a crazy row?
        }
        result::get(res).map() { |s|
            m.insert(cols[col], s);
            col += 1u;
        };
        f(m);
    }
}

// reads the first row as a header, to derive keys for a hashmap
// emitted for each subsequent row
fn hashmap_iter(r: rowreader, f: fn(map::hashmap<str, str>)) {
    let res = r.readrow();
    alt res {
        result::ok(row) {
            hashmap_iter_cols(r, result::get(res).getall(), f);
        }
        result::err(_) { }
    }
}

// as hashmap_iter, but first apply 'hc' to each header; allows
// cleaning up headers; also allows verification that heads are
// satisfactory
fn hashmap_iter_full(r: rowreader, hmap: fn(&&h: str) -> str, hver: fn(cols: [str]) -> bool, f: fn(map::hashmap<str, str>)) {
    let res = r.readrow();
    alt res {
        result::ok(row) {
            let cols : [str] = vec::map(result::get(res).getall(), hmap);
            if !hver(cols) {
                ret;
            }
            hashmap_iter_cols(r, cols, f);
        }
        result::err(_) { }
    }
}

#[cfg(test)]
mod test {
    fn rowmatch(testdata: str, expected: [[str]]) {
        let chk = fn@(mk: fn(io::reader) -> rowreader) {
            let f = io::str_reader(testdata);
            let r = mk(f);
            let i = 0u;
            while true {
                let res = r.readrow();
                if result::failure(res) {
                    break;
                }
                let row = result::get(res);
                let expect = expected[i];

                assert(row.len() == vec::len(expect));
                let j = 0u;
                while j < row.len() {
                    assert(row.getstr(j) == expect[j]);
                    j += 1u;
                }
                i += 1u;
            }
            assert(i == vec::len(expected));
        };
        let runchecks = fn@(testdata: str) {
            // test default reader params
            chk() { |inp|
                new_reader_readlen(inp, ',', '"', 2u)
            };
            // test continuations over read buffers
            let j = 1u;
            while j < str::len(testdata) {
                chk() { |inp|
                    new_reader_readlen(inp, ',', '"', j)
                };
                j += 1u;
            }
        };
        runchecks(testdata);
        runchecks(str::replace(testdata, "\n", "\r\n"));
    }

    #[test]
    fn test_simple() {
        rowmatch("a,b,c,d\n1,2,3,4\n",
                 [["a", "b", "c", "d"], ["1", "2", "3", "4"]]);
    }

    #[test]
    fn test_trailing_comma() {
        rowmatch("a,b,c,d\n1,2,3,4,\n",
                 [["a", "b", "c", "d"], ["1", "2", "3", "4", ""]]);
    }

    #[test]
    fn test_leading_comma() {
        rowmatch("a,b,c,d\n,1,2,3,4\n",
                 [["a", "b", "c", "d"], ["", "1", "2", "3", "4"]]);
    }

    #[test]
    fn test_quote_simple() {
        rowmatch("\"Hello\",\"There\"\na,b,\"c\",d\n",
                 [["Hello", "There"], ["a", "b", "c", "d"]]);
    }

    #[test]
    fn test_quote_nested() {
        rowmatch("\"Hello\",\"There is a \"\"fly\"\" in my soup\"\na,b,\"c\",d\n",
                 [["Hello", "There is a \"fly\" in my soup"], ["a", "b", "c", "d"]]);
    }

    #[test]
    fn test_quote_with_comma() {
        rowmatch("\"1,2\"\n",
                 [["1,2"]])
    }

    #[test]
    fn test_quote_with_other_comma() {
        rowmatch("1,2,3,\"a,b,c\"",
                 [["1", "2", "3", "a,b,c"]])
    }

    #[test]
    fn test_blank_line() {
        rowmatch("\n\n", [[], []]);
    }
}

<|endoftext|>