<commit_before>#![deny(missing_docs)]
#![cfg_attr(test, feature(test))]

//! A DSL parsing library for human readable text documents

extern crate read_token;
extern crate range;

pub use parse_error_handler::{
    stderr_unwrap,
    ParseErrorHandler
};
pub use parse_error::ParseError;
pub use meta_rules::{ parse, parse_errstr, Rule };

/// The type of debug id used to track down errors in rules.
pub type DebugId = usize;

use std::sync::Arc;
use std::fs::File;
use std::path::Path;
use range::Range;

pub mod bootstrap;
pub mod json;
pub mod meta_rules;
pub mod tokenizer;

mod parse_error;
mod parse_error_handler;

mod all {
    pub use super::*;
}

/// Represents meta data.
#[derive(PartialEq, Clone, Debug)]
pub enum MetaData {
    /// Starts node.
    StartNode(Arc<String>),
    /// Ends node.
    EndNode(Arc<String>),
    /// Sets bool property.
    Bool(Arc<String>, bool),
    /// Sets f64 property.
    F64(Arc<String>, f64),
    /// Sets string property.
    String(Arc<String>, Arc<String>),
}

/// Stores syntax.
#[derive(Clone, Debug, PartialEq)]
pub struct Syntax {
    /// Rule data.
    pub rules: Vec<Rule>,
    /// Name of rules.
    pub names: Vec<Arc<String>>,
}

impl Syntax {
    /// Creates a new syntax.
    pub fn new() -> Syntax {
        Syntax {
            rules: vec![],
            names: vec![]
        }
    }

    /// Adds a new rule.
    pub fn push(&mut self, name: Arc<String>, rule: Rule) {
        self.rules.push(rule);
        self.names.push(name);
    }
}

/// Reads syntax from text.
pub fn syntax(rules: &str) -> Result<Syntax, Range<ParseError>> {
    let mut tokens = vec![];
    try!(parse(&bootstrap::rules(), rules, &mut tokens));
    let mut ignored_meta_data = vec![];
    match bootstrap::convert(&tokens, &mut ignored_meta_data) {
        Ok(res) => Ok(res),
        Err(()) => Err((Range::empty(0).wrap(ParseError::Conversion(
            format!("Bootstrapping rules are incorrect")))))
    }
}

/// Reads syntax from text, formatting the error as `String`.
pub fn syntax_errstr(rules: &str) -> Result<Syntax, String> {
    match syntax(rules) {
        Ok(syntax) => Ok(syntax),
        Err(range_err) => {
            let mut w: Vec<u8> = vec![];
            ParseErrorHandler::new(&rules).write(&mut w, range_err).unwrap();
            Err(String::from_utf8(w).unwrap())
        }
    }
}

/// Convenience method for loading data, using the meta language.
/// Panics if there is an error, and writes error message to
/// standard error output.
pub fn load_syntax_data<A, B>(
    syntax_path: A,
    data_path: B
) -> Vec<Range<MetaData>>
    where A: AsRef<Path>, B: AsRef<Path>
{
    use std::io::Read;

    let mut syntax_file = File::open(syntax_path).unwrap();
    let mut s = String::new();
    syntax_file.read_to_string(&mut s).unwrap();
    let rules = stderr_unwrap(&s, syntax(&s));

    let mut data_file = File::open(data_path).unwrap();
    let mut d = String::new();
    data_file.read_to_string(&mut d).unwrap();
    let mut tokens = vec![];
    stderr_unwrap(&d, parse(&rules, &d, &mut tokens));
    tokens
}

#[cfg(test)]
mod tests {
    extern crate test;

    use self::test::Bencher;
    use super::*;

    fn is_thread_safe<T: Send + Sync>() {}

    #[test]
    fn meta_data_thread_safe() {
        is_thread_safe::<MetaData>();
    }

    #[test]
    fn parse_error_thread_safe() {
        is_thread_safe::<ParseError>();
    }

    #[test]
    fn rule_thread_safe() {
        is_thread_safe::<Rule>();
    }

    #[test]
    fn syntax_thread_safe() {
        is_thread_safe::<Syntax>();
    }

    #[bench]
    fn bench_hello(b: &mut Bencher) {
        b.iter(|| {
            let text = r#"hi James!"#;
            let rules = r#"
                1 say_hi = ["hi" .w? {"James":"james" "Peter":"peter"} "!"]
                2 document = say_hi
            "#;
            // Parse rules with meta language and convert to rules for parsing text.
            let rules = match syntax_errstr(rules) {
                Err(err) => {
                    println!("{}", err);
                    return;
                }
                Ok(rules) => rules
            };
            let mut data = vec![];
            match parse_errstr(&rules, text, &mut data) {
                Err(err) => {
                    println!("{}", err);
                    return;
                }
                Ok(()) => {}
            };
        });
    }
}
<commit_msg>Added "bench_deep_string"<commit_after>#![deny(missing_docs)]
#![cfg_attr(test, feature(test))]

//! A DSL parsing library for human readable text documents

extern crate read_token;
extern crate range;

pub use parse_error_handler::{
    stderr_unwrap,
    ParseErrorHandler
};
pub use parse_error::ParseError;
pub use meta_rules::{ parse, parse_errstr, Rule };

/// The type of debug id used to track down errors in rules.
pub type DebugId = usize;

use std::sync::Arc;
use std::fs::File;
use std::path::Path;
use range::Range;

pub mod bootstrap;
pub mod json;
pub mod meta_rules;
pub mod tokenizer;

mod parse_error;
mod parse_error_handler;

mod all {
    pub use super::*;
}

/// Represents meta data.
#[derive(PartialEq, Clone, Debug)]
pub enum MetaData {
    /// Starts node.
    StartNode(Arc<String>),
    /// Ends node.
    EndNode(Arc<String>),
    /// Sets bool property.
    Bool(Arc<String>, bool),
    /// Sets f64 property.
    F64(Arc<String>, f64),
    /// Sets string property.
    String(Arc<String>, Arc<String>),
}

/// Stores syntax.
#[derive(Clone, Debug, PartialEq)]
pub struct Syntax {
    /// Rule data.
    pub rules: Vec<Rule>,
    /// Name of rules.
    pub names: Vec<Arc<String>>,
}

impl Syntax {
    /// Creates a new syntax.
    pub fn new() -> Syntax {
        Syntax {
            rules: vec![],
            names: vec![]
        }
    }

    /// Adds a new rule.
    pub fn push(&mut self, name: Arc<String>, rule: Rule) {
        self.rules.push(rule);
        self.names.push(name);
    }
}

/// Reads syntax from text.
pub fn syntax(rules: &str) -> Result<Syntax, Range<ParseError>> {
    let mut tokens = vec![];
    try!(parse(&bootstrap::rules(), rules, &mut tokens));
    let mut ignored_meta_data = vec![];
    match bootstrap::convert(&tokens, &mut ignored_meta_data) {
        Ok(res) => Ok(res),
        Err(()) => Err((Range::empty(0).wrap(ParseError::Conversion(
            format!("Bootstrapping rules are incorrect")))))
    }
}

/// Reads syntax from text, formatting the error as `String`.
pub fn syntax_errstr(rules: &str) -> Result<Syntax, String> {
    match syntax(rules) {
        Ok(syntax) => Ok(syntax),
        Err(range_err) => {
            let mut w: Vec<u8> = vec![];
            ParseErrorHandler::new(&rules).write(&mut w, range_err).unwrap();
            Err(String::from_utf8(w).unwrap())
        }
    }
}

/// Convenience method for loading data, using the meta language.
/// Panics if there is an error, and writes error message to
/// standard error output.
pub fn load_syntax_data<A, B>(
    syntax_path: A,
    data_path: B
) -> Vec<Range<MetaData>>
    where A: AsRef<Path>, B: AsRef<Path>
{
    use std::io::Read;

    let mut syntax_file = File::open(syntax_path).unwrap();
    let mut s = String::new();
    syntax_file.read_to_string(&mut s).unwrap();
    let rules = stderr_unwrap(&s, syntax(&s));

    let mut data_file = File::open(data_path).unwrap();
    let mut d = String::new();
    data_file.read_to_string(&mut d).unwrap();
    let mut tokens = vec![];
    stderr_unwrap(&d, parse(&rules, &d, &mut tokens));
    tokens
}

#[cfg(test)]
mod tests {
    extern crate test;

    use self::test::Bencher;
    use super::*;

    fn is_thread_safe<T: Send + Sync>() {}

    #[test]
    fn meta_data_thread_safe() {
        is_thread_safe::<MetaData>();
    }

    #[test]
    fn parse_error_thread_safe() {
        is_thread_safe::<ParseError>();
    }

    #[test]
    fn rule_thread_safe() {
        is_thread_safe::<Rule>();
    }

    #[test]
    fn syntax_thread_safe() {
        is_thread_safe::<Syntax>();
    }

    #[bench]
    fn bench_hello(b: &mut Bencher) {
        b.iter(|| {
            let text = r#"hi James!"#;
            let rules = r#"
                1 say_hi = ["hi" .w? {"James":"james" "Peter":"peter"} "!"]
                2 document = say_hi
            "#;
            // Parse rules with meta language and convert to rules for parsing text.
            let rules = match syntax_errstr(rules) {
                Err(err) => {
                    panic!("{}", err);
                }
                Ok(rules) => rules
            };
            let mut data = vec![];
            match parse_errstr(&rules, text, &mut data) {
                Err(err) => {
                    panic!("{}", err);
                }
                Ok(()) => {}
            };
        });
    }

    #[bench]
    fn bench_deep_string(b: &mut Bencher) {
        b.iter(|| {
            let text = r#"hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi hi
            foo"#;
            let rules = r#"
                1 foo = [.." "!:"foo" .w! {"foo" foo}]
                2 document = foo
            "#;
            // Parse rules with meta language and convert to rules for parsing text.
            let rules = match syntax_errstr(rules) {
                Err(err) => panic!("{}", err),
                Ok(rules) => rules
            };
            let mut data = vec![];
            match parse_errstr(&rules, text, &mut data) {
                Err(err) => panic!("{}", err),
                Ok(()) => {}
            };
        });
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate lazy_static;
extern crate regex;

use regex::Regex;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;
use std::result;
use std::fmt::{Display, Formatter};

pub type Result<T> = result::Result<T, String>;

#[derive(Clone, Debug, PartialEq)]
pub struct SubLine {
    pub text: String,
    pub index: u32,
    pub start: u32,
    pub end: u32,
}

impl Display for SubLine {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        let start = recover_timestamp(&self.start);
        let end = recover_timestamp(&self.end);
        write!(f,
               "{index}\r\n\
{s_h:02}:{s_m:02}:{s_s:02},{s_ms:03} --> {e_h:02}:{e_m:02}:{e_s:02},{e_ms:03}\r\n\
{text}\r\n\r\n",
               index = self.index,
               s_h = start[0],
               s_m = start[1],
               s_s = start[2],
               s_ms = start[3],
               e_h = end[0],
               e_m = end[1],
               e_s = end[2],
               e_ms = end[3],
               text = self.text)
    }
}

impl SubLine {
    pub fn get_start_timestamp(&self) -> [u32; 4] {
        recover_timestamp(&self.start)
    }

    pub fn get_end_timestamp(&self) -> [u32; 4] {
        recover_timestamp(&self.end)
    }

    pub fn get_duration(&self) -> u32 {
        self.end - self.start
    }

    pub fn shift(&mut self, offset: i32) -> Result<()> {
        if offset > 0 {
            self.start += offset as u32;
            self.end += offset as u32;
            Ok(())
        } else if offset < 0 {
            if (-offset as u32 >= self.start) || (-offset as u32 >= self.end) {
                return Err("start(or end) can't be negative".to_string());
            };
            self.start -= -offset as u32;
            self.end -= -offset as u32;
            Ok(())
        } else {
            Err("offset can't be 0".to_string())
        }
    }
}

pub trait SubLineVector {
    fn insert_line(&mut self, item: SubLine);

    fn by_index(&self, index: usize) -> Option<&SubLine>;
    fn by_index_mut(&mut self, index: usize) -> Option<&mut SubLine>;

    fn by_timestamp(&self, timestamp: &[u32; 4]) -> Option<&SubLine> {
        let miliseconds = to_miliseconds(&timestamp);
        self.by_miliseconds(miliseconds)
    }

    fn by_timestamp_mut(&mut self, timestamp: &[u32; 4]) -> Option<&mut SubLine> {
        let miliseconds = to_miliseconds(&timestamp);
        self.by_miliseconds_mut(miliseconds)
    }

    fn by_miliseconds(&self, time: u32) -> Option<&SubLine>;
    fn by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine>;

    fn nearest_by_miliseconds(&self, time: u32) -> Option<&SubLine>;
    fn nearest_by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine>;

    fn save_to(&self, path: &str) -> Result<()>;
}

pub type Subtitles = Vec<SubLine>;

impl SubLineVector for Subtitles {
    fn insert_line(&mut self, item: SubLine) {
        let vector_index = (item.index - 1) as usize;
        let index = item.index as usize;
        self.insert(vector_index, item);
        for i in &mut self[index..] {
            i.index += 1
        }
    }

    fn save_to(&self, path: &str) -> Result<()> {
        let mut file = try!(File::create(&path).map_err(|e| e.to_string()));
        for line in self {
            let start = line.get_start_timestamp();
            let end = line.get_end_timestamp();
            try!(write!(&mut file,
                        "{index}\r\n{s_h:02}:{s_m:02}:{s_s:02},{s_ms:03} --> \
                         {e_h:02}:{e_m:02}:{e_s:02},{e_ms:03}\r\n{text}\r\n\r\n",
                        index = line.index,
                        s_h = start[0],
                        s_m = start[1],
                        s_s = start[2],
                        s_ms = start[3],
                        e_h = end[0],
                        e_m = end[1],
                        e_s = end[2],
                        e_ms = end[3],
                        text = line.text)
                .map_err(|e| e.to_string()));
        }
        try!(write!(&mut file, "\r\n").map_err(|e| e.to_string()));
        Ok(())
    }

    fn by_index(&self, index: usize) -> Option<&SubLine> {
        self.get(index - 1)
    }

    fn by_index_mut(&mut self, index: usize) -> Option<&mut SubLine> {
        self.get_mut(index - 1)
    }

    fn nearest_by_miliseconds(&self, time: u32) -> Option<&SubLine> {
        // binary search
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;

        while max >= min {
            guess_index = (max + min) / 2;
            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                return Some(&guess);
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        self.get(min - 1)
    }

    fn nearest_by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine> {
        // binary search
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;
        let mut result: usize = 0;
        let mut state = false;

        while max >= min {
            guess_index = (max + min) / 2;

            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                result = guess_index;
                state = true;
                break;
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        if state { Some(&mut self[result]) } else { self.get_mut(min - 1) }
    }

    fn by_miliseconds(&self, time: u32) -> Option<&SubLine> {
        // binary search
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;

        while max >= min {
            guess_index = (max + min) / 2;
            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                return Some(&guess);
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        None
    }

    fn by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine> {
        // binary search
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;
        let mut result: usize = 0;
        let mut state = false;

        while max >= min {
            guess_index = (max + min) / 2;

            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                result = guess_index;
                state = true;
                break;
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        if state { Some(&mut self[result]) } else { None }
    }
}

fn to_miliseconds(timestamp: &[u32; 4]) -> u32 {
    let mut result: u32 = 0;
    result += timestamp[3];
    result += timestamp[2] * 1000;
    result += timestamp[1] * 1000 * 60;
    result += timestamp[0] * 1000 * 60 * 60;
    result
}

fn recover_timestamp(miliseconds: &u32) -> [u32; 4] {
    let hours = miliseconds / 3600000;
    let minutes = (miliseconds - hours * 3600000) / 60000;
    let seconds = (miliseconds - (hours * 3600000 + minutes * 60000)) / 1000;
    let miliseconds = miliseconds - (hours * 3600000 + minutes * 60000 + seconds * 1000);
    [hours, minutes, seconds, miliseconds]
}

lazy_static! {
    pub static ref SUBS: Regex = Regex::new(r"(?x)
        (\d+)
        (\r\n)
        (\d{2}):(\d{2}):(\d{2}),(\d{3})
        \s-->\s
        (\d{2}):(\d{2}):(\d{2}),(\d{3})
        (\r\n)
        ([\S\s]*?)
        (\r\n){2}?").unwrap();
    static ref EOF_EMPTY_LINES: Regex = Regex::new(r"\s*?\z").unwrap();
    static ref UNIFY_NEWLINE_STYLE: Regex = Regex::new(r"(?x)
            ([^\n(\r\n)]\r[^\n(\r\n)])|
            ([^\r(\r\n)]\n[^\r(\r\n)])").unwrap();

}

fn read_file(sub_path: &str) -> Result<String> {
    let path = Path::new(sub_path);
    let mut file = try!(File::open(&path).map_err(|e| e.to_string()));
    let mut content = String::new();
    try!(file.read_to_string(&mut content).map_err(|e| e.to_string()));
    Ok(content)
}

fn parse(content: String) -> Subtitles {
    let mut result: Subtitles = Subtitles::new();

    for cap in SUBS.captures_iter(&content) {
        let start_timestamp = [cap.at(3).unwrap().parse::<u32>().unwrap(),
                               cap.at(4).unwrap().parse::<u32>().unwrap(),
                               cap.at(5).unwrap().parse::<u32>().unwrap(),
                               cap.at(6).unwrap().parse::<u32>().unwrap()];
        let end_timestamp = [cap.at(7).unwrap().parse::<u32>().unwrap(),
                             cap.at(8).unwrap().parse::<u32>().unwrap(),
                             cap.at(9).unwrap().parse::<u32>().unwrap(),
                             cap.at(10).unwrap().parse::<u32>().unwrap()];

        let start = to_miliseconds(&start_timestamp);
        let end = to_miliseconds(&end_timestamp);

        let line = SubLine {
            index: cap.at(1).unwrap().parse::<u32>().unwrap(),
            text: cap.at(12).unwrap().to_string(),
            start: start,
            end: end,
        };
        result.push(line)
    }
    result
}

pub fn prepare(path: &str) -> Result<Subtitles> {
    let mut content = try!(read_file(&path).map_err(|e| e.to_string()));
    content = EOF_EMPTY_LINES.replace_all(&content, "\r\n\r\n");
    content = UNIFY_NEWLINE_STYLE.replace_all(&content, "\r\n");

    if !SUBS.is_match(&content) {
        return Err("Bad srt file".to_string());
    }

    Ok(parse(content))
}<commit_msg>added some compatibility with std::io::Error and separete trait for saving with generic path parameters<commit_after>#[macro_use]
extern crate lazy_static;
extern crate regex;

use regex::Regex;
use std::fs::File;
use std::path::Path;
use std::fmt::{Display, Formatter};
use std::convert::AsRef;
use std::io::{Error, ErrorKind, Read, Write};

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SubLine {
    pub text: String,
    pub index: u32,
    pub start: u32,
    pub end: u32,
    pub start_timestamp: [u32; 4],
    pub end_timestamp: [u32; 4],
}

impl Display for SubLine {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        write!(f,
               "{index}\r\n{s_h:02}:{s_m:02}:{s_s:02},{s_ms:03} --> \
                {e_h:02}:{e_m:02}:{e_s:02},{e_ms:03}\r\n{text}\r\n\r\n",
               index = self.index,
               s_h = self.start_timestamp[0],
               s_m = self.start_timestamp[1],
               s_s = self.start_timestamp[2],
               s_ms = self.start_timestamp[3],
               e_h = self.end_timestamp[0],
               e_m = self.end_timestamp[1],
               e_s = self.end_timestamp[2],
               e_ms = self.end_timestamp[3],
               text = self.text)
    }
}

impl SubLine {
    pub fn get_duration(&self) -> u32 {
        self.end - self.start
    }

    pub fn shift(&mut self, offset: i32) -> Result<(), &'static str> {
        if offset > 0 {
            self.start += offset as u32;
            self.end += offset as u32;
            Ok(())
        } else if offset < 0 {
            if (-offset as u32 >= self.start) || (-offset as u32 >= self.end) {
                return Err("start(or end) is going to be negative with given offset");
            };
            self.start -= -offset as u32;
            self.end -= -offset as u32;
            Ok(())
        } else {
            Err("offset can't be 0")
        }
    }
}

pub trait SaveSub<P: AsRef<Path>> {
    fn save_to(&self, path: P) -> Result<(), Error>;
}

pub trait SubLineVector {
    fn insert_line(&mut self, item: SubLine);

    fn by_index(&self, index: usize) -> Option<&SubLine>;
    fn by_index_mut(&mut self, index: usize) -> Option<&mut SubLine>;

    fn by_timestamp(&self, timestamp: &[u32; 4]) -> Option<&SubLine> {
        let miliseconds = to_miliseconds(&timestamp);
        self.by_miliseconds(miliseconds)
    }

    fn by_timestamp_mut(&mut self, timestamp: &[u32; 4]) -> Option<&mut SubLine> {
        let miliseconds = to_miliseconds(&timestamp);
        self.by_miliseconds_mut(miliseconds)
    }

    fn by_miliseconds(&self, time: u32) -> Option<&SubLine>;
    fn by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine>;

    fn nearest_by_miliseconds(&self, time: u32) -> Option<&SubLine>;
    fn nearest_by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine>;
}

pub type Subtitles = Vec<SubLine>;

impl <P: AsRef<Path>> SaveSub<P> for Subtitles {
    fn save_to(&self, path: P) -> Result<(), Error> {
        let mut file = try!(File::create(&path));
        for line in self {
            try!(write!(&mut file,"{repr}", repr = line));
        }
        try!(write!(&mut file, "\r\n\r\n"));
        Ok(())
    }
}

impl SubLineVector for Subtitles {
    fn insert_line(&mut self, item: SubLine) {
        let vector_index = (item.index - 1) as usize;
        let index = item.index as usize;
        self.insert(vector_index, item);
        for i in &mut self[index..] {
            i.index += 1;
        }
    }

    fn by_index(&self, index: usize) -> Option<&SubLine> {
        self.get(index - 1)
    }

    fn by_index_mut(&mut self, index: usize) -> Option<&mut SubLine> {
        self.get_mut(index - 1)
    }

    fn nearest_by_miliseconds(&self, time: u32) -> Option<&SubLine> {
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;

        while max >= min {
            guess_index = (max + min) / 2;
            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                return Some(&guess);
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        self.get(max)
    }

    fn nearest_by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine> {
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;
        let mut result: usize = 0;
        let mut state = false;

        while max >= min {
            guess_index = (max + min) / 2;

            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                result = guess_index;
                state = true;
                break;
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        if state {
            self.get_mut(result)
        } else {
            self.get_mut(max)
        }
    }

    fn by_miliseconds(&self, time: u32) -> Option<&SubLine> {
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;

        while max >= min {
            guess_index = (max + min) / 2;
            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                return Some(&guess);
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        None
    }

    fn by_miliseconds_mut(&mut self, time: u32) -> Option<&mut SubLine> {
        let mut min = 0;
        let mut max = self.len() - 1;
        let mut guess_index;
        let mut result: usize = 0;
        let mut state = false;

        while max >= min {
            guess_index = (max + min) / 2;

            let guess = self.get(guess_index).unwrap();

            if (guess.start <= time) && (time <= guess.end) {
                result = guess_index;
                state = true;
                break;
            } else if time > guess.end {
                min = guess_index + 1;
            } else {
                max = guess_index - 1;
            }
        }
        if state { self.get_mut(result) } else { None }
    }
}

pub fn to_miliseconds(timestamp: &[u32; 4]) -> u32 {
    let mut result: u32 = 0;
    result += timestamp[3];
    result += timestamp[2] * 1000;
    result += timestamp[1] * 1000 * 60;
    result += timestamp[0] * 1000 * 60 * 60;
    result
}

lazy_static! {
    pub static ref SUBS: Regex = Regex::new(r"(?x)
        (\d+)
        (\r\n)
        (\d{2}):(\d{2}):(\d{2}),(\d{3})
        \s-->\s
        (\d{2}):(\d{2}):(\d{2}),(\d{3})
        (\r\n)
        ([\S\s]*?)
        (\r\n){2}?").unwrap();
    static ref EOF_EMPTY_LINES: Regex = Regex::new(r"\s*?\z").unwrap();
    static ref UNIFY_NEWLINE_STYLE: Regex = Regex::new(r"(?x)
            ([^\n(\r\n)]\r[^\n(\r\n)])|
            ([^\r(\r\n)]\n[^\r(\r\n)])").unwrap();

}

fn read_file<P: AsRef<Path>> (path: P) -> Result<String, Error> {
    let mut file = try!(File::open(&path));
    let mut content = String::new();
    try!(file.read_to_string(&mut content));
    Ok(content)
}

fn parse(content: &str) -> Subtitles {
    let mut result: Subtitles = Subtitles::new();

    for cap in SUBS.captures_iter(&content) {
        let start_timestamp = [cap.at(3).unwrap().parse::<u32>().unwrap(),
                               cap.at(4).unwrap().parse::<u32>().unwrap(),
                               cap.at(5).unwrap().parse::<u32>().unwrap(),
                               cap.at(6).unwrap().parse::<u32>().unwrap()];
        let end_timestamp = [cap.at(7).unwrap().parse::<u32>().unwrap(),
                             cap.at(8).unwrap().parse::<u32>().unwrap(),
                             cap.at(9).unwrap().parse::<u32>().unwrap(),
                             cap.at(10).unwrap().parse::<u32>().unwrap()];

        let start = to_miliseconds(&start_timestamp);
        let end = to_miliseconds(&end_timestamp);

        let line = SubLine {
            index: cap.at(1).unwrap().parse::<u32>().unwrap(),
            text: cap.at(12).unwrap().to_string(),
            start: start,
            end: end,
            start_timestamp: start_timestamp,
            end_timestamp: end_timestamp,
        };
        result.push(line)
    }
    result
}

pub fn prepare<P: AsRef<Path>> (path: P) -> Result<Subtitles, Error> {
    let mut content = try!(read_file(&path));
    content = EOF_EMPTY_LINES.replace_all(&content, "\r\n\r\n");
    content = UNIFY_NEWLINE_STYLE.replace_all(&content, "\r\n");

    if !SUBS.is_match(&content) {
        return Err(Error::new(ErrorKind::InvalidData,
                              "Given file does not match with srt format specification"));
    }

    Ok(parse(&content))
}
<|endoftext|>


<commit_before>extern crate futures;

use futures::{Async, Poll, Future, Stream};
use std::mem;

pub type BoxStateStream<T, S, E> = Box<StateStream<Item = T, State = S, Error = E> + Send>;

pub enum StreamEvent<I, S> {
    Next(I),
    Done(S),
}

pub trait StateStream {
    type Item;
    type State;
    type Error;

    fn poll(&mut self) -> Poll<StreamEvent<Self::Item, Self::State>, Self::Error>;

    #[inline]
    fn boxed(self) -> BoxStateStream<Self::Item, Self::State, Self::Error>
        where Self: Sized + Send + 'static
    {
        Box::new(self)
    }

    #[inline]
    fn into_future(self) -> IntoFuture<Self>
        where Self: Sized
    {
        IntoFuture(Some(self))
    }

    #[inline]
    fn into_stream(self) -> IntoStream<Self>
        where Self: Sized
    {
        IntoStream(self)
    }

    #[inline]
    fn map<F, B>(self, f: F) -> Map<Self, F>
        where Self: Sized,
              F: FnMut(Self::Item) -> B
    {
        Map {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn map_err<F, B>(self, f: F) -> MapErr<Self, F>
        where Self: Sized,
              F: FnMut(Self::Error) -> B
    {
        MapErr {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn map_state<F, B>(self, f: F) -> MapState<Self, F>
        where Self: Sized,
              F: FnOnce(Self::State) -> B
    {
        MapState {
            stream: self,
            f: Some(f),
        }
    }

    #[inline]
    fn filter<F>(self, f: F) -> Filter<Self, F>
        where Self: Sized,
              F: FnMut(&Self::Item) -> bool
    {
        Filter {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn filter_map<F, B>(self, f: F) -> FilterMap<Self, F>
        where Self: Sized,
              F: FnMut(Self::Item) -> Option<B>
    {

        FilterMap {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn collect(self) -> Collect<Self>
        where Self: Sized
    {
        Collect {
            stream: self,
            items: vec![],
        }
    }
}

impl<S: ?Sized> StateStream for Box<S>
    where S: StateStream
{
    type Item = S::Item;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, S::State>, S::Error> {
        S::poll(self)
    }
}

pub trait FutureExt: Future {
    #[inline]
    fn flatten_state_stream(self) -> FlattenStateStream<Self>
        where Self: Sized,
              Self::Item: StateStream<Error = Self::Error>
    {
        FlattenStateStream(FlattenStateStreamState::Future(self))
    }
}

impl<F: ?Sized> FutureExt for F
    where F: Future
{}

#[inline]
pub fn stream<S>(stream: S) -> FromStream<S>
    where S: Stream
{
    FromStream(stream)
}

#[inline]
pub fn unfold<T, F, Fut, It, St>(init: T, f: F) -> Unfold<T, F, Fut>
     where F: FnMut(T) -> Fut,
           Fut: futures::IntoFuture<Item = StreamEvent<(It, T), St>>
{
    Unfold {
        state: UnfoldState::Ready(init),
        f: f,
    }
}

pub struct FromStream<S>(S);

impl<S> StateStream for FromStream<S>
    where S: Stream
{
    type Item = S::Item;
    type State = ();
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, ()>, S::Error> {
        self.0.poll().map(|a| {
            match a {
                Async::Ready(Some(i)) => Async::Ready(StreamEvent::Next(i)),
                Async::Ready(None) => Async::Ready(StreamEvent::Done(())),
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct IntoFuture<S>(Option<S>);

impl<S> Future for IntoFuture<S>
    where S: StateStream
{
    type Item = (StreamEvent<S::Item, S::State>, S);
    type Error = (S::Error, S);

    #[inline]
    fn poll(&mut self) -> Poll<(StreamEvent<S::Item, S::State>, S), (S::Error, S)> {
        let item = match self.0.as_mut().expect("polling IntoFuture twice").poll() {
            Ok(Async::NotReady) => return Ok(Async::NotReady),
            Ok(Async::Ready(i)) => Ok(i),
            Err(e) => Err(e),
        };
        let stream = self.0.take().unwrap();
        match item {
            Ok(i) => Ok(Async::Ready((i, stream))),
            Err(e) => Err((e, stream)),
        }
    }
}

pub struct IntoStream<S>(S);

impl<S> Stream for IntoStream<S>
    where S: StateStream
{
    type Item = S::Item;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<Option<S::Item>, S::Error> {
        self.0.poll().map(|a| {
            match a {
                Async::Ready(StreamEvent::Next(i)) => Async::Ready(Some(i)),
                Async::Ready(StreamEvent::Done(_)) => Async::Ready(None),
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct Map<S, F> {
    stream: S,
    f: F,
}

impl<S, F, B> StateStream for Map<S, F>
    where S: StateStream,
          F: FnMut(S::Item) -> B
{
    type Item = B;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<B, S::State>, S::Error> {
        self.stream.poll().map(|a| {
            match a {
                Async::Ready(StreamEvent::Next(i)) => Async::Ready(StreamEvent::Next((self.f)(i))),
                Async::Ready(StreamEvent::Done(s)) => Async::Ready(StreamEvent::Done(s)),
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct MapErr<S, F> {
    stream: S,
    f: F,
}

impl<S, F, B> StateStream for MapErr<S, F>
    where S: StateStream,
          F: FnMut(S::Error) -> B
{
    type Item = S::Item;
    type State = S::State;
    type Error = B;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, S::State>, B> {
        match self.stream.poll() {
            Ok(a) => Ok(a),
            Err(e) => Err((self.f)(e))
        }
    }
}

pub struct MapState<S, F> {
    stream: S,
    f: Option<F>,
}

impl<S, F, B> StateStream for MapState<S, F>
    where S: StateStream,
          F: FnOnce(S::State) -> B
{
    type Item = S::Item;
    type State = B;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, B>, S::Error> {
        self.stream.poll().map(|a| {
            match a {
                Async::Ready(StreamEvent::Next(i)) => Async::Ready(StreamEvent::Next(i)),
                Async::Ready(StreamEvent::Done(s)) => {
                    let f = self.f.take().expect("polled MapState after completion");
                    Async::Ready(StreamEvent::Done(f(s)))
                }
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct Collect<S>
    where S: StateStream
{
    stream: S,
    items: Vec<S::Item>,
}

impl<S> Future for Collect<S>
    where S: StateStream
{
    type Item = (Vec<S::Item>, S::State);
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<(Vec<S::Item>, S::State), S::Error> {
        loop {
            match self.stream.poll() {
                Ok(Async::Ready(StreamEvent::Next(i))) => self.items.push(i),
                Ok(Async::Ready(StreamEvent::Done(s))) => {
                    let items = mem::replace(&mut self.items, vec![]);
                    return Ok(Async::Ready((items, s)))
                }
                Ok(Async::NotReady) => return Ok(Async::NotReady),
                Err(e) => return Err(e),
            }
        }
    }
}

pub struct Unfold<T, F, Fut>
    where Fut: futures::IntoFuture
{
    state: UnfoldState<T, Fut::Future>,
    f: F,
}

enum UnfoldState<T, F>
{
    Empty,
    Ready(T),
    Processing(F),
}

impl<T, F, Fut, It, St> StateStream for Unfold<T, F, Fut>
    where F: FnMut(T) -> Fut,
          Fut: futures::IntoFuture<Item = StreamEvent<(It, T), St>>
{
    type Item = It;
    type State = St;
    type Error = Fut::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<It, St>, Fut::Error> {
        loop {
            match mem::replace(&mut self.state, UnfoldState::Empty) {
                UnfoldState::Empty => panic!("polled an Unfold after completion"),
                UnfoldState::Ready(state) => {
                    self.state = UnfoldState::Processing((self.f)(state).into_future())
                }
                UnfoldState::Processing(mut fut) => {
                    match try!(fut.poll()) {
                        Async::Ready(StreamEvent::Next((i, state))) => {
                            self.state = UnfoldState::Ready(state);
                            return Ok(Async::Ready(StreamEvent::Next(i)));
                        }
                        Async::Ready(StreamEvent::Done(s)) => {
                            return Ok(Async::Ready(StreamEvent::Done(s)));
                        }
                        Async::NotReady => {
                            self.state = UnfoldState::Processing(fut);
                            return Ok(Async::NotReady);
                        }
                    }
                }
            }
        }
    }
}

enum FlattenStateStreamState<F>
    where F: Future
{
    Future(F),
    Stream(F::Item),
}

pub struct FlattenStateStream<F>(FlattenStateStreamState<F>)
    where F: Future;

impl<F> StateStream for FlattenStateStream<F>
    where F: Future,
          F::Item: StateStream<Error = F::Error>
{
    type Item = <F::Item as StateStream>::Item;
    type State = <F::Item as StateStream>::State;
    type Error = F::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<Self::Item, Self::State>, Self::Error> {
        loop {
            self.0 = match self.0 {
                FlattenStateStreamState::Future(ref mut f) => {
                    match f.poll() {
                        Ok(Async::NotReady) => return Ok(Async::NotReady),
                        Ok(Async::Ready(stream)) => FlattenStateStreamState::Stream(stream),
                        Err(e) => return Err(e),
                    }
                }
                FlattenStateStreamState::Stream(ref mut s) => return s.poll(),
            };
        }
    }
}

pub struct Filter<S, F> {
    stream: S,
    f: F,
}

impl<S, F> StateStream for Filter<S, F>
    where S: StateStream,
          F: FnMut(&S::Item) -> bool
{
    type Item = S::Item;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, S::State>, S::Error> {
        loop {
            match self.stream.poll() {
                Ok(Async::Ready(StreamEvent::Next(i))) => {
                    if (self.f)(&i) {
                        return Ok(Async::Ready(StreamEvent::Next(i)));
                    }
                }
                s => return s,
            }
        }
    }
}

pub struct FilterMap<S, F> {
    stream: S,
    f: F,
}

impl<S, F, B> StateStream for FilterMap<S, F>
    where S: StateStream,
          F: FnMut(S::Item) -> Option<B>
{
    type Item = B;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<B, S::State>, S::Error> {
        loop {
            match self.stream.poll() {
                Ok(Async::Ready(StreamEvent::Next(i))) => {
                    if let Some(i) = (self.f)(i) {
                        return Ok(Async::Ready(StreamEvent::Next(i)));
                    }
                }
                Ok(Async::Ready(StreamEvent::Done(s))) => {
                    return Ok(Async::Ready(StreamEvent::Done(s)));
                }
                Ok(Async::NotReady) => return Ok(Async::NotReady),
                Err(e) => return Err(e),
            }
        }
    }
}<commit_msg>Add then<commit_after>extern crate futures;

use futures::{Async, Poll, Future, Stream};
use std::mem;

pub type BoxStateStream<T, S, E> = Box<StateStream<Item = T, State = S, Error = E> + Send>;

pub enum StreamEvent<I, S> {
    Next(I),
    Done(S),
}

pub trait StateStream {
    type Item;
    type State;
    type Error;

    fn poll(&mut self) -> Poll<StreamEvent<Self::Item, Self::State>, Self::Error>;

    #[inline]
    fn boxed(self) -> BoxStateStream<Self::Item, Self::State, Self::Error>
        where Self: Sized + Send + 'static
    {
        Box::new(self)
    }

    #[inline]
    fn into_future(self) -> IntoFuture<Self>
        where Self: Sized
    {
        IntoFuture(Some(self))
    }

    #[inline]
    fn into_stream(self) -> IntoStream<Self>
        where Self: Sized
    {
        IntoStream(self)
    }

    #[inline]
    fn map<F, B>(self, f: F) -> Map<Self, F>
        where Self: Sized,
              F: FnMut(Self::Item) -> B
    {
        Map {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn map_err<F, B>(self, f: F) -> MapErr<Self, F>
        where Self: Sized,
              F: FnMut(Self::Error) -> B
    {
        MapErr {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn map_state<F, B>(self, f: F) -> MapState<Self, F>
        where Self: Sized,
              F: FnOnce(Self::State) -> B
    {
        MapState {
            stream: self,
            f: Some(f),
        }
    }

    #[inline]
    fn filter<F>(self, f: F) -> Filter<Self, F>
        where Self: Sized,
              F: FnMut(&Self::Item) -> bool
    {
        Filter {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn filter_map<F, B>(self, f: F) -> FilterMap<Self, F>
        where Self: Sized,
              F: FnMut(Self::Item) -> Option<B>
    {

        FilterMap {
            stream: self,
            f: f,
        }
    }

    #[inline]
    fn then<F, U>(self, f: F) -> Then<Self, F, U>
        where Self: Sized,
              F: FnMut(Result<Self::Item, Self::Error>) -> U,
              U: futures::IntoFuture,
              Self::Error: From<U::Error>
    {
        Then {
            stream: self,
            f: f,
            fut: None,
        }
    }

    #[inline]
    fn collect(self) -> Collect<Self>
        where Self: Sized
    {
        Collect {
            stream: self,
            items: vec![],
        }
    }
}

impl<S: ?Sized> StateStream for Box<S>
    where S: StateStream
{
    type Item = S::Item;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, S::State>, S::Error> {
        S::poll(self)
    }
}

pub trait FutureExt: Future {
    #[inline]
    fn flatten_state_stream(self) -> FlattenStateStream<Self>
        where Self: Sized,
              Self::Item: StateStream<Error = Self::Error>
    {
        FlattenStateStream(FlattenStateStreamState::Future(self))
    }
}

impl<F: ?Sized> FutureExt for F
    where F: Future
{}

#[inline]
pub fn stream<S>(stream: S) -> FromStream<S>
    where S: Stream
{
    FromStream(stream)
}

#[inline]
pub fn unfold<T, F, Fut, It, St>(init: T, f: F) -> Unfold<T, F, Fut>
     where F: FnMut(T) -> Fut,
           Fut: futures::IntoFuture<Item = StreamEvent<(It, T), St>>
{
    Unfold {
        state: UnfoldState::Ready(init),
        f: f,
    }
}

pub struct FromStream<S>(S);

impl<S> StateStream for FromStream<S>
    where S: Stream
{
    type Item = S::Item;
    type State = ();
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, ()>, S::Error> {
        self.0.poll().map(|a| {
            match a {
                Async::Ready(Some(i)) => Async::Ready(StreamEvent::Next(i)),
                Async::Ready(None) => Async::Ready(StreamEvent::Done(())),
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct IntoFuture<S>(Option<S>);

impl<S> Future for IntoFuture<S>
    where S: StateStream
{
    type Item = (StreamEvent<S::Item, S::State>, S);
    type Error = (S::Error, S);

    #[inline]
    fn poll(&mut self) -> Poll<(StreamEvent<S::Item, S::State>, S), (S::Error, S)> {
        let item = match self.0.as_mut().expect("polling IntoFuture twice").poll() {
            Ok(Async::NotReady) => return Ok(Async::NotReady),
            Ok(Async::Ready(i)) => Ok(i),
            Err(e) => Err(e),
        };
        let stream = self.0.take().unwrap();
        match item {
            Ok(i) => Ok(Async::Ready((i, stream))),
            Err(e) => Err((e, stream)),
        }
    }
}

pub struct IntoStream<S>(S);

impl<S> Stream for IntoStream<S>
    where S: StateStream
{
    type Item = S::Item;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<Option<S::Item>, S::Error> {
        self.0.poll().map(|a| {
            match a {
                Async::Ready(StreamEvent::Next(i)) => Async::Ready(Some(i)),
                Async::Ready(StreamEvent::Done(_)) => Async::Ready(None),
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct Map<S, F> {
    stream: S,
    f: F,
}

impl<S, F, B> StateStream for Map<S, F>
    where S: StateStream,
          F: FnMut(S::Item) -> B
{
    type Item = B;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<B, S::State>, S::Error> {
        self.stream.poll().map(|a| {
            match a {
                Async::Ready(StreamEvent::Next(i)) => Async::Ready(StreamEvent::Next((self.f)(i))),
                Async::Ready(StreamEvent::Done(s)) => Async::Ready(StreamEvent::Done(s)),
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct MapErr<S, F> {
    stream: S,
    f: F,
}

impl<S, F, B> StateStream for MapErr<S, F>
    where S: StateStream,
          F: FnMut(S::Error) -> B
{
    type Item = S::Item;
    type State = S::State;
    type Error = B;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, S::State>, B> {
        match self.stream.poll() {
            Ok(a) => Ok(a),
            Err(e) => Err((self.f)(e))
        }
    }
}

pub struct MapState<S, F> {
    stream: S,
    f: Option<F>,
}

impl<S, F, B> StateStream for MapState<S, F>
    where S: StateStream,
          F: FnOnce(S::State) -> B
{
    type Item = S::Item;
    type State = B;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, B>, S::Error> {
        self.stream.poll().map(|a| {
            match a {
                Async::Ready(StreamEvent::Next(i)) => Async::Ready(StreamEvent::Next(i)),
                Async::Ready(StreamEvent::Done(s)) => {
                    let f = self.f.take().expect("polled MapState after completion");
                    Async::Ready(StreamEvent::Done(f(s)))
                }
                Async::NotReady => Async::NotReady,
            }
        })
    }
}

pub struct Collect<S>
    where S: StateStream
{
    stream: S,
    items: Vec<S::Item>,
}

impl<S> Future for Collect<S>
    where S: StateStream
{
    type Item = (Vec<S::Item>, S::State);
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<(Vec<S::Item>, S::State), S::Error> {
        loop {
            match self.stream.poll() {
                Ok(Async::Ready(StreamEvent::Next(i))) => self.items.push(i),
                Ok(Async::Ready(StreamEvent::Done(s))) => {
                    let items = mem::replace(&mut self.items, vec![]);
                    return Ok(Async::Ready((items, s)))
                }
                Ok(Async::NotReady) => return Ok(Async::NotReady),
                Err(e) => return Err(e),
            }
        }
    }
}

pub struct Unfold<T, F, Fut>
    where Fut: futures::IntoFuture
{
    state: UnfoldState<T, Fut::Future>,
    f: F,
}

enum UnfoldState<T, F>
{
    Empty,
    Ready(T),
    Processing(F),
}

impl<T, F, Fut, It, St> StateStream for Unfold<T, F, Fut>
    where F: FnMut(T) -> Fut,
          Fut: futures::IntoFuture<Item = StreamEvent<(It, T), St>>
{
    type Item = It;
    type State = St;
    type Error = Fut::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<It, St>, Fut::Error> {
        loop {
            match mem::replace(&mut self.state, UnfoldState::Empty) {
                UnfoldState::Empty => panic!("polled an Unfold after completion"),
                UnfoldState::Ready(state) => {
                    self.state = UnfoldState::Processing((self.f)(state).into_future())
                }
                UnfoldState::Processing(mut fut) => {
                    match try!(fut.poll()) {
                        Async::Ready(StreamEvent::Next((i, state))) => {
                            self.state = UnfoldState::Ready(state);
                            return Ok(Async::Ready(StreamEvent::Next(i)));
                        }
                        Async::Ready(StreamEvent::Done(s)) => {
                            return Ok(Async::Ready(StreamEvent::Done(s)));
                        }
                        Async::NotReady => {
                            self.state = UnfoldState::Processing(fut);
                            return Ok(Async::NotReady);
                        }
                    }
                }
            }
        }
    }
}

enum FlattenStateStreamState<F>
    where F: Future
{
    Future(F),
    Stream(F::Item),
}

pub struct FlattenStateStream<F>(FlattenStateStreamState<F>)
    where F: Future;

impl<F> StateStream for FlattenStateStream<F>
    where F: Future,
          F::Item: StateStream<Error = F::Error>
{
    type Item = <F::Item as StateStream>::Item;
    type State = <F::Item as StateStream>::State;
    type Error = F::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<Self::Item, Self::State>, Self::Error> {
        loop {
            self.0 = match self.0 {
                FlattenStateStreamState::Future(ref mut f) => {
                    match f.poll() {
                        Ok(Async::NotReady) => return Ok(Async::NotReady),
                        Ok(Async::Ready(stream)) => FlattenStateStreamState::Stream(stream),
                        Err(e) => return Err(e),
                    }
                }
                FlattenStateStreamState::Stream(ref mut s) => return s.poll(),
            };
        }
    }
}

pub struct Filter<S, F> {
    stream: S,
    f: F,
}

impl<S, F> StateStream for Filter<S, F>
    where S: StateStream,
          F: FnMut(&S::Item) -> bool
{
    type Item = S::Item;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<S::Item, S::State>, S::Error> {
        loop {
            match self.stream.poll() {
                Ok(Async::Ready(StreamEvent::Next(i))) => {
                    if (self.f)(&i) {
                        return Ok(Async::Ready(StreamEvent::Next(i)));
                    }
                }
                s => return s,
            }
        }
    }
}

pub struct FilterMap<S, F> {
    stream: S,
    f: F,
}

impl<S, F, B> StateStream for FilterMap<S, F>
    where S: StateStream,
          F: FnMut(S::Item) -> Option<B>
{
    type Item = B;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<B, S::State>, S::Error> {
        loop {
            match try!(self.stream.poll()) {
                Async::Ready(StreamEvent::Next(i)) => {
                    if let Some(i) = (self.f)(i) {
                        return Ok(Async::Ready(StreamEvent::Next(i)));
                    }
                }
                Async::Ready(StreamEvent::Done(s)) => {
                    return Ok(Async::Ready(StreamEvent::Done(s)));
                }
                Async::NotReady => return Ok(Async::NotReady),
            }
        }
    }
}

pub struct Then<S, F, U>
    where U: futures::IntoFuture
{
    stream: S,
    f: F,
    fut: Option<U::Future>,
}

impl<S, F, U> StateStream for Then<S, F, U>
    where S: StateStream,
          F: FnMut(Result<S::Item, S::Error>) -> U,
          U: futures::IntoFuture,
          S::Error: From<U::Error>,
{
    type Item = U::Item;
    type State = S::State;
    type Error = S::Error;

    #[inline]
    fn poll(&mut self) -> Poll<StreamEvent<U::Item, S::State>, S::Error> {
        loop {
            if let Some(mut fut) = self.fut.take() {
                match try!(fut.poll()) {
                    Async::Ready(i) => return Ok(Async::Ready(StreamEvent::Next(i))),
                    Async::NotReady => {
                        self.fut = Some(fut);
                        return Ok(Async::NotReady);
                    }
                }
            }

            match self.stream.poll() {
                Ok(Async::Ready(StreamEvent::Next(i))) => {
                    self.fut = Some((self.f)(Ok(i)).into_future());
                }
                Ok(Async::Ready(StreamEvent::Done(s))) => {
                    return Ok(Async::Ready(StreamEvent::Done(s)));
                }
                Ok(Async::NotReady) => return Ok(Async::NotReady),
                Err(e) => self.fut = Some((self.f)(Err(e)).into_future()),
            }
        }
    }
}<|endoftext|>


<commit_before>// use variable names from paper, namely registers M
#![allow(non_snake_case)]

#![crate_name = "basichll"]
#![crate_type = "lib"]

extern crate byteorder;

/* -------------------- std libs ------------------- */
use std::cmp;
use std::fmt;
use std::hash::{ Hash, Hasher, SipHasher };
use std::io::Cursor;
use std::iter::{ repeat, Iterator };
/* ------------------------------------------------- */

use byteorder::{BigEndian, WriteBytesExt, ReadBytesExt};

/* -------------------- helpers -------------------- */

fn alpha (m: usize) -> f64 {
    match m {
        16 => 0.673,
        32 => 0.697,
        64 => 0.709,
        _  => 0.7213/(1f64 + 1.079/(m as f64))
    }
}

fn leftmost_one_bit (v: u64) -> usize {
    let mut counter: usize = 0;
    let mut shiftable = v;

    while shiftable > 0 {
        shiftable >>= 1;
        counter += 1;
    }

    64 - counter
}

/* ------------------------------------------------- */

/* --------------- hyper log log ------------------- */

/// The `HLL` struct stores the underlying registers
/// of the HyperLogLog, with `m` many registers.
///
/// The value for `m` is derived from the desired error
/// of the estimation. The error is considered to be
/// 1.04 / sqrt(2 ^ bits). So, given a desired 1% error
/// rate, we result in around 12 bits per register,
/// producing an underlying M of 2^12 bits (i.e. 4kb).
#[derive(Clone, PartialEq)]
pub struct HLL {
    alpha: f64,
    b: u32,
    m: usize,
    M: Vec<u8>,
    // [!] this is pretty much a hack until I can think of something
    //     better to do for the monoid instance. Basically we need the
    //     zero value to always satisfy the left and right identity laws
    //     but because you need to have the number of registers be
    //     equivalent to do addition, it implies you need a distinct
    //     monoid instance per HLL configuration (i.e. an HLL of 4
    //     registers has a distinct zero from a HLL of 16).
    //
    //     Because I can't really do dependent types in rust, we instead
    //     have a distinguished zero element, that is allowed to bypass
    //     the assertions on the equivalent sizes...
    //
    //     For reference, Ed Kmett's haskell implementation actually does
    //     use higher kinded types and some tricky reflection to make the
    //     monoid instance be parameterised by a config type that is itself
    //     typed to a Peano natural encoded type. Twitter's algebird instance
    //     declares a separate class that encapsulates the notion of
    //     a "HyperLogLogMonoid" instance, and instances of that class are
    //     instantiated with the register size (i.e. the 'instance' for monoid
    //     is just an 'instance' of the class, for two interpretations of
    //     'instance').
    isZero: bool
}

impl std::cmp::Eq for HLL {}

impl HLL {
    /// Convenience function to produce a HLL with
    /// one hundred and twenty eight registers.
    pub fn one_hundred_twenty_eight () -> HLL {
        HLL::new(0.09192)
    }

    /// Create a new HLL with the desired standard error.
    /// Some examples might be:
    ///
    ///```ignore
    ///     bits    size    error
    ///     12      4096    0.0163
    ///     13      8192    0.0115
    ///     14      16384   0.0081
    ///```
    ///
    /// The error must be between 0.0 and 1.0. Beware using a stupidly
    /// small error size will grow beyond e.g. isize at less than 0.03
    /// (if you need something that isn't an estimate probably don't use
    /// a hyperloglog).
    pub fn new(error: f64) -> HLL {
        assert!(error > 0.0 && error < 1.0);
        // error = 1.04 / sqrt(m)
        let m = ((1.04/error) * (1.04/error)).floor() as usize;
        let b = (m as f64).log2() as u32;

        HLL {
            alpha: alpha(m),
            b: b,
            m: m,
            M: repeat(0u8).take(m).collect(),
            isZero: false
        }
    }

    /// Add an element into the hyperloglog estimate.
    /// We require the type of value to be able to be hashed.
    /// Returns whether the insertion altered the hyperloglog
    pub fn insert<T: Hash>(&mut self, val: &T) -> bool {
        let mut hasher = SipHasher::new();
        val.hash(&mut hasher);
        let hash = hasher.finish();

        // j is the first b many bits
        let j = (hash >> (64 - self.b)) as usize;
        // w is the remaining bits (i.e. b -> 64)
        let w = hash & (2u64.pow(64 - self.b) - 1) as u64;
        let rho = leftmost_one_bit(w) as u8;

        let prev = self.M[j];
        self.M[j] = cmp::max(self.M[j], rho);

        prev != self.M[j]
    }

    /// Return the estimated cardinality of the observed set
    /// of elements.
    pub fn count(&self) -> f64 {
        self.range_correction(self.raw_estimate())
    }

    fn raw_estimate(&self) -> f64 {
        let sum: f64 = self.M.iter().fold (0.0, |p, &r| p + 2.0f64.powi(-(r as i32)));

        self.alpha as f64 * ((self.m  * self.m) as f64) * (1.0 / sum)
    }

    fn empty_registers (&self) -> usize {
        self.M.iter().filter(|&r| *r == 0).count()
    }

    fn range_correction(&self, e: f64) -> f64 {
        let twoTo32: f64 = 2.0f64.powi(32) as f64;

        // small range correction
        if e <= (5.0*(self.m as f64))/2.0 {
            let v  = self.empty_registers() as f64;
            let fm = self.m as f64;

            return match v {
                0.0 => e,
                _   => fm * (fm / v).ln()
            }
        }

        // medium range (no) correction
        if e <= twoTo32/30.0 {
            return e;
        }

        -twoTo32 * (1.0 - e/twoTo32).ln()
    }

    /// Access a copy of the underlying registers (this is mainly
    /// here just for debugging/testing... its unlikely you'll need
    /// this access typically).
    pub fn registers(&self) -> Vec<u8> {
        self.M.clone()
    }

    /// A completely zeroed HLL. Not particularly useful
    /// except as an identity element in `Add`.
    pub fn empty () -> HLL {
        HLL {
            alpha: 0.0,
            m: 0,
            b: 0,
            M: Vec::new(),
            isZero: true
        }
    }

    pub fn into_vec(self) -> Vec<u8> {
        let mut data = self.M;
        data.reserve(21);
        data.write_f64::<BigEndian>(self.alpha).unwrap();
        data.write_u32::<BigEndian>(self.b).unwrap();
        data.write_u64::<BigEndian>(self.m as u64).unwrap();
        data.write_u8(if self.isZero { 0 } else { 1 }).unwrap();
        data
    }

    pub fn from_vec(data: Vec<u8>) -> HLL {
        assert!(data.len() >= 21);
        let mut M = data;
        // drain or split_off would be useful, but are not stable yet
        let mut metadata = [0u8; 21];
        for i in 0..21 {
            metadata[20 - i] = M.pop().unwrap();
        }
        let mut readable = Cursor::new(metadata.to_vec());
        let alpha = readable.read_f64::<BigEndian>().unwrap();
        let b = readable.read_u32::<BigEndian>().unwrap();
        let m = readable.read_u64::<BigEndian>().unwrap() as usize;
        let isZero = readable.read_u8().unwrap() == 0;
        HLL {
            M: M,
            alpha: alpha,
            b: b,
            m: m,
            isZero: isZero,
        }
    }
}

/* ------------------------------------------------- */

/* --------------- trait instances ----------------- */

impl std::fmt::Display for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

impl std::fmt::Debug for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

fn zipWith<R, U: Iterator, C: Fn(U::Item, U::Item) -> R> (combo: C, left: U, right: U)
   -> Vec<R> {
    left.zip(right).map(| (l, r) | combo(l, r)).collect()
}

fn mergeRegisters (first: &Vec<u8>, second: &Vec<u8>) -> Vec<u8> {
    zipWith(|&l, &r| std::cmp::max(l, r), first.iter(), second.iter())
}

/// Adding together two HLLs produces a new HLL where
/// the larger value of each register has been selected.
impl<'a> std::ops::Add<&'a HLL> for &'a HLL {
    type Output = HLL;

    fn add(self, other: &'a HLL) -> HLL {
        // [!] gross...
        if self.isZero { return other.clone(); }
        if other.isZero { return self.clone(); }

        assert!(self.alpha == other.alpha);
        assert!(self.b == other.b);
        assert!(self.m == other.m);

        HLL {
            alpha: self.alpha,
            b: self.b,
            m: self.m,
            M: mergeRegisters(&self.M, &other.M),
            isZero: false
        }
    }
}

/* ------------------------------------------------- */
<commit_msg>:facepunch: whitespace<commit_after>// use variable names from paper, namely registers M
#![allow(non_snake_case)]

#![crate_name = "basichll"]
#![crate_type = "lib"]

extern crate byteorder;

/* -------------------- std libs ------------------- */
use std::cmp;
use std::fmt;
use std::hash::{ Hash, Hasher, SipHasher };
use std::io::Cursor;
use std::iter::{ repeat, Iterator };
/* ------------------------------------------------- */

use byteorder::{ BigEndian, WriteBytesExt, ReadBytesExt };

/* -------------------- helpers -------------------- */

fn alpha (m: usize) -> f64 {
    match m {
        16 => 0.673,
        32 => 0.697,
        64 => 0.709,
        _  => 0.7213/(1f64 + 1.079/(m as f64))
    }
}

fn leftmost_one_bit (v: u64) -> usize {
    let mut counter: usize = 0;
    let mut shiftable = v;

    while shiftable > 0 {
        shiftable >>= 1;
        counter += 1;
    }

    64 - counter
}

/* ------------------------------------------------- */

/* --------------- hyper log log ------------------- */

/// The `HLL` struct stores the underlying registers
/// of the HyperLogLog, with `m` many registers.
///
/// The value for `m` is derived from the desired error
/// of the estimation. The error is considered to be
/// 1.04 / sqrt(2 ^ bits). So, given a desired 1% error
/// rate, we result in around 12 bits per register,
/// producing an underlying M of 2^12 bits (i.e. 4kb).
#[derive(Clone, PartialEq)]
pub struct HLL {
    alpha: f64,
    b: u32,
    m: usize,
    M: Vec<u8>,
    // [!] this is pretty much a hack until I can think of something
    //     better to do for the monoid instance. Basically we need the
    //     zero value to always satisfy the left and right identity laws
    //     but because you need to have the number of registers be
    //     equivalent to do addition, it implies you need a distinct
    //     monoid instance per HLL configuration (i.e. an HLL of 4
    //     registers has a distinct zero from a HLL of 16).
    //
    //     Because I can't really do dependent types in rust, we instead
    //     have a distinguished zero element, that is allowed to bypass
    //     the assertions on the equivalent sizes...
    //
    //     For reference, Ed Kmett's haskell implementation actually does
    //     use higher kinded types and some tricky reflection to make the
    //     monoid instance be parameterised by a config type that is itself
    //     typed to a Peano natural encoded type. Twitter's algebird instance
    //     declares a separate class that encapsulates the notion of
    //     a "HyperLogLogMonoid" instance, and instances of that class are
    //     instantiated with the register size (i.e. the 'instance' for monoid
    //     is just an 'instance' of the class, for two interpretations of
    //     'instance').
    isZero: bool
}

impl std::cmp::Eq for HLL {}

impl HLL {
    /// Convenience function to produce a HLL with
    /// one hundred and twenty eight registers.
    pub fn one_hundred_twenty_eight () -> HLL {
        HLL::new(0.09192)
    }

    /// Create a new HLL with the desired standard error.
    /// Some examples might be:
    ///
    ///```ignore
    ///     bits    size    error
    ///     12      4096    0.0163
    ///     13      8192    0.0115
    ///     14      16384   0.0081
    ///```
    ///
    /// The error must be between 0.0 and 1.0. Beware using a stupidly
    /// small error size will grow beyond e.g. isize at less than 0.03
    /// (if you need something that isn't an estimate probably don't use
    /// a hyperloglog).
    pub fn new(error: f64) -> HLL {
        assert!(error > 0.0 && error < 1.0);
        // error = 1.04 / sqrt(m)
        let m = ((1.04/error) * (1.04/error)).floor() as usize;
        let b = (m as f64).log2() as u32;

        HLL {
            alpha: alpha(m),
            b: b,
            m: m,
            M: repeat(0u8).take(m).collect(),
            isZero: false
        }
    }

    /// Add an element into the hyperloglog estimate.
    /// We require the type of value to be able to be hashed.
    /// Returns whether the insertion altered the hyperloglog
    pub fn insert<T: Hash>(&mut self, val: &T) -> bool {
        let mut hasher = SipHasher::new();
        val.hash(&mut hasher);
        let hash = hasher.finish();

        // j is the first b many bits
        let j = (hash >> (64 - self.b)) as usize;
        // w is the remaining bits (i.e. b -> 64)
        let w = hash & (2u64.pow(64 - self.b) - 1) as u64;
        let rho = leftmost_one_bit(w) as u8;

        let prev = self.M[j];
        self.M[j] = cmp::max(self.M[j], rho);

        prev != self.M[j]
    }

    /// Return the estimated cardinality of the observed set
    /// of elements.
    pub fn count(&self) -> f64 {
        self.range_correction(self.raw_estimate())
    }

    fn raw_estimate(&self) -> f64 {
        let sum: f64 = self.M.iter().fold (0.0, |p, &r| p + 2.0f64.powi(-(r as i32)));

        self.alpha as f64 * ((self.m  * self.m) as f64) * (1.0 / sum)
    }

    fn empty_registers (&self) -> usize {
        self.M.iter().filter(|&r| *r == 0).count()
    }

    fn range_correction(&self, e: f64) -> f64 {
        let twoTo32: f64 = 2.0f64.powi(32) as f64;

        // small range correction
        if e <= (5.0*(self.m as f64))/2.0 {
            let v  = self.empty_registers() as f64;
            let fm = self.m as f64;

            return match v {
                0.0 => e,
                _   => fm * (fm / v).ln()
            }
        }

        // medium range (no) correction
        if e <= twoTo32/30.0 {
            return e;
        }

        -twoTo32 * (1.0 - e/twoTo32).ln()
    }

    /// Access a copy of the underlying registers (this is mainly
    /// here just for debugging/testing... its unlikely you'll need
    /// this access typically).
    pub fn registers(&self) -> Vec<u8> {
        self.M.clone()
    }

    /// A completely zeroed HLL. Not particularly useful
    /// except as an identity element in `Add`.
    pub fn empty () -> HLL {
        HLL {
            alpha: 0.0,
            m: 0,
            b: 0,
            M: Vec::new(),
            isZero: true
        }
    }

    pub fn into_vec(self) -> Vec<u8> {
        let mut data = self.M;
        data.reserve(21);
        data.write_f64::<BigEndian>(self.alpha).unwrap();
        data.write_u32::<BigEndian>(self.b).unwrap();
        data.write_u64::<BigEndian>(self.m as u64).unwrap();
        data.write_u8(if self.isZero { 0 } else { 1 }).unwrap();
        data
    }

    pub fn from_vec(data: Vec<u8>) -> HLL {
        assert!(data.len() >= 21);
        let mut M = data;
        // drain or split_off would be useful, but are not stable yet
        let mut metadata = [0u8; 21];
        for i in 0..21 {
            metadata[20 - i] = M.pop().unwrap();
        }
        let mut readable = Cursor::new(metadata.to_vec());
        let alpha = readable.read_f64::<BigEndian>().unwrap();
        let b = readable.read_u32::<BigEndian>().unwrap();
        let m = readable.read_u64::<BigEndian>().unwrap() as usize;
        let isZero = readable.read_u8().unwrap() == 0;
        HLL {
            M: M,
            alpha: alpha,
            b: b,
            m: m,
            isZero: isZero,
        }
    }
}

/* ------------------------------------------------- */

/* --------------- trait instances ----------------- */

impl std::fmt::Display for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

impl std::fmt::Debug for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

fn zipWith<R, U: Iterator, C: Fn(U::Item, U::Item) -> R> (combo: C, left: U, right: U)
   -> Vec<R> {
    left.zip(right).map(| (l, r) | combo(l, r)).collect()
}

fn mergeRegisters (first: &Vec<u8>, second: &Vec<u8>) -> Vec<u8> {
    zipWith(|&l, &r| std::cmp::max(l, r), first.iter(), second.iter())
}

/// Adding together two HLLs produces a new HLL where
/// the larger value of each register has been selected.
impl<'a> std::ops::Add<&'a HLL> for &'a HLL {
    type Output = HLL;

    fn add(self, other: &'a HLL) -> HLL {
        // [!] gross...
        if self.isZero { return other.clone(); }
        if other.isZero { return self.clone(); }

        assert!(self.alpha == other.alpha);
        assert!(self.b == other.b);
        assert!(self.m == other.m);

        HLL {
            alpha: self.alpha,
            b: self.b,
            m: self.m,
            M: mergeRegisters(&self.M, &other.M),
            isZero: false
        }
    }
}

/* ------------------------------------------------- */
<|endoftext|>


<commit_before>extern crate getopts;

use getopts::{Options, ParsingStyle};
use std::io::{stdout, stderr, Write};
use std::fmt;
use std::iter::repeat;

/// Version of this crate. It is parsed from the environment variable that
/// cargo sets when building the crate.
pub const VERSION: &'static str         = env!("CARGO_PKG_VERSION");

/// Name of this tool.
pub const NAME: &'static str            = "yes";

/// A description of this tool.
pub const DESCRIPTION: &'static str     = "Repeats a given string and a newline \
character infinitely on stdout.";

/// The default string to repeat.
pub const DEFAULT_STRING: &'static str  = "y";

/// The buffer size to use when printing the repeated string.
pub const BUFSIZE: usize                = 16384;

/// Represents a request, this tells the yes utility what it has to do.
/// Depending on the options passed on the command line, we might take
/// different actions, which are represented here.
#[derive(Debug, PartialEq)]
pub enum Request {
    /// Main purpose of this utility: send a given string followed by a newline
    /// repeatedly to `stdout`.
    RepeatString(String),

    /// Print help information to `stdout`.
    ShowHelp,

    /// Print the utility’s version number to `stdout`.
    ShowVersion
}

/// These are all the errors that can occur in this utility.
#[derive(Debug, PartialEq)]
pub enum Error {

    /// An invalid option was encountered while parsing the options.
    InvalidOption(getopts::Fail)
}

use Request::*;
use Error::*;

impl Error {
    /// Returns the error message associated with the error.
    pub fn message(&self) -> String {
        match *self {
            InvalidOption(ref fail) => fail.to_string()
        }
    }

    /// Prints the error to stderr.
    pub fn show(&self) {
        println!("{}", self.message());
    }
}

impl fmt::Display for Request {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ShowHelp            => write!(f, "{}", help_text()),
            ShowVersion         => write!(f, "{}", version_text()),
            RepeatString(ref s) => write!(f, "{}", s)
        }
    }
}

impl Request {
    /// Do the request. This performs the action suggested by the request.
    pub fn do_it(&self) {
        match *self {
            ShowHelp => println!("{}", self),
            ShowVersion => println!("{}", self),
            RepeatString(ref s) => repeat_string(s)
        }
    }
}

/// Create an options object.
fn options() -> Options {
    let mut opts = Options::new();
    opts.parsing_style(ParsingStyle::StopAtFirstFree);

    // command-line options that this utility supports
    opts.optflag("h", "help", "display help text and exit");
    opts.optflag("v", "version", "output version information and exit");

    opts
}

/// Parse the command line arguments as options, returning either the parsed
/// `Request` or an `Error`, depending on the arguments.
pub fn parse_args(args: Vec<String>) -> Result<Request, Error> {
    // parse the command-line arguments, and return on error.
    let matches = match options().parse(args) {
        Ok(matches)  => matches,
        Err(failure) => return Err(InvalidOption(failure))
    };

    // if the help flag is set, we want to show the help text.
    if matches.opt_present("help") {
        return Ok(ShowHelp);
    }

    // if the version flag is set, we want to show the version.
    if matches.opt_present("version") {
        return Ok(ShowVersion);
    }

    // if there is no string set, we use the default one, otherwise we take the
    // free strings and join them together.
    let string = if matches.free.is_empty() {
        DEFAULT_STRING.to_owned()
    } else {
        matches.free.join(" ")
    };

    Ok(RepeatString(string))
}

#[test]
fn test_parse_args() {
    // test help options
    assert_eq!(parse_args(vec!["-h".to_string()]), Ok(ShowHelp));
    assert_eq!(parse_args(vec!["--help".to_string()]), Ok(ShowHelp));

    // test version options
    assert_eq!(parse_args(vec!["-v".to_string()]), Ok(ShowVersion));
    assert_eq!(parse_args(vec!["--version".to_string()]), Ok(ShowVersion));
}

/// Generates a help text using the `getopts` crate. It is derived from the
/// `NAME`, the `DESCRIPTION`, as well as from the options.
pub fn help_text() -> String {
    let opts = options();
    let short_usage = format!("{} [STRING]", opts.short_usage(NAME));
    opts.usage(&short_usage)
}

/// Generates a version text, uses the `VERSION` as well as the `NAME` consts.
pub fn version_text() -> String {
    format!("{} version {}", NAME, VERSION)
}

/// Print a string repeatedly to `stdout`.
pub fn repeat_string(string: &str) {
    // creates a buffer, containing a lot of repetitions of our string
    let buf = repeat_buffer(string);

    // create a handle to stdout
    let stdout = stdout();
    let mut stdout = stdout.lock();

    // continuously print to stdout
    loop {
        stdout.write_all(buf.as_bytes());
    }
}

/// Generates a buffer, which is maximally `BUFSIZE` big, consisting of the
/// string repeated as many times as will fit into the buffer, joined with
/// newline characters.
pub fn repeat_buffer(string: &str) -> String {
    // append newline to string
    let mut s = string.to_owned();
    s.push('\n');

    // calculate how many times our string fits into a bufsize
    let times = BUFSIZE / s.len();

    // repeat the string this many times and join it together
    repeat(s).take(times).collect::<Vec<String>>().join("")
}
<commit_msg>expands command option tests<commit_after>extern crate getopts;

use getopts::{Options, ParsingStyle};
use std::io::{stdout, Write};
use std::fmt;
use std::iter::repeat;

/// Version of this crate. It is parsed from the environment variable that
/// cargo sets when building the crate.
pub const VERSION: &'static str         = env!("CARGO_PKG_VERSION");

/// Name of this tool.
pub const NAME: &'static str            = "yes";

/// A description of this tool.
pub const DESCRIPTION: &'static str     = "Repeats a given string and a newline \
character infinitely on stdout.";

/// The default string to repeat.
pub const DEFAULT_STRING: &'static str  = "y";

/// The buffer size to use when printing the repeated string.
pub const BUFSIZE: usize                = 16384;

/// Represents a request, this tells the yes utility what it has to do.
/// Depending on the options passed on the command line, we might take
/// different actions, which are represented here.
#[derive(Debug, PartialEq)]
pub enum Request {
    /// Main purpose of this utility: send a given string followed by a newline
    /// repeatedly to `stdout`.
    RepeatString(String),

    /// Print help information to `stdout`.
    ShowHelp,

    /// Print the utility’s version number to `stdout`.
    ShowVersion
}

/// These are all the errors that can occur in this utility.
#[derive(Debug, PartialEq)]
pub enum Error {

    /// An invalid option was encountered while parsing the options.
    InvalidOption(getopts::Fail)
}

use Request::*;
use Error::*;

impl Error {
    /// Returns the error message associated with the error.
    pub fn message(&self) -> String {
        match *self {
            InvalidOption(ref fail) => fail.to_string()
        }
    }

    /// Prints the error to stderr.
    pub fn show(&self) {
        eprintln!("{}", self.message());
    }
}

impl fmt::Display for Request {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ShowHelp            => write!(f, "{}", help_text()),
            ShowVersion         => write!(f, "{}", version_text()),
            RepeatString(ref s) => write!(f, "{}", s)
        }
    }
}

impl Request {
    /// Do the request. This performs the action suggested by the request.
    pub fn do_it(&self) {
        match *self {
            ShowHelp => println!("{}", self),
            ShowVersion => println!("{}", self),
            RepeatString(ref s) => repeat_string(s)
        }
    }
}

/// Create an options object.
fn options() -> Options {
    let mut opts = Options::new();
    opts.parsing_style(ParsingStyle::StopAtFirstFree);

    // command-line options that this utility supports
    opts.optflag("h", "help", "display help text and exit");
    opts.optflag("v", "version", "output version information and exit");

    opts
}

/// Parse the command line arguments as options, returning either the parsed
/// `Request` or an `Error`, depending on the arguments.
pub fn parse_args(args: Vec<String>) -> Result<Request, Error> {
    // parse the command-line arguments, and return on error.
    let matches = match options().parse(args) {
        Ok(matches)  => matches,
        Err(failure) => return Err(InvalidOption(failure))
    };

    // if the help flag is set, we want to show the help text.
    if matches.opt_present("help") {
        return Ok(ShowHelp);
    }

    // if the version flag is set, we want to show the version.
    if matches.opt_present("version") {
        return Ok(ShowVersion);
    }

    // if there is no string set, we use the default one, otherwise we take the
    // free strings and join them together.
    let string = if matches.free.is_empty() {
        DEFAULT_STRING.to_owned()
    } else {
        matches.free.join(" ")
    };

    Ok(RepeatString(string))
}

#[test]
fn test_parse_args() {
    // test help options
    assert_eq!(parse_args(vec!["-h".to_string()]), Ok(ShowHelp));
    assert_eq!(parse_args(vec!["--help".to_string()]), Ok(ShowHelp));

    // test version options
    assert_eq!(parse_args(vec!["-v".to_string()]), Ok(ShowVersion));
    assert_eq!(parse_args(vec!["--version".to_string()]), Ok(ShowVersion));

    // use the DEAFULT_STRING if nothing is specified
    assert_eq!(parse_args(vec![]), Ok(RepeatString(DEFAULT_STRING.to_string())));

    // use whatever is given on command line
    assert_eq!(parse_args(vec!["str".to_string()]),
               Ok(RepeatString("str".to_string())));

    // join multiple arguments together
    assert_eq!(parse_args(vec!["str".to_string(), "dup".to_string()]),
               Ok(RepeatString("str dup".to_string())));

    // allow command line flags as string
    assert_eq!(parse_args(vec!["--".to_string(), "-h".to_string()]),
               Ok(RepeatString("-h".to_string())));
}

/// Generates a help text using the `getopts` crate. It is derived from the
/// `NAME`, the `DESCRIPTION`, as well as from the options.
pub fn help_text() -> String {
    let opts = options();
    let short_usage = format!("{} [STRING]", opts.short_usage(NAME));
    opts.usage(&short_usage)
}

/// Generates a version text, uses the `VERSION` as well as the `NAME` consts.
pub fn version_text() -> String {
    format!("{} version {}", NAME, VERSION)
}

/// Print a string repeatedly to `stdout`.
pub fn repeat_string(string: &str) {
    // creates a buffer, containing a lot of repetitions of our string
    let buf = repeat_buffer(string);

    // create a handle to stdout
    let stdout = stdout();
    let mut stdout = stdout.lock();

    // continuously print to stdout
    loop {
        stdout.write_all(buf.as_bytes());
    }
}

/// Generates a buffer, which is maximally `BUFSIZE` big, consisting of the
/// string repeated as many times as will fit into the buffer, joined with
/// newline characters.
pub fn repeat_buffer(string: &str) -> String {
    // append newline to string
    let mut s = string.to_owned();
    s.push('\n');

    // calculate how many times our string fits into a bufsize
    let times = BUFSIZE / s.len();

    // repeat the string this many times and join it together
    repeat(s).take(times).collect::<Vec<String>>().join("")
}
<|endoftext|>


<commit_before>#![feature(zero_one, box_syntax)]

extern crate asexp;

pub mod num_expr;
pub mod cond;

use std::fmt::Debug;
use std::cmp::{PartialEq, PartialOrd};

#[derive(Debug, Eq, PartialEq)]
pub enum ExpressionError {
    /// In case of division by zero.
    DivByZero,
    /// In case an invalid variable in references from the expression.
    InvalidVariable,
    /// In case of an invalid operation.
    InvalidOperation,
}

pub trait ElementType: Debug + Copy + Clone + PartialEq + PartialOrd {}

pub trait Expression: Debug + Clone + PartialEq
{
    type Element: ElementType;
    /// Evaluates the expression with the given variables bound.
    fn evaluate(&self, variables: &[Self]) -> Result<Self::Element, ExpressionError>;

    /// Constructs a constant expression. Required by lindenmayer-system.
    fn make_const(elm: Self::Element) -> Self;
}

pub trait Condition: Debug + Clone + PartialEq
{
    type Expr: Expression;
    /// Evaluate the condition with the given variables bound.
    fn evaluate(&self, variables: &[Self::Expr]) -> Result<bool, ExpressionError>;
}
<commit_msg>Add Default to ElementType trait. Required by lindenmayer-system for now<commit_after>#![feature(zero_one, box_syntax)]

extern crate asexp;

pub mod num_expr;
pub mod cond;

use std::fmt::Debug;
use std::cmp::{PartialEq, PartialOrd};

#[derive(Debug, Eq, PartialEq)]
pub enum ExpressionError {
    /// In case of division by zero.
    DivByZero,
    /// In case an invalid variable in references from the expression.
    InvalidVariable,
    /// In case of an invalid operation.
    InvalidOperation,
}

pub trait ElementType: Debug + Copy + Clone + PartialEq + PartialOrd + Default {}

pub trait Expression: Debug + Clone + PartialEq
{
    type Element: ElementType;
    /// Evaluates the expression with the given variables bound.
    fn evaluate(&self, variables: &[Self]) -> Result<Self::Element, ExpressionError>;

    /// Constructs a constant expression. Required by lindenmayer-system.
    fn make_const(elm: Self::Element) -> Self;
}

pub trait Condition: Debug + Clone + PartialEq
{
    type Expr: Expression;
    /// Evaluate the condition with the given variables bound.
    fn evaluate(&self, variables: &[Self::Expr]) -> Result<bool, ExpressionError>;
}
<|endoftext|>