<commit_before>#[cfg(feature="trace_log")] extern crate env_logger;

extern crate radeco_lib;
extern crate r2pipe;
extern crate r2api;
extern crate petgraph;

mod cli;

use std::env;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
use std::process;

use radeco_lib::analysis::cse::cse::CSE;
use radeco_lib::analysis::sccp;
//use radeco_lib::analysis::valueset::analyzer_wysinwyx::FnAnalyzer;
//use radeco_lib::analysis::valueset::mem_structs::{A_Loc,AbstractAddress};
use radeco_lib::analysis::interproc::fixcall::CallFixer;
use radeco_lib::frontend::radeco_containers::ProjectLoader;
use radeco_lib::middle::{dce, dot};
use radeco_lib::middle::ir_writer;
use radeco_lib::middle::ir_reader::parse_il;
use radeco_lib::middle::ssa::verifier;
use radeco_lib::backend::lang_c::{c_simple_ast, c_simple_ast_builder};

//use radeco_lib::analysis::mark_refs;

const USAGE: &'static str = "
Usage: minidec [-f <names>...] <target>

Options:
    -f, --functions  Analayze only some functions
";

fn main() {
    #[cfg(feature="trace_log")] env_logger::init();

    let requested_functions = cli::init_for_args(USAGE);

    let proj_name = env::args().nth(env::args().len() - 1).unwrap();
    let mut rproj = {
        ProjectLoader::new().path(&proj_name).load()
    };
    let regfile = rproj.regfile().clone();
    for mut xy in rproj.iter_mut() {
        let rmod = &mut xy.module;
        let mut dir = PathBuf::from(".");
        dir.push(format!("{}_out", proj_name));
        fs::create_dir_all(&dir).expect("Failed to create directory");

        // Reduce the complexity of rmod.functions to just a vec of (u64,&String)
        // for easier extraction and matching

        // Analyze preserved for all functions.
        {
            println!("[*] Fixing Callee Information");
            let bp_name = regfile.get_name_by_alias(&"BP".to_string());
            let bp_name = bp_name.map(|s| s.to_owned());
            let sp_name = regfile.get_name_by_alias(&"SP".to_string());
            let sp_name = sp_name.map(|s| s.to_owned());
            let mut callfixer = CallFixer::new(rmod, bp_name, sp_name);
            callfixer.rounded_analysis();
        }

        // Fix call sites
        radeco_lib::analysis::functions::fix_ssa_opcalls::go(rmod);

        // Infer calling conventions
        radeco_lib::analysis::functions::infer_regusage::run(rmod, &*regfile);

        // Filter the data if the user provided some args to be matched upon
        let matched_func_addrs = if requested_functions.len() != 0 {

            //XXX Impl in more efficient way
            let mut matched_func_vec: Vec<(u64, &str)> =
                rmod.iter().map(|_f| {
                    let f = _f.function.1;
                    (f.offset.clone(), &*f.name)
                }).collect();

            let all_func_names: Vec<(&str)> =
                matched_func_vec.iter().map(|&(_, name)| name).collect();
            matched_func_vec = filter_with(&matched_func_vec,
                                           &requested_functions.iter().map(|s| &s[..]).collect::<Vec<_>>());
            cli::print_match_summary(&matched_func_vec,
                                     &requested_functions.iter().map(|s| &s[..]).collect::<Vec<_>>(),
                                     &all_func_names);
            matched_func_vec.into_iter().map(|(addr, _)| addr).collect::<Vec<_>>()
        } else {
            Vec::new()
        };

        // Main file to contain IRs of all rfns
        let mut ffm = {
            let mut fname = PathBuf::from(&dir);
            fname.push("main");
            File::create(&fname).expect("Unable to create file")
        };

        for addr in matched_func_addrs {

            let ref mut rfn = rmod.functions.get_mut(&addr).unwrap();

            println!("[+] Analyzing: {} @ {:#x}", rfn.name, addr);
            {
                println!("  [*] Eliminating Dead Code");
                dce::collect(rfn.ssa_mut());
            }
            let mut ssa = {
                // Constant Propagation (sccp)
                println!("  [*] Propagating Constants");
                let mut analyzer = sccp::Analyzer::new(rfn.ssa_mut());
                analyzer.analyze();
                analyzer.emit_ssa()
            };
            {
                println!("  [*] Eliminating More DeadCode");
                dce::collect(&mut ssa);
            }
            *rfn.ssa_mut() = ssa;
            {
                // Common SubExpression Elimination (cse)
                println!("  [*] Eliminating Common SubExpressions");
                let mut cse = CSE::new(rfn.ssa_mut());
                cse.run();
            }
            {
                // Verify SSA
                println!("  [*] Verifying SSA's Validity");
                match verifier::verify(rfn.ssa()) {
                    Err(e) => {
                        println!("  [*] Found Error: {}", e);
                        process::exit(255);
                    }
                    Ok(_) => {  }
                }
            }
            {
                // TODO
                // // Building memory SSA.
                // let _memory_ssa = {
                //     // Generate MemorySSA
                //     println!("  [*] Generating Memory SSA");
                //     let mut mssa = MemorySSA::new(rfn.ssa());
                //     mssa.gather_variables(rfn.datarefs(), &rfn.locals(),
                //                           &rfn.call_refs(&rmod.callgraph));
                //     mssa.run();
                //     mssa
                // };
            }
            //if false {
            //    if (!rfn.name.eq("sym.main")) & (!rfn.name.eq("main")) {
            //        continue;
            //    }
            //    println!("  [*] Analyzing Value Sets");
            //    let fn_analyzer = FnAnalyzer::from((*rfn).clone());
            //    let a_store_fn = fn_analyzer.analyze_rfn();
            //    for (a_loc, strided_interval) in a_store_fn.store {
            //        if let A_Loc{addr: AbstractAddress::Node{..}, ..} = a_loc {
            //            continue;
            //        };
            //        println!("{}", a_loc);
            //        println!("Strided Interval: {}", strided_interval);
            //    };
            //}

            //{
                //// Expreimental reference marking pass
                //println!("  [*] Unstable> Marking References");
                //let mut rmark = mark_refs::ReferenceMarker { };
                //rmark.resolve_refs(&mut rfn.ssa);
            //}

            let mut fname = PathBuf::from(&dir);
            fname.push(rfn.name.as_ref());

            let fn_ir_str = {
                ///////////////////////////
                // Write out the IR file
                //////////////////////////
                println!("  [*] Writing out IR");
                let mut res = String::new();
                ir_writer::emit_il(&mut res, Some(rfn.name.to_string()), rfn.ssa()).unwrap();
                let mut ff = File::create(&fname).expect("Unable to create file");
                write!(ff, "{}", res).expect("Error writing to file");
                writeln!(ffm, "{}", res).expect("Error writing to file");
                // Set as a comment in radare2
                match rmod.source.as_mut().unwrap().send(format!("CC, {} @ {}", fname.to_str().unwrap(), addr)) {
                    Ok(_) => {},
                    Err(e) => println!("{:?}", e),
                };
                res
            };

            {
                //////////////////////////////
                // Try parsing the IR file
                /////////////////////////////
                println!("  [*] Testing IR parser");
                let parsed = parse_il(&fn_ir_str, regfile.clone());
                let mut res = String::new();
                ir_writer::emit_il(&mut res, Some((*rfn.name).to_owned()), &parsed).unwrap();
                for (orig, roundtrip) in fn_ir_str.lines().zip(res.lines()) {
                    if orig != roundtrip {
                        println!("  FAILED TO ROUND-TRIP: \"{}\" => \"{}\"", orig, roundtrip);
                    }
                }
            }

            {
                ////////////////////////
                // Generate DOT file
                ///////////////////////
                println!("  [*] Generating dot");
                fname.set_extension("dot");
                let mut df = File::create(&fname).expect("Unable to create .dot file");
                let dot = dot::emit_dot(rfn.ssa());
                writeln!(df, "{}", dot).expect("Error writing to file");
            }

            {
                ////////////////////////
                // Generate pseudo-C code
                ///////////////////////
                println!("  [*] Generating psuedo code");
                fname.set_extension("c");
                let mut df = File::create(&fname).expect("Unable to create .c file");
                let ast = c_simple_ast_builder::recover_simple_ast(&rfn);
                let code = ast.to_c_ast().print();
                writeln!(df, "{}", code).expect("Error writing to file");
            }
        }

        match rmod.source.as_mut().unwrap().send("e scr.color=true".to_string()) {
            Ok(()) => {},
            Err(e) => println!("{:?}", e),
        }
    }
}

// Filters the functions that were in Radeco output AND requested by user
fn filter_with<'a>(all_funcs: &Vec<(u64, &'a str)>,
                   requested: &Vec<&'a str>)
                   -> Vec<(u64, &'a str)> {

    all_funcs.iter()
        .filter(|&&(_, name)| requested.iter().any(|user_req| &name == user_req))
        .map(|&(addr, name)| (addr, name))
        .collect()
}
<commit_msg>Fix filepath<commit_after>#[cfg(feature="trace_log")] extern crate env_logger;

extern crate radeco_lib;
extern crate r2pipe;
extern crate r2api;
extern crate petgraph;

mod cli;

use std::env;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
use std::process;

use radeco_lib::analysis::cse::cse::CSE;
use radeco_lib::analysis::sccp;
//use radeco_lib::analysis::valueset::analyzer_wysinwyx::FnAnalyzer;
//use radeco_lib::analysis::valueset::mem_structs::{A_Loc,AbstractAddress};
use radeco_lib::analysis::interproc::fixcall::CallFixer;
use radeco_lib::frontend::radeco_containers::ProjectLoader;
use radeco_lib::middle::{dce, dot};
use radeco_lib::middle::ir_writer;
use radeco_lib::middle::ir_reader::parse_il;
use radeco_lib::middle::ssa::verifier;
use radeco_lib::backend::lang_c::{c_simple_ast, c_simple_ast_builder};

//use radeco_lib::analysis::mark_refs;

const USAGE: &'static str = "
Usage: minidec [-f <names>...] <target>

Options:
    -f, --functions  Analayze only some functions
";

fn main() {
    #[cfg(feature="trace_log")] env_logger::init();

    let requested_functions = cli::init_for_args(USAGE);

    let proj_name = env::args().nth(env::args().len() - 1).unwrap();
    let mut rproj = {
        ProjectLoader::new().path(&proj_name).load()
    };
    let regfile = rproj.regfile().clone();
    for mut xy in rproj.iter_mut() {
        let rmod = &mut xy.module;
        let mut dir = PathBuf::from(".");
        dir.push(format!("{}_out", proj_name));
        fs::create_dir_all(&dir).expect("Failed to create directory");

        // Reduce the complexity of rmod.functions to just a vec of (u64,&String)
        // for easier extraction and matching

        // Analyze preserved for all functions.
        {
            println!("[*] Fixing Callee Information");
            let bp_name = regfile.get_name_by_alias(&"BP".to_string());
            let bp_name = bp_name.map(|s| s.to_owned());
            let sp_name = regfile.get_name_by_alias(&"SP".to_string());
            let sp_name = sp_name.map(|s| s.to_owned());
            let mut callfixer = CallFixer::new(rmod, bp_name, sp_name);
            callfixer.rounded_analysis();
        }

        // Fix call sites
        radeco_lib::analysis::functions::fix_ssa_opcalls::go(rmod);

        // Infer calling conventions
        radeco_lib::analysis::functions::infer_regusage::run(rmod, &*regfile);

        // Filter the data if the user provided some args to be matched upon
        let matched_func_addrs = if requested_functions.len() != 0 {

            //XXX Impl in more efficient way
            let mut matched_func_vec: Vec<(u64, &str)> =
                rmod.iter().map(|_f| {
                    let f = _f.function.1;
                    (f.offset.clone(), &*f.name)
                }).collect();

            let all_func_names: Vec<(&str)> =
                matched_func_vec.iter().map(|&(_, name)| name).collect();
            matched_func_vec = filter_with(&matched_func_vec,
                                           &requested_functions.iter().map(|s| &s[..]).collect::<Vec<_>>());
            cli::print_match_summary(&matched_func_vec,
                                     &requested_functions.iter().map(|s| &s[..]).collect::<Vec<_>>(),
                                     &all_func_names);
            matched_func_vec.into_iter().map(|(addr, _)| addr).collect::<Vec<_>>()
        } else {
            Vec::new()
        };

        // Main file to contain IRs of all rfns
        let mut ffm = {
            let mut fname = PathBuf::from(&dir);
            fname.push("main");
            File::create(&fname).expect("Unable to create file")
        };

        for addr in matched_func_addrs {

            let ref mut rfn = rmod.functions.get_mut(&addr).unwrap();

            println!("[+] Analyzing: {} @ {:#x}", rfn.name, addr);
            {
                println!("  [*] Eliminating Dead Code");
                dce::collect(rfn.ssa_mut());
            }
            let mut ssa = {
                // Constant Propagation (sccp)
                println!("  [*] Propagating Constants");
                let mut analyzer = sccp::Analyzer::new(rfn.ssa_mut());
                analyzer.analyze();
                analyzer.emit_ssa()
            };
            {
                println!("  [*] Eliminating More DeadCode");
                dce::collect(&mut ssa);
            }
            *rfn.ssa_mut() = ssa;
            {
                // Common SubExpression Elimination (cse)
                println!("  [*] Eliminating Common SubExpressions");
                let mut cse = CSE::new(rfn.ssa_mut());
                cse.run();
            }
            {
                // Verify SSA
                println!("  [*] Verifying SSA's Validity");
                match verifier::verify(rfn.ssa()) {
                    Err(e) => {
                        println!("  [*] Found Error: {}", e);
                        process::exit(255);
                    }
                    Ok(_) => {  }
                }
            }
            {
                // TODO
                // // Building memory SSA.
                // let _memory_ssa = {
                //     // Generate MemorySSA
                //     println!("  [*] Generating Memory SSA");
                //     let mut mssa = MemorySSA::new(rfn.ssa());
                //     mssa.gather_variables(rfn.datarefs(), &rfn.locals(),
                //                           &rfn.call_refs(&rmod.callgraph));
                //     mssa.run();
                //     mssa
                // };
            }
            //if false {
            //    if (!rfn.name.eq("sym.main")) & (!rfn.name.eq("main")) {
            //        continue;
            //    }
            //    println!("  [*] Analyzing Value Sets");
            //    let fn_analyzer = FnAnalyzer::from((*rfn).clone());
            //    let a_store_fn = fn_analyzer.analyze_rfn();
            //    for (a_loc, strided_interval) in a_store_fn.store {
            //        if let A_Loc{addr: AbstractAddress::Node{..}, ..} = a_loc {
            //            continue;
            //        };
            //        println!("{}", a_loc);
            //        println!("Strided Interval: {}", strided_interval);
            //    };
            //}

            //{
                //// Expreimental reference marking pass
                //println!("  [*] Unstable> Marking References");
                //let mut rmark = mark_refs::ReferenceMarker { };
                //rmark.resolve_refs(&mut rfn.ssa);
            //}

            let mut fname = PathBuf::from(&dir);
            fname.push(rfn.name.as_ref());

            let fn_ir_str = {
                ///////////////////////////
                // Write out the IR file
                //////////////////////////
                println!("  [*] Writing out IR");
                let mut res = String::new();
                ir_writer::emit_il(&mut res, Some(rfn.name.to_string()), rfn.ssa()).unwrap();
                let mut ff = File::create(&fname).expect("Unable to create file");
                write!(ff, "{}", res).expect("Error writing to file");
                writeln!(ffm, "{}", res).expect("Error writing to file");
                // Set as a comment in radare2
                match rmod.source.as_mut().unwrap().send(format!("CC, {} @ {}", fname.to_str().unwrap(), addr)) {
                    Ok(_) => {},
                    Err(e) => println!("{:?}", e),
                };
                res
            };

            {
                //////////////////////////////
                // Try parsing the IR file
                /////////////////////////////
                println!("  [*] Testing IR parser");
                let parsed = parse_il(&fn_ir_str, regfile.clone());
                let mut res = String::new();
                ir_writer::emit_il(&mut res, Some((*rfn.name).to_owned()), &parsed).unwrap();
                for (orig, roundtrip) in fn_ir_str.lines().zip(res.lines()) {
                    if orig != roundtrip {
                        println!("  FAILED TO ROUND-TRIP: \"{}\" => \"{}\"", orig, roundtrip);
                    }
                }
            }

            {
                ////////////////////////
                // Generate DOT file
                ///////////////////////
                println!("  [*] Generating dot");
                let mut df = File::create(format!("{}.dot", fname.to_string_lossy())).expect("Unable to create .dot file");
                let dot = dot::emit_dot(rfn.ssa());
                writeln!(df, "{}", dot).expect("Error writing to file");
            }

            {
                ////////////////////////
                // Generate pseudo-C code
                ///////////////////////
                println!("  [*] Generating psuedo code");
                let mut df = File::create(format!("{}.c", fname.to_string_lossy())).expect("Unable to create .c file");
                let ast = c_simple_ast_builder::recover_simple_ast(&rfn);
                let code = ast.to_c_ast().print();
                writeln!(df, "{}", code).expect("Error writing to file");
            }
        }

        match rmod.source.as_mut().unwrap().send("e scr.color=true".to_string()) {
            Ok(()) => {},
            Err(e) => println!("{:?}", e),
        }
    }
}

// Filters the functions that were in Radeco output AND requested by user
fn filter_with<'a>(all_funcs: &Vec<(u64, &'a str)>,
                   requested: &Vec<&'a str>)
                   -> Vec<(u64, &'a str)> {

    all_funcs.iter()
        .filter(|&&(_, name)| requested.iter().any(|user_req| &name == user_req))
        .map(|&(addr, name)| (addr, name))
        .collect()
}
<|endoftext|>


<commit_before>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::sstable::*;
use super::*;
use client::AsyncClient;
use itertools::Itertools;
use parking_lot::RwLock;
use std::sync::Arc;
use std::{mem, ptr};
use index::btree::Ordering;

const LEVEL_PAGES_MULTIPLIER: usize = 1000;
const LEVEL_DIFF_MULTIPLIER: usize = 10;
const LEVEL_M: usize = super::btree::NUM_KEYS;
const LEVEL_1: usize = LEVEL_M * LEVEL_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_DIFF_MULTIPLIER;
// TODO: debug assert the last one will not overflow MAX_SEGMENT_SIZE

type LevelTrees = Vec<Box<Tree>>;

macro_rules! with_levels {
    ($($sym:ident, $level:ident;)*) => {
        $(
            type $sym = [EntryKey; $level];
            impl_sspage_slice!($sym, EntryKey, $level);
        )*

        fn init_lsm_level_trees(neb_client: &Arc<AsyncClient>) -> (LevelTrees, Vec<usize>) {
            let mut trees = LevelTrees::new();
            let mut sizes = Vec::new();
            $(
                trees.push(box LevelTree::<$sym>::new(neb_client));
                sizes.push($level);
            )*
            return (trees, sizes);
        }
    };
}

with_levels!{
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    L4, LEVEL_4;
}

pub struct LSMTree {
    level_m: BPlusTree,
    trees: LevelTrees,
    // use Vec here for convenience
    sizes: Vec<usize>,
}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Self {
        let (trees, sizes) = init_lsm_level_trees(neb_client);
        LSMTree {
            level_m: BPlusTree::new(neb_client),
            trees,
            sizes,
        }
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) -> Result<(), ()> {
        key_with_id(&mut key, id);
        self.level_m.insert(&key).map_err(|e| ())
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> Result<bool, ()> {
        key_with_id(&mut key, id);
        let m_deleted = self.level_m.remove(&key).map_err(|e| ())?;
        let levels_deleted = self
            .trees
            .iter()
            .map(|tree| tree.mark_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d);
        Ok(m_deleted || levels_deleted)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX))
        };
        let mut cursors = vec![self.level_m.seek(&key, ordering).unwrap()];
        for tree in &self.trees {
            cursors.push(tree.seek(&key, ordering));
        }
        return LSMTreeCursor::new(cursors);
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>) -> Self {
        LSMTreeCursor {
            level_cursors: cursors,
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        let min_tree = self
            .level_cursors
            .iter()
            .enumerate()
            .map(|(i, cursor)| (i, cursor.current()))
            .filter_map(|(i, current)| current.map(|current_val| (i, current_val)))
            .min_by_key(|(i, val)| *val)
            .map(|(id, _)| id);
        if let Some(id) = min_tree {
            self.level_cursors[id].next();
            return true;
        } else {
            return false;
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}
<commit_msg>cope with end of min tree<commit_after>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::sstable::*;
use super::*;
use client::AsyncClient;
use itertools::Itertools;
use parking_lot::RwLock;
use std::sync::Arc;
use std::{mem, ptr};
use index::btree::Ordering;

const LEVEL_PAGES_MULTIPLIER: usize = 1000;
const LEVEL_DIFF_MULTIPLIER: usize = 10;
const LEVEL_M: usize = super::btree::NUM_KEYS;
const LEVEL_1: usize = LEVEL_M * LEVEL_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_DIFF_MULTIPLIER;
// TODO: debug assert the last one will not overflow MAX_SEGMENT_SIZE

type LevelTrees = Vec<Box<Tree>>;

macro_rules! with_levels {
    ($($sym:ident, $level:ident;)*) => {
        $(
            type $sym = [EntryKey; $level];
            impl_sspage_slice!($sym, EntryKey, $level);
        )*

        fn init_lsm_level_trees(neb_client: &Arc<AsyncClient>) -> (LevelTrees, Vec<usize>) {
            let mut trees = LevelTrees::new();
            let mut sizes = Vec::new();
            $(
                trees.push(box LevelTree::<$sym>::new(neb_client));
                sizes.push($level);
            )*
            return (trees, sizes);
        }
    };
}

with_levels!{
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    L4, LEVEL_4;
}

pub struct LSMTree {
    level_m: BPlusTree,
    trees: LevelTrees,
    // use Vec here for convenience
    sizes: Vec<usize>,
}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Self {
        let (trees, sizes) = init_lsm_level_trees(neb_client);
        LSMTree {
            level_m: BPlusTree::new(neb_client),
            trees,
            sizes,
        }
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) -> Result<(), ()> {
        key_with_id(&mut key, id);
        self.level_m.insert(&key).map_err(|e| ())
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> Result<bool, ()> {
        key_with_id(&mut key, id);
        let m_deleted = self.level_m.remove(&key).map_err(|e| ())?;
        let levels_deleted = self
            .trees
            .iter()
            .map(|tree| tree.mark_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d);
        Ok(m_deleted || levels_deleted)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX))
        };
        let mut cursors = vec![self.level_m.seek(&key, ordering).unwrap()];
        for tree in &self.trees {
            cursors.push(tree.seek(&key, ordering));
        }
        return LSMTreeCursor::new(cursors);
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>) -> Self {
        LSMTreeCursor {
            level_cursors: cursors,
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        let min_tree = self
            .level_cursors
            .iter()
            .enumerate()
            .map(|(i, cursor)| (i, cursor.current()))
            .filter_map(|(i, current)| current.map(|current_val| (i, current_val)))
            .min_by_key(|(i, val)| *val)
            .map(|(id, _)| id);
        if let Some(id) = min_tree {
            let min_has_next = self.level_cursors[id].next();
            if !min_has_next {
                return self.level_cursors.iter().any(|level| level.current().is_some());
            } else {
                return true;
            }
        } else {
            return false;
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}
<|endoftext|>


<commit_before>use std::io::{FileAccess, FileMode, Open, Read, Truncate};
use std::io::{ReadWrite};
use std::io::{SeekEnd, SeekSet};
use std::io::fs::File;
use std::mem::size_of;
use std::io::Decorator;

static version: u8 = 1;

// some definitions from kissdb.h

struct Kissdb {
    hash_table_size : u64,
    key_size : u64,
    value_size : u64,
    num_hash_tables : u64,
    hash_tables : ~[u64],
    f : std::io::fs::File
}

enum OpenMode {
    ReadOnly,
    RW,
    RWCreate,
    RWReplace
}

fn kissdb_hash(b : &~[u8]) -> u64
{
    let mut hash : u64 = 5381;
    for &bx in b.iter() {
        hash = ((hash << 5) + hash) + (bx as u64);
    }
    return hash
}

fn kissdb_open(
    path : &Path,
    orig_mode : OpenMode,
    mut hash_table_size : u64,
    mut key_size : u64,
    mut value_size : u64
        ) -> Option<~ Kissdb>
{
    let (mode, access) : (FileMode, FileAccess) =
        match orig_mode {
        ReadOnly =>  (Open, Read), // rb
        RW => (Open, ReadWrite), // r+b
        RWCreate =>  (Open, ReadWrite), // r+b
        RWReplace => (Truncate, ReadWrite) // w+b
    };
    let mut f_ : Option<File>
        = std::io::fs::File::open_mode(path, mode, access);

    if f_.is_none() {
        match (orig_mode) {
            RWCreate =>
                f_ = std::io::fs::File::open_mode(path, Truncate, ReadWrite), // w+b
            _ => f_ = f_
        }
    };

    let mut f =
        match f_ {
        None => return None,
        Some (f) => f
    };
    let kissdb_header_size : u64 = ((size_of::<u64>() * 3) + 4) as u64;

    f.seek(0, SeekEnd);
    if (f.tell() < kissdb_header_size) {
        if hash_table_size > 0 && key_size > 0
            && value_size >0
            {
            f.seek(0, SeekSet);
            let tmp2 : [u8, ..4] = ['K' as u8, 'd' as u8, 'R' as u8, version];
            f.write(tmp2);
            f.write_le_u64(hash_table_size);
            f.write_le_u64(key_size);
            f.write_le_u64(value_size);
            f.flush();
        }
            else {
            fail!()
        }
    }
        else {
        f.seek(0, SeekSet);
        let mut tmp2 : [u8, ..4] = [0,0,0,0];
        if f.read(tmp2) != Some(4) { fail! (); }
        if tmp2 != ['K' as u8, 'd' as u8, 'R' as u8, version] {fail! ();}
        hash_table_size = f.read_le_u64();
        key_size = f.read_le_u64();
        value_size = f.read_le_u64();
    }

    let mut num_hash_tables = 0;

    let mut hash_tables : ~[u64] = std::vec::from_elem(0, 0 as u64);
    let mut h : Option<~[u64]>;

    // do we do realloc in rust? with append, apparently.
    while({h = f.read_one_hash_table(hash_table_size); h.is_some()})
        {
        let httmp = h.unwrap();
        hash_tables = std::vec::append(hash_tables, httmp);
        num_hash_tables = num_hash_tables + 1;
        if (httmp[hash_table_size] > 0) {
            f.seek(httmp[hash_table_size] as i64, SeekSet)
        }
            else {
            break;
        }
    }

    debug!("key_size = {:s}", key_size.to_str());

    return Some(~Kissdb {hash_table_size : hash_table_size,
                         key_size : key_size,
                         value_size : value_size,
                         num_hash_tables : num_hash_tables,
                         hash_tables : hash_tables,
                         f : f})
}

trait File2 {
    fn read_one_hash_table(&mut self, u64) -> Option<~[u64]>;
    fn write_one_hash_table(&mut self, &[u64]);
}

impl File2 for File {
    fn read_one_hash_table(&mut self,
                           hash_table_size : u64) -> Option<~[u64]> {
        let bs = (hash_table_size + 1) as uint * size_of::<u64>();
        let mut buf = std::vec::from_elem(bs, 0 as u8);
        let mut result = std::vec::from_elem((hash_table_size + 1) as uint, 0 as u64);
        match self.read(buf) {
            None => None,
            Some(buf_size) if buf_size == bs => {
                let mut br = std::io::mem::BufReader::new(buf);
                for i in range(0, hash_table_size + 1) {
                    result[i] = br.read_le_u64();
                }
                Some((result))
            },
            _ => None
        }
    }
    fn write_one_hash_table(&mut self,
                            ht : &[u64]) {
        for &x in ht.iter() {
            self.write_le_u64(x);
        }
    }
}

trait Kdb {
    fn kissdb_get(&mut self, &~[u8]) -> Option<~[u8]>;
    fn kissdb_put(&mut self, key : &~[u8], value : &[u8]) -> bool;
}

impl Kdb for Kissdb {
    fn kissdb_get(&mut self, key : &~[u8]) -> Option<~[u8]> {
        if key.len() != self.key_size as uint { return None }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let hash_tables = &self.hash_tables;
        for i in range(0, self.num_hash_tables) {
            let offset : i64 = hash_tables[i * (self.hash_table_size + 1) + hash] as i64;
            if (offset == 0) { return None }; // yeah, 0 is being special. never used normally.
            self.f.seek(offset as i64, SeekSet);

            let klen = self.key_size as uint;
            // isn't compare available?? read_bytes available
            let tmp = self.f.read_bytes(klen);

            if std::vec::bytes::memcmp(key, &tmp) == 0 {
                return Some(self.f.read_bytes(self.value_size as uint))
            }
        }
        return None
    }

    fn kissdb_put(&mut self, key : &~[u8], value : &[u8]) -> bool
        {
        if key.len() != self.key_size as uint { fail!() }
        if value.len() != self.value_size as uint { fail!() }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let mut lasthtoffset : u64 = ((size_of::<u64>() * 3) + 4) as u64;
        let mut htoffset = lasthtoffset;

        for i in range(0, self.num_hash_tables) {
            let offset : i64 = self.hash_tables[(self.hash_table_size + 1) * i + hash] as i64;
            if (offset != 0) { // yes, 0 is treated special. 0 will never used normally
                /* rewrite if already exists */
                self.f.seek(offset, SeekSet);

                let klen = self.key_size as uint;
                let tmp = self.f.read_bytes(klen);
                if std::vec::bytes::memcmp(key, &tmp) == 0 {
                    self.f.write(value);
                    self.f.flush();
                    return true; /* success */
                }
                // put_no_match_next_hash_table:
                lasthtoffset = htoffset;
                htoffset = self.hash_tables[(self.hash_table_size + 1) * i + self.hash_table_size];
                //cmp != , should be put_no_match_next_hash_table
            } else {
                /* add if an empty hash table slot is discovered */
                self.f.seek(0, SeekEnd);
                let endoffset : u64 = self.f.tell();

                // debug!("endoffset = {:s}",endoffset.to_str());
                self.f.write(*key);
                self.f.write(value);

                self.f.seek(htoffset as i64 + (size_of::<u64>() as i64 * hash as i64), SeekSet);
                self.f.write_le_u64(endoffset);
                self.hash_tables[(self.hash_table_size + 1) * i + hash] = endoffset;

                self.f.flush();

                return true; /* success */
            }
        }

        /* if no existing slots, add a new page of hash table entries */
        debug!("add new page");
        self.f.seek(0, SeekEnd);
        let endoffset = self.f.tell();

        let mut new_table : ~[u64] = std::vec::from_elem(self.hash_table_size as uint + 1, 0 as u64);
        let hash_table_size_bytes : u64 = size_of::<u64>() as u64 * (self.hash_table_size + 1);
        new_table[hash] = endoffset + hash_table_size_bytes; // where key, value are to be written
        self.f.write_one_hash_table(new_table);
        let longer = std::vec::append(self.hash_tables.clone(), new_table); // slow!!
        self.hash_tables = longer;

        debug!("writing at {:s}",self.f.tell().to_str());
        self.f.write(*key);
        self.f.write(value);

        if (self.num_hash_tables > 0) {
            self.f.seek(lasthtoffset as i64 + (size_of::<u64>() as i64 * self.hash_table_size as i64), SeekSet);
            self.f.write_le_u64(endoffset);
            // mimick rereading
            self.hash_tables[((self.hash_table_size + 1) * (self.num_hash_tables - 1)) + self.hash_table_size] = endoffset;
        }

        self.num_hash_tables = self.num_hash_tables + 1;
        self.f.flush();
        return true; /* success */
    }

}

fn main()
{
    println("Opening new empty database test.db...");
    {
        let mut db = kissdb_open(&Path::new("test.db"), RWReplace, 1024, 8, size_of::<u64>() as u64).unwrap();

        println("Adding and then re-getting 10000 64-byte values...");

        let mut v : [u8, .. 8] = [0,0,0,0,0,0,0,0];

        for i in range(0, 10000) {
            for j in range(0, 8) {
                v[j] = i as u8;
            }
            let mut key = std::io::mem::MemWriter::new();
            key.write_le_u64(i as u64);
            db.kissdb_put(key.inner_ref(), v);
            let gotten = db.kissdb_get(key.inner_ref()).unwrap();
            for j in range(0, 8) {
                if gotten[j] != i as u8 { fail!() }
            }
        }
        println("Getting 10000 64-byte values...");

        for i in range(0, 10000) {
            let mut key = std::io::mem::MemWriter::new();
            key.write_le_u64(i as u64);
            let gotten = db.kissdb_get(key.inner_ref()).unwrap();
            for j in range(0,8) {
                if gotten[j] != i as u8 { fail!() }
            }
        }

        println("closing and re-opening database in read-only mode...");
    }

    {
        let mut db = kissdb_open(&Path::new("test.db"), ReadOnly, 1024, 8, size_of::<u64>() as u64).unwrap();

        println("Getting 10000 64-byte values...");

        for i in range(0, 10000) {
            let mut key = std::io::mem::MemWriter::new();
            key.write_le_u64(i as u64);
            let gotten = db.kissdb_get(key.inner_ref()).unwrap();
            for j in range(0,8) {
                if gotten[j] != i as u8 { fail!() }
            }
        }
    }

    println("Iterator not implemented ...");

    println("All tests OK!");
}
<commit_msg>made code more idiomatic<commit_after>use std::io::{Open, Read, Truncate, ReadWrite, SeekEnd, SeekSet};
use std::io::fs::File;
use std::mem::size_of;
use std::io::Decorator;
use std::vec;
use std::io::mem::{BufReader, MemWriter};

static VERSION: u8 = 1;

// some definitions from kissdb.h

struct Kissdb {
    hash_table_size : u64,
    key_size : u64,
    value_size : u64,
    num_hash_tables : u64,
    hash_tables : ~[u64],
    f : File
}

enum OpenMode {
    ReadOnly,
    RW,
    RWCreate,
    RWReplace
}

fn kissdb_hash(b : &[u8]) -> u64 {
    let mut hash = 5381;
    for &bx in b.iter() {
        hash = ((hash << 5) + hash) + (bx as u64);
    }
    return hash
}

fn kissdb_open(
    path : &Path,
    orig_mode : OpenMode,
    mut hash_table_size : u64,
    mut key_size : u64,
    mut value_size : u64
        ) -> Option<~Kissdb>
{
    let (mode, access) = match orig_mode {
        ReadOnly =>  (Open, Read), // rb
        RW => (Open, ReadWrite), // r+b
        RWCreate =>  (Open, ReadWrite), // r+b
        RWReplace => (Truncate, ReadWrite) // w+b
    };
    let f_ = File::open_mode(path, mode, access).or_else(|| {
            match orig_mode {
                RWCreate => File::open_mode(path, Truncate, ReadWrite),
                _ => None
            }
        });

    let mut f = match f_ {
        Some(f) => f,
        None => return None
    };
    let kissdb_header_size = ((size_of::<u64>() * 3) + 4) as u64;

    f.seek(0, SeekEnd);
    if (f.tell() < kissdb_header_size) {
        if hash_table_size > 0 && key_size > 0
            && value_size >0
            {
            f.seek(0, SeekSet);
            let tmp2 = ['K' as u8, 'd' as u8, 'R' as u8, VERSION];
            f.write(tmp2);
            f.write_le_u64(hash_table_size);
            f.write_le_u64(key_size);
            f.write_le_u64(value_size);
            f.flush();
        }
        else {
            fail!()
        }
    }
    else {
        f.seek(0, SeekSet);
        let mut tmp2 = [0, ..4];
        if f.read(tmp2) != Some(4) { fail!() }
        if tmp2 != ['K' as u8, 'd' as u8, 'R' as u8, VERSION] { fail!() }
        hash_table_size = f.read_le_u64();
        key_size = f.read_le_u64();
        value_size = f.read_le_u64();
    }

    let mut num_hash_tables = 0;

    let mut hash_tables = ~[];
    let mut h;

    while ({h = f.read_one_hash_table(hash_table_size); h.is_some()})
    {
        let httmp = h.unwrap();
        hash_tables.push_all(httmp);
        num_hash_tables = num_hash_tables + 1;
        if (httmp[hash_table_size] > 0) {
            f.seek(httmp[hash_table_size] as i64, SeekSet)
        }
        else {
            break;
        }
    }

    debug!("key_size = {:s}", key_size.to_str());

    Some(~Kissdb {hash_table_size : hash_table_size,
                  key_size : key_size,
                  value_size : value_size,
                  num_hash_tables : num_hash_tables,
                  hash_tables : hash_tables,
                  f : f})
}

trait File2 {
    fn read_one_hash_table(&mut self, u64) -> Option<~[u64]>;
    fn write_one_hash_table(&mut self, &[u64]);
}

impl File2 for File {
    fn read_one_hash_table(&mut self,
                           hash_table_size : u64) -> Option<~[u64]> {
        let bs = (hash_table_size + 1) as uint * size_of::<u64>();
        let mut buf = vec::from_elem(bs, 0 as u8);

        match self.read(buf) {
            Some(buf_size) if buf_size == bs => {
                let mut br = BufReader::new(buf);
                Some(vec::from_fn((hash_table_size + 1) as uint,
                                  |_| br.read_le_u64()))
            },
            _ => None
        }
    }
    fn write_one_hash_table(&mut self, ht : &[u64]) {
        for &x in ht.iter() {
            self.write_le_u64(x);
        }
    }
}

trait Kdb {
    fn kissdb_get(&mut self, key : &[u8]) -> Option<~[u8]>;
    fn kissdb_put(&mut self, key : &[u8], value : &[u8]) -> bool;
}

impl Kdb for Kissdb {
    fn kissdb_get(&mut self, key : &[u8]) -> Option<~[u8]> {
        if key.len() != self.key_size as uint { return None }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let hash_tables = &self.hash_tables;
        for i in range(0, self.num_hash_tables) {
            let offset = hash_tables[i * (self.hash_table_size + 1) + hash] as i64;
            if (offset == 0) { return None }; // yeah, 0 is being special. never used normally.
            self.f.seek(offset as i64, SeekSet);

            let klen = self.key_size as uint;
            let tmp = self.f.read_bytes(klen);

            if key == tmp {
                return Some(self.f.read_bytes(self.value_size as uint))
            }
        }
        None
    }

    fn kissdb_put(&mut self, key : &[u8], value : &[u8]) -> bool {
        if key.len() != self.key_size as uint { fail!() }
        if value.len() != self.value_size as uint { fail!() }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let mut last_ht_offset = ((size_of::<u64>() * 3) + 4) as u64;
        let mut ht_offset = last_ht_offset;

        for i in range(0, self.num_hash_tables) {
            let offset = self.hash_tables[(self.hash_table_size + 1) * i + hash] as i64;
            if (offset != 0) { // yes, 0 is treated special. 0 will never used normally
                /* rewrite if already exists */
                self.f.seek(offset, SeekSet);

                let klen = self.key_size as uint;
                let tmp = self.f.read_bytes(klen);
                if key == tmp {
                    self.f.write(value);
                    self.f.flush();
                    return true; /* success */
                }
                // put_no_match_next_hash_table:
                last_ht_offset = ht_offset;
                ht_offset = self.hash_tables[(self.hash_table_size + 1) * i + self.hash_table_size];
                //cmp != , should be put_no_match_next_hash_table
            } else {
                /* add if an empty hash table slot is discovered */
                self.f.seek(0, SeekEnd);
                let end_offset = self.f.tell();

                // debug!("endoffset = {:s}",endoffset.to_str());
                self.f.write(key);
                self.f.write(value);

                self.f.seek(ht_offset as i64 + (size_of::<u64>() as i64 * hash as i64), SeekSet);
                self.f.write_le_u64(end_offset);
                self.hash_tables[(self.hash_table_size + 1) * i + hash] = end_offset;

                self.f.flush();

                return true; /* success */
            }
        }

        /* if no existing slots, add a new page of hash table entries */
        debug!("add new page");
        self.f.seek(0, SeekEnd);
        let endoffset = self.f.tell();

        let mut new_table = vec::from_elem(self.hash_table_size as uint + 1, 0 as u64);
        let hash_table_size_bytes = size_of::<u64>() as u64 * (self.hash_table_size + 1);
        new_table[hash] = endoffset + hash_table_size_bytes; // where key, value are to be written
        self.f.write_one_hash_table(new_table);
        self.hash_tables.push_all_move(new_table);

        debug!("writing at {:s}", self.f.tell().to_str());
        self.f.write(key);
        self.f.write(value);

        if self.num_hash_tables > 0 {
            self.f.seek(last_ht_offset as i64 + (size_of::<u64>() as i64 * self.hash_table_size as i64), SeekSet);
            self.f.write_le_u64(endoffset);
            // mimick rereading
            self.hash_tables[((self.hash_table_size + 1) * (self.num_hash_tables - 1)) + self.hash_table_size] = endoffset;
        }

        self.num_hash_tables = self.num_hash_tables + 1;
        self.f.flush();
        return true; /* success */
    }

}

fn main()
{
    println("Opening new empty database test.db...");
    {
        let mut db = kissdb_open(&Path::new("test.db"), RWReplace, 1024, 8, size_of::<u64>() as u64).unwrap();

        println("Adding and then re-getting 10000 64-byte values...");

        let mut v = [0,0,0,0,0,0,0,0];

        for i in range(0, 10000) {
            for j in range(0, 8) {
                v[j] = i as u8;
            }
            let mut key = MemWriter::new();
            key.write_le_u64(i as u64);
            let key_buf = key.inner();
            db.kissdb_put(key_buf, v);
            let gotten = db.kissdb_get(key_buf).unwrap();
            for j in range(0, 8) {
                if gotten[j] != i as u8 { fail!() }
            }
        }
        println("Getting 10000 64-byte values...");

        for i in range(0, 10000) {
            let mut key = MemWriter::new();
            key.write_le_u64(i as u64);
            let gotten = db.kissdb_get(key.inner()).unwrap();
            for j in range(0,8) {
                if gotten[j] != i as u8 { fail!() }
            }
        }

        println("closing and re-opening database in read-only mode...");
    }

    {
        let mut db = kissdb_open(&Path::new("test.db"), ReadOnly, 1024, 8, size_of::<u64>() as u64).unwrap();

        println("Getting 10000 64-byte values...");

        for i in range(0, 10000) {
            let mut key = MemWriter::new();
            key.write_le_u64(i as u64);
            let gotten = db.kissdb_get(key.inner()).unwrap();
            for j in range(0,8) {
                if gotten[j] != i as u8 { fail!() }
            }
        }
    }

    println("Iterator not implemented ...");

    println("All tests OK!");
}
<|endoftext|>


<commit_before>use super::internal::*;
use super::*;

/// `FlatMap` maps each element to an iterator, then flattens these iterators together.
/// This struct is created by the [`flat_map()`] method on [`ParallelIterator`]
///
/// [`flap_map()`]: trait.ParallelIterator.html#method.flat_map
/// [`ParallelIterator`]: trait.ParallelIterator.html
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct FlatMap<I: ParallelIterator, F> {
    base: I,
    map_op: F,
}

/// Create a new `FlatMap` iterator.
///
/// NB: a free fn because it is NOT part of the end-user API.
pub fn new<I, F>(base: I, map_op: F) -> FlatMap<I, F>
    where I: ParallelIterator
{
    FlatMap {
        base: base,
        map_op: map_op,
    }
}

impl<I, F, PI> ParallelIterator for FlatMap<I, F>
    where I: ParallelIterator,
          F: Fn(I::Item) -> PI + Sync + Send,
          PI: IntoParallelIterator
{
    type Item = PI::Item;

    fn drive_unindexed<C>(self, consumer: C) -> C::Result
        where C: UnindexedConsumer<Self::Item>
    {
        let consumer = FlatMapConsumer {
            base: consumer,
            map_op: &self.map_op,
        };
        self.base.drive_unindexed(consumer)
    }
}

/// ////////////////////////////////////////////////////////////////////////
/// Consumer implementation

struct FlatMapConsumer<'f, C, F: 'f> {
    base: C,
    map_op: &'f F,
}

impl<'f, C, F> FlatMapConsumer<'f, C, F> {
    fn new(base: C, map_op: &'f F) -> Self {
        FlatMapConsumer {
            base: base,
            map_op: map_op,
        }
    }
}

impl<'f, T, U, C, F> Consumer<T> for FlatMapConsumer<'f, C, F>
    where C: UnindexedConsumer<U::Item>,
          F: Fn(T) -> U + Sync,
          U: IntoParallelIterator
{
    type Folder = FlatMapFolder<'f, C, F, C::Result>;
    type Reducer = C::Reducer;
    type Result = C::Result;

    fn split_at(self, index: usize) -> (Self, Self, C::Reducer) {
        let (left, right, reducer) = self.base.split_at(index);
        (FlatMapConsumer::new(left, self.map_op),
         FlatMapConsumer::new(right, self.map_op),
         reducer)
    }

    fn into_folder(self) -> Self::Folder {
        FlatMapFolder {
            base: self.base,
            map_op: self.map_op,
            previous: None,
        }
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}

impl<'f, T, U, C, F> UnindexedConsumer<T> for FlatMapConsumer<'f, C, F>
    where C: UnindexedConsumer<U::Item>,
          F: Fn(T) -> U + Sync,
          U: IntoParallelIterator
{
    fn split_off_left(&self) -> Self {
        FlatMapConsumer::new(self.base.split_off_left(), self.map_op)
    }

    fn to_reducer(&self) -> Self::Reducer {
        self.base.to_reducer()
    }
}

struct FlatMapFolder<'f, C, F: 'f, R> {
    base: C,
    map_op: &'f F,
    previous: Option<R>,
}

impl<'f, T, U, C, F> Folder<T> for FlatMapFolder<'f, C, F, C::Result>
    where C: UnindexedConsumer<U::Item>,
          F: Fn(T) -> U + Sync,
          U: IntoParallelIterator
{
    type Result = C::Result;

    fn consume(self, item: T) -> Self {
        let map_op = self.map_op;
        let par_iter = map_op(item).into_par_iter();
        let result = par_iter.drive_unindexed(self.base.split_off_left());

        // We expect that `previous` is `None`, because we drive
        // the cost up so high, but just in case.
        let previous = match self.previous {
            None => Some(result),
            Some(previous) => {
                let reducer = self.base.to_reducer();
                Some(reducer.reduce(previous, result))
            }
        };

        FlatMapFolder {
            base: self.base,
            map_op: map_op,
            previous: previous,
        }
    }

    fn complete(self) -> Self::Result {
        match self.previous {
            Some(previous) => previous,
            None => self.base.into_folder().complete(),
        }
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}
<commit_msg>Fix doc error in flat_map<commit_after>use super::internal::*;
use super::*;

/// `FlatMap` maps each element to an iterator, then flattens these iterators together.
/// This struct is created by the [`flat_map()`] method on [`ParallelIterator`]
///
/// [`flat_map()`]: trait.ParallelIterator.html#method.flat_map
/// [`ParallelIterator`]: trait.ParallelIterator.html
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct FlatMap<I: ParallelIterator, F> {
    base: I,
    map_op: F,
}

/// Create a new `FlatMap` iterator.
///
/// NB: a free fn because it is NOT part of the end-user API.
pub fn new<I, F>(base: I, map_op: F) -> FlatMap<I, F>
    where I: ParallelIterator
{
    FlatMap {
        base: base,
        map_op: map_op,
    }
}

impl<I, F, PI> ParallelIterator for FlatMap<I, F>
    where I: ParallelIterator,
          F: Fn(I::Item) -> PI + Sync + Send,
          PI: IntoParallelIterator
{
    type Item = PI::Item;

    fn drive_unindexed<C>(self, consumer: C) -> C::Result
        where C: UnindexedConsumer<Self::Item>
    {
        let consumer = FlatMapConsumer {
            base: consumer,
            map_op: &self.map_op,
        };
        self.base.drive_unindexed(consumer)
    }
}

/// ////////////////////////////////////////////////////////////////////////
/// Consumer implementation

struct FlatMapConsumer<'f, C, F: 'f> {
    base: C,
    map_op: &'f F,
}

impl<'f, C, F> FlatMapConsumer<'f, C, F> {
    fn new(base: C, map_op: &'f F) -> Self {
        FlatMapConsumer {
            base: base,
            map_op: map_op,
        }
    }
}

impl<'f, T, U, C, F> Consumer<T> for FlatMapConsumer<'f, C, F>
    where C: UnindexedConsumer<U::Item>,
          F: Fn(T) -> U + Sync,
          U: IntoParallelIterator
{
    type Folder = FlatMapFolder<'f, C, F, C::Result>;
    type Reducer = C::Reducer;
    type Result = C::Result;

    fn split_at(self, index: usize) -> (Self, Self, C::Reducer) {
        let (left, right, reducer) = self.base.split_at(index);
        (FlatMapConsumer::new(left, self.map_op),
         FlatMapConsumer::new(right, self.map_op),
         reducer)
    }

    fn into_folder(self) -> Self::Folder {
        FlatMapFolder {
            base: self.base,
            map_op: self.map_op,
            previous: None,
        }
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}

impl<'f, T, U, C, F> UnindexedConsumer<T> for FlatMapConsumer<'f, C, F>
    where C: UnindexedConsumer<U::Item>,
          F: Fn(T) -> U + Sync,
          U: IntoParallelIterator
{
    fn split_off_left(&self) -> Self {
        FlatMapConsumer::new(self.base.split_off_left(), self.map_op)
    }

    fn to_reducer(&self) -> Self::Reducer {
        self.base.to_reducer()
    }
}

struct FlatMapFolder<'f, C, F: 'f, R> {
    base: C,
    map_op: &'f F,
    previous: Option<R>,
}

impl<'f, T, U, C, F> Folder<T> for FlatMapFolder<'f, C, F, C::Result>
    where C: UnindexedConsumer<U::Item>,
          F: Fn(T) -> U + Sync,
          U: IntoParallelIterator
{
    type Result = C::Result;

    fn consume(self, item: T) -> Self {
        let map_op = self.map_op;
        let par_iter = map_op(item).into_par_iter();
        let result = par_iter.drive_unindexed(self.base.split_off_left());

        // We expect that `previous` is `None`, because we drive
        // the cost up so high, but just in case.
        let previous = match self.previous {
            None => Some(result),
            Some(previous) => {
                let reducer = self.base.to_reducer();
                Some(reducer.reduce(previous, result))
            }
        };

        FlatMapFolder {
            base: self.base,
            map_op: map_op,
            previous: previous,
        }
    }

    fn complete(self) -> Self::Result {
        match self.previous {
            Some(previous) => previous,
            None => self.base.into_folder().complete(),
        }
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}
<|endoftext|>


<commit_before>//! Internal traits and functions used to implement parallel
//! iteration. These should be considered highly unstable: users of
//! parallel iterators should not need to interact with them directly.
//! See `README.md` for a high-level overview.

use rayon_core::join;
use super::IndexedParallelIterator;

pub trait ProducerCallback<T> {
    type Output;
    fn callback<P>(self, producer: P) -> Self::Output where P: Producer<Item = T>;
}

/// A producer which will produce a fixed number of items N. This is
/// not queryable through the API; the consumer is expected to track
/// it.
pub trait Producer: Send + Sized {
    // Rust issue https://github.com/rust-lang/rust/issues/20671
    // prevents us from declaring the DoubleEndedIterator and
    // ExactSizeIterator constraints on a required IntoIterator trait,
    // so we inline IntoIterator here until that issue is fixed.
    type Item;
    type IntoIter: Iterator<Item = Self::Item> + DoubleEndedIterator + ExactSizeIterator;

    fn into_iter(self) -> Self::IntoIter;

    /// Split into two producers; one produces items `0..index`, the
    /// other `index..N`. Index must be less than or equal to `N`.
    fn split_at(self, index: usize) -> (Self, Self);

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    ///
    /// The provided implementation is sufficient for most iterables.
    fn fold_with<F>(self, folder: F) -> F
        where F: Folder<Self::Item>
    {
        folder.consume_iter(self.into_iter())
    }
}

/// A consumer which consumes items that are fed to it.
pub trait Consumer<Item>: Send + Sized {
    type Folder: Folder<Item, Result = Self::Result>;
    type Reducer: Reducer<Self::Result>;
    type Result: Send;

    /// Divide the consumer into two consumers, one processing items
    /// `0..index` and one processing items from `index..`. Also
    /// produces a reducer that can be used to reduce the results at
    /// the end.
    fn split_at(self, index: usize) -> (Self, Self, Self::Reducer);

    /// Convert the consumer into a folder that can consume items
    /// sequentially, eventually producing a final result.
    fn into_folder(self) -> Self::Folder;

    /// Hint whether this `Consumer` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Folder<Item>: Sized {
    type Result;

    /// Consume next item and return new sequential state.
    fn consume(self, item: Item) -> Self;

    /// Consume items from the iterator until full, and return new sequential state.
    fn consume_iter<I>(mut self, iter: I) -> Self
        where I: IntoIterator<Item = Item>
    {
        for item in iter {
            self = self.consume(item);
            if self.full() {
                break;
            }
        }
        self
    }

    /// Finish consuming items, produce final result.
    fn complete(self) -> Self::Result;

    /// Hint whether this `Folder` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Reducer<Result> {
    /// Reduce two final results into one; this is executed after a
    /// split.
    fn reduce(self, left: Result, right: Result) -> Result;
}

/// A stateless consumer can be freely copied.
pub trait UnindexedConsumer<I>: Consumer<I> {
    // The result of split_off_left should be used for the left side of the
    // data it consumes, and the remaining consumer for the right side
    // (this matters for methods like find_first).
    fn split_off_left(&self) -> Self;
    fn to_reducer(&self) -> Self::Reducer;
}

/// An unindexed producer that doesn't know its exact length.
/// (or can't represent its known length in a `usize`)
pub trait UnindexedProducer: Send + Sized {
    type Item;

    /// Split midway into a new producer if possible, otherwise return `None`.
    fn split(self) -> (Self, Option<Self>);

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    fn fold_with<F>(self, folder: F) -> F where F: Folder<Self::Item>;
}

/// A splitter controls the policy for splitting into smaller work items.
///
/// Thief-splitting is an adaptive policy that starts by splitting into
/// enough jobs for every worker thread, and then resets itself whenever a
/// job is actually stolen into a different thread.
#[derive(Clone, Copy)]
struct Splitter {
    origin: usize,
    splits: usize,
}

impl Splitter {
    #[inline]
    fn thief_id() -> usize {
        // The actual `ID` value is irrelevant.  We're just using its TLS
        // address as a unique thread key, faster than a real thread-id call.
        thread_local!{ static ID: bool = false }
        ID.with(|id| id as *const bool as usize)
    }

    #[inline]
    fn new() -> Splitter {
        Splitter {
            origin: Splitter::thief_id(),
            splits: ::current_num_threads(),
        }
    }

    #[inline]
    fn try(&mut self) -> bool {
        let Splitter { origin, splits } = *self;

        let id = Splitter::thief_id();
        if origin != id {
            self.origin = id;
            self.splits = ::current_num_threads();
            true
        } else if splits > 0 {
            self.splits /= 2;
            true
        } else {
            false
        }
    }
}

pub fn bridge<I, C>(mut par_iter: I, consumer: C) -> C::Result
    where I: IndexedParallelIterator,
          C: Consumer<I::Item>
{
    let len = par_iter.len();
    return par_iter.with_producer(Callback {
                                      len: len,
                                      consumer: consumer,
                                  });

    struct Callback<C> {
        len: usize,
        consumer: C,
    }

    impl<C, I> ProducerCallback<I> for Callback<C>
        where C: Consumer<I>
    {
        type Output = C::Result;
        fn callback<P>(self, producer: P) -> C::Result
            where P: Producer<Item = I>
        {
            bridge_producer_consumer(self.len, producer, self.consumer)
        }
    }
}

pub fn bridge_producer_consumer<P, C>(len: usize, producer: P, consumer: C) -> C::Result
    where P: Producer,
          C: Consumer<P::Item>
{
    let splitter = Splitter::new();
    return helper(len, splitter, producer, consumer);

    fn helper<P, C>(len: usize, mut splitter: Splitter, producer: P, consumer: C) -> C::Result
        where P: Producer,
              C: Consumer<P::Item>
    {
        if consumer.full() {
            consumer.into_folder().complete()
        } else if len > 1 && splitter.try() {
            let mid = len / 2;
            let (left_producer, right_producer) = producer.split_at(mid);
            let (left_consumer, right_consumer, reducer) = consumer.split_at(mid);
            let (left_result, right_result) =
                join(|| helper(mid, splitter, left_producer, left_consumer),
                     || helper(len - mid, splitter, right_producer, right_consumer));
            reducer.reduce(left_result, right_result)
        } else {
            producer.fold_with(consumer.into_folder()).complete()
        }
    }
}

pub fn bridge_unindexed<P, C>(producer: P, consumer: C) -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer<P::Item>
{
    let splitter = Splitter::new();
    bridge_unindexed_producer_consumer(splitter, producer, consumer)
}

fn bridge_unindexed_producer_consumer<P, C>(mut splitter: Splitter,
                                            producer: P,
                                            consumer: C)
                                            -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer<P::Item>
{
    if consumer.full() {
        consumer.into_folder().complete()
    } else if splitter.try() {
        match producer.split() {
            (left_producer, Some(right_producer)) => {
                let (reducer, left_consumer, right_consumer) =
                    (consumer.to_reducer(), consumer.split_off_left(), consumer);
                let bridge = bridge_unindexed_producer_consumer;
                let (left_result, right_result) =
                    join(|| bridge(splitter, left_producer, left_consumer),
                         || bridge(splitter, right_producer, right_consumer));
                reducer.reduce(left_result, right_result)
            }
            (producer, None) => producer.fold_with(consumer.into_folder()).complete(),
        }
    } else {
        producer.fold_with(consumer.into_folder()).complete()
    }
}

<commit_msg>Add producer methods for min/max length<commit_after>//! Internal traits and functions used to implement parallel
//! iteration. These should be considered highly unstable: users of
//! parallel iterators should not need to interact with them directly.
//! See `README.md` for a high-level overview.

use rayon_core::join;
use super::IndexedParallelIterator;

use std::cmp;
use std::usize;

pub trait ProducerCallback<T> {
    type Output;
    fn callback<P>(self, producer: P) -> Self::Output where P: Producer<Item = T>;
}

/// A producer which will produce a fixed number of items N. This is
/// not queryable through the API; the consumer is expected to track
/// it.
pub trait Producer: Send + Sized {
    // Rust issue https://github.com/rust-lang/rust/issues/20671
    // prevents us from declaring the DoubleEndedIterator and
    // ExactSizeIterator constraints on a required IntoIterator trait,
    // so we inline IntoIterator here until that issue is fixed.
    type Item;
    type IntoIter: Iterator<Item = Self::Item> + DoubleEndedIterator + ExactSizeIterator;

    fn into_iter(self) -> Self::IntoIter;

    fn min_len(&self) -> usize { 1 }
    fn max_len(&self) -> usize { usize::MAX }

    /// Split into two producers; one produces items `0..index`, the
    /// other `index..N`. Index must be less than or equal to `N`.
    fn split_at(self, index: usize) -> (Self, Self);

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    ///
    /// The provided implementation is sufficient for most iterables.
    fn fold_with<F>(self, folder: F) -> F
        where F: Folder<Self::Item>
    {
        folder.consume_iter(self.into_iter())
    }
}

/// A consumer which consumes items that are fed to it.
pub trait Consumer<Item>: Send + Sized {
    type Folder: Folder<Item, Result = Self::Result>;
    type Reducer: Reducer<Self::Result>;
    type Result: Send;

    /// Divide the consumer into two consumers, one processing items
    /// `0..index` and one processing items from `index..`. Also
    /// produces a reducer that can be used to reduce the results at
    /// the end.
    fn split_at(self, index: usize) -> (Self, Self, Self::Reducer);

    /// Convert the consumer into a folder that can consume items
    /// sequentially, eventually producing a final result.
    fn into_folder(self) -> Self::Folder;

    /// Hint whether this `Consumer` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Folder<Item>: Sized {
    type Result;

    /// Consume next item and return new sequential state.
    fn consume(self, item: Item) -> Self;

    /// Consume items from the iterator until full, and return new sequential state.
    fn consume_iter<I>(mut self, iter: I) -> Self
        where I: IntoIterator<Item = Item>
    {
        for item in iter {
            self = self.consume(item);
            if self.full() {
                break;
            }
        }
        self
    }

    /// Finish consuming items, produce final result.
    fn complete(self) -> Self::Result;

    /// Hint whether this `Folder` would like to stop processing
    /// further items, e.g. if a search has been completed.
    fn full(&self) -> bool {
        false
    }
}

pub trait Reducer<Result> {
    /// Reduce two final results into one; this is executed after a
    /// split.
    fn reduce(self, left: Result, right: Result) -> Result;
}

/// A stateless consumer can be freely copied.
pub trait UnindexedConsumer<I>: Consumer<I> {
    // The result of split_off_left should be used for the left side of the
    // data it consumes, and the remaining consumer for the right side
    // (this matters for methods like find_first).
    fn split_off_left(&self) -> Self;
    fn to_reducer(&self) -> Self::Reducer;
}

/// An unindexed producer that doesn't know its exact length.
/// (or can't represent its known length in a `usize`)
pub trait UnindexedProducer: Send + Sized {
    type Item;

    /// Split midway into a new producer if possible, otherwise return `None`.
    fn split(self) -> (Self, Option<Self>);

    /// Iterate the producer, feeding each element to `folder`, and
    /// stop when the folder is full (or all elements have been consumed).
    fn fold_with<F>(self, folder: F) -> F where F: Folder<Self::Item>;
}

/// A splitter controls the policy for splitting into smaller work items.
///
/// Thief-splitting is an adaptive policy that starts by splitting into
/// enough jobs for every worker thread, and then resets itself whenever a
/// job is actually stolen into a different thread.
#[derive(Clone, Copy)]
struct Splitter {
    origin: usize,
    splits: usize,
}

impl Splitter {
    #[inline]
    fn thief_id() -> usize {
        // The actual `ID` value is irrelevant.  We're just using its TLS
        // address as a unique thread key, faster than a real thread-id call.
        thread_local!{ static ID: bool = false }
        ID.with(|id| id as *const bool as usize)
    }

    #[inline]
    fn new() -> Splitter {
        Splitter {
            origin: Splitter::thief_id(),
            splits: ::current_num_threads(),
        }
    }

    #[inline]
    fn try(&mut self) -> bool {
        let Splitter { origin, splits } = *self;

        let id = Splitter::thief_id();
        if origin != id {
            self.origin = id;
            self.splits = cmp::max(::current_num_threads(), self.splits / 2);
            true
        } else if splits > 0 {
            self.splits /= 2;
            true
        } else {
            false
        }
    }
}

#[derive(Clone, Copy)]
struct LengthSplitter {
    inner: Splitter,
    min: usize,
}

impl LengthSplitter {
    #[inline]
    fn new(min: usize, max: usize, len: usize) -> LengthSplitter {
        let mut splitter = LengthSplitter {
            inner: Splitter::new(),
            min: cmp::max(min, 1),
        };

        let max_splits = len / cmp::max(max, 1);
        if max_splits > splitter.inner.splits {
            splitter.inner.splits = max_splits;
        }

        splitter
    }

    #[inline]
    fn try(&mut self, len: usize) -> bool {
        len / 2 >= self.min && self.inner.try()
    }
}

pub fn bridge<I, C>(mut par_iter: I, consumer: C) -> C::Result
    where I: IndexedParallelIterator,
          C: Consumer<I::Item>
{
    let len = par_iter.len();
    return par_iter.with_producer(Callback {
                                      len: len,
                                      consumer: consumer,
                                  });

    struct Callback<C> {
        len: usize,
        consumer: C,
    }

    impl<C, I> ProducerCallback<I> for Callback<C>
        where C: Consumer<I>
    {
        type Output = C::Result;
        fn callback<P>(self, producer: P) -> C::Result
            where P: Producer<Item = I>
        {
            bridge_producer_consumer(self.len, producer, self.consumer)
        }
    }
}

pub fn bridge_producer_consumer<P, C>(len: usize, producer: P, consumer: C) -> C::Result
    where P: Producer,
          C: Consumer<P::Item>
{
    let splitter = LengthSplitter::new(producer.min_len(), producer.max_len(), len);
    return helper(len, splitter, producer, consumer);

    fn helper<P, C>(len: usize, mut splitter: LengthSplitter, producer: P, consumer: C) -> C::Result
        where P: Producer,
              C: Consumer<P::Item>
    {
        if consumer.full() {
            consumer.into_folder().complete()
        } else if splitter.try(len) {
            let mid = len / 2;
            let (left_producer, right_producer) = producer.split_at(mid);
            let (left_consumer, right_consumer, reducer) = consumer.split_at(mid);
            let (left_result, right_result) =
                join(|| helper(mid, splitter, left_producer, left_consumer),
                     || helper(len - mid, splitter, right_producer, right_consumer));
            reducer.reduce(left_result, right_result)
        } else {
            producer.fold_with(consumer.into_folder()).complete()
        }
    }
}

pub fn bridge_unindexed<P, C>(producer: P, consumer: C) -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer<P::Item>
{
    let splitter = Splitter::new();
    bridge_unindexed_producer_consumer(splitter, producer, consumer)
}

fn bridge_unindexed_producer_consumer<P, C>(mut splitter: Splitter,
                                            producer: P,
                                            consumer: C)
                                            -> C::Result
    where P: UnindexedProducer,
          C: UnindexedConsumer<P::Item>
{
    if consumer.full() {
        consumer.into_folder().complete()
    } else if splitter.try() {
        match producer.split() {
            (left_producer, Some(right_producer)) => {
                let (reducer, left_consumer, right_consumer) =
                    (consumer.to_reducer(), consumer.split_off_left(), consumer);
                let bridge = bridge_unindexed_producer_consumer;
                let (left_result, right_result) =
                    join(|| bridge(splitter, left_producer, left_consumer),
                         || bridge(splitter, right_producer, right_consumer));
                reducer.reduce(left_result, right_result)
            }
            (producer, None) => producer.fold_with(consumer.into_folder()).complete(),
        }
    } else {
        producer.fold_with(consumer.into_folder()).complete()
    }
}

<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Regular language
//!
//! This provides a data structure representing a regular language. This is a more generalised form of a regular
//! expression (it supports symbol types other than simple strings). A regular language using the `u8` symbol forms
//! a simple regular expression.
//!

use std::iter::FromIterator;
use std::ops::Range;

///
/// A Pattern represents a matching pattern in a regular language
///
#[derive(Clone, PartialEq, Eq)]
pub enum Pattern<Symbol> {
    ///
    /// Matches nothing
    ///
    Epsilon,

    ///
    /// Matches a specific literal phrase
    ///
    Match(Vec<Symbol>),

    ///
    /// Matches at least a particular number of repetitions of a pattern
    ///
    /// `RepeatInfinite(0, X)` is the equivalent of the regular expression `X*`, `RepeatInfinite(1, X)` is the equivalent of the regular expression `X+`
    ///
    RepeatInfinite(u32, Box<Pattern<Symbol>>),

    ///
    /// Matches a range of repetitions of a pattern
    ///
    Repeat(Range<u32>, Box<Pattern<Symbol>>),

    ///
    /// Matches a set of sub-patterns in order
    ///
    MatchAll(Vec<Pattern<Symbol>>),

    ///
    /// Matches any one of a set of patterns
    ///
    MatchAny(Vec<Pattern<Symbol>>)
}

pub use Pattern::*;

///
/// Implemented by things that can be converted into a pattern
///
pub trait ToPattern<Symbol> {
    ///
    /// Converts a particular object into a pattern that will match it
    ///
    fn to_pattern(&self) -> Pattern<Symbol>;
}

///
/// Implemented by things that can be converted into a pattern
///
pub trait IntoPattern<Symbol> {
    ///
    /// Converts a particular object into a pattern that will match it
    ///
    fn into_pattern(self) -> Pattern<Symbol>;
}

impl<Symbol> IntoPattern<Symbol> for Pattern<Symbol> {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        self
    }
}

impl<Symbol> IntoPattern<Symbol> for Box<Pattern<Symbol>> {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        *self
    }
}

impl<'a, Symbol: Clone, PatternType: ToPattern<Symbol>> IntoPattern<Symbol> for &'a PatternType {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        self.to_pattern()
    }
}

impl<'a, Symbol: Clone> IntoPattern<Symbol> for &'a [Symbol] {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        self.to_pattern()
    }
}

impl<'a> IntoPattern<char> for &'a str {
    #[inline]
    fn into_pattern(self) -> Pattern<char> {
        self.to_pattern()
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for Pattern<Symbol> {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        self.clone()
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for Box<Pattern<Symbol>> {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        (**self).clone()
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for Vec<Symbol> {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        Match(self.clone())
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for [Symbol] {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        Match(self.to_vec())
    }
}

impl ToPattern<char> for str {
    #[inline]
    fn to_pattern(&self) -> Pattern<char> {
        Match(Vec::from_iter(self.chars()))
    }
}

///
/// Implemented by things that can modify existing patterns into other forms
///
/// Pattern transformers act by altering a single pattern object into a new form
///
pub trait PatternTransformer<Symbol> {
    /// Repeats the current pattern forever
    fn repeat_forever(self, min_count: u32) -> Pattern<Symbol>;

    /// Repeats the current pattern for a certain number of iterations
    fn repeat(self, count: Range<u32>) -> Pattern<Symbol>;
}

///
/// Implemented by things that combine patterns together to create new patterns
///
pub trait PatternCombiner<Symbol, SecondPattern: IntoPattern<Symbol>> {
    /// Appends a pattern to this one
    fn append(self, pattern: SecondPattern) -> Pattern<Symbol>;

    /// Matches either this pattern or the specified pattern
    fn or(self, pattern: SecondPattern) -> Pattern<Symbol>;
}

impl<Symbol> Pattern<Symbol> {

}

impl<Symbol, PatternType: IntoPattern<Symbol>> PatternTransformer<Symbol> for PatternType {
    fn repeat_forever(self, min_count: u32) -> Pattern<Symbol> {
        RepeatInfinite(min_count, Box::new(self.into_pattern()))
    }

    fn repeat(self, count: Range<u32>) -> Pattern<Symbol> {
        Repeat(count, Box::new(self.into_pattern()))
    }
}

impl<Symbol, PatternType: IntoPattern<Symbol>, SecondPatternType: IntoPattern<Symbol>> PatternCombiner<Symbol, SecondPatternType> for PatternType {
    fn append(self, pattern: SecondPatternType) -> Pattern<Symbol> {
        MatchAll(vec![self.into_pattern(), pattern.into_pattern()])
    }

    fn or(self, pattern: SecondPatternType) -> Pattern<Symbol> {
        MatchAny(vec![self.into_pattern(), pattern.into_pattern()])
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn can_convert_vec_to_pattern() {
        let pattern = vec![0, 1, 2].into_pattern();

        assert!(pattern == Match(vec![0, 1, 2]));
    }

    #[test]
    fn can_convert_array_to_pattern() {
        let pattern = [0, 1, 2].into_pattern();

        assert!(pattern == Match(vec![0, 1, 2]));
    }

    #[test]
    fn can_convert_string_to_pattern() {
        let pattern = "abc".into_pattern();

        assert!(pattern == Match(vec!['a', 'b', 'c']));
    }

    #[test]
    fn can_repeat_pattern() {
        let pattern = "abc".repeat(1..2);

        assert!(pattern == Repeat(1..2, Box::new(Match(vec!['a', 'b', 'c']))));
    }

    #[test]
    fn can_repeat_pattern_forever() {
        let pattern = "abc".repeat_forever(0);

        assert!(pattern == RepeatInfinite(0, Box::new(Match(vec!['a', 'b', 'c']))));
    }

    #[test]
    fn can_append_pattern() {
        let pattern = "abc".append("def");

        assert!(pattern == MatchAll(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
        //assert!(pattern == MatchAll(vec![Match(vec!['a', 'b', 'c', 'd', 'e', 'f'])]));
    }

    #[test]
    fn can_or_pattern() {
        let pattern = "abc".or("def");

        assert!(pattern == MatchAny(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
    }
}
<commit_msg>Add a bunch of combining rules so we develop more efficient patterns<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Regular language
//!
//! This provides a data structure representing a regular language. This is a more generalised form of a regular
//! expression (it supports symbol types other than simple strings). A regular language using the `u8` symbol forms
//! a simple regular expression.
//!

use std::iter::FromIterator;
use std::ops::Range;

///
/// A Pattern represents a matching pattern in a regular language
///
#[derive(Clone, PartialEq, Eq)]
pub enum Pattern<Symbol> {
    ///
    /// Matches nothing
    ///
    Epsilon,

    ///
    /// Matches a specific literal phrase
    ///
    Match(Vec<Symbol>),

    ///
    /// Matches at least a particular number of repetitions of a pattern
    ///
    /// `RepeatInfinite(0, X)` is the equivalent of the regular expression `X*`, `RepeatInfinite(1, X)` is the equivalent of the regular expression `X+`
    ///
    RepeatInfinite(u32, Box<Pattern<Symbol>>),

    ///
    /// Matches a range of repetitions of a pattern
    ///
    Repeat(Range<u32>, Box<Pattern<Symbol>>),

    ///
    /// Matches a set of sub-patterns in order
    ///
    MatchAll(Vec<Pattern<Symbol>>),

    ///
    /// Matches any one of a set of patterns
    ///
    MatchAny(Vec<Pattern<Symbol>>)
}

pub use Pattern::*;

///
/// Implemented by things that can be converted into a pattern
///
pub trait ToPattern<Symbol> {
    ///
    /// Converts a particular object into a pattern that will match it
    ///
    fn to_pattern(&self) -> Pattern<Symbol>;
}

///
/// Implemented by things that can be converted into a pattern
///
pub trait IntoPattern<Symbol> {
    ///
    /// Converts a particular object into a pattern that will match it
    ///
    fn into_pattern(self) -> Pattern<Symbol>;
}

impl<Symbol> IntoPattern<Symbol> for Pattern<Symbol> {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        self
    }
}

impl<Symbol> IntoPattern<Symbol> for Box<Pattern<Symbol>> {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        *self
    }
}

impl<'a, Symbol: Clone, PatternType: ToPattern<Symbol>> IntoPattern<Symbol> for &'a PatternType {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        self.to_pattern()
    }
}

impl<'a, Symbol: Clone> IntoPattern<Symbol> for &'a [Symbol] {
    #[inline]
    fn into_pattern(self) -> Pattern<Symbol> {
        self.to_pattern()
    }
}

impl<'a> IntoPattern<char> for &'a str {
    #[inline]
    fn into_pattern(self) -> Pattern<char> {
        self.to_pattern()
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for Pattern<Symbol> {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        self.clone()
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for Box<Pattern<Symbol>> {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        (**self).clone()
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for Vec<Symbol> {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        Match(self.clone())
    }
}

impl<Symbol: Clone> ToPattern<Symbol> for [Symbol] {
    #[inline]
    fn to_pattern(&self) -> Pattern<Symbol> {
        Match(self.to_vec())
    }
}

impl ToPattern<char> for str {
    #[inline]
    fn to_pattern(&self) -> Pattern<char> {
        Match(Vec::from_iter(self.chars()))
    }
}

///
/// Implemented by things that can modify existing patterns into other forms
///
/// Pattern transformers act by altering a single pattern object into a new form
///
pub trait PatternTransformer<Symbol> {
    /// Repeats the current pattern forever
    fn repeat_forever(self, min_count: u32) -> Pattern<Symbol>;

    /// Repeats the current pattern for a certain number of iterations
    fn repeat(self, count: Range<u32>) -> Pattern<Symbol>;
}

///
/// Implemented by things that combine patterns together to create new patterns
///
pub trait PatternCombiner<Symbol, SecondPattern: IntoPattern<Symbol>> {
    /// Appends a pattern to this one
    fn append(self, pattern: SecondPattern) -> Pattern<Symbol>;

    /// Matches either this pattern or the specified pattern
    fn or(self, pattern: SecondPattern) -> Pattern<Symbol>;
}

impl<Symbol> Pattern<Symbol> {

}

impl<Symbol, PatternType: IntoPattern<Symbol>> PatternTransformer<Symbol> for PatternType {
    fn repeat_forever(self, min_count: u32) -> Pattern<Symbol> {
        RepeatInfinite(min_count, Box::new(self.into_pattern()))
    }

    fn repeat(self, count: Range<u32>) -> Pattern<Symbol> {
        Repeat(count, Box::new(self.into_pattern()))
    }
}

impl<Symbol: Clone, PatternType: IntoPattern<Symbol>, SecondPatternType: IntoPattern<Symbol>> PatternCombiner<Symbol, SecondPatternType> for PatternType {
    fn append(self, pattern: SecondPatternType) -> Pattern<Symbol> {
        // Get the two patterns to combine
        let first_pattern   = self.into_pattern();
        let second_pattern  = pattern.into_pattern();

        // Combination rules depend on what the patterns are
        match (first_pattern, second_pattern) {
            // Combining 'Match(x)' and 'Match(y)' should produce 'Match(xy)'
            (Match(first_string), Match(second_string)) => Match(first_string.into_iter().chain(second_string.into_iter()).collect()),

            // Combining 'MatchAll(x)' and 'MatchAll(y)' should produce 'MatchAll(xy)'
            (MatchAll(first_string), MatchAll(second_string)) => MatchAll(first_string.into_iter().chain(second_string.into_iter()).collect()),

            // Combining 'MatchAll(x)' and 'y' should produce 'MatchAll(xy)'
            (MatchAll(first_string), second) => {
                let mut result = first_string.clone();
                result.push(second);
                MatchAll(result)
            },

            // Combining 'x' and 'MatchAll(y)' should produce 'MatchAll(xy)'
            (first, MatchAll(second_string)) => {
                let mut result: Vec<Pattern<Symbol>> = vec![first];
                for p in second_string {
                    result.push(p.clone());
                }
                MatchAll(result)
            },

            // Everything else is just MatchAll(xy)
            (first, second) => MatchAll(vec![first, second])
        }
    }

    fn or(self, pattern: SecondPatternType) -> Pattern<Symbol> {
        // Get the two patterns to combine
        let first_pattern   = self.into_pattern();
        let second_pattern  = pattern.into_pattern();

        // Combination rules depend on what the patterns are
        match (first_pattern, second_pattern) {
            // Combining 'MatchAny(x)' and 'MatchAny(y)' should produce 'MatchAny(xy)'
            (MatchAny(first_string), MatchAny(second_string)) => MatchAny(first_string.into_iter().chain(second_string.into_iter()).collect()),

            // Everything else is just MatchAny(xy)
            (first, second) => MatchAny(vec![first, second])
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn can_convert_vec_to_pattern() {
        let pattern = vec![0, 1, 2].into_pattern();

        assert!(pattern == Match(vec![0, 1, 2]));
    }

    #[test]
    fn can_convert_array_to_pattern() {
        let pattern = [0, 1, 2].into_pattern();

        assert!(pattern == Match(vec![0, 1, 2]));
    }

    #[test]
    fn can_convert_string_to_pattern() {
        let pattern = "abc".into_pattern();

        assert!(pattern == Match(vec!['a', 'b', 'c']));
    }

    #[test]
    fn can_repeat_pattern() {
        let pattern = "abc".repeat(1..2);

        assert!(pattern == Repeat(1..2, Box::new(Match(vec!['a', 'b', 'c']))));
    }

    #[test]
    fn can_repeat_pattern_forever() {
        let pattern = "abc".repeat_forever(0);

        assert!(pattern == RepeatInfinite(0, Box::new(Match(vec!['a', 'b', 'c']))));
    }

    #[test]
    fn can_append_pattern_combine_matches() {
        let pattern = "abc".append("def");

        assert!(pattern == Match(vec!['a', 'b', 'c', 'd', 'e', 'f']));
    }

    #[test]
    fn can_append_pattern_combine_matchalls_left() {
        let pattern = MatchAll(vec!["abc".to_pattern()]).append("def");

        assert!(pattern == MatchAll(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
    }

    #[test]
    fn can_append_pattern_combine_matchalls_right() {
        let pattern = "abc".append(MatchAll(vec!["def".to_pattern()]));

        assert!(pattern == MatchAll(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
    }

    #[test]
    fn can_append_pattern_combine_matchalls_both() {
        let pattern = MatchAll(vec!["abc".to_pattern()]).append(MatchAll(vec!["def".to_pattern()]));

        assert!(pattern == MatchAll(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
    }

    #[test]
    fn can_append_pattern_combine_not_matchall_or_match() {
        let pattern: Pattern<char> = MatchAny(vec![Epsilon]).append(MatchAny(vec![Epsilon]));

        assert!(pattern == MatchAll(vec![MatchAny(vec![Epsilon]), MatchAny(vec![Epsilon])]));
    }

    #[test]
    fn can_or_pattern() {
        let pattern = "abc".or("def");

        assert!(pattern == MatchAny(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
    }

    #[test]
    fn can_or_pattern_combine_matchanys() {
        let pattern = MatchAny(vec!["abc".to_pattern()]).or(MatchAny(vec!["def".to_pattern()]));

        assert!(pattern == MatchAny(vec![Match(vec!['a', 'b', 'c']), Match(vec!['d', 'e', 'f'])]));
    }
}
<|endoftext|>