<commit_before>use std::fmt::{Debug, Formatter};
use std::fmt;
use std::ops::Deref;

use clap::ArgMatches;

use runtime::Runtime;
use module::Module;

use storage::parser::FileHeaderParser;
use storage::parser::Parser;
use storage::json::parser::JsonHeaderParser;
use module::helpers::cli::create_tag_filter;
use module::helpers::cli::create_hash_filter;
use module::helpers::cli::create_text_header_field_grep_filter;
use module::helpers::cli::CliFileFilter;

mod header;

use self::header::get_url_from_header;
use self::header::get_tags_from_header;

pub struct BM<'a> {
    rt: &'a Runtime<'a>,
}

impl<'a> BM<'a> {

    pub fn new(rt: &'a Runtime<'a>) -> BM<'a> {
        BM {
            rt: rt,
        }
    }

    /**
     * Subcommand: add
     */
    fn command_add(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};
        use std::process::exit;
        use self::header::build_header;

        let parser = Parser::new(JsonHeaderParser::new(None));

        let url    = matches.value_of("url").map(String::from).unwrap(); // clap ensures this is present

        if !self.validate_url(&url, &parser) {
            error!("URL validation failed, exiting.");
            exit(1);
        } else {
            debug!("Verification succeeded");
        }

        let tags   = matches.value_of("tags").and_then(|s| {
            Some(s.split(",").map(String::from).collect())
        }).unwrap_or(vec![]);

        debug!("Building header with");
        debug!("    url  = '{:?}'", url);
        debug!("    tags = '{:?}'", tags);
        let header = build_header(url, tags);

        let fileid = self.rt
                         .store()
                         .new_file_with_header(self, header);

        let result = self.rt
            .store()
            .load(self, &parser, &fileid)
            .map(|file| {
                info!("{}", Yellow.paint(format!("Created file in memory: {}", fileid)));
                self.rt
                    .store()
                    .persist(&parser, file)
            })
            .unwrap_or(false);

        if result {
            info!("{}", Red.paint("Adding worked"));
        } else {
            info!("{}", Green.paint("Adding failed"));
        }

        result
    }

    fn validate_url<HP>(&self, url: &String, parser: &Parser<HP>) -> bool
        where HP: FileHeaderParser
    {
        use util::is_url;

        if !is_url(url) {
            error!("Url '{}' is not a valid URL. Will not store.", url);
            return false;
        }

        let is_in_store = self.rt
            .store()
            .load_for_module(self, parser)
            .iter()
            .any(|file| {
                let f = file.deref().borrow();
                get_url_from_header(f.header()).map(|url_in_store| {
                    &url_in_store == url
                }).unwrap_or(false)
            });

        if is_in_store {
            error!("URL '{}' seems to be in the store already", url);
            return false;
        }

        return true;
    }

    /**
     * Subcommand: list
     */
    fn command_list(&self, matches: &ArgMatches) -> bool {
        use ui::file::{FilePrinter, TablePrinter};
        use std::ops::Deref;

        let parser = Parser::new(JsonHeaderParser::new(None));
        let filter = {
            let hash_filter = create_hash_filter(matches, "id", true);
            let text_filter = create_text_header_field_grep_filter(matches, "match", "URL", true);
            let tags_filter = create_tag_filter(matches, "tags", true);
            hash_filter.and(Box::new(text_filter)).and(Box::new(tags_filter))
        };

        let files  = self.rt
            .store()
            .load_for_module(self, &parser)
            .into_iter()
            .filter(|file| filter.filter_file(file));
        let printer = TablePrinter::new(self.rt.is_verbose(), self.rt.is_debugging());

        printer.print_files_custom(files,
            &|file| {
                let fl = file.deref().borrow();
                let hdr = fl.header();
                let url = get_url_from_header(hdr).unwrap_or(String::from("Parser error"));
                let tags = get_tags_from_header(hdr);

                debug!("Custom printer field: url  = '{:?}'", url);
                debug!("Custom printer field: tags = '{:?}'", tags);

                vec![url, tags.join(", ")]
            }
        );
        true
    }

    /**
     * Subcommand: open
     */
    fn command_open(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};
        use open;

        let parser = Parser::new(JsonHeaderParser::new(None));
        let filter : Box<CliFileFilter> = {
            let hash_filter = create_hash_filter(matches, "id", true);
            let text_filter = create_text_header_field_grep_filter(matches, "match", "URL", true);
            let tags_filter = create_tag_filter(matches, "tags", true);
            Box::new(hash_filter.and(Box::new(text_filter)).and(Box::new(tags_filter)))
        };
        let result = self.rt
            .store()
            .load_for_module(self, &parser)
            .iter()
            .filter(|file| filter.filter_file(file))
            .map(|file| {
                debug!("File loaded, can open now: {:?}", file);
                let f = file.deref().borrow();
                get_url_from_header(f.header()).map(|url| {
                    if open::that(&url[..]).is_ok() {
                        info!("{}", Green.paint(format!("open({})", url)));
                        true
                    } else {
                        info!("{}", Red.paint(format!("could not open({})", url)));
                        false
                    }
                })
                .unwrap_or(false)
            })
            .fold((0, 0), |acc, succeeded| {
                let (worked, failed) = acc;
                if succeeded {
                    (worked + 1, failed)
                } else {
                    (worked, failed + 1)
                }
            });

        let (succ, fail) = result;
        info!("{}", Green.paint(format!("open() succeeded for {} files", succ)));
        info!("{}", Red.paint(format!(  "open() failed    for {} files", fail)));
        return fail == 0;
    }

    /**
     * Subcommand: remove
     */
    fn command_remove(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};

        let parser = Parser::new(JsonHeaderParser::new(None));

        let filter = {
            let hash_filter = create_hash_filter(matches, "id", false);
            let text_filter = create_text_header_field_grep_filter(matches, "match", "URL", false);
            let tags_filter = create_tag_filter(matches, "tags", false);
            hash_filter.or(Box::new(text_filter)).or(Box::new(tags_filter))
        };

        let result = self.rt
            .store()
            .load_for_module(self, &parser)
            .iter()
            .filter(|file| filter.filter_file(file))
            .map(|file| {
                debug!("File loaded, can remove now: {:?}", file);
                let f = file.deref().borrow();
                self.rt.store().remove(f.id().clone())
            })
            .fold((0, 0), |acc, succeeded| {
                let (worked, failed) = acc;
                if succeeded {
                    (worked + 1, failed)
                } else {
                    (worked, failed + 1)
                }
            });

        let (worked, failed) = result;

        info!("{}", Green.paint(format!("Removing succeeded for {} files", worked)));
        info!("{}", Red.paint(format!(  "Removing failed for {} files", failed)));

        return failed == 0;
    }

    /**
     * Subcommand: add_tags
     */
    fn command_add_tags(&self, matches: &ArgMatches) -> bool {
        use module::helpers::header::tags::data::alter_tags_in_files;
        use self::header::rebuild_header_with_tags;

        let parser = Parser::new(JsonHeaderParser::new(None));
        alter_tags_in_files(self, matches, &parser, |old_tags, cli_tags| {
            let mut new_tags = old_tags.clone();
            new_tags.append(&mut cli_tags.clone());
            new_tags
        }, rebuild_header_with_tags)
    }

    /**
     * Subcommand: rm_tags
     */
    fn command_rm_tags(&self, matches: &ArgMatches) -> bool {
        use module::helpers::header::tags::data::alter_tags_in_files;
        use self::header::rebuild_header_with_tags;

        let parser = Parser::new(JsonHeaderParser::new(None));
        alter_tags_in_files(self, matches, &parser, |old_tags, cli_tags| {
            old_tags.clone()
                .into_iter()
                .filter(|tag| !cli_tags.contains(tag))
                .collect()
        }, rebuild_header_with_tags)
    }

    /**
     * Subcommand: set_tags
     */
    fn command_set_tags(&self, matches: &ArgMatches) -> bool {
        use module::helpers::header::tags::data::alter_tags_in_files;
        use self::header::rebuild_header_with_tags;

        let parser = Parser::new(JsonHeaderParser::new(None));
        alter_tags_in_files(self, matches, &parser, |old_tags, cli_tags| {
            cli_tags.clone()
        }, rebuild_header_with_tags)
    }

}

/**
 * Trait implementation for BM module
 */
impl<'a> Module<'a> for BM<'a> {

    fn exec(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};

        match matches.subcommand_name() {
            Some("add") => {
                self.command_add(matches.subcommand_matches("add").unwrap())
            },

            Some("list") => {
                self.command_list(matches.subcommand_matches("list").unwrap())
            },

            Some("open") => {
                self.command_open(matches.subcommand_matches("open").unwrap())
            },

            Some("remove") => {
                self.command_remove(matches.subcommand_matches("remove").unwrap())
            },

            Some("add_tags") => {
                self.command_add_tags(matches.subcommand_matches("add_tags").unwrap())
            },

            Some("rm_tags") => {
                self.command_rm_tags(matches.subcommand_matches("rm_tags").unwrap())
            },

            Some("set_tags") => {
                self.command_set_tags(matches.subcommand_matches("set_tags").unwrap())
            },

            Some(_) | None => {
                info!("{}", Red.bold().paint("No command given, doing nothing"));
                false
            },
        }
    }

    fn name(&self) -> &'static str {
        "bookmark"
    }

    fn runtime(&self) -> &Runtime {
        self.rt
    }
}

impl<'a> Debug for BM<'a> {

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        try!(write!(fmt, "BM"));
        Ok(())
    }

}

<commit_msg>Remove unused import<commit_after>use std::fmt::{Debug, Formatter};
use std::fmt;
use std::ops::Deref;

use clap::ArgMatches;

use runtime::Runtime;
use module::Module;

use storage::parser::FileHeaderParser;
use storage::parser::Parser;
use storage::json::parser::JsonHeaderParser;
use module::helpers::cli::create_tag_filter;
use module::helpers::cli::create_hash_filter;
use module::helpers::cli::create_text_header_field_grep_filter;
use module::helpers::cli::CliFileFilter;

mod header;

use self::header::get_url_from_header;
use self::header::get_tags_from_header;

pub struct BM<'a> {
    rt: &'a Runtime<'a>,
}

impl<'a> BM<'a> {

    pub fn new(rt: &'a Runtime<'a>) -> BM<'a> {
        BM {
            rt: rt,
        }
    }

    /**
     * Subcommand: add
     */
    fn command_add(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};
        use std::process::exit;
        use self::header::build_header;

        let parser = Parser::new(JsonHeaderParser::new(None));

        let url    = matches.value_of("url").map(String::from).unwrap(); // clap ensures this is present

        if !self.validate_url(&url, &parser) {
            error!("URL validation failed, exiting.");
            exit(1);
        } else {
            debug!("Verification succeeded");
        }

        let tags   = matches.value_of("tags").and_then(|s| {
            Some(s.split(",").map(String::from).collect())
        }).unwrap_or(vec![]);

        debug!("Building header with");
        debug!("    url  = '{:?}'", url);
        debug!("    tags = '{:?}'", tags);
        let header = build_header(url, tags);

        let fileid = self.rt
                         .store()
                         .new_file_with_header(self, header);

        let result = self.rt
            .store()
            .load(self, &parser, &fileid)
            .map(|file| {
                info!("{}", Yellow.paint(format!("Created file in memory: {}", fileid)));
                self.rt
                    .store()
                    .persist(&parser, file)
            })
            .unwrap_or(false);

        if result {
            info!("{}", Red.paint("Adding worked"));
        } else {
            info!("{}", Green.paint("Adding failed"));
        }

        result
    }

    fn validate_url<HP>(&self, url: &String, parser: &Parser<HP>) -> bool
        where HP: FileHeaderParser
    {
        use util::is_url;

        if !is_url(url) {
            error!("Url '{}' is not a valid URL. Will not store.", url);
            return false;
        }

        let is_in_store = self.rt
            .store()
            .load_for_module(self, parser)
            .iter()
            .any(|file| {
                let f = file.deref().borrow();
                get_url_from_header(f.header()).map(|url_in_store| {
                    &url_in_store == url
                }).unwrap_or(false)
            });

        if is_in_store {
            error!("URL '{}' seems to be in the store already", url);
            return false;
        }

        return true;
    }

    /**
     * Subcommand: list
     */
    fn command_list(&self, matches: &ArgMatches) -> bool {
        use ui::file::{FilePrinter, TablePrinter};
        use std::ops::Deref;

        let parser = Parser::new(JsonHeaderParser::new(None));
        let filter = {
            let hash_filter = create_hash_filter(matches, "id", true);
            let text_filter = create_text_header_field_grep_filter(matches, "match", "URL", true);
            let tags_filter = create_tag_filter(matches, "tags", true);
            hash_filter.and(Box::new(text_filter)).and(Box::new(tags_filter))
        };

        let files  = self.rt
            .store()
            .load_for_module(self, &parser)
            .into_iter()
            .filter(|file| filter.filter_file(file));
        let printer = TablePrinter::new(self.rt.is_verbose(), self.rt.is_debugging());

        printer.print_files_custom(files,
            &|file| {
                let fl = file.deref().borrow();
                let hdr = fl.header();
                let url = get_url_from_header(hdr).unwrap_or(String::from("Parser error"));
                let tags = get_tags_from_header(hdr);

                debug!("Custom printer field: url  = '{:?}'", url);
                debug!("Custom printer field: tags = '{:?}'", tags);

                vec![url, tags.join(", ")]
            }
        );
        true
    }

    /**
     * Subcommand: open
     */
    fn command_open(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Red};
        use open;

        let parser = Parser::new(JsonHeaderParser::new(None));
        let filter : Box<CliFileFilter> = {
            let hash_filter = create_hash_filter(matches, "id", true);
            let text_filter = create_text_header_field_grep_filter(matches, "match", "URL", true);
            let tags_filter = create_tag_filter(matches, "tags", true);
            Box::new(hash_filter.and(Box::new(text_filter)).and(Box::new(tags_filter)))
        };
        let result = self.rt
            .store()
            .load_for_module(self, &parser)
            .iter()
            .filter(|file| filter.filter_file(file))
            .map(|file| {
                debug!("File loaded, can open now: {:?}", file);
                let f = file.deref().borrow();
                get_url_from_header(f.header()).map(|url| {
                    if open::that(&url[..]).is_ok() {
                        info!("{}", Green.paint(format!("open({})", url)));
                        true
                    } else {
                        info!("{}", Red.paint(format!("could not open({})", url)));
                        false
                    }
                })
                .unwrap_or(false)
            })
            .fold((0, 0), |acc, succeeded| {
                let (worked, failed) = acc;
                if succeeded {
                    (worked + 1, failed)
                } else {
                    (worked, failed + 1)
                }
            });

        let (succ, fail) = result;
        info!("{}", Green.paint(format!("open() succeeded for {} files", succ)));
        info!("{}", Red.paint(format!(  "open() failed    for {} files", fail)));
        return fail == 0;
    }

    /**
     * Subcommand: remove
     */
    fn command_remove(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};

        let parser = Parser::new(JsonHeaderParser::new(None));

        let filter = {
            let hash_filter = create_hash_filter(matches, "id", false);
            let text_filter = create_text_header_field_grep_filter(matches, "match", "URL", false);
            let tags_filter = create_tag_filter(matches, "tags", false);
            hash_filter.or(Box::new(text_filter)).or(Box::new(tags_filter))
        };

        let result = self.rt
            .store()
            .load_for_module(self, &parser)
            .iter()
            .filter(|file| filter.filter_file(file))
            .map(|file| {
                debug!("File loaded, can remove now: {:?}", file);
                let f = file.deref().borrow();
                self.rt.store().remove(f.id().clone())
            })
            .fold((0, 0), |acc, succeeded| {
                let (worked, failed) = acc;
                if succeeded {
                    (worked + 1, failed)
                } else {
                    (worked, failed + 1)
                }
            });

        let (worked, failed) = result;

        info!("{}", Green.paint(format!("Removing succeeded for {} files", worked)));
        info!("{}", Red.paint(format!(  "Removing failed for {} files", failed)));

        return failed == 0;
    }

    /**
     * Subcommand: add_tags
     */
    fn command_add_tags(&self, matches: &ArgMatches) -> bool {
        use module::helpers::header::tags::data::alter_tags_in_files;
        use self::header::rebuild_header_with_tags;

        let parser = Parser::new(JsonHeaderParser::new(None));
        alter_tags_in_files(self, matches, &parser, |old_tags, cli_tags| {
            let mut new_tags = old_tags.clone();
            new_tags.append(&mut cli_tags.clone());
            new_tags
        }, rebuild_header_with_tags)
    }

    /**
     * Subcommand: rm_tags
     */
    fn command_rm_tags(&self, matches: &ArgMatches) -> bool {
        use module::helpers::header::tags::data::alter_tags_in_files;
        use self::header::rebuild_header_with_tags;

        let parser = Parser::new(JsonHeaderParser::new(None));
        alter_tags_in_files(self, matches, &parser, |old_tags, cli_tags| {
            old_tags.clone()
                .into_iter()
                .filter(|tag| !cli_tags.contains(tag))
                .collect()
        }, rebuild_header_with_tags)
    }

    /**
     * Subcommand: set_tags
     */
    fn command_set_tags(&self, matches: &ArgMatches) -> bool {
        use module::helpers::header::tags::data::alter_tags_in_files;
        use self::header::rebuild_header_with_tags;

        let parser = Parser::new(JsonHeaderParser::new(None));
        alter_tags_in_files(self, matches, &parser, |old_tags, cli_tags| {
            cli_tags.clone()
        }, rebuild_header_with_tags)
    }

}

/**
 * Trait implementation for BM module
 */
impl<'a> Module<'a> for BM<'a> {

    fn exec(&self, matches: &ArgMatches) -> bool {
        use ansi_term::Colour::{Green, Yellow, Red};

        match matches.subcommand_name() {
            Some("add") => {
                self.command_add(matches.subcommand_matches("add").unwrap())
            },

            Some("list") => {
                self.command_list(matches.subcommand_matches("list").unwrap())
            },

            Some("open") => {
                self.command_open(matches.subcommand_matches("open").unwrap())
            },

            Some("remove") => {
                self.command_remove(matches.subcommand_matches("remove").unwrap())
            },

            Some("add_tags") => {
                self.command_add_tags(matches.subcommand_matches("add_tags").unwrap())
            },

            Some("rm_tags") => {
                self.command_rm_tags(matches.subcommand_matches("rm_tags").unwrap())
            },

            Some("set_tags") => {
                self.command_set_tags(matches.subcommand_matches("set_tags").unwrap())
            },

            Some(_) | None => {
                info!("{}", Red.bold().paint("No command given, doing nothing"));
                false
            },
        }
    }

    fn name(&self) -> &'static str {
        "bookmark"
    }

    fn runtime(&self) -> &Runtime {
        self.rt
    }
}

impl<'a> Debug for BM<'a> {

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        try!(write!(fmt, "BM"));
        Ok(())
    }

}

<|endoftext|>


<commit_before>use std::collections::HashMap;
// use num::traits::Num;

use core::{Block , BlockType , MutBlock , BlockId};
use graph::{Graph , NodeIndex};

// pub struct Buffer
// {
//
// }

// we could pull the vector graph out .. eg NodeData<T> and have module hold a graph ..

pub struct Module<'b>
{
    graph: Graph<& 'b BlockType<'b>>,
    buffers: Vec<Vec<u8>>,   ///TODO try a single buffer .
    buffers_for_node: HashMap< usize, Vec<usize>>

    // std::slice::from_raw_parts
}

impl<'b> Module<'b>
{
    pub fn new() -> Module<'b> {
        Module { graph: Graph::<& 'b BlockType<'b>>::new() , buffers: Vec::new() , buffers_for_node: HashMap::new() }
    }

    //TODO NodeIndex replace with nodeid IF unique
    pub fn add_box_block(& mut self, block: & 'b mut BlockType<'b>) -> NodeIndex {

        self.graph.add_node(block)
    }

    pub fn add_link(& mut self, from: NodeIndex , to: NodeIndex) {  self.graph.add_edge(from, to); }

    /// this will get WAY more complicated ..
    /// we need to handle loops which never finish , re run on completion , threading , timed and priority etc
    pub fn process_blocks(& 'b mut self)
    {
        self.process_rec(0);

    }

    pub fn process_rec(& mut self , index: NodeIndex )
    {
        let mut successor_ids: Vec<NodeIndex> = Vec::new();

        {
            let node: & BlockType = { self.graph.get_node(index)};
            self.process( node);
            successor_ids  = { self.graph.successors(index).collect()};
        }

        //let successors: Vec<NodeIndex> = self.graph.successors(0).collect();

        for i in successor_ids {
            self.process_rec ( i);
        }

    }

    fn process(& self ,  block: & 'b BlockType)
    {
        match *block {
            BlockType::Block(b) => self.process_block (b ),
            BlockType::MutBlock(ref b) => process_mut_block( *b ),
            BlockType::FunctionBlock(_) => {}
        }
    }

    // fn get_buffer_ids(&self ,  block_id: BlockId) -> Vec< (usize , usize , bool)>
    // {
    //
    //     self.buffers_for_node.get(block_id).map( )
    // }

    // fn get_buffers(& self ,  block_id: BlockId) -> Vec<& 'b [u8]>
    // {
    //
    //     let bufs = self.buffers_for_node.get(& (block_id as usize) ).iter( )
    //         .map(|x| & (self.buffers[block_id as usize] ) [..] );
    //
    //     bufs.collect::<Vec<_>>()
    //     //a.iter().map(|&x| 2 * x);
    //     //buffer_for_node: HashMap< usize, Vec< (usize , usize , bool)>>
    // }

    fn get_buffers(&self,  block_id: BlockId) -> Vec<&[u8]>
     {

         let bufs = self.buffers_for_node[&(block_id as usize)].iter( )
             .map(|x| &self.buffers[block_id as usize][..] );

         bufs.collect::<Vec<_>>()
     }

    //    Block (Box<Block + 'b> ),
    fn process_block(& self ,  blockType: & 'b Block)
    {
        let buffers = self.get_buffers(blockType.get_id());
        //let mut_buffers = get_mut_buffers(blockType.get_id());

        //  process(&mut self , data: & [u8] , inputs: & [u8] , outputs: & mut [u8])
        //*blockType.
    }

}

fn process_mut_block( blockType: &MutBlock)
{

}

// we dont really need an interface , maybe for network
pub trait IModule
{
    //fn process(<Vec<O>>) -> Vec<O>;
    //(args: &[&str])
    //fn process(&[ &[O] ]) -> Vec<O>;
    fn process_blocks(&mut self) ; // or return slice
    // void LoadInputsIntoModules();
    //  void ProcessFirstBlock();
    //  INeuronBlock GetEntryBlock();
    //  INeuronBlock GetBlock(uint id);
    //  void AddRunningTasks(Task task);
    //  void RemoveRunningTasks(Task task);

}
<commit_msg>pre test rebuild 2<commit_after>use std::collections::HashMap;
// use num::traits::Num;

use core::{Block , BlockType , MutBlock , BlockId};
use graph::{Graph , NodeIndex};

// pub struct Buffer
// {
//
// }

// we could pull the vector graph out .. eg NodeData<T> and have module hold a graph ..

pub struct Module<'b>
{
    graph: Graph<& 'b BlockType<'b>>,
    buffers: Vec<Vec<u8>>,   ///TODO try a single buffer .
    buffers_for_node: HashMap< usize, Vec<usize>>

}

impl<'b> Module<'b>
{
    pub fn new() -> Module<'b> {
        Module { graph: Graph::<& 'b BlockType<'b>>::new() , buffers: Vec::new() , buffers_for_node: HashMap::new() }
    }

    //TODO NodeIndex replace with nodeid IF unique
    pub fn add_box_block(& mut self, block: & 'b mut BlockType<'b>) -> NodeIndex {

        self.graph.add_node(block)
    }

    pub fn add_link(& mut self, from: NodeIndex , to: NodeIndex) {  self.graph.add_edge(from, to); }

    /// this will get WAY more complicated ..
    /// we need to handle loops which never finish , re run on completion , threading , timed and priority etc
    pub fn process_blocks(& 'b mut self)
    {
        self.process_rec(0);

    }

    pub fn process_rec(&mut self , index: NodeIndex )
    {
        let successor_ids: Vec<NodeIndex> = { self.graph.successors(index).collect()};

        {
            let node: & BlockType = { self.graph.get_node(index)};
            self.process( node);

        }

        for i in successor_ids {
            self.process_rec ( i);
        }

    }

    fn process(& self ,  block: & 'b BlockType)
    {
        match *block {
            BlockType::Block(b) => self.process_block (b ),
            BlockType::MutBlock(ref b) => process_mut_block( *b ),
            BlockType::FunctionBlock(_) => {}
        }
    }

    // fn get_buffer_ids(&self ,  block_id: BlockId) -> Vec< (usize , usize , bool)>
    // {
    //
    //     self.buffers_for_node.get(block_id).map( )
    // }

    // fn get_buffers(& self ,  block_id: BlockId) -> Vec<& 'b [u8]>
    // {
    //
    //     let bufs = self.buffers_for_node.get(& (block_id as usize) ).iter( )
    //         .map(|x| & (self.buffers[block_id as usize] ) [..] );
    //
    //     bufs.collect::<Vec<_>>()
    //     //a.iter().map(|&x| 2 * x);
    //     //buffer_for_node: HashMap< usize, Vec< (usize , usize , bool)>>
    // }

    fn get_buffers(&self,  block_id: BlockId) -> Vec<&[u8]>
     {

         let bufs = self.buffers_for_node[&(block_id as usize)].iter( )
             .map(|x| &self.buffers[block_id as usize][..] );

         bufs.collect::<Vec<_>>()
     }

    //    Block (Box<Block + 'b> ),
    fn process_block(& self ,  blockType: & 'b Block)
    {
        let buffers = self.get_buffers(blockType.get_id());
        //let mut_buffers = get_mut_buffers(blockType.get_id());

        //  process(&mut self , data: & [u8] , inputs: & [u8] , outputs: & mut [u8])
        //*blockType.
    }

}

fn process_mut_block( blockType: &MutBlock)
{

}

// we dont really need an interface , maybe for network
pub trait IModule
{
    //fn process(<Vec<O>>) -> Vec<O>;
    //(args: &[&str])
    //fn process(&[ &[O] ]) -> Vec<O>;
    fn process_blocks(&mut self) ; // or return slice
    // void LoadInputsIntoModules();
    //  void ProcessFirstBlock();
    //  INeuronBlock GetEntryBlock();
    //  INeuronBlock GetBlock(uint id);
    //  void AddRunningTasks(Task task);
    //  void RemoveRunningTasks(Task task);

}
<|endoftext|>


<commit_before>use tokenizer::Tokenizer;
use top_level_event::TopLevelEvent;
use variable_mapper::VariableMapper;

pub fn expanded<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event.clone() {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.expanded());
                output.push_str("\n\n");
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(comment) => {
                output.push_str(&comment.expanded());
                output.push_str("\n");
            },
        }
    }

    output
}

pub fn nested<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event.clone() {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.nested());
                output.push_str("\n\n");
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(comment) => {
                output.push_str(&comment.nested());
                output.push_str("\n");
            },
        }
    }

    output
}

pub fn compact<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event.clone() {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.compact());
                output.push_str("\n\n");
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(comment) => {
                output.push_str(&comment.compact());
                output.push_str("\n");
            },
        }
    }

    output
}

pub fn compressed<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event.clone() {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.compressed());
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(..) => {},
        }
    }

    output
}

pub fn debug(tokenizer: &mut Tokenizer) -> String {
    let mut output = String::new();

    while let Some(event) = tokenizer.next() {
        output.push_str(&format!("{:?}\n", event));
    }
    output
}
<commit_msg>Get rid of some unnecessary clones in output<commit_after>use tokenizer::Tokenizer;
use top_level_event::TopLevelEvent;
use variable_mapper::VariableMapper;

pub fn expanded<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.expanded());
                output.push_str("\n\n");
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(comment) => {
                output.push_str(&comment.expanded());
                output.push_str("\n");
            },
        }
    }

    output
}

pub fn nested<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.nested());
                output.push_str("\n\n");
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(comment) => {
                output.push_str(&comment.nested());
                output.push_str("\n");
            },
        }
    }

    output
}

pub fn compact<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.compact());
                output.push_str("\n\n");
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(comment) => {
                output.push_str(&comment.compact());
                output.push_str("\n");
            },
        }
    }

    output
}

pub fn compressed<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = TopLevelEvent<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut output = String::new();

    while let Some(event) = vm.next() {
        match event {
            TopLevelEvent::Rule(rule) => {
                output.push_str(&rule.compressed());
            },
            TopLevelEvent::Variable(..) => {},
            TopLevelEvent::Comment(..) => {},
        }
    }

    output
}

pub fn debug(tokenizer: &mut Tokenizer) -> String {
    let mut output = String::new();

    while let Some(event) = tokenizer.next() {
        output.push_str(&format!("{:?}\n", event));
    }
    output
}
<|endoftext|>


<commit_before>use regex::Regex;
use std::collections::BTreeMap;
use serde_json::{Value};
use url::percent_encoding::percent_decode;

use merge::merge;
use helpers::{create_array, push_item_to_array};

lazy_static! {
    static ref PARENT_REGEX: Regex = Regex::new(r"^([^][]+)").unwrap();
    static ref CHILD_REGEX: Regex = Regex::new(r"(\[[^][]*\])").unwrap();
}

#[derive(Debug)]
#[allow(missing_copy_implementations)]
pub enum ParseErrorKind {
    DecodingError,
    Other
}

#[derive(Debug)]
pub struct ParseError {
    pub kind: ParseErrorKind,
    pub message: String
}

pub type ParseResult<T> = Result<T,ParseError>;

pub fn decode_component(source: &str) -> Result<String,String> {
    let result = percent_decode(source.as_bytes()).decode_utf8_lossy().to_string();
    return Ok(result);
}

fn parse_pairs(body: &str) -> Vec<(&str, Option<&str>)> {

    let mut pairs = vec![];

    for part in body.split("&") {
        let separator = part.find("]=")
                            .and_then(|pos| Some(pos+1))
                            .or_else(|| part.find("="));

        match separator {
            None => pairs.push((part, None)),
            Some(pos) => {
                let key = &part[..pos];
                let val = &part[(pos + 1)..];
                pairs.push((key, Some(val)));
            }
        }
    }

    return pairs
}

fn parse_key(key: &str) -> ParseResult<Vec<String>> {
    let mut keys: Vec<String> = vec![];

    match PARENT_REGEX.captures(key) {
        Some(captures) => {
            match decode_component(captures.at(1).unwrap()) {
                Ok(decoded_key) => keys.push(decoded_key),
                Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
            }
        }
        None => ()
    };

    for captures in CHILD_REGEX.captures_iter(key) {
        match decode_component(captures.at(1).unwrap()) {
            Ok(decoded_key) => keys.push(decoded_key),
            Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
        }
    }

    Ok(keys)
}

fn cleanup_key(key: &str) -> &str {
    if key.starts_with("[") && key.ends_with("]") {
        &key[1..(key.len()-1)]
    } else {
        key
    }
}

fn create_idx_merger(idx: u64, obj: Value) -> Value {
    let mut tree: BTreeMap<String,Value> = BTreeMap::new();
    tree.insert("__idx".to_string(), Value::U64(idx));
    tree.insert("__object".to_string(), obj);
    return Value::Object(tree)
}

fn create_object_with_key(key: String, obj: Value) -> Value {
    let mut tree: BTreeMap<String,Value> = BTreeMap::new();
    tree.insert(key, obj);
    return Value::Object(tree)
}

fn apply_object(keys: &[String], val: Value) -> Value {

    if keys.len() > 0 {
        let key = keys.get(0).unwrap();
        if key == "[]" {
            let mut new_array = create_array();
            let item = apply_object(&keys[1..], val);
            push_item_to_array(&mut new_array, item);
            return new_array;
        } else {
            let key = cleanup_key(key);
            let array_index = key.parse();

            match array_index {
                Ok(idx) => {
                    let result = apply_object(&keys[1..], val);
                    let item = create_idx_merger(idx, result);
                    return item;
                },
                Err(_) => {
                    return create_object_with_key(key.to_string(), apply_object(&keys[1..], val));
                }
            }
        }

    } else {
        return val;
    }
}

pub fn parse(params: &str) -> ParseResult<Value> {
    let tree: BTreeMap<String,Value> = BTreeMap::new();
    let mut obj = Value::Object(tree);
    let pairs = parse_pairs(params);
    for &(key, value) in pairs.iter() {
        let parse_key_res = try!(parse_key(key));
        let key_chain = &parse_key_res[0..];
        let decoded_value = match decode_component(value.unwrap_or("")) {
            Ok(val) => val,
            Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
        };
        let partial = apply_object(key_chain, Value::String(decoded_value));
        merge(&mut obj, &partial);
    }

    Ok(obj)
}

#[cfg(test)]
mod tests {
    use parse;
    use serde_json::{Value, to_string};

    fn eq_str(value: Value, string: &str) {
        assert_eq!(&to_string(&value).unwrap(), string)
    }

    #[test]
    fn it_parses_simple_string() {
        eq_str(parse("0=foo").unwrap(), r#"{"0":"foo"}"#);
        eq_str(parse("a[<=>]==23").unwrap(), r#"{"a":{"<=>":"=23"}}"#);
        eq_str(parse(" foo = bar = baz ").unwrap(), r#"{" foo ":" bar = baz "}"#);
    }

    #[test]
    fn it_parses_nested_string() {
        eq_str(parse("a[b][c][d][e][f][g][h]=i").unwrap(),
            r#"{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":"i"}}}}}}}}"#);
    }

    #[test]
    fn it_parses_simple_array() {
        eq_str(parse("a=b&a=c&a=d&a=e").unwrap(),
            r#"{"a":["b","c","d","e"]}"#);
    }

    #[test]
    fn it_parses_explicit_array() {
        eq_str(parse("a[]=b&a[]=c&a[]=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }

    #[test]
    fn it_parses_nested_array() {
        eq_str(parse("a[b][]=c&a[b][]=d").unwrap(),
            r#"{"a":{"b":["c","d"]}}"#);
    }

    #[test]
    fn it_allows_to_specify_array_indexes() {
        eq_str(parse("a[0][]=c&a[1][]=d").unwrap(),
            r#"{"a":[["c"],["d"]]}"#);
    }

    #[test]
    fn it_transforms_arrays_to_object() {
        eq_str(parse("foo[0]=bar&foo[bad]=baz").unwrap(),
            r#"{"foo":{"0":"bar","bad":"baz"}}"#);

        eq_str(parse("foo[0][a]=a&foo[0][b]=b&foo[1][a]=aa&foo[1][b]=bb").unwrap(),
            r#"{"foo":[{"a":"a","b":"b"},{"a":"aa","b":"bb"}]}"#);
    }

    #[test]
    fn it_doesnt_produce_empty_keys() {
        eq_str(parse("_r=1&").unwrap(),
            r#"{"_r":"1"}"#);
    }

    #[test]
    fn it_supports_encoded_strings() {
        eq_str(parse("a[b%20c]=c%20d").unwrap(),
            r#"{"a":{"b c":"c d"}}"#);
    }
}

<commit_msg>fix(parser): decode params before parsing<commit_after>use regex::Regex;
use std::collections::BTreeMap;
use serde_json::{Value};
use url::percent_encoding::percent_decode;

use merge::merge;
use helpers::{create_array, push_item_to_array};

lazy_static! {
    static ref PARENT_REGEX: Regex = Regex::new(r"^([^][]+)").unwrap();
    static ref CHILD_REGEX: Regex = Regex::new(r"(\[[^][]*\])").unwrap();
}

#[derive(Debug)]
#[allow(missing_copy_implementations)]
pub enum ParseErrorKind {
    DecodingError,
    Other
}

#[derive(Debug)]
pub struct ParseError {
    pub kind: ParseErrorKind,
    pub message: String
}

pub type ParseResult<T> = Result<T,ParseError>;

pub fn decode_component(source: &str) -> Result<String,String> {
    let result = percent_decode(source.as_bytes()).decode_utf8_lossy().to_string();
    return Ok(result);
}

fn parse_pairs(body: &str) -> Vec<(&str, Option<&str>)> {

    let mut pairs = vec![];

    for part in body.split("&") {
        let separator = part.find("]=")
                            .and_then(|pos| Some(pos+1))
                            .or_else(|| part.find("="));

        match separator {
            None => pairs.push((part, None)),
            Some(pos) => {
                let key = &part[..pos];
                let val = &part[(pos + 1)..];
                pairs.push((key, Some(val)));
            }
        }
    }

    return pairs
}

fn parse_key(key: &str) -> ParseResult<Vec<String>> {
    let mut keys: Vec<String> = vec![];

    match PARENT_REGEX.captures(key) {
        Some(captures) => {
            match decode_component(captures.at(1).unwrap()) {
                Ok(decoded_key) => keys.push(decoded_key),
                Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
            }
        }
        None => ()
    };

    for captures in CHILD_REGEX.captures_iter(key) {
        match decode_component(captures.at(1).unwrap()) {
            Ok(decoded_key) => keys.push(decoded_key),
            Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
        }
    }

    Ok(keys)
}

fn cleanup_key(key: &str) -> &str {
    if key.starts_with("[") && key.ends_with("]") {
        &key[1..(key.len()-1)]
    } else {
        key
    }
}

fn create_idx_merger(idx: u64, obj: Value) -> Value {
    let mut tree: BTreeMap<String,Value> = BTreeMap::new();
    tree.insert("__idx".to_string(), Value::U64(idx));
    tree.insert("__object".to_string(), obj);
    return Value::Object(tree)
}

fn create_object_with_key(key: String, obj: Value) -> Value {
    let mut tree: BTreeMap<String,Value> = BTreeMap::new();
    tree.insert(key, obj);
    return Value::Object(tree)
}

fn apply_object(keys: &[String], val: Value) -> Value {

    if keys.len() > 0 {
        let key = keys.get(0).unwrap();
        if key == "[]" {
            let mut new_array = create_array();
            let item = apply_object(&keys[1..], val);
            push_item_to_array(&mut new_array, item);
            return new_array;
        } else {
            let key = cleanup_key(key);
            let array_index = key.parse();

            match array_index {
                Ok(idx) => {
                    let result = apply_object(&keys[1..], val);
                    let item = create_idx_merger(idx, result);
                    return item;
                },
                Err(_) => {
                    return create_object_with_key(key.to_string(), apply_object(&keys[1..], val));
                }
            }
        }

    } else {
        return val;
    }
}

pub fn parse(params: &str) -> ParseResult<Value> {
    let tree: BTreeMap<String,Value> = BTreeMap::new();
    let mut obj = Value::Object(tree);
    let decoded_params = match decode_component(params) {
        Ok(val) => val,
        Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
    };
    let pairs = parse_pairs(&decoded_params);
    for &(key, value) in pairs.iter() {
        let parse_key_res = try!(parse_key(key));
        let key_chain = &parse_key_res[0..];
        let decoded_value = match decode_component(value.unwrap_or("")) {
            Ok(val) => val,
            Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
        };
        let partial = apply_object(key_chain, Value::String(decoded_value));
        merge(&mut obj, &partial);
    }

    Ok(obj)
}

#[cfg(test)]
mod tests {
    use parse;
    use serde_json::{Value, to_string};

    fn eq_str(value: Value, string: &str) {
        assert_eq!(&to_string(&value).unwrap(), string)
    }

    #[test]
    fn it_parses_simple_string() {
        eq_str(parse("0=foo").unwrap(), r#"{"0":"foo"}"#);
        eq_str(parse("a[<=>]==23").unwrap(), r#"{"a":{"<=>":"=23"}}"#);
        eq_str(parse(" foo = bar = baz ").unwrap(), r#"{" foo ":" bar = baz "}"#);
    }

    #[test]
    fn it_parses_nested_string() {
        eq_str(parse("a[b][c][d][e][f][g][h]=i").unwrap(),
            r#"{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":"i"}}}}}}}}"#);
    }

    #[test]
    fn it_parses_simple_array() {
        eq_str(parse("a=b&a=c&a=d&a=e").unwrap(),
            r#"{"a":["b","c","d","e"]}"#);
    }

    #[test]
    fn it_parses_explicit_array() {
        eq_str(parse("a[]=b&a[]=c&a[]=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }

    #[test]
    fn it_parses_nested_array() {
        eq_str(parse("a[b][]=c&a[b][]=d").unwrap(),
            r#"{"a":{"b":["c","d"]}}"#);
    }

    #[test]
    fn it_allows_to_specify_array_indexes() {
        eq_str(parse("a[0][]=c&a[1][]=d").unwrap(),
            r#"{"a":[["c"],["d"]]}"#);
    }

    #[test]
    fn it_transforms_arrays_to_object() {
        eq_str(parse("foo[0]=bar&foo[bad]=baz").unwrap(),
            r#"{"foo":{"0":"bar","bad":"baz"}}"#);

        eq_str(parse("foo[0][a]=a&foo[0][b]=b&foo[1][a]=aa&foo[1][b]=bb").unwrap(),
            r#"{"foo":[{"a":"a","b":"b"},{"a":"aa","b":"bb"}]}"#);
    }

    #[test]
    fn it_doesnt_produce_empty_keys() {
        eq_str(parse("_r=1&").unwrap(),
            r#"{"_r":"1"}"#);
    }

    #[test]
    fn it_supports_encoded_strings() {
        eq_str(parse("a[b%20c]=c%20d").unwrap(),
            r#"{"a":{"b c":"c d"}}"#);
    }

    #[test]
    fn it_parses_explicit_encoded_array() {
        eq_str(parse("a%5B%5D=b&a%5B%5D=c&a%5B%5D=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::str::FromStr;

use ipnet::{IpNet, Ipv4Net, Ipv6Net};

pub struct Parser<'a> {
    // parsing as ASCII, so can use byte array
    s: &'a [u8],
    pos: usize,
}

impl<'a> Parser<'a> {
    fn new(s: &'a str) -> Parser<'a> {
        Parser {
            s: s.as_bytes(),
            pos: 0,
        }
    }

    fn is_eof(&self) -> bool {
        self.pos == self.s.len()
    }

    // Commit only if parser returns Some
    fn read_atomically<T, F>(&mut self, cb: F) -> Option<T> where
        F: FnOnce(&mut Parser) -> Option<T>,
    {
        let pos = self.pos;
        let r = cb(self);
        if r.is_none() {
            self.pos = pos;
        }
        r
    }

    // Commit only if parser read till EOF
    fn read_till_eof<T, F>(&mut self, cb: F) -> Option<T> where
        F: FnOnce(&mut Parser) -> Option<T>,
    {
        self.read_atomically(move |p| {
            match cb(p) {
                Some(x) => if p.is_eof() {Some(x)} else {None},
                None => None,
            }
        })
    }

    // Return result of first successful parser
    fn read_or<T>(&mut self, parsers: &mut [Box<FnMut(&mut Parser) -> Option<T> + 'static>])
               -> Option<T> {
        for pf in parsers {
            if let Some(r) = self.read_atomically(|p: &mut Parser| pf(p)) {
                return Some(r);
            }
        }
        None
    }

    // Apply 3 parsers sequentially
    fn read_seq_3<A, B, C, PA, PB, PC>(&mut self,
                                       pa: PA,
                                       pb: PB,
                                       pc: PC)
                                       -> Option<(A, B, C)> where
        PA: FnOnce(&mut Parser) -> Option<A>,
        PB: FnOnce(&mut Parser) -> Option<B>,
        PC: FnOnce(&mut Parser) -> Option<C>,
    {
        self.read_atomically(move |p| {
            let a = pa(p);
            let b = if a.is_some() { pb(p) } else { None };
            let c = if b.is_some() { pc(p) } else { None };
            match (a, b, c) {
                (Some(a), Some(b), Some(c)) => Some((a, b, c)),
                _ => None
            }
        })
    }

    // Read next char
    fn read_char(&mut self) -> Option<char> {
        if self.is_eof() {
            None
        } else {
            let r = self.s[self.pos] as char;
            self.pos += 1;
            Some(r)
        }
    }

    // Return char and advance iff next char is equal to requested
    fn read_given_char(&mut self, c: char) -> Option<char> {
        self.read_atomically(|p| {
            match p.read_char() {
                Some(next) if next == c => Some(next),
                _ => None,
            }
        })
    }

    // Read digit
    fn read_digit(&mut self, radix: u8) -> Option<u8> {
        fn parse_digit(c: char, radix: u8) -> Option<u8> {
            let c = c as u8;
            // assuming radix is either 10 or 16
            if c >= b'0' && c <= b'9' {
                Some(c - b'0')
            } else if radix > 10 && c >= b'a' && c < b'a' + (radix - 10) {
                Some(c - b'a' + 10)
            } else if radix > 10 && c >= b'A' && c < b'A' + (radix - 10) {
                Some(c - b'A' + 10)
            } else {
                None
            }
        }

        self.read_atomically(|p| {
            p.read_char().and_then(|c| parse_digit(c, radix))
        })
    }

    fn read_number_impl(&mut self, radix: u8, max_digits: u32, upto: u32) -> Option<u32> {
        let mut r = 0;
        let mut digit_count = 0;
        loop {
            match self.read_digit(radix) {
                Some(d) => {
                    r = r * (radix as u32) + (d as u32);
                    digit_count += 1;
                    if digit_count > max_digits || r >= upto {
                        return None
                    }
                }
                None => {
                    if digit_count == 0 {
                        return None
                    } else {
                        return Some(r)
                    }
                }
            };
        }
    }

    // Read number, failing if max_digits of number value exceeded
    fn read_number(&mut self, radix: u8, max_digits: u32, upto: u32) -> Option<u32> {
        self.read_atomically(|p| p.read_number_impl(radix, max_digits, upto))
    }

    fn read_ipv4_addr_impl(&mut self) -> Option<Ipv4Addr> {
        let mut bs = [0; 4];
        let mut i = 0;
        while i < 4 {
            if i != 0 && self.read_given_char('.').is_none() {
                return None;
            }

            let octet = self.read_number(10, 3, 0x100).map(|n| n as u8);
            match octet {
                Some(d) => bs[i] = d,
                None => return None,
            };
            i += 1;
        }
        Some(Ipv4Addr::new(bs[0], bs[1], bs[2], bs[3]))
    }

    // Read IPv4 address
    fn read_ipv4_addr(&mut self) -> Option<Ipv4Addr> {
        self.read_atomically(|p| p.read_ipv4_addr_impl())
    }

    fn read_ipv6_addr_impl(&mut self) -> Option<Ipv6Addr> {
        fn ipv6_addr_from_head_tail(head: &[u16], tail: &[u16]) -> Ipv6Addr {
            assert!(head.len() + tail.len() <= 8);
            let mut gs = [0; 8];
            gs[..head.len()].copy_from_slice(head);
            gs[(8 - tail.len()) .. 8].copy_from_slice(tail);
            Ipv6Addr::new(gs[0], gs[1], gs[2], gs[3], gs[4], gs[5], gs[6], gs[7])
        }

        fn read_groups(p: &mut Parser, groups: &mut [u16; 8], limit: usize)
                       -> (usize, bool) {
            let mut i = 0;
            while i < limit {
                if i < limit - 1 {
                    let ipv4 = p.read_atomically(|p| {
                        if i == 0 || p.read_given_char(':').is_some() {
                            p.read_ipv4_addr()
                        } else {
                            None
                        }
                    });
                    if let Some(v4_addr) = ipv4 {
                        let octets = v4_addr.octets();
                        groups[i + 0] = ((octets[0] as u16) << 8) | (octets[1] as u16);
                        groups[i + 1] = ((octets[2] as u16) << 8) | (octets[3] as u16);
                        return (i + 2, true);
                    }
                }

                let group = p.read_atomically(|p| {
                    if i == 0 || p.read_given_char(':').is_some() {
                        p.read_number(16, 4, 0x10000).map(|n| n as u16)
                    } else {
                        None
                    }
                });
                match group {
                    Some(g) => groups[i] = g,
                    None => return (i, false)
                }
                i += 1;
            }
            (i, false)
        }

        let mut head = [0; 8];
        let (head_size, head_ipv4) = read_groups(self, &mut head, 8);

        if head_size == 8 {
            return Some(Ipv6Addr::new(
                head[0], head[1], head[2], head[3],
                head[4], head[5], head[6], head[7]))
        }

        // IPv4 part is not allowed before `::`
        if head_ipv4 {
            return None
        }

        // read `::` if previous code parsed less than 8 groups
        if !self.read_given_char(':').is_some() || !self.read_given_char(':').is_some() {
            return None;
        }

        let mut tail = [0; 8];
        let (tail_size, _) = read_groups(self, &mut tail, 8 - head_size);
        Some(ipv6_addr_from_head_tail(&head[..head_size], &tail[..tail_size]))
    }

    fn read_ipv6_addr(&mut self) -> Option<Ipv6Addr> {
        self.read_atomically(|p| p.read_ipv6_addr_impl())
    }

    fn read_ip_addr(&mut self) -> Option<IpAddr> {
        let ipv4_addr = |p: &mut Parser| p.read_ipv4_addr().map(IpAddr::V4);
        let ipv6_addr = |p: &mut Parser| p.read_ipv6_addr().map(IpAddr::V6);
        self.read_or(&mut [Box::new(ipv4_addr), Box::new(ipv6_addr)])
    }

    /* Additions for IpNet below. */

    // Read IPv4 network
    fn read_ipv4_net(&mut self) -> Option<Ipv4Net> {
        let ip_addr = |p: &mut Parser| p.read_ipv4_addr();
        let slash = |p: &mut Parser| p.read_given_char('/');
        let prefix_len = |p: &mut Parser| {
            p.read_number(10, 2, 33).map(|n| n as u8)
        };

        self.read_seq_3(ip_addr, slash, prefix_len).map(|t| {
            let (ip, _, prefix_len): (Ipv4Addr, char, u8) = t;
            Ipv4Net::new(ip, prefix_len)
        })
    }

    // Read Ipv6 network
    fn read_ipv6_net(&mut self) -> Option<Ipv6Net> {
        let ip_addr = |p: &mut Parser| p.read_ipv6_addr();
        let slash = |p: &mut Parser| p.read_given_char('/');
        let prefix_len = |p: &mut Parser| {
            p.read_number(10, 3, 129).map(|n| n as u8)
        };

        self.read_seq_3(ip_addr, slash, prefix_len).map(|t| {
            let (ip, _, prefix_len): (Ipv6Addr, char, u8) = t;
            Ipv6Net::new(ip, prefix_len)
        })
    }

    fn read_ip_net(&mut self) -> Option<IpNet> {
        let ipv4_net = |p: &mut Parser| p.read_ipv4_net().map(IpNet::V4);
        let ipv6_net = |p: &mut Parser| p.read_ipv6_net().map(IpNet::V6);
        self.read_or(&mut [Box::new(ipv4_net), Box::new(ipv6_net)])
    }

    /* Additions for IpNet above. */
}

/* Additions for IpNet below. */

impl FromStr for IpNet {
    type Err = AddrParseError;
    fn from_str(s: &str) -> Result<IpNet, AddrParseError> {
        match Parser::new(s).read_till_eof(|p| p.read_ip_net()) {
            Some(s) => Ok(s),
            None => Err(AddrParseError(()))
        }
    }
}

impl FromStr for Ipv4Net {
    type Err = AddrParseError;
    fn from_str(s: &str) -> Result<Ipv4Net, AddrParseError> {
        match Parser::new(s).read_till_eof(|p| p.read_ipv4_net()) {
            Some(s) => Ok(s),
            None => Err(AddrParseError(()))
        }
    }
}

impl FromStr for Ipv6Net {
    type Err = AddrParseError;
    fn from_str(s: &str) -> Result<Ipv6Net, AddrParseError> {
        match Parser::new(s).read_till_eof(|p| p.read_ipv6_net()) {
            Some(s) => Ok(s),
            None => Err(AddrParseError(()))
        }
    }
}

/* Additions for IpNet above. */

/// An error which can be returned when parsing an IP address or a socket address.
///
/// This error is used as the error type for the [`FromStr`] implementation for
/// [`IpAddr`], [`Ipv4Addr`], [`Ipv6Addr`], [`SocketAddr`], [`SocketAddrV4`], and
/// [`SocketAddrV6`].
///
/// [`FromStr`]: ../../std/str/trait.FromStr.html
/// [`IpAddr`]: ../../std/net/enum.IpAddr.html
/// [`Ipv4Addr`]: ../../std/net/struct.Ipv4Addr.html
/// [`Ipv6Addr`]: ../../std/net/struct.Ipv6Addr.html
/// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html
/// [`SocketAddrV4`]: ../../std/net/struct.SocketAddrV4.html
/// [`SocketAddrV6`]: ../../std/net/struct.SocketAddrV6.html
//#[stable(feature = "rust1", since = "1.0.0")]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AddrParseError(());

//#[stable(feature = "addr_parse_error_error", since = "1.4.0")]
impl fmt::Display for AddrParseError {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.write_str(self.description())
    }
}

//#[stable(feature = "addr_parse_error_error", since = "1.4.0")]
impl Error for AddrParseError {
    fn description(&self) -> &str {
        "invalid IP address syntax"
    }
}
<commit_msg>Adding documentation.<commit_after>use std::error::Error;
use std::fmt;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::str::FromStr;

use ipnet::{IpNet, Ipv4Net, Ipv6Net};

//! A private parser implementation of IPv4 and IPv6 network addresses.
//!
//! The std::net::parser module cannot be extended because it is private.
//! This module duplicates it and adds methods for parsing the IP network
//! addresses.
//!
//! This module is "publicly exported" through the `FromStr` implementations
//! below.

pub struct Parser<'a> {
    // parsing as ASCII, so can use byte array
    s: &'a [u8],
    pos: usize,
}

impl<'a> Parser<'a> {
    fn new(s: &'a str) -> Parser<'a> {
        Parser {
            s: s.as_bytes(),
            pos: 0,
        }
    }

    fn is_eof(&self) -> bool {
        self.pos == self.s.len()
    }

    // Commit only if parser returns Some
    fn read_atomically<T, F>(&mut self, cb: F) -> Option<T> where
        F: FnOnce(&mut Parser) -> Option<T>,
    {
        let pos = self.pos;
        let r = cb(self);
        if r.is_none() {
            self.pos = pos;
        }
        r
    }

    // Commit only if parser read till EOF
    fn read_till_eof<T, F>(&mut self, cb: F) -> Option<T> where
        F: FnOnce(&mut Parser) -> Option<T>,
    {
        self.read_atomically(move |p| {
            match cb(p) {
                Some(x) => if p.is_eof() {Some(x)} else {None},
                None => None,
            }
        })
    }

    // Return result of first successful parser
    fn read_or<T>(&mut self, parsers: &mut [Box<FnMut(&mut Parser) -> Option<T> + 'static>])
               -> Option<T> {
        for pf in parsers {
            if let Some(r) = self.read_atomically(|p: &mut Parser| pf(p)) {
                return Some(r);
            }
        }
        None
    }

    // Apply 3 parsers sequentially
    fn read_seq_3<A, B, C, PA, PB, PC>(&mut self,
                                       pa: PA,
                                       pb: PB,
                                       pc: PC)
                                       -> Option<(A, B, C)> where
        PA: FnOnce(&mut Parser) -> Option<A>,
        PB: FnOnce(&mut Parser) -> Option<B>,
        PC: FnOnce(&mut Parser) -> Option<C>,
    {
        self.read_atomically(move |p| {
            let a = pa(p);
            let b = if a.is_some() { pb(p) } else { None };
            let c = if b.is_some() { pc(p) } else { None };
            match (a, b, c) {
                (Some(a), Some(b), Some(c)) => Some((a, b, c)),
                _ => None
            }
        })
    }

    // Read next char
    fn read_char(&mut self) -> Option<char> {
        if self.is_eof() {
            None
        } else {
            let r = self.s[self.pos] as char;
            self.pos += 1;
            Some(r)
        }
    }

    // Return char and advance iff next char is equal to requested
    fn read_given_char(&mut self, c: char) -> Option<char> {
        self.read_atomically(|p| {
            match p.read_char() {
                Some(next) if next == c => Some(next),
                _ => None,
            }
        })
    }

    // Read digit
    fn read_digit(&mut self, radix: u8) -> Option<u8> {
        fn parse_digit(c: char, radix: u8) -> Option<u8> {
            let c = c as u8;
            // assuming radix is either 10 or 16
            if c >= b'0' && c <= b'9' {
                Some(c - b'0')
            } else if radix > 10 && c >= b'a' && c < b'a' + (radix - 10) {
                Some(c - b'a' + 10)
            } else if radix > 10 && c >= b'A' && c < b'A' + (radix - 10) {
                Some(c - b'A' + 10)
            } else {
                None
            }
        }

        self.read_atomically(|p| {
            p.read_char().and_then(|c| parse_digit(c, radix))
        })
    }

    fn read_number_impl(&mut self, radix: u8, max_digits: u32, upto: u32) -> Option<u32> {
        let mut r = 0;
        let mut digit_count = 0;
        loop {
            match self.read_digit(radix) {
                Some(d) => {
                    r = r * (radix as u32) + (d as u32);
                    digit_count += 1;
                    if digit_count > max_digits || r >= upto {
                        return None
                    }
                }
                None => {
                    if digit_count == 0 {
                        return None
                    } else {
                        return Some(r)
                    }
                }
            };
        }
    }

    // Read number, failing if max_digits of number value exceeded
    fn read_number(&mut self, radix: u8, max_digits: u32, upto: u32) -> Option<u32> {
        self.read_atomically(|p| p.read_number_impl(radix, max_digits, upto))
    }

    fn read_ipv4_addr_impl(&mut self) -> Option<Ipv4Addr> {
        let mut bs = [0; 4];
        let mut i = 0;
        while i < 4 {
            if i != 0 && self.read_given_char('.').is_none() {
                return None;
            }

            let octet = self.read_number(10, 3, 0x100).map(|n| n as u8);
            match octet {
                Some(d) => bs[i] = d,
                None => return None,
            };
            i += 1;
        }
        Some(Ipv4Addr::new(bs[0], bs[1], bs[2], bs[3]))
    }

    // Read IPv4 address
    fn read_ipv4_addr(&mut self) -> Option<Ipv4Addr> {
        self.read_atomically(|p| p.read_ipv4_addr_impl())
    }

    fn read_ipv6_addr_impl(&mut self) -> Option<Ipv6Addr> {
        fn ipv6_addr_from_head_tail(head: &[u16], tail: &[u16]) -> Ipv6Addr {
            assert!(head.len() + tail.len() <= 8);
            let mut gs = [0; 8];
            gs[..head.len()].copy_from_slice(head);
            gs[(8 - tail.len()) .. 8].copy_from_slice(tail);
            Ipv6Addr::new(gs[0], gs[1], gs[2], gs[3], gs[4], gs[5], gs[6], gs[7])
        }

        fn read_groups(p: &mut Parser, groups: &mut [u16; 8], limit: usize)
                       -> (usize, bool) {
            let mut i = 0;
            while i < limit {
                if i < limit - 1 {
                    let ipv4 = p.read_atomically(|p| {
                        if i == 0 || p.read_given_char(':').is_some() {
                            p.read_ipv4_addr()
                        } else {
                            None
                        }
                    });
                    if let Some(v4_addr) = ipv4 {
                        let octets = v4_addr.octets();
                        groups[i + 0] = ((octets[0] as u16) << 8) | (octets[1] as u16);
                        groups[i + 1] = ((octets[2] as u16) << 8) | (octets[3] as u16);
                        return (i + 2, true);
                    }
                }

                let group = p.read_atomically(|p| {
                    if i == 0 || p.read_given_char(':').is_some() {
                        p.read_number(16, 4, 0x10000).map(|n| n as u16)
                    } else {
                        None
                    }
                });
                match group {
                    Some(g) => groups[i] = g,
                    None => return (i, false)
                }
                i += 1;
            }
            (i, false)
        }

        let mut head = [0; 8];
        let (head_size, head_ipv4) = read_groups(self, &mut head, 8);

        if head_size == 8 {
            return Some(Ipv6Addr::new(
                head[0], head[1], head[2], head[3],
                head[4], head[5], head[6], head[7]))
        }

        // IPv4 part is not allowed before `::`
        if head_ipv4 {
            return None
        }

        // read `::` if previous code parsed less than 8 groups
        if !self.read_given_char(':').is_some() || !self.read_given_char(':').is_some() {
            return None;
        }

        let mut tail = [0; 8];
        let (tail_size, _) = read_groups(self, &mut tail, 8 - head_size);
        Some(ipv6_addr_from_head_tail(&head[..head_size], &tail[..tail_size]))
    }

    fn read_ipv6_addr(&mut self) -> Option<Ipv6Addr> {
        self.read_atomically(|p| p.read_ipv6_addr_impl())
    }

    fn read_ip_addr(&mut self) -> Option<IpAddr> {
        let ipv4_addr = |p: &mut Parser| p.read_ipv4_addr().map(IpAddr::V4);
        let ipv6_addr = |p: &mut Parser| p.read_ipv6_addr().map(IpAddr::V6);
        self.read_or(&mut [Box::new(ipv4_addr), Box::new(ipv6_addr)])
    }

    /* Additions for IpNet below. */

    // Read IPv4 network
    fn read_ipv4_net(&mut self) -> Option<Ipv4Net> {
        let ip_addr = |p: &mut Parser| p.read_ipv4_addr();
        let slash = |p: &mut Parser| p.read_given_char('/');
        let prefix_len = |p: &mut Parser| {
            p.read_number(10, 2, 33).map(|n| n as u8)
        };

        self.read_seq_3(ip_addr, slash, prefix_len).map(|t| {
            let (ip, _, prefix_len): (Ipv4Addr, char, u8) = t;
            Ipv4Net::new(ip, prefix_len)
        })
    }

    // Read Ipv6 network
    fn read_ipv6_net(&mut self) -> Option<Ipv6Net> {
        let ip_addr = |p: &mut Parser| p.read_ipv6_addr();
        let slash = |p: &mut Parser| p.read_given_char('/');
        let prefix_len = |p: &mut Parser| {
            p.read_number(10, 3, 129).map(|n| n as u8)
        };

        self.read_seq_3(ip_addr, slash, prefix_len).map(|t| {
            let (ip, _, prefix_len): (Ipv6Addr, char, u8) = t;
            Ipv6Net::new(ip, prefix_len)
        })
    }

    fn read_ip_net(&mut self) -> Option<IpNet> {
        let ipv4_net = |p: &mut Parser| p.read_ipv4_net().map(IpNet::V4);
        let ipv6_net = |p: &mut Parser| p.read_ipv6_net().map(IpNet::V6);
        self.read_or(&mut [Box::new(ipv4_net), Box::new(ipv6_net)])
    }

    /* Additions for IpNet above. */
}

/* Additions for IpNet below. */

impl FromStr for IpNet {
    type Err = AddrParseError;
    fn from_str(s: &str) -> Result<IpNet, AddrParseError> {
        match Parser::new(s).read_till_eof(|p| p.read_ip_net()) {
            Some(s) => Ok(s),
            None => Err(AddrParseError(()))
        }
    }
}

impl FromStr for Ipv4Net {
    type Err = AddrParseError;
    fn from_str(s: &str) -> Result<Ipv4Net, AddrParseError> {
        match Parser::new(s).read_till_eof(|p| p.read_ipv4_net()) {
            Some(s) => Ok(s),
            None => Err(AddrParseError(()))
        }
    }
}

impl FromStr for Ipv6Net {
    type Err = AddrParseError;
    fn from_str(s: &str) -> Result<Ipv6Net, AddrParseError> {
        match Parser::new(s).read_till_eof(|p| p.read_ipv6_net()) {
            Some(s) => Ok(s),
            None => Err(AddrParseError(()))
        }
    }
}

/* Additions for IpNet above. */

/// An error which can be returned when parsing an IP network address.
///
/// This error is used as the error type for the [`FromStr`] implementation for
/// [`IpNet`], [`Ipv4Net`], and [`Ipv6Net`].
///
/// This is copied from the implementation
///
/// [`FromStr`]: https://doc.rust-lang.org/std/str/trait.FromStr.html
/// [`IpNet`]: enum.IpNet.html
/// [`Ipv4Net`]: struct.Ipv4Net.html
/// [`Ipv6Net`]: struct.Ipv6Net.html
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AddrParseError(());

impl fmt::Display for AddrParseError {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.write_str(self.description())
    }
}

impl Error for AddrParseError {
    fn description(&self) -> &str {
        "invalid IP address syntax"
    }
}
<|endoftext|>


<commit_before>use token::Token;
use tokenizer::{Tokenizer, LexerError};

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

// Private type just for internal functions since the public interface
// cannot return option
type ParseResult<T> = Result<Option<T>, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Statement> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return self.dispatch_statement(line)
        }
    }

    fn line(&mut self) -> ParseResult<Vec<Token>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&mut self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        if statement_tokens[0] == Token::Backslash {
            return self.anonymous_function(statement_tokens);
        }

        // looking for either assignment or a named function
        // both are a collection of symbols followed by equals
        let mut lhs: Vec<Token> = Vec::new();
        let mut rhs: Vec<Token> = Vec::new();
        let mut found = false;
        for token in &statement_tokens {
            if found {
                rhs.push(token.clone());
                continue;
            }

            if let Token::Symbol(_) = *token {
                lhs.push(token.clone());
            }
            else if *token == Token::Equals && !lhs.is_empty() {
                found = true;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function
                break;
            }
        }

        if found {
            match lhs.len() {
                1 => self.assignment(lhs, rhs),
                _ => self.named_function(lhs, rhs),
            }
        }
        else {
            //TODO
            Ok(None)
        }
    }

    fn anonymous_function(&mut self, tokens: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn assignment(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn named_function(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Completed dispatch statement<commit_after>use token::Token;
use tokenizer::{Tokenizer, LexerError};

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

// Private type just for internal functions since the public interface
// cannot return option
type ParseResult<T> = Result<Option<T>, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Statement> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return self.dispatch_statement(line)
        }
    }

    fn line(&mut self) -> ParseResult<Vec<Token>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&mut self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        if statement_tokens[0] == Token::Backslash {
            return self.anonymous_function(statement_tokens);
        }

        // looking for either assignment or a named function
        // both are a collection of symbols followed by equals
        let mut lhs: Vec<Token> = Vec::new();
        let mut rhs: Vec<Token> = Vec::new();
        let mut found = false;
        for token in &statement_tokens {
            if found {
                rhs.push(token.clone());
                continue;
            }

            if let Token::Symbol(_) = *token {
                lhs.push(token.clone());
            }
            else if *token == Token::Equals && !lhs.is_empty() {
                found = true;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function
                break;
            }
        }

        if found {
            match lhs.len() {
                // only one argument: name = ...
                1 => self.assignment(lhs, rhs),
                // more than one argument: name arg1 ... = ...
                _ => self.named_function(lhs, rhs),
            }
        }
        else {
            self.expr(statement_tokens)
        }
    }

    fn anonymous_function(&mut self, tokens: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn assignment(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn named_function(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn expr(&mut self, tokens: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>