<commit_before>#![feature(plugin)]
#![plugin(docopt_macros)]

#![feature(test)]

extern crate jsrs_common;
extern crate jsrs_parser;
extern crate french_press;

extern crate walkdir;
extern crate unescape;
extern crate rustyline;
extern crate rustc_serialize;
extern crate docopt;

extern crate test;

mod eval;
mod native;
mod number;
mod bench;
mod preprocess;
mod var;

use std::cell::RefCell;
use std::fs::{File, metadata};
use std::io::{self, BufReader};
use std::io::prelude::*;
use std::path::Path;
use std::process::exit;
use std::rc::Rc;

use rustyline::error::ReadlineError;
use rustyline::Editor;
use walkdir::WalkDir;

use jsrs_common::types::coerce::AsString;
use french_press::{init_gc, ScopeManager};

use eval::eval_string;
use native::add_pervasives;
use preprocess::{clean_string, add_semicolon};
use jsrs_common::js_error::{self, JsError};

docopt!(Args derive Debug, "
js.rs - a javascript interpreter

Usage:
jsrs
jsrs <file>
jsrs --test
");

fn eval_file(filename: String, debug: bool, should_repl: bool,
             scope_manager: Rc<RefCell<ScopeManager>>) -> js_error::Result<()> {
    if debug {
        println!("Reading from \"{}\"", filename);
    }

    let path = Path::new(&filename);
    let file = File::open(&path)
        .expect(&format!("Cannot open \"{}\": no such file or directory", filename));
    let file_buffer = BufReader::new(file);

    let mut line_builder = String::new();
    let mut braces = Vec::new();

    let mut file_iter = file_buffer.lines();
    let mut negative_test = false;
    loop {
        if let Some(line) = file_iter.next() {
            let input = String::from(line.expect(&format!("Cannot read from {}", filename)));
            let input = clean_string(input);
            if input.contains("@negative") {
                negative_test = true;
            }

            let mut last = '\0';
            // Match braces to see if we should wait for more input
            for c in input.chars() {
                if c == '(' {
                    braces.push('(');
                } else if c == '{' {
                    braces.push('{');
                } else if c == '*' && last == '/' {
                    braces.push('/');
                } else if c == ')' {
                    if braces.pop() != Some('(') {
                        return Err(JsError::ParseError(format!("Unexpected token {}", c)));
                    }
                } else if c == '}' {
                    if braces.pop() != Some('{') {
                        return Err(JsError::ParseError(format!("Unexpected token {}", c)));
                    }
                } else if c == '/' && last == '*' {
                    if braces.pop() != Some('/') {
                        return Err(JsError::ParseError(format!("Unexpected token {}", c)));
                    }
                }
                last = c;
            }

            line_builder.push_str(&input);

            if braces.len() == 0 {
                let js_string = clean_string(line_builder.clone());
                if js_string == "" {
                    continue;
                }

                if debug {
                    println!("\n{}", js_string);
                }

                let ret;
                let eval_result = eval_string(&js_string, scope_manager.clone());
                if let Err(JsError::ParseError(_)) = eval_result {
                    continue;
                }
                if negative_test {
                    match eval_result {
                        Ok((var, ptr)) => ret = (var, ptr),
                        Err(e) => {
                            if !e.is_meta_error() {
                                continue;
                            } else {
                                return Err(e);
                            }
                        }
                    }
                } else {
                    ret = try!(eval_result);
                }

                if debug {
                    println!("=> {:?}", ret);
                }
                line_builder = String::new();
            }
        } else {
            break;
        }
    }
    if should_repl {
        repl(scope_manager);
    }
    Ok(())
}

fn test_dir(dir_name: String) {
    for entry in WalkDir::new(dir_name) {
        let entry = entry.unwrap();
        if !entry.path().is_dir() {
            let entry_path = entry.path().display().to_string();
            let scope_manager = Rc::new(RefCell::new(init_gc()));
            add_pervasives(scope_manager.clone());
            println!("{}", entry_path);
            match eval_file(entry_path.clone(), false, false, scope_manager.clone()) {
                Ok(_) => {
                    println!("{}: {}", entry_path, "OK");
                }
                Err(e) => {
                    println!("{}: {}", entry_path, e);
                }
            }
        }
    }
}

fn repl(scope_manager: Rc<RefCell<ScopeManager>>) -> i32 {
    let mut rl = Editor::new();
    let mut stderr = io::stderr();

    if metadata(".history").is_ok() && rl.load_history(".history").is_err() {
        writeln!(stderr, "Error: unable to load history on startup").unwrap();
    }

    loop {
        // prompt
        let readline = rl.readline(">> ");

        match readline {
            Ok(line) => {
                rl.add_history_entry(&line);
                let input = add_semicolon(clean_string(String::from(line)));
                if input == "" {
                    continue;
                }
                rl.add_history_entry(&input);

                match eval_string(&input, scope_manager.clone()) {
                    Ok((var, ptr)) => println!("{}", ptr.map(|p| p.as_string()).unwrap_or(var.t.as_string())),
                    Err(e) => println!("{:?}", e),
                }
            },
            Err(ReadlineError::Interrupted) => {
                if rl.save_history(".history").is_err() {
                    writeln!(stderr, "Error: unable to save history on exit").unwrap();
                    return 2;
                }
                return 1;
            },
            Err(ReadlineError::Eof) => {
                if rl.save_history(".history").is_err() {
                    writeln!(stderr, "Error: unable to save history on exit").unwrap();
                    return 2;
                }
                break
            },
            Err(err) => {
                println!("Error: {:?}", err);

                if rl.save_history(".history").is_err() {
                    writeln!(stderr, "Error: unable to save history on exit").unwrap();
                    return 2;
                }
                return 3;
            }
        }
    }
    0
}

fn main() {
    let args: Args = Args::docopt().decode().unwrap_or_else(|e| e.exit());
    if args.flag_test {
        let dir_name = "sputnik";

        test_dir(String::from(dir_name))
    } else {
        let scope_manager = Rc::new(RefCell::new(init_gc()));
        add_pervasives(scope_manager.clone());

        if args.arg_file == "" {
            let ret = repl(scope_manager.clone());
            exit(ret)
        } else {
            eval_file(args.arg_file, true, true, scope_manager.clone())
                .expect("Error evaluating file");
        }
    }
}
<commit_msg>Revert "On parse errors, read in more of the file."<commit_after>#![feature(plugin)]
#![plugin(docopt_macros)]

#![feature(test)]

extern crate jsrs_common;
extern crate jsrs_parser;
extern crate french_press;

extern crate walkdir;
extern crate unescape;
extern crate rustyline;
extern crate rustc_serialize;
extern crate docopt;

extern crate test;

mod eval;
mod native;
mod number;
mod bench;
mod preprocess;
mod var;

use std::cell::RefCell;
use std::fs::{File, metadata};
use std::io::{self, BufReader};
use std::io::prelude::*;
use std::path::Path;
use std::process::exit;
use std::rc::Rc;

use rustyline::error::ReadlineError;
use rustyline::Editor;
use walkdir::WalkDir;

use jsrs_common::types::coerce::AsString;
use french_press::{init_gc, ScopeManager};

use eval::eval_string;
use native::add_pervasives;
use preprocess::{clean_string, add_semicolon};
use jsrs_common::js_error::{self, JsError};

docopt!(Args derive Debug, "
js.rs - a javascript interpreter

Usage:
jsrs
jsrs <file>
jsrs --test
");

fn eval_file(filename: String, debug: bool, should_repl: bool,
             scope_manager: Rc<RefCell<ScopeManager>>) -> js_error::Result<()> {
    if debug {
        println!("Reading from \"{}\"", filename);
    }

    let path = Path::new(&filename);
    let file = File::open(&path)
        .expect(&format!("Cannot open \"{}\": no such file or directory", filename));
    let file_buffer = BufReader::new(file);

    let mut line_builder = String::new();
    let mut braces = Vec::new();

    let mut file_iter = file_buffer.lines();
    let mut negative_test = false;
    loop {
        if let Some(line) = file_iter.next() {
            let input = String::from(line.expect(&format!("Cannot read from {}", filename)));
            let input = clean_string(input);
            if input.contains("@negative") {
                negative_test = true;
            }

            let mut last = '\0';
            // Match braces to see if we should wait for more input
            for c in input.chars() {
                if c == '(' {
                    braces.push('(');
                } else if c == '{' {
                    braces.push('{');
                } else if c == '*' && last == '/' {
                    braces.push('/');
                } else if c == ')' {
                    if braces.pop() != Some('(') {
                        return Err(JsError::ParseError(format!("Unexpected token {}", c)));
                    }
                } else if c == '}' {
                    if braces.pop() != Some('{') {
                        return Err(JsError::ParseError(format!("Unexpected token {}", c)));
                    }
                } else if c == '/' && last == '*' {
                    if braces.pop() != Some('/') {
                        return Err(JsError::ParseError(format!("Unexpected token {}", c)));
                    }
                }
                last = c;
            }

            line_builder.push_str(&input);

            if braces.len() == 0 {
                let js_string = clean_string(line_builder.clone());
                line_builder = String::new();
                if js_string == "" {
                    continue;
                }

                if debug {
                    println!("\n{}", js_string);
                }

                let ret;
                let eval_result = eval_string(&js_string, scope_manager.clone());
                if negative_test {
                    match eval_result {
                        Ok((var, ptr)) => ret = (var, ptr),
                        Err(e) => {
                            if !e.is_meta_error() {
                                continue;
                            } else {
                                return Err(e);
                            }
                        }
                    }
                } else {
                    ret = try!(eval_result);
                }

                if debug {
                    println!("=> {:?}", ret);
                }
            }
        } else {
            break;
        }
    }
    if should_repl {
        repl(scope_manager);
    }
    Ok(())
}

fn test_dir(dir_name: String) {
    for entry in WalkDir::new(dir_name) {
        let entry = entry.unwrap();
        if !entry.path().is_dir() {
            let entry_path = entry.path().display().to_string();
            let scope_manager = Rc::new(RefCell::new(init_gc()));
            add_pervasives(scope_manager.clone());
            println!("{}", entry_path);
            match eval_file(entry_path.clone(), false, false, scope_manager.clone()) {
                Ok(_) => {
                    println!("{}: {}", entry_path, "OK");
                }
                Err(e) => {
                    println!("{}: {}", entry_path, e);
                }
            }
        }
    }
}

fn repl(scope_manager: Rc<RefCell<ScopeManager>>) -> i32 {
    let mut rl = Editor::new();
    let mut stderr = io::stderr();

    if metadata(".history").is_ok() && rl.load_history(".history").is_err() {
        writeln!(stderr, "Error: unable to load history on startup").unwrap();
    }

    loop {
        // prompt
        let readline = rl.readline(">> ");

        match readline {
            Ok(line) => {
                rl.add_history_entry(&line);
                let input = add_semicolon(clean_string(String::from(line)));
                if input == "" {
                    continue;
                }
                rl.add_history_entry(&input);

                match eval_string(&input, scope_manager.clone()) {
                    Ok((var, ptr)) => println!("{}", ptr.map(|p| p.as_string()).unwrap_or(var.t.as_string())),
                    Err(e) => println!("{:?}", e),
                }
            },
            Err(ReadlineError::Interrupted) => {
                if rl.save_history(".history").is_err() {
                    writeln!(stderr, "Error: unable to save history on exit").unwrap();
                    return 2;
                }
                return 1;
            },
            Err(ReadlineError::Eof) => {
                if rl.save_history(".history").is_err() {
                    writeln!(stderr, "Error: unable to save history on exit").unwrap();
                    return 2;
                }
                break
            },
            Err(err) => {
                println!("Error: {:?}", err);

                if rl.save_history(".history").is_err() {
                    writeln!(stderr, "Error: unable to save history on exit").unwrap();
                    return 2;
                }
                return 3;
            }
        }
    }
    0
}

fn main() {
    let args: Args = Args::docopt().decode().unwrap_or_else(|e| e.exit());
    if args.flag_test {
        let dir_name = "sputnik";

        test_dir(String::from(dir_name))
    } else {
        let scope_manager = Rc::new(RefCell::new(init_gc()));
        add_pervasives(scope_manager.clone());

        if args.arg_file == "" {
            let ret = repl(scope_manager.clone());
            exit(ret)
        } else {
            eval_file(args.arg_file, true, true, scope_manager.clone())
                .expect("Error evaluating file");
        }
    }
}
<|endoftext|>


<commit_before>use address::{Address, Addressable};
use messages;
use protobuf::Message;
use std::net::{SocketAddr, ToSocketAddrs, UdpSocket, Ipv4Addr};
use time;

pub trait Serialize {
    fn serialize(&self) -> messages::Node;
}

pub trait Deserialize {
    fn deserialize(message: &messages::Node) -> Self;
}

#[derive(Debug)]
pub struct UdpNode {
    address: Address,
    socket_address: SocketAddr,
    last_seen: time::Tm
}

impl UdpNode {
    pub fn new<S: ToSocketAddrs>(address: Address, socket_address: S) -> UdpNode {
        let socket_address = socket_address
            .to_socket_addrs()
            .unwrap()
            .next()
            .unwrap();
        UdpNode {
            address: address,
            socket_address: socket_address,
            last_seen: time::now_utc()
        }
    }
}

pub trait Node {
    fn update(&mut self);
    fn send<M: Message>(&self, message: M);
}

impl Node for UdpNode {
    fn update(&mut self) {
        self.last_seen = time::now_utc()
    }

    fn send<M: Message>(&self, message: M) {
        let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
        let ref buf = message.write_to_bytes().unwrap()[..];
        socket.send_to(buf, self.socket_address).unwrap();
    }
}

impl Deserialize for UdpNode {
    fn deserialize(message: &messages::Node) -> UdpNode {
        let ip = message.get_ip_address();
        let ip = Ipv4Addr::new(ip[0], ip[1], ip[2], ip[3]);
        let port = message.get_port() as u16;
        let address = Address::from_str(message.get_id());
        Self::new(address, (ip, port))
    }
}

impl Serialize for UdpNode {
    fn serialize(&self) -> messages::Node {
        let mut message = messages::Node::new();
        message.set_id(self.address.to_str());
        message.set_ip_address(vec![127, 0, 0, 1]); // TODO: use actual IP address
        message.set_port(self.socket_address.port() as u32);
        message
    }
}

impl Addressable for UdpNode {
    fn get_address(&self) -> Address {
        self.address
    }
}

#[cfg(test)]
mod tests {
    use super::{Node, UdpNode};
    use address::Address;

    #[test]
    fn test_update() {
        let address = Address::for_content("some string");
        let mut node = UdpNode::new(address, ("0.0.0.0", 9000));
        let last_seen_before = node.last_seen;
        node.update();
        let last_seen_after = node.last_seen;
        assert!(last_seen_before < last_seen_after);
    }
}
<commit_msg>Make Node supertrait : Debug + Addressable<commit_after>use address::{Address, Addressable};
use messages;
use protobuf::Message;
use std::net::{SocketAddr, ToSocketAddrs, UdpSocket, Ipv4Addr};
use std::fmt::Debug;
use time;

pub trait Serialize {
    fn serialize(&self) -> messages::Node;
}

pub trait Deserialize {
    fn deserialize(message: &messages::Node) -> Self;
}

pub trait Node : Addressable + Debug + Serialize {
    fn update(&mut self);
    fn send<M: Message>(&self, message: M) where Self : Sized;
}

#[derive(Debug)]
pub struct UdpNode {
    address: Address,
    socket_address: SocketAddr,
    last_seen: time::Tm
}

impl UdpNode {
    pub fn new<S: ToSocketAddrs>(address: Address, socket_address: S) -> UdpNode {
        let socket_address = socket_address
            .to_socket_addrs()
            .unwrap()
            .next()
            .unwrap();
        UdpNode {
            address: address,
            socket_address: socket_address,
            last_seen: time::now_utc()
        }
    }
}

impl Node for UdpNode {
    fn update(&mut self) {
        self.last_seen = time::now_utc()
    }

    fn send<M: Message>(&self, message: M) {
        let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
        let ref buf = message.write_to_bytes().unwrap()[..];
        socket.send_to(buf, self.socket_address).unwrap();
    }
}

impl Deserialize for UdpNode {
    fn deserialize(message: &messages::Node) -> UdpNode {
        let ip = message.get_ip_address();
        let ip = Ipv4Addr::new(ip[0], ip[1], ip[2], ip[3]);
        let port = message.get_port() as u16;
        let address = Address::from_str(message.get_id());
        Self::new(address, (ip, port))
    }
}

impl Serialize for UdpNode {
    fn serialize(&self) -> messages::Node {
        let mut message = messages::Node::new();
        message.set_id(self.address.to_str());
        message.set_ip_address(vec![127, 0, 0, 1]); // TODO: use actual IP address
        message.set_port(self.socket_address.port() as u32);
        message
    }
}

impl Addressable for UdpNode {
    fn get_address(&self) -> Address {
        self.address
    }
}

#[cfg(test)]
mod tests {
    use super::{Node, UdpNode};
    use address::Address;

    #[test]
    fn test_update() {
        let address = Address::for_content("some string");
        let mut node = UdpNode::new(address, ("0.0.0.0", 9000));
        let last_seen_before = node.last_seen;
        node.update();
        let last_seen_after = node.last_seen;
        assert!(last_seen_before < last_seen_after);
    }
}
<|endoftext|>


<commit_before>/// A page, can be a blog post or a basic page
use std::cmp::Ordering;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;
use std::result::Result as StdResult;

use pulldown_cmark as cmark;
use regex::Regex;
use tera::{Tera, Context};
use serde::ser::{SerializeStruct, self};
use slug::slugify;

use errors::{Result, ResultExt};
use config::Config;
use front_matter::{FrontMatter};

lazy_static! {
    static ref PAGE_RE: Regex = Regex::new(r"^\n?\+\+\+\n((?s).*(?-s))\+\+\+\n((?s).*(?-s))$").unwrap();
}

#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct Page {
    /// .md filepath, excluding the content/ bit
    #[serde(skip_serializing)]
    pub filepath: String,
    /// The name of the .md file
    #[serde(skip_serializing)]
    pub filename: String,
    /// The directories above our .md file are called sections
    /// for example a file at content/kb/solutions/blabla.md will have 2 sections:
    /// `kb` and `solutions`
    #[serde(skip_serializing)]
    pub sections: Vec<String>,
    /// The actual content of the page, in markdown
    #[serde(skip_serializing)]
    pub raw_content: String,
    /// The HTML rendered of the page
    pub content: String,
    /// The front matter meta-data
    pub meta: FrontMatter,
    /// The previous page, by date
    pub previous: Option<Box<Page>>,
    /// The next page, by date
    pub next: Option<Box<Page>>,
}

impl Page {
    pub fn new(meta: FrontMatter) -> Page {
        Page {
            filepath: "".to_string(),
            filename: "".to_string(),
            sections: vec![],
            raw_content: "".to_string(),
            content: "".to_string(),
            meta: meta,
            previous: None,
            next: None,
        }
    }

    /// Get the slug for the page.
    /// First tries to find the slug in the meta and defaults to filename otherwise
    pub fn get_slug(&self) -> String {
        if let Some(ref slug) = self.meta.slug {
            slug.to_string()
        } else {
            slugify(self.filename.clone())
        }
    }

    pub fn get_url(&self) -> String {
        if let Some(ref u) = self.meta.url {
            return u.to_string();
        }

        if !self.sections.is_empty() {
            return format!("/{}/{}", self.sections.join("/"), self.get_slug());
        }

        format!("/{}", self.get_slug())
    }

    // Parse a page given the content of the .md file
    // Files without front matter or with invalid front matter are considered
    // erroneous
    pub fn parse(filepath: &str, content: &str) -> Result<Page> {
        // 1. separate front matter from content
        if !PAGE_RE.is_match(content) {
            bail!("Couldn't find front matter in `{}`. Did you forget to add `+++`?", filepath);
        }

        // 2. extract the front matter and the content
        let caps = PAGE_RE.captures(content).unwrap();
        // caps[0] is the full match
        let front_matter = &caps[1];
        let content = &caps[2];

        // 3. create our page, parse front matter and assign all of that
        let meta = FrontMatter::parse(&front_matter)
            .chain_err(|| format!("Error when parsing front matter of file `{}`", filepath))?;

        let mut page = Page::new(meta);
        page.filepath = filepath.to_string();
        page.raw_content = content.to_string();
        page.content = {
            let mut html = String::new();
            let parser = cmark::Parser::new(&page.raw_content);
            cmark::html::push_html(&mut html, parser);
            html
        };

        // 4. Find sections
        // Pages with custom urls exists outside of sections
        if page.meta.url.is_none() {
            let path = Path::new(filepath);
            page.filename = path.file_stem().expect("Couldn't get filename").to_string_lossy().to_string();

            // find out if we have sections
            for section in path.parent().unwrap().components() {
                page.sections.push(section.as_ref().to_string_lossy().to_string());
            }
        }

        Ok(page)
    }

    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Page> {
        let path = path.as_ref();

        let mut content = String::new();
        File::open(path)
            .chain_err(|| format!("Failed to open '{:?}'", path.display()))?
            .read_to_string(&mut content)?;

        // Remove the content string from name
        // Maybe get a path as an arg instead and use strip_prefix?
        Page::parse(&path.strip_prefix("content").unwrap().to_string_lossy(), &content)
    }

    fn get_layout_name(&self) -> String {
        match self.meta.layout {
            Some(ref l) => l.to_string(),
            None => "page.html".to_string()
        }
    }

    pub fn render_html(&self, tera: &Tera, config: &Config) -> Result<String> {
        let tpl = self.get_layout_name();
        let mut context = Context::new();
        context.add("site", config);
        context.add("page", self);

        tera.render(&tpl, &context)
            .chain_err(|| "Error while rendering template")
    }
}

impl ser::Serialize for Page {
    fn serialize<S>(&self, serializer: S) -> StdResult<S::Ok, S::Error> where S: ser::Serializer {
        let mut state = serializer.serialize_struct("page", 10)?;
        state.serialize_field("content", &self.content)?;
        state.serialize_field("title", &self.meta.title)?;
        state.serialize_field("description", &self.meta.description)?;
        state.serialize_field("date", &self.meta.date)?;
        state.serialize_field("slug", &self.get_slug())?;
        state.serialize_field("url", &self.get_url())?;
        state.serialize_field("tags", &self.meta.tags)?;
        state.serialize_field("draft", &self.meta.draft)?;
        state.serialize_field("category", &self.meta.category)?;
        state.serialize_field("extra", &self.meta.extra)?;
        state.end()
    }
}

impl PartialOrd for Page {
    fn partial_cmp(&self, other: &Page) -> Option<Ordering> {
        if self.meta.date.is_none() {
            println!("No self data");
            return Some(Ordering::Less);
        }

        if other.meta.date.is_none() {
            println!("No other date");
            return Some(Ordering::Greater);
        }

        let this_date = self.meta.parse_date().unwrap();
        let other_date = other.meta.parse_date().unwrap();

        if this_date > other_date {
            return Some(Ordering::Less);
        }
        if this_date < other_date {
            return Some(Ordering::Greater);
        }

        Some(Ordering::Equal)
    }
}

#[cfg(test)]
mod tests {
    use super::{Page};

    #[test]
    fn test_can_parse_a_valid_page() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("post.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();

        assert_eq!(page.meta.title, "Hello".to_string());
        assert_eq!(page.meta.slug.unwrap(), "hello-world".to_string());
        assert_eq!(page.raw_content, "Hello world".to_string());
        assert_eq!(page.content, "
Hello world
\n".to_string());
    }

    #[test]
    fn test_can_find_one_parent_directory() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("posts/intro.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.sections, vec!["posts".to_string()]);
    }

    #[test]
    fn test_can_find_multiple_parent_directories() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("posts/intro/start.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.sections, vec!["posts".to_string(), "intro".to_string()]);
    }

    #[test]
    fn test_can_make_url_from_sections_and_slug() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("posts/intro/start.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.get_url(), "/posts/intro/hello-world");
    }

    #[test]
    fn test_can_make_url_from_sections_and_slug_root() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("start.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.get_url(), "/hello-world");
    }

    #[test]
    fn test_errors_on_invalid_front_matter_format() {
        let content = r#"
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("start.md", content);
        assert!(res.is_err());
    }

    #[test]
    fn test_can_make_slug_from_non_slug_filename() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
+++
Hello world"#;
        let res = Page::parse("file with space.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.get_slug(), "file-with-space");
    }
}
<commit_msg>Wordcount + reading time<commit_after>/// A page, can be a blog post or a basic page
use std::cmp::Ordering;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;
use std::result::Result as StdResult;

use pulldown_cmark as cmark;
use regex::Regex;
use tera::{Tera, Context};
use serde::ser::{SerializeStruct, self};
use slug::slugify;

use errors::{Result, ResultExt};
use config::Config;
use front_matter::{FrontMatter};

lazy_static! {
    static ref PAGE_RE: Regex = Regex::new(r"^\n?\+\+\+\n((?s).*(?-s))\+\+\+\n((?s).*(?-s))$").unwrap();
}

#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct Page {
    /// .md filepath, excluding the content/ bit
    #[serde(skip_serializing)]
    pub filepath: String,
    /// The name of the .md file
    #[serde(skip_serializing)]
    pub filename: String,
    /// The directories above our .md file are called sections
    /// for example a file at content/kb/solutions/blabla.md will have 2 sections:
    /// `kb` and `solutions`
    #[serde(skip_serializing)]
    pub sections: Vec<String>,
    /// The actual content of the page, in markdown
    #[serde(skip_serializing)]
    pub raw_content: String,
    /// The HTML rendered of the page
    pub content: String,
    /// The front matter meta-data
    pub meta: FrontMatter,
    /// The previous page, by date
    pub previous: Option<Box<Page>>,
    /// The next page, by date
    pub next: Option<Box<Page>>,
}

impl Page {
    pub fn new(meta: FrontMatter) -> Page {
        Page {
            filepath: "".to_string(),
            filename: "".to_string(),
            sections: vec![],
            raw_content: "".to_string(),
            content: "".to_string(),
            meta: meta,
            previous: None,
            next: None,
        }
    }

    /// Get the slug for the page.
    /// First tries to find the slug in the meta and defaults to filename otherwise
    pub fn get_slug(&self) -> String {
        if let Some(ref slug) = self.meta.slug {
            slug.to_string()
        } else {
            slugify(self.filename.clone())
        }
    }

    /// Get the URL (without the base URL) to that page
    pub fn get_url(&self) -> String {
        if let Some(ref u) = self.meta.url {
            return u.to_string();
        }

        if !self.sections.is_empty() {
            return format!("/{}/{}", self.sections.join("/"), self.get_slug());
        }

        format!("/{}", self.get_slug())
    }

    // Get word count and estimated reading time
    pub fn get_reading_analytics(&self) -> (usize, usize) {
        // Only works for latin language but good enough for a start
        let word_count: usize = self.raw_content.split_whitespace().count();

        // https://help.medium.com/hc/en-us/articles/214991667-Read-time
        // 275 seems a bit too high though
        (word_count, (word_count / 200))
    }

    // Parse a page given the content of the .md file
    // Files without front matter or with invalid front matter are considered
    // erroneous
    pub fn parse(filepath: &str, content: &str) -> Result<Page> {
        // 1. separate front matter from content
        if !PAGE_RE.is_match(content) {
            bail!("Couldn't find front matter in `{}`. Did you forget to add `+++`?", filepath);
        }

        // 2. extract the front matter and the content
        let caps = PAGE_RE.captures(content).unwrap();
        // caps[0] is the full match
        let front_matter = &caps[1];
        let content = &caps[2];

        // 3. create our page, parse front matter and assign all of that
        let meta = FrontMatter::parse(&front_matter)
            .chain_err(|| format!("Error when parsing front matter of file `{}`", filepath))?;

        let mut page = Page::new(meta);
        page.filepath = filepath.to_string();
        page.raw_content = content.to_string();
        page.content = {
            let mut html = String::new();
            let parser = cmark::Parser::new(&page.raw_content);
            cmark::html::push_html(&mut html, parser);
            html
        };

        // 4. Find sections
        // Pages with custom urls exists outside of sections
        if page.meta.url.is_none() {
            let path = Path::new(filepath);
            page.filename = path.file_stem().expect("Couldn't get filename").to_string_lossy().to_string();

            // find out if we have sections
            for section in path.parent().unwrap().components() {
                page.sections.push(section.as_ref().to_string_lossy().to_string());
            }
        }

        Ok(page)
    }

    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Page> {
        let path = path.as_ref();

        let mut content = String::new();
        File::open(path)
            .chain_err(|| format!("Failed to open '{:?}'", path.display()))?
            .read_to_string(&mut content)?;

        // Remove the content string from name
        // Maybe get a path as an arg instead and use strip_prefix?
        Page::parse(&path.strip_prefix("content").unwrap().to_string_lossy(), &content)
    }

    fn get_layout_name(&self) -> String {
        match self.meta.layout {
            Some(ref l) => l.to_string(),
            None => "page.html".to_string()
        }
    }

    /// Renders the page using the default layout, unless specified in front-matter
    pub fn render_html(&self, tera: &Tera, config: &Config) -> Result<String> {
        let tpl = self.get_layout_name();
        let mut context = Context::new();
        context.add("site", config);
        context.add("page", self);

        tera.render(&tpl, &context)
            .chain_err(|| "Error while rendering template")
    }
}

impl ser::Serialize for Page {
    fn serialize<S>(&self, serializer: S) -> StdResult<S::Ok, S::Error> where S: ser::Serializer {
        let mut state = serializer.serialize_struct("page", 12)?;
        state.serialize_field("content", &self.content)?;
        state.serialize_field("title", &self.meta.title)?;
        state.serialize_field("description", &self.meta.description)?;
        state.serialize_field("date", &self.meta.date)?;
        state.serialize_field("slug", &self.get_slug())?;
        state.serialize_field("url", &self.get_url())?;
        state.serialize_field("tags", &self.meta.tags)?;
        state.serialize_field("draft", &self.meta.draft)?;
        state.serialize_field("category", &self.meta.category)?;
        state.serialize_field("extra", &self.meta.extra)?;
        let (word_count, reading_time) = self.get_reading_analytics();
        state.serialize_field("word_count", &word_count)?;
        state.serialize_field("reading_time", &reading_time)?;
        state.end()
    }
}

impl PartialOrd for Page {
    fn partial_cmp(&self, other: &Page) -> Option<Ordering> {
        if self.meta.date.is_none() {
            println!("No self data");
            return Some(Ordering::Less);
        }

        if other.meta.date.is_none() {
            println!("No other date");
            return Some(Ordering::Greater);
        }

        let this_date = self.meta.parse_date().unwrap();
        let other_date = other.meta.parse_date().unwrap();

        if this_date > other_date {
            return Some(Ordering::Less);
        }
        if this_date < other_date {
            return Some(Ordering::Greater);
        }

        Some(Ordering::Equal)
    }
}

#[cfg(test)]
mod tests {
    use super::{Page};

    #[test]
    fn test_can_parse_a_valid_page() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("post.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();

        assert_eq!(page.meta.title, "Hello".to_string());
        assert_eq!(page.meta.slug.unwrap(), "hello-world".to_string());
        assert_eq!(page.raw_content, "Hello world".to_string());
        assert_eq!(page.content, "
Hello world
\n".to_string());
    }

    #[test]
    fn test_can_find_one_parent_directory() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("posts/intro.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.sections, vec!["posts".to_string()]);
    }

    #[test]
    fn test_can_find_multiple_parent_directories() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("posts/intro/start.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.sections, vec!["posts".to_string(), "intro".to_string()]);
    }

    #[test]
    fn test_can_make_url_from_sections_and_slug() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("posts/intro/start.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.get_url(), "/posts/intro/hello-world");
    }

    #[test]
    fn test_can_make_url_from_sections_and_slug_root() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("start.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.get_url(), "/hello-world");
    }

    #[test]
    fn test_errors_on_invalid_front_matter_format() {
        let content = r#"
title = "Hello"
description = "hey there"
slug = "hello-world"
+++
Hello world"#;
        let res = Page::parse("start.md", content);
        assert!(res.is_err());
    }

    #[test]
    fn test_can_make_slug_from_non_slug_filename() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
+++
Hello world"#;
        let res = Page::parse("file with space.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        assert_eq!(page.get_slug(), "file-with-space");
    }

    #[test]
    fn test_reading_analytics_short() {
        let content = r#"
+++
title = "Hello"
description = "hey there"
+++
Hello world"#;
        let res = Page::parse("file with space.md", content);
        assert!(res.is_ok());
        let page = res.unwrap();
        let (word_count, reading_time) = page.get_reading_analytics();
        assert_eq!(word_count, 2);
        assert_eq!(reading_time, 0);
    }

    #[test]
    fn test_reading_analytics_long() {
        let mut content = r#"
+++
title = "Hello"
description = "hey there"
+++
Hello world"#.to_string();
        for _ in 0..1000 {
            content.push_str(" Hello world");
        }
        let res = Page::parse("hello.md", &content);
        assert!(res.is_ok());
        let page = res.unwrap();
        let (word_count, reading_time) = page.get_reading_analytics();
        assert_eq!(word_count, 2002);
        assert_eq!(reading_time, 10);
    }
}
<|endoftext|>


<commit_before>use combine;
use rustyline::Editor;
use rustyline::error::ReadlineError as RLError;
use {lexer, parser, ast, ops};
use eval::Env;
use error::*;
use token::Token;

pub fn repl() -> Result<()> {
    // Prompt constants
    let header = format!(r"telescope v{}", env!("CARGO_PKG_VERSION"));
    let prompt = "> ";

    let mut rl = Editor::<()>::new();

    println!("{}", header);

    let mut env = ops::env();
    let mut token_buf = Vec::<Token>::with_capacity(1024);

    loop {
        match rl.readline(prompt) {
            Ok(line) => {
                rl.add_history_entry(&line);
                exec(&line, &mut token_buf, &mut env)?;
            }
            Err(RLError::Interrupted) |
            Err(RLError::Eof) => bail!(ErrorKind::Eof),
            Err(err) => {
                println!("Error: {:?}", err);
                bail!(ErrorKind::Eof);
            }
        };
    }
}

fn exec(line: &str, token_buf: &mut Vec<Token>, mut env: &mut Env) -> Result<()> {
    let (mut tokens, _unlexed) = lexer::lex(line.trim_right()).unwrap();
    let mut all_tokens = token_buf.drain(..).collect::<Vec<_>>();
    all_tokens.append(&mut tokens);
    match parser::parse(combine::State::new(&*all_tokens)) {
        Ok((expr, unparsed)) => {
            token_buf.extend_from_slice(unparsed.input);
            match eval(&expr, &mut env) {
                Ok(result) => Ok(print(result)),
                Err(err) => match err.kind() {
                    &ErrorKind::Exit(_) => Err(err),
                    _ => Ok(println!("Error: {}", err)),
                }
            }
        },
        Err(err) => Ok(println!("Parsing error: {:?}", err)),
    }
}

fn read(line: &str) -> ::std::result::Result<(ast::Expr, &[Token]), combine::ParseError<&[Token]>> {
    unimplemented!()
}

fn eval(expr: &ast::Expr, mut env: &mut Env) -> Result<ast::Expr> {
    expr.eval(&mut env)
}

fn print(value: ast::Expr) {
    if value != ast::Expr::Nil {
        println!("{}", value);
    }
}
<commit_msg>Non-alien error messages<commit_after>use combine;
use rustyline::Editor;
use rustyline::error::ReadlineError as RLError;
use {lexer, parser, ast, ops};
use eval::Env;
use error::*;
use token::Token;

pub fn repl() -> Result<()> {
    // Prompt constants
    let header = format!(r"telescope v{}", env!("CARGO_PKG_VERSION"));
    let prompt = "> ";

    let mut rl = Editor::<()>::new();

    println!("{}", header);

    let mut env = ops::env();
    let mut token_buf = Vec::<Token>::with_capacity(1024);

    loop {
        match rl.readline(prompt) {
            Ok(line) => {
                rl.add_history_entry(&line);
                exec(&line, &mut token_buf, &mut env)?;
            }
            Err(RLError::Interrupted) |
            Err(RLError::Eof) => bail!(ErrorKind::Eof),
            Err(err) => {
                println!("Error: {:?}", err);
                bail!(ErrorKind::Eof);
            }
        };
    }
}

fn exec(line: &str, token_buf: &mut Vec<Token>, mut env: &mut Env) -> Result<()> {
    let (tokens, _unlexed) = lexer::lex(line.trim_right()).unwrap();
    let all_tokens = token_buf.drain(..).chain(tokens).collect::<Vec<_>>();
    let token_iter = combine::from_iter(all_tokens.into_iter());
    match parser::parse(combine::State::new(token_iter)) {
        Ok((expr, unparsed)) => {
            token_buf.extend(unparsed.input);
            match eval(&expr, &mut env) {
                Ok(result) => Ok(print(result)),
                Err(err) => match err.kind() {
                    &ErrorKind::Exit(_) => Err(err),
                    _ => Ok(println!("Error: {}", err)),
                }
            }
        },
        Err(err) => Ok(println!("Parsing error: {}", err)),
    }
}

fn read(line: &str) -> ::std::result::Result<(ast::Expr, &[Token]), combine::ParseError<&[Token]>> {
    unimplemented!()
}

fn eval(expr: &ast::Expr, mut env: &mut Env) -> Result<ast::Expr> {
    expr.eval(&mut env)
}

fn print(value: ast::Expr) {
    if value != ast::Expr::Nil {
        println!("{}", value);
    }
}
<|endoftext|>


<commit_before>use std::bool;
use std::ascii;

pub struct Stemmer {
    priv word: ~[ascii::Ascii],
    priv j: uint,
}

impl Stemmer {
    pub fn new(word: &str) -> Result<Stemmer, ~str> {
        if bool::not(word.is_ascii()) {
            Err(~"Only support English words with ASCII characters")
        } else {
            Ok(Stemmer {
                word: unsafe { word.to_ascii_nocheck().to_lower() },
                j: 0,
            })
        }
    }

    /// stem.is_consonant(i) is true <=> stem[i] is a consonant
    pub fn is_consonant(&self, i: uint) -> bool {
        match self.word[i].to_char() {
            'a' | 'e' | 'i' | 'o' | 'u' => false,
            'y' => if i == 0 {
                true
            } else {
                !self.is_consonant(i - 1)
            },
            _ => true,
        }
    }

    pub fn get(&self) -> ~str {
        self.word.to_str_ascii()
    }
}

pub fn get(word: &str) -> Result<~str, ~str> {
    if word.len() > 2 {
        match Stemmer::new(word) {
            Ok(w) => Ok(w.get()),
            Err(e) => Err(e),
        }
    } else {
        Ok(word.to_owned())
    }
}
<commit_msg>working code<commit_after>use std::bool;
use std::ascii;

/// Member b is a vector of std::ascii::Ascii holding a word to be stemmed.
/// The letters are in b[0], b[1] ... ending at b[z->k]. Member k is readjusted
/// downwards as the stemming progresses. Zero termination is not in fact used
/// in the algorithm.
///
/// Note that only lower case sequences are stemmed. Forcing to lower case
/// should be done before get(...) is called.
///
///
/// Typical usage is:
///
///     let b = ~"pencils";
///     let res = stem::get(b);
///         /- stem the 7 characters of b[0] to b[6]. The result, res,
///            will be "pencil" (the 's' is removed). -/
///
pub struct Stemmer {
    priv b: ~[ascii::Ascii],
    priv k: uint,
    priv j: uint,
}

impl Stemmer {
    pub fn new(word: &str) -> Result<Stemmer, ~str> {
        if bool::not(word.is_ascii()) {
            Err(~"Only support English words with ASCII characters")
        } else {
            let b = unsafe { word.to_ascii_nocheck().to_lower() };
            let k = b.len();
            Ok(Stemmer {
                b: b,
                k: k,
                j: 0,
            })
        }
    }

    /// stem.is_consonant(i) is true <=> stem[i] is a consonant
    pub fn is_consonant(&self, i: uint) -> bool {
        match self.b[i].to_char() {
            'a' | 'e' | 'i' | 'o' | 'u' => false,
            'y' => if i == 0 {
                true
            } else {
                bool::not(self.is_consonant(i - 1))
            },
            _ => true,
        }
    }

    /// stem.measure() measures the number of consonant sequences in [0, j).
    /// if c is a consonant sequence and v a vowel sequence, and <..> indicates
    /// arbitrary presence,
    ///
    ///    <c><v>       gives 0
    ///    <c>vc<v>     gives 1
    ///    <c>vcvc<v>   gives 2
    ///    <c>vcvcvc<v> gives 3
    ///    ....
    ///
    pub fn measure(&self) -> uint {
        let mut n = 0u;
        let mut i = 0u;
        let j = self.j;
        loop {
            if i >= j { return n }
            if bool::not(self.is_consonant(i)) { break }
            i += 1;
        }
        i += 1;
        loop {
            loop {
                if i >= j { return n }
                if self.is_consonant(i) { break }
                i += 1;
            }
            i += 1;
            n += 1;
            loop {
                if i >= j { return n }
                if bool::not(self.is_consonant(i)) { break }
                i += 1;
            }
            i += 1;
        }
    }

    /// stem.has_vowel() is TRUE <=> [0, j-1) contains a vowel
    pub fn has_vowel(&self) -> bool {
        for i in range(0, self.j) {
            if bool::not(self.is_consonant(i)) {
                return true;
            }
        }
        return false;
    }

    /// stem.double_consonant(i) is TRUE <=> i,(i-1) contain a double consonant.
    pub fn double_consonant(&self, i: uint) -> bool {
        if i < 1 {
            false
        } else if self.b[i] != self.b[i - 1] {
            false
        } else {
            self.is_consonant(i)
        }
    }

    /// cvc(z, i) is TRUE <=> i-2,i-1,i has the form consonant - vowel - consonant
    /// and also if the second c is not w,x or y. this is used when trying to
    /// restore an e at the end of a short word. e.g.
    ///
    ///    cav(e), lov(e), hop(e), crim(e), but
    ///    snow, box, tray.
    ///
    pub fn cvc(&self, i: uint) -> bool {
        if i < 2 || !self.is_consonant(i) || self.is_consonant(i - 1)
            || !self.is_consonant(i - 2) { return false }
        match self.b[i].to_char() {
            'w' | 'x' | 'y' => false,
            _ => true,
        }
    }

    /// stem.ends(s) is true <=> [0, k) ends with the string s.
    pub fn ends(&mut self, s: &str) -> bool {
        let len = s.len();
        let k = self.k;
        if s[len - 1] != self.b[k-1].to_byte() { return false } /* tiny speed-up */
        if len > k { return false }
        let mut iter = s.byte_iter();
        for ac in self.b.slice(k - len, k).iter() {
            if ac.to_byte() != iter.next().unwrap() { return false }
        }
        self.j = k - len;
        return true;
    }

    /// stem.setto(s) sets [j,k) to the characters in the string s,
    /// readjusting k.
    fn set_to(&mut self, s: &str) {
        let length = s.len();
        let j = self.j;
        for i in range(0, length) {
            self.b[j + i] = s[i].to_ascii();
        }
        self.k = j + length;
    }

    /// self.replace(s) is used further down.
    fn r(&mut self, s: &str) {
        if self.measure() > 0 {
            self.set_to(s);
        }
    }

    /// stem.step1ab() gets rid of plurals and -ed or -ing. e.g.
    ///
    ///     caresses  ->  caress
    ///     ponies    ->  poni
    ///     ties      ->  ti
    ///     caress    ->  caress
    ///     cats      ->  cat
    ///
    ///     feed      ->  feed
    ///     agreed    ->  agree
    ///     disabled  ->  disable
    ///
    ///     matting   ->  mat
    ///     mating    ->  mate
    ///     meeting   ->  meet
    ///     milling   ->  mill
    ///     messing   ->  mess
    ///
    ///     meetings  ->  meet
    ///
    pub fn step1ab(&mut self) {
        if self.b[self.k - 1].to_char() == 's' {
            if self.ends("sses") {
                self.k -= 2;
            } else if self.ends("ies") {
                self.set_to("i");
            } else if self.b[self.k - 2].to_char() != 's' {
                self.k -= 1;
            }
        }
        if self.ends("eed") {
            if self.measure() > 0 { self.k -= 1 }
        } else if (self.ends("ed") || self.ends("ing")) && self.has_vowel() {
            self.k = self.j;
            if self.ends("at") {
                self.set_to("ate");
            } else if self.ends("bl") {
                self.set_to("ble");
            } else if self.ends("iz") {
                self.set_to("ize");
            } else if self.double_consonant(self.k - 1) {
                self.k -= 1;
                match self.b[self.k - 1].to_char() {
                    'l' | 's' | 'z' => self.k += 1,
                    _ => (),
                }
            } else if self.measure() == 1 && self.cvc(self.k - 1) {
                self.set_to("e");
            }
        }
    }

    /// stem.step1c() turns terminal y to i when there is another vowel in the stem.
    pub fn step1c(&mut self) {
       if self.ends("y") && self.has_vowel() {
           self.b[self.k-1] = 'i'.to_ascii();
        }
    }

    /// stem.step2() maps double suffices to single ones. so -ization ( = -ize
    /// plus -ation) maps to -ize etc. note that the string before the suffix
    /// must give m(z) > 0.
    pub fn step2(&mut self) {
        match self.b[self.k-2].to_char() {
            'a' => {
                if self.ends("ational") { self.r("ate"); return }
                if self.ends("tional") { self.r("tion"); return }
            },
            'c' => {
                if self.ends("enci") { self.r("ence"); return }
                if self.ends("anci") { self.r("ance"); return }
            },
            'e' => if self.ends("izer") { self.r("ize"); return },
            'l' => {
                if self.ends("bli") { self.r("ble"); return } /*-DEPARTURE-*/

             /* To match the published algorithm, replace this line with
                'l' => {
                    if self.ends("abli") { self.r("able"); return } */

                 if self.ends("alli") { self.r("al"); return }
                 if self.ends("entli") { self.r("ent"); return }
                 if self.ends("eli") { self.r("e"); return }
                 if self.ends("ousli") { self.r("ous"); return }
            },
            'o' => {
                if self.ends("ization") { self.r("ize"); return }
                if self.ends("ation") { self.r("ate"); return }
                if self.ends("ator") { self.r("ate"); return }
            },
            's' => {
                if self.ends("alism") { self.r("al"); return }
                if self.ends("iveness") { self.r("ive"); return }
                if self.ends("fulness") { self.r("ful"); return }
                if self.ends("ousness") { self.r("ous"); return }
            },
            't' => {
                if self.ends("aliti") { self.r("al"); return }
                if self.ends("iviti") { self.r("ive"); return }
                if self.ends("biliti") { self.r("ble"); return }
            },
            'g' => if self.ends("logi") { self.r("log"); return }, /*-DEPARTURE-*/
             /* To match the published algorithm, delete this line */
            _ => (),
        }
    }

    /// stem.step3() deals with -ic-, -full, -ness etc. similar strategy to step2.
    pub fn step3(&mut self) {
        match self.b[self.k-1].to_char() {
            'e' => {
                if self.ends("icate") { self.r("ic"); return }
                if self.ends("ative") { self.r(""); return }
                if self.ends("alize") { self.r("al"); return }
            },
            'i' => if self.ends("iciti") { self.r("ic"); return },
            'l' => {
                if self.ends("ical") { self.r("ic"); return }
                if self.ends("ful") { self.r(""); return }
            },
            's' => if self.ends("ness") { self.r(""); return },
            _ => (),
        }
    }

    /// stem.step4() takes off -ant, -ence etc., in context <c>vcvc<v>.
    pub fn step4(&mut self) {
        match self.b[self.k-2].to_char() {
            'a' => {
                if self.ends("al") {}
                else { return }
            }
            'c' => {
                if self.ends("ance") {}
                else if self.ends("ence") {}
                else { return }
            },
            'e' => {
                if self.ends("er") {}
                else { return }
            }
            'i' => {
                if self.ends("ic") {}
                else { return }
            },
            'l' => {
                if self.ends("able") {}
                else if self.ends("ible") {}
                else { return }
            },
            'n' => {
                if self.ends("ant") {}
                else if self.ends("ement") {}
                else if self.ends("ment") {}
                else if self.ends("ent") {}
                else { return }
            },
            'o' => {
                if self.ends("ion")
                    && (self.b[self.j-1].to_char() == 's' || self.b[self.j-1].to_char() == 't') {}
                else if self.ends("ou") {}
                else { return }
                /* takes care of -ous */
            },
            's' => {
                if self.ends("ism") {}
                else { return }
            },
            't' => {
                if self.ends("ate") {}
                else if self.ends("iti") {}
                else { return }
            },
            'u' => {
                if self.ends("ous") {}
                else { return }
            },
            'v' => {
                if self.ends("ive") {}
                else { return }
            },
            'z' => {
                if self.ends("ize") {}
                else { return }
            },
            _ => return,
        }
        if self.measure() > 1 { self.k = self.j }
    }

    /// stem.step5() removes a final -e if self.measure() > 1, and changes -ll
    /// to -l if self.measure() > 1.
    pub fn step5(&mut self) {
       self.j = self.k;
       if self.b[self.k - 1].to_char() == 'e' {
           let a = self.measure();
           if a > 1 || a == 1 && bool::not(self.cvc(self.k - 2)) { self.k -= 1 }
       }
       if self.b[self.k-1].to_char() == 'l'
           && self.double_consonant(self.k-1) && self.measure() > 1 {
           self.k-=1;
       }
    }

    pub fn get(&self) -> ~str {
        self.b.slice_to(self.k).to_str_ascii()
    }
}

pub fn get(word: &str) -> Result<~str, ~str> {
    if word.len() > 2 {
        match Stemmer::new(word) {
            Ok(w) => {
                let mut mw = w;
                mw.step1ab();
                mw.step1c();
                mw.step2();
                mw.step3();
                mw.step4();
                mw.step5();
                Ok(mw.get())
            }
            Err(e) => Err(e),
        }
    } else {
        Ok(word.to_owned())
    }
}
<|endoftext|>


<commit_before>#[cfg(test)] extern crate test;

use std::borrow::ToOwned;

/// Member b is a vector of bytes holding a word to be stemmed.
/// The letters are in b[0], b[1] ... ending at b[z->k]. Member k is readjusted
/// downwards as the stemming progresses. Zero termination is not in fact used
/// in the algorithm.
///
/// Note that only lower case sequences are stemmed. Forcing to lower case
/// should be done before get(...) is called.
///
///
/// Typical usage is:
///
///     let b = "pencils";
///     let res = stem::get(b);
///     assert_eq!(res, Ok("pencil".to_string()));
///
struct Stemmer {
    b: Vec<u8>,
    k: uint,
    j: uint
}

impl Stemmer {
    fn new(word: &str) -> Result<Stemmer, &str> {
        if !word.is_ascii() {
            Err("Only support English words with ASCII characters")
        } else {
            let b: Vec<u8> = unsafe {
                word.to_ascii_nocheck()
                    .iter()
                    .map(|c| c.to_lowercase().as_byte())
                    .collect()
            };
            let k = b.len();
            Ok(Stemmer {
                b: b,
                k: k,
                j: 0
            })
        }
    }

    /// stem.is_consonant(i) is true <=> stem[i] is a consonant
    #[inline]
    fn is_consonant(&self, i: uint) -> bool {
        match self.b[i] {
            b'a' | b'e' | b'i' | b'o' | b'u' => false,
            b'y' => if i == 0 {
                true
            } else {
                !self.is_consonant(i - 1)
            },
            _ => true
        }
    }

    /// stem.measure() measures the number of consonant sequences in [0, j).
    /// if c is a consonant sequence and v a vowel sequence, and <..> indicates
    /// arbitrary presence,
    ///
    /// ~~~notrust
    ///    <c><v>       gives 0
    ///    <c>vc<v>     gives 1
    ///    <c>vcvc<v>   gives 2
    ///    <c>vcvcvc<v> gives 3
    ///    ....
    /// ~~~
    fn measure(&self) -> uint {
        let mut n = 0u;
        let mut i = 0u;
        let j = self.j;
        loop {
            if i >= j { return n }
            if !self.is_consonant(i) { break }
            i += 1;
        }
        i += 1;
        loop {
            loop {
                if i >= j { return n }
                if self.is_consonant(i) { break }
                i += 1;
            }
            i += 1;
            n += 1;
            loop {
                if i >= j { return n }
                if !self.is_consonant(i) { break }
                i += 1;
            }
            i += 1;
        }
    }

    /// stem.has_vowel() is TRUE <=> [0, j-1) contains a vowel
    fn has_vowel(&self) -> bool {
        for i in range(0, self.j) {
            if !self.is_consonant(i) {
                return true;
            }
        }
        false
    }

    /// stem.double_consonant(i) is TRUE <=> i,(i-1) contain a double consonant.
    #[inline]
    fn double_consonant(&self, i: uint) -> bool {
        if i < 1 {
            false
        } else if self.b[i] != self.b[i - 1] {
            false
        } else {
            self.is_consonant(i)
        }
    }

    /// cvc(z, i) is TRUE <=> i-2,i-1,i has the form consonant - vowel - consonant
    /// and also if the second c is not w,x or y. this is used when trying to
    /// restore an e at the end of a short word. e.g.
    ///
    /// ~~~notrust
    ///    cav(e), lov(e), hop(e), crim(e), but
    ///    snow, box, tray.
    /// ~~~
    fn cvc(&self, i: uint) -> bool {
        if i < 2
            || !self.is_consonant(i)
            || self.is_consonant(i - 1)
            || !self.is_consonant(i - 2)
        {
            false
        } else {
            match self.b[i] {
                b'w' | b'x' | b'y' => false,
                _ => true
            }
        }
    }

    /// stem.ends(s) is true <=> [0, k) ends with the string s.
    fn ends(&mut self, _s: &str) -> bool {
        let s = _s.as_bytes();
        let len = s.len();
        if len > self.k {
            false
        } else {
            if self.b.slice(self.k - len, self.k) == s {
                self.j = self.k - len;
                true
            } else {
                false
            }
        }
    }

    /// stem.setto(s) sets [j,k) to the characters in the string s,
    /// readjusting k.
    fn set_to(&mut self, s: &str) {
        let s = s.as_bytes();
        let len = s.len();
        for i in range(0, len) {
            self.b.as_mut_slice()[self.j + i] = s[i];
        }
        self.k = self.j + len;
    }

    /// self.replace(s) is used further down.
    #[inline]
    fn r(&mut self, s: &str) {
        if self.measure() > 0 {
            self.set_to(s);
        }
    }

    /// stem.step1ab() gets rid of plurals and -ed or -ing. e.g.
    ///
    /// ~~~~notrust
    ///     caresses  ->  caress
    ///     ponies    ->  poni
    ///     ties      ->  ti
    ///     caress    ->  caress
    ///     cats      ->  cat
    ///
    ///     feed      ->  feed
    ///     agreed    ->  agree
    ///     disabled  ->  disable
    ///
    ///     matting   ->  mat
    ///     mating    ->  mate
    ///     meeting   ->  meet
    ///     milling   ->  mill
    ///     messing   ->  mess
    ///
    ///     meetings  ->  meet
    /// ~~~~
    fn step1ab(&mut self) {
        if self.b[self.k - 1] == b's' {
            if self.ends("sses") {
                self.k -= 2;
            } else if self.ends("ies") {
                self.set_to("i");
            } else if self.b[self.k - 2] != b's' {
                self.k -= 1;
            }
        }
        if self.ends("eed") {
            if self.measure() > 0 { self.k -= 1 }
        } else if (self.ends("ed") || self.ends("ing")) && self.has_vowel() {
            self.k = self.j;
            if self.ends("at") {
                self.set_to("ate");
            } else if self.ends("bl") {
                self.set_to("ble");
            } else if self.ends("iz") {
                self.set_to("ize");
            } else if self.double_consonant(self.k - 1) {
                self.k -= 1;
                match self.b[self.k - 1] {
                    b'l' | b's' | b'z' => self.k += 1,
                    _ => ()
                }
            } else if self.measure() == 1 && self.cvc(self.k - 1) {
                self.set_to("e");
            }
        }
    }

    /// stem.step1c() turns terminal y to i when there is another vowel in the stem.
    fn step1c(&mut self) {
        if self.ends("y") && self.has_vowel() {
           self.b.as_mut_slice()[self.k - 1] = b'i';
        }
    }

    /// stem.step2() maps double suffices to single ones. so -ization ( = -ize
    /// plus -ation) maps to -ize etc. note that the string before the suffix
    /// must give m(z) > 0.
    fn step2(&mut self) {
        match self.b[self.k - 2] {
            b'a' => {
                if self.ends("ational") { self.r("ate"); return }
                if self.ends("tional") { self.r("tion"); return }
            }
            b'c' => {
                if self.ends("enci") { self.r("ence"); return }
                if self.ends("anci") { self.r("ance"); return }
            }
            b'e' => if self.ends("izer") { self.r("ize"); return },
            b'l' => {
                if self.ends("bli") { self.r("ble"); return } /*-DEPARTURE-*/

             /* To match the published algorithm, replace this line with
                'l' => {
                    if self.ends("abli") { self.r("able"); return } */

                if self.ends("alli") { self.r("al"); return }
                if self.ends("entli") { self.r("ent"); return }
                if self.ends("eli") { self.r("e"); return }
                if self.ends("ousli") { self.r("ous"); return }
            }
            b'o' => {
                if self.ends("ization") { self.r("ize"); return }
                if self.ends("ation") { self.r("ate"); return }
                if self.ends("ator") { self.r("ate"); return }
            }
            b's' => {
                if self.ends("alism") { self.r("al"); return }
                if self.ends("iveness") { self.r("ive"); return }
                if self.ends("fulness") { self.r("ful"); return }
                if self.ends("ousness") { self.r("ous"); return }
            }
            b't' => {
                if self.ends("aliti") { self.r("al"); return }
                if self.ends("iviti") { self.r("ive"); return }
                if self.ends("biliti") { self.r("ble"); return }
            }
            b'g' => if self.ends("logi") { self.r("log"); return }, /*-DEPARTURE-*/
             /* To match the published algorithm, delete this line */
            _ => ()
        }
    }

    /// stem.step3() deals with -ic-, -full, -ness etc. similar strategy to step2.
    fn step3(&mut self) {
        match self.b[self.k - 1] {
            b'e' => {
                if self.ends("icate") { self.r("ic"); return }
                if self.ends("ative") { self.r(""); return }
                if self.ends("alize") { self.r("al"); return }
            }
            b'i' => if self.ends("iciti") { self.r("ic"); return },
            b'l' => {
                if self.ends("ical") { self.r("ic"); return }
                if self.ends("ful") { self.r(""); return }
            }
            b's' => if self.ends("ness") { self.r(""); return },
            _ => ()
        }
    }

    /// stem.step4() takes off -ant, -ence etc., in context <c>vcvc<v>.
    fn step4(&mut self) {
        match self.b[self.k - 2] {
            b'a' => {
                if self.ends("al") {}
                else { return }
            }
            b'c' => {
                if self.ends("ance") {}
                else if self.ends("ence") {}
                else { return }
            }
            b'e' => {
                if self.ends("er") {}
                else { return }
            }
            b'i' => {
                if self.ends("ic") {}
                else { return }
            }
            b'l' => {
                if self.ends("able") {}
                else if self.ends("ible") {}
                else { return }
            }
            b'n' => {
                if self.ends("ant") {}
                else if self.ends("ement") {}
                else if self.ends("ment") {}
                else if self.ends("ent") {}
                else { return }
            }
            b'o' => {
                if self.ends("ion") &&
                    (self.b[self.j - 1] == b's' || self.b[self.j - 1] == b't') {}
                else if self.ends("ou") {}
                else { return }
                /* takes care of -ous */
            }
            b's' => {
                if self.ends("ism") {}
                else { return }
            }
            b't' => {
                if self.ends("ate") {}
                else if self.ends("iti") {}
                else { return }
            }
            b'u' => {
                if self.ends("ous") {}
                else { return }
            }
            b'v' => {
                if self.ends("ive") {}
                else { return }
            }
            b'z' => {
                if self.ends("ize") {}
                else { return }
            }
            _ => return
        }
        if self.measure() > 1 { self.k = self.j }
    }

    /// stem.step5() removes a final -e if self.measure() > 0, and changes -ll
    /// to -l if self.measure() > 1.
    fn step5(&mut self) {
        self.j = self.k;
        if self.b[self.k - 1] == b'e' {
            let a = self.measure();
            if a > 1 || a == 1 && !self.cvc(self.k - 2) { self.k -= 1 }
        }
        if self.b[self.k - 1] == b'l' &&
            self.double_consonant(self.k - 1) &&
            self.measure() > 1
        {
            self.k -= 1;
        }
    }

    #[inline]
    fn get(&self) -> String {
        unsafe { String::from_raw_buf_len(self.b.slice_to(self.k).as_ptr(), self.k) }
    }
}

pub fn get(word: &str) -> Result<String, &str> {
    if word.len() > 2 {
        match Stemmer::new(word) {
            Ok(w) => {
                let mut mw = w;
                mw.step1ab();
                mw.step1c();
                mw.step2();
                mw.step3();
                mw.step4();
                mw.step5();
                Ok(mw.get())
            }
            Err(e) => Err(e),
        }
    } else {
        Ok(word.to_owned())
    }
}

#[cfg(test)]
mod test_stem {
    use super::get;
    use test::Bencher;

    pub static INPUT: &'static str  = include_str!("../test-data/voc.txt");
    pub static RESULT: &'static str = include_str!("../test-data/output.txt");

    fn test_loop<
        's,
        I0: Iterator<T>,
        I1: Iterator<T>,
        T: Deref<str>>(
        tests: I0,
        results: I1
    ) {
        for (test, expect) in tests.zip(results) {
            let stemmed = get(test.trim_right_chars('\r'));

            assert!(stemmed.is_ok(), "[FAILED] Expected stem for '{}'", test.trim_right_chars('\r'));
            assert_eq!(stemmed.unwrap().trim_right_chars('\r'), expect.trim_right_chars('\r'));
        }
    }

    #[test]
    fn lexicon() {
        let input_s = INPUT.split('\n');
        let result_s = RESULT.split('\n');

        test_loop(input_s, result_s);
    }

    #[bench]
    fn lexicon_bench(b: &mut Bencher) {
        let input_v: Vec<&str> = INPUT.split('\n').collect();

        b.iter(|| {
            for t in input_v.iter() {
                let stemmed = get(t.trim_right_chars('\r'));

                assert!(stemmed.is_ok());
            }
        });
    }

    #[bench]
    fn single_bench(b: &mut Bencher) {
        b.iter(|| {
            let stemmed = get("testing");

            assert!(stemmed.is_ok());
        });
    }
}
<commit_msg>Update to 4874ca36f 2015-01-23 00:18:57 +0000.<commit_after>#[cfg(test)] extern crate test;

use std::ascii::AsciiExt;
use std::borrow::ToOwned;
use std::str;

/// Member b is a vector of bytes holding a word to be stemmed.
/// The letters are in b[0], b[1] ... ending at b[z->k]. Member k is readjusted
/// downwards as the stemming progresses. Zero termination is not in fact used
/// in the algorithm.
///
/// Note that only lower case sequences are stemmed. Forcing to lower case
/// should be done before get(...) is called.
///
///
/// Typical usage is:
///
///     let b = "pencils";
///     let res = stem::get(b);
///     assert_eq!(res, Ok("pencil".to_string()));
///
struct Stemmer {
    b: Vec<u8>,
    k: usize,
    j: usize
}

impl Stemmer {
    fn new(word: &str) -> Result<Stemmer, &'static str> {
        if !word.is_ascii() {
            Err("Only support English words with ASCII characters")
        } else {
            let b = word.to_ascii_lowercase().into_bytes();
            let k = b.len();
            Ok(Stemmer {
                b: b,
                k: k,
                j: 0
            })
        }
    }

    /// stem.is_consonant(i) is true <=> stem[i] is a consonant
    #[inline]
    fn is_consonant(&self, i: usize) -> bool {
        match self.b[i] {
            b'a' | b'e' | b'i' | b'o' | b'u' => false,
            b'y' => if i == 0 {
                true
            } else {
                !self.is_consonant(i - 1)
            },
            _ => true
        }
    }

    /// stem.measure() measures the number of consonant sequences in [0, j).
    /// if c is a consonant sequence and v a vowel sequence, and <..> indicates
    /// arbitrary presence,
    ///
    /// ~~~notrust
    ///    <c><v>       gives 0
    ///    <c>vc<v>     gives 1
    ///    <c>vcvc<v>   gives 2
    ///    <c>vcvcvc<v> gives 3
    ///    ....
    /// ~~~
    fn measure(&self) -> usize {
        let mut n = 0;
        let mut i = 0;
        let j = self.j;
        loop {
            if i >= j { return n }
            if !self.is_consonant(i) { break }
            i += 1;
        }
        i += 1;
        loop {
            loop {
                if i >= j { return n }
                if self.is_consonant(i) { break }
                i += 1;
            }
            i += 1;
            n += 1;
            loop {
                if i >= j { return n }
                if !self.is_consonant(i) { break }
                i += 1;
            }
            i += 1;
        }
    }

    /// stem.has_vowel() is TRUE <=> [0, j-1) contains a vowel
    fn has_vowel(&self) -> bool {
        for i in 0..self.j {
            if !self.is_consonant(i) {
                return true;
            }
        }
        false
    }

    /// stem.double_consonant(i) is TRUE <=> i,(i-1) contain a double consonant.
    #[inline]
    fn double_consonant(&self, i: usize) -> bool {
        if i < 1 {
            false
        } else if self.b[i] != self.b[i - 1] {
            false
        } else {
            self.is_consonant(i)
        }
    }

    /// cvc(z, i) is TRUE <=> i-2,i-1,i has the form consonant - vowel - consonant
    /// and also if the second c is not w,x or y. this is used when trying to
    /// restore an e at the end of a short word. e.g.
    ///
    /// ~~~notrust
    ///    cav(e), lov(e), hop(e), crim(e), but
    ///    snow, box, tray.
    /// ~~~
    fn cvc(&self, i: usize) -> bool {
        if i < 2
            || !self.is_consonant(i)
            || self.is_consonant(i - 1)
            || !self.is_consonant(i - 2)
        {
            false
        } else {
            match self.b[i] {
                b'w' | b'x' | b'y' => false,
                _ => true
            }
        }
    }

    /// stem.ends(s) is true <=> [0, k) ends with the string s.
    fn ends(&mut self, _s: &str) -> bool {
        let s = _s.as_bytes();
        let len = s.len();
        if len > self.k {
            false
        } else {
            if &self.b[self.k - len..self.k] == s {
                self.j = self.k - len;
                true
            } else {
                false
            }
        }
    }

    /// stem.setto(s) sets [j,k) to the characters in the string s,
    /// readjusting k.
    fn set_to(&mut self, s: &str) {
        let s = s.as_bytes();
        let len = s.len();
        for i in 0..(len) {
            self.b[self.j + i] = s[i];
        }
        self.k = self.j + len;
    }

    /// self.replace(s) is used further down.
    #[inline]
    fn r(&mut self, s: &str) {
        if self.measure() > 0 {
            self.set_to(s);
        }
    }

    /// stem.step1ab() gets rid of plurals and -ed or -ing. e.g.
    ///
    /// ~~~~notrust
    ///     caresses  ->  caress
    ///     ponies    ->  poni
    ///     ties      ->  ti
    ///     caress    ->  caress
    ///     cats      ->  cat
    ///
    ///     feed      ->  feed
    ///     agreed    ->  agree
    ///     disabled  ->  disable
    ///
    ///     matting   ->  mat
    ///     mating    ->  mate
    ///     meeting   ->  meet
    ///     milling   ->  mill
    ///     messing   ->  mess
    ///
    ///     meetings  ->  meet
    /// ~~~~
    fn step1ab(&mut self) {
        if self.b[self.k - 1] == b's' {
            if self.ends("sses") {
                self.k -= 2;
            } else if self.ends("ies") {
                self.set_to("i");
            } else if self.b[self.k - 2] != b's' {
                self.k -= 1;
            }
        }
        if self.ends("eed") {
            if self.measure() > 0 { self.k -= 1 }
        } else if (self.ends("ed") || self.ends("ing")) && self.has_vowel() {
            self.k = self.j;
            if self.ends("at") {
                self.set_to("ate");
            } else if self.ends("bl") {
                self.set_to("ble");
            } else if self.ends("iz") {
                self.set_to("ize");
            } else if self.double_consonant(self.k - 1) {
                self.k -= 1;
                match self.b[self.k - 1] {
                    b'l' | b's' | b'z' => self.k += 1,
                    _ => ()
                }
            } else if self.measure() == 1 && self.cvc(self.k - 1) {
                self.set_to("e");
            }
        }
    }

    /// stem.step1c() turns terminal y to i when there is another vowel in the stem.
    fn step1c(&mut self) {
        if self.ends("y") && self.has_vowel() {
           self.b.as_mut_slice()[self.k - 1] = b'i';
        }
    }

    /// stem.step2() maps double suffices to single ones. so -ization ( = -ize
    /// plus -ation) maps to -ize etc. note that the string before the suffix
    /// must give m(z) > 0.
    fn step2(&mut self) {
        match self.b[self.k - 2] {
            b'a' => {
                if self.ends("ational") { self.r("ate"); return }
                if self.ends("tional") { self.r("tion"); return }
            }
            b'c' => {
                if self.ends("enci") { self.r("ence"); return }
                if self.ends("anci") { self.r("ance"); return }
            }
            b'e' => if self.ends("izer") { self.r("ize"); return },
            b'l' => {
                if self.ends("bli") { self.r("ble"); return } /*-DEPARTURE-*/

             /* To match the published algorithm, replace this line with
                'l' => {
                    if self.ends("abli") { self.r("able"); return } */

                if self.ends("alli") { self.r("al"); return }
                if self.ends("entli") { self.r("ent"); return }
                if self.ends("eli") { self.r("e"); return }
                if self.ends("ousli") { self.r("ous"); return }
            }
            b'o' => {
                if self.ends("ization") { self.r("ize"); return }
                if self.ends("ation") { self.r("ate"); return }
                if self.ends("ator") { self.r("ate"); return }
            }
            b's' => {
                if self.ends("alism") { self.r("al"); return }
                if self.ends("iveness") { self.r("ive"); return }
                if self.ends("fulness") { self.r("ful"); return }
                if self.ends("ousness") { self.r("ous"); return }
            }
            b't' => {
                if self.ends("aliti") { self.r("al"); return }
                if self.ends("iviti") { self.r("ive"); return }
                if self.ends("biliti") { self.r("ble"); return }
            }
            b'g' => if self.ends("logi") { self.r("log"); return }, /*-DEPARTURE-*/
             /* To match the published algorithm, delete this line */
            _ => ()
        }
    }

    /// stem.step3() deals with -ic-, -full, -ness etc. similar strategy to step2.
    fn step3(&mut self) {
        match self.b[self.k - 1] {
            b'e' => {
                if self.ends("icate") { self.r("ic"); return }
                if self.ends("ative") { self.r(""); return }
                if self.ends("alize") { self.r("al"); return }
            }
            b'i' => if self.ends("iciti") { self.r("ic"); return },
            b'l' => {
                if self.ends("ical") { self.r("ic"); return }
                if self.ends("ful") { self.r(""); return }
            }
            b's' => if self.ends("ness") { self.r(""); return },
            _ => ()
        }
    }

    /// stem.step4() takes off -ant, -ence etc., in context <c>vcvc<v>.
    fn step4(&mut self) {
        match self.b[self.k - 2] {
            b'a' => {
                if self.ends("al") {}
                else { return }
            }
            b'c' => {
                if self.ends("ance") {}
                else if self.ends("ence") {}
                else { return }
            }
            b'e' => {
                if self.ends("er") {}
                else { return }
            }
            b'i' => {
                if self.ends("ic") {}
                else { return }
            }
            b'l' => {
                if self.ends("able") {}
                else if self.ends("ible") {}
                else { return }
            }
            b'n' => {
                if self.ends("ant") {}
                else if self.ends("ement") {}
                else if self.ends("ment") {}
                else if self.ends("ent") {}
                else { return }
            }
            b'o' => {
                if self.ends("ion") &&
                    (self.b[self.j - 1] == b's' || self.b[self.j - 1] == b't') {}
                else if self.ends("ou") {}
                else { return }
                /* takes care of -ous */
            }
            b's' => {
                if self.ends("ism") {}
                else { return }
            }
            b't' => {
                if self.ends("ate") {}
                else if self.ends("iti") {}
                else { return }
            }
            b'u' => {
                if self.ends("ous") {}
                else { return }
            }
            b'v' => {
                if self.ends("ive") {}
                else { return }
            }
            b'z' => {
                if self.ends("ize") {}
                else { return }
            }
            _ => return
        }
        if self.measure() > 1 { self.k = self.j }
    }

    /// stem.step5() removes a final -e if self.measure() > 0, and changes -ll
    /// to -l if self.measure() > 1.
    fn step5(&mut self) {
        self.j = self.k;
        if self.b[self.k - 1] == b'e' {
            let a = self.measure();
            if a > 1 || a == 1 && !self.cvc(self.k - 2) { self.k -= 1 }
        }
        if self.b[self.k - 1] == b'l' &&
            self.double_consonant(self.k - 1) &&
            self.measure() > 1
        {
            self.k -= 1;
        }
    }

    #[inline]
    fn get(&self) -> String {
        unsafe {
            str::from_utf8_unchecked(&self.b[..self.k]).to_owned()
        }
    }
}

pub fn get(word: &str) -> Result<String, &str> {
    if word.len() > 2 {
        match Stemmer::new(word) {
            Ok(w) => {
                let mut mw = w;
                mw.step1ab();
                mw.step1c();
                mw.step2();
                mw.step3();
                mw.step4();
                mw.step5();
                Ok(mw.get())
            }
            Err(e) => Err(e),
        }
    } else {
        Ok(word.to_owned())
    }
}

#[cfg(test)]
mod test_stem {
    use super::get;
    use test::Bencher;

    pub static INPUT: &'static str  = include_str!("../test-data/voc.txt");
    pub static RESULT: &'static str = include_str!("../test-data/output.txt");

    fn test_loop<
        's,
        I0: Iterator<T>,
        I1: Iterator<T>,
        T: Deref<str>>(
        tests: I0,
        results: I1
    ) {
        for (test, expect) in tests.zip(results) {
            let stemmed = get(test.trim_right_chars('\r'));

            assert!(stemmed.is_ok(), "[FAILED] Expected stem for '{}'", test.trim_right_chars('\r'));
            assert_eq!(stemmed.unwrap().trim_right_chars('\r'), expect.trim_right_chars('\r'));
        }
    }

    #[test]
    fn lexicon() {
        let input_s = INPUT.split('\n');
        let result_s = RESULT.split('\n');

        test_loop(input_s, result_s);
    }

    #[bench]
    fn lexicon_bench(b: &mut Bencher) {
        let input_v: Vec<&str> = INPUT.split('\n').collect();

        b.iter(|| {
            for t in input_v.iter() {
                let stemmed = get(t.trim_right_chars('\r'));

                assert!(stemmed.is_ok());
            }
        });
    }

    #[bench]
    fn single_bench(b: &mut Bencher) {
        b.iter(|| {
            let stemmed = get("testing");

            assert!(stemmed.is_ok());
        });
    }
}
<|endoftext|>


<commit_before>use std::collections::BTreeMap;

use serde_json::value::Value;

use priority::TaskPriority;

pub type DateTime= String; // FIXME
pub type Project = String;
pub type Status  = String;
pub type Tag     = String;
pub type UUID    = String; // FIXME
pub type Urgency = f64;

#[derive(Debug, Clone)]
pub struct Task {
    id: u64,
    desc: String,
    entry: DateTime,
    modified: DateTime,
    priority: TaskPriority,
    project: Project,
    status: Status,
    tags: Vec<Tag>,
    uuid: UUID,
    urgency: Urgency,
}

/*
 * TODO: We do not fail if the JSON parsing fails. This panics. We rely on taskwarrior to be nice
 * to us. I guess this should be fixed.
 */
impl Task {

    pub fn new(id: u64, desc: String, entry: DateTime, modified: DateTime,
        priority: TaskPriority, project: Project, status: Status,
        tags: Vec<Tag>, uuid: UUID, urgency: Urgency)
        -> Task
    {
        Task {
            id       : id,
            desc     : desc,
            entry    : entry,
            modified : modified,
            priority : priority,
            project  : project,
            status   : status,
            tags     : tags,
            uuid     : uuid,
            urgency  : urgency,
        }
    }

    pub fn from_value(v: Value) -> Option<Task> {
        if !v.is_object() {
            return None;
        }

        let map = v.as_object().unwrap();

        let keys = [ "id", "description", "modified", "priority", "project",
            "status", "tags", "uuid", "urgency"];

        if keys.iter().any(|x| map.contains_key(*x)) {
            return None
        }

        Some(Task {
            id       : get_id(&map),
            desc     : get_desc(&map),
            entry    : get_entry(&map),
            modified : get_modified(&map),
            priority : TaskPriority::from(get_priority(&map)),
            project  : get_project(&map),
            status   : get_status(&map),
            tags     : get_tags(&map),
            uuid     : get_uuid(&map),
            urgency  : get_urgency(&map),
        })
    }

}

impl Into<Value> for Task {

    fn into(self) -> Value {
        let id       = Value::U64(self.id);
        let desc     = Value::String(self.desc);
        let entry    = Value::String(self.entry);
        let modified = Value::String(self.modified);
        let priority = Value::String(self.priority.into());
        let project  = Value::String(self.project);
        let status   = Value::String(self.status);
        let tags     = Value::Array(self.tags
                                        .into_iter()
                                        .map(|s| Value::String(String::from(s)))
                                        .collect());
        let uuid     = Value::String(self.uuid);
        let urgency  = Value::F64(self.urgency);

        let mut map = BTreeMap::new();
        map.insert(String::from("id")       , id);
        map.insert(String::from("desc")     , desc);
        map.insert(String::from("entry")    , entry);
        map.insert(String::from("modified") , modified);
        map.insert(String::from("priority") , priority);
        map.insert(String::from("project")  , project);
        map.insert(String::from("status")   , status);
        map.insert(String::from("tags")     , tags);
        map.insert(String::from("uuid")     , uuid);
        map.insert(String::from("urgency")  , urgency);

        Value::Object(map)
    }

}

fn get_id(map: &BTreeMap<String, Value>) -> u64 {
    map.get("id").unwrap().as_u64().unwrap()
}

fn get_desc(map: &BTreeMap<String, Value>) -> String {
    map.get("description").unwrap().as_string().map(String::from).unwrap()
}

fn get_entry(map: &BTreeMap<String, Value>) -> String {
    map.get("entry").unwrap().as_string().map(String::from).unwrap()
}

fn get_modified(map: &BTreeMap<String, Value>) -> String {
    map.get("modified").unwrap().as_string().map(String::from).unwrap()
}

fn get_priority(map: &BTreeMap<String, Value>) -> &str {
    map.get("priority").unwrap().as_string().unwrap()
}

fn get_project(map: &BTreeMap<String, Value>) -> String {
    map.get("project").unwrap().as_string().map(String::from).unwrap()
}

fn get_status(map: &BTreeMap<String, Value>) -> String {
    map.get("status").unwrap().as_string().map(String::from).unwrap()
}

fn get_tags(map: &BTreeMap<String, Value>) -> Vec<Tag> {
    map.get("tags")
        .unwrap()
        .as_array()
        .unwrap()
        .clone()
        .into_iter()
        .map(|item| item.as_string().map(String::from).unwrap())
        .collect()
}

fn get_uuid(map: &BTreeMap<String, Value>) -> String {
    map.get("uuid").unwrap().as_string().map(String::from).unwrap()
}

fn get_urgency(map: &BTreeMap<String, Value>) -> f64 {
    map.get("urgency").unwrap().as_f64().unwrap()
}

<commit_msg>Add data getters for Task<commit_after>use std::collections::BTreeMap;

use serde_json::value::Value;

use priority::TaskPriority;

pub type DateTime= String; // FIXME
pub type Project = String;
pub type Status  = String;
pub type Tag     = String;
pub type UUID    = String; // FIXME
pub type Urgency = f64;

#[derive(Debug, Clone)]
pub struct Task {
    id: u64,
    desc: String,
    entry: DateTime,
    modified: DateTime,
    priority: TaskPriority,
    project: Project,
    status: Status,
    tags: Vec<Tag>,
    uuid: UUID,
    urgency: Urgency,
}

/*
 * TODO: We do not fail if the JSON parsing fails. This panics. We rely on taskwarrior to be nice
 * to us. I guess this should be fixed.
 */
impl Task {

    pub fn new(id: u64, desc: String, entry: DateTime, modified: DateTime,
        priority: TaskPriority, project: Project, status: Status,
        tags: Vec<Tag>, uuid: UUID, urgency: Urgency)
        -> Task
    {
        Task {
            id       : id,
            desc     : desc,
            entry    : entry,
            modified : modified,
            priority : priority,
            project  : project,
            status   : status,
            tags     : tags,
            uuid     : uuid,
            urgency  : urgency,
        }
    }

    pub fn from_value(v: Value) -> Option<Task> {
        if !v.is_object() {
            return None;
        }

        let map = v.as_object().unwrap();

        let keys = [ "id", "description", "modified", "priority", "project",
            "status", "tags", "uuid", "urgency"];

        if keys.iter().any(|x| map.contains_key(*x)) {
            return None
        }

        Some(Task {
            id       : get_id(&map),
            desc     : get_desc(&map),
            entry    : get_entry(&map),
            modified : get_modified(&map),
            priority : TaskPriority::from(get_priority(&map)),
            project  : get_project(&map),
            status   : get_status(&map),
            tags     : get_tags(&map),
            uuid     : get_uuid(&map),
            urgency  : get_urgency(&map),
        })
    }

    pub fn id(&self) -> u64 {
        self.id
    }

    pub fn desc(&self) -> &String {
        &self.desc
    }

    pub fn entry(&self) -> &DateTime {
        &self.entry
    }

    pub fn modified(&self) -> &DateTime {
        &self.modified
    }

    pub fn priority(&self) -> TaskPriority {
        self.priority.clone()
    }

    pub fn project(&self) -> &Project {
        &self.project
    }

    pub fn status(&self) -> &Status {
        &self.status
    }

    pub fn tags(&self) -> &Vec<Tag> {
        &self.tags
    }

    pub fn uuid(&self) -> &UUID {
        &self.uuid
    }

    pub fn urgency(&self) -> Urgency {
        self.urgency
    }

}

impl Into<Value> for Task {

    fn into(self) -> Value {
        let id       = Value::U64(self.id);
        let desc     = Value::String(self.desc);
        let entry    = Value::String(self.entry);
        let modified = Value::String(self.modified);
        let priority = Value::String(self.priority.into());
        let project  = Value::String(self.project);
        let status   = Value::String(self.status);
        let tags     = Value::Array(self.tags
                                        .into_iter()
                                        .map(|s| Value::String(String::from(s)))
                                        .collect());
        let uuid     = Value::String(self.uuid);
        let urgency  = Value::F64(self.urgency);

        let mut map = BTreeMap::new();
        map.insert(String::from("id")       , id);
        map.insert(String::from("desc")     , desc);
        map.insert(String::from("entry")    , entry);
        map.insert(String::from("modified") , modified);
        map.insert(String::from("priority") , priority);
        map.insert(String::from("project")  , project);
        map.insert(String::from("status")   , status);
        map.insert(String::from("tags")     , tags);
        map.insert(String::from("uuid")     , uuid);
        map.insert(String::from("urgency")  , urgency);

        Value::Object(map)
    }

}

fn get_id(map: &BTreeMap<String, Value>) -> u64 {
    map.get("id").unwrap().as_u64().unwrap()
}

fn get_desc(map: &BTreeMap<String, Value>) -> String {
    map.get("description").unwrap().as_string().map(String::from).unwrap()
}

fn get_entry(map: &BTreeMap<String, Value>) -> String {
    map.get("entry").unwrap().as_string().map(String::from).unwrap()
}

fn get_modified(map: &BTreeMap<String, Value>) -> String {
    map.get("modified").unwrap().as_string().map(String::from).unwrap()
}

fn get_priority(map: &BTreeMap<String, Value>) -> &str {
    map.get("priority").unwrap().as_string().unwrap()
}

fn get_project(map: &BTreeMap<String, Value>) -> String {
    map.get("project").unwrap().as_string().map(String::from).unwrap()
}

fn get_status(map: &BTreeMap<String, Value>) -> String {
    map.get("status").unwrap().as_string().map(String::from).unwrap()
}

fn get_tags(map: &BTreeMap<String, Value>) -> Vec<Tag> {
    map.get("tags")
        .unwrap()
        .as_array()
        .unwrap()
        .clone()
        .into_iter()
        .map(|item| item.as_string().map(String::from).unwrap())
        .collect()
}

fn get_uuid(map: &BTreeMap<String, Value>) -> String {
    map.get("uuid").unwrap().as_string().map(String::from).unwrap()
}

fn get_urgency(map: &BTreeMap<String, Value>) -> f64 {
    map.get("urgency").unwrap().as_f64().unwrap()
}

<|endoftext|>