<commit_before>
pub struct Csv {
    header: Vec<String>,
    rows: Vec<Vec<String>>,
}

impl Csv {
    pub fn new(mut rows: Vec<Vec<String>>) -> Csv {
        let header = rows.remove(0);
        Csv {
            header: header,
            rows: rows,
        }
    }

    pub fn sort_by(&mut self, col_name: &String) {
        let col_index = self.header.iter().position(|h| h == col_name).unwrap();

        self.rows
            .sort_by(|a, b| a[col_index].cmp(&(b[col_index])));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn build_csv() -> Csv {
        let header = vec!["Col1", "Col2"];
        let row1 = vec!["foo", "bar"];
        let row2 = vec!["baz", "qux"];
        let csv_data = vec![header, row1, row2];

        let string_csv_data = csv_data
            .iter()
            .map(|row| row.iter().map(|col| col.to_string()).collect())
            .collect();

        Csv::new(string_csv_data)
    }

    #[test]
    fn new_csv() {
        let csv = build_csv();

        assert_eq!("foo".to_string(), csv.rows[0][0]);
    }

    #[test]
    fn first_row_is_header() {
        let csv = build_csv();

        assert_eq!("Col1".to_string(), csv.header[0]);
    }

    #[test]
    fn sort_by_col_name_sorts_csv() {
        let mut csv = build_csv();

        let col: &String = &"Col1".to_string();
        csv.sort_by(col);

        assert_eq!("baz", csv.rows[0][0]);
    }
}
<commit_msg>cleanup sort_by method<commit_after>
pub struct Csv {
    header: Vec<String>,
    rows: Vec<Vec<String>>,
}

impl Csv {
    pub fn new(mut rows: Vec<Vec<String>>) -> Csv {
        let header = rows.remove(0);
        Csv {
            header: header,
            rows: rows,
        }
    }

    pub fn sort_by(&mut self, col_name: String) {
        let col_index = self.header
            .iter()
            .position(|h| h == &col_name)
            .unwrap();

        self.rows
            .sort_by(|a, b| a[col_index].cmp(&(b[col_index])));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn build_csv() -> Csv {
        let header = vec!["Col1", "Col2"];
        let row1 = vec!["foo", "bar"];
        let row2 = vec!["baz", "qux"];
        let csv_data = vec![header, row1, row2];

        let string_csv_data = csv_data
            .iter()
            .map(|row| row.iter().map(|col| col.to_string()).collect())
            .collect();

        Csv::new(string_csv_data)
    }

    #[test]
    fn new_csv() {
        let csv = build_csv();

        assert_eq!("foo".to_string(), csv.rows[0][0]);
    }

    #[test]
    fn first_row_is_header() {
        let csv = build_csv();

        assert_eq!("Col1".to_string(), csv.header[0]);
    }

    #[test]
    fn sort_by_col_name_sorts_csv() {
        let mut csv = build_csv();

        let col = "Col1".to_string();
        csv.sort_by(col);

        assert_eq!("baz", csv.rows[0][0]);
    }
}
<|endoftext|>


<commit_before>use std::result::Result;

use serde_json::*;
use gossyp_base::environment::Environment;

use super::bound_script::*;
use super::evaluate_expression::*;
use super::binding_environment::*;
use super::bind_statement::*;
use super::script::*;
use super::script_interpreter::*;

///
/// Enumeration representing how a failed bound statement can be described
///
#[derive(Deserialize, Serialize, Clone)]
pub enum FailedBoundStatement {
    RunCommand(FailedBoundExpression),
    Sequence(Vec<FailedBoundStatement>),
    Let(ScriptToken),
    Var(ScriptToken),
    Assign(ScriptToken),
    Loop(Box<FailedBoundStatement>),
    While(FailedBoundExpression),
    Using(FailedBoundExpression),
    Def(ScriptToken)
}

///
/// Generates a JS value indicating the statement that failed
///
fn generate_failed_bound_statement(script: &BoundScript) -> FailedBoundStatement {
    use self::FailedBoundStatement::*;

    match script {
        &BoundScript::AllocateVariables(_, ref script)  => generate_failed_bound_statement(&**script),
        &BoundScript::RunCommand(ref expr)              => RunCommand(generate_failed_bound_expression(expr)),
        &BoundScript::Sequence(ref statements)          => Sequence(statements.iter().map(|statement| generate_failed_bound_statement(statement)).collect()),
        &BoundScript::Assign(_, _, ref token)           => Assign(token.clone()),
        &BoundScript::Let(_, _, ref token)              => Let(token.clone()),
        &BoundScript::Var(_, _, ref token)              => Var(token.clone()),
        &BoundScript::Loop(ref loop_box)                => Loop(Box::new(generate_failed_bound_statement(&**loop_box))),
        &BoundScript::While(ref expr, _)                => While(generate_failed_bound_expression(expr)),
        &BoundScript::Using(ref expr, _)                => Using(generate_failed_bound_expression(expr)),
        &BoundScript::Def(ref token, _, _)              => Def(token.clone()),
    }
}

///
/// Creates an execution error
///
fn generate_script_error(error: ScriptEvaluationError, script: &BoundScript) -> Value {
    json![{
        "error":                    error,
        "failed-bound-statement":   generate_failed_bound_statement(script)
    }]
}

///
/// Evaluates the result of executing a sequence of steps
///
pub fn evaluate_sequence(sequence: &Vec<BoundScript>, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    // Execute the script
    let mut result = vec![];
    for statement in sequence.iter() {
        // Evaluate the next statement
        let next_result = evaluate_statement(statement, environment, execution_environment)?;

        // The script result is built up from the result of each statement
        // TODO: unless there's something like a return statement?
        result.push(next_result);
    }

    // Script is done
    Ok(Value::Array(result))
}

///
/// Allocates variables before continuing
///
fn evaluate_allocate_variables(num_variables: u32, continuation: &BoundScript, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    execution_environment.allocate_variables(num_variables);
    evaluate_statement(continuation, environment, execution_environment)
}

///
/// Assigns a value to a particular variable
///
fn evaluate_assignment(variable_index: u32, expr: &BoundExpression, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    let expression_value = evaluate_expression(expr, environment, execution_environment)?;
    execution_environment.set_variable(variable_index, Box::new(expression_value.clone()));

    Ok(expression_value)
}

///
/// Evaluates the result of executing a single statement
///
pub fn evaluate_statement(statement: &BoundScript, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    match statement {
        &BoundScript::AllocateVariables(num, ref continuation)  => evaluate_allocate_variables(num, &**continuation, environment, execution_environment),
        &BoundScript::RunCommand(ref expr)                      => evaluate_expression(expr, environment, execution_environment),
        &BoundScript::Sequence(ref steps)                       => evaluate_sequence(steps, environment, execution_environment),
        &BoundScript::Var(index, ref expr, _)                   => evaluate_assignment(index, expr, environment, execution_environment),
        &BoundScript::Assign(index, ref expr, _)                => evaluate_assignment(index, expr, environment, execution_environment),

        _                                                       => Err(generate_script_error(ScriptEvaluationError::StatementNotImplemented, statement))
    }
}

///
/// Evaluates the result of executing a single statement
///
pub fn evaluate_unbound_statement(statement: &Script, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    let mut binding_environment = BindingEnvironment::from_environment(environment);
    let bound                   = bind_statement(statement, &mut *binding_environment)?;

    evaluate_statement(&bound, environment, execution_environment)
}

#[cfg(test)]
mod test {
    use gossyp_base::basic::*;
    use super::*;
    use super::super::evaluate::*;

    #[test]
    fn can_execute_run_command() {
        let tool_expr           = Script::RunCommand(Expression::identifier("test"));
        let tool_environment    = DynamicEnvironment::new();

        tool_environment.define("test", Box::new(make_pure_tool(|_: ()| "Success")));

        let mut env             = ScriptExecutionEnvironment::new();
        let result              = evaluate_unbound_statement(&tool_expr, &tool_environment, &mut env);

        assert!(result == Ok(Value::String(String::from("Success"))));
    }

    #[test]
    fn can_execute_sequence() {
        let tool_expr           = Script::Sequence(
            vec![
                Script::RunCommand(Expression::string("\"test 1\"")),
                Script::RunCommand(Expression::string("\"test 2\""))
            ]);
        let tool_environment    = DynamicEnvironment::new();

        let mut env             = ScriptExecutionEnvironment::new();
        let result              = evaluate_unbound_statement(&tool_expr, &tool_environment, &mut env);

        assert!(result == Ok(Value::Array(vec![
            Value::String(String::from("test 1")),
            Value::String(String::from("test 2"))
        ])));
    }

    #[test]
    fn commands_have_own_environment() {
        let environment = DynamicEnvironment::new();

        // Tool that defines a new tool in its environment
        assert!(define_dynamic_tool(&environment, "make_tool", |_: (), env| define_pure_tool(env, "subtool", |_: ()| ())).is_ok());

        // Tools should get their own sub-environment so the new tool should not 'escape'. Need to do make_tool().subtool() to call the subtool
        assert!(gossyp_eval("subtool", &environment).is_err());
        assert!(gossyp_eval("make_tool", &environment).is_ok());
        assert!(gossyp_eval("subtool", &environment).is_err());
    }

    #[test]
    fn can_call_subtools() {
        let environment = DynamicEnvironment::new();

        // Tool that defines a new tool in its environment
        assert!(define_dynamic_tool(&environment, "make_tool", |_: (), env| define_pure_tool(env, "subtool", |_: ()| ())).is_ok());

        let subtool_result = gossyp_eval("make_tool().subtool()", &environment);
        if !subtool_result.is_ok() { println!("{:?}", subtool_result); }
        assert!(subtool_result.is_ok());
    }
}
<commit_msg>We’ll need to implement this later on<commit_after>use std::result::Result;

use serde_json::*;
use gossyp_base::environment::Environment;

use super::bound_script::*;
use super::evaluate_expression::*;
use super::binding_environment::*;
use super::bind_statement::*;
use super::script::*;
use super::script_interpreter::*;

///
/// Enumeration representing how a failed bound statement can be described
///
#[derive(Deserialize, Serialize, Clone)]
pub enum FailedBoundStatement {
    RunCommand(FailedBoundExpression),
    Sequence(Vec<FailedBoundStatement>),
    Let(ScriptToken),
    Var(ScriptToken),
    Assign(ScriptToken),
    Loop(Box<FailedBoundStatement>),
    While(FailedBoundExpression),
    Using(FailedBoundExpression),
    Def(ScriptToken)
}

///
/// Generates a JS value indicating the statement that failed
///
fn generate_failed_bound_statement(script: &BoundScript) -> FailedBoundStatement {
    use self::FailedBoundStatement::*;

    match script {
        &BoundScript::AllocateVariables(_, ref script)  => generate_failed_bound_statement(&**script),
        &BoundScript::RunCommand(ref expr)              => RunCommand(generate_failed_bound_expression(expr)),
        &BoundScript::Sequence(ref statements)          => Sequence(statements.iter().map(|statement| generate_failed_bound_statement(statement)).collect()),
        &BoundScript::Assign(_, _, ref token)           => Assign(token.clone()),
        &BoundScript::Let(_, _, ref token)              => Let(token.clone()),
        &BoundScript::Var(_, _, ref token)              => Var(token.clone()),
        &BoundScript::Loop(ref loop_box)                => Loop(Box::new(generate_failed_bound_statement(&**loop_box))),
        &BoundScript::While(ref expr, _)                => While(generate_failed_bound_expression(expr)),
        &BoundScript::Using(ref expr, _)                => Using(generate_failed_bound_expression(expr)),
        &BoundScript::Def(ref token, _, _)              => Def(token.clone()),
    }
}

///
/// Creates an execution error
///
fn generate_script_error(error: ScriptEvaluationError, script: &BoundScript) -> Value {
    json![{
        "error":                    error,
        "failed-bound-statement":   generate_failed_bound_statement(script)
    }]
}

///
/// Evaluates the result of executing a sequence of steps
///
pub fn evaluate_sequence(sequence: &Vec<BoundScript>, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    // Execute the script
    let mut result = vec![];
    for statement in sequence.iter() {
        // Evaluate the next statement
        let next_result = evaluate_statement(statement, environment, execution_environment)?;

        // The script result is built up from the result of each statement
        // TODO: unless there's something like a return statement?
        result.push(next_result);
    }

    // Script is done
    Ok(Value::Array(result))
}

///
/// Allocates variables before continuing
///
fn evaluate_allocate_variables(num_variables: u32, continuation: &BoundScript, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    execution_environment.allocate_variables(num_variables);
    evaluate_statement(continuation, environment, execution_environment)
}

///
/// Assigns a value to a particular variable
///
fn evaluate_assignment(variable_index: u32, expr: &BoundExpression, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    let expression_value = evaluate_expression(expr, environment, execution_environment)?;
    execution_environment.set_variable(variable_index, Box::new(expression_value.clone()));

    Ok(expression_value)
}

///
/// Evaluates the result of executing a single statement
///
pub fn evaluate_statement(statement: &BoundScript, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    match statement {
        &BoundScript::AllocateVariables(num, ref continuation)  => evaluate_allocate_variables(num, &**continuation, environment, execution_environment),
        &BoundScript::RunCommand(ref expr)                      => evaluate_expression(expr, environment, execution_environment),
        &BoundScript::Sequence(ref steps)                       => evaluate_sequence(steps, environment, execution_environment),
        &BoundScript::Var(index, ref expr, _)                   => evaluate_assignment(index, expr, environment, execution_environment),
        &BoundScript::Assign(index, ref expr, _)                => evaluate_assignment(index, expr, environment, execution_environment),

        _                                                       => Err(generate_script_error(ScriptEvaluationError::StatementNotImplemented, statement))
    }
}

///
/// Evaluates the result of executing a single statement
///
pub fn evaluate_unbound_statement(statement: &Script, environment: &Environment, execution_environment: &mut ScriptExecutionEnvironment) -> Result<Value, Value> {
    let mut binding_environment = BindingEnvironment::from_environment(environment);
    let bound                   = bind_statement(statement, &mut *binding_environment)?;

    evaluate_statement(&bound, environment, execution_environment)
}

#[cfg(test)]
mod test {
    use gossyp_base::basic::*;
    use super::*;
    use super::super::evaluate::*;

    #[test]
    fn can_execute_run_command() {
        let tool_expr           = Script::RunCommand(Expression::identifier("test"));
        let tool_environment    = DynamicEnvironment::new();

        tool_environment.define("test", Box::new(make_pure_tool(|_: ()| "Success")));

        let mut env             = ScriptExecutionEnvironment::new();
        let result              = evaluate_unbound_statement(&tool_expr, &tool_environment, &mut env);

        assert!(result == Ok(Value::String(String::from("Success"))));
    }

    #[test]
    fn can_execute_sequence() {
        let tool_expr           = Script::Sequence(
            vec![
                Script::RunCommand(Expression::string("\"test 1\"")),
                Script::RunCommand(Expression::string("\"test 2\""))
            ]);
        let tool_environment    = DynamicEnvironment::new();

        let mut env             = ScriptExecutionEnvironment::new();
        let result              = evaluate_unbound_statement(&tool_expr, &tool_environment, &mut env);

        assert!(result == Ok(Value::Array(vec![
            Value::String(String::from("test 1")),
            Value::String(String::from("test 2"))
        ])));
    }

    #[test]
    fn commands_have_own_environment() {
        let environment = DynamicEnvironment::new();

        // Tool that defines a new tool in its environment
        assert!(define_dynamic_tool(&environment, "make_tool", |_: (), env| define_pure_tool(env, "subtool", |_: ()| ())).is_ok());

        // Tools should get their own sub-environment so the new tool should not 'escape'. Need to do make_tool().subtool() to call the subtool
        assert!(gossyp_eval("subtool", &environment).is_err());
        assert!(gossyp_eval("make_tool", &environment).is_ok());
        assert!(gossyp_eval("subtool", &environment).is_err());
    }

    /*
    #[test]
    fn can_call_subtools() {
        let environment = DynamicEnvironment::new();

        // Tool that defines a new tool in its environment
        assert!(define_dynamic_tool(&environment, "make_tool", |_: (), env| define_pure_tool(env, "subtool", |_: ()| ())).is_ok());

        let subtool_result = gossyp_eval("make_tool().subtool()", &environment);
        if !subtool_result.is_ok() { println!("{:?}", subtool_result); }
        assert!(subtool_result.is_ok());
    }
    */
}
<|endoftext|>


<commit_before>use regex::Regex;
use database::*;

macro_rules! regex { ($re:expr) => { ::regex::Regex::new($re).unwrap() } }

#[derive(Debug)]
struct Film {
    name: String,
    votes: String,
    rating: String,
    year: String,
}

impl Film {
    fn new() -> Self {
        Film {
            name: String::new(),
            votes: String::new(),
            rating: String::new(),
            year: String::new(),
        }
    }
}

pub fn parse_rating(string: String) {
    let mut all_films: Vec<Film> = Vec::new();

    let re =
        Regex::new(r"(\n [\s]+ [\d|\.]{10}[\s]+ (\d+)[\s]+ ([\d|\.]{3}) (.+) ([(](\d{4})[)]))")
            .unwrap();
    for cap in re.captures_iter(string.as_str()) {
        let mut film = Film::new();
        // Debug
        // println!("Name:{} Votecount:{} Rating:{} Year:{}", &cap[4], &cap[2], &cap[3], &cap[5] );
        film.name.push_str(&cap[4]);
        film.votes.push_str(&cap[2]);
        film.rating.push_str(&cap[3]);
        film.year.push_str(&cap[5]);
        all_films.push(film);
    }

    println!("{:?}", all_films[5]);
}

pub fn parse_movies(string: String) {
    // let re = Regex::new(r"(\n\u{0022}?[#|\+|\-|\*|\/|\.|\,|\!|\:|\&|\%|\$|\§|\w|\d|\s]*\u{0022}? \([\d]{4}\))").unwrap();
    let re = Regex::new("(\\n\u{0022}?[#|\\+|\\-|\\*|\\041|\\057|\\054|\\056|\\072|\\073|\\044|\\045|\\046|\\w|\\d|\\s]*\u{0022}? \\([\\d]{4}\\))")
        .unwrap();
    println!("{:?}", re);
    for cap in re.captures_iter(string.as_str()) {
        let mut movie = &cap[0];
        movie = movie.trim();
        let tmp: Vec<&str> = movie.rsplitn(2, ' ').collect();
        let mut year = tmp[0];
        year = year.trim();
        import_movie(tmp[1], &year[1..5]);
    }
}
<commit_msg>add parse_genre()<commit_after>use regex::Regex;
use database::*;

macro_rules! regex { ($re:expr) => { ::regex::Regex::new($re).unwrap() } }

#[derive(Debug)]
struct Film {
    name: String,
    votes: String,
    rating: String,
    year: String,
}

impl Film {
    fn new() -> Self {
        Film {
            name: String::new(),
            votes: String::new(),
            rating: String::new(),
            year: String::new(),
        }
    }
}

#[derive(Debug)]
struct Genre {
    movie: String,
    genre: String,
}
impl Genre {
    fn new() -> Self {
        Genre {
            movie: String::new(),
            genre: String::new(),
        }
    }
}

pub fn parse_rating(string: String) {
    let mut all_films: Vec<Film> = Vec::new();

    let re = Regex::new(
        r"(\n [\s]+ [\d|\.]{10}[\s]+ (\d+)[\s]+ ([\d|\.]{3}) (.+) [(](\d{4})[)])"
    ).unwrap();

    for cap in re.captures_iter(string.as_str()) {
        let mut film = Film::new();
        // Debug
        // println!("Name:{} Votecount:{} Rating:{} Year:{}", &cap[4], &cap[2], &cap[3], &cap[5] );
        film.name.push_str(&cap[4]);
        film.votes.push_str(&cap[2]);
        film.rating.push_str(&cap[3]);
        film.year.push_str(&cap[5]);
        all_films.push(film);
    }

    println!("{:?}", all_films[5]);
}

fn parse_genre(string: String) {

    // // hardcoded test file
    // let mut f = File::open("test2");
    // let mut buffer = String::new();
    // // TODO: Error handling
    // f.unwrap().read_to_string(&mut buffer);

    let mut all_genres:Vec<Genre> = Vec::new();

    let re = Regex::new(r"(\n(.+)[\s]+[(]\d{4}[)][\s]+(?:\(V\)|\(TV\))?[\s]*([^\{].+))").unwrap();
    //                     ^ title     ^ year          ^ (TV|V) filter            ^ Genre
    for cap in re.captures_iter(string.as_str()) {

        let mut genre = Genre::new();
        // Debug
        // println!("Name: {} Genre: {}", &cap[2], &cap[3] );
        genre.movie.push_str(&cap[2]);
        genre.genre.push_str(&cap[3]);
        all_genres.push(genre);
    }

    // all_genres
    // println!("{:?}", all_films);
}

pub fn parse_movies(string: String) {
    // let re = Regex::new(r"(\n\u{0022}?[#|\+|\-|\*|\/|\.|\,|\!|\:|\&|\%|\$|\§|\w|\d|\s]*\u{0022}? \([\d]{4}\))").unwrap();
    let re = Regex::new("(\\n\u{0022}?[#|\\+|\\-|\\*|\\041|\\057|\\054|\\056|\\072|\\073|\\044|\\045|\\046|\\w|\\d|\\s]*\u{0022}? \\([\\d]{4}\\))")
        .unwrap();
    println!("{:?}", re);
    for cap in re.captures_iter(string.as_str()) {
        let mut movie = &cap[0];
        movie = movie.trim();
        let tmp: Vec<&str> = movie.rsplitn(2, ' ').collect();
        let mut year = tmp[0];
        year = year.trim();
        import_movie(tmp[1], &year[1..5]);
    }
}
<|endoftext|>


<commit_before>use std::io::{Read, BufReader};
use std::str::from_utf8;

use types::{RedisResult, Value, ErrorKind, make_extension_error};

/// The internal redis response parser.
pub struct Parser<T> {
    reader: T,
}

/// The parser can be used to parse redis responses into values.  Generally
/// you normally do not use this directly as it's already done for you by
/// the client but in some more complex situations it might be useful to be
/// able to parse the redis responses.
impl<'a, T: Read> Parser<T> {

    /// Creates a new parser that parses the data behind the reader.  More
    /// than one value can be behind the reader in which case the parser can
    /// be invoked multiple times.  In other words: the stream does not have
    /// to be terminated.
    pub fn new(reader: T) -> Parser<T> {
        Parser { reader: reader }
    }

    /* public api */

    /// parses a single value out of the stream.  If there are multiple
    /// values you can call this multiple times.  If the reader is not yet
    /// ready this will block.
    pub fn parse_value(&mut self) -> RedisResult<Value> {
        let b = try!(self.read_byte());
        match b as char {
            '+' => self.parse_status(),
            ':' => self.parse_int(),
            '$' => self.parse_data(),
            '*' => self.parse_bulk(),
            '-' => self.parse_error(),
            _ => fail!((ErrorKind::ResponseError, "Invalid response when parsing value")),
        }
    }

    /* internal helpers */

    #[inline]
    fn expect_char(&mut self, refchar: char) -> RedisResult<()> {
        if try!(self.read_byte()) as char == refchar {
            Ok(())
        } else {
            fail!((ErrorKind::ResponseError, "Invalid byte in response"));
        }
    }

    #[inline]
    fn expect_newline(&mut self) -> RedisResult<()> {
        match try!(self.read_byte()) as char {
            '\n' => Ok(()),
            '\r' => self.expect_char('\n'),
            _ => fail!((ErrorKind::ResponseError, "Invalid byte in response")),
        }
    }

    fn read_line(&mut self) -> RedisResult<Vec<u8>> {
        let mut rv = vec![];

        loop {
            let b = try!(self.read_byte());
            match b as char {
                '\n' => { break; }
                '\r' => {
                    try!(self.expect_char('\n'));
                    break;
                },
                _ => { rv.push(b) }
            };
        }

        Ok(rv)
    }

    fn read_string_line(&mut self) -> RedisResult<String> {
        match String::from_utf8(try!(self.read_line())) {
            Err(_) => {
                fail!((ErrorKind::ResponseError, "Expected valid string, got garbage"))
            }
            Ok(value) => Ok(value),
        }
    }

    fn read_byte(&mut self) -> RedisResult<u8> {
        let buf: &mut [u8; 1] = &mut [0];
        let nread = try!(self.reader.read(buf));

        if nread < 1 {
            fail!((ErrorKind::ResponseError, "Could not read enough bytes"))
        } else {
            Ok(buf[0])
        }
    }

    fn read(&mut self, bytes: usize) -> RedisResult<Vec<u8>> {
        let mut rv = vec![0; bytes];
        let mut i = 0;
        while i < bytes {
            let res_nread = {
                let ref mut buf = &mut rv[i..];
                self.reader.read(buf)
            };
            match res_nread {
                Ok(nread) if nread > 0 =>
                    i += nread,
                Ok(_) =>
                    return fail!((ErrorKind::ResponseError, "Could not read enough bytes")),
                Err(e) =>
                    return Err(From::from(e))
            }
        };
        Ok(rv)
    }

    fn read_int_line(&mut self) -> RedisResult<i64> {
        let line = try!(self.read_string_line());
        match line.trim().parse::<i64>() {
            Err(_) => fail!((ErrorKind::ResponseError, "Expected integer, got garbage")),
            Ok(value) => Ok(value)
        }
    }

    fn parse_status(&mut self) -> RedisResult<Value> {
        let line = try!(self.read_string_line());
        if line == "OK" {
            Ok(Value::Okay)
        } else {
            Ok(Value::Status(line))
        }
    }

    fn parse_int(&mut self) -> RedisResult<Value> {
        Ok(Value::Int(try!(self.read_int_line())))
    }

    fn parse_data(&mut self) -> RedisResult<Value> {
        let length = try!(self.read_int_line());
        if length < 0 {
            Ok(Value::Nil)
        } else {
            let data = try!(self.read(length as usize));
            try!(self.expect_newline());
            Ok(Value::Data(data))
        }
    }

    fn parse_bulk(&mut self) -> RedisResult<Value> {
        let length = try!(self.read_int_line());
        if length < 0 {
            Ok(Value::Nil)
        } else {
            let mut rv = vec![];
            rv.reserve(length as usize);
            for _ in 0..length {
                rv.push(try!(self.parse_value()));
            }
            Ok(Value::Bulk(rv))
        }
    }

    fn parse_error(&mut self) -> RedisResult<Value> {
        let desc = "An error was signalled by the server";
        let line = try!(self.read_string_line());
        let mut pieces = line.splitn(2, ' ');
        let kind = match pieces.next().unwrap() {
            "ERR" => ErrorKind::ResponseError,
            "EXECABORT" => ErrorKind::ExecAbortError,
            "LOADING" => ErrorKind::BusyLoadingError,
            "NOSCRIPT" => ErrorKind::NoScriptError,
            code => { fail!(make_extension_error(code, pieces.next())); }
        };
        match pieces.next() {
            Some(detail) => fail!((kind, desc, detail.to_string())),
            None => fail!((kind, desc))
        }
    }
}

/// Parses bytes into a redis value.
///
/// This is the most straightforward way to parse something into a low
/// level redis value instead of having to use a whole parser.
pub fn parse_redis_value(bytes: &[u8]) -> RedisResult<Value> {
    let mut parser = Parser::new(BufReader::new(bytes));
    parser.parse_value()
}
<commit_msg>Remove unused import<commit_after>use std::io::{Read, BufReader};

use types::{RedisResult, Value, ErrorKind, make_extension_error};

/// The internal redis response parser.
pub struct Parser<T> {
    reader: T,
}

/// The parser can be used to parse redis responses into values.  Generally
/// you normally do not use this directly as it's already done for you by
/// the client but in some more complex situations it might be useful to be
/// able to parse the redis responses.
impl<'a, T: Read> Parser<T> {

    /// Creates a new parser that parses the data behind the reader.  More
    /// than one value can be behind the reader in which case the parser can
    /// be invoked multiple times.  In other words: the stream does not have
    /// to be terminated.
    pub fn new(reader: T) -> Parser<T> {
        Parser { reader: reader }
    }

    /* public api */

    /// parses a single value out of the stream.  If there are multiple
    /// values you can call this multiple times.  If the reader is not yet
    /// ready this will block.
    pub fn parse_value(&mut self) -> RedisResult<Value> {
        let b = try!(self.read_byte());
        match b as char {
            '+' => self.parse_status(),
            ':' => self.parse_int(),
            '$' => self.parse_data(),
            '*' => self.parse_bulk(),
            '-' => self.parse_error(),
            _ => fail!((ErrorKind::ResponseError, "Invalid response when parsing value")),
        }
    }

    /* internal helpers */

    #[inline]
    fn expect_char(&mut self, refchar: char) -> RedisResult<()> {
        if try!(self.read_byte()) as char == refchar {
            Ok(())
        } else {
            fail!((ErrorKind::ResponseError, "Invalid byte in response"));
        }
    }

    #[inline]
    fn expect_newline(&mut self) -> RedisResult<()> {
        match try!(self.read_byte()) as char {
            '\n' => Ok(()),
            '\r' => self.expect_char('\n'),
            _ => fail!((ErrorKind::ResponseError, "Invalid byte in response")),
        }
    }

    fn read_line(&mut self) -> RedisResult<Vec<u8>> {
        let mut rv = vec![];

        loop {
            let b = try!(self.read_byte());
            match b as char {
                '\n' => { break; }
                '\r' => {
                    try!(self.expect_char('\n'));
                    break;
                },
                _ => { rv.push(b) }
            };
        }

        Ok(rv)
    }

    fn read_string_line(&mut self) -> RedisResult<String> {
        match String::from_utf8(try!(self.read_line())) {
            Err(_) => {
                fail!((ErrorKind::ResponseError, "Expected valid string, got garbage"))
            }
            Ok(value) => Ok(value),
        }
    }

    fn read_byte(&mut self) -> RedisResult<u8> {
        let buf: &mut [u8; 1] = &mut [0];
        let nread = try!(self.reader.read(buf));

        if nread < 1 {
            fail!((ErrorKind::ResponseError, "Could not read enough bytes"))
        } else {
            Ok(buf[0])
        }
    }

    fn read(&mut self, bytes: usize) -> RedisResult<Vec<u8>> {
        let mut rv = vec![0; bytes];
        let mut i = 0;
        while i < bytes {
            let res_nread = {
                let ref mut buf = &mut rv[i..];
                self.reader.read(buf)
            };
            match res_nread {
                Ok(nread) if nread > 0 =>
                    i += nread,
                Ok(_) =>
                    return fail!((ErrorKind::ResponseError, "Could not read enough bytes")),
                Err(e) =>
                    return Err(From::from(e))
            }
        };
        Ok(rv)
    }

    fn read_int_line(&mut self) -> RedisResult<i64> {
        let line = try!(self.read_string_line());
        match line.trim().parse::<i64>() {
            Err(_) => fail!((ErrorKind::ResponseError, "Expected integer, got garbage")),
            Ok(value) => Ok(value)
        }
    }

    fn parse_status(&mut self) -> RedisResult<Value> {
        let line = try!(self.read_string_line());
        if line == "OK" {
            Ok(Value::Okay)
        } else {
            Ok(Value::Status(line))
        }
    }

    fn parse_int(&mut self) -> RedisResult<Value> {
        Ok(Value::Int(try!(self.read_int_line())))
    }

    fn parse_data(&mut self) -> RedisResult<Value> {
        let length = try!(self.read_int_line());
        if length < 0 {
            Ok(Value::Nil)
        } else {
            let data = try!(self.read(length as usize));
            try!(self.expect_newline());
            Ok(Value::Data(data))
        }
    }

    fn parse_bulk(&mut self) -> RedisResult<Value> {
        let length = try!(self.read_int_line());
        if length < 0 {
            Ok(Value::Nil)
        } else {
            let mut rv = vec![];
            rv.reserve(length as usize);
            for _ in 0..length {
                rv.push(try!(self.parse_value()));
            }
            Ok(Value::Bulk(rv))
        }
    }

    fn parse_error(&mut self) -> RedisResult<Value> {
        let desc = "An error was signalled by the server";
        let line = try!(self.read_string_line());
        let mut pieces = line.splitn(2, ' ');
        let kind = match pieces.next().unwrap() {
            "ERR" => ErrorKind::ResponseError,
            "EXECABORT" => ErrorKind::ExecAbortError,
            "LOADING" => ErrorKind::BusyLoadingError,
            "NOSCRIPT" => ErrorKind::NoScriptError,
            code => { fail!(make_extension_error(code, pieces.next())); }
        };
        match pieces.next() {
            Some(detail) => fail!((kind, desc, detail.to_string())),
            None => fail!((kind, desc))
        }
    }
}

/// Parses bytes into a redis value.
///
/// This is the most straightforward way to parse something into a low
/// level redis value instead of having to use a whole parser.
pub fn parse_redis_value(bytes: &[u8]) -> RedisResult<Value> {
    let mut parser = Parser::new(BufReader::new(bytes));
    parser.parse_value()
}
<|endoftext|>


<commit_before>use token::Token;
use tokenizer::{Tokenizer, LexerError};
pub use ast::*;

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    UnexpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        let function_parts = self.partition_function(&statement_tokens);

        match function_parts {
            // no problem if we just expected equals, it is probably
            // an expr
            Err(ParseError::ExpectedToken(Token::Equals)) => {
                Ok(Statement::Expression(
                    try!(self.expr(&statement_tokens[..]))
                ))
            },
            Err(e) => return Err(e),
            Ok(parts) => {
                let (lhs, rhs) = parts;

                // partition_function is guarenteed to return non-empty slices
                debug_assert!(!lhs.is_empty() && !rhs.is_empty());

                if lhs[0] == Token::Backslash {
                    return self.anonymous_function(lhs, rhs);
                }

                match lhs.len() {
                    // only one argument: name = ...
                    1 => self.assignment(lhs, rhs),
                    // more than one argument: name arg1 ... = ...
                    _ => self.named_function(lhs, rhs),
                }
            },
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[0..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = lhs[0].clone().unwrap_symbol();

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[0..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> ParseResult<Function> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(|ref sym| (*sym).clone().unwrap_symbol()).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs.len() == 1);

        Ok(Statement::Assignment {
            name: lhs[0].clone().unwrap_symbol(),
            value: try!(self.expr(rhs)),
        })
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        let mut result = Expr::new();

        let mut remaining = tokens;
        loop {
            let (item, leftover) = try!(self.expr_item(remaining));
            remaining = leftover;

            result.push(item);

            if remaining.is_empty() {
                break;
            }
        }

        Ok(result)
    }

    fn expr_item<'a>(&self, tokens: &'a [Token]) -> ParseResult<(ExprItem, &'a [Token])> {
        debug_assert!(!tokens.is_empty());

        if tokens[0] == Token::Backslash {
            let (lhs, rhs) = try!(self.partition_function(tokens));

            debug_assert!(lhs[0] == Token::Backslash);

            Ok((
                ExprItem::AnonymousFunction(
                    try!(self.function(&lhs[1..], rhs))
                ),
                &[]
            ))
        }
        else if tokens[0] == Token::ParenOpen {
            let close = tokens.iter().position(|ref t| **t == Token::ParenClose);
            if close.is_none() {
                return Err(ParseError::ExpectedToken(Token::ParenClose));
            }
            let close = close.unwrap();

            Ok((
                ExprItem::Group(
                    try!(self.expr(&tokens[1..close]))
                ),
                &tokens[(close+1)..]
            ))
        }
        else {
            Ok((ExprItem::SingleTerm(
                try!(self.term(&tokens[0]))
            ), &tokens[1..]))
        }
    }

    fn term(&self, token: &Token) -> ParseResult<Term> {
        Ok(Term::Symbol("ok".to_owned()))
    }

    // partitions a function by its Equals token, returning None if no
    // valid function is found
    // Returns (lhs, rhs)
    fn partition_function<'a>(&self, tokens: &'a [Token]) -> ParseResult<(&'a [Token], &'a [Token])> {
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                return Err(ParseError::UnexpectedToken(token.clone()));
            }
        }

        if equals.is_none() {
            Err(ParseError::ExpectedToken(Token::Equals))
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            Ok((&tokens[..equals], &tokens[(equals+1)..]))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Finished implementing term and added string boundary<commit_after>use token::Token;
use tokenizer::{Tokenizer, LexerError};
pub use ast::*;

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    UnexpectedToken(Token),
    InvalidStringLiteral,
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        let function_parts = self.partition_function(&statement_tokens);

        match function_parts {
            // no problem if we just expected equals, it is probably
            // an expr
            Err(ParseError::ExpectedToken(Token::Equals)) => {
                Ok(Statement::Expression(
                    try!(self.expr(&statement_tokens[..]))
                ))
            },
            Err(e) => return Err(e),
            Ok(parts) => {
                let (lhs, rhs) = parts;

                // partition_function is guarenteed to return non-empty slices
                debug_assert!(!lhs.is_empty() && !rhs.is_empty());

                if lhs[0] == Token::Backslash {
                    return self.anonymous_function(lhs, rhs);
                }

                match lhs.len() {
                    // only one argument: name = ...
                    1 => self.assignment(lhs, rhs),
                    // more than one argument: name arg1 ... = ...
                    _ => self.named_function(lhs, rhs),
                }
            },
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[0..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = lhs[0].clone().unwrap_symbol();

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[0..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> ParseResult<Function> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(|ref sym| (*sym).clone().unwrap_symbol()).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs.len() == 1);

        Ok(Statement::Assignment {
            name: lhs[0].clone().unwrap_symbol(),
            value: try!(self.expr(rhs)),
        })
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        let mut result = Expr::new();

        let mut remaining = tokens;
        loop {
            let (item, leftover) = try!(self.expr_item(remaining));
            remaining = leftover;

            result.push(item);

            if remaining.is_empty() {
                break;
            }
        }

        Ok(result)
    }

    fn expr_item<'a>(&self, tokens: &'a [Token]) -> ParseResult<(ExprItem, &'a [Token])> {
        debug_assert!(!tokens.is_empty());

        if tokens[0] == Token::Backslash {
            let (lhs, rhs) = try!(self.partition_function(tokens));

            debug_assert!(lhs[0] == Token::Backslash);

            Ok((
                ExprItem::AnonymousFunction(
                    try!(self.function(&lhs[1..], rhs))
                ),
                &[]
            ))
        }
        else if tokens[0] == Token::ParenOpen {
            let close = tokens.iter().position(|ref t| **t == Token::ParenClose);
            if close.is_none() {
                return Err(ParseError::ExpectedToken(Token::ParenClose));
            }
            let close = close.unwrap();

            Ok((
                ExprItem::Group(
                    try!(self.expr(&tokens[1..close]))
                ),
                &tokens[(close+1)..]
            ))
        }
        else if tokens[0] == Token::StringBoundary {
            // since strings can only be a single word, not a number
            const STRING_VALUE: usize = 1;
            const STRING_END: usize = 2;
            const AFTER_STRING: usize = 3;

            if tokens[STRING_END] != Token::StringBoundary {
                Err(ParseError::ExpectedToken(Token::StringBoundary))
            }
            else if let Token::Symbol(ref value) = tokens[STRING_VALUE] {
                Ok((ExprItem::SingleTerm(
                    Term::StringLiteral(value.clone())
                ), &tokens[AFTER_STRING..]))
            }
            else {
                Err(ParseError::InvalidStringLiteral)
            }
        }
        else {
            Ok((ExprItem::SingleTerm(
                try!(self.simple_term(&tokens[0]))
            ), &tokens[1..]))
        }
    }

    fn simple_term(&self, token: &Token) -> ParseResult<Term> {
        match *token {
            Token::Symbol(ref s) => Ok(Term::Symbol(s.clone())),
            Token::Number(ref n) => Ok(Term::Number(n.clone())),
            _ => Err(ParseError::UnexpectedToken(token.clone())),
        }
    }

    // partitions a function by its Equals token, returning None if no
    // valid function is found
    // Returns (lhs, rhs)
    fn partition_function<'a>(&self, tokens: &'a [Token]) -> ParseResult<(&'a [Token], &'a [Token])> {
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                return Err(ParseError::UnexpectedToken(token.clone()));
            }
        }

        if equals.is_none() {
            Err(ParseError::ExpectedToken(Token::Equals))
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            Ok((&tokens[..equals], &tokens[(equals+1)..]))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>


<commit_before>// Copyright (c) 2014 Corey Richardson
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

use std::collections::hashmap::HashMap;
use std;

#[deriving(Show)]
pub struct Protocol {
    types: HashMap<String, Type>,
    categories: HashMap<String, Category>,
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum Type {
    I8, U8,
    I16, U16,
    I32, U32, F32,
    I64, U64, F64,
    Array(Box<Type>),
    Aggregate(Object),
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub struct Object {
    fields: Vec<(String, Type)>,
}

#[deriving(Show)]
pub struct Category {
    name: String,
    methods: HashMap<String, Method>,
}

#[deriving(Show, Encodable, Decodable)]
pub struct Method {
    comment: String,
    properties: HashMap<String, Property>,
    attributes: Vec<Attr>,
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum Property {
    In(Type),
    Out(Type)
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum ParseError {
    UnexpectedToken(String),
    UnexpectedEof(String),
    Other(String)
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum Attr {
    Auth,
    Unauth,
    Admin,
    Global,
    Map,
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
enum Token {
    ATTR(Attr),
    STRING(String),
    IDENT(String),
    PRIM(Type),

    NEWTYPE,
    CATEGORY,
    INCLUDE,
    METHOD,
    IN,
    OUT,

    SEMI,
    LBRACE,
    RBRACE,
    COMMA,
    EQ,

    NL,
}

struct Lexer<R> {
    reader: R,
}

impl<R: std::io::Buffer> Iterator<Token> for Lexer<R> {
    fn next(&mut self) -> Option<Token> {
        // IO errors are interesting and never expected.
        let mut chrs = self.reader.chars().map(|io| io.unwrap()).peekable();
        for c in chrs {
            // could be anything...
            match c {
                // string literal
                '"' => {
                    return Some(STRING(chrs.take_while(|&c| c != '"').collect()));
                }
                '\n' => {
                    return Some(NL);
                },
                '\r' => {
                    if chrs.next().unwrap() != '\n' {
                        error!("CR found not followed by LF!");
                    }
                    return Some(NL);
                }
                // keyword or identifier
                c @ 'a'..'z' | c @ 'A'..'Z' | c @ '_' => {
                    let ident = Some(c).move_iter().chain(chrs.take_while(|&c| c.is_alphanumeric() || c == '_')).collect::<String>();
                    match ident.as_slice() {
                        "auth" => return Some(ATTR(Auth)),
                        "unauth" => return Some(ATTR(Unauth)),
                        "admin" => return Some(ATTR(Admin)),
                        "global" => return Some(ATTR(Global)),
                        "map" => return Some(ATTR(Map)),
                        "newtype" => return Some(NEWTYPE),
                        "category" => return Some(CATEGORY),
                        "include" => return Some(INCLUDE),
                        "method" => return Some(METHOD),
                        "in" => return Some(IN),
                        "out" => return Some(OUT),
                        non_keyword => {
                            match try_parse_type(non_keyword) {
                                Some(t) => return Some(PRIM(t)),
                                None => return Some(IDENT(non_keyword.to_string()))
                            }
                        }
                    }
                },
                ';' => return Some(SEMI),
                '{' => return Some(LBRACE),
                '}' => return Some(RBRACE),
                ',' => return Some(COMMA),
                '=' => return Some(EQ),
                ' ' | '\t' => continue,
                wat => error!("Found unexpected character when lexing: {}", wat),
            }
        }
        None
    }
}

pub fn parse<R: std::io::Buffer>(f: R) -> Result<Protocol, ParseError> {
    let mut lex = Lexer { reader: f };
    for tok in lex {
        println!("{}", tok);
    }

    let mut proto = Protocol {
        types: HashMap::new(),
        categories: HashMap::new(),
    };

    Ok(proto)
}

fn try_parse_type(s: &str) -> Option<Type> {
    match s {
        "i8" => Some(I8),
        "u8" => Some(U8),
        "i16" => Some(I16),
        "u16" => Some(U16),
        "i32" => Some(I32),
        "u32" => Some(U32),
        "f32" => Some(F32),
        "i64" => Some(I64),
        "u64" => Some(U64),
        "f64" => Some(F64),
        other if other.starts_with("array<") => {
            Some(Array(box try_parse_type(other.slice(6, other.len() - 2)).unwrap()))
        }
        _ => {
            None
        }
    }
}

<commit_msg>Add COLON token<commit_after>// Copyright (c) 2014 Corey Richardson
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

use std::collections::hashmap::HashMap;
use std;

#[deriving(Show)]
pub struct Protocol {
    types: HashMap<String, Type>,
    categories: HashMap<String, Category>,
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum Type {
    I8, U8,
    I16, U16,
    I32, U32, F32,
    I64, U64, F64,
    Array(Box<Type>),
    Aggregate(Object),
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub struct Object {
    fields: Vec<(String, Type)>,
}

#[deriving(Show)]
pub struct Category {
    name: String,
    methods: HashMap<String, Method>,
}

#[deriving(Show, Encodable, Decodable)]
pub struct Method {
    comment: String,
    properties: HashMap<String, Property>,
    attributes: Vec<Attr>,
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum Property {
    In(Type),
    Out(Type)
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum ParseError {
    UnexpectedToken(String),
    UnexpectedEof(String),
    Other(String)
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
pub enum Attr {
    Auth,
    Unauth,
    Admin,
    Global,
    Map,
}

#[deriving(Show, Ord, Eq, PartialOrd, PartialEq, Hash, Encodable, Decodable)]
enum Token {
    ATTR(Attr),
    STRING(String),
    IDENT(String),
    PRIM(Type),

    NEWTYPE,
    CATEGORY,
    INCLUDE,
    METHOD,
    IN,
    OUT,

    SEMI,
    LBRACE,
    RBRACE,
    COMMA,
    EQ,
    COLON,

    NL,
}

struct Lexer<R> {
    reader: R,
}

impl<R: std::io::Buffer> Iterator<Token> for Lexer<R> {
    fn next(&mut self) -> Option<Token> {
        // IO errors are interesting and never expected.
        let mut chrs = self.reader.chars().map(|io| io.unwrap()).peekable();
        for c in chrs {
            // could be anything...
            match c {
                // string literal
                '"' => {
                    return Some(STRING(chrs.take_while(|&c| c != '"').collect()));
                }
                '\n' => {
                    return Some(NL);
                },
                '\r' => {
                    if chrs.next().unwrap() != '\n' {
                        error!("CR found not followed by LF!");
                    }
                    return Some(NL);
                }
                // keyword or identifier
                c @ 'a'..'z' | c @ 'A'..'Z' | c @ '_' => {
                    let ident = Some(c).move_iter().chain(chrs.take_while(|&c| c.is_alphanumeric() || c == '_')).collect::<String>();
                    match ident.as_slice() {
                        "auth" => return Some(ATTR(Auth)),
                        "unauth" => return Some(ATTR(Unauth)),
                        "admin" => return Some(ATTR(Admin)),
                        "global" => return Some(ATTR(Global)),
                        "map" => return Some(ATTR(Map)),
                        "newtype" => return Some(NEWTYPE),
                        "category" => return Some(CATEGORY),
                        "include" => return Some(INCLUDE),
                        "method" => return Some(METHOD),
                        "in" => return Some(IN),
                        "out" => return Some(OUT),
                        non_keyword => {
                            match try_parse_type(non_keyword) {
                                Some(t) => return Some(PRIM(t)),
                                None => return Some(IDENT(non_keyword.to_string()))
                            }
                        }
                    }
                },
                ';' => return Some(SEMI),
                ':' => return Some(COLON),
                '{' => return Some(LBRACE),
                '}' => return Some(RBRACE),
                ',' => return Some(COMMA),
                '=' => return Some(EQ),
                ' ' | '\t' => continue,
                wat => error!("Found unexpected character when lexing: {}", wat),
            }
        }
        None
    }
}

pub fn parse<R: std::io::Buffer>(f: R) -> Result<Protocol, ParseError> {
    let mut lex = Lexer { reader: f };
    for tok in lex {
        println!("{}", tok);
    }

    let mut proto = Protocol {
        types: HashMap::new(),
        categories: HashMap::new(),
    };

    Ok(proto)
}

fn try_parse_type(s: &str) -> Option<Type> {
    match s {
        "i8" => Some(I8),
        "u8" => Some(U8),
        "i16" => Some(I16),
        "u16" => Some(U16),
        "i32" => Some(I32),
        "u32" => Some(U32),
        "f32" => Some(F32),
        "i64" => Some(I64),
        "u64" => Some(U64),
        "f64" => Some(F64),
        other if other.starts_with("array<") => {
            Some(Array(box try_parse_type(other.slice(6, other.len() - 2)).unwrap()))
        }
        _ => {
            None
        }
    }
}

<|endoftext|>


<commit_before>use super::models::{LanguageModel, WordVec};
use self::Token::*;
use super::mayberef::MaybeRef::{self, Val, Ref};
use std::fmt::{Display, Formatter, Error};

pub fn parse(expr: &str, model: &LanguageModel) -> Result<WordVec, String> {
    expression(&mut Tokens::from(expr.chars()), model).map(|w| w.take())
}

#[derive(PartialEq, Eq, Debug, Clone)]
enum Token {
    RParen,
    LParen,
    Plus,
    Minus,
    Divide,
    Word(String),
    Invalid(char),
    Number(i32),
}

impl Token {
    fn apply_addop(&self, lhs: WordVec, rhs: &WordVec) -> Result<WordVec, String> {
        match *self {
            Plus => Ok(lhs + rhs),
            Minus => Ok(lhs - rhs),
            _ => Err(format!("Internal error: tried calling add using {}", self)),
        }
    }
}

impl Display for Token {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        let string;
        fmt.write_str(match *self {
            RParen => ")",
            LParen => "(",
            Plus => "+",
            Minus => "-",
            Divide => "/",
            Word(ref w) => &*w,
            Number(i) => {
                string = format!("{}", i);
                &*string
            }
            Invalid(w) => {
                string = w.to_string();
                &*string
            },
        })
    }
}

fn get_word<'a>(model: &'a LanguageModel, word: &str) -> Result<&'a WordVec, String> {
    model.get(&*word).ok_or_else(||
        format!("'{}' is not present in the language model", &word)
    )
}

fn expression<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let atom = try!(atom(tokens, model));
    match tokens.peek() {
        Some(&Plus) | Some(&Minus) => rhs(atom, tokens, model),
        None => Ok(atom),
        _ => Err(format!("Invalid token '{:?}'", tokens.peek().unwrap()))
    }
    // let token = try!(tokens.next().ok_or("An expression may not be empty"));
    // match token {
    //     LParen => {
    //         let vec = try!(expression(tokens, model));
    //         let rparen = try!(tokens.next().ok_or("Unbalanced parentheses"));
    //         if rparen != RParen {
    //             return Err(format!("Expected ')', found '{}'", rparen))
    //         }
    //         rhs(vec, tokens, model)
    //     },
    //     Word(word) => {
    //         let vec = try!(get_word(model, &*word));
    //         rhs(Ref(vec), tokens, model)
    //     },
    //     _ => Err(format!("An expression may not start with '{}'", token)),
    // }
}

fn rhs<'a, I>(lhs: MaybeRef<'a, WordVec>, tokens: &mut Tokens<I>, model: &LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    match tokens.peek() {
        Some(&RParen) | None => return Ok(lhs),
        _ => {}
    }

    let token = tokens.next().unwrap();
    match token {
        Plus | Minus => {
            let lhs = lhs.take();
            let atom = try!(atom(tokens, model));
            let result = try!(token.apply_addop(lhs, &*atom));
            rhs(Val(result), tokens, model)
        }
        Word(_) | LParen => Err(format!("'{}' found in invalid position", token)),
        Invalid(s) => Err(format!("'{}' could not be tokenized", s)),
        Divide | Number(_) => Err("Not sure how we got here!".to_string()),
        RParen => unreachable!(),
    }
}

fn atom<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    {
        let token = try!(tokens.peek().ok_or("Invalid end of expression"));
        match *token {
            Plus | Minus | Invalid(_) | RParen => {
                return Err(format!("Invalid token {}", token));
            }
            _ => {},
        }
    }

    let token = tokens.next().unwrap();
    let atom = match token {
        LParen => {
            let expr = try!(expression(tokens, model));
            match tokens.next() {
                Some(RParen) => expr,
                Some(t) => return Err(format!("Expected ')', found '{}'", t)),
                None => return Err("Unclosed parentheses".to_string()),
            }
        }
        Word(word) =>  {
            let vec = try!(get_word(model, &*word));
            Ref(vec)
        }
        _ => unreachable!(),
    };

    if tokens.peek() != Some(&Divide) {
        return Ok(atom);
    }

    tokens.next();

    let num = try!(tokens.next().ok_or(
        "An expression may not end with a division operator"));

    match num {
        Number(num) => Ok(Val(atom.take() / num)),
        _ => Err(format!("'{}' is not a valid numerator: must be an integer", num)),
    }

}

struct Tokens<I> {
    iter: I,
    next_char: Option<Option<char>>,
    next_token: Option<Option<Token>>
}

impl<I> Tokens<I> where I: Iterator<Item = char> {
    fn from(iter: I) -> Tokens<I> {
        Tokens {
            iter: iter,
            next_char: None,
            next_token: None,
        }
    }

    fn get_next_token(&mut self) -> Option<Token> {
        while let Some(' ') = self.peek_char() {
            self.take();
        }

        match self.peek_char() {
            Some('a'...'z') => return Some(self.word()),
            Some('0'...'9') => return Some(self.number()),
            // Some('0'..'9') | Some('.') => return Some(self.number()),
            None => return None,
            _ => {}
        }

        Some(match self.take().unwrap() {
            '+' => Plus,
            '-' => Minus,
            '(' => LParen,
            ')' => RParen,
            '/' => Divide,
            c => Invalid(c),
        })
    }

    fn peek(&mut self) -> Option<&Token> {
        if let Some(ref t) = self.next_token {
            return t.as_ref();
        }

        self.next_token = Some(self.get_next_token());
        return self.next_token.as_ref().unwrap().as_ref();
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.next_char {
            Some(x) => x,
            None => {
                let next = self.iter.next();
                self.next_char = Some(next);
                next
            }
        }
    }

    fn take(&mut self) -> Option<char> {
        self.peek_char();
        self.next_char.take().unwrap()
    }

    fn word(&mut self) -> Token {
        let mut token = String::new();
        while let Some('a'...'z') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Word(token)
    }

    fn number(&mut self) -> Token {
        let mut token = String::new();
        while let Some('0'...'9') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Number(token.parse().unwrap())
    }

}

impl<I> Iterator for Tokens<I> where I: Iterator<Item = char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.peek();
        self.next_token.take().unwrap()
    }
}

mod test {
    use super::Tokens;
    use super::Token::*;
    use super::parse;
    use super::super::models::{WordVec, LanguageModel, LanguageModelBuilder};

    #[test]
    fn single_character() {
        let c = Tokens::from("a".chars()).next().unwrap();
        assert_eq!(c, Word("a".to_string()));
    }

    #[test]
    fn single_token() {
        let s = Tokens::from("abc".chars()).next().unwrap();
        assert_eq!(s, Word("abc".to_string()));
    }

    #[test]
    fn two_tokens() {
        let s = Tokens::from("abc def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Word("def".to_string())]);
    }

    #[test]
    fn plus_tokens() {
        let s = Tokens::from("   abc  +def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Plus, Word("def".to_string())]);
    }

    #[test]
    fn bracket_tokens() {
        let s = Tokens::from(" ()  (abc  +)def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![LParen, RParen, LParen, Word("abc".to_string()),
            Plus, RParen, Word("def".to_string())]);
    }

    fn get_model() -> LanguageModel {
        let words = vec!("a".to_string(), "b".to_string(), "c".to_string());
        let mut builder = LanguageModelBuilder::new(1, words);

        {
            let mut acc = builder.new_file();
            let text = "x x a a b a c b x x";
            for word in text.split(' ') {
                acc.add_word(word.to_string());
            }
        }

        builder.build()
    }

    #[test]
    fn test_addition() {
        let model = get_model();
        assert_eq!(parse("a - b + c", &model).unwrap(), parse("(a - b) + c", &model).unwrap());
        assert_eq!(parse("a + (b - c)", &model).unwrap(), parse("(a + b) - c", &model).unwrap());
        assert_eq!(parse("a + b - c)", &model).unwrap(), parse("a - c + b", &model).unwrap());
        assert!(parse("a + b", &model) != parse("a - b", &model));
    }

    #[test]
    fn test_division() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        println!("{:?} == {:?}", a, a.clone() + &a);
        assert!(a == (a.clone() + &a) / 2);
    }

    #[test]
    fn test_div_expression() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        assert_eq!(a, parse("(a + a) / 2", &model).unwrap());
    }

}<commit_msg>Simplification<commit_after>use super::models::{LanguageModel, WordVec};
use self::Token::*;
use super::mayberef::MaybeRef::{self, Val, Ref};
use std::fmt::{Display, Formatter, Error};

pub fn parse(expr: &str, model: &LanguageModel) -> Result<WordVec, String> {
    expression(&mut Tokens::from(expr.chars()), model).map(|w| w.take())
}

#[derive(PartialEq, Eq, Debug, Clone)]
enum Token {
    RParen,
    LParen,
    Plus,
    Minus,
    Divide,
    Word(String),
    Invalid(char),
    Number(i32),
}

impl Token {
    fn apply_addop(&self, lhs: WordVec, rhs: &WordVec) -> Result<WordVec, String> {
        match *self {
            Plus => Ok(lhs + rhs),
            Minus => Ok(lhs - rhs),
            _ => Err(format!("Internal error: tried calling add using {}", self)),
        }
    }
}

impl Display for Token {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        let string;
        fmt.write_str(match *self {
            RParen => ")",
            LParen => "(",
            Plus => "+",
            Minus => "-",
            Divide => "/",
            Word(ref w) => &*w,
            Number(i) => {
                string = format!("{}", i);
                &*string
            }
            Invalid(w) => {
                string = w.to_string();
                &*string
            },
        })
    }
}

fn get_word<'a>(model: &'a LanguageModel, word: &str) -> Result<&'a WordVec, String> {
    model.get(&*word).ok_or_else(||
        format!("'{}' is not present in the language model", &word)
    )
}

fn expression<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let atom = try!(atom(tokens, model));
    match tokens.peek() {
        Some(&Plus) | Some(&Minus) => rhs(atom, tokens, model),
        None => Ok(atom),
        _ => Err(format!("Invalid token '{:?}'", tokens.peek().unwrap()))
    }
}

fn rhs<'a, I>(lhs: MaybeRef<'a, WordVec>, tokens: &mut Tokens<I>, model: &LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    match tokens.peek() {
        Some(&RParen) | None => return Ok(lhs),
        _ => {}
    }

    let token = tokens.next().unwrap();
    match token {
        Plus | Minus => {
            let lhs = lhs.take();
            let atom = try!(atom(tokens, model));
            let result = try!(token.apply_addop(lhs, &*atom));
            rhs(Val(result), tokens, model)
        }
        Word(_) | LParen => Err(format!("'{}' found in invalid position", token)),
        Invalid(s) => Err(format!("'{}' could not be tokenized", s)),
        Divide | Number(_) => Err("Not sure how we got here!".to_string()),
        RParen => unreachable!(),
    }
}

fn atom<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let token = try!(tokens.next().ok_or("Invalid end of expression"));

    let atom = match token {
        LParen => {
            let expr = try!(expression(tokens, model));
            match tokens.next() {
                Some(RParen) => expr,
                Some(t) => return Err(format!("Expected ')', found '{}'", t)),
                None => return Err("Unclosed parentheses".to_string()),
            }
        }
        Word(word) =>  {
            let vec = try!(get_word(model, &*word));
            Ref(vec)
        }
        _ => return Err(format!("Invalid token {}", token)),
    };

    if tokens.peek() != Some(&Divide) {
        return Ok(atom);
    }

    tokens.next();

    let num = try!(tokens.next().ok_or(
        "An expression may not end with a division operator"));

    match num {
        Number(num) => Ok(Val(atom.take() / num)),
        _ => Err(format!("'{}' is not a valid numerator: must be an integer", num)),
    }

}

struct Tokens<I> {
    iter: I,
    next_char: Option<Option<char>>,
    next_token: Option<Option<Token>>
}

impl<I> Tokens<I> where I: Iterator<Item = char> {
    fn from(iter: I) -> Tokens<I> {
        Tokens {
            iter: iter,
            next_char: None,
            next_token: None,
        }
    }

    fn get_next_token(&mut self) -> Option<Token> {
        while let Some(' ') = self.peek_char() {
            self.take();
        }

        match self.peek_char() {
            Some('a'...'z') => return Some(self.word()),
            Some('0'...'9') => return Some(self.number()),
            None => return None,
            _ => {}
        }

        Some(match self.take().unwrap() {
            '+' => Plus,
            '-' => Minus,
            '(' => LParen,
            ')' => RParen,
            '/' => Divide,
            c => Invalid(c),
        })
    }

    fn peek(&mut self) -> Option<&Token> {
        if let Some(ref t) = self.next_token {
            return t.as_ref();
        }

        self.next_token = Some(self.get_next_token());
        return self.next_token.as_ref().unwrap().as_ref();
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.next_char {
            Some(x) => x,
            None => {
                let next = self.iter.next();
                self.next_char = Some(next);
                next
            }
        }
    }

    fn take(&mut self) -> Option<char> {
        self.peek_char();
        self.next_char.take().unwrap()
    }

    fn word(&mut self) -> Token {
        let mut token = String::new();
        while let Some('a'...'z') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Word(token)
    }

    fn number(&mut self) -> Token {
        let mut token = String::new();
        while let Some('0'...'9') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Number(token.parse().unwrap())
    }

}

impl<I> Iterator for Tokens<I> where I: Iterator<Item = char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.peek();
        self.next_token.take().unwrap()
    }
}

mod test {
    use super::Tokens;
    use super::Token::*;
    use super::parse;
    use super::super::models::{WordVec, LanguageModel, LanguageModelBuilder};

    #[test]
    fn single_character() {
        let c = Tokens::from("a".chars()).next().unwrap();
        assert_eq!(c, Word("a".to_string()));
    }

    #[test]
    fn single_token() {
        let s = Tokens::from("abc".chars()).next().unwrap();
        assert_eq!(s, Word("abc".to_string()));
    }

    #[test]
    fn two_tokens() {
        let s = Tokens::from("abc def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Word("def".to_string())]);
    }

    #[test]
    fn plus_tokens() {
        let s = Tokens::from("   abc  +def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Plus, Word("def".to_string())]);
    }

    #[test]
    fn bracket_tokens() {
        let s = Tokens::from(" ()  (abc  +)def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![LParen, RParen, LParen, Word("abc".to_string()),
            Plus, RParen, Word("def".to_string())]);
    }

    fn get_model() -> LanguageModel {
        let words = vec!("a".to_string(), "b".to_string(), "c".to_string());
        let mut builder = LanguageModelBuilder::new(1, words);

        {
            let mut acc = builder.new_file();
            let text = "x x a a b a c b x x";
            for word in text.split(' ') {
                acc.add_word(word.to_string());
            }
        }

        builder.build()
    }

    #[test]
    fn test_addition() {
        let model = get_model();
        assert_eq!(parse("a - b + c", &model).unwrap(), parse("(a - b) + c", &model).unwrap());
        assert_eq!(parse("a + (b - c)", &model).unwrap(), parse("(a + b) - c", &model).unwrap());
        assert_eq!(parse("a + b - c)", &model).unwrap(), parse("a - c + b", &model).unwrap());
        assert!(parse("a + b", &model) != parse("a - b", &model));
    }

    #[test]
    fn test_division() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        println!("{:?} == {:?}", a, a.clone() + &a);
        assert!(a == (a.clone() + &a) / 2);
    }

    #[test]
    fn test_div_expression() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        assert_eq!(a, parse("(a + a) / 2", &model).unwrap());
    }

}<|endoftext|>


<commit_before>use tokenizer::Token;
use std::fmt::Debug;
use std::iter::Peekable;

trait Expr:Debug + PartialEq {
}

#[derive(PartialEq)]
#[derive(Debug)]
struct SExpr<'x> {
    elements:Vec<Box<Expr + 'x>>,
}

impl<'x> Expr for SExpr<'x> {

}

fn parse_s_expr<'a, I>(iter: I) -> SExpr<'a> where I: Iterator<Item=Token>{
    let mut elements = Vec::new();

    return SExpr { elements: elements };
}

fn parse<'a, 'b, I>(iter: I) -> Vec<Box<Expr + 'a>> where I: Iterator<Item=(&'b Token)>{
    let mut piter = iter.peekable();
    let mut exprs: Vec<Box<Expr>> = Vec::new();

    loop {
        match piter.peek() {
            Some(token) => {
                match **token {
                    Token::OpenParen => {}, //exprs.push(Box::new(parse_s_expr(iter))),
                    _ => {}
                }
            },
            None => { break }
        }
    }

    return exprs;
}

//fn parse<'a>(tokens: Vec<Token>) -> Vec<Box<Expr + 'a>> {
    //let mut iter = tokens.iter();
    //return parse_iter(tokens);
//}

#[test]
fn test_parse_empty_s_expr() {
    assert_eq!(
        parse(vec!(Token::OpenParen, Token::CloseParen).iter()),
        vec!(Box::new(SExpr {elements: vec!()}))
    );

}
<commit_msg>Working on getting peeked tokens copied<commit_after>use tokenizer::Token;
use std::fmt::Debug;
use std::iter::Peekable;

trait Expr:Debug + PartialEq {
}

#[derive(PartialEq)]
#[derive(Debug)]
struct SExpr<'x> {
    elements:Vec<Box<Expr + 'x>>,
}

impl<'x> Expr for SExpr<'x> {

}

fn parse_s_expr<'a, 'b, I>(iter: &I) -> Box<Expr + 'a> where I: Iterator<Item=(&'b Token)>{
    let mut elements = Vec::new();

    return Box::new(SExpr { elements: elements });
}

fn parse<'a, 'b, I>(iter: I) -> Vec<Box<Expr + 'a>> where I: Iterator<Item=(&'b Token)>{
    let mut piter = iter.peekable();
    let mut exprs: Vec<Box<Expr>> = Vec::new();

    loop {
        let tokenResult:Option<Token> = match piter.peek() {
            Some(token) => {
                let borrowedToken:&Token = token;
                Some(*borrowedToken.clone())
            },
            None => None
        };

        match tokenResult {
            Some(token) => {
                match token {
                    Token::OpenParen => exprs.push(parse_s_expr(&piter)),
                    _ => {
                        piter.next();
                    }
                }
            },
            None => { break }
        }
    }

    return exprs;
}

//fn parse<'a>(tokens: Vec<Token>) -> Vec<Box<Expr + 'a>> {
    //let mut iter = tokens.iter();
    //return parse_iter(tokens);
//}

#[test]
fn test_parse_empty_s_expr() {
    let expected:Box<Expr> = Box::new(SExpr {elements: vec!()});

    assert_eq!(
        parse(vec!(Token::OpenParen, Token::CloseParen).iter()),
        vec!(expected)
    );

}
<|endoftext|>