<commit_before>use nom::multispace;
use nom::{IResult, Err, ErrorKind, Needed};
use std::str;

use common::{field_list, statement_terminator, table_reference, value_list};
use parser::{Column, Table};

#[derive(Clone, Debug, Default, PartialEq)]
pub struct InsertStatement {
    pub table: Table,
    pub fields: Vec<(Column, String)>,
}

/// Parse rule for a SQL insert query.
/// TODO(malte): support REPLACE, multiple parens expr, nested selection, DEFAULT VALUES
named!(pub insertion<&[u8], InsertStatement>,
    complete!(chain!(
        caseless_tag!("insert") ~
        multispace ~
        caseless_tag!("into") ~
        multispace ~
        table: table_reference ~
        multispace ~
        fields: opt!(chain!(
                tag!("(") ~
                fields: field_list ~
                tag!(")") ~
                multispace,
                || { fields }
                )
            ) ~
        caseless_tag!("values") ~
        multispace ~
        tag!("(") ~
        values: value_list ~
        tag!(")") ~
        statement_terminator,
        || {
            // "table AS alias" isn't legal in INSERT statements
            assert!(table.alias.is_none());
            InsertStatement {
                table: table,
                fields: match fields {
                    Some(ref f) =>
                        f.iter()
                         .cloned()
                         .zip(values.into_iter()
                                    .map(|s| String::from(s))
                                    )
                         .collect(),
                    None =>
                        values.into_iter()
                              .enumerate()
                              .map(|(i, v)| (Column::from(format!("{}", i).as_str()), String::from(v)))
                              .collect(),
                },
            }
        }
    ))
);

#[cfg(test)]
mod tests {
    use super::*;
    use parser::{Column, Table};

    #[test]
    fn simple_insert() {
        let qstring = "INSERT INTO users VALUES (42, test);";

        let res = insertion(qstring.as_bytes());
        assert_eq!(res.unwrap().1,
                   InsertStatement {
                       table: Table::from("users"),
                       fields: vec![(Column::from("0"), "42".into()), (Column::from("1"), "test".into())],
                       ..Default::default()
                   });
    }

    #[test]
    fn placeholder_insert() {
        let qstring = "INSERT INTO users VALUES (?, ?);";

        let res = insertion(qstring.as_bytes());
        assert_eq!(res.unwrap().1,
                   InsertStatement {
                       table: Table::from("users"),
                       fields: vec![(Column::from("0"), "?".into()), (Column::from("1"), "?".into())],
                       ..Default::default()
                   });
    }

    #[test]
    fn insert_with_field_names() {
        let qstring = "INSERT INTO users (id, name) VALUES (42, test);";

        let res = insertion(qstring.as_bytes());
        assert_eq!(res.unwrap().1,
                   InsertStatement {
                       table: Table::from("users"),
                       fields: vec![(Column::from("id"), "42".into()), (Column::from("name"), "test".into())],
                       ..Default::default()
                   });
    }
}
<commit_msg>Allow spaces at start/end of parenthetical lists.<commit_after>use nom::multispace;
use nom::{IResult, Err, ErrorKind, Needed};
use std::str;

use common::{field_list, statement_terminator, table_reference, value_list};
use parser::{Column, Table};

#[derive(Clone, Debug, Default, PartialEq)]
pub struct InsertStatement {
    pub table: Table,
    pub fields: Vec<(Column, String)>,
}

/// Parse rule for a SQL insert query.
/// TODO(malte): support REPLACE, multiple parens expr, nested selection, DEFAULT VALUES
named!(pub insertion<&[u8], InsertStatement>,
    complete!(chain!(
        caseless_tag!("insert") ~
        multispace ~
        caseless_tag!("into") ~
        multispace ~
        table: table_reference ~
        multispace ~
        fields: opt!(chain!(
                tag!("(") ~
                multispace? ~
                fields: field_list ~
                multispace? ~
                tag!(")") ~
                multispace,
                || { fields }
                )
            ) ~
        caseless_tag!("values") ~
        multispace ~
        tag!("(") ~
        values: value_list ~
        tag!(")") ~
        statement_terminator,
        || {
            // "table AS alias" isn't legal in INSERT statements
            assert!(table.alias.is_none());
            InsertStatement {
                table: table,
                fields: match fields {
                    Some(ref f) =>
                        f.iter()
                         .cloned()
                         .zip(values.into_iter()
                                    .map(|s| String::from(s))
                                    )
                         .collect(),
                    None =>
                        values.into_iter()
                              .enumerate()
                              .map(|(i, v)| (Column::from(format!("{}", i).as_str()), String::from(v)))
                              .collect(),
                },
            }
        }
    ))
);

#[cfg(test)]
mod tests {
    use super::*;
    use parser::{Column, Table};

    #[test]
    fn simple_insert() {
        let qstring = "INSERT INTO users VALUES (42, test);";

        let res = insertion(qstring.as_bytes());
        assert_eq!(res.unwrap().1,
                   InsertStatement {
                       table: Table::from("users"),
                       fields: vec![(Column::from("0"), "42".into()), (Column::from("1"), "test".into())],
                       ..Default::default()
                   });
    }

    #[test]
    fn placeholder_insert() {
        let qstring = "INSERT INTO users VALUES (?, ?);";

        let res = insertion(qstring.as_bytes());
        assert_eq!(res.unwrap().1,
                   InsertStatement {
                       table: Table::from("users"),
                       fields: vec![(Column::from("0"), "?".into()), (Column::from("1"), "?".into())],
                       ..Default::default()
                   });
    }

    #[test]
    fn insert_with_field_names() {
        let qstring = "INSERT INTO users (id, name) VALUES (42, test);";

        let res = insertion(qstring.as_bytes());
        assert_eq!(res.unwrap().1,
                   InsertStatement {
                       table: Table::from("users"),
                       fields: vec![(Column::from("id"), "42".into()), (Column::from("name"), "test".into())],
                       ..Default::default()
                   });
    }
}
<|endoftext|>


<commit_before>use clap::{ArgMatches};
use std::path::Path;
use db;
use std::fs::File;
use std::io::Read;

use rustc_serialize::Encodable;
use rustc_serialize::json::{self, Encoder};
use rustc_serialize::json::Json;

#[derive(Debug)]
struct Person {
    id: i32,
    name: String,
    data: Option<Vec<u8>>
}

pub fn listy(direcory: &str) {
    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            println!("aaaa{:?}", entry.path());
        }
    }
}

pub fn job_files_list(direcory: &str)  {
    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            println!("job_files_list{:?}", entry.path());
        }
    }
}

pub fn listy2(direcory: &str) -> Vec<String>{
    println!("listy2{:?}", direcory);
    let mut items = Vec::<String>::new();

    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            let path = entry.path();
            if path.is_dir() {
                continue;
            }
            let pb_2 = path.as_path();
            let as_path_buf = pb_2.as_os_str();
            let path = String::new() + as_path_buf.to_str().unwrap() ;
            items.push(path);

        }
    }
    return items;
}

fn elephanc() {
    let conn = db::connect();
    db::create_tables(&conn);
    db::insert_fs_dir(&conn, "Steven".to_string());
    db::insert_fs_dir(&conn, "Steven2".to_string());
    db::insert_fs_file(&conn, 0, "Steven".to_string());
    db::insert_provider(&conn, "Steven".to_string());
    db::provider_list(&conn);
    db::insert_job(&conn, "Steven".to_string(), "".to_string());
    db::insert_job_provide(&conn, 1, 1);
    db::insert_job_depend(&conn, 1, 1,10);
    db::insert_job_variable_name(&conn, "Steven".to_string());
    db::insert_job_require_variable(&conn, 1, 1);
    db::insert_job_variable_pair(&conn, 1, "Steven".to_string());
    db::insert_job_require_variable_pair(&conn, 1, 1);
    conn.execute("CREATE TABLE person (
                  id              INTEGER PRIMARY KEY,
                  name            TEXT NOT NULL,
                  data            BLOB
                  )", &[]).unwrap();
    let me = Person {
        id: 0,
        name: "Steven".to_string(),
        data: None
    };
    conn.execute("INSERT INTO person (name, data)
                  VALUES (?1, ?2)",
                 &[&me.name, &me.data]).unwrap();

    let mut stmt = conn.prepare("SELECT id, name, data FROM person").unwrap();
    let person_iter = stmt.query_map(&[], |row| {
        Person {
            id: row.get(0),
            name: row.get(1),
            data: row.get(2)
        }
    }).unwrap();

    for person in person_iter {
        println!("Found person {:?}", person.unwrap());
    }
}

pub fn deligate(matches : ArgMatches) {

    let conn = db::connect();
    db::create_tables(&conn);

    if let Some(in_v) = matches.values_of("dir-scripts") {
        for in_file in in_v {
            //println!("An input dir-scripts: {}", in_file);
           listy(&in_file);

        }
    }

        // If we specified the multiple() setting we can get all the values
    if let Some(in_v) = matches.values_of("env") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
            job_files_list(&in_file);
        }
    }

    if let Some(in_v) = matches.values_of("dir-jobs") {
        for in_file in in_v {
            db::insert_fs_dir(&conn, in_file.to_string());
        }
    }
    for directory in db::list_fs_dir(&conn) {
        let foo = directory.id;
        let foo_name = directory.name;
        let file_list = listy2(&foo_name);
        for fnccc in file_list {
            println!("An input file: {}", fnccc);
            db::insert_fs_file(&conn, foo, fnccc);
        }
    }
    for filename in db::list_fs_file(&conn) {
        println!("list_fs_file{:?}", filename);
        let name = filename.name;
        let mut f = File::open(name).expect("file not found");

        let mut contents = String::new();
        f.read_to_string(&mut contents).expect("something went wrong reading the file");

        let json = Json::from_str(&contents).unwrap();
        //println!("json={}", json);
        let found = json.find_path(&["provides"]);
        if found != None {
            for item in found {
                println!("provides={}", item);
                let as_a_str = item.to_string();
                db::insert_provider(&conn, as_a_str);
            }
        }
        let found = json.find_path(&["name"]);
        if found != None {
            for item in found {
                println!("name={}", item);
            }
        }
        let found = json.find_path(&["script"]);
        if found != None {
            for item in found {
                println!("script={}", item);
            }
        }
        let found = json.find_path(&["variables"]);
        if found != None {

            for item in found {
                println!("variables={}", item);
                let bill = item.find_path(&["require_keys"]);
                if bill != None {
                    for item2 in bill {
                        println!("variables::require_keys={}", item2);
                    }
                }
                let bill = item.find_path(&["provides_keys"]);
                if bill != None {
                    for item2 in bill {
                        println!("variables::provides_keys={}", item2);
                    }
                }
                let bill = item.find_path(&["require_values"]);
                if bill != None {
                    for item2 in bill {
                        let ben = item2.find_path(&[""]);

                        for itme3 in ben {
                            println!("variables::itme3={}", itme3);
                        }
                    }
                }
            }

        }
        let found = json.find_path(&["depends"]);
        if found != None {
            for item in found {
                println!("depends={}", item);
            }
        }

    }
    for filename in db::list_provider(&conn) {
        println!("list_provider{:?}", filename);
    }
}

<commit_msg>Slowly learning json and data structures<commit_after>use clap::{ArgMatches};
use std::path::Path;
use db;
use std::fs::File;
use std::io::Read;

use rustc_serialize::{Encodable};
use rustc_serialize::json::{self, Encoder};
use rustc_serialize::json::Json;

use std::collections::HashMap;
use std::result::Result;
use rusqlite::Connection;
use rusqlite::Error;

use dbFsFile;

#[derive(Debug)]
struct Person {
    id: i32,
    name: String,
    data: Option<Vec<u8>>
}

pub fn listy(direcory: &str) {
    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            println!("aaaa{:?}", entry.path());
        }
    }
}

pub fn job_files_list(direcory: &str)  {
    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            println!("job_files_list{:?}", entry.path());
        }
    }
}

pub fn listy2(direcory: &str) -> Vec<String>{
    println!("listy2{:?}", direcory);
    let mut items = Vec::<String>::new();

    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            let path = entry.path();
            if path.is_dir() {
                continue;
            }
            let pb_2 = path.as_path();
            let as_path_buf = pb_2.as_os_str();
            let path = String::new() + as_path_buf.to_str().unwrap() ;
            items.push(path);

        }
    }
    return items;
}

pub fn json_loader_old(name: &str)  {
    let mut f = File::open(name).expect("file not found");

    let mut contents = String::new();
    f.read_to_string(&mut contents).expect("something went wrong reading the file");

    let json = Json::from_str(&contents).unwrap();
    //println!("json={}", json);
    let found = json.find_path(&["provides"]);
    if found != None {
        for item in found {
            println!("provides={}", item);
            let as_a_str = item.to_string();
            //db::insert_provider(&conn, as_a_str);
        }
    }
    let found = json.find_path(&["name"]);
    if found != None {
        for item in found {
            println!("name={}", item);
        }
    }
    let found = json.find_path(&["script"]);
    if found != None {
        for item in found {
            println!("script={}", item);
        }
    }
    let found = json.find_path(&["variables"]);
    if found != None {

        for item in found {
            println!("variables={}", item);
            let bill = item.find_path(&["require_keys"]);
            if bill != None {
                for item2 in bill {
                    println!("variables::require_keys={}", item2);
                }
            }
            let bill = item.find_path(&["provides_keys"]);
            if bill != None {
                for item2 in bill {
                    println!("variables::provides_keys={}", item2);
                }
            }
            let bill = item.find_path(&["require_values"]);
            if bill != None {
                for item2 in bill {
                    let ben = item2.find_path(&[""]);

                    for itme3 in ben {
                        println!("variables::itme3={}", itme3);
                    }
                }
            }
        }

    }
    let found = json.find_path(&["depends"]);
    if found != None {
        for item in found {
            println!("depends={}", item);
        }
    }

}

pub fn loader(name: &str) -> String {
    let mut f = File::open(name).expect("file not found");

    let mut contents = String::new();
    f.read_to_string(&mut contents).expect("something went wrong reading the file");

    return contents;

}

pub fn json_loader_name(conn: &Connection, content: &str)  {
    let mut contents = String::new();
    let json = Json::from_str(&content);

    match json {
        Ok(json) => {
            let found = json.find_path(&["name"]);
            if found != None {
                for item in found {

                    db::insert_job(conn, item.to_string());

                    //println!("name={}", item);
                }
            }
        }
        Err(_)=> {}
    }
    //return json.unwrap();
}

pub fn json_loader_provides_keys(conn: &Connection, content: &str)  {
    let mut contents = String::new();
    let json = Json::from_str(&content);

    match json {
        Ok(json) => {
            let found = json.find_path(&["provides_keys"]);
            if found != None {
                for item in found {
                    let str_item = item.to_string();
                    let str_item_clone1 = str_item.clone();
                    db::insert_provider(conn, item.to_string());

                    //println!("provides_keys={}", item);
                }
            }
        }
        Err(_)=> {}
    }
    //return json.unwrap();
}

pub fn json_loader_require_keys(conn: &Connection, content: &str)  {
    let mut contents = String::new();
    let json = Json::from_str(&content);
    match json {
        Ok(json) => {
            let found = json.find_path(&["require_keys"]);
            if found != None {
                for item in found {

                    db::insert_job_variable_name(conn, item.to_string());

                    //println!("require_keys={}", item);
                }
            }
        }
        Err(_)=> {}
    }
    //return json.unwrap();
}

pub fn deligate(matches : ArgMatches) {

    let conn = db::connect();
    db::create_tables(&conn);

    if let Some(in_v) = matches.values_of("dir-scripts") {
        for in_file in in_v {
            //println!("An input dir-scripts: {}", in_file);
           listy(&in_file);

        }
    }

        // If we specified the multiple() setting we can get all the values
    if let Some(in_v) = matches.values_of("env") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
            job_files_list(&in_file);
        }
    }

    if let Some(in_v) = matches.values_of("dir-jobs") {
        for in_file in in_v {
            let filename = in_file.to_string();
            db::insert_fs_dir(&conn, 0, filename);
        }
    }
    for directory in db::list_fs_dir(&conn) {
        let foo = directory.id;
        let foo_name = directory.name;
        let file_list = listy2(&foo_name);
        for fnccc in file_list {
            println!("An input file: {}", fnccc);
            let s1 = fnccc.clone();
            db::insert_fs_file(&conn, foo, fnccc);
            let mut pk = 10;
            db::pk_fs_file_by_name(&conn, s1, &mut pk);
            println!("ssss: {}", pk);
        }
    }

    let mut scores = HashMap::new();
    for filename in db::list_fs_file(&conn) {
        let mut name = String::from(filename.name);
        let name2 = name.clone();
        let loader_rc = loader(name2.trim());
        scores.insert(name,loader_rc );
    }

    for filename in db::list_fs_file(&conn) {
        let name = filename.name;
        json_loader_name(&conn, &loader(&name));

    }
    for filename in db::list_fs_file(&conn) {
        let name = filename.name;
        json_loader_provides_keys(&conn, &loader(&name));

    }

    for filename in db::list_fs_file(&conn) {
        let name = filename.name;
        json_loader_require_keys(&conn, &loader(&name));

    }

    for filename in db::list_fs_file(&conn) {
        let name = filename.name;
        json_loader_require_keys(&conn, &loader(&name));

    }

    for filename in db::list_provider(&conn) {
        println!("list_provider{:?}", filename);
    }
}

<|endoftext|>


<commit_before>
use rusqlite::Connection;

use clap::{ArgMatches};
use std::path::Path;

#[derive(Debug)]
struct Person {
    id: i32,
    name: String,
    data: Option<Vec<u8>>
}

#[derive(Debug)]
struct Provider {
    id: i32,
    name: String,
}

#[derive(Debug)]
struct Job {
    id: i32,
    name: String,
    shell: String,
}

#[derive(Debug)]
struct Variable_ID {
    id: i32,
    name: String
}

#[derive(Debug)]
struct Variable_Value {
    id: i32,
    value: String
}

#[derive(Debug)]
struct Variable_Pair {
    id: i32,
    variable_id: i32,
    value: String
}

#[derive(Debug)]
struct JobRequireVariable_ID {
    id: i32,
    variable_id: i32,
}

#[derive(Debug)]
struct JobRequireVariable_Pair {
    id: i32,
    variable_pair: i32,
}

#[derive(Debug)]
struct JobProvide {
    id: i32,
    provider: i32,
    job: i32,
}

pub fn listy(direcory: &str) {
    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            println!("{:?}", entry.path());
        }
    }
}

fn elephanc() {
    let conn = Connection::open_in_memory().unwrap();

    conn.execute("CREATE TABLE person (
                  id              INTEGER PRIMARY KEY,
                  name            TEXT NOT NULL,
                  data            BLOB
                  )", &[]).unwrap();
    let me = Person {
        id: 0,
        name: "Steven".to_string(),
        data: None
    };
    conn.execute("INSERT INTO person (name, data)
                  VALUES (?1, ?2)",
                 &[&me.name, &me.data]).unwrap();

    let mut stmt = conn.prepare("SELECT id, name, data FROM person").unwrap();
    let person_iter = stmt.query_map(&[], |row| {
        Person {
            id: row.get(0),
            name: row.get(1),
            data: row.get(2)
        }
    }).unwrap();

    for person in person_iter {
        println!("Found person {:?}", person.unwrap());
    }
}

pub fn deligate(matches : ArgMatches) {
    if let Some(in_v) = matches.values_of("dir-scripts") {
        for in_file in in_v {
            println!("An input dir-scripts: {}", in_file);
           listy(&in_file);
        }
    }

        // If we specified the multiple() setting we can get all the values
    if let Some(in_v) = matches.values_of("env") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
        }
    }

    if let Some(in_v) = matches.values_of("dir-jobs") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
            listy(&in_file);
        }
    }
    elephanc();
}

<commit_msg>Use initial db structure<commit_after>

use clap::{ArgMatches};
use std::path::Path;

use db;

#[derive(Debug)]
struct Person {
    id: i32,
    name: String,
    data: Option<Vec<u8>>
}

pub fn listy(direcory: &str) {
    let path = Path::new(direcory);
    for entry in path.read_dir().expect("read_dir call failed") {
        if let Ok(entry) = entry {
            println!("{:?}", entry.path());
        }
    }
}

fn elephanc() {
    let conn = db::connect();
    db::create_tables(&conn);
    db::provider_list(&conn);
    db::insert_provider(&conn, "Steven".to_string());
    db::provider_list(&conn);
    db::insert_job(&conn, "Steven".to_string(), "".to_string());
    db::insert_job_provide(&conn, 1, 1);
    db::insert_job_depend(&conn, 1, 1,10);
    conn.execute("CREATE TABLE person (
                  id              INTEGER PRIMARY KEY,
                  name            TEXT NOT NULL,
                  data            BLOB
                  )", &[]).unwrap();
    let me = Person {
        id: 0,
        name: "Steven".to_string(),
        data: None
    };
    conn.execute("INSERT INTO person (name, data)
                  VALUES (?1, ?2)",
                 &[&me.name, &me.data]).unwrap();

    let mut stmt = conn.prepare("SELECT id, name, data FROM person").unwrap();
    let person_iter = stmt.query_map(&[], |row| {
        Person {
            id: row.get(0),
            name: row.get(1),
            data: row.get(2)
        }
    }).unwrap();

    for person in person_iter {
        println!("Found person {:?}", person.unwrap());
    }
}

pub fn deligate(matches : ArgMatches) {
    if let Some(in_v) = matches.values_of("dir-scripts") {
        for in_file in in_v {
            println!("An input dir-scripts: {}", in_file);
           listy(&in_file);
        }
    }

        // If we specified the multiple() setting we can get all the values
    if let Some(in_v) = matches.values_of("env") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
        }
    }

    if let Some(in_v) = matches.values_of("dir-jobs") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
            listy(&in_file);
        }
    }
    elephanc();
}

<|endoftext|>


<commit_before>#[macro_export]
macro_rules! string_enum {
    ($ty:ident: $($key:ident => $value:expr,)*) => {
        #[allow(non_camel_case_types)]
        #[derive(Copy, Clone, Debug, PartialEq, Eq)]
        pub enum $ty {
            $($key),*
        }

        impl<'a> Into<&'static str> for &'a $ty {
            fn into(self) -> &'static str {
                match *self {
                    $($ty::$key => $value),*
                }
            }
        }

        impl $ty {
            fn try_from(str: &str) -> Option<Self> {
                match str {
                    $($value => Some($ty::$key),)*
                    _ => None,
                }
            }
        }

        impl ::serde::de::Deserialize for $ty {
            fn deserialize<D: ::serde::de::Deserializer>(d: &mut D) -> Result<Self, D::Error> {
                struct V;

                impl ::serde::de::Visitor for V {
                    type Value = $ty;

                    fn visit_str<E: ::serde::de::Error>(&mut self, value: &str) -> Result<Self::Value, E> {
                        $ty::try_from(value).ok_or_else(|| E::syntax("unknown value"))
                    }
                }

                d.visit(V)
            }
        }

        impl ::serde::ser::Serialize for $ty {
            fn serialize<S: ::serde::ser::Serializer>(&self, s: &mut S) -> Result<(), S::Error> {
                ::serde::ser::Serialize::serialize(Into::<&'static str>::into(self), s)
            }
        }
    };
}
<commit_msg>Don't export that macro...<commit_after>macro_rules! string_enum {
    ($ty:ident: $($key:ident => $value:expr,)*) => {
        #[allow(non_camel_case_types)]
        #[derive(Copy, Clone, Debug, PartialEq, Eq)]
        pub enum $ty {
            $($key),*
        }

        impl<'a> Into<&'static str> for &'a $ty {
            fn into(self) -> &'static str {
                match *self {
                    $($ty::$key => $value),*
                }
            }
        }

        impl $ty {
            fn try_from(str: &str) -> Option<Self> {
                match str {
                    $($value => Some($ty::$key),)*
                    _ => None,
                }
            }
        }

        impl ::serde::de::Deserialize for $ty {
            fn deserialize<D: ::serde::de::Deserializer>(d: &mut D) -> Result<Self, D::Error> {
                struct V;

                impl ::serde::de::Visitor for V {
                    type Value = $ty;

                    fn visit_str<E: ::serde::de::Error>(&mut self, value: &str) -> Result<Self::Value, E> {
                        $ty::try_from(value).ok_or_else(|| E::syntax("unknown value"))
                    }
                }

                d.visit(V)
            }
        }

        impl ::serde::ser::Serialize for $ty {
            fn serialize<S: ::serde::ser::Serializer>(&self, s: &mut S) -> Result<(), S::Error> {
                ::serde::ser::Serialize::serialize(Into::<&'static str>::into(self), s)
            }
        }
    };
}
<|endoftext|>


<commit_before>use rand;
use rand::Rng;
use rand::distributions::normal::StandardNormal;
use nalgebra::{DMatrix,DVector, IterableMut};

/// Artificial Neural Network
///
/// This struct represents a simple Artificial Neural Network (ANN) using
/// feedforward and backpropagation. It uses Stochastic Gradient Descent(SGD)
/// and a sigmoid activation function.
///
/// [Source](http://neuralnetworksanddeeplearning.com/chap1.html)
///
/// # Example
///
/// ```rust
/// // This will create a new ANN with the following topology:
/// // 3 "neurons" in the input layer
/// // 5 "neurons" in the first hidden layer
/// // 3 "neurons" in the second hidden layer
/// // 2 "neurons" in the output layer
/// let nnet = Network::new(vec![3, 5, 2]);
/// ```
#[derive(Debug, Clone)]
pub struct Network {
    /// a Vec outlining the topology of the ANN
    /// the first entry corresponds to the inputlayer,
    /// the intermediate entries correspond to the hidden layers
    /// and the last entry corresponds to the outputlayer.
    layers: Vec<u8>,
    /// a Vec that contains the weights of the respective layer
    weights: Vec<DMatrix<f32>>,
    /// a Vec cointaining the biases of the respective layer
    biases: Vec<DVector<f32>>,
}

impl Network {
    /// build a new Network with a topology Vector
    pub fn new(sizes: Vec<u8>) -> Result<Network, &'static str> {
        if sizes.len() < 3 {
            return Err("Not enough layers");
        }

        // Store the weights and biases in lists
        // We will not need weights or biases for input layer, so ignore that (hence -1)
        let mut weights: Vec<DMatrix<f32>> = Vec::with_capacity(sizes.len() - 1);
        let mut biases: Vec<DVector<f32>> = Vec::with_capacity(sizes.len() - 1);

        let mut rng = rand::thread_rng();

        // we use the standard normal distribution to initialize weights and biases - why?
        for (i, layer) in sizes.iter().enumerate().skip(1) {
            // initialize weight matrices
            weights.push(DMatrix::from_fn(*layer as usize, sizes[i - 1] as usize, |_, _| {
                let StandardNormal(x) = rng.gen();
                x as f32
            }));

            // initialize biases
            biases.push(DVector::from_fn(*layer as usize, |_| {
                let StandardNormal(x) = rng.gen();
                x as f32
            }));
        }

        Ok(Network {
            layers: sizes,
            weights: weights,
            biases: biases,
        })
    }

    /// Feed input through network, return output layer activation level
    pub fn feedforward(&self, mut a:  DVector<f32>) -> DVector<f32> {
        for (weight, bias) in self.weights.iter().zip(self.biases.iter()) {
            // TODO: get rid of ugly clone
            a = sigmoid(weight * a + bias.clone());
        }
        a
    }

    /// return the layers used to initialize the ANN
    pub fn get_layers(&self) -> &Vec<u8> {
        &self.layers
    }

    /// return a vector of the weight matrices of the ANN
    pub fn get_weights(&self) -> &Vec<DMatrix<f32>> {
        &self.weights
    }

    /// return a vector of the bias matrices of the ANN
    pub fn get_biases(&self) -> &Vec<DVector<f32>> {
        &self.biases
    }
}

// calculate elementwise sigmoid function
fn sigmoid(arr: DVector<f32>) -> DVector<f32> {
    let mut sig = arr.clone();
    for elem in sig.iter_mut() {
        *elem = 1.0/(1.0+(elem).exp());
    }
    sig
}
<commit_msg>Added unit test for sigmoid<commit_after>use rand;
use rand::Rng;
use rand::distributions::normal::StandardNormal;
use nalgebra::{DMatrix,DVector, IterableMut};

/// Artificial Neural Network
///
/// This struct represents a simple Artificial Neural Network (ANN) using
/// feedforward and backpropagation. It uses Stochastic Gradient Descent(SGD)
/// and a sigmoid activation function.
///
/// [Source](http://neuralnetworksanddeeplearning.com/chap1.html)
///
/// # Example
///
/// ```rust
/// // This will create a new ANN with the following topology:
/// // 3 "neurons" in the input layer
/// // 5 "neurons" in the first hidden layer
/// // 3 "neurons" in the second hidden layer
/// // 2 "neurons" in the output layer
/// let nnet = Network::new(vec![3, 5, 2]);
/// ```
#[derive(Debug, Clone)]
pub struct Network {
    /// a Vec outlining the topology of the ANN
    /// the first entry corresponds to the inputlayer,
    /// the intermediate entries correspond to the hidden layers
    /// and the last entry corresponds to the outputlayer.
    layers: Vec<u8>,
    /// a Vec that contains the weights of the respective layer
    weights: Vec<DMatrix<f32>>,
    /// a Vec cointaining the biases of the respective layer
    biases: Vec<DVector<f32>>,
}

impl Network {
    /// build a new Network with a topology Vector
    pub fn new(sizes: Vec<u8>) -> Result<Network, &'static str> {
        if sizes.len() < 3 {
            return Err("Not enough layers");
        }

        // Store the weights and biases in lists
        // We will not need weights or biases for input layer, so ignore that (hence -1)
        let mut weights: Vec<DMatrix<f32>> = Vec::with_capacity(sizes.len() - 1);
        let mut biases: Vec<DVector<f32>> = Vec::with_capacity(sizes.len() - 1);

        let mut rng = rand::thread_rng();

        // we use the standard normal distribution to initialize weights and biases - why?
        for (i, layer) in sizes.iter().enumerate().skip(1) {
            // initialize weight matrices
            weights.push(DMatrix::from_fn(*layer as usize, sizes[i - 1] as usize, |_, _| {
                let StandardNormal(x) = rng.gen();
                x as f32
            }));

            // initialize biases
            biases.push(DVector::from_fn(*layer as usize, |_| {
                let StandardNormal(x) = rng.gen();
                x as f32
            }));
        }

        Ok(Network {
            layers: sizes,
            weights: weights,
            biases: biases,
        })
    }

    /// Feed input through network, return output layer activation level
    pub fn feedforward(&self, mut a:  DVector<f32>) -> DVector<f32> {
        for (weight, bias) in self.weights.iter().zip(self.biases.iter()) {
            // TODO: get rid of ugly clone
            a = sigmoid(weight * a + bias.clone());
        }
        a
    }

    /// return the layers used to initialize the ANN
    pub fn get_layers(&self) -> &Vec<u8> {
        &self.layers
    }

    /// return a vector of the weight matrices of the ANN
    pub fn get_weights(&self) -> &Vec<DMatrix<f32>> {
        &self.weights
    }

    /// return a vector of the bias matrices of the ANN
    pub fn get_biases(&self) -> &Vec<DVector<f32>> {
        &self.biases
    }
}

// calculate elementwise sigmoid function
fn sigmoid(arr: DVector<f32>) -> DVector<f32> {
    let mut sig = arr.clone();
    for elem in sig.iter_mut() {
        *elem = 1.0/(1.0+(elem).exp());
    }
    sig
}

#[test]
fn test_sigmoid() {
    let mut arr_orig = DVector::from_element(3, 1.0f32);
    arr_orig[2] = 2.4137;
    let arr = sigmoid(arr_orig);
    assert_eq!(arr[0], arr[1]);
    assert_eq!(arr[0], 0.26894142137f32);
    assert_eq!(0.082133951f32, arr[2]);
}
<|endoftext|>


<commit_before>use std;
use std::ffi::CString;
use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::ptr;
use std::slice;
use std::str;

use class::{ClassDefId, ClassSize};
use ctxt::SemContext;
use gc::Address;
use gc::root::IndirectObj;
use handle::Rooted;
use mem;
use vtable::VTable;

#[repr(C)]
pub struct Header {
    // ptr to class
    vtable: *mut VTable,
}

impl Header {
    pub fn size() -> i32 {
        std::mem::size_of::<Header>() as i32
    }

    pub fn vtblptr(&self) -> *mut VTable {
        self.vtable
    }

    pub fn vtbl(&self) -> &mut VTable {
        unsafe { &mut *self.vtable }
    }

    pub fn forward_to(&mut self, address: Address) {
        self.vtable = (address.to_usize() | 1) as *mut VTable;
    }

    pub fn forwarded(&self) -> Option<Address> {
        let addr = self.vtable as usize;

        if (addr & 1) == 1 {
            Some((addr & !1).into())
        } else {
            None
        }
    }
}

// is used to reference any object
#[repr(C)]
pub struct Obj {
    header: Header,
    data: u8,
}

impl Obj {
    pub fn header(&self) -> &Header {
        &self.header
    }

    pub fn header_mut(&mut self) -> &mut Header {
        &mut self.header
    }

    pub fn data(&self) -> *const u8 {
        &self.data as *const u8
    }

    pub fn is_array_ref(&self) -> bool {
        let cls = self.header().vtbl().class();

        match cls.size {
            ClassSize::ObjArray => true,
            _ => false,
        }
    }

    pub fn size(&self) -> usize {
        let cls = self.header().vtbl().class();

        match cls.size {
            ClassSize::Fixed(size) => size as usize,

            ClassSize::ObjArray => determine_array_size(self, mem::ptr_width()),

            ClassSize::Array(element_size) => determine_array_size(self, element_size),

            ClassSize::Str => {
                let handle: Handle<Str> = Handle { ptr: self as *const Obj as *const Str };
                mem::align_usize(handle.size(), mem::ptr_width() as usize)
            }
        }
    }

    pub fn visit_reference_fields<F>(&mut self, mut f: F)
    where
        F: FnMut(IndirectObj),
    {
        let classptr = self.header().vtbl().classptr;
        let cls = unsafe { &*classptr };

        if let ClassSize::ObjArray = cls.size {
            let array = unsafe { &*(self as *const _ as *const StrArray) };

            // walk through all objects in array
            let mut ptr = array.data() as *mut *mut Obj;
            let last = unsafe { ptr.offset(array.len() as isize) };

            while ptr < last {
                f((ptr as usize).into());

                unsafe { ptr = ptr.offset(1) }
            }

            return;
        }

        for &offset in &cls.ref_fields {
            let obj = (self as *mut Obj as usize) + offset as usize;
            f(obj.into());
        }
    }

    pub fn copy_to(&self, dest: Address, size: usize) {
        unsafe {
            ptr::copy_nonoverlapping(
                self as *const Obj as *const u8,
                dest.to_mut_ptr::<u8>(),
                size,
            );
        }
    }
}

fn determine_array_size(obj: &Obj, element_size: i32) -> usize {
    let handle: Handle<ByteArray> = Handle { ptr: obj as *const Obj as *const ByteArray };

    let value = Header::size() as usize + mem::ptr_width() as usize +
        element_size as usize * handle.len() as usize;

    mem::align_usize(value, mem::ptr_width() as usize)
}

#[repr(C)]
pub struct Handle<T> {
    ptr: *const T,
}

unsafe impl<T> Send for Handle<T> {}

impl<T> Handle<T> {
    pub fn null() -> Handle<T> {
        Handle { ptr: ptr::null() }
    }

    pub fn cast<R>(&self) -> Handle<R> {
        Handle { ptr: self.ptr as *const R }
    }

    pub fn raw(&self) -> *const T {
        self.ptr
    }
}

// known limitation of #[derive(Copy, Clone)]
// traits need to be implemented manually
impl<T> Copy for Handle<T> {}
impl<T> Clone for Handle<T> {
    fn clone(&self) -> Handle<T> {
        *self
    }
}

impl<T> Deref for Handle<T> {
    type Target = T;

    fn deref(&self) -> &T {
        unsafe { &*self.ptr }
    }
}

impl<T> DerefMut for Handle<T> {
    fn deref_mut(&mut self) -> &mut T {
        unsafe { &mut *(self.ptr as *mut T) }
    }
}

impl<T> Into<Handle<T>> for usize {
    fn into(self) -> Handle<T> {
        Handle { ptr: self as *const T }
    }
}

#[repr(C)]
pub struct Testing {
    header: Header,
    failed: bool,
}

impl Testing {
    pub fn has_failed(&self) -> bool {
        self.failed
    }
}

#[repr(C)]
pub struct Str {
    header: Header,
    length: usize,
    data: u8,
}

impl Str {
    pub fn header(&self) -> &Header {
        &self.header
    }

    pub fn header_mut(&mut self) -> &mut Header {
        &mut self.header
    }

    pub fn len(&self) -> usize {
        self.length
    }

    pub fn data(&self) -> *const u8 {
        &self.data as *const u8
    }

    pub fn content(&self) -> &[u8] {
        unsafe { slice::from_raw_parts(self.data(), self.len()) }
    }

    pub fn to_cstring(&self) -> CString {
        let view = self.content();

        CString::new(view).unwrap()
    }

    pub fn size(&self) -> usize {
        Header::size() as usize         // Object header
            + mem::ptr_width() as usize // length field
            + self.len() // string content
    }

    /// allocates string from buffer in permanent space
    pub fn from_buffer_in_perm(ctxt: &SemContext, buf: &[u8]) -> Handle<Str> {
        let mut handle = str_alloc_perm(ctxt, buf.len());
        handle.length = buf.len();

        unsafe {
            let data = handle.data() as *mut u8;

            // copy buffer content into Str
            ptr::copy_nonoverlapping(buf.as_ptr(), data, buf.len());
        }

        handle
    }

    /// allocates string from buffer in heap
    pub fn from_buffer(ctxt: &SemContext, buf: &[u8]) -> Handle<Str> {
        let mut handle = str_alloc_heap(ctxt, buf.len());
        handle.length = buf.len();

        unsafe {
            let data = handle.data() as *mut u8;

            // copy buffer content into Str
            ptr::copy_nonoverlapping(buf.as_ptr(), data, buf.len());
        }

        handle
    }

    pub fn from_str(ctxt: &SemContext, val: Rooted<Str>, offset: usize, len: usize) -> Handle<Str> {
        let total_len = val.len();

        if offset > total_len {
            return Handle::null();
        }

        let len = std::cmp::min(total_len - offset, len);

        let slice = unsafe {
            let data = val.data().offset(offset as isize);
            slice::from_raw_parts(data, len)
        };

        if let Ok(_) = str::from_utf8(slice) {
            let mut handle = str_alloc_heap(ctxt, len);
            handle.length = len;

            unsafe {
                let dest = handle.data() as *mut u8;
                let src = val.data().offset(offset as isize);

                // copy buffer content into Str
                ptr::copy_nonoverlapping(src, dest, len);
            }

            handle
        } else {
            Handle::null()
        }
    }

    pub fn concat(ctxt: &SemContext, lhs: Rooted<Str>, rhs: Rooted<Str>) -> Rooted<Str> {
        let len = lhs.len() + rhs.len();
        let mut handle = ctxt.handles.root(str_alloc_heap(ctxt, len));

        unsafe {
            handle.length = len;

            ptr::copy_nonoverlapping(lhs.data(), handle.data() as *mut u8, lhs.len());
            ptr::copy_nonoverlapping(
                rhs.data(),
                handle.data().offset(lhs.len() as isize) as *mut u8,
                rhs.len(),
            );
        }

        handle
    }

    // duplicate string into a new object
    pub fn dup(&self, ctxt: &SemContext) -> Handle<Str> {
        let len = self.len();
        let mut handle = str_alloc_heap(ctxt, len);

        unsafe {
            handle.length = len;

            ptr::copy_nonoverlapping(self.data(), handle.data() as *mut u8, len);
        }

        handle
    }
}

fn str_alloc_heap(ctxt: &SemContext, len: usize) -> Handle<Str> {
    str_alloc(ctxt, len, |ctxt, size| {
        ctxt.gc.alloc(ctxt, size, false) as *const u8
    })
}

fn str_alloc_perm(ctxt: &SemContext, len: usize) -> Handle<Str> {
    str_alloc(ctxt, len, |ctxt, size| ctxt.gc.alloc_perm(size))
}

fn str_alloc<F>(ctxt: &SemContext, len: usize, alloc: F) -> Handle<Str>
where
    F: FnOnce(&SemContext, usize) -> *const u8,
{
    let size = Header::size() as usize      // Object header
                + mem::ptr_width() as usize // length field
                + len; // string content

    let size = mem::align_usize(size, mem::ptr_width() as usize);
    let ptr = alloc(ctxt, size) as usize;

    let clsid = ctxt.vips.str(ctxt);
    let cls = ctxt.class_defs[clsid].borrow();
    let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();
    let mut handle: Handle<Str> = ptr.into();
    handle.header_mut().vtable = vtable as *mut VTable;

    handle
}

#[repr(C)]
pub struct Array<T: Copy> {
    header: Header,
    length: usize,
    data: u8,
    phantom: PhantomData<T>,
}

impl<T> Array<T>
where
    T: Copy,
{
    pub fn header(&self) -> &Header {
        &self.header
    }

    pub fn header_mut(&mut self) -> &mut Header {
        &mut self.header
    }

    pub fn len(&self) -> usize {
        self.length
    }

    pub fn data(&self) -> *const T {
        &self.data as *const u8 as *const T
    }

    pub fn data_mut(&mut self) -> *mut T {
        &self.data as *const u8 as *mut T
    }

    pub fn get_at(&self, idx: usize) -> T {
        unsafe { *self.data().offset(idx as isize) }
    }

    pub fn set_at(&mut self, idx: usize, val: T) {
        unsafe {
            *self.data_mut().offset(idx as isize) = val;
        }
    }

    pub fn size(&self) -> usize {
        Header::size() as usize         // Object header
            + mem::ptr_width() as usize // length field
            + self.len() * std::mem::size_of::<T>() // array content
    }

    pub fn alloc(ctxt: &SemContext, len: usize, elem: T, clsid: ClassDefId) -> Handle<Array<T>> {
        let size = Header::size() as usize        // Object header
                   + mem::ptr_width() as usize    // length field
                   + len * std::mem::size_of::<T>(); // array content

        let ptr = ctxt.gc.alloc(ctxt, size, false) as usize;
        let cls = ctxt.class_defs[clsid].borrow();
        let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();
        let mut handle: Handle<Array<T>> = ptr.into();
        handle.header_mut().vtable = vtable as *mut VTable;
        handle.length = len;

        for i in 0..handle.len() {
            unsafe {
                *handle.data_mut().offset(i as isize) = elem;
            }
        }

        handle
    }
}

pub fn offset_of_array_length() -> i32 {
    offset_of!(Array<i32>, length) as i32
}

pub fn offset_of_array_data() -> i32 {
    offset_of!(Array<i32>, data) as i32
}

pub type BoolArray = Array<bool>;
pub type ByteArray = Array<u8>;
pub type CharArray = Array<char>;
pub type IntArray = Array<i32>;
pub type LongArray = Array<i64>;
pub type FloatArray = Array<f32>;
pub type DoubleArray = Array<f64>;
pub type StrArray = Array<Handle<Str>>;

pub fn alloc(ctxt: &SemContext, clsid: ClassDefId) -> Handle<Obj> {
    let cls_def = ctxt.class_defs[clsid].borrow();

    let size = match cls_def.size {
        ClassSize::Fixed(size) => size as usize,
        _ => panic!("alloc only supports fix-sized types"),
    };

    let size = mem::align_usize(size, mem::ptr_width() as usize);

    let ptr = ctxt.gc.alloc(ctxt, size, false) as usize;
    let vtable: *const VTable = &**cls_def.vtable.as_ref().unwrap();
    let mut handle: Handle<Obj> = ptr.into();
    handle.header_mut().vtable = vtable as *mut VTable;

    handle
}

pub struct Exception {
    pub header: Header,
    pub msg: Handle<Str>,
    pub backtrace: Handle<IntArray>,
    pub elements: Handle<Obj>,
}

pub struct StackTraceElement {
    pub header: Header,
    pub name: Handle<Str>,
    pub line: i32,
}
<commit_msg>set array_ref for array allocation<commit_after>use std;
use std::ffi::CString;
use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::ptr;
use std::slice;
use std::str;

use class::{ClassDefId, ClassSize};
use ctxt::SemContext;
use gc::Address;
use gc::root::IndirectObj;
use handle::Rooted;
use mem;
use vtable::VTable;

#[repr(C)]
pub struct Header {
    // ptr to class
    vtable: *mut VTable,
}

impl Header {
    pub fn size() -> i32 {
        std::mem::size_of::<Header>() as i32
    }

    pub fn vtblptr(&self) -> *mut VTable {
        self.vtable
    }

    pub fn vtbl(&self) -> &mut VTable {
        unsafe { &mut *self.vtable }
    }

    pub fn forward_to(&mut self, address: Address) {
        self.vtable = (address.to_usize() | 1) as *mut VTable;
    }

    pub fn forwarded(&self) -> Option<Address> {
        let addr = self.vtable as usize;

        if (addr & 1) == 1 {
            Some((addr & !1).into())
        } else {
            None
        }
    }
}

// is used to reference any object
#[repr(C)]
pub struct Obj {
    header: Header,
    data: u8,
}

impl Obj {
    pub fn header(&self) -> &Header {
        &self.header
    }

    pub fn header_mut(&mut self) -> &mut Header {
        &mut self.header
    }

    pub fn data(&self) -> *const u8 {
        &self.data as *const u8
    }

    pub fn is_array_ref(&self) -> bool {
        let cls = self.header().vtbl().class();

        match cls.size {
            ClassSize::ObjArray => true,
            _ => false,
        }
    }

    pub fn size(&self) -> usize {
        let cls = self.header().vtbl().class();

        match cls.size {
            ClassSize::Fixed(size) => size as usize,

            ClassSize::ObjArray => determine_array_size(self, mem::ptr_width()),

            ClassSize::Array(element_size) => determine_array_size(self, element_size),

            ClassSize::Str => {
                let handle: Handle<Str> = Handle { ptr: self as *const Obj as *const Str };
                mem::align_usize(handle.size(), mem::ptr_width() as usize)
            }
        }
    }

    pub fn visit_reference_fields<F>(&mut self, mut f: F)
    where
        F: FnMut(IndirectObj),
    {
        let classptr = self.header().vtbl().classptr;
        let cls = unsafe { &*classptr };

        if let ClassSize::ObjArray = cls.size {
            let array = unsafe { &*(self as *const _ as *const StrArray) };

            // walk through all objects in array
            let mut ptr = array.data() as *mut *mut Obj;
            let last = unsafe { ptr.offset(array.len() as isize) };

            while ptr < last {
                f((ptr as usize).into());

                unsafe { ptr = ptr.offset(1) }
            }

            return;
        }

        for &offset in &cls.ref_fields {
            let obj = (self as *mut Obj as usize) + offset as usize;
            f(obj.into());
        }
    }

    pub fn copy_to(&self, dest: Address, size: usize) {
        unsafe {
            ptr::copy_nonoverlapping(
                self as *const Obj as *const u8,
                dest.to_mut_ptr::<u8>(),
                size,
            );
        }
    }
}

fn determine_array_size(obj: &Obj, element_size: i32) -> usize {
    let handle: Handle<ByteArray> = Handle { ptr: obj as *const Obj as *const ByteArray };

    let value = Header::size() as usize + mem::ptr_width() as usize +
        element_size as usize * handle.len() as usize;

    mem::align_usize(value, mem::ptr_width() as usize)
}

#[repr(C)]
pub struct Handle<T> {
    ptr: *const T,
}

unsafe impl<T> Send for Handle<T> {}

impl<T> Handle<T> {
    pub fn null() -> Handle<T> {
        Handle { ptr: ptr::null() }
    }

    pub fn cast<R>(&self) -> Handle<R> {
        Handle { ptr: self.ptr as *const R }
    }

    pub fn raw(&self) -> *const T {
        self.ptr
    }
}

// known limitation of #[derive(Copy, Clone)]
// traits need to be implemented manually
impl<T> Copy for Handle<T> {}
impl<T> Clone for Handle<T> {
    fn clone(&self) -> Handle<T> {
        *self
    }
}

impl<T> Deref for Handle<T> {
    type Target = T;

    fn deref(&self) -> &T {
        unsafe { &*self.ptr }
    }
}

impl<T> DerefMut for Handle<T> {
    fn deref_mut(&mut self) -> &mut T {
        unsafe { &mut *(self.ptr as *mut T) }
    }
}

impl<T> Into<Handle<T>> for usize {
    fn into(self) -> Handle<T> {
        Handle { ptr: self as *const T }
    }
}

#[repr(C)]
pub struct Testing {
    header: Header,
    failed: bool,
}

impl Testing {
    pub fn has_failed(&self) -> bool {
        self.failed
    }
}

#[repr(C)]
pub struct Str {
    header: Header,
    length: usize,
    data: u8,
}

impl Str {
    pub fn header(&self) -> &Header {
        &self.header
    }

    pub fn header_mut(&mut self) -> &mut Header {
        &mut self.header
    }

    pub fn len(&self) -> usize {
        self.length
    }

    pub fn data(&self) -> *const u8 {
        &self.data as *const u8
    }

    pub fn content(&self) -> &[u8] {
        unsafe { slice::from_raw_parts(self.data(), self.len()) }
    }

    pub fn to_cstring(&self) -> CString {
        let view = self.content();

        CString::new(view).unwrap()
    }

    pub fn size(&self) -> usize {
        Header::size() as usize         // Object header
            + mem::ptr_width() as usize // length field
            + self.len() // string content
    }

    /// allocates string from buffer in permanent space
    pub fn from_buffer_in_perm(ctxt: &SemContext, buf: &[u8]) -> Handle<Str> {
        let mut handle = str_alloc_perm(ctxt, buf.len());
        handle.length = buf.len();

        unsafe {
            let data = handle.data() as *mut u8;

            // copy buffer content into Str
            ptr::copy_nonoverlapping(buf.as_ptr(), data, buf.len());
        }

        handle
    }

    /// allocates string from buffer in heap
    pub fn from_buffer(ctxt: &SemContext, buf: &[u8]) -> Handle<Str> {
        let mut handle = str_alloc_heap(ctxt, buf.len());
        handle.length = buf.len();

        unsafe {
            let data = handle.data() as *mut u8;

            // copy buffer content into Str
            ptr::copy_nonoverlapping(buf.as_ptr(), data, buf.len());
        }

        handle
    }

    pub fn from_str(ctxt: &SemContext, val: Rooted<Str>, offset: usize, len: usize) -> Handle<Str> {
        let total_len = val.len();

        if offset > total_len {
            return Handle::null();
        }

        let len = std::cmp::min(total_len - offset, len);

        let slice = unsafe {
            let data = val.data().offset(offset as isize);
            slice::from_raw_parts(data, len)
        };

        if let Ok(_) = str::from_utf8(slice) {
            let mut handle = str_alloc_heap(ctxt, len);
            handle.length = len;

            unsafe {
                let dest = handle.data() as *mut u8;
                let src = val.data().offset(offset as isize);

                // copy buffer content into Str
                ptr::copy_nonoverlapping(src, dest, len);
            }

            handle
        } else {
            Handle::null()
        }
    }

    pub fn concat(ctxt: &SemContext, lhs: Rooted<Str>, rhs: Rooted<Str>) -> Rooted<Str> {
        let len = lhs.len() + rhs.len();
        let mut handle = ctxt.handles.root(str_alloc_heap(ctxt, len));

        unsafe {
            handle.length = len;

            ptr::copy_nonoverlapping(lhs.data(), handle.data() as *mut u8, lhs.len());
            ptr::copy_nonoverlapping(
                rhs.data(),
                handle.data().offset(lhs.len() as isize) as *mut u8,
                rhs.len(),
            );
        }

        handle
    }

    // duplicate string into a new object
    pub fn dup(&self, ctxt: &SemContext) -> Handle<Str> {
        let len = self.len();
        let mut handle = str_alloc_heap(ctxt, len);

        unsafe {
            handle.length = len;

            ptr::copy_nonoverlapping(self.data(), handle.data() as *mut u8, len);
        }

        handle
    }
}

fn str_alloc_heap(ctxt: &SemContext, len: usize) -> Handle<Str> {
    str_alloc(ctxt, len, |ctxt, size| {
        ctxt.gc.alloc(ctxt, size, false) as *const u8
    })
}

fn str_alloc_perm(ctxt: &SemContext, len: usize) -> Handle<Str> {
    str_alloc(ctxt, len, |ctxt, size| ctxt.gc.alloc_perm(size))
}

fn str_alloc<F>(ctxt: &SemContext, len: usize, alloc: F) -> Handle<Str>
where
    F: FnOnce(&SemContext, usize) -> *const u8,
{
    let size = Header::size() as usize      // Object header
                + mem::ptr_width() as usize // length field
                + len; // string content

    let size = mem::align_usize(size, mem::ptr_width() as usize);
    let ptr = alloc(ctxt, size) as usize;

    let clsid = ctxt.vips.str(ctxt);
    let cls = ctxt.class_defs[clsid].borrow();
    let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();
    let mut handle: Handle<Str> = ptr.into();
    handle.header_mut().vtable = vtable as *mut VTable;

    handle
}

pub trait ArrayElement {
    const REF: bool;
}

impl ArrayElement for bool {
    const REF: bool = false;
}

impl ArrayElement for u8 {
    const REF: bool = false;
}

impl ArrayElement for char {
    const REF: bool = false;
}

impl ArrayElement for i32 {
    const REF: bool = false;
}

impl ArrayElement for i64 {
    const REF: bool = false;
}

impl ArrayElement for f32 {
    const REF: bool = false;
}

impl ArrayElement for f64 {
    const REF: bool = false;
}

impl ArrayElement for Handle<Str> {
    const REF: bool = true;
}

#[repr(C)]
pub struct Array<T: Copy> {
    header: Header,
    length: usize,
    data: u8,
    phantom: PhantomData<T>,
}

impl<T> Array<T>
where
    T: Copy + ArrayElement,
{
    pub fn header(&self) -> &Header {
        &self.header
    }

    pub fn header_mut(&mut self) -> &mut Header {
        &mut self.header
    }

    pub fn len(&self) -> usize {
        self.length
    }

    pub fn data(&self) -> *const T {
        &self.data as *const u8 as *const T
    }

    pub fn data_mut(&mut self) -> *mut T {
        &self.data as *const u8 as *mut T
    }

    pub fn get_at(&self, idx: usize) -> T {
        unsafe { *self.data().offset(idx as isize) }
    }

    pub fn set_at(&mut self, idx: usize, val: T) {
        unsafe {
            *self.data_mut().offset(idx as isize) = val;
        }
    }

    pub fn size(&self) -> usize {
        Header::size() as usize         // Object header
            + mem::ptr_width() as usize // length field
            + self.len() * std::mem::size_of::<T>() // array content
    }

    pub fn alloc(ctxt: &SemContext, len: usize, elem: T, clsid: ClassDefId) -> Handle<Array<T>> {
        let size = Header::size() as usize        // Object header
                   + mem::ptr_width() as usize    // length field
                   + len * std::mem::size_of::<T>(); // array content

        let ptr = ctxt.gc.alloc(ctxt, size, T::REF) as usize;
        let cls = ctxt.class_defs[clsid].borrow();
        let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();
        let mut handle: Handle<Array<T>> = ptr.into();
        handle.header_mut().vtable = vtable as *mut VTable;
        handle.length = len;

        for i in 0..handle.len() {
            unsafe {
                *handle.data_mut().offset(i as isize) = elem;
            }
        }

        handle
    }
}

pub fn offset_of_array_length() -> i32 {
    offset_of!(Array<i32>, length) as i32
}

pub fn offset_of_array_data() -> i32 {
    offset_of!(Array<i32>, data) as i32
}

pub type BoolArray = Array<bool>;
pub type ByteArray = Array<u8>;
pub type CharArray = Array<char>;
pub type IntArray = Array<i32>;
pub type LongArray = Array<i64>;
pub type FloatArray = Array<f32>;
pub type DoubleArray = Array<f64>;
pub type StrArray = Array<Handle<Str>>;

pub fn alloc(ctxt: &SemContext, clsid: ClassDefId) -> Handle<Obj> {
    let cls_def = ctxt.class_defs[clsid].borrow();

    let size = match cls_def.size {
        ClassSize::Fixed(size) => size as usize,
        _ => panic!("alloc only supports fix-sized types"),
    };

    let size = mem::align_usize(size, mem::ptr_width() as usize);

    let ptr = ctxt.gc.alloc(ctxt, size, false) as usize;
    let vtable: *const VTable = &**cls_def.vtable.as_ref().unwrap();
    let mut handle: Handle<Obj> = ptr.into();
    handle.header_mut().vtable = vtable as *mut VTable;

    handle
}

pub struct Exception {
    pub header: Header,
    pub msg: Handle<Str>,
    pub backtrace: Handle<IntArray>,
    pub elements: Handle<Obj>,
}

pub struct StackTraceElement {
    pub header: Header,
    pub name: Handle<Str>,
    pub line: i32,
}
<|endoftext|>


<commit_before>use token::Token;
use tokenizer::{Tokenizer, LexerError};
pub use ast::*;

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    UnexpectedToken(Token),
    InvalidStringLiteral,
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        let function_parts = self.partition_function(&statement_tokens);

        // Error here is okay and does not need to be propagated yet because
        // it probably just means that this is an expression
        if function_parts.is_err() {
            Ok(Statement::Expression(
                try!(self.expr(&statement_tokens[..]))
            ))
        }
        else {
            let (lhs, rhs) = function_parts.unwrap();

            // partition_function is guarenteed to return non-empty slices
            debug_assert!(!lhs.is_empty() && !rhs.is_empty());

            if lhs[0] == Token::Backslash {
                return self.anonymous_function(lhs, rhs);
            }

            match lhs.len() {
                // only one argument: name = ...
                1 => self.assignment(lhs, rhs),
                // more than one argument: name arg1 ... = ...
                _ => self.named_function(lhs, rhs),
            }
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[1..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = lhs[0].clone().unwrap_symbol();

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[1..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> ParseResult<Function> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(|ref sym| (*sym).clone().unwrap_symbol()).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs.len() == 1);

        Ok(Statement::Assignment {
            name: lhs[0].clone().unwrap_symbol(),
            value: try!(self.expr(rhs)),
        })
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        let mut result = Expr::new();

        let mut remaining = tokens;
        loop {
            let (item, leftover) = try!(self.expr_item(remaining));
            remaining = leftover;

            result.push(item);

            if remaining.is_empty() {
                break;
            }
        }

        Ok(result)
    }

    fn expr_item<'a>(&self, tokens: &'a [Token]) -> ParseResult<(ExprItem, &'a [Token])> {
        debug_assert!(!tokens.is_empty());

        if tokens[0] == Token::Backslash {
            let (lhs, rhs) = try!(self.partition_function(tokens));

            debug_assert!(lhs[0] == Token::Backslash);

            Ok((
                ExprItem::AnonymousFunction(
                    try!(self.function(&lhs[1..], rhs))
                ),
                &[]
            ))
        }
        else if tokens[0] == Token::ParenOpen {
            let close = tokens.iter().position(|ref t| **t == Token::ParenClose);
            if close.is_none() {
                return Err(ParseError::ExpectedToken(Token::ParenClose));
            }
            let close = close.unwrap();

            Ok((
                ExprItem::Group(
                    try!(self.expr(&tokens[1..close]))
                ),
                &tokens[(close+1)..]
            ))
        }
        else if tokens[0] == Token::StringBoundary {
            // since strings can only be a single word, not a number
            const STRING_VALUE: usize = 1;
            const STRING_END: usize = 2;
            const AFTER_STRING: usize = 3;

            if tokens[STRING_END] != Token::StringBoundary {
                Err(ParseError::ExpectedToken(Token::StringBoundary))
            }
            else if let Token::Symbol(ref value) = tokens[STRING_VALUE] {
                Ok((ExprItem::SingleTerm(
                    Term::StringLiteral(value.clone())
                ), &tokens[AFTER_STRING..]))
            }
            else {
                Err(ParseError::InvalidStringLiteral)
            }
        }
        else {
            Ok((ExprItem::SingleTerm(
                try!(self.simple_term(&tokens[0]))
            ), &tokens[1..]))
        }
    }

    fn simple_term(&self, token: &Token) -> ParseResult<Term> {
        match *token {
            Token::Symbol(ref s) => Ok(Term::Symbol(s.clone())),
            Token::Number(ref n) => Ok(Term::Number(n.clone())),
            _ => Err(ParseError::UnexpectedToken(token.clone())),
        }
    }

    // partitions a function by its Equals token, returning None if no
    // valid function is found
    // Returns (lhs, rhs)
    fn partition_function<'a>(&self, tokens: &'a [Token]) -> ParseResult<(&'a [Token], &'a [Token])> {
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                return Err(ParseError::UnexpectedToken(token.clone()));
            }
        }

        if equals.is_none() {
            Err(ParseError::ExpectedToken(Token::Equals))
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            Ok((&tokens[..equals], &tokens[(equals+1)..]))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn call_function_with_string() {
        test_statement(r#"defineUnit "km/h""#,
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ])
        );
    }

    #[test]
    fn define_simple_variable() {
        test_statement(r"my_var = 30 km/h",
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
        );
    }

    #[test]
    fn named_function_complex_expression() {
        test_statement(r"f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3",
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
        );
    }

    #[test]
    fn complete_program() {
        let program = r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#;

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),

            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },

            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },

            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),

            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),

            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },

            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        test_program(program, syntax_tree);
    }

    fn test_program(string: &str, expected: Program) {
        let mut parser = parser_for(string);
        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, expected);
    }

    fn test_statement(string: &str, expected: Statement) {
        let mut parser = parser_for(string);
        let parsed = parser.parse().unwrap();
        assert_eq!(parsed[0], expected);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Fixed test that had an extra group<commit_after>use token::Token;
use tokenizer::{Tokenizer, LexerError};
pub use ast::*;

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    UnexpectedToken(Token),
    InvalidStringLiteral,
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        let function_parts = self.partition_function(&statement_tokens);

        // Error here is okay and does not need to be propagated yet because
        // it probably just means that this is an expression
        if function_parts.is_err() {
            Ok(Statement::Expression(
                try!(self.expr(&statement_tokens[..]))
            ))
        }
        else {
            let (lhs, rhs) = function_parts.unwrap();

            // partition_function is guarenteed to return non-empty slices
            debug_assert!(!lhs.is_empty() && !rhs.is_empty());

            if lhs[0] == Token::Backslash {
                return self.anonymous_function(lhs, rhs);
            }

            match lhs.len() {
                // only one argument: name = ...
                1 => self.assignment(lhs, rhs),
                // more than one argument: name arg1 ... = ...
                _ => self.named_function(lhs, rhs),
            }
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[1..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = lhs[0].clone().unwrap_symbol();

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[1..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> ParseResult<Function> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(|ref sym| (*sym).clone().unwrap_symbol()).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs.len() == 1);

        Ok(Statement::Assignment {
            name: lhs[0].clone().unwrap_symbol(),
            value: try!(self.expr(rhs)),
        })
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        let mut result = Expr::new();

        let mut remaining = tokens;
        loop {
            let (item, leftover) = try!(self.expr_item(remaining));
            remaining = leftover;

            result.push(item);

            if remaining.is_empty() {
                break;
            }
        }

        Ok(result)
    }

    fn expr_item<'a>(&self, tokens: &'a [Token]) -> ParseResult<(ExprItem, &'a [Token])> {
        debug_assert!(!tokens.is_empty());

        if tokens[0] == Token::Backslash {
            let (lhs, rhs) = try!(self.partition_function(tokens));

            debug_assert!(lhs[0] == Token::Backslash);

            Ok((
                ExprItem::AnonymousFunction(
                    try!(self.function(&lhs[1..], rhs))
                ),
                &[]
            ))
        }
        else if tokens[0] == Token::ParenOpen {
            let close = tokens.iter().position(|ref t| **t == Token::ParenClose);
            if close.is_none() {
                return Err(ParseError::ExpectedToken(Token::ParenClose));
            }
            let close = close.unwrap();

            Ok((
                ExprItem::Group(
                    try!(self.expr(&tokens[1..close]))
                ),
                &tokens[(close+1)..]
            ))
        }
        else if tokens[0] == Token::StringBoundary {
            // since strings can only be a single word, not a number
            const STRING_VALUE: usize = 1;
            const STRING_END: usize = 2;
            const AFTER_STRING: usize = 3;

            if tokens[STRING_END] != Token::StringBoundary {
                Err(ParseError::ExpectedToken(Token::StringBoundary))
            }
            else if let Token::Symbol(ref value) = tokens[STRING_VALUE] {
                Ok((ExprItem::SingleTerm(
                    Term::StringLiteral(value.clone())
                ), &tokens[AFTER_STRING..]))
            }
            else {
                Err(ParseError::InvalidStringLiteral)
            }
        }
        else {
            Ok((ExprItem::SingleTerm(
                try!(self.simple_term(&tokens[0]))
            ), &tokens[1..]))
        }
    }

    fn simple_term(&self, token: &Token) -> ParseResult<Term> {
        match *token {
            Token::Symbol(ref s) => Ok(Term::Symbol(s.clone())),
            Token::Number(ref n) => Ok(Term::Number(n.clone())),
            _ => Err(ParseError::UnexpectedToken(token.clone())),
        }
    }

    // partitions a function by its Equals token, returning None if no
    // valid function is found
    // Returns (lhs, rhs)
    fn partition_function<'a>(&self, tokens: &'a [Token]) -> ParseResult<(&'a [Token], &'a [Token])> {
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                return Err(ParseError::UnexpectedToken(token.clone()));
            }
        }

        if equals.is_none() {
            Err(ParseError::ExpectedToken(Token::Equals))
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            Ok((&tokens[..equals], &tokens[(equals+1)..]))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn call_function_with_string() {
        test_statement(r#"defineUnit "km/h""#,
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ])
        );
    }

    #[test]
    fn define_simple_variable() {
        test_statement(r"my_var = 30 km/h",
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
        );
    }

    #[test]
    fn named_function_complex_expression() {
        test_statement(r"f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3",
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
        );
    }

    #[test]
    fn complete_program() {
        let program = r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#;

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),

            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },

            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },

            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),

            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),

            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },

            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("$$".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::AnonymousFunction(
                        Function {
                            params: vec![
                                "x".to_owned(),
                                "y".to_owned()
                            ],
                            body: vec![
                                ExprItem::SingleTerm(
                                    Term::Symbol("x".to_owned())
                                ),
                                ExprItem::SingleTerm(
                                    Term::Symbol("*".to_owned())
                                ),
                                ExprItem::SingleTerm(
                                    Term::Symbol("y".to_owned())
                                ),
                                ExprItem::SingleTerm(
                                    Term::Symbol("+".to_owned())
                                ),
                                ExprItem::SingleTerm(
                                    Term::Symbol("x".to_owned())
                                ),
                            ],
                        },
                    ),
                ]),
            ]),
        ];

        test_program(program, syntax_tree);
    }

    fn test_program(string: &str, expected: Program) {
        let mut parser = parser_for(string);
        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, expected);
    }

    fn test_statement(string: &str, expected: Statement) {
        let mut parser = parser_for(string);
        let parsed = parser.parse().unwrap();
        assert_eq!(parsed[0], expected);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>