<commit_before>//! Lexer
//!
//! This module contains elements than can be used for writing plugins
//! but can be ignored for simple usage.

use token::Token;
use token::Token::*;
use token::ComparisonOperator::*;
use self::Element::*;
use regex::Regex;
use error::{Error, Result};

#[derive(Clone, Debug, PartialEq)]
pub enum Element {
    Expression(Vec<Token>, String),
    Tag(Vec<Token>, String),
    Raw(String),
}

lazy_static! {
    static ref MARKUP: Regex = Regex::new("\\{%.*?%\\}|\\{\\{.*?\\}\\}").unwrap();
}

fn split_blocks(text: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for mat in MARKUP.find_iter(text) {
        let start = mat.start();
        let end = mat.end();
        match &text[current..start] {
            "" => {}
            t => tokens.push(t),
        }
        tokens.push(&text[start..end]);
        current = end;
    }
    match &text[current..text.len()] {
        "" => {}
        t => tokens.push(t),
    }
    tokens
}

lazy_static! {
    static ref EXPRESSION: Regex = Regex::new("\\{\\{(.*?)\\}\\}").unwrap();
    static ref TAG: Regex = Regex::new("\\{%(.*?)%\\}").unwrap();
}

pub fn tokenize(text: &str) -> Result<Vec<Element>> {
    let mut blocks = vec![];

    for block in split_blocks(text) {
        if let Some(caps) = TAG.captures(block) {
            blocks.push(Tag(try!(granularize(caps.get(1).map(|x| x.as_str()).unwrap_or(""))),
                            block.to_owned()));
        } else if let Some(caps) = EXPRESSION.captures(block) {
            blocks.push(Expression(try!(granularize(caps.get(1)
                                       .map(|x| x.as_str())
                                       .unwrap_or(""))),
                                   block.to_owned()));
        } else {
            blocks.push(Raw(block.to_owned()));
        }
    }

    Ok(blocks)
}

lazy_static! {
    static ref SPLIT: Regex = Regex::new(
        r#"'.*?'|".*?"|\s+|[\|:,\[\]\(\)\?]|\.\.|={1,2}|!=|<=|>=|[<>]"#).unwrap();
}

fn split_atom(block: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for mat in SPLIT.find_iter(block) {
        let start = mat.start();
        let end = mat.end();
        // insert the stuff between identifiers
        tokens.push(&block[current..start]);
        // insert the identifier
        tokens.push(&block[start..end]);
        current = end;
    }
    // insert remaining things
    tokens.push(&block[current..block.len()]);
    tokens
}

lazy_static! {
    static ref IDENTIFIER: Regex = Regex::new(r"[a-zA-Z_][\w-]*\??").unwrap();
    static ref SINGLE_STRING_LITERAL: Regex = Regex::new(r"'[^']*'").unwrap();
    static ref DOUBLE_STRING_LITERAL: Regex = Regex::new("\"[^\"]*\"").unwrap();
    static ref NUMBER_LITERAL: Regex = Regex::new(r"^-?\d+(\.\d+)?$").unwrap();
    static ref BOOLEAN_LITERAL: Regex = Regex::new(r"^true|false$").unwrap();
}

pub fn granularize(block: &str) -> Result<Vec<Token>> {
    let mut result = vec![];

    for el in split_atom(block) {
        result.push(match &*el.trim() {
            "" => continue,

            "|" => Pipe,
            "." => Dot,
            ":" => Colon,
            "," => Comma,
            "[" => OpenSquare,
            "]" => CloseSquare,
            "(" => OpenRound,
            ")" => CloseRound,
            "?" => Question,
            "-" => Dash,
            "=" => Assignment,
            "or" => Or,

            "==" => Comparison(Equals),
            "!=" => Comparison(NotEquals),
            "<=" => Comparison(LessThanEquals),
            ">=" => Comparison(GreaterThanEquals),
            "<" => Comparison(LessThan),
            ">" => Comparison(GreaterThan),
            "contains" => Comparison(Contains),
            ".." => DotDot,

            x if SINGLE_STRING_LITERAL.is_match(x) || DOUBLE_STRING_LITERAL.is_match(x) => {
                StringLiteral(x[1..x.len() - 1].to_owned())
            }
            x if NUMBER_LITERAL.is_match(x) => {
                NumberLiteral(x.parse::<f32>().expect(&format!("Could not parse {:?} as float", x)))
            }
            x if BOOLEAN_LITERAL.is_match(x) => {
                BooleanLiteral(x.parse::<bool>()
                    .expect(&format!("Could not parse {:?} as bool", x)))
            }
            x if IDENTIFIER.is_match(x) => Identifier(x.to_owned()),
            x => return Err(Error::Lexer(format!("{} is not a valid identifier", x))),
        });
    }

    Ok(result)
}

#[test]
fn test_split_blocks() {
    assert_eq!(split_blocks("asdlkjfn\n{{askdljfbalkjsdbf}} asdjlfb"),
               vec!["asdlkjfn\n", "{{askdljfbalkjsdbf}}", " asdjlfb"]);
    assert_eq!(split_blocks("asdlkjfn\n{%askdljfbalkjsdbf%} asdjlfb"),
               vec!["asdlkjfn\n", "{%askdljfbalkjsdbf%}", " asdjlfb"]);
}

#[test]
fn test_split_atom() {
    assert_eq!(split_atom("truc | arg:val"),
               vec!["truc", " ", "", "|", "", " ", "arg", ":", "val"]);
    assert_eq!(split_atom("truc | filter:arg1,arg2"),
               vec!["truc", " ", "", "|", "", " ", "filter", ":", "arg1", ",", "arg2"]);
}

#[test]
fn test_tokenize() {
    assert_eq!(tokenize("{{hello 'world'}}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned())]);
    assert_eq!(tokenize("{{hello.world}}").unwrap(),
               vec![Expression(vec![Identifier("hello.world".to_owned())],
                               "{{hello.world}}".to_owned())]);
    assert_eq!(tokenize("{{ hello 'world' }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{ hello 'world' }}".to_owned())]);
    assert_eq!(tokenize("{{   hello   'world'    }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{   hello   'world'    }}".to_owned())]);
    assert_eq!(tokenize("wat\n{{hello 'world'}} test").unwrap(),
               vec![Raw("wat\n".to_owned()),
                    Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned()),
                    Raw(" test".to_owned())]);
}

#[test]
fn test_granularize() {
    assert_eq!(granularize("include my-file.html").unwrap(),
               vec![Identifier("include".to_owned()), Identifier("my-file.html".to_owned())]);
    assert_eq!(granularize("test | me").unwrap(),
               vec![Identifier("test".to_owned()), Pipe, Identifier("me".to_owned())]);
    assert_eq!(granularize("test .. me").unwrap(),
               vec![Identifier("test".to_owned()), DotDot, Identifier("me".to_owned())]);
    assert_eq!(granularize("test : me").unwrap(),
               vec![Identifier("test".to_owned()), Colon, Identifier("me".to_owned())]);
    assert_eq!(granularize("test , me").unwrap(),
               vec![Identifier("test".to_owned()), Comma, Identifier("me".to_owned())]);
    assert_eq!(granularize("test [ me").unwrap(),
               vec![Identifier("test".to_owned()), OpenSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ] me").unwrap(),
               vec![Identifier("test".to_owned()), CloseSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ( me").unwrap(),
               vec![Identifier("test".to_owned()), OpenRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ) me").unwrap(),
               vec![Identifier("test".to_owned()), CloseRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ? me").unwrap(),
               vec![Identifier("test".to_owned()), Question, Identifier("me".to_owned())]);
    assert_eq!(granularize("test - me").unwrap(),
               vec![Identifier("test".to_owned()), Dash, Identifier("me".to_owned())]);
    assert_eq!(granularize("test me").unwrap(),
               vec![Identifier("test".to_owned()), Identifier("me".to_owned())]);
    assert_eq!(granularize("test = me").unwrap(),
               vec![Identifier("test".to_owned()), Assignment, Identifier("me".to_owned())]);
    assert_eq!(granularize("test == me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(Equals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test >= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test > me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test < me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test != me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(NotEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test <= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test.me").unwrap(),
               vec![Identifier("test.me".to_owned())]);
    assert_eq!(granularize("'test' == \"me\"").unwrap(),
               vec![StringLiteral("test".to_owned()),
                    Comparison(Equals),
                    StringLiteral("me".to_owned())]);
    assert_eq!(granularize("test | me:arg").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg".to_owned())]);
    assert_eq!(granularize("test | me:arg1,arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("test | me : arg1, arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("multiply 5 3").unwrap(),
               vec![Identifier("multiply".to_owned()), NumberLiteral(5f32), NumberLiteral(3f32)]);
    assert_eq!(granularize("for i in (1..5)").unwrap(),
               vec![Identifier("for".to_owned()),
                    Identifier("i".to_owned()),
                    Identifier("in".to_owned()),
                    OpenRound,
                    NumberLiteral(1f32),
                    DotDot,
                    NumberLiteral(5f32),
                    CloseRound]);
    assert_eq!(granularize("\"1, '2', 3, 4\"").unwrap(),
               vec![StringLiteral("1, '2', 3, 4".to_owned())]);
    assert_eq!(granularize("'1, \"2\", 3, 4'").unwrap(),
               vec![StringLiteral("1, \"2\", 3, 4".to_owned())]);
    assert_eq!(granularize("\"1, '2', 3, 4\"\"1, '2', 3, 4\"").unwrap(),
               vec![StringLiteral("1, '2', 3, 4".to_owned()),
                    StringLiteral("1, '2', 3, 4".to_owned())]);
    assert_eq!(granularize("abc : \"1, '2', 3, 4\"").unwrap(),
               vec![Identifier("abc".to_owned()), Colon, StringLiteral("1, '2', 3, 4".to_owned())]);
}
<commit_msg>whitespace control<commit_after>//! Lexer
//!
//! This module contains elements than can be used for writing plugins
//! but can be ignored for simple usage.

use token::Token;
use token::Token::*;
use token::ComparisonOperator::*;
use self::Element::*;
use regex::Regex;
use error::{Error, Result};

#[derive(Clone, Debug, PartialEq)]
pub enum Element {
    Expression(Vec<Token>, String),
    Tag(Vec<Token>, String),
    Raw(String),
}

lazy_static! {
    static ref MARKUP: Regex = {
        let t = "(?:[[:space:]]*\\{\\{-|\\{\\{).*?(?:-\\}\\}[[:space:]]*|\\}\\})";
        let e = "(?:[[:space:]]*\\{%-|\\{%).*?(?:-%\\}[[:space:]]*|%\\})";
        Regex::new(&format!("{}|{}", t, e)).unwrap()
    };
}

fn split_blocks(text: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for mat in MARKUP.find_iter(text) {
        let start = mat.start();
        let end = mat.end();
        match &text[current..start] {
            "" => {}
            t => tokens.push(t),
        }
        tokens.push(&text[start..end]);
        current = end;
    }
    match &text[current..text.len()] {
        "" => {}
        t => tokens.push(t),
    }
    tokens
}

lazy_static! {
    static ref EXPRESSION: Regex = {
        let t = "(?:[[:space:]]*\\{\\{-|\\{\\{)(.*?)(?:-\\}\\}[[:space:]]*|\\}\\})";
        Regex::new(t).unwrap()
    };
    static ref TAG: Regex = {
        let e = "(?:[[:space:]]*\\{%-|\\{%)(.*?)(?:-%\\}[[:space:]]*|%\\})";
        Regex::new(e).unwrap()
    };
}

pub fn tokenize(text: &str) -> Result<Vec<Element>> {
    let mut blocks = vec![];

    for block in split_blocks(text) {
        if let Some(caps) = TAG.captures(block) {
            blocks.push(Tag(try!(granularize(caps.get(1).map(|x| x.as_str()).unwrap_or(""))),
                            block.to_owned()));
        } else if let Some(caps) = EXPRESSION.captures(block) {
            blocks.push(Expression(try!(granularize(caps.get(1)
                                       .map(|x| x.as_str())
                                       .unwrap_or(""))),
                                   block.to_owned()));
        } else {
            blocks.push(Raw(block.to_owned()));
        }
    }

    Ok(blocks)
}

lazy_static! {
    static ref SPLIT: Regex = Regex::new(
        r#"'.*?'|".*?"|\s+|[\|:,\[\]\(\)\?]|\.\.|={1,2}|!=|<=|>=|[<>]"#).unwrap();
}

fn split_atom(block: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for mat in SPLIT.find_iter(block) {
        let start = mat.start();
        let end = mat.end();
        // insert the stuff between identifiers
        tokens.push(&block[current..start]);
        // insert the identifier
        tokens.push(&block[start..end]);
        current = end;
    }
    // insert remaining things
    tokens.push(&block[current..block.len()]);
    tokens
}

lazy_static! {
    static ref IDENTIFIER: Regex = Regex::new(r"[a-zA-Z_][\w-]*\??").unwrap();
    static ref SINGLE_STRING_LITERAL: Regex = Regex::new(r"'[^']*'").unwrap();
    static ref DOUBLE_STRING_LITERAL: Regex = Regex::new("\"[^\"]*\"").unwrap();
    static ref NUMBER_LITERAL: Regex = Regex::new(r"^-?\d+(\.\d+)?$").unwrap();
    static ref BOOLEAN_LITERAL: Regex = Regex::new(r"^true|false$").unwrap();
}

pub fn granularize(block: &str) -> Result<Vec<Token>> {
    let mut result = vec![];

    for el in split_atom(block) {
        result.push(match &*el.trim() {
            "" => continue,

            "|" => Pipe,
            "." => Dot,
            ":" => Colon,
            "," => Comma,
            "[" => OpenSquare,
            "]" => CloseSquare,
            "(" => OpenRound,
            ")" => CloseRound,
            "?" => Question,
            "-" => Dash,
            "=" => Assignment,
            "or" => Or,

            "==" => Comparison(Equals),
            "!=" => Comparison(NotEquals),
            "<=" => Comparison(LessThanEquals),
            ">=" => Comparison(GreaterThanEquals),
            "<" => Comparison(LessThan),
            ">" => Comparison(GreaterThan),
            "contains" => Comparison(Contains),
            ".." => DotDot,

            x if SINGLE_STRING_LITERAL.is_match(x) || DOUBLE_STRING_LITERAL.is_match(x) => {
                StringLiteral(x[1..x.len() - 1].to_owned())
            }
            x if NUMBER_LITERAL.is_match(x) => {
                NumberLiteral(x.parse::<f32>().expect(&format!("Could not parse {:?} as float", x)))
            }
            x if BOOLEAN_LITERAL.is_match(x) => {
                BooleanLiteral(x.parse::<bool>()
                    .expect(&format!("Could not parse {:?} as bool", x)))
            }
            x if IDENTIFIER.is_match(x) => Identifier(x.to_owned()),
            x => return Err(Error::Lexer(format!("{} is not a valid identifier", x))),
        });
    }

    Ok(result)
}

#[test]
fn test_split_blocks() {
    assert_eq!(split_blocks("asdlkjfn\n{{askdljfbalkjsdbf}} asdjlfb"),
               vec!["asdlkjfn\n", "{{askdljfbalkjsdbf}}", " asdjlfb"]);
    assert_eq!(split_blocks("asdlkjfn\n{%askdljfbalkjsdbf%} asdjlfb"),
               vec!["asdlkjfn\n", "{%askdljfbalkjsdbf%}", " asdjlfb"]);
}
#[test]
fn test_whitespace_control() {
    assert_eq!(split_blocks("foo {{ bar }} 2000"),
               vec!["foo ", "{{ bar }}", " 2000"]);
    assert_eq!(split_blocks("foo {{- bar -}} 2000"),
               vec!["foo", " {{- bar -}} ", "2000"]);
    assert_eq!(split_blocks("foo \n{{- bar }} 2000"),
               vec!["foo", " \n{{- bar }}", " 2000"]);
    assert_eq!(split_blocks("foo {% bar %} 2000"),
               vec!["foo ", "{% bar %}", " 2000"]);
    assert_eq!(split_blocks("foo {%- bar -%} 2000"),
               vec!["foo", " {%- bar -%} ", "2000"]);
    assert_eq!(split_blocks("foo \n{%- bar %} 2000"),
               vec!["foo", " \n{%- bar %}", " 2000"]);
}

#[test]
fn test_split_atom() {
    assert_eq!(split_atom("truc | arg:val"),
               vec!["truc", " ", "", "|", "", " ", "arg", ":", "val"]);
    assert_eq!(split_atom("truc | filter:arg1,arg2"),
               vec!["truc", " ", "", "|", "", " ", "filter", ":", "arg1", ",", "arg2"]);
}

#[test]
fn test_tokenize() {
    assert_eq!(tokenize("{{hello 'world'}}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned())]);
    assert_eq!(tokenize("{{hello.world}}").unwrap(),
               vec![Expression(vec![Identifier("hello.world".to_owned())],
                               "{{hello.world}}".to_owned())]);
    assert_eq!(tokenize("{{ hello 'world' }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{ hello 'world' }}".to_owned())]);
    assert_eq!(tokenize("{{   hello   'world'    }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{   hello   'world'    }}".to_owned())]);
    assert_eq!(tokenize("wat\n{{hello 'world'}} test").unwrap(),
               vec![Raw("wat\n".to_owned()),
                    Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned()),
                    Raw(" test".to_owned())]);
    assert_eq!(tokenize("wat \n {{-hello 'world'-}} test").unwrap(),
               vec![Raw("wat".to_owned()),
                    Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               " \n {{-hello 'world'-}} ".to_owned()),
                    Raw("test".to_owned())]);
}

#[test]
fn test_granularize() {
    assert_eq!(granularize("include my-file.html").unwrap(),
               vec![Identifier("include".to_owned()), Identifier("my-file.html".to_owned())]);
    assert_eq!(granularize("test | me").unwrap(),
               vec![Identifier("test".to_owned()), Pipe, Identifier("me".to_owned())]);
    assert_eq!(granularize("test .. me").unwrap(),
               vec![Identifier("test".to_owned()), DotDot, Identifier("me".to_owned())]);
    assert_eq!(granularize("test : me").unwrap(),
               vec![Identifier("test".to_owned()), Colon, Identifier("me".to_owned())]);
    assert_eq!(granularize("test , me").unwrap(),
               vec![Identifier("test".to_owned()), Comma, Identifier("me".to_owned())]);
    assert_eq!(granularize("test [ me").unwrap(),
               vec![Identifier("test".to_owned()), OpenSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ] me").unwrap(),
               vec![Identifier("test".to_owned()), CloseSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ( me").unwrap(),
               vec![Identifier("test".to_owned()), OpenRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ) me").unwrap(),
               vec![Identifier("test".to_owned()), CloseRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ? me").unwrap(),
               vec![Identifier("test".to_owned()), Question, Identifier("me".to_owned())]);
    assert_eq!(granularize("test - me").unwrap(),
               vec![Identifier("test".to_owned()), Dash, Identifier("me".to_owned())]);
    assert_eq!(granularize("test me").unwrap(),
               vec![Identifier("test".to_owned()), Identifier("me".to_owned())]);
    assert_eq!(granularize("test = me").unwrap(),
               vec![Identifier("test".to_owned()), Assignment, Identifier("me".to_owned())]);
    assert_eq!(granularize("test == me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(Equals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test >= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test > me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test < me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test != me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(NotEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test <= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test.me").unwrap(),
               vec![Identifier("test.me".to_owned())]);
    assert_eq!(granularize("'test' == \"me\"").unwrap(),
               vec![StringLiteral("test".to_owned()),
                    Comparison(Equals),
                    StringLiteral("me".to_owned())]);
    assert_eq!(granularize("test | me:arg").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg".to_owned())]);
    assert_eq!(granularize("test | me:arg1,arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("test | me : arg1, arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("multiply 5 3").unwrap(),
               vec![Identifier("multiply".to_owned()), NumberLiteral(5f32), NumberLiteral(3f32)]);
    assert_eq!(granularize("for i in (1..5)").unwrap(),
               vec![Identifier("for".to_owned()),
                    Identifier("i".to_owned()),
                    Identifier("in".to_owned()),
                    OpenRound,
                    NumberLiteral(1f32),
                    DotDot,
                    NumberLiteral(5f32),
                    CloseRound]);
    assert_eq!(granularize("\"1, '2', 3, 4\"").unwrap(),
               vec![StringLiteral("1, '2', 3, 4".to_owned())]);
    assert_eq!(granularize("'1, \"2\", 3, 4'").unwrap(),
               vec![StringLiteral("1, \"2\", 3, 4".to_owned())]);
    assert_eq!(granularize("\"1, '2', 3, 4\"\"1, '2', 3, 4\"").unwrap(),
               vec![StringLiteral("1, '2', 3, 4".to_owned()),
                    StringLiteral("1, '2', 3, 4".to_owned())]);
    assert_eq!(granularize("abc : \"1, '2', 3, 4\"").unwrap(),
               vec![Identifier("abc".to_owned()), Colon, StringLiteral("1, '2', 3, 4".to_owned())]);
}
<|endoftext|>


<commit_before>//! Lexer
//!
//! This module contains elements than can be used for writing plugins
//! but can be ignored for simple usage.

use token::Token;
use token::Token::*;
use token::ComparisonOperator::*;
use self::Element::*;
use regex::Regex;
use error::{Error, Result};

#[derive(Clone, Debug, PartialEq)]
pub enum Element {
    Expression(Vec<Token>, String),
    Tag(Vec<Token>, String),
    Raw(String),
}

lazy_static! {
    static ref MARKUP: Regex = Regex::new("\\{%.*?%\\}|\\{\\{.*?\\}\\}").unwrap();
}

fn split_blocks(text: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in MARKUP.find_iter(text) {
        match &text[current..begin] {
            "" => {}
            t => tokens.push(t),
        }
        tokens.push(&text[begin..end]);
        current = end;
    }
    match &text[current..text.len()] {
        "" => {}
        t => tokens.push(t),
    }
    tokens
}

lazy_static! {
    static ref EXPRESSION: Regex = Regex::new("\\{\\{(.*?)\\}\\}").unwrap();
    static ref TAG: Regex = Regex::new("\\{%(.*?)%\\}").unwrap();
}

pub fn tokenize(text: &str) -> Result<Vec<Element>> {
    let mut blocks = vec![];

    for block in split_blocks(text) {
        if let Some(caps) = TAG.captures(block) {
            blocks.push(Tag(try!(granularize(caps.at(1).unwrap_or(""))),
                            block.to_owned()));
        } else if let Some(caps) = EXPRESSION.captures(block) {
            blocks.push(Expression(try!(granularize(caps.at(1).unwrap_or(""))),
                                   block.to_owned()));
        } else {
            blocks.push(Raw(block.to_owned()));
        }
    }

    Ok(blocks)
}

lazy_static! {
    static ref SPLIT: Regex = Regex::new(
        r"\s+|[\|:,\[\]\(\)\?-]|\.\.|={1,2}|!=|<=|>=|[<>]").unwrap();
}

fn split_atom(block: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in SPLIT.find_iter(block) {
        // insert the stuff between identifiers
        tokens.push(&block[current..begin]);
        // insert the identifier
        tokens.push(&block[begin..end]);
        current = end;
    }
    // insert remaining things
    tokens.push(&block[current..block.len()]);
    tokens
}

lazy_static! {
    static ref IDENTIFIER: Regex = Regex::new(r"[a-zA-Z_][\w-]*\??").unwrap();
    static ref SINGLE_STRING_LITERAL: Regex = Regex::new(r"'[^']*'").unwrap();
    static ref DOUBLE_STRING_LITERAL: Regex = Regex::new("\"[^\"]*\"").unwrap();
    static ref NUMBER_LITERAL: Regex = Regex::new(r"^-?\d+(\.\d+)?$").unwrap();
    static ref BOOLEAN_LITERAL: Regex = Regex::new(r"^true|false$").unwrap();
}

pub fn granularize(block: &str) -> Result<Vec<Token>> {
    let mut result = vec![];

    for el in split_atom(block) {
        result.push(match &*el.trim() {
            "" => continue,

            "|" => Pipe,
            "." => Dot,
            ":" => Colon,
            "," => Comma,
            "[" => OpenSquare,
            "]" => CloseSquare,
            "(" => OpenRound,
            ")" => CloseRound,
            "?" => Question,
            "-" => Dash,
            "=" => Assignment,
            "or" => Or,

            "==" => Comparison(Equals),
            "!=" => Comparison(NotEquals),
            "<=" => Comparison(LessThanEquals),
            ">=" => Comparison(GreaterThanEquals),
            "<" => Comparison(LessThan),
            ">" => Comparison(GreaterThan),
            "contains" => Comparison(Contains),
            ".." => DotDot,

            x if SINGLE_STRING_LITERAL.is_match(x) || DOUBLE_STRING_LITERAL.is_match(x) => {
                StringLiteral(x[1..x.len() - 1].to_owned())
            }
            x if NUMBER_LITERAL.is_match(x) => {
                NumberLiteral(x.parse::<f32>().expect(&format!("Could not parse {:?} as float", x)))
            }
            x if BOOLEAN_LITERAL.is_match(x) => {
                BooleanLiteral(x.parse::<bool>()
                    .expect(&format!("Could not parse {:?} as bool", x)))
            }
            x if IDENTIFIER.is_match(x) => Identifier(x.to_owned()),
            x => return Err(Error::Lexer(format!("{} is not a valid identifier", x))),
        });
    }

    Ok(result)
}

#[test]
fn test_split_blocks() {
    assert_eq!(split_blocks("asdlkjfn\n{{askdljfbalkjsdbf}} asdjlfb"),
               vec!["asdlkjfn\n", "{{askdljfbalkjsdbf}}", " asdjlfb"]);
    assert_eq!(split_blocks("asdlkjfn\n{%askdljfbalkjsdbf%} asdjlfb"),
               vec!["asdlkjfn\n", "{%askdljfbalkjsdbf%}", " asdjlfb"]);
}

#[test]
fn test_split_atom() {
    assert_eq!(split_atom("truc | arg:val"),
               vec!["truc", " ", "", "|", "", " ", "arg", ":", "val"]);
    assert_eq!(split_atom("truc | filter:arg1,arg2"),
               vec!["truc", " ", "", "|", "", " ", "filter", ":", "arg1", ",", "arg2"]);
}

#[test]
fn test_tokenize() {
    assert_eq!(tokenize("{{hello 'world'}}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned())]);
    assert_eq!(tokenize("{{hello.world}}").unwrap(),
               vec![Expression(vec![Identifier("hello.world".to_owned())],
                               "{{hello.world}}".to_owned())]);
    assert_eq!(tokenize("{{ hello 'world' }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{ hello 'world' }}".to_owned())]);
    assert_eq!(tokenize("{{   hello   'world'    }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{   hello   'world'    }}".to_owned())]);
    assert_eq!(tokenize("wat\n{{hello 'world'}} test").unwrap(),
               vec![Raw("wat\n".to_owned()),
                    Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned()),
                    Raw(" test".to_owned())]);
}

#[test]
fn test_granularize() {
    assert_eq!(granularize("test | me").unwrap(),
               vec![Identifier("test".to_owned()), Pipe, Identifier("me".to_owned())]);
    assert_eq!(granularize("test .. me").unwrap(),
               vec![Identifier("test".to_owned()), DotDot, Identifier("me".to_owned())]);
    assert_eq!(granularize("test : me").unwrap(),
               vec![Identifier("test".to_owned()), Colon, Identifier("me".to_owned())]);
    assert_eq!(granularize("test , me").unwrap(),
               vec![Identifier("test".to_owned()), Comma, Identifier("me".to_owned())]);
    assert_eq!(granularize("test [ me").unwrap(),
               vec![Identifier("test".to_owned()), OpenSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ] me").unwrap(),
               vec![Identifier("test".to_owned()), CloseSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ( me").unwrap(),
               vec![Identifier("test".to_owned()), OpenRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ) me").unwrap(),
               vec![Identifier("test".to_owned()), CloseRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ? me").unwrap(),
               vec![Identifier("test".to_owned()), Question, Identifier("me".to_owned())]);
    assert_eq!(granularize("test - me").unwrap(),
               vec![Identifier("test".to_owned()), Dash, Identifier("me".to_owned())]);
    assert_eq!(granularize("test me").unwrap(),
               vec![Identifier("test".to_owned()), Identifier("me".to_owned())]);
    assert_eq!(granularize("test = me").unwrap(),
               vec![Identifier("test".to_owned()), Assignment, Identifier("me".to_owned())]);
    assert_eq!(granularize("test == me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(Equals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test >= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test > me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test < me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test != me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(NotEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test <= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test.me").unwrap(),
               vec![Identifier("test.me".to_owned())]);
    assert_eq!(granularize("'test' == \"me\"").unwrap(),
               vec![StringLiteral("test".to_owned()),
                    Comparison(Equals),
                    StringLiteral("me".to_owned())]);
    assert_eq!(granularize("test | me:arg").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg".to_owned())]);
    assert_eq!(granularize("test | me:arg1,arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("test | me : arg1, arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("multiply 5 3").unwrap(),
               vec![Identifier("multiply".to_owned()), NumberLiteral(5f32), NumberLiteral(3f32)]);
    assert_eq!(granularize("for i in (1..5)").unwrap(),
               vec![Identifier("for".to_owned()),
                    Identifier("i".to_owned()),
                    Identifier("in".to_owned()),
                    OpenRound,
                    NumberLiteral(1f32),
                    DotDot,
                    NumberLiteral(5f32),
                    CloseRound]);
}
<commit_msg>Add quotes to split regex (fixes #41)<commit_after>//! Lexer
//!
//! This module contains elements than can be used for writing plugins
//! but can be ignored for simple usage.

use token::Token;
use token::Token::*;
use token::ComparisonOperator::*;
use self::Element::*;
use regex::Regex;
use error::{Error, Result};

#[derive(Clone, Debug, PartialEq)]
pub enum Element {
    Expression(Vec<Token>, String),
    Tag(Vec<Token>, String),
    Raw(String),
}

lazy_static! {
    static ref MARKUP: Regex = Regex::new("\\{%.*?%\\}|\\{\\{.*?\\}\\}").unwrap();
}

fn split_blocks(text: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in MARKUP.find_iter(text) {
        match &text[current..begin] {
            "" => {}
            t => tokens.push(t),
        }
        tokens.push(&text[begin..end]);
        current = end;
    }
    match &text[current..text.len()] {
        "" => {}
        t => tokens.push(t),
    }
    tokens
}

lazy_static! {
    static ref EXPRESSION: Regex = Regex::new("\\{\\{(.*?)\\}\\}").unwrap();
    static ref TAG: Regex = Regex::new("\\{%(.*?)%\\}").unwrap();
}

pub fn tokenize(text: &str) -> Result<Vec<Element>> {
    let mut blocks = vec![];

    for block in split_blocks(text) {
        if let Some(caps) = TAG.captures(block) {
            blocks.push(Tag(try!(granularize(caps.at(1).unwrap_or(""))),
                            block.to_owned()));
        } else if let Some(caps) = EXPRESSION.captures(block) {
            blocks.push(Expression(try!(granularize(caps.at(1).unwrap_or(""))),
                                   block.to_owned()));
        } else {
            blocks.push(Raw(block.to_owned()));
        }
    }

    Ok(blocks)
}

lazy_static! {
    static ref SPLIT: Regex = Regex::new(
        r#"'.*?'|".*?"|\s+|[\|:,\[\]\(\)\?-]|\.\.|={1,2}|!=|<=|>=|[<>]"#).unwrap();
}

fn split_atom(block: &str) -> Vec<&str> {
    let mut tokens = vec![];
    let mut current = 0;
    for (begin, end) in SPLIT.find_iter(block) {
        // insert the stuff between identifiers
        tokens.push(&block[current..begin]);
        // insert the identifier
        tokens.push(&block[begin..end]);
        current = end;
    }
    // insert remaining things
    tokens.push(&block[current..block.len()]);
    tokens
}

lazy_static! {
    static ref IDENTIFIER: Regex = Regex::new(r"[a-zA-Z_][\w-]*\??").unwrap();
    static ref SINGLE_STRING_LITERAL: Regex = Regex::new(r"'[^']*'").unwrap();
    static ref DOUBLE_STRING_LITERAL: Regex = Regex::new("\"[^\"]*\"").unwrap();
    static ref NUMBER_LITERAL: Regex = Regex::new(r"^-?\d+(\.\d+)?$").unwrap();
    static ref BOOLEAN_LITERAL: Regex = Regex::new(r"^true|false$").unwrap();
}

pub fn granularize(block: &str) -> Result<Vec<Token>> {
    let mut result = vec![];

    for el in split_atom(block) {
        result.push(match &*el.trim() {
            "" => continue,

            "|" => Pipe,
            "." => Dot,
            ":" => Colon,
            "," => Comma,
            "[" => OpenSquare,
            "]" => CloseSquare,
            "(" => OpenRound,
            ")" => CloseRound,
            "?" => Question,
            "-" => Dash,
            "=" => Assignment,
            "or" => Or,

            "==" => Comparison(Equals),
            "!=" => Comparison(NotEquals),
            "<=" => Comparison(LessThanEquals),
            ">=" => Comparison(GreaterThanEquals),
            "<" => Comparison(LessThan),
            ">" => Comparison(GreaterThan),
            "contains" => Comparison(Contains),
            ".." => DotDot,

            x if SINGLE_STRING_LITERAL.is_match(x) || DOUBLE_STRING_LITERAL.is_match(x) => {
                StringLiteral(x[1..x.len() - 1].to_owned())
            }
            x if NUMBER_LITERAL.is_match(x) => {
                NumberLiteral(x.parse::<f32>().expect(&format!("Could not parse {:?} as float", x)))
            }
            x if BOOLEAN_LITERAL.is_match(x) => {
                BooleanLiteral(x.parse::<bool>()
                    .expect(&format!("Could not parse {:?} as bool", x)))
            }
            x if IDENTIFIER.is_match(x) => Identifier(x.to_owned()),
            x => return Err(Error::Lexer(format!("{} is not a valid identifier", x))),
        });
    }

    Ok(result)
}

#[test]
fn test_split_blocks() {
    assert_eq!(split_blocks("asdlkjfn\n{{askdljfbalkjsdbf}} asdjlfb"),
               vec!["asdlkjfn\n", "{{askdljfbalkjsdbf}}", " asdjlfb"]);
    assert_eq!(split_blocks("asdlkjfn\n{%askdljfbalkjsdbf%} asdjlfb"),
               vec!["asdlkjfn\n", "{%askdljfbalkjsdbf%}", " asdjlfb"]);
}

#[test]
fn test_split_atom() {
    assert_eq!(split_atom("truc | arg:val"),
               vec!["truc", " ", "", "|", "", " ", "arg", ":", "val"]);
    assert_eq!(split_atom("truc | filter:arg1,arg2"),
               vec!["truc", " ", "", "|", "", " ", "filter", ":", "arg1", ",", "arg2"]);
}

#[test]
fn test_tokenize() {
    assert_eq!(tokenize("{{hello 'world'}}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned())]);
    assert_eq!(tokenize("{{hello.world}}").unwrap(),
               vec![Expression(vec![Identifier("hello.world".to_owned())],
                               "{{hello.world}}".to_owned())]);
    assert_eq!(tokenize("{{ hello 'world' }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{ hello 'world' }}".to_owned())]);
    assert_eq!(tokenize("{{   hello   'world'    }}").unwrap(),
               vec![Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{   hello   'world'    }}".to_owned())]);
    assert_eq!(tokenize("wat\n{{hello 'world'}} test").unwrap(),
               vec![Raw("wat\n".to_owned()),
                    Expression(vec![Identifier("hello".to_owned()),
                                    StringLiteral("world".to_owned())],
                               "{{hello 'world'}}".to_owned()),
                    Raw(" test".to_owned())]);
}

#[test]
fn test_granularize() {
    assert_eq!(granularize("test | me").unwrap(),
               vec![Identifier("test".to_owned()), Pipe, Identifier("me".to_owned())]);
    assert_eq!(granularize("test .. me").unwrap(),
               vec![Identifier("test".to_owned()), DotDot, Identifier("me".to_owned())]);
    assert_eq!(granularize("test : me").unwrap(),
               vec![Identifier("test".to_owned()), Colon, Identifier("me".to_owned())]);
    assert_eq!(granularize("test , me").unwrap(),
               vec![Identifier("test".to_owned()), Comma, Identifier("me".to_owned())]);
    assert_eq!(granularize("test [ me").unwrap(),
               vec![Identifier("test".to_owned()), OpenSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ] me").unwrap(),
               vec![Identifier("test".to_owned()), CloseSquare, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ( me").unwrap(),
               vec![Identifier("test".to_owned()), OpenRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ) me").unwrap(),
               vec![Identifier("test".to_owned()), CloseRound, Identifier("me".to_owned())]);
    assert_eq!(granularize("test ? me").unwrap(),
               vec![Identifier("test".to_owned()), Question, Identifier("me".to_owned())]);
    assert_eq!(granularize("test - me").unwrap(),
               vec![Identifier("test".to_owned()), Dash, Identifier("me".to_owned())]);
    assert_eq!(granularize("test me").unwrap(),
               vec![Identifier("test".to_owned()), Identifier("me".to_owned())]);
    assert_eq!(granularize("test = me").unwrap(),
               vec![Identifier("test".to_owned()), Assignment, Identifier("me".to_owned())]);
    assert_eq!(granularize("test == me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(Equals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test >= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test > me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(GreaterThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test < me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThan),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test != me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(NotEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test <= me").unwrap(),
               vec![Identifier("test".to_owned()),
                    Comparison(LessThanEquals),
                    Identifier("me".to_owned())]);
    assert_eq!(granularize("test.me").unwrap(),
               vec![Identifier("test.me".to_owned())]);
    assert_eq!(granularize("'test' == \"me\"").unwrap(),
               vec![StringLiteral("test".to_owned()),
                    Comparison(Equals),
                    StringLiteral("me".to_owned())]);
    assert_eq!(granularize("test | me:arg").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg".to_owned())]);
    assert_eq!(granularize("test | me:arg1,arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("test | me : arg1, arg2").unwrap(),
               vec![Identifier("test".to_owned()),
                    Pipe,
                    Identifier("me".to_owned()),
                    Colon,
                    Identifier("arg1".to_owned()),
                    Comma,
                    Identifier("arg2".to_owned())]);
    assert_eq!(granularize("multiply 5 3").unwrap(),
               vec![Identifier("multiply".to_owned()), NumberLiteral(5f32), NumberLiteral(3f32)]);
    assert_eq!(granularize("for i in (1..5)").unwrap(),
               vec![Identifier("for".to_owned()),
                    Identifier("i".to_owned()),
                    Identifier("in".to_owned()),
                    OpenRound,
                    NumberLiteral(1f32),
                    DotDot,
                    NumberLiteral(5f32),
                    CloseRound]);
    assert_eq!(granularize("\"1, '2', 3, 4\"").unwrap(),
               vec![StringLiteral("1, '2', 3, 4".to_owned())]);
    assert_eq!(granularize("'1, \"2\", 3, 4'").unwrap(),
               vec![StringLiteral("1, \"2\", 3, 4".to_owned())]);
    assert_eq!(granularize("\"1, '2', 3, 4\"\"1, '2', 3, 4\"").unwrap(),
               vec![StringLiteral("1, '2', 3, 4".to_owned()),
                    StringLiteral("1, '2', 3, 4".to_owned())]);
    assert_eq!(granularize("abc : \"1, '2', 3, 4\"").unwrap(),
               vec![Identifier("abc".to_owned()), Colon, StringLiteral("1, '2', 3, 4".to_owned())]);
}
<|endoftext|>


<commit_before><commit_msg>save sorts are correct highest to lowest order<commit_after><|endoftext|>


<commit_before><commit_msg>start tracking status effects<commit_after><|endoftext|>


<commit_before><commit_msg>Really flimsy (and incorrect) mutex skeleton<commit_after>use std::cell::UnsafeCell;
use std::sync::atomic::{AtomicUsize, Ordering};

use futex;

pub struct Mutex {
    inner: UnsafeCell<MutexInner>
}

struct MutexInner {
    lock: AtomicUsize,
    guard: i32
}

impl Mutex {
    pub fn new() -> Mutex {
        Mutex { inner: UnsafeCell::new(MutexInner::new()) }
    }

    pub fn lock(&mut self) {
        unsafe {
            let ref mut inner = *self.inner.get();
            inner.lock()
        }
    }

    pub fn unlock(&mut self) {
        unsafe {
            let ref mut inner = *self.inner.get();
            inner.unlock()
        }
    }
}

impl MutexInner {
    pub fn new() -> MutexInner {
        MutexInner {
            lock: AtomicUsize::new(0),
            guard: 0
        }
    }

    pub fn unlock(&mut self) {
        let old_val = self.lock.load(Ordering::Relaxed);

        if old_val != /* TODO: get thread id */ 0 {
            panic!("trying to unlock a mutex we don't own");
        }

        // TODO: handle this
        let _ = futex::wake(&mut self.guard, 1);
    }

    pub fn lock(&mut self) {
        let old_val = self.lock.load(Ordering::Relaxed);

        loop {
            // TODO: get thread id.
            let new_val = 0;

            let swap = self.lock.compare_and_swap(old_val, new_val, Ordering::Relaxed);

            // If CAS didn't return old_val, it didn't succeed, so futex and spin
            if swap != old_val {
                // TODO: handle possible error
                let _ = futex::wait(&mut self.guard, 0);
            }

            // Otherwise we got the mutex
            else { break; }
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Add merge.<commit_after><|endoftext|>


<commit_before><commit_msg>Starting to flesh out timer library<commit_after><|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::borrow::Cow;
use std::error::Error;
use std::fmt;
use std::io;

use dbus;
use devicemapper;
use nix;
use term;

pub type StratisResult<T> = Result<T, StratisError>;

// An error type for errors generated within Stratis
//
#[derive(Debug)]
pub struct InternalError(pub Cow<'static, str>);

impl fmt::Display for InternalError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{}", self.0)
    }
}

impl Error for InternalError {
    fn description(&self) -> &str {
        &self.0
    }
}

// Define a common error enum.
// See http://blog.burntsushi.net/rust-error-handling/
#[derive(Debug)]
pub enum StratisError {
    Stratis(InternalError),
    Io(io::Error),
    Nix(nix::Error),
    Dbus(dbus::Error),
    Term(term::Error),
    DM(devicemapper::DmError),
}

impl fmt::Display for StratisError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            StratisError::Stratis(ref err) => write!(f, "Stratis error: {}", err.0),
            StratisError::Io(ref err) => write!(f, "IO error: {}", err),
            StratisError::Nix(ref err) => write!(f, "Nix error: {}", err.errno().desc()),
            StratisError::Dbus(ref err) => {
                write!(f, "Dbus error: {}", err.message().unwrap_or("Unknown"))
            }
            StratisError::Term(ref err) => write!(f, "Term error: {}", err),
            StratisError::DM(ref err) => write!(f, "DM error: {}", err),
        }
    }
}

impl Error for StratisError {
    fn description(&self) -> &str {
        match *self {
            StratisError::Stratis(ref err) => &err.0,
            StratisError::Io(ref err) => err.description(),
            StratisError::Nix(ref err) => err.errno().desc(),
            StratisError::Dbus(ref err) => err.message().unwrap_or("D-Bus Error"),
            StratisError::Term(ref err) => Error::description(err),
            StratisError::DM(ref err) => Error::description(err),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            StratisError::Stratis(ref err) => Some(err),
            StratisError::Io(ref err) => Some(err),
            StratisError::Nix(ref err) => Some(err),
            StratisError::Dbus(ref err) => Some(err),
            StratisError::Term(ref err) => Some(err),
            StratisError::DM(ref err) => Some(err),
        }
    }
}

impl From<InternalError> for StratisError {
    fn from(err: InternalError) -> StratisError {
        StratisError::Stratis(err)
    }
}

impl From<io::Error> for StratisError {
    fn from(err: io::Error) -> StratisError {
        StratisError::Io(err)
    }
}

impl From<nix::Error> for StratisError {
    fn from(err: nix::Error) -> StratisError {
        StratisError::Nix(err)
    }
}

impl From<dbus::Error> for StratisError {
    fn from(err: dbus::Error) -> StratisError {
        StratisError::Dbus(err)
    }
}

impl From<term::Error> for StratisError {
    fn from(err: term::Error) -> StratisError {
        StratisError::Term(err)
    }
}

impl From<devicemapper::DmError> for StratisError {
    fn from(err: devicemapper::DmError) -> StratisError {
        StratisError::DM(err)
    }
}
<commit_msg>Remove obsoleted comment<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::borrow::Cow;
use std::error::Error;
use std::fmt;
use std::io;

use dbus;
use devicemapper;
use nix;
use term;

pub type StratisResult<T> = Result<T, StratisError>;

// An error type for errors generated within Stratis
//
#[derive(Debug)]
pub struct InternalError(pub Cow<'static, str>);

impl fmt::Display for InternalError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{}", self.0)
    }
}

impl Error for InternalError {
    fn description(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
pub enum StratisError {
    Stratis(InternalError),
    Io(io::Error),
    Nix(nix::Error),
    Dbus(dbus::Error),
    Term(term::Error),
    DM(devicemapper::DmError),
}

impl fmt::Display for StratisError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            StratisError::Stratis(ref err) => write!(f, "Stratis error: {}", err.0),
            StratisError::Io(ref err) => write!(f, "IO error: {}", err),
            StratisError::Nix(ref err) => write!(f, "Nix error: {}", err.errno().desc()),
            StratisError::Dbus(ref err) => {
                write!(f, "Dbus error: {}", err.message().unwrap_or("Unknown"))
            }
            StratisError::Term(ref err) => write!(f, "Term error: {}", err),
            StratisError::DM(ref err) => write!(f, "DM error: {}", err),
        }
    }
}

impl Error for StratisError {
    fn description(&self) -> &str {
        match *self {
            StratisError::Stratis(ref err) => &err.0,
            StratisError::Io(ref err) => err.description(),
            StratisError::Nix(ref err) => err.errno().desc(),
            StratisError::Dbus(ref err) => err.message().unwrap_or("D-Bus Error"),
            StratisError::Term(ref err) => Error::description(err),
            StratisError::DM(ref err) => Error::description(err),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            StratisError::Stratis(ref err) => Some(err),
            StratisError::Io(ref err) => Some(err),
            StratisError::Nix(ref err) => Some(err),
            StratisError::Dbus(ref err) => Some(err),
            StratisError::Term(ref err) => Some(err),
            StratisError::DM(ref err) => Some(err),
        }
    }
}

impl From<InternalError> for StratisError {
    fn from(err: InternalError) -> StratisError {
        StratisError::Stratis(err)
    }
}

impl From<io::Error> for StratisError {
    fn from(err: io::Error) -> StratisError {
        StratisError::Io(err)
    }
}

impl From<nix::Error> for StratisError {
    fn from(err: nix::Error) -> StratisError {
        StratisError::Nix(err)
    }
}

impl From<dbus::Error> for StratisError {
    fn from(err: dbus::Error) -> StratisError {
        StratisError::Dbus(err)
    }
}

impl From<term::Error> for StratisError {
    fn from(err: term::Error) -> StratisError {
        StratisError::Term(err)
    }
}

impl From<devicemapper::DmError> for StratisError {
    fn from(err: devicemapper::DmError) -> StratisError {
        StratisError::DM(err)
    }
}
<|endoftext|>


<commit_before><commit_msg>started epoll_ctl<commit_after><|endoftext|>


<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! # Lattice Variables
//!
//! This file contains generic code for operating on inference variables
//! that are characterized by an upper- and lower-bound.  The logic and
//! reasoning is explained in detail in the large comment in `infer.rs`.
//!
//! The code in here is defined quite generically so that it can be
//! applied both to type variables, which represent types being inferred,
//! and fn variables, which represent function types being inferred.
//! It may eventually be applied to their types as well, who knows.
//! In some cases, the functions are also generic with respect to the
//! operation on the lattice (GLB vs LUB).
//!
//! Although all the functions are generic, we generally write the
//! comments in a way that is specific to type variables and the LUB
//! operation.  It's just easier that way.
//!
//! In general all of the functions are defined parametrically
//! over a `LatticeValue`, which is a value defined with respect to
//! a lattice.

use super::InferCtxt;
use super::type_variable::TypeVariableOrigin;

use traits::ObligationCause;
use ty::TyVar;
use ty::{self, Ty};
use ty::relate::{RelateResult, TypeRelation};

pub trait LatticeDir<'f, 'gcx: 'f+'tcx, 'tcx: 'f> : TypeRelation<'f, 'gcx, 'tcx> {
    fn infcx(&self) -> &'f InferCtxt<'f, 'gcx, 'tcx>;

    fn cause(&self) -> &ObligationCause<'tcx>;

    // Relates the type `v` to `a` and `b` such that `v` represents
    // the LUB/GLB of `a` and `b` as appropriate.
    fn relate_bound(&mut self, v: Ty<'tcx>, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, ()>;
}

pub fn super_lattice_tys<'a, 'gcx, 'tcx, L>(this: &mut L,
                                            a: Ty<'tcx>,
                                            b: Ty<'tcx>)
                                            -> RelateResult<'tcx, Ty<'tcx>>
    where L: LatticeDir<'a, 'gcx, 'tcx>, 'gcx: 'a+'tcx, 'tcx: 'a
{
    debug!("{}.lattice_tys({:?}, {:?})",
           this.tag(),
           a,
           b);

    if a == b {
        return Ok(a);
    }

    let infcx = this.infcx();
    let a = infcx.type_variables.borrow_mut().replace_if_possible(a);
    let b = infcx.type_variables.borrow_mut().replace_if_possible(b);
    match (&a.sty, &b.sty) {
        (&ty::TyInfer(TyVar(..)), &ty::TyInfer(TyVar(..)))
            if infcx.type_var_diverges(a) && infcx.type_var_diverges(b) => {
            let v = infcx.next_diverging_ty_var(
                TypeVariableOrigin::LatticeVariable(this.cause().span));
            this.relate_bound(v, a, b)?;
            Ok(v)
        }

        (&ty::TyInfer(TyVar(..)), _) |
        (_, &ty::TyInfer(TyVar(..))) => {
            let v = infcx.next_ty_var(TypeVariableOrigin::LatticeVariable(this.cause().span));
            this.relate_bound(v, a, b)?;
            Ok(v)
        }

        _ => {
            infcx.super_combine_tys(this, a, b)
        }
    }
}
<commit_msg>avoid unneeded subtype obligations in lub/glb<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! # Lattice Variables
//!
//! This file contains generic code for operating on inference variables
//! that are characterized by an upper- and lower-bound.  The logic and
//! reasoning is explained in detail in the large comment in `infer.rs`.
//!
//! The code in here is defined quite generically so that it can be
//! applied both to type variables, which represent types being inferred,
//! and fn variables, which represent function types being inferred.
//! It may eventually be applied to their types as well, who knows.
//! In some cases, the functions are also generic with respect to the
//! operation on the lattice (GLB vs LUB).
//!
//! Although all the functions are generic, we generally write the
//! comments in a way that is specific to type variables and the LUB
//! operation.  It's just easier that way.
//!
//! In general all of the functions are defined parametrically
//! over a `LatticeValue`, which is a value defined with respect to
//! a lattice.

use super::InferCtxt;
use super::type_variable::TypeVariableOrigin;

use traits::ObligationCause;
use ty::TyVar;
use ty::{self, Ty};
use ty::relate::{RelateResult, TypeRelation};

pub trait LatticeDir<'f, 'gcx: 'f+'tcx, 'tcx: 'f> : TypeRelation<'f, 'gcx, 'tcx> {
    fn infcx(&self) -> &'f InferCtxt<'f, 'gcx, 'tcx>;

    fn cause(&self) -> &ObligationCause<'tcx>;

    // Relates the type `v` to `a` and `b` such that `v` represents
    // the LUB/GLB of `a` and `b` as appropriate.
    //
    // Subtle hack: ordering *may* be significant here. This method
    // relates `v` to `a` first, which may help us to avoid unecessary
    // type variable obligations. See caller for details.
    fn relate_bound(&mut self, v: Ty<'tcx>, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, ()>;
}

pub fn super_lattice_tys<'a, 'gcx, 'tcx, L>(this: &mut L,
                                            a: Ty<'tcx>,
                                            b: Ty<'tcx>)
                                            -> RelateResult<'tcx, Ty<'tcx>>
    where L: LatticeDir<'a, 'gcx, 'tcx>, 'gcx: 'a+'tcx, 'tcx: 'a
{
    debug!("{}.lattice_tys({:?}, {:?})",
           this.tag(),
           a,
           b);

    if a == b {
        return Ok(a);
    }

    let infcx = this.infcx();
    let a = infcx.type_variables.borrow_mut().replace_if_possible(a);
    let b = infcx.type_variables.borrow_mut().replace_if_possible(b);
    match (&a.sty, &b.sty) {
        (&ty::TyInfer(TyVar(..)), &ty::TyInfer(TyVar(..)))
            if infcx.type_var_diverges(a) && infcx.type_var_diverges(b) => {
            let v = infcx.next_diverging_ty_var(
                TypeVariableOrigin::LatticeVariable(this.cause().span));
            this.relate_bound(v, a, b)?;
            Ok(v)
        }

        // If one side is known to be a variable and one is not,
        // create a variable (`v`) to represent the LUB. Make sure to
        // relate `v` to the non-type-variable first (by passing it
        // first to `relate_bound`). Otherwise, we would produce a
        // subtype obligation that must then be processed.
        //
        // Example: if the LHS is a type variable, and RHS is
        // `Box<i32>`, then we current compare `v` to the RHS first,
        // which will instantiate `v` with `Box<i32>`.  Then when `v`
        // is compared to the LHS, we instantiate LHS with `Box<i32>`.
        // But if we did in reverse order, we would create a `v <:
        // LHS` (or vice versa) constraint and then instantiate
        // `v`. This would require further processing to achieve same
        // end-result; in partiular, this screws up some of the logic
        // in coercion, which expects LUB to figure out that the LHS
        // is (e.g.) `Box<i32>`. A more obvious solution might be to
        // iterate on the subtype obligations that are returned, but I
        // think this suffices. -nmatsakis
        (&ty::TyInfer(TyVar(..)), _) => {
            let v = infcx.next_ty_var(TypeVariableOrigin::LatticeVariable(this.cause().span));
            this.relate_bound(v, b, a)?;
            Ok(v)
        }
        (_, &ty::TyInfer(TyVar(..))) => {
            let v = infcx.next_ty_var(TypeVariableOrigin::LatticeVariable(this.cause().span));
            this.relate_bound(v, a, b)?;
            Ok(v)
        }

        _ => {
            infcx.super_combine_tys(this, a, b)
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Remove Timer.as_f64() and reimplement PartialCmp<commit_after><|endoftext|>


<commit_before><commit_msg>[add] simply realization of fibonacci number<commit_after>fn fib_seq(n: u32) -> u64 {
    let mut a: u64 = 0;
    let mut b: u64 = 1;
    for _ in 0..n {
        let c = b;
        b = a + b;
        a = c;
    }
    a
}

fn main() {
    println!("fib_seq(5) = {}", fib_seq(5));
}<|endoftext|>


<commit_before><commit_msg>use a separate shim for detached threads<commit_after><|endoftext|>


<commit_before>import lib.llvm.llvm;
import lib.llvm.llvm.ModuleRef;
import std._str;
import std._vec;
import util.common.istr;

const int wordsz = 4;

fn wstr(int i) -> str {
    ret istr(i * wordsz);
}

fn save_callee_saves() -> vec[str] {
    ret vec("pushl %ebp",
            "pushl %edi",
            "pushl %esi",
            "pushl %ebx");
}

fn restore_callee_saves() -> vec[str] {
    ret vec("popl  %ebx",
            "popl  %esi",
            "popl  %edi",
            "popl  %ebp");
}

fn load_esp_from_rust_sp() -> vec[str] {
    ret vec("movl  " + wstr(abi.task_field_rust_sp) + "(%edx), %esp");
}

fn load_esp_from_runtime_sp() -> vec[str] {
    ret vec("movl  " + wstr(abi.task_field_runtime_sp) + "(%edx), %esp");
}

fn store_esp_to_rust_sp() -> vec[str] {
    ret vec("movl  %esp, " + wstr(abi.task_field_rust_sp) + "(%edx)");
}

fn store_esp_to_runtime_sp() -> vec[str] {
    ret vec("movl  %esp, " + wstr(abi.task_field_runtime_sp) + "(%edx)");
}

fn rust_activate_glue() -> vec[str] {
    ret vec("movl  4(%esp), %edx    # edx = rust_task")
        + save_callee_saves()
        + store_esp_to_runtime_sp()
        + load_esp_from_rust_sp()

        // This 'add' instruction is a bit surprising.
        // See lengthy comment in boot/be/x86.ml activate_glue.
        + vec("addl  $20, " + wstr(abi.task_field_rust_sp) + "(%edx)")

        + restore_callee_saves()
        + vec("ret");
}

fn rust_yield_glue() -> vec[str] {
    ret vec("movl  0(%esp), %edx    # edx = rust_task")
        + load_esp_from_rust_sp()
        + save_callee_saves()
        + store_esp_to_rust_sp()
        + load_esp_from_runtime_sp()
        + restore_callee_saves()
        + vec("ret");
}

fn upcall_glue(int n_args) -> vec[str] {

    /*
     * 0, 4, 8, 12 are callee-saves
     * 16 is retpc
     * 20 is taskptr
     * 24 is callee
     * 28 .. (7+i) * 4 are args
     */

    fn copy_arg(uint i) -> str {
        auto off = wstr(7 + (i as int));
        auto m = vec("movl  " + off + "(%ebp),%edx",
                     "movl  %edx," + off + "(%esp)");
        ret _str.connect(m, "\n\t");
    }

    auto carg = copy_arg;

    ret
        save_callee_saves()

        + vec("movl  %esp, %ebp     # ebp = rust_sp",
              "movl  20(%esp), %edx # edx = rust_task")

        + store_esp_to_rust_sp()
        + load_esp_from_runtime_sp()

        + vec("subl  $" + wstr(n_args + 1) + ", %esp   # esp -= args",
              "andl  $~0xf, %esp    # align esp down",
              "movl  %edx, (%esp)   # arg[0] = rust_task ")

        + _vec.init_fn[str](carg, n_args as uint)

        +  vec("movl  24(%ebp), %edx # edx = callee",
               "call  *%edx          # call *%edx",
               "movl  20(%ebp), %edx # edx = rust_task")

        + load_esp_from_rust_sp()
        + restore_callee_saves()
        + vec("ret");

}

fn decl_glue(int align, str prefix, str name, vec[str] insns) -> str {
    auto sym = prefix + name;
    ret "\t.globl " + sym + "\n" +
        "\t.balign " + istr(align) + "\n" +
        sym + ":\n" +
        "\t" + _str.connect(insns, "\n\t");
}

fn decl_upcall_glue(int align, str prefix, uint n) -> str {
    let int i = n as int;
    ret decl_glue(align, prefix,
                  abi.upcall_glue_name(i),
                  upcall_glue(i));
}

fn get_module_asm() -> str {
    auto align = 4;
    auto prefix = "";

    auto glues =
        vec(decl_glue(align, prefix,
                      abi.activate_glue_name(),
                      rust_activate_glue()),

            decl_glue(align, prefix,
                      abi.yield_glue_name(),
                      rust_yield_glue()))

        + _vec.init_fn[str](bind decl_upcall_glue(align, prefix, _),
                            abi.n_upcall_glues as uint);

    ret _str.connect(glues, "\n\n");
}

//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C ../.. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
//
<commit_msg>Fix indexing bug in rustc's indirect-upcall arg-copying loops.<commit_after>import lib.llvm.llvm;
import lib.llvm.llvm.ModuleRef;
import std._str;
import std._vec;
import util.common.istr;

const int wordsz = 4;

fn wstr(int i) -> str {
    ret istr(i * wordsz);
}

fn save_callee_saves() -> vec[str] {
    ret vec("pushl %ebp",
            "pushl %edi",
            "pushl %esi",
            "pushl %ebx");
}

fn restore_callee_saves() -> vec[str] {
    ret vec("popl  %ebx",
            "popl  %esi",
            "popl  %edi",
            "popl  %ebp");
}

fn load_esp_from_rust_sp() -> vec[str] {
    ret vec("movl  " + wstr(abi.task_field_rust_sp) + "(%edx), %esp");
}

fn load_esp_from_runtime_sp() -> vec[str] {
    ret vec("movl  " + wstr(abi.task_field_runtime_sp) + "(%edx), %esp");
}

fn store_esp_to_rust_sp() -> vec[str] {
    ret vec("movl  %esp, " + wstr(abi.task_field_rust_sp) + "(%edx)");
}

fn store_esp_to_runtime_sp() -> vec[str] {
    ret vec("movl  %esp, " + wstr(abi.task_field_runtime_sp) + "(%edx)");
}

fn rust_activate_glue() -> vec[str] {
    ret vec("movl  4(%esp), %edx    # edx = rust_task")
        + save_callee_saves()
        + store_esp_to_runtime_sp()
        + load_esp_from_rust_sp()

        // This 'add' instruction is a bit surprising.
        // See lengthy comment in boot/be/x86.ml activate_glue.
        + vec("addl  $20, " + wstr(abi.task_field_rust_sp) + "(%edx)")

        + restore_callee_saves()
        + vec("ret");
}

fn rust_yield_glue() -> vec[str] {
    ret vec("movl  0(%esp), %edx    # edx = rust_task")
        + load_esp_from_rust_sp()
        + save_callee_saves()
        + store_esp_to_rust_sp()
        + load_esp_from_runtime_sp()
        + restore_callee_saves()
        + vec("ret");
}

fn upcall_glue(int n_args) -> vec[str] {

    /*
     * 0, 4, 8, 12 are callee-saves
     * 16 is retpc
     * 20 is taskptr
     * 24 is callee
     * 28 .. (7+i) * 4 are args
     */

    fn copy_arg(uint i) -> str {
        auto src_off = wstr(7 + (i as int));
        auto dst_off = wstr(1 + (i as int));
        auto m = vec("movl  " + src_off + "(%ebp),%edx",
                     "movl  %edx," + dst_off + "(%esp)");
        ret _str.connect(m, "\n\t");
    }

    auto carg = copy_arg;

    ret
        save_callee_saves()

        + vec("movl  %esp, %ebp     # ebp = rust_sp",
              "movl  20(%esp), %edx # edx = rust_task")

        + store_esp_to_rust_sp()
        + load_esp_from_runtime_sp()

        + vec("subl  $" + wstr(n_args + 1) + ", %esp   # esp -= args",
              "andl  $~0xf, %esp    # align esp down",
              "movl  %edx, (%esp)   # arg[0] = rust_task ")

        + _vec.init_fn[str](carg, n_args as uint)

        +  vec("movl  24(%ebp), %edx # edx = callee",
               "call  *%edx          # call *%edx",
               "movl  20(%ebp), %edx # edx = rust_task")

        + load_esp_from_rust_sp()
        + restore_callee_saves()
        + vec("ret");

}

fn decl_glue(int align, str prefix, str name, vec[str] insns) -> str {
    auto sym = prefix + name;
    ret "\t.globl " + sym + "\n" +
        "\t.balign " + istr(align) + "\n" +
        sym + ":\n" +
        "\t" + _str.connect(insns, "\n\t");
}

fn decl_upcall_glue(int align, str prefix, uint n) -> str {
    let int i = n as int;
    ret decl_glue(align, prefix,
                  abi.upcall_glue_name(i),
                  upcall_glue(i));
}

fn get_module_asm() -> str {
    auto align = 4;
    auto prefix = "";

    auto glues =
        vec(decl_glue(align, prefix,
                      abi.activate_glue_name(),
                      rust_activate_glue()),

            decl_glue(align, prefix,
                      abi.yield_glue_name(),
                      rust_yield_glue()))

        + _vec.init_fn[str](bind decl_upcall_glue(align, prefix, _),
                            abi.n_upcall_glues as uint);

    ret _str.connect(glues, "\n\n");
}

//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C ../.. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
//
<|endoftext|>


<commit_before><commit_msg>Added a version of msgsend that uses pipes and select. Here, select is way too slow to be useful, but this can be optimized.<commit_after>// A port of the simplistic benchmark from
//
//    http://github.com/PaulKeeble/ScalaVErlangAgents
//
// I *think* it's the same, more or less.

use std;
import io::writer;
import io::writer_util;

import pipes::{port, chan};

macro_rules! move {
    { $x:expr } => { unsafe { let y <- *ptr::addr_of($x); y } }
}

enum request {
    get_count,
    bytes(uint),
    stop
}

fn server(requests: port_set<request>, responses: pipes::chan<uint>) {
    let mut count = 0u;
    let mut done = false;
    while !done {
        alt requests.try_recv() {
          some(get_count) { responses.send(copy count); }
          some(bytes(b)) {
            //#error("server: received %? bytes", b);
            count += b;
          }
          none { done = true; }
          _ { }
        }
    }
    responses.send(count);
    //#error("server exiting");
}

fn run(args: &[str]) {
    let (to_parent, from_child) = pipes::stream();
    let (to_child, from_parent_) = pipes::stream();
    let from_parent = port_set();
    from_parent.add(from_parent_);

    let size = option::get(uint::from_str(args[1]));
    let workers = option::get(uint::from_str(args[2]));
    let num_bytes = 100;
    let start = std::time::precise_time_s();
    let mut worker_results = ~[];
    for uint::range(0u, workers) |i| {
        let builder = task::builder();
        vec::push(worker_results, task::future_result(builder));
        let (to_child, from_parent_) = pipes::stream();
        from_parent.add(from_parent_);
        do task::run(builder) {
            for uint::range(0u, size / workers) |_i| {
                //#error("worker %?: sending %? bytes", i, num_bytes);
                to_child.send(bytes(num_bytes));
            }
            //#error("worker %? exiting", i);
        };
    }
    do task::spawn {
        server(from_parent, to_parent);
    }

    vec::iter(worker_results, |r| { future::get(r); } );
    //#error("sending stop message");
    to_child.send(stop);
    move!{to_child};
    let result = from_child.recv();
    let end = std::time::precise_time_s();
    let elapsed = end - start;
    io::stdout().write_str(#fmt("Count is %?\n", result));
    io::stdout().write_str(#fmt("Test took %? seconds\n", elapsed));
    let thruput = ((size / workers * workers) as float) / (elapsed as float);
    io::stdout().write_str(#fmt("Throughput=%f per sec\n", thruput));
    assert result == num_bytes * size;
}

fn main(args: ~[str]) {
    let args = if os::getenv("RUST_BENCH").is_some() {
        ~["", "1000000", "10000"]
    } else if args.len() <= 1u {
        ~["", "10000", "4"]
    } else {
        copy args
    };

    #debug("%?", args);
    run(args);
}

// Treat a whole bunch of ports as one.
class box<T> {
    let mut contents: option<T>;
    new(+x: T) { self.contents = some(x); }

    fn swap(f: fn(+T) -> T) {
        let mut tmp = none;
        self.contents <-> tmp;
        self.contents = some(f(option::unwrap(tmp)));
    }

    fn unwrap() -> T {
        let mut tmp = none;
        self.contents <-> tmp;
        option::unwrap(tmp)
    }
}

class port_set<T: send> {
    let ports: box<~[pipes::streamp::server::open<T>]>;

    new() { self.ports = box(~[]); }

    fn add(+port: pipes::port<T>) {
        let pipes::port_(port) <- port;
        let mut p = none;
        port.endp <-> p;
        do self.ports.swap |ports| {
            let mut p_ = none;
            p <-> p_;
            vec::append_one(ports, option::unwrap(p_))
        }
    }

    fn try_recv() -> option<T> {
        let mut result = none;
        let mut done = false;
        while !done {
            do self.ports.swap |ports| {
                if ports.len() > 0 {
                    let old_len = ports.len();
                    let (_, m, ports) = pipes::select(ports);
                    alt m {
                      some(pipes::streamp::data(x, next)) {
                        result = some(move!{x});
                        done = true;
                        assert ports.len() == old_len - 1;
                        vec::append_one(ports, move!{next})
                      }
                      none {
                        //#error("pipe closed");
                        assert ports.len() == old_len - 1;
                        ports
                      }
                    }
                }
                else {
                    //#error("no more pipes");
                    done = true;
                    ~[]
                }
            }
        }
        result
    }

    fn recv() -> T {
        option::unwrap(self.try_recv())
    }
}
<|endoftext|>


<commit_before><commit_msg>Test statement macros.<commit_after>// xfail-pretty - token trees can't pretty print

macro_rules! myfn(
    ( $f:ident, ( $( $x:ident ),* ), $body:block ) => (
        fn $f( $( $x : int),* ) -> int $body
    )
)

myfn!(add, (a,b), { return a+b; } )

fn main() {

    macro_rules! mylet(
        ($x:ident, $val:expr) => (
            let $x = $val;
        )
    );

    mylet!(y, 8*2);
    assert(y == 16);

    myfn!(mult, (a,b), { a*b } );

    assert (mult(2, add(4,4)) == 16);

    macro_rules! actually_an_expr_macro (
        () => ( 16 )
    )

    assert { actually_an_expr_macro!() } == 16;

}
<|endoftext|>


<commit_before><commit_msg>fetcher: fix broken genericity<commit_after><|endoftext|>