<commit_before>#![feature(asm)]

// External libraries
extern crate dylib;
use dylib::DynamicLibrary;

use std::path::Path;
use std::mem::transmute;

use std::process::Command;
use std::thread;

use std::sync::atomic::{AtomicBool, Ordering, AtomicPtr};
use std::option::Option;

use std::sync::{Arc, RwLock};
use std::fmt;

// Internal libraries
extern crate gr;
use gr::*;

#[derive(Clone)]
enum OpType {
    Func(String),
    Const(usize),
    Var(usize),
    UVar(usize),
    Op(String),
    Pow(i32)
}

impl fmt::Display for OpType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

        let outstring = match self {
            &OpType::Func(ref s) => format!("f{}", s),
            &OpType::Const(i) => format!("c{}", i),
            &OpType::Var(i) => format!("i{}", i),
            &OpType::UVar(i) => format!("v{}", i),
            &OpType::Op(ref s) => format!("o{}", s),
            &OpType::Pow(i) => format!("p{}", i)
        };
        write!(f, "{}", outstring)
    }

}

#[derive(Clone)]
pub struct FuncObj {
    handle: Arc<RwLock<Option<DynamicLibrary>>>,
    user_vars: Vec<GI>,
    constants: Vec<GI>,
    instructions: Vec<OpType>,
    switched: Arc<AtomicBool>,
    function: Arc<AtomicPtr<fn(&Vec<GI>, &Vec<GI>) -> (GI, Option<Vec<GI>>)>>,

}

unsafe impl Sync for FuncObj {}
unsafe impl Send for FuncObj {}
fn dummy(_x: &Vec<GI>, _c: &Vec<GI>) -> (GI, Option<Vec<GI>>) {
    (GI::new_c("1.0").unwrap(), None)
}

impl FuncObj {
    pub fn call(&self, _x: &Vec<GI>) -> (GI, Option<Vec<GI>>) {
        if self.switched.load(Ordering::Acquire) {
            let real_func = unsafe{
                std::mem::transmute::<*mut fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>),
                                      fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(
                    (self.function.load(Ordering::Acquire)))};
            real_func(_x, &self.constants)
        }
        else {
            self.interpreted(_x, &self.constants)
        }
    }

    fn set(&self, f: fn(&Vec<GI>,&Vec<GI>) -> (GI, Option<Vec<GI>>), handle: DynamicLibrary) {
        // This FuncObj owns the handle from the dynamic lib as the lifetime of
        // f is tied to the lifetime of the handle.
        self.function.store(unsafe{std::mem::transmute::<fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>),
                                                         *mut fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(f)}, Ordering::Release);
        let mut h = self.handle.write().unwrap();
        *h = Some(handle);
        self.switched.store(true, Ordering::Release);
    }

    fn interpreted(&self, _x: &Vec<GI>, _c: &Vec<GI>) -> (GI, Option<Vec<GI>>) {
        let mut stack: Vec<GI> = Vec::new();
        for inst in &self.instructions {
            match inst {
                &OpType::Func(ref s) => {
                    let op = stack.last_mut().unwrap();
                    match s.as_str() {
                        "abs" => op.abs(),
                        "sin" => op.sin(),
                        "asin"=> op.asin(),
                        "cos" => op.cos(),
                        "acos" => op.acos(),
                        "tan" => op.tan(),
                        "atan" => op.atan(),
                        "exp" => op.exp(),
                        "log" => op.log(),
                        "neg" => op.neg(),
                        "sqrt" => op.sqrt(),
                        "sinh" => op.sinh(),
                        "cosh" => op.cosh(),
                        "tanh" => op.tanh(),
                        "asinh" => op.asinh(),
                        "acosh" => op.acosh(),
                        "atanh" => op.atanh(),
                        "floor_power2" => op.floor_power2(),
                        "sym_interval" => op.sym_interval(),
                        _     => unreachable!()
                    };
                },
                &OpType::Const(i) => {
                    stack.push(self.constants[i]);
                },
                &OpType::Var(i) => {
                    stack.push(_x[i]);
                },
                &OpType::UVar(i) => {
                    stack.push(self.user_vars[i]);
                },
                &OpType::Op(ref s) => {
                    let right = stack.pop().unwrap();
                    let left = stack.last_mut().unwrap();
                    match s.as_str() {
                        "+" => left.add(right),
                        "-" => left.sub(right),
                        "*" => left.mul(right),
                        "/" => left.div(right),
                        "p" => left.powi(right),
                        _   => unreachable!()
                    };
                },
                &OpType::Pow(exp) => {
                    let arg = stack.last_mut().unwrap();
                    arg.pow(exp);
                }
            }
        }
        (stack[0], None)
    }

    pub fn new(consts: &Vec<GI>, instructions: &String, debug: bool, suffix: String) -> FuncObj {
        let mut insts = vec![];

        for inst in instructions.split(',') {
            let dummy = inst.trim().to_string();
            let (first, rest) = dummy.split_at(1);
            insts.push(match first {
                "c" => OpType::Const(rest.to_string().parse::<usize>().unwrap()),
                "i" => OpType::Var(rest.to_string().parse::<usize>().unwrap()),
                "v" => OpType::UVar(rest.to_string().parse::<usize>().unwrap()),
                "o" => OpType::Op(rest.to_string()),
                "f" => OpType::Func(rest.to_string()),
                "p" => OpType::Pow(rest.to_string().parse::<i32>().unwrap()),
                _   => panic!()
            });
        }
        let result =
            FuncObj{handle: Arc::new(RwLock::new(Option::None)),
                user_vars: vec![],
                constants: consts.clone(),
                instructions: insts,
                switched: Arc::new(AtomicBool::new(false)),
                function: Arc::new(AtomicPtr::new(unsafe{std::mem::transmute::<fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>),
                                                                               *mut fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(dummy)}))
        };

        {
            let fo_c = result.clone();
            thread::spawn(move || {
                &fo_c.compile(debug, &suffix);
            })
        };
        result
    }

    fn compile(&self, debug: bool, suffix: &String) {
        let mut process = Command::new("build_func.sh");
        if debug {
            println!("compile suffix: {}", suffix);
        }
        let ignore = process.arg(suffix.clone()).output()
            .unwrap_or_else(|e| {panic!("Could not compile: {}", e)});

        if !ignore.status.success() {
            panic!("Could not compile: {}\n----\n{}", String::from_utf8(ignore.stdout).unwrap(),
                   String::from_utf8(ignore.stderr).unwrap());
        }

        DynamicLibrary::prepend_search_path(Path::new("./.compiled"));
        let dylib: String = "libfunc_".to_string() + suffix + ".so".into();

        let f = match DynamicLibrary::open(Some(Path::new(&dylib)))
        {
            Ok(lib) => lib,
            Err(err) => panic!("Could not load library: {}", err)
        };
        let g = unsafe{match f.symbol("gelpia_func") {
            Ok(func) => transmute::<*mut u32, fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(func),
            Err(err) => panic!("Could not load function: {}", err),
        }};

        self.set(g, f);
    }
}

#[test]
fn test1() {
    let constants = vec![GI::new_c("1"), GI::new_c("2"), GI::new_c("3")];
    let f = FuncObj::new(&constants,
                         &"c0,c1,o-,c2,o+".to_string());
    let expected = ((GI::new_c("1") - GI::new_c("2")) + GI::new_c("3")).to_string();
    assert!(f.call(&vec![]).to_string() == expected);
}

#[test]
fn test2() {
    let consts = vec![GI::new_c("1"), GI::new_c("2"), GI::new_c("3")];
    let f = FuncObj::new(&consts,
                         &"c0,fsin,c1,o*,fcos,c2,o/".to_string());
    let expected = cos(sin(consts[0]) * consts[1])/consts[2];
    assert!(f.call(&vec![]).to_string() == expected.to_string());
}

#[test]
fn sum_inputs() {
    let consts = vec![];
    let v = vec![GI::new_d(1.0, 1.0), GI::new_d(1.0, 2.0), GI::new_d(1.0, 3.0),
                 GI::new_d(1.0, 4.0), GI::new_d(1.0, 5.0), GI::new_d(1.0, 6.0)];
    let f = FuncObj::new(&consts,
                         &"i0,i1, o+, i2, o+, i3, o+, i4, o+, i5, o+".to_string());
    let expected = (v[0] + v[1] + v[2] + v[3] + v[4] + v[5]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test3() {
    let consts = vec![GI::new_c("0.1")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fsin".to_string());
    let expected = sin(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test4() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fabs".to_string());
    let expected = abs(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test5() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fcos".to_string());
    let expected = cos(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test6() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, ftan".to_string());
    let expected = tan(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test7() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fexp".to_string());
    let expected = exp(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test8() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fneg".to_string());
    let expected = (-consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test9() {
    let consts = vec![GI::new_c("[0.1, .2]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, flog".to_string());
    let expected = log(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}
#[test]
fn test10() {
    let consts = vec![GI::new_c("[2.0, 3.0]"),
                      GI::new_c("[1.5, 2.5]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                         &"c0, c1, op".to_string());
    let expected = powi(consts[0], consts[1]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}
#[test]
fn test_power() {
    let consts = vec![GI::new_c("[2.0, 3.0]"),
                      GI::new_c("[1.5, 2.5]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                         &"c0, p-1, c1, p2, o+".to_string());
    let expected = (pow(consts[0], -1) + pow(consts[1], 2)).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}
<commit_msg>Don't panic when failing to compile<commit_after>#![feature(asm)]

// External libraries
extern crate dylib;
use dylib::DynamicLibrary;

use std::path::Path;
use std::mem::transmute;

use std::process::Command;
use std::thread;

use std::sync::atomic::{AtomicBool, Ordering, AtomicPtr};
use std::option::Option;

use std::sync::{Arc, RwLock};
use std::fmt;

// Internal libraries
extern crate gr;
use gr::*;

#[derive(Clone)]
enum OpType {
    Func(String),
    Const(usize),
    Var(usize),
    UVar(usize),
    Op(String),
    Pow(i32)
}

impl fmt::Display for OpType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

        let outstring = match self {
            &OpType::Func(ref s) => format!("f{}", s),
            &OpType::Const(i) => format!("c{}", i),
            &OpType::Var(i) => format!("i{}", i),
            &OpType::UVar(i) => format!("v{}", i),
            &OpType::Op(ref s) => format!("o{}", s),
            &OpType::Pow(i) => format!("p{}", i)
        };
        write!(f, "{}", outstring)
    }

}

#[derive(Clone)]
pub struct FuncObj {
    handle: Arc<RwLock<Option<DynamicLibrary>>>,
    user_vars: Vec<GI>,
    constants: Vec<GI>,
    instructions: Vec<OpType>,
    switched: Arc<AtomicBool>,
    function: Arc<AtomicPtr<fn(&Vec<GI>, &Vec<GI>) -> (GI, Option<Vec<GI>>)>>,

}

unsafe impl Sync for FuncObj {}
unsafe impl Send for FuncObj {}
fn dummy(_x: &Vec<GI>, _c: &Vec<GI>) -> (GI, Option<Vec<GI>>) {
    (GI::new_c("1.0").unwrap(), None)
}

impl FuncObj {
    pub fn call(&self, _x: &Vec<GI>) -> (GI, Option<Vec<GI>>) {
        if self.switched.load(Ordering::Acquire) {
            let real_func = unsafe{
                std::mem::transmute::<*mut fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>),
                                      fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(
                    (self.function.load(Ordering::Acquire)))};
            real_func(_x, &self.constants)
        }
        else {
            self.interpreted(_x, &self.constants)
        }
    }

    fn set(&self, f: fn(&Vec<GI>,&Vec<GI>) -> (GI, Option<Vec<GI>>), handle: DynamicLibrary) {
        // This FuncObj owns the handle from the dynamic lib as the lifetime of
        // f is tied to the lifetime of the handle.
        self.function.store(unsafe{std::mem::transmute::<fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>),
                                                         *mut fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(f)}, Ordering::Release);
        let mut h = self.handle.write().unwrap();
        *h = Some(handle);
        self.switched.store(true, Ordering::Release);
    }

    fn interpreted(&self, _x: &Vec<GI>, _c: &Vec<GI>) -> (GI, Option<Vec<GI>>) {
        let mut stack: Vec<GI> = Vec::new();
        for inst in &self.instructions {
            match inst {
                &OpType::Func(ref s) => {
                    let op = stack.last_mut().unwrap();
                    match s.as_str() {
                        "abs" => op.abs(),
                        "sin" => op.sin(),
                        "asin"=> op.asin(),
                        "cos" => op.cos(),
                        "acos" => op.acos(),
                        "tan" => op.tan(),
                        "atan" => op.atan(),
                        "exp" => op.exp(),
                        "log" => op.log(),
                        "neg" => op.neg(),
                        "sqrt" => op.sqrt(),
                        "sinh" => op.sinh(),
                        "cosh" => op.cosh(),
                        "tanh" => op.tanh(),
                        "asinh" => op.asinh(),
                        "acosh" => op.acosh(),
                        "atanh" => op.atanh(),
                        "floor_power2" => op.floor_power2(),
                        "sym_interval" => op.sym_interval(),
                        _     => unreachable!()
                    };
                },
                &OpType::Const(i) => {
                    stack.push(self.constants[i]);
                },
                &OpType::Var(i) => {
                    stack.push(_x[i]);
                },
                &OpType::UVar(i) => {
                    stack.push(self.user_vars[i]);
                },
                &OpType::Op(ref s) => {
                    let right = stack.pop().unwrap();
                    let left = stack.last_mut().unwrap();
                    match s.as_str() {
                        "+" => left.add(right),
                        "-" => left.sub(right),
                        "*" => left.mul(right),
                        "/" => left.div(right),
                        "p" => left.powi(right),
                        _   => unreachable!()
                    };
                },
                &OpType::Pow(exp) => {
                    let arg = stack.last_mut().unwrap();
                    arg.pow(exp);
                }
            }
        }
        (stack[0], None)
    }

    pub fn new(consts: &Vec<GI>, instructions: &String, debug: bool, suffix: String) -> FuncObj {
        let mut insts = vec![];

        for inst in instructions.split(',') {
            let dummy = inst.trim().to_string();
            let (first, rest) = dummy.split_at(1);
            insts.push(match first {
                "c" => OpType::Const(rest.to_string().parse::<usize>().unwrap()),
                "i" => OpType::Var(rest.to_string().parse::<usize>().unwrap()),
                "v" => OpType::UVar(rest.to_string().parse::<usize>().unwrap()),
                "o" => OpType::Op(rest.to_string()),
                "f" => OpType::Func(rest.to_string()),
                "p" => OpType::Pow(rest.to_string().parse::<i32>().unwrap()),
                _   => panic!()
            });
        }
        let result =
            FuncObj{handle: Arc::new(RwLock::new(Option::None)),
                user_vars: vec![],
                constants: consts.clone(),
                instructions: insts,
                switched: Arc::new(AtomicBool::new(false)),
                function: Arc::new(AtomicPtr::new(unsafe{std::mem::transmute::<fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>),
                                                                               *mut fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(dummy)}))
        };

        {
            let fo_c = result.clone();
            thread::spawn(move || {
                &fo_c.compile(debug, &suffix);
            })
        };
        result
    }

    fn compile(&self, debug: bool, suffix: &String) {
        let mut process = Command::new("build_func.sh");
        if debug {
            println!("compile suffix: {}", suffix);
        }
        let ignore = process.arg(suffix.clone()).output();
        if !ignore.is_ok() {
            return;
        }
        let ignore2 = ignore.unwrap();
        if !ignore2.status.success() {
            return;
        }
        DynamicLibrary::prepend_search_path(Path::new("./.compiled"));
        let dylib: String = "libfunc_".to_string() + suffix + ".so".into();

        let fl = DynamicLibrary::open(Some(Path::new(&dylib)));
        if !fl.is_ok() {
            return;
        }
        let f = fl.unwrap();

        let g = unsafe{match f.symbol("gelpia_func") {
            Ok(func) => transmute::<*mut u32, fn(&Vec<GI>, &Vec<GI>)->(GI, Option<Vec<GI>>)>(func),
            Err(err) => panic!("Could not load function: {}", err),
        }};
        self.set(g, f);
    }
}

#[test]
fn test1() {
    let constants = vec![GI::new_c("1"), GI::new_c("2"), GI::new_c("3")];
    let f = FuncObj::new(&constants,
                         &"c0,c1,o-,c2,o+".to_string());
    let expected = ((GI::new_c("1") - GI::new_c("2")) + GI::new_c("3")).to_string();
    assert!(f.call(&vec![]).to_string() == expected);
}

#[test]
fn test2() {
    let consts = vec![GI::new_c("1"), GI::new_c("2"), GI::new_c("3")];
    let f = FuncObj::new(&consts,
                         &"c0,fsin,c1,o*,fcos,c2,o/".to_string());
    let expected = cos(sin(consts[0]) * consts[1])/consts[2];
    assert!(f.call(&vec![]).to_string() == expected.to_string());
}

#[test]
fn sum_inputs() {
    let consts = vec![];
    let v = vec![GI::new_d(1.0, 1.0), GI::new_d(1.0, 2.0), GI::new_d(1.0, 3.0),
                 GI::new_d(1.0, 4.0), GI::new_d(1.0, 5.0), GI::new_d(1.0, 6.0)];
    let f = FuncObj::new(&consts,
                         &"i0,i1, o+, i2, o+, i3, o+, i4, o+, i5, o+".to_string());
    let expected = (v[0] + v[1] + v[2] + v[3] + v[4] + v[5]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test3() {
    let consts = vec![GI::new_c("0.1")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fsin".to_string());
    let expected = sin(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test4() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fabs".to_string());
    let expected = abs(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test5() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fcos".to_string());
    let expected = cos(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test6() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, ftan".to_string());
    let expected = tan(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test7() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fexp".to_string());
    let expected = exp(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test8() {
    let consts = vec![GI::new_c("[-3, 4]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, fneg".to_string());
    let expected = (-consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}

#[test]
fn test9() {
    let consts = vec![GI::new_c("[0.1, .2]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                &"c0, flog".to_string());
    let expected = log(consts[0]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}
#[test]
fn test10() {
    let consts = vec![GI::new_c("[2.0, 3.0]"),
                      GI::new_c("[1.5, 2.5]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                         &"c0, c1, op".to_string());
    let expected = powi(consts[0], consts[1]).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}
#[test]
fn test_power() {
    let consts = vec![GI::new_c("[2.0, 3.0]"),
                      GI::new_c("[1.5, 2.5]")];
    let v = vec![];
    let f = FuncObj::new(&consts,
                         &"c0, p-1, c1, p2, o+".to_string());
    let expected = (pow(consts[0], -1) + pow(consts[1], 2)).to_string();
    let real = f.call(&v).to_string();
    assert!(real == expected, "real = {}, expected = {}", real, expected);
}
<|endoftext|>


<commit_before>use path;

use std::borrow::Cow;
use std::path::PathBuf;

use syn;

pub struct TemplateInput<'a> {
    pub ast: &'a syn::DeriveInput,
    pub meta: TemplateMeta<'a>,
    pub path: PathBuf,
    pub source: Cow<'a, str>,
}

impl<'a> TemplateInput<'a> {
    pub fn new(ast: &'a syn::DeriveInput) -> TemplateInput<'a> {
        let meta = TemplateMeta::new(ast);
        let (path, source) = match meta.source {
            Source::Source(s) => (PathBuf::new(), Cow::Borrowed(s)),
            Source::Path(s) => {
                let path = path::find_template_from_path(&s, None);
                let src = path::get_template_source(&path);
                (path, Cow::Owned(src))
            },
        };
        TemplateInput { ast, meta, path, source }
    }
}

// Holds metadata for the template, based on the `template()` attribute.
pub struct TemplateMeta<'a> {
    source: Source<'a>,
    pub print: Print,
    pub escaping: EscapeMode,
}

impl<'a> TemplateMeta<'a> {
    fn new(ast: &'a syn::DeriveInput) -> TemplateMeta<'a> {
        let attr = ast.attrs.iter().find(|a| a.name() == "template");
        if attr.is_none() {
            let msg = format!("'template' attribute not found on struct '{}'",
                              ast.ident.as_ref());
            panic!(msg);
        }

        let attr = attr.unwrap();
        let mut source = None;
        let mut print = Print::None;
        let mut escaping = EscapeMode::Html;
        if let syn::MetaItem::List(_, ref inner) = attr.value {
            for nm_item in inner {
                if let syn::NestedMetaItem::MetaItem(ref item) = *nm_item {
                    if let syn::MetaItem::NameValue(ref key, ref val) = *item {
                        match key.as_ref() {
                            "path" => if let syn::Lit::Str(ref s, _) = *val {
                                source = Some(Source::Path(s.as_ref()));
                            } else {
                                panic!("template path must be string literal");
                            },
                            "source" => if let syn::Lit::Str(ref s, _) = *val {
                                source = Some(Source::Source(s.as_ref()));
                            } else {
                                panic!("template source must be string literal");
                            },
                            "print" => if let syn::Lit::Str(ref s, _) = *val {
                                print = s.into();
                            } else {
                                panic!("print value must be string literal");
                            },
                            "escape" => if let syn::Lit::Str(ref s, _) = *val {
                                escaping = s.into();
                            } else {
                                panic!("escape value must be string literal");
                            },
                            _ => { panic!("unsupported annotation key found") }
                        }
                    }
                }
            }
        }

        match source {
            Some(s) => TemplateMeta { source: s, print, escaping },
            None => panic!("template path or source not found in struct attributes"),
        }
    }
}

enum Source<'a> {
    Path(&'a str),
    Source(&'a str),
}

#[derive(PartialEq)]
pub enum EscapeMode {
    Html,
    None,
}

impl<'a> From<&'a String> for EscapeMode {
    fn from(s: &'a String) -> EscapeMode {
        use self::EscapeMode::*;
        match s.as_ref() {
            "html" => Html,
            "none" => None,
            v => panic!("invalid value for escape option: {}", v),
        }
    }
}

#[derive(PartialEq)]
pub enum Print {
    All,
    Ast,
    Code,
    None,
}

impl<'a> From<&'a String> for Print {
    fn from(s: &'a String) -> Print {
        use self::Print::*;
        match s.as_ref() {
            "all" => All,
            "ast" => Ast,
            "code" => Code,
            "none" => None,
            v => panic!("invalid value for print option: {}", v),
        }
    }
}
<commit_msg>Refactor type handling for attribute parameters<commit_after>use path;

use std::borrow::Cow;
use std::path::PathBuf;

use syn;

pub struct TemplateInput<'a> {
    pub ast: &'a syn::DeriveInput,
    pub meta: TemplateMeta<'a>,
    pub path: PathBuf,
    pub source: Cow<'a, str>,
}

impl<'a> TemplateInput<'a> {
    pub fn new(ast: &'a syn::DeriveInput) -> TemplateInput<'a> {
        let meta = TemplateMeta::new(ast);
        let (path, source) = match meta.source {
            Source::Source(s) => (PathBuf::new(), Cow::Borrowed(s)),
            Source::Path(s) => {
                let path = path::find_template_from_path(&s, None);
                let src = path::get_template_source(&path);
                (path, Cow::Owned(src))
            },
        };
        TemplateInput { ast, meta, path, source }
    }
}

// Holds metadata for the template, based on the `template()` attribute.
pub struct TemplateMeta<'a> {
    source: Source<'a>,
    pub print: Print,
    pub escaping: EscapeMode,
}

impl<'a> TemplateMeta<'a> {
    fn new(ast: &'a syn::DeriveInput) -> TemplateMeta<'a> {
        let attr = ast.attrs.iter().find(|a| a.name() == "template");
        if attr.is_none() {
            let msg = format!("'template' attribute not found on struct '{}'",
                              ast.ident.as_ref());
            panic!(msg);
        }

        let attr = attr.unwrap();
        let mut source = None;
        let mut print = Print::None;
        let mut escaping = EscapeMode::Html;
        if let syn::MetaItem::List(_, ref inner) = attr.value {
            for nm_item in inner {
                if let syn::NestedMetaItem::MetaItem(ref item) = *nm_item {
                    if let syn::MetaItem::NameValue(ref key, ref val) = *item {
                        match key.as_ref() {
                            "path" => if let syn::Lit::Str(ref s, _) = *val {
                                source = Some(Source::Path(s.as_ref()));
                            } else {
                                panic!("template path must be string literal");
                            },
                            "source" => if let syn::Lit::Str(ref s, _) = *val {
                                source = Some(Source::Source(s.as_ref()));
                            } else {
                                panic!("template source must be string literal");
                            },
                            "print" => if let syn::Lit::Str(ref s, _) = *val {
                                print = (s.as_ref() as &str).into();
                            } else {
                                panic!("print value must be string literal");
                            },
                            "escape" => if let syn::Lit::Str(ref s, _) = *val {
                                escaping = (s.as_ref() as &str).into();
                            } else {
                                panic!("escape value must be string literal");
                            },
                            _ => { panic!("unsupported annotation key found") }
                        }
                    }
                }
            }
        }

        match source {
            Some(s) => TemplateMeta { source: s, print, escaping },
            None => panic!("template path or source not found in struct attributes"),
        }
    }
}

enum Source<'a> {
    Path(&'a str),
    Source(&'a str),
}

#[derive(PartialEq)]
pub enum EscapeMode {
    Html,
    None,
}

impl<'a> From<&'a str> for EscapeMode {
    fn from(s: &'a str) -> EscapeMode {
        use self::EscapeMode::*;
        match s {
            "html" => Html,
            "none" => None,
            v => panic!("invalid value for escape option: {}", v),
        }
    }
}

#[derive(PartialEq)]
pub enum Print {
    All,
    Ast,
    Code,
    None,
}

impl<'a> From<&'a str> for Print {
    fn from(s: &'a str) -> Print {
        use self::Print::*;
        match s {
            "all" => All,
            "ast" => Ast,
            "code" => Code,
            "none" => None,
            v => panic!("invalid value for print option: {}", v),
        }
    }
}
<|endoftext|>


<commit_before>#![feature(phase)]

#[phase(plugin)]
extern crate static_fragments;
extern crate fragments;

template! greeting {"Hello, [[:name]]!"}

fn main() {
    let mut template = greeting::new();

    //Insert something into the `name` placeholder
    template.insert_name("Peter");

    //Templates can be printed as they are
    //Result: 'Hello, Peter!'
    println!("Result: '{}'", template);
}<commit_msg>Update example for macro reform<commit_after>#![feature(plugin)]

#[plugin]
#[no_link]
extern crate static_fragments;
extern crate fragments;

template! greeting {"Hello, [[:name]]!"}

fn main() {
    let mut template = greeting::new();

    //Insert something into the `name` placeholder
    template.insert_name("Peter");

    //Templates can be printed as they are
    //Result: 'Hello, Peter!'
    println!("Result: '{}'", template);
}<|endoftext|>


<commit_before>#[cfg(test)]
use game;
use game::VictoryState;
use ai;
use ai::run_match;
use constants::LINES;
use std::rc::Rc;

#[test]
fn match_mc() {
    let structure = Rc::new(game::Structure::new(&LINES));
    let mut white_player = ai::mc::MonteCarloAI::new(structure.clone(), 1000);
    let mut black_player = ai::mc::MonteCarloAI::new(structure.clone(), 1000);
    run_match(structure, &mut white_player, &mut black_player);
}

#[test]
fn match_mc_tree() {
    let structure = Rc::new(game::Structure::new(&LINES));
    let mut white_player = ai::mc::MonteCarloAI::new(structure.clone(), 1000);
    let mut black_player =
        ai::tree::TreeJudgementAI::new(structure.clone(), 2, ai::value::Simple::Subsets);
    run_match(structure, &mut white_player, &mut black_player);
}

#[test]
fn match_tree() {
    let structure = Rc::new(game::Structure::new(&LINES));
    let mut white_player =
        ai::tree::TreeJudgementAI::new(structure.clone(), 2, ai::value::Simple::Subsets);
    let mut black_player =
        ai::tree::TreeJudgementAI::new(structure.clone(), 2, ai::value::Simple::Subsets);
    run_match(structure, &mut white_player, &mut black_player);
}

#[test]
fn subset_coherence() {
    // This is a property based test, see QuickCheck for more information.
    use rand::{thread_rng, Rng};

    let mut rng = thread_rng();

    for _ in 0..10000 {
        // Ensure that all positions returned by the Subset iterator are
        // contained in the Subset.
        let subset = game::Subset(rng.next_u64());
        for position in subset.iter() {
            println!("{:?}", position);
            assert!(subset.contains(position));
        }
    }
}
<commit_msg>Fix tests.<commit_after>#[cfg(test)]
use game;
use ai;
use ai::run_match;
use constants::LINES;
use std::rc::Rc;

#[test]
fn match_mc() {
    let structure = Rc::new(game::Structure::new(&LINES));
    let mut white_player = ai::mc::MonteCarloAI::new(1000);
    let mut black_player = ai::mc::MonteCarloAI::new(1000);
    run_match(structure, &mut white_player, &mut black_player);
}

#[test]
fn match_mc_tree() {
    let structure = Rc::new(game::Structure::new(&LINES));
    let mut white_player = ai::mc::MonteCarloAI::new(1000);
    let mut black_player =
        ai::tree::TreeJudgementAI::new(structure.clone(), 2, ai::value::Simple::Subsets);
    run_match(structure, &mut white_player, &mut black_player);
}

#[test]
fn match_tree() {
    let structure = Rc::new(game::Structure::new(&LINES));
    let mut white_player =
        ai::tree::TreeJudgementAI::new(structure.clone(), 2, ai::value::Simple::Subsets);
    let mut black_player =
        ai::tree::TreeJudgementAI::new(structure.clone(), 2, ai::value::Simple::Subsets);
    run_match(structure, &mut white_player, &mut black_player);
}

#[test]
fn subset_coherence() {
    // This is a property based test, see QuickCheck for more information.
    use rand::{thread_rng, Rng};

    let mut rng = thread_rng();

    for _ in 0..10000 {
        // Ensure that all positions returned by the Subset iterator are
        // contained in the Subset.
        let subset = game::Subset(rng.next_u64());
        for position in subset.iter() {
            println!("{:?}", position);
            assert!(subset.contains(position));
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use std::collections::BinaryHeap;
use std::sync::Mutex;
use std::usize;

/// Thread ID manager which allocates thread IDs. It attempts to aggressively
/// reuse thread IDs where possible to avoid cases where a ThreadLocal grows
/// indefinitely when it is used by many short-lived threads.
struct ThreadIdManager {
    free_from: usize,
    free_list: BinaryHeap<usize>,
}
impl ThreadIdManager {
    fn new() -> ThreadIdManager {
        ThreadIdManager {
            free_from: 0,
            free_list: BinaryHeap::new(),
        }
    }
    fn alloc(&mut self) -> usize {
        if let Some(id) = self.free_list.pop() {
            id
        } else {
            let id = self.free_from;
            self.free_from = self
                .free_from
                .checked_add(1)
                .expect("Ran out of thread IDs");
            id
        }
    }
    fn free(&mut self, id: usize) {
        self.free_list.push(id);
    }
}
lazy_static! {
    static ref THREAD_ID_MANAGER: Mutex<ThreadIdManager> = Mutex::new(ThreadIdManager::new());
}

/// Non-zero integer which is unique to the current thread while it is running.
/// A thread ID may be reused after a thread exits.
struct ThreadId(usize);
impl ThreadId {
    fn new() -> ThreadId {
        ThreadId(THREAD_ID_MANAGER.lock().unwrap().alloc())
    }
}
impl Drop for ThreadId {
    fn drop(&mut self) {
        THREAD_ID_MANAGER.lock().unwrap().free(self.0);
    }
}
thread_local!(static THREAD_ID: ThreadId = ThreadId::new());

/// Returns a non-zero ID for the current thread
pub(crate) fn get() -> usize {
    THREAD_ID.with(|x| x.0)
}
<commit_msg>Reverse the thread id binary heap<commit_after>// Copyright 2017 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use std::cmp::Reverse;
use std::collections::BinaryHeap;
use std::sync::Mutex;
use std::usize;

/// Thread ID manager which allocates thread IDs. It attempts to aggressively
/// reuse thread IDs where possible to avoid cases where a ThreadLocal grows
/// indefinitely when it is used by many short-lived threads.
struct ThreadIdManager {
    free_from: usize,
    free_list: BinaryHeap<Reverse<usize>>,
}
impl ThreadIdManager {
    fn new() -> ThreadIdManager {
        ThreadIdManager {
            free_from: 0,
            free_list: BinaryHeap::new(),
        }
    }
    fn alloc(&mut self) -> usize {
        if let Some(id) = self.free_list.pop() {
            id.0
        } else {
            let id = self.free_from;
            self.free_from = self
                .free_from
                .checked_add(1)
                .expect("Ran out of thread IDs");
            id
        }
    }
    fn free(&mut self, id: usize) {
        self.free_list.push(Reverse(id));
    }
}
lazy_static! {
    static ref THREAD_ID_MANAGER: Mutex<ThreadIdManager> = Mutex::new(ThreadIdManager::new());
}

/// Non-zero integer which is unique to the current thread while it is running.
/// A thread ID may be reused after a thread exits.
struct ThreadId(usize);
impl ThreadId {
    fn new() -> ThreadId {
        ThreadId(THREAD_ID_MANAGER.lock().unwrap().alloc())
    }
}
impl Drop for ThreadId {
    fn drop(&mut self) {
        THREAD_ID_MANAGER.lock().unwrap().free(self.0);
    }
}
thread_local!(static THREAD_ID: ThreadId = ThreadId::new());

/// Returns a non-zero ID for the current thread
pub(crate) fn get() -> usize {
    THREAD_ID.with(|x| x.0)
}
<|endoftext|>


<commit_before>use std::cell::RefCell;
use image;
use image::{SubImage, GenericImage};
use calx::{V2, Canvas, Image, Rgb};
use calx;

local_data_key!(TILE_CACHE: RefCell<Vec<Image>>)

fn batch(tiles: &mut Vec<Image>, ctx: &mut Canvas, data: &[u8],
       elt_dim: (int, int), offset: (int, int)) {
    let mut image = calx::color_key(
        &image::load_from_memory(data, image::PNG).unwrap(),
        &Rgb::new(0x80u8, 0x80u8, 0x80u8));
    let (w, h) = image.dimensions();
    let (columns, rows) = (w / elt_dim.0 as u32, h / elt_dim.1 as u32);

    for y in range(0, rows) {
        for x in range(0, columns) {
            tiles.push(ctx.add_image(V2(offset.0, offset.1), SubImage::new(
                &mut image,
                x * elt_dim.0 as u32, y * elt_dim.1 as u32,
                elt_dim.0 as u32, elt_dim.1 as u32)));
        }
    }
}

/// Initialize global tile cache.
pub fn init(ctx: &mut Canvas) {
    let mut tiles: Vec<Image> = vec![];
    batch(&mut tiles, ctx, include_bin!("../assets/tile.png"), (32, 32), (-16, -16));
    batch(&mut tiles, ctx, include_bin!("../assets/icon.png"), (8, 8), (0, -8));
    batch(&mut tiles, ctx, include_bin!("../assets/logo.png"), (92, 25), (0, 0));
    TILE_CACHE.replace(Some(RefCell::new(tiles)));
}

pub fn get(idx: uint) -> Image {
    assert!(TILE_CACHE.get().is_some(), "Tile cache not initialized");
    (*TILE_CACHE.get().unwrap().borrow())[idx].clone()
}

pub mod tile {
    pub static CUBE : uint = 0;
    pub static CURSOR_BOTTOM : uint = 1;
    pub static CURSOR_TOP : uint = 2;
    pub static BLOCK_NW : uint = 3;
    pub static BLOCK_N : uint = 4;
    pub static BLOCK_NE : uint = 5;
    pub static BLOCK_DARK : uint = 6;
    pub static CHASM : uint = 7;
    pub static SHALLOWS : uint = 8;
    pub static PORTAL : uint = 9;
    pub static BLANK_FLOOR : uint = 10;
    pub static FLOOR : uint = 11;
    pub static GRASS : uint = 12;
    pub static WATER : uint = 13;
    pub static MAGMA : uint = 14;
    pub static DOWNSTAIRS : uint = 15;
    pub static ROCKWALL : uint = 16;
    pub static WALL : uint = 20;
    pub static FENCE : uint = 24;
    pub static BARS : uint = 28;
    pub static WINDOW : uint = 32;
    pub static DOOR : uint = 36;
    pub static TREE_TRUNK : uint = 48;
    pub static TREE_FOLIAGE : uint = 49;
    pub static TABLE : uint = 50;
    pub static AVATAR : uint = 51;
    pub static BLOCK : uint = 52;
    pub static FOUNTAIN : uint = 53;
    pub static ALTAR : uint = 54;
    pub static BARREL : uint = 55;
    pub static STALAGMITE : uint = 56;
    pub static GRAVE : uint = 58;
    pub static STONE : uint = 69;
    pub static MENHIR : uint = 70;
    pub static TALLGRASS : uint = 80;
}

pub mod icon {
    static ICON_OFFSET: uint = 256;

    pub static HEART : uint = ICON_OFFSET + 0;
    pub static HALF_HEART : uint = ICON_OFFSET + 1;
    pub static NO_HEART : uint = ICON_OFFSET + 2;
    pub static SHARD : uint = ICON_OFFSET + 3;
    pub static HALF_SHARD : uint = ICON_OFFSET + 4;
    pub static NO_SHARD : uint = ICON_OFFSET + 5;
}

pub static LOGO: uint = 256 + 128;
<commit_msg>Upgrade to latest Rust<commit_after>use std::cell::RefCell;
use image;
use image::{SubImage, GenericImage};
use calx::{V2, Canvas, Image, Rgb};
use calx;

local_data_key!(TILE_CACHE: RefCell<Vec<Image>>)

fn batch(tiles: &mut Vec<Image>, ctx: &mut Canvas, data: &[u8],
       elt_dim: (int, int), offset: (int, int)) {
    let mut image = calx::color_key(
        &image::load_from_memory(data, image::PNG).unwrap(),
        &Rgb::new(0x80u8, 0x80u8, 0x80u8));
    let (w, h) = image.dimensions();
    let (columns, rows) = (w / elt_dim.0 as u32, h / elt_dim.1 as u32);

    for y in range(0, rows) {
        for x in range(0, columns) {
            tiles.push(ctx.add_image(V2(offset.0, offset.1), SubImage::new(
                &mut image,
                x * elt_dim.0 as u32, y * elt_dim.1 as u32,
                elt_dim.0 as u32, elt_dim.1 as u32)));
        }
    }
}

/// Initialize global tile cache.
pub fn init(ctx: &mut Canvas) {
    let mut tiles: Vec<Image> = vec![];
    batch(&mut tiles, ctx, include_bin!("../assets/tile.png"), (32, 32), (-16, -16));
    batch(&mut tiles, ctx, include_bin!("../assets/icon.png"), (8, 8), (0, -8));
    batch(&mut tiles, ctx, include_bin!("../assets/logo.png"), (92, 25), (0, 0));
    TILE_CACHE.replace(Some(RefCell::new(tiles)));
}

pub fn get(idx: uint) -> Image {
    assert!(TILE_CACHE.get().is_some(), "Tile cache not initialized");
    (*TILE_CACHE.get().unwrap().borrow())[idx].clone()
}

pub mod tile {
    pub static CUBE : uint = 0;
    pub static CURSOR_BOTTOM : uint = 1;
    pub static CURSOR_TOP : uint = 2;
    pub static BLOCK_NW : uint = 3;
    pub static BLOCK_N : uint = 4;
    pub static BLOCK_NE : uint = 5;
    pub static BLOCK_DARK : uint = 6;
    pub static CHASM : uint = 7;
    pub static SHALLOWS : uint = 8;
    pub static PORTAL : uint = 9;
    pub static BLANK_FLOOR : uint = 10;
    pub static FLOOR : uint = 11;
    pub static GRASS : uint = 12;
    pub static WATER : uint = 13;
    pub static MAGMA : uint = 14;
    pub static DOWNSTAIRS : uint = 15;
    pub static ROCKWALL : uint = 16;
    pub static WALL : uint = 20;
    pub static FENCE : uint = 24;
    pub static BARS : uint = 28;
    pub static WINDOW : uint = 32;
    pub static DOOR : uint = 36;
    pub static TREE_TRUNK : uint = 48;
    pub static TREE_FOLIAGE : uint = 49;
    pub static TABLE : uint = 50;
    pub static AVATAR : uint = 51;
    pub static BLOCK : uint = 52;
    pub static FOUNTAIN : uint = 53;
    pub static ALTAR : uint = 54;
    pub static BARREL : uint = 55;
    pub static STALAGMITE : uint = 56;
    pub static GRAVE : uint = 58;
    pub static STONE : uint = 69;
    pub static MENHIR : uint = 70;
    pub static TALLGRASS : uint = 80;
}

pub mod icon {
    pub static HEART : uint = 256;
    pub static HALF_HEART : uint = 257;
    pub static NO_HEART : uint = 258;
    pub static SHARD : uint = 259;
    pub static HALF_SHARD : uint = 260;
    pub static NO_SHARD : uint = 261;
}

pub static LOGO: uint = 256 + 128;
<|endoftext|>


<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    InvalidSymbol,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let char = self.scanner.get_char();
            if char.is_none() {
                break;
            }

            let char = char.unwrap();
            if !char.is_whitespace() || char == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert!(tokenizer.next().unwrap().unwrap() == StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::InvalidSymbol);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "12[3abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh#*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<commit_msg>implementing a basic tokenizer next() method<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    SymbolCannotBeNumber,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }

    fn next_symbol(&mut self, start: char) -> Option<TokenResult> {
        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else {
                if let Err(message) = self.scanner.unget_char() {
                    return Some(Err(TokenError::ScannerError(message.to_owned())));
                }
                break;
            }

            next_char = self.scanner.get_char();
        }

        if symbol.len() == 0 {
            Some(Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            }))
        }
        else if symbol.chars().next().map_or(false, |c| c.is_digit(10)) {
            Some(Err(TokenError::SymbolCannotBeNumber))
        }
        else if symbol == "=" {
            Some(Ok(Token::Equals))
        }
        else {
            Some(Ok(Token::Symbol(symbol)))
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            '\\' => Token::Backslash,
            '(' => Token::ParenOpen,
            ')' => Token::ParenClose,
            '"' => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => {
                if let Some(symbol_result) = self.next_symbol(c) {
                    return Some(symbol_result);
                }
                else {
                    return None;
                }
            },
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert!(tokenizer.next().unwrap().unwrap() == StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::SymbolCannotBeNumber);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "12[3abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh#*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<|endoftext|>


<commit_before>use error::{SassError, ErrorKind, Result};
use event::Event;
use sass::comment::SassComment;
use sass::rule::SassRule;
use sass::selector::SassSelector;
use sass::variable::SassVariable;
use top_level_event::TopLevelEvent;
use tokenizer_utils::*;

use std::borrow::Cow::Borrowed;
use std::borrow::Cow;

#[derive(Debug)]
pub struct Tokenizer<'a> {
    toker: Toker<'a>,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule_selectors_done: bool,
}

#[derive(PartialEq, Debug, Copy, Clone)]
enum State {
    OutsideRules,
    InVariable,
    InComment,
    InRule,
    InSelectors,
    InProperties,
    EndRule,
    Eof,
}

impl<'a> Tokenizer<'a> {
    pub fn new(inner_str: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            toker: Toker {
                inner_str: &inner_str,
                bytes: &inner_str.as_bytes(),
                offset: 0,
            },
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule_selectors_done: false,
        }
    }

    fn limit(&self) -> usize {
        self.toker.limit()
    }

    fn start_something(&mut self) -> Result<Option<TopLevelEvent<'a>>> {
        let mut current_sass_rule = SassRule::new();
        self.current_sass_rule_selectors_done = false;

        self.pick_something();
        while self.state != State::OutsideRules {
            if self.state == State::Eof { return Ok(None) }

            // See if this works now that State is Copy
            // self.state = match self.state

            if self.state == State::InSelectors {
                match try!(self.next_selector()) {
                    Some(sel) => current_sass_rule.selectors.push(sel),
                    None => {},
                }
            } else if self.state == State::InProperties {
                match try!(self.next_property()) {
                    Some(prop) => current_sass_rule.children.push(prop),
                    None => {},
                }
            } else if self.state == State::EndRule {
                try!(self.toker.eat("}"));

                match self.sass_rule_stack.pop() {
                    Some(mut rule) => {
                        rule.children.push(Event::ChildRule(current_sass_rule));
                        current_sass_rule = rule;
                        self.pick_something();
                    },
                    None => self.state = State::OutsideRules,
                }
            } else if self.state == State::InRule {
                self.sass_rule_stack.push(current_sass_rule);
                current_sass_rule = SassRule::new();
                self.current_sass_rule_selectors_done = false;
                self.pick_something();
            } else if self.state == State::InVariable {
                return self.next_variable()
            } else if self.state == State::InComment {
                let comment = try!(self.next_comment());
                if comment.is_some() {
                    if self.sass_rule_stack.len() == 0 &&
                       current_sass_rule.selectors.len() == 0 &&
                       current_sass_rule.children.len() == 0 {
                           return Ok(Some(
                               TopLevelEvent::Comment(SassComment { comment: comment.unwrap() })
                           ))
                    } else {
                        current_sass_rule.children.push(comment.unwrap());
                        self.pick_something();
                    }
                } else {
                    return Ok(None)
                }
            } else {
                return Err(SassError {
                    kind: ErrorKind::TokenizerError,
                    message: format!(
                        "Something unexpected happened in tokenization! Current tokenization state: {:?}. Current sass rule = {:?}",
                        self.state,
                        current_sass_rule
                    ),
                })
            }
        }

        Ok(Some(TopLevelEvent::Rule(current_sass_rule)))
    }

    fn pick_something(&mut self) {
        self.toker.skip_leading_whitespace();

        if self.toker.at_eof() {
            self.state = State::Eof;
            return
        }

        let c = self.toker.bytes[self.toker.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.toker.offset + 1) < self.limit() {
            let d = self.toker.bytes[self.toker.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        if self.current_sass_rule_selectors_done {
            self.state = State::InProperties;
            return
        }

        self.state = State::InSelectors;
    }

    fn next_comment(&mut self) -> Result<Option<Event<'a>>> {
        let comment_body_beginning = self.toker.offset;
        let mut i = comment_body_beginning + 2;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, isnt_asterisk);
            self.toker.offset = i;

            if self.toker.eat("*/").is_ok() {
                return Ok(Some(
                    Event::Comment(Borrowed(
                        &self.toker.inner_str[comment_body_beginning..self.toker.offset]
                    ))
                ))
            } else {
                i += 1;
            }
        }
        self.toker.offset = self.limit();
        Err(SassError {
            kind: ErrorKind::UnexpectedEof,
            message: String::from(
                "Expected comment; reached EOF instead."
            ),
        })
    }

    fn next_name(&mut self) -> Result<Cow<'a, str>> {
        let name_beginning = self.toker.offset;
        let mut i = name_beginning;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, valid_name_char);
            let name_end = i;
            self.toker.offset = i;
            return Ok(Borrowed(&self.toker.inner_str[name_beginning..name_end]))
        }
        self.toker.offset = self.limit();
        Err(SassError {
            kind: ErrorKind::UnexpectedEof,
            message: String::from(
                "Expected a valid name; reached EOF instead."
            ),
        })
    }

    fn next_value(&mut self) -> Result<Cow<'a, str>> {
        let value_beginning = self.toker.offset;
        let mut i = value_beginning;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, isnt_semicolon);
            let value_end = i;
            self.toker.offset = i;
            return Ok(Borrowed(&self.toker.inner_str[value_beginning..value_end]))
        }
        self.toker.offset = self.limit();
        Err(SassError {
            kind: ErrorKind::UnexpectedEof,
            message: String::from(
                "Expected a valid value; reached EOF instead."
            ),
        })
    }

    fn next_variable(&mut self) -> Result<Option<TopLevelEvent<'a>>> {
        let var_name = try!(self.next_name());

        try!(self.toker.eat(":"));
        self.toker.skip_leading_whitespace();

        let var_value = try!(self.next_value());

        try!(self.toker.eat(";"));
        self.toker.skip_leading_whitespace();

        Ok(Some(TopLevelEvent::Variable(SassVariable {
            name:  var_name,
            value: var_value,
        })))
    }

    fn next_property(&mut self) -> Result<Option<Event<'a>>> {
        self.toker.skip_leading_whitespace();

        if self.toker.at_eof() {
            self.state = State::Eof;
            return Ok(None)
        }

        let c = self.toker.curr_byte();
        if c == b'}' {
            self.state = State::EndRule;
            return Ok(None)
        }

        let d = self.toker.next_byte();
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return Ok(None)
        }

        let saved_offset = self.toker.offset;

        let prop_name = try!(self.next_name());

        let c = self.toker.curr_byte();
        if c == b'{' {
            self.state = State::InRule;
            self.toker.offset = saved_offset;
            return Ok(None)
        }

        try!(self.toker.eat(":"));
        self.toker.skip_leading_whitespace();

        let prop_value = try!(self.next_value());

        try!(self.toker.eat(";"));
        self.toker.skip_leading_whitespace();

        if prop_name.as_bytes()[0] == b'$' {
            Ok(Some(Event::Variable(SassVariable {
                name:  prop_name,
                value: prop_value,
            })))
        } else {
            Ok(Some(Event::UnevaluatedProperty(
                prop_name,
                prop_value,
            )))
        }
    }

    fn next_selector(&mut self) -> Result<Option<SassSelector<'a>>> {
        self.toker.skip_leading_whitespace();

        let beginning = self.toker.offset;
        let mut i = beginning;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, valid_selector_char);
            let c = self.toker.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return Ok(None)
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.toker.inner_str[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule_selectors_done {
                            self.state = State::InRule;
                            return Ok(None)
                        } else {
                            self.current_sass_rule_selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.toker.offset = i + 1;
                    return Ok(Some(SassSelector::new(&self.toker.inner_str[beginning..end])))
                } else {
                    // only whitespace between commas
                    self.toker.offset += 1;
                    return self.next_selector()
                }
            }

            self.toker.offset = i;
            if i > beginning {
                return Ok(Some(SassSelector::new(&self.toker.inner_str[beginning..i])))
            }
            i += 1;
        }

        if i > beginning {
            self.toker.offset = i;
            Ok(Some(SassSelector::new(&self.toker.inner_str[beginning..i])))
        } else {
            self.state = State::Eof;
            Ok(None)
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = Result<TopLevelEvent<'a>>;

    fn next(&mut self) -> Option<Result<TopLevelEvent<'a>>> {
        if !self.toker.at_eof() {
            return match self.start_something() {
                Ok(Some(t)) => Some(Ok(t)),
                Ok(None) => None,
                Err(e) => Some(Err(e)),
            }
        }
        None
    }
}
<commit_msg>Remove unnecessary check<commit_after>use error::{SassError, ErrorKind, Result};
use event::Event;
use sass::comment::SassComment;
use sass::rule::SassRule;
use sass::selector::SassSelector;
use sass::variable::SassVariable;
use top_level_event::TopLevelEvent;
use tokenizer_utils::*;

use std::borrow::Cow::Borrowed;
use std::borrow::Cow;

#[derive(Debug)]
pub struct Tokenizer<'a> {
    toker: Toker<'a>,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule_selectors_done: bool,
}

#[derive(PartialEq, Debug, Copy, Clone)]
enum State {
    OutsideRules,
    InVariable,
    InComment,
    InRule,
    InSelectors,
    InProperties,
    EndRule,
    Eof,
}

impl<'a> Tokenizer<'a> {
    pub fn new(inner_str: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            toker: Toker {
                inner_str: &inner_str,
                bytes: &inner_str.as_bytes(),
                offset: 0,
            },
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule_selectors_done: false,
        }
    }

    fn limit(&self) -> usize {
        self.toker.limit()
    }

    fn start_something(&mut self) -> Result<Option<TopLevelEvent<'a>>> {
        let mut current_sass_rule = SassRule::new();
        self.current_sass_rule_selectors_done = false;

        self.pick_something();
        while self.state != State::OutsideRules {
            if self.state == State::Eof { return Ok(None) }

            // See if this works now that State is Copy
            // self.state = match self.state

            if self.state == State::InSelectors {
                match try!(self.next_selector()) {
                    Some(sel) => current_sass_rule.selectors.push(sel),
                    None => {},
                }
            } else if self.state == State::InProperties {
                match try!(self.next_property()) {
                    Some(prop) => current_sass_rule.children.push(prop),
                    None => {},
                }
            } else if self.state == State::EndRule {
                try!(self.toker.eat("}"));

                match self.sass_rule_stack.pop() {
                    Some(mut rule) => {
                        rule.children.push(Event::ChildRule(current_sass_rule));
                        current_sass_rule = rule;
                        self.pick_something();
                    },
                    None => self.state = State::OutsideRules,
                }
            } else if self.state == State::InRule {
                self.sass_rule_stack.push(current_sass_rule);
                current_sass_rule = SassRule::new();
                self.current_sass_rule_selectors_done = false;
                self.pick_something();
            } else if self.state == State::InVariable {
                return self.next_variable()
            } else if self.state == State::InComment {
                let comment = try!(self.next_comment());
                if comment.is_some() {
                    if self.sass_rule_stack.len() == 0 &&
                       current_sass_rule.selectors.len() == 0 &&
                       current_sass_rule.children.len() == 0 {
                           return Ok(Some(
                               TopLevelEvent::Comment(SassComment { comment: comment.unwrap() })
                           ))
                    } else {
                        current_sass_rule.children.push(comment.unwrap());
                        self.pick_something();
                    }
                } else {
                    return Ok(None)
                }
            } else {
                return Err(SassError {
                    kind: ErrorKind::TokenizerError,
                    message: format!(
                        "Something unexpected happened in tokenization! Current tokenization state: {:?}. Current sass rule = {:?}",
                        self.state,
                        current_sass_rule
                    ),
                })
            }
        }

        Ok(Some(TopLevelEvent::Rule(current_sass_rule)))
    }

    fn pick_something(&mut self) {
        self.toker.skip_leading_whitespace();

        if self.toker.at_eof() {
            self.state = State::Eof;
            return
        }

        let c = self.toker.bytes[self.toker.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.toker.offset + 1) < self.limit() {
            let d = self.toker.bytes[self.toker.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        if self.current_sass_rule_selectors_done {
            self.state = State::InProperties;
            return
        }

        self.state = State::InSelectors;
    }

    fn next_comment(&mut self) -> Result<Option<Event<'a>>> {
        let comment_body_beginning = self.toker.offset;
        let mut i = comment_body_beginning + 2;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, isnt_asterisk);
            self.toker.offset = i;

            if self.toker.eat("*/").is_ok() {
                return Ok(Some(
                    Event::Comment(Borrowed(
                        &self.toker.inner_str[comment_body_beginning..self.toker.offset]
                    ))
                ))
            } else {
                i += 1;
            }
        }
        self.toker.offset = self.limit();
        Err(SassError {
            kind: ErrorKind::UnexpectedEof,
            message: String::from(
                "Expected comment; reached EOF instead."
            ),
        })
    }

    fn next_name(&mut self) -> Result<Cow<'a, str>> {
        let name_beginning = self.toker.offset;
        let mut i = name_beginning;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, valid_name_char);
            let name_end = i;
            self.toker.offset = i;
            return Ok(Borrowed(&self.toker.inner_str[name_beginning..name_end]))
        }
        self.toker.offset = self.limit();
        Err(SassError {
            kind: ErrorKind::UnexpectedEof,
            message: String::from(
                "Expected a valid name; reached EOF instead."
            ),
        })
    }

    fn next_value(&mut self) -> Result<Cow<'a, str>> {
        let value_beginning = self.toker.offset;
        let mut i = value_beginning;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, isnt_semicolon);
            let value_end = i;
            self.toker.offset = i;
            return Ok(Borrowed(&self.toker.inner_str[value_beginning..value_end]))
        }
        self.toker.offset = self.limit();
        Err(SassError {
            kind: ErrorKind::UnexpectedEof,
            message: String::from(
                "Expected a valid value; reached EOF instead."
            ),
        })
    }

    fn next_variable(&mut self) -> Result<Option<TopLevelEvent<'a>>> {
        let var_name = try!(self.next_name());

        try!(self.toker.eat(":"));
        self.toker.skip_leading_whitespace();

        let var_value = try!(self.next_value());

        try!(self.toker.eat(";"));
        self.toker.skip_leading_whitespace();

        Ok(Some(TopLevelEvent::Variable(SassVariable {
            name:  var_name,
            value: var_value,
        })))
    }

    fn next_property(&mut self) -> Result<Option<Event<'a>>> {
        self.toker.skip_leading_whitespace();

        if self.toker.at_eof() {
            self.state = State::Eof;
            return Ok(None)
        }

        let c = self.toker.curr_byte();
        if c == b'}' {
            self.state = State::EndRule;
            return Ok(None)
        }

        let d = self.toker.next_byte();
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return Ok(None)
        }

        let saved_offset = self.toker.offset;

        let prop_name = try!(self.next_name());

        let c = self.toker.curr_byte();
        if c == b'{' {
            self.state = State::InRule;
            self.toker.offset = saved_offset;
            return Ok(None)
        }

        try!(self.toker.eat(":"));
        self.toker.skip_leading_whitespace();

        let prop_value = try!(self.next_value());

        try!(self.toker.eat(";"));
        self.toker.skip_leading_whitespace();

        if prop_name.as_bytes()[0] == b'$' {
            Ok(Some(Event::Variable(SassVariable {
                name:  prop_name,
                value: prop_value,
            })))
        } else {
            Ok(Some(Event::UnevaluatedProperty(
                prop_name,
                prop_value,
            )))
        }
    }

    fn next_selector(&mut self) -> Result<Option<SassSelector<'a>>> {
        self.toker.skip_leading_whitespace();

        let beginning = self.toker.offset;
        let mut i = beginning;

        while i < self.limit() {
            i += self.toker.scan_while_or_end(i, valid_selector_char);
            let c = self.toker.bytes[i];

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.toker.inner_str[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule_selectors_done {
                            self.state = State::InRule;
                            return Ok(None)
                        } else {
                            self.current_sass_rule_selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.toker.offset = i + 1;
                    return Ok(Some(SassSelector::new(&self.toker.inner_str[beginning..end])))
                } else {
                    // only whitespace between commas
                    self.toker.offset += 1;
                    return self.next_selector()
                }
            }

            self.toker.offset = i;
            if i > beginning {
                return Ok(Some(SassSelector::new(&self.toker.inner_str[beginning..i])))
            }
            i += 1;
        }

        if i > beginning {
            self.toker.offset = i;
            Ok(Some(SassSelector::new(&self.toker.inner_str[beginning..i])))
        } else {
            self.state = State::Eof;
            Ok(None)
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = Result<TopLevelEvent<'a>>;

    fn next(&mut self) -> Option<Result<TopLevelEvent<'a>>> {
        if !self.toker.at_eof() {
            return match self.start_something() {
                Ok(Some(t)) => Some(Ok(t)),
                Ok(None) => None,
                Err(e) => Some(Err(e)),
            }
        }
        None
    }
}
<|endoftext|>