<commit_before>mod bxdf;
mod fresnel;
mod lambertian;
mod oren_nayar;
mod microfacet;

pub use self::bxdf::*;
pub use self::fresnel::*;
pub use self::lambertian::*;
pub use self::oren_nayar::*;
pub use self::microfacet::*;

use std::cmp;

use na::{Cross, Dot, Norm};

use ::{Vector3f, Point2f, ONE_MINUS_EPSILON};
use spectrum::Spectrum;
use interaction::SurfaceInteraction;

bitflags! {
    pub flags BxDFType: u32 {
        const BSDF_REFLECTION   = 0b_00000001,
        const BSDF_TRANSMISSION = 0b_00000010,
        const BSDF_DIFFUSE      = 0b_00000100,
        const BSDF_GLOSSY       = 0b_00001000,
        const BSDF_SPECULAR     = 0b_00010000,
    }
}

/// Represents the Bidirectional Scattering Distribution Function.
/// It represents the properties of a material at a given point.
pub struct BSDF {
    /// Index of refraction of the surface
    eta: f32,
    /// Shading normal (i.e. potentially affected by bump-mapping)
    ns: Vector3f,
    /// Geometry normal
    ng: Vector3f,
    ss: Vector3f,
    ts: Vector3f,
    bxdfs: Vec<Box<BxDF + Sync + Send>>,
}

impl BSDF {
    pub fn new(isect: &SurfaceInteraction, eta: f32, bxdfs: Vec<Box<BxDF + Sync + Send>>) -> BSDF {
        let ss = isect.dpdu.normalize();
        BSDF {
            eta: eta,
            ns: isect.shading.n,
            ng: isect.n,
            ss: ss,
            ts: isect.shading.n.cross(&ss),
            bxdfs: bxdfs,
        }
    }

    /// Evaluate the BSDF for the given incoming light direction and outgoing light direction.
    pub fn f(&self, wo_w: &Vector3f, wi_w: &Vector3f, flags: BxDFType) -> Spectrum {
        let wi = self.world_to_local(wi_w);
        let wo = self.world_to_local(wo_w);
        if wo.z == 0.0 {
            return Spectrum::black();
        }
        let reflect = wi_w.dot(&self.ng) * wo_w.dot(&self.ng) > 0.0;
        self.bxdfs
            .iter()
            .filter(|b| {
                // Make sure we only evaluate reflection or transmission based on whether wi and wo
                // lie in the same hemisphere.
                b.matches(flags) &&
                ((reflect && (b.get_type().contains(BSDF_REFLECTION))) ||
                 (!reflect && (b.get_type().contains(BSDF_TRANSMISSION))))
            })
            .fold(Spectrum::black(), |c, b| c + b.f(&wo, &wi))
    }

    pub fn pdf(&self, wo_w: &Vector3f, wi_w: &Vector3f, flags: BxDFType) -> f32 {
        if self.bxdfs.is_empty() {
            return 0.0;
        }
        let wo = self.world_to_local(wo_w);
        if wo.z == 0.0 {
            return 0.0;
        }
        let wi = self.world_to_local(wi_w);

        let mut matched_comps = 0;
        let mut pdf = 0.0;
        for bxdf in &self.bxdfs {
            if bxdf.matches(flags) {
                matched_comps += 1;
                pdf += bxdf.pdf(&wo, &wi);
            }
        }
        if matched_comps == 0 {
            0.0
        } else {
            pdf / matched_comps as f32
        }
    }

    pub fn sample_f(&self,
                    wo_w: &Vector3f,
                    u: &Point2f,
                    flags: BxDFType)
                    -> (Spectrum, Vector3f, f32, BxDFType) {

        let matching_comps = self.bxdfs
            .iter()
            .filter(|b| b.matches(flags))
            .collect::<Vec<&Box<BxDF + Sync + Send>>>();
        if matching_comps.is_empty() {
            return (Spectrum::black(), Vector3f::new(0.0, 0.0, 0.0), 0.0, BxDFType::empty());
        }
        // Chose which BxDF to sample
        let comp = cmp::min((u[0] * matching_comps.len() as f32).floor() as usize,
                            matching_comps.len() - 1);
        let bxdf = matching_comps.get(comp).expect("Was expecting a BxDF with this index!");

        debug!("BDDF::sample_f chose comp = {} / matching {}",
               comp,
               matching_comps.len());

        // Remap BxDF sample u to [0,1)^2
        let u_remapped = Point2f::new((u[0] * matching_comps.len() as f32 - comp as f32)
                                          .min(ONE_MINUS_EPSILON),
                                      u[1]);
        // Sample chosen BxDF
        let wo = self.world_to_local(wo_w);
        if wo.z == 0.0 {
            return (Spectrum::black(), Vector3f::new(0.0, 0.0, 0.0), 0.0, bxdf.get_type());
        }
        let (mut f, wi, mut pdf, sampled_type) = bxdf.sample_f(&wo, &u_remapped);
        debug!("For wo = {:?}, sampled f = {}, pdf = {}, ratio = {}, wi = {:?}",
               wo,
               f,
               pdf,
               if pdf > 0.0 {
                   f / pdf
               } else {
                   Spectrum::black()
               },
               wi);
        if pdf == 0.0 {
            return (Spectrum::black(), Vector3f::new(0.0, 0.0, 0.0), 0.0, BxDFType::empty());
        }
        let wi_w = self.local_to_world(&wi);

        // Compute overall PDF with all matching BxDF
        if !bxdf.get_type().contains(BSDF_SPECULAR) && matching_comps.len() > 1 {
            for i in 0..matching_comps.len() {
                if i != comp {
                    pdf += matching_comps[i].pdf(&wo, &wi);
                }
            }
        }
        if matching_comps.len() > 1 {
            pdf /= matching_comps.len() as f32;
        }

        // Compute value of BSDF for sampled direction
        if !bxdf.get_type().contains(BSDF_SPECULAR) && matching_comps.len() > 1 {
            let reflect = wi_w.dot(&self.ng) * wo_w.dot(&self.ng) > 0.0;
            f = matching_comps.iter()
                .filter(|b| {
                    (reflect && b.get_type().contains(BSDF_REFLECTION)) ||
                    (!reflect && b.get_type().contains(BSDF_TRANSMISSION))
                })
                .fold(Spectrum::black(), |f, b| f + b.f(&wo, &wi));
        }

        debug!("Overall f = {}, pdf = {}, ratio = {}",
               f,
               pdf,
               if pdf > 0.0 {
                   f / pdf
               } else {
                   Spectrum::black()
               });
        (f, wi_w, pdf, sampled_type)
    }

    fn world_to_local(&self, v: &Vector3f) -> Vector3f {
        Vector3f::new(v.dot(&self.ss), v.dot(&self.ts), v.dot(&self.ns))
    }

    fn local_to_world(&self, v: &Vector3f) -> Vector3f {
        Vector3f::new(self.ss.x * v.x + self.ts.x * v.y + self.ns.x * v.z,
                      self.ss.y * v.x + self.ts.y * v.y + self.ns.y * v.z,
                      self.ss.z * v.z + self.ts.z * v.y + self.ns.z * v.z)
    }

    /// Return the number of BxDFs matching the given flags
    fn num_components(&self, flags: BxDFType) -> usize {
        self.bxdfs.iter().filter(|b| b.matches(flags)).count()
    }
}

#[test]
fn test_flags() {
    let flags = BSDF_SPECULAR | BSDF_REFLECTION;
    let bxdf_type = BSDF_SPECULAR | BSDF_REFLECTION | BSDF_TRANSMISSION;

    assert!((bxdf_type & flags) == flags);
}
<commit_msg>Expose some stuff needed by PathIntegrator<commit_after>mod bxdf;
mod fresnel;
mod lambertian;
mod oren_nayar;
mod microfacet;

pub use self::bxdf::*;
pub use self::fresnel::*;
pub use self::lambertian::*;
pub use self::oren_nayar::*;
pub use self::microfacet::*;

use std::cmp;

use na::{Cross, Dot, Norm};

use ::{Vector3f, Point2f, ONE_MINUS_EPSILON};
use spectrum::Spectrum;
use interaction::SurfaceInteraction;

bitflags! {
    pub flags BxDFType: u32 {
        const BSDF_REFLECTION   = 0b_00000001,
        const BSDF_TRANSMISSION = 0b_00000010,
        const BSDF_DIFFUSE      = 0b_00000100,
        const BSDF_GLOSSY       = 0b_00001000,
        const BSDF_SPECULAR     = 0b_00010000,
    }
}

/// Represents the Bidirectional Scattering Distribution Function.
/// It represents the properties of a material at a given point.
pub struct BSDF {
    /// Index of refraction of the surface
    pub eta: f32,
    /// Shading normal (i.e. potentially affected by bump-mapping)
    ns: Vector3f,
    /// Geometry normal
    ng: Vector3f,
    ss: Vector3f,
    ts: Vector3f,
    bxdfs: Vec<Box<BxDF + Sync + Send>>,
}

impl BSDF {
    pub fn new(isect: &SurfaceInteraction, eta: f32, bxdfs: Vec<Box<BxDF + Sync + Send>>) -> BSDF {
        let ss = isect.dpdu.normalize();
        BSDF {
            eta: eta,
            ns: isect.shading.n,
            ng: isect.n,
            ss: ss,
            ts: isect.shading.n.cross(&ss),
            bxdfs: bxdfs,
        }
    }

    /// Evaluate the BSDF for the given incoming light direction and outgoing light direction.
    pub fn f(&self, wo_w: &Vector3f, wi_w: &Vector3f, flags: BxDFType) -> Spectrum {
        let wi = self.world_to_local(wi_w);
        let wo = self.world_to_local(wo_w);
        if wo.z == 0.0 {
            return Spectrum::black();
        }
        let reflect = wi_w.dot(&self.ng) * wo_w.dot(&self.ng) > 0.0;
        self.bxdfs
            .iter()
            .filter(|b| {
                // Make sure we only evaluate reflection or transmission based on whether wi and wo
                // lie in the same hemisphere.
                b.matches(flags) &&
                ((reflect && (b.get_type().contains(BSDF_REFLECTION))) ||
                 (!reflect && (b.get_type().contains(BSDF_TRANSMISSION))))
            })
            .fold(Spectrum::black(), |c, b| c + b.f(&wo, &wi))
    }

    pub fn pdf(&self, wo_w: &Vector3f, wi_w: &Vector3f, flags: BxDFType) -> f32 {
        if self.bxdfs.is_empty() {
            return 0.0;
        }
        let wo = self.world_to_local(wo_w);
        if wo.z == 0.0 {
            return 0.0;
        }
        let wi = self.world_to_local(wi_w);

        let mut matched_comps = 0;
        let mut pdf = 0.0;
        for bxdf in &self.bxdfs {
            if bxdf.matches(flags) {
                matched_comps += 1;
                pdf += bxdf.pdf(&wo, &wi);
            }
        }
        if matched_comps == 0 {
            0.0
        } else {
            pdf / matched_comps as f32
        }
    }

    pub fn sample_f(&self,
                    wo_w: &Vector3f,
                    u: &Point2f,
                    flags: BxDFType)
                    -> (Spectrum, Vector3f, f32, BxDFType) {

        let matching_comps = self.bxdfs
            .iter()
            .filter(|b| b.matches(flags))
            .collect::<Vec<&Box<BxDF + Sync + Send>>>();
        if matching_comps.is_empty() {
            return (Spectrum::black(), Vector3f::new(0.0, 0.0, 0.0), 0.0, BxDFType::empty());
        }
        // Chose which BxDF to sample
        let comp = cmp::min((u[0] * matching_comps.len() as f32).floor() as usize,
                            matching_comps.len() - 1);
        let bxdf = matching_comps.get(comp).expect("Was expecting a BxDF with this index!");

        debug!("BDDF::sample_f chose comp = {} / matching {}",
               comp,
               matching_comps.len());

        // Remap BxDF sample u to [0,1)^2
        let u_remapped = Point2f::new((u[0] * matching_comps.len() as f32 - comp as f32)
                                          .min(ONE_MINUS_EPSILON),
                                      u[1]);
        // Sample chosen BxDF
        let wo = self.world_to_local(wo_w);
        if wo.z == 0.0 {
            return (Spectrum::black(), Vector3f::new(0.0, 0.0, 0.0), 0.0, bxdf.get_type());
        }
        let (mut f, wi, mut pdf, sampled_type) = bxdf.sample_f(&wo, &u_remapped);
        debug!("For wo = {:?}, sampled f = {}, pdf = {}, ratio = {}, wi = {:?}",
               wo,
               f,
               pdf,
               if pdf > 0.0 {
                   f / pdf
               } else {
                   Spectrum::black()
               },
               wi);
        if pdf == 0.0 {
            return (Spectrum::black(), Vector3f::new(0.0, 0.0, 0.0), 0.0, BxDFType::empty());
        }
        let wi_w = self.local_to_world(&wi);

        // Compute overall PDF with all matching BxDF
        if !bxdf.get_type().contains(BSDF_SPECULAR) && matching_comps.len() > 1 {
            for i in 0..matching_comps.len() {
                if i != comp {
                    pdf += matching_comps[i].pdf(&wo, &wi);
                }
            }
        }
        if matching_comps.len() > 1 {
            pdf /= matching_comps.len() as f32;
        }

        // Compute value of BSDF for sampled direction
        if !bxdf.get_type().contains(BSDF_SPECULAR) && matching_comps.len() > 1 {
            let reflect = wi_w.dot(&self.ng) * wo_w.dot(&self.ng) > 0.0;
            f = matching_comps.iter()
                .filter(|b| {
                    (reflect && b.get_type().contains(BSDF_REFLECTION)) ||
                    (!reflect && b.get_type().contains(BSDF_TRANSMISSION))
                })
                .fold(Spectrum::black(), |f, b| f + b.f(&wo, &wi));
        }

        debug!("Overall f = {}, pdf = {}, ratio = {}",
               f,
               pdf,
               if pdf > 0.0 {
                   f / pdf
               } else {
                   Spectrum::black()
               });
        (f, wi_w, pdf, sampled_type)
    }

    fn world_to_local(&self, v: &Vector3f) -> Vector3f {
        Vector3f::new(v.dot(&self.ss), v.dot(&self.ts), v.dot(&self.ns))
    }

    fn local_to_world(&self, v: &Vector3f) -> Vector3f {
        Vector3f::new(self.ss.x * v.x + self.ts.x * v.y + self.ns.x * v.z,
                      self.ss.y * v.x + self.ts.y * v.y + self.ns.y * v.z,
                      self.ss.z * v.z + self.ts.z * v.y + self.ns.z * v.z)
    }

    /// Return the number of BxDFs matching the given flags
    pub fn num_components(&self, flags: BxDFType) -> usize {
        self.bxdfs.iter().filter(|b| b.matches(flags)).count()
    }
}

#[test]
fn test_flags() {
    let flags = BSDF_SPECULAR | BSDF_REFLECTION;
    let bxdf_type = BSDF_SPECULAR | BSDF_REFLECTION | BSDF_TRANSMISSION;

    assert!((bxdf_type & flags) == flags);
}
<|endoftext|>


<commit_before>use error::Error;
use pos::Pos;
use token::{Token, TokenType};

use std::str::Chars;
use std::iter::Peekable;

pub struct Scanner<'a> {
    data: Peekable<Chars<'a>>,
    start_pos: Pos,
    curr_pos: Pos,
}

impl<'a> Scanner<'a> {
    // Create a new scanner from a given program represented as a
    // String.
    pub fn new<'b>(data: &'b str) -> Scanner<'b> {
        Scanner {
            data: data.chars().peekable(),
            start_pos: Pos { line: 1, col: 1 },
            curr_pos: Pos { line: 1, col: 1 },
        }
    }

    // Internal function: return the character at the current index.
    fn peek(&mut self) -> char {
        self.data.peek().map(|&c|c).unwrap_or('\x00')
    }

    // Internal function: return the character at the current index
    // and increment the index by one.
    fn advance(&mut self) -> char {
        let c = self.peek();
        if c == '\n' {
            self.curr_pos.line += 1;
            self.curr_pos.col = 1;
        } else {
            self.curr_pos.col += 1
        }
        let _ = self.data.next();
        c
    }

    // Internal function: verify if the end of the program has been
    // reached.
    fn is_eof(&mut self) -> bool {
        !self.data.peek().is_some()
    }

    pub fn next_token(&mut self) -> Result<Token, Error> {
        // Discard blanks and comments.
        self.skip_comments_and_whitespace();

        // Set start_pos (starting position of the next token) to the
        // value of curr_pos (current position in the text stream).
        self.start_pos = self.curr_pos;

        // Return Eof if the end of the file has been reached.
        if self.is_eof() {
            return Ok(self.empty_tok(TokenType::Eof));
        }

        // Scanning dispatch.
        match self.peek() {
            '+' => { Ok(self.single_char_tok(TokenType::Plus)) }
            '-' => { Ok(self.single_char_tok(TokenType::Minus)) }
            '*' => { Ok(self.single_char_tok(TokenType::Star)) }
            '/' => { Ok(self.single_char_tok(TokenType::Slash)) }
            '=' => { Ok(self.single_char_tok(TokenType::Equal)) }
            '(' => { Ok(self.single_char_tok(TokenType::LParen)) }
            ')' => { Ok(self.single_char_tok(TokenType::RParen)) }
            ':' => { Ok(self.single_char_tok(TokenType::Colon)) }
            ';' => { Ok(self.single_char_tok(TokenType::Semicolon)) }
            c if c.is_digit(10) => { self.scan_int_or_float() }
            c if is_id_start(c) => { self.scan_id_or_keyword() }
            c   => { Err(Error::IllegalCharacter(self.curr_pos, c)) }
        }
    }

    // Scan digits into an Int or Float token.
    fn scan_int_or_float(&mut self) -> Result<Token, Error> {
        let mut val = String::new();
        while self.peek().is_digit(10) {
            val.push(self.advance());
        }

        if self.peek() != '.' {
            return Ok(self.lexeme_tok(TokenType::Int, val));
        }

        val.push(self.advance()); // Add decimal point.

        while self.peek().is_digit(10) {
            val.push(self.advance());
        }

        Ok(self.lexeme_tok(TokenType::Float, val))
    }

    // Scan alpha-numeric characters into an Id or a keyword token.
    fn scan_id_or_keyword(&mut self) -> Result<Token, Error> {
        let mut lexeme = String::new();
        while is_id_char(self.peek()) {
            lexeme.push(self.advance());
        }

        let token_type = match &*lexeme {
            "if" => TokenType::If,
            "then" => TokenType::Then,
            "else" => TokenType::Else,
            "end" => TokenType::End,
            "while" => TokenType::While,
            "do" => TokenType::Do,
            "done" => TokenType::Done,
            "read" => TokenType::Read,
            "print" => TokenType::Print,
            "var" => TokenType::Var,
            "int" => TokenType::TypeInt,
            "float" => TokenType::TypeFloat,
            "function" => TokenType::Function,
            "return" => TokenType::Return,
            _ => TokenType::Id,
        };

        let token = if token_type == TokenType::Id {
            self.lexeme_tok(token_type, lexeme)
        } else {
            self.empty_tok(token_type)
        };

        Ok(token)
    }

    fn skip_comments_and_whitespace(&mut self) {
        loop {
            self.skip_whitespace();
            if self.peek() == '#' {
                self.skip_comment();
            } else {
                break;
            }
        }
    }

    fn skip_whitespace(&mut self) {
        while self.peek().is_whitespace() {
            self.advance();
        }
    }

    fn skip_comment(&mut self) {
        while self.peek() != '\n' {
            self.advance();
        }
    }

    fn empty_tok(&self, t: TokenType) -> Token {
        Token {
            typ: t,
            lexeme: None,
            pos: self.start_pos,
        }
    }

    fn lexeme_tok(&self, t: TokenType, lexeme: String) -> Token {
        Token {
            typ: t,
            lexeme: Some(lexeme),
            pos: self.start_pos,
        }
    }

    fn single_char_tok(&mut self, t: TokenType) -> Token {
        let t = self.empty_tok(t);
        self.advance();
        t
    }
}

fn is_id_start(c: char) -> bool {
    (c >= 'a' && c <= 'z') ||
    c == '_' ||
    (c >= 'A' && c <= 'Z')
}

fn is_id_char(c: char) -> bool {
    (c >= 'a' && c <= 'z') ||
    c == '_' ||
    c.is_digit(10) ||
    (c >= 'A' && c <= 'Z')
}
<commit_msg>Recognize comma (',') as a token<commit_after>use error::Error;
use pos::Pos;
use token::{Token, TokenType};

use std::str::Chars;
use std::iter::Peekable;

pub struct Scanner<'a> {
    data: Peekable<Chars<'a>>,
    start_pos: Pos,
    curr_pos: Pos,
}

impl<'a> Scanner<'a> {
    // Create a new scanner from a given program represented as a
    // String.
    pub fn new<'b>(data: &'b str) -> Scanner<'b> {
        Scanner {
            data: data.chars().peekable(),
            start_pos: Pos { line: 1, col: 1 },
            curr_pos: Pos { line: 1, col: 1 },
        }
    }

    // Internal function: return the character at the current index.
    fn peek(&mut self) -> char {
        self.data.peek().map(|&c|c).unwrap_or('\x00')
    }

    // Internal function: return the character at the current index
    // and increment the index by one.
    fn advance(&mut self) -> char {
        let c = self.peek();
        if c == '\n' {
            self.curr_pos.line += 1;
            self.curr_pos.col = 1;
        } else {
            self.curr_pos.col += 1
        }
        let _ = self.data.next();
        c
    }

    // Internal function: verify if the end of the program has been
    // reached.
    fn is_eof(&mut self) -> bool {
        !self.data.peek().is_some()
    }

    pub fn next_token(&mut self) -> Result<Token, Error> {
        // Discard blanks and comments.
        self.skip_comments_and_whitespace();

        // Set start_pos (starting position of the next token) to the
        // value of curr_pos (current position in the text stream).
        self.start_pos = self.curr_pos;

        // Return Eof if the end of the file has been reached.
        if self.is_eof() {
            return Ok(self.empty_tok(TokenType::Eof));
        }

        // Scanning dispatch.
        match self.peek() {
            '+' => { Ok(self.single_char_tok(TokenType::Plus)) }
            '-' => { Ok(self.single_char_tok(TokenType::Minus)) }
            '*' => { Ok(self.single_char_tok(TokenType::Star)) }
            '/' => { Ok(self.single_char_tok(TokenType::Slash)) }
            '=' => { Ok(self.single_char_tok(TokenType::Equal)) }
            '(' => { Ok(self.single_char_tok(TokenType::LParen)) }
            ')' => { Ok(self.single_char_tok(TokenType::RParen)) }
            ':' => { Ok(self.single_char_tok(TokenType::Colon)) }
            ';' => { Ok(self.single_char_tok(TokenType::Semicolon)) }
            ',' => { Ok(self.single_char_tok(TokenType::Comma)) }
            c if c.is_digit(10) => { self.scan_int_or_float() }
            c if is_id_start(c) => { self.scan_id_or_keyword() }
            c   => { Err(Error::IllegalCharacter(self.curr_pos, c)) }
        }
    }

    // Scan digits into an Int or Float token.
    fn scan_int_or_float(&mut self) -> Result<Token, Error> {
        let mut val = String::new();
        while self.peek().is_digit(10) {
            val.push(self.advance());
        }

        if self.peek() != '.' {
            return Ok(self.lexeme_tok(TokenType::Int, val));
        }

        val.push(self.advance()); // Add decimal point.

        while self.peek().is_digit(10) {
            val.push(self.advance());
        }

        Ok(self.lexeme_tok(TokenType::Float, val))
    }

    // Scan alpha-numeric characters into an Id or a keyword token.
    fn scan_id_or_keyword(&mut self) -> Result<Token, Error> {
        let mut lexeme = String::new();
        while is_id_char(self.peek()) {
            lexeme.push(self.advance());
        }

        let token_type = match &*lexeme {
            "if" => TokenType::If,
            "then" => TokenType::Then,
            "else" => TokenType::Else,
            "end" => TokenType::End,
            "while" => TokenType::While,
            "do" => TokenType::Do,
            "done" => TokenType::Done,
            "read" => TokenType::Read,
            "print" => TokenType::Print,
            "var" => TokenType::Var,
            "int" => TokenType::TypeInt,
            "float" => TokenType::TypeFloat,
            "function" => TokenType::Function,
            "return" => TokenType::Return,
            _ => TokenType::Id,
        };

        let token = if token_type == TokenType::Id {
            self.lexeme_tok(token_type, lexeme)
        } else {
            self.empty_tok(token_type)
        };

        Ok(token)
    }

    fn skip_comments_and_whitespace(&mut self) {
        loop {
            self.skip_whitespace();
            if self.peek() == '#' {
                self.skip_comment();
            } else {
                break;
            }
        }
    }

    fn skip_whitespace(&mut self) {
        while self.peek().is_whitespace() {
            self.advance();
        }
    }

    fn skip_comment(&mut self) {
        while self.peek() != '\n' {
            self.advance();
        }
    }

    fn empty_tok(&self, t: TokenType) -> Token {
        Token {
            typ: t,
            lexeme: None,
            pos: self.start_pos,
        }
    }

    fn lexeme_tok(&self, t: TokenType, lexeme: String) -> Token {
        Token {
            typ: t,
            lexeme: Some(lexeme),
            pos: self.start_pos,
        }
    }

    fn single_char_tok(&mut self, t: TokenType) -> Token {
        let t = self.empty_tok(t);
        self.advance();
        t
    }
}

fn is_id_start(c: char) -> bool {
    (c >= 'a' && c <= 'z') ||
    c == '_' ||
    (c >= 'A' && c <= 'Z')
}

fn is_id_char(c: char) -> bool {
    (c >= 'a' && c <= 'z') ||
    c == '_' ||
    c.is_digit(10) ||
    (c >= 'A' && c <= 'Z')
}
<|endoftext|>


<commit_before>use std::iter::{Iterator, Peekable};
use std::result;
use std::str::Chars;

#[derive(Clone,Debug,PartialEq)]
pub enum Token {
    OpenParen,
    CloseParen,
    Eq,
    DoubleEq,
    Lt,
    LtEq,
    Gt,
    GtEq,
    Plus,
    Minus,
    Times,
    Divide,
    Nil,
    Boolean(bool),
    Identifier(String),
}

#[derive(Clone,Debug,PartialEq)]
pub enum TokenError {
    UnexpectedChar(char),
}

pub type Result<T> = result::Result<T, TokenError>;

pub struct Scanner<'a> {
    input: Peekable<Chars<'a>>,
}

impl<'a> Scanner<'a> {
    pub fn new(input: &'a str) -> Self {
        Scanner{input: input.chars().peekable()}
    }

    fn read_word(&mut self) -> Token {
        let mut word = String::new();
        while let Some(&c) = self.input.peek() {
            if !Self::is_digit(c) && !Self::is_alpha(c) {
                break;
            }

            self.input.next();
            word.push(c);
        }

        match word.as_ref() {
            "nil" => Token::Nil,
            "true" => Token::Boolean(true),
            "false" => Token::Boolean(false),
            _ => Token::Identifier(word),
        }
    }

    fn is_space(c: char) -> bool {
        c == ' ' || c == '\t' || c == '\n' || c == '\r'
    }

    fn is_alpha(c: char) -> bool {
        c == '_' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
    }

    fn is_digit(c: char) -> bool {
        ('0' <= c && c <= '9')
    }
}

impl<'a> Iterator for Scanner<'a> {
    type Item = Result<Token>;

    fn next(&mut self) -> Option<Self::Item> {
        while let Some(&c) = self.input.peek() {
            if Self::is_space(c) {
                self.input.next();
            } else {
                break;
            }
        }

        match self.input.peek() {
            None => None,
            Some(&'(') => {
                self.input.next();
                Some(Ok(Token::OpenParen))
            },
            Some(&')') => {
                self.input.next();
                Some(Ok(Token::CloseParen))
            },
            Some(&'=') => {
                self.input.next();
                if let Some(&'=') = self.input.peek() {
                    self.input.next();
                    Some(Ok(Token::DoubleEq))
                } else {
                    Some(Ok(Token::Eq))
                }
            },
            Some(&'<') => {
                self.input.next();
                if let Some(&'=') = self.input.peek() {
                    self.input.next();
                    Some(Ok(Token::LtEq))
                } else {
                    Some(Ok(Token::Lt))
                }
            },
            Some(&'>') => {
                self.input.next();
                if let Some(&'=') = self.input.peek() {
                    self.input.next();
                    Some(Ok(Token::GtEq))
                } else {
                    Some(Ok(Token::Gt))
                }
            },
            Some(&'+') => {
                self.input.next();
                Some(Ok(Token::Plus))
            },
            Some(&'-') => {
                self.input.next();
                Some(Ok(Token::Minus))
            },
            Some(&'*') => {
                self.input.next();
                Some(Ok(Token::Times))
            },
            Some(&'/') => {
                self.input.next();
                Some(Ok(Token::Divide))
            },
            Some(&c) if Self::is_alpha(c) => Some(Ok(self.read_word())),
            Some(&c) => {
                self.input.next();
                Some(Err(TokenError::UnexpectedChar(c)))
            },
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::Token::*;

    #[test]
    fn test_punctuation() {
        let mut s = Scanner::new("() = == < <= > >= +-*/");
        assert_eq!(s.next(), Some(Ok(OpenParen)));
        assert_eq!(s.next(), Some(Ok(CloseParen)));
        assert_eq!(s.next(), Some(Ok(Eq)));
        assert_eq!(s.next(), Some(Ok(DoubleEq)));
        assert_eq!(s.next(), Some(Ok(Lt)));
        assert_eq!(s.next(), Some(Ok(LtEq)));
        assert_eq!(s.next(), Some(Ok(Gt)));
        assert_eq!(s.next(), Some(Ok(GtEq)));
        assert_eq!(s.next(), Some(Ok(Plus)));
        assert_eq!(s.next(), Some(Ok(Minus)));
        assert_eq!(s.next(), Some(Ok(Times)));
        assert_eq!(s.next(), Some(Ok(Divide)));
        assert_eq!(s.next(), None);
    }

    #[test]
    fn test_unexpected_char() {
        let mut s = Scanner::new("($)");
        assert_eq!(s.next(), Some(Ok(OpenParen)));
        assert_eq!(s.next(), Some(Err(TokenError::UnexpectedChar('$'))));
    }

    #[test]
    fn test_words() {
        let mut s = Scanner::new("foo FOO _123_ Nil nil false true");
        assert_eq!(s.next(), Some(Ok(Identifier("foo".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Identifier("FOO".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Identifier("_123_".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Identifier("Nil".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Nil)));
        assert_eq!(s.next(), Some(Ok(Boolean(false))));
        assert_eq!(s.next(), Some(Ok(Boolean(true))));
        assert_eq!(s.next(), None);
    }
}
<commit_msg>Add number to scanner<commit_after>use std::iter::{Iterator, Peekable};
use std::result;
use std::str::Chars;

#[derive(Clone,Debug,PartialEq)]
pub enum Token {
    OpenParen,
    CloseParen,
    Eq,
    DoubleEq,
    Lt,
    LtEq,
    Gt,
    GtEq,
    Plus,
    Minus,
    Times,
    Divide,
    Nil,
    Boolean(bool),
    Identifier(String),
    Number(f64),
}

#[derive(Clone,Debug,PartialEq)]
pub enum TokenError {
    UnexpectedChar(char),
}

pub type Result<T> = result::Result<T, TokenError>;

pub struct Scanner<'a> {
    input: Peekable<Chars<'a>>,
}

impl<'a> Scanner<'a> {
    pub fn new(input: &'a str) -> Self {
        Scanner{input: input.chars().peekable()}
    }

    fn read_word(&mut self) -> Token {
        let mut word = String::new();
        while let Some(&c) = self.input.peek() {
            if !Self::is_digit(c) && !Self::is_alpha(c) {
                break;
            }

            self.input.next();
            word.push(c);
        }

        match word.as_ref() {
            "nil" => Token::Nil,
            "true" => Token::Boolean(true),
            "false" => Token::Boolean(false),
            _ => Token::Identifier(word),
        }
    }

    fn read_number(&mut self) -> f64 {
        let mut num = String::new();
        while let Some(&c) = self.input.peek() {
            if !Self::is_digit(c) {
                break;
            }

            self.input.next();
            num.push(c);
        }

        if let Some(&'.') = self.input.peek() {
            self.input.next();
            num.push('.');

            while let Some(&c) = self.input.peek() {
                if !Self::is_digit(c) {
                    break;
                }

                self.input.next();
                num.push(c);
            }
        }

        num.parse().unwrap()
    }

    fn is_space(c: char) -> bool {
        c == ' ' || c == '\t' || c == '\n' || c == '\r'
    }

    fn is_alpha(c: char) -> bool {
        c == '_' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
    }

    fn is_digit(c: char) -> bool {
        ('0' <= c && c <= '9')
    }
}

impl<'a> Iterator for Scanner<'a> {
    type Item = Result<Token>;

    fn next(&mut self) -> Option<Self::Item> {
        while let Some(&c) = self.input.peek() {
            if Self::is_space(c) {
                self.input.next();
            } else {
                break;
            }
        }

        match self.input.peek() {
            None => None,
            Some(&'(') => {
                self.input.next();
                Some(Ok(Token::OpenParen))
            },
            Some(&')') => {
                self.input.next();
                Some(Ok(Token::CloseParen))
            },
            Some(&'=') => {
                self.input.next();
                if let Some(&'=') = self.input.peek() {
                    self.input.next();
                    Some(Ok(Token::DoubleEq))
                } else {
                    Some(Ok(Token::Eq))
                }
            },
            Some(&'<') => {
                self.input.next();
                if let Some(&'=') = self.input.peek() {
                    self.input.next();
                    Some(Ok(Token::LtEq))
                } else {
                    Some(Ok(Token::Lt))
                }
            },
            Some(&'>') => {
                self.input.next();
                if let Some(&'=') = self.input.peek() {
                    self.input.next();
                    Some(Ok(Token::GtEq))
                } else {
                    Some(Ok(Token::Gt))
                }
            },
            Some(&'+') => {
                self.input.next();
                match self.input.peek() {
                    Some(&c) if Self::is_digit(c) => Some(Ok(Token::Number(self.read_number()))),
                    _ => Some(Ok(Token::Plus)),
                }
            },
            Some(&'-') => {
                self.input.next();
                match self.input.peek() {
                    Some(&c) if Self::is_digit(c) => Some(Ok(Token::Number(self.read_number() * -1.0))),
                    _ => Some(Ok(Token::Minus)),
                }
            },
            Some(&'*') => {
                self.input.next();
                Some(Ok(Token::Times))
            },
            Some(&'/') => {
                self.input.next();
                Some(Ok(Token::Divide))
            },
            Some(&c) if Self::is_alpha(c) => Some(Ok(self.read_word())),
            Some(&c) if Self::is_digit(c) => Some(Ok(Token::Number(self.read_number()))),
            Some(&c) => {
                self.input.next();
                Some(Err(TokenError::UnexpectedChar(c)))
            },
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::Token::*;

    #[test]
    fn test_punctuation() {
        let mut s = Scanner::new("() = == < <= > >= +-*/");
        assert_eq!(s.next(), Some(Ok(OpenParen)));
        assert_eq!(s.next(), Some(Ok(CloseParen)));
        assert_eq!(s.next(), Some(Ok(Eq)));
        assert_eq!(s.next(), Some(Ok(DoubleEq)));
        assert_eq!(s.next(), Some(Ok(Lt)));
        assert_eq!(s.next(), Some(Ok(LtEq)));
        assert_eq!(s.next(), Some(Ok(Gt)));
        assert_eq!(s.next(), Some(Ok(GtEq)));
        assert_eq!(s.next(), Some(Ok(Plus)));
        assert_eq!(s.next(), Some(Ok(Minus)));
        assert_eq!(s.next(), Some(Ok(Times)));
        assert_eq!(s.next(), Some(Ok(Divide)));
        assert_eq!(s.next(), None);
    }

    #[test]
    fn test_unexpected_char() {
        let mut s = Scanner::new("($)");
        assert_eq!(s.next(), Some(Ok(OpenParen)));
        assert_eq!(s.next(), Some(Err(TokenError::UnexpectedChar('$'))));
    }

    #[test]
    fn test_words() {
        let mut s = Scanner::new("foo FOO _123_ Nil nil false true");
        assert_eq!(s.next(), Some(Ok(Identifier("foo".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Identifier("FOO".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Identifier("_123_".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Identifier("Nil".to_owned()))));
        assert_eq!(s.next(), Some(Ok(Nil)));
        assert_eq!(s.next(), Some(Ok(Boolean(false))));
        assert_eq!(s.next(), Some(Ok(Boolean(true))));
        assert_eq!(s.next(), None);
    }

    #[test]
    fn test_number() {
        let mut s = Scanner::new("0 -0 -1.2 +2.3 999 1.");
        assert_eq!(s.next(), Some(Ok(Number(0.0))));
        assert_eq!(s.next(), Some(Ok(Number(0.0))));
        assert_eq!(s.next(), Some(Ok(Number(-1.2))));
        assert_eq!(s.next(), Some(Ok(Number(2.3))));
        assert_eq!(s.next(), Some(Ok(Number(999.0))));
        assert_eq!(s.next(), Some(Ok(Number(1.0))));
        assert_eq!(s.next(), None);
    }
}
<|endoftext|>


<commit_before>
use syntax::Token;

#[test]
fn test_trivial(){
    assert_eq!(scan(" "), vec![Token::Spaces(" ")]);
    assert_eq!(scan(" \n\t"), vec![Token::Spaces(" \n\t")]);
    assert_eq!(scan("//x"), vec![Token::Comment("//x")]);
    assert_eq!(scan("// one"), vec![Token::Comment("// one")]);
    assert_eq!(scan("/* \n*/"), vec![Token::Comment("/* \n*/")]);
    assert_eq!(scan("123"), vec![Token::Integer("123")]);
    assert_eq!(scan("123.456"), vec![Token::Float("123.456")]);
    assert_eq!(scan("foo_bar2"), vec![Token::Identifier("foo_bar2")]);
    assert_eq!(scan("THEN"), vec![Token::Keyword("THEN")]);
    assert_eq!(scan("THENxxx"), vec![Token::Identifier("THENxxx")]);
    assert_eq!(scan("-"), vec![Token::Operator("-")]);
    assert_eq!(scan("-->"), vec![Token::Operator("-->")]);
}

#[test]
fn test_double(){
    assert_eq!(scan("//xy z\n  "), vec![
        Token::Comment("//xy z"),
        Token::Spaces("\n  "),
    ]);
    assert_eq!(scan("\n  //xy z"), vec![
        Token::Spaces("\n  "),
        Token::Comment("//xy z"),
    ]);
}

#[test]
fn test_unicode_1(){
    // one char and one grapheme

    let mut reconstruct = "".to_owned();
    reconstruct.push('é'); // one
    assert_eq!("é", reconstruct);

    assert_eq!(scan("é"), vec![Token::Identifier("é")]);
}

#[test]
fn test_unicode_2(){
    // two chars but one graphemes

    let mut reconstruct = "".to_owned();
    reconstruct.push('e');
    reconstruct.push('́');
    assert_eq!("é", reconstruct);

    // the result should be 2 chars long
    // (if we consider composite char as legal string for identifier)
    //assert_eq!(scan("é"), vec![Token::Identifier("é")]);
    // TODO doesn't work for the moment
}

#[test]
fn test_unicode_normalisation(){
    use unicode_normalization::UnicodeNormalization;

    let mut composite = "".to_owned();
    composite.push('e');
    composite.push('́');

    assert!("é" != composite); // one grapheme
    assert!("é" == composite); // two graphemes

    let normalized : String = composite.nfc().collect();

    assert!("é" == normalized); // one grapheme
    assert!("é" != normalized); // two graphemes

}

#[test]
fn test_unicode_alphab(){
    // composable char are not alphabetic ...
    assert!( ! '́'.is_alphabetic())
}

#[test]
fn test_composed(){
    assert_eq!(scan("1..2"), vec![Token::Integer("1"), Token::Operator(".."), Token::Integer("2")]);
}

struct ScannerState<'a> {
    i : usize,
    j : usize,
    token : Token<'a>,
    source : &'a str,

    size_left : usize, // in bytes
    size_right : usize, // in bytes
}

pub fn scan(source : &str) -> Vec<Token> {
    let mut state = ScannerState{
        i:0,
        j:0,
        token : Token::Error,
        source : source,

        size_left : 0,
        size_right : 0,
    };
    let mut res = vec![];
    loop {

        state.scan_spaces();
        state.scan_comment_monoline();
        state.scan_comment_multiline();
        state.scan_number();
        for keyword in ::syntax::KEYWORDS.iter() {
            state.scan_keyword(keyword);
        }
        state.scan_identifier();
        for operator in ::syntax::OPERATORS.iter() {
            state.scan_operator(operator);
        }

        if state.token == Token::Error {
            return res
        }
        res.push(state.token);
        state.i = state.j;
        state.size_left = state.size_right;
        state.token = Token::Error;
    }
}

impl<'a> ScannerState<'a> {

    fn scan_spaces(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        loop {
            match self.source.char_indices().nth(x) {
                Some((i,' ')) | Some((i,'\t')) | Some((i,'\n')) => {
                    x += 1;
                    new_right = i + ' '.len_utf8();
                },
                _ => break,
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Spaces(content);
        }
    }

    fn scan_comment_monoline(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        if self.source.chars().nth(x) == Some('/') &&
           self.source.chars().nth(x+1) == Some('/') {
            x += 2;
            new_right +=  '/'.len_utf8()*2;
            loop {
                match self.source.char_indices().nth(x) {
                    None => break,
                    Some((_,'\n')) => break,
                    Some((i,c)) => {
                        x += 1;
                        new_right = i + c.len_utf8();
                    },
                }
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Comment(content);
        }
    }

    fn scan_comment_multiline(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let mut iter = self.source.char_indices();
        if iter.nth(x) == Some((new_right,'/')) &&
           iter.next() == Some((new_right+'/'.len_utf8(),'*')) {
            x += 2;
            new_right +=  '/'.len_utf8()  + '*'.len_utf8();
            'outer: loop {
                match iter.next() {
                    Some((i,'*')) => {
                        x += 1;
                        new_right = i + '*'.len_utf8();
                        'inner: loop {
                            match iter.next() {
                                Some((i,'/')) => {
                                    x += 1;
                                    new_right = i + '/'.len_utf8();
                                    break 'outer;
                                },
                                Some((i,'*')) => {
                                    x += 1;
                                    new_right = i + '*'.len_utf8();
                                    continue 'inner;
                                },
                                Some((i,c)) => {
                                    x += 1;
                                    new_right = i + c.len_utf8();
                                    continue 'outer;
                                },
                                None => {
                                    break 'outer;
                                },
                            }
                        }
                    },
                    Some((i,c)) => {
                        x += 1;
                        new_right = i + c.len_utf8();
                        continue 'outer;
                    },
                    None => {
                        break 'outer;
                    },
                }
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Comment(content)
        }
    }

    fn scan_number(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let mut float = false;
        let mut iter = self.source.char_indices().skip(x);
        loop {
            match iter.next() {
                Some((i,'0' ... '9')) => {
                    x += 1;
                    new_right = i + '0'.len_utf8();
                },
                Some((i,'.')) => {
                    if float == true { break; }
                    float = true;
                    x += 1;
                    new_right = i + '.'.len_utf8();
                }
                _ => break,
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = if float {
                Token::Float(content)
            } else {
                Token::Integer(content)
            }
        }
    }

    fn scan_identifier(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let mut iter = self.source.char_indices().skip(self.i);
        'outer: loop {
            match iter.next() {
                Some((i,c)) if c.is_alphabetic() => {
                    println!("({},{})",i,c);
                    x += 1;
                    new_right = i + c.len_utf8();
                    'inner: loop {
                        match iter.next() {
                            Some((i,c)) if c.is_alphabetic() ||
                                       c.is_numeric() ||
                                       c == '_' => {
                                println!("({},{})",i,c);
                                x += 1;
                                new_right = i + c.len_utf8();
                                continue 'inner;
                            },
                            _ => break 'outer,
                        }
                    }
                },
                _ => {
                    break 'outer
                },
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Identifier(content)
        }
    }

    fn scan_keyword(&mut self, keyword : &str){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let iter = self.source.char_indices().skip(self.i);
        let ik = keyword.chars();
        for ((i,a),b) in iter.zip(ik) {
            if a == b {
                x += 1;
                new_right = i + a.len_utf8();
            } else {
                break;
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Keyword(content)
        }
    }

    // TODO factorize with scan_keyword
    fn scan_operator(&mut self, operator : &str){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let iter = self.source.char_indices().skip(self.i);
        let ik = operator.chars();
        for ((i,a),b) in iter.zip(ik) {
            if a == b {
                x += 1;
                new_right = i + a.len_utf8();
            } else {
                break;
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Operator(content)
        }
    }

}

// TODO scan_operator

<commit_msg>manage ambiguity between '..' operator and '1.' float<commit_after>
use syntax::Token;

#[test]
fn test_trivial(){
    assert_eq!(scan(" "), vec![Token::Spaces(" ")]);
    assert_eq!(scan(" \n\t"), vec![Token::Spaces(" \n\t")]);
    assert_eq!(scan("//x"), vec![Token::Comment("//x")]);
    assert_eq!(scan("// one"), vec![Token::Comment("// one")]);
    assert_eq!(scan("/* \n*/"), vec![Token::Comment("/* \n*/")]);
    assert_eq!(scan("123"), vec![Token::Integer("123")]);
    assert_eq!(scan("123."), vec![Token::Float("123.")]);
    assert_eq!(scan("123.456"), vec![Token::Float("123.456")]);
    assert_eq!(scan("foo_bar2"), vec![Token::Identifier("foo_bar2")]);
    assert_eq!(scan("THEN"), vec![Token::Keyword("THEN")]);
    assert_eq!(scan("THENxxx"), vec![Token::Identifier("THENxxx")]);
    assert_eq!(scan("-"), vec![Token::Operator("-")]);
    assert_eq!(scan("-->"), vec![Token::Operator("-->")]);
}

#[test]
fn test_double(){
    assert_eq!(scan("//xy z\n  "), vec![
        Token::Comment("//xy z"),
        Token::Spaces("\n  "),
    ]);
    assert_eq!(scan("\n  //xy z"), vec![
        Token::Spaces("\n  "),
        Token::Comment("//xy z"),
    ]);
}

#[test]
fn test_unicode_1(){
    // one char and one grapheme

    let mut reconstruct = "".to_owned();
    reconstruct.push('é'); // one
    assert_eq!("é", reconstruct);

    assert_eq!(scan("é"), vec![Token::Identifier("é")]);
}

#[test]
fn test_unicode_2(){
    // two chars but one graphemes

    let mut reconstruct = "".to_owned();
    reconstruct.push('e');
    reconstruct.push('́');
    assert_eq!("é", reconstruct);

    // the result should be 2 chars long
    // (if we consider composite char as legal string for identifier)
    //assert_eq!(scan("é"), vec![Token::Identifier("é")]);
    // TODO doesn't work for the moment
}

#[test]
fn test_unicode_normalisation(){
    use unicode_normalization::UnicodeNormalization;

    let mut composite = "".to_owned();
    composite.push('e');
    composite.push('́');

    assert!("é" != composite); // one grapheme
    assert!("é" == composite); // two graphemes

    let normalized : String = composite.nfc().collect();

    assert!("é" == normalized); // one grapheme
    assert!("é" != normalized); // two graphemes

}

#[test]
fn test_unicode_alphab(){
    // composable char are not alphabetic ...
    assert!( ! '́'.is_alphabetic())
}

#[test]
fn test_composed(){
    assert_eq!(scan("1..2"), vec![Token::Integer("1"), Token::Operator(".."), Token::Integer("2")]);
}

struct ScannerState<'a> {
    i : usize,
    j : usize,
    token : Token<'a>,
    source : &'a str,

    size_left : usize, // in bytes
    size_right : usize, // in bytes
}

pub fn scan(source : &str) -> Vec<Token> {
    let mut state = ScannerState{
        i:0,
        j:0,
        token : Token::Error,
        source : source,

        size_left : 0,
        size_right : 0,
    };
    let mut res = vec![];
    loop {

        state.scan_spaces();
        state.scan_comment_monoline();
        state.scan_comment_multiline();
        state.scan_number();
        for keyword in ::syntax::KEYWORDS.iter() {
            state.scan_keyword(keyword);
        }
        state.scan_identifier();
        for operator in ::syntax::OPERATORS.iter() {
            state.scan_operator(operator);
        }

        if state.token == Token::Error {
            return res
        }
        res.push(state.token);
        state.i = state.j;
        state.size_left = state.size_right;
        state.token = Token::Error;
    }
}

impl<'a> ScannerState<'a> {

    fn scan_spaces(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        loop {
            match self.source.char_indices().nth(x) {
                Some((i,' ')) | Some((i,'\t')) | Some((i,'\n')) => {
                    x += 1;
                    new_right = i + ' '.len_utf8();
                },
                _ => break,
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Spaces(content);
        }
    }

    fn scan_comment_monoline(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        if self.source.chars().nth(x) == Some('/') &&
           self.source.chars().nth(x+1) == Some('/') {
            x += 2;
            new_right +=  '/'.len_utf8()*2;
            loop {
                match self.source.char_indices().nth(x) {
                    None => break,
                    Some((_,'\n')) => break,
                    Some((i,c)) => {
                        x += 1;
                        new_right = i + c.len_utf8();
                    },
                }
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Comment(content);
        }
    }

    fn scan_comment_multiline(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let mut iter = self.source.char_indices();
        if iter.nth(x) == Some((new_right,'/')) &&
           iter.next() == Some((new_right+'/'.len_utf8(),'*')) {
            x += 2;
            new_right +=  '/'.len_utf8()  + '*'.len_utf8();
            'outer: loop {
                match iter.next() {
                    Some((i,'*')) => {
                        x += 1;
                        new_right = i + '*'.len_utf8();
                        'inner: loop {
                            match iter.next() {
                                Some((i,'/')) => {
                                    x += 1;
                                    new_right = i + '/'.len_utf8();
                                    break 'outer;
                                },
                                Some((i,'*')) => {
                                    x += 1;
                                    new_right = i + '*'.len_utf8();
                                    continue 'inner;
                                },
                                Some((i,c)) => {
                                    x += 1;
                                    new_right = i + c.len_utf8();
                                    continue 'outer;
                                },
                                None => {
                                    break 'outer;
                                },
                            }
                        }
                    },
                    Some((i,c)) => {
                        x += 1;
                        new_right = i + c.len_utf8();
                        continue 'outer;
                    },
                    None => {
                        break 'outer;
                    },
                }
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Comment(content)
        }
    }

    fn scan_number(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let mut float = false;
        let mut iter = self.source.char_indices().skip(x);
        loop {
            match iter.next() {
                Some((i,'0' ... '9')) => {
                    x += 1;
                    new_right = i + '0'.len_utf8();
                },
                Some((i,'.')) => {
                    let mut iter_tmp = iter.clone();
                    // Looking for a '..' operator which can confuse with a float like "1."
                    match iter_tmp.next() {
                        Some((_, '.')) => break,
                        _ => {}
                    }
                    if float == true { break; }
                    float = true;
                    x += 1;
                    new_right = i + '.'.len_utf8();
                }
                _ => break,
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = if float {
                Token::Float(content)
            } else {
                Token::Integer(content)
            }
        }
    }

    fn scan_identifier(&mut self){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let mut iter = self.source.char_indices().skip(self.i);
        'outer: loop {
            match iter.next() {
                Some((i,c)) if c.is_alphabetic() => {
                    println!("({},{})",i,c);
                    x += 1;
                    new_right = i + c.len_utf8();
                    'inner: loop {
                        match iter.next() {
                            Some((i,c)) if c.is_alphabetic() ||
                                       c.is_numeric() ||
                                       c == '_' => {
                                println!("({},{})",i,c);
                                x += 1;
                                new_right = i + c.len_utf8();
                                continue 'inner;
                            },
                            _ => break 'outer,
                        }
                    }
                },
                _ => {
                    break 'outer
                },
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Identifier(content)
        }
    }

    fn scan_keyword(&mut self, keyword : &str){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let iter = self.source.char_indices().skip(self.i);
        let ik = keyword.chars();
        for ((i,a),b) in iter.zip(ik) {
            if a == b {
                x += 1;
                new_right = i + a.len_utf8();
            } else {
                break;
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Keyword(content)
        }
    }

    // TODO factorize with scan_keyword
    fn scan_operator(&mut self, operator : &str){
        let mut x = self.i;
        let mut new_right = self.size_left;
        let iter = self.source.char_indices().skip(self.i);
        let ik = operator.chars();
        for ((i,a),b) in iter.zip(ik) {
            if a == b {
                x += 1;
                new_right = i + a.len_utf8();
            } else {
                break;
            }
        }
        if self.j < x {
            self.j = x;
            self.size_right = new_right;
            let content = &self.source[self.size_left..self.size_right];
            self.token = Token::Operator(content)
        }
    }

}

// TODO scan_operator

<|endoftext|>


<commit_before>use state::InterpState;

/// Represents a builtin function
#[derive(Debug, Clone, PartialEq, Variants)]
pub enum Builtin {
    Dot,
    Plus,
    Minus,
    Star,
    Slash,
    Abs,
    And,
    Or,
    Xor,
    LShift,
    RShift,
    Equals,
    NotEquals,
    LessThan,
    GreaterThan,
    LessEqual,
    GreaterEqual,
    Emit,
    Dup,
    Swap,
    Over,
    Rot,
    Tuck,
    Drop,
    Fetch,
    Store,
    Here,
    Comma,
    Depth,
    Allot,
}

impl Builtin {
    pub fn word(&self) -> &'static str {
        match *self {
            Builtin::Dot => ".",
            Builtin::Plus => "+",
            Builtin::Minus => "-",
            Builtin::Star => "*",
            Builtin::Slash => "/",
            Builtin::Abs => "abs",
            Builtin::And => "and",
            Builtin::Or => "or",
            Builtin::Xor => "xor",
            Builtin::LShift => "lshift",
            Builtin::RShift => "rshift",
            Builtin::Equals => "=",
            Builtin::NotEquals => "<>",
            Builtin::LessThan => "<",
            Builtin::GreaterThan => ">",
            Builtin::LessEqual => "<=",
            Builtin::GreaterEqual => ">=",
            Builtin::Emit => "emit",
            Builtin::Dup => "dup",
            Builtin::Swap => "swap",
            Builtin::Over => "over",
            Builtin::Rot => "rot",
            Builtin::Tuck => "tuck",
            Builtin::Drop => "drop",
            Builtin::Here => "here",
            Builtin::Fetch => "@",
            Builtin::Store => "!",
            Builtin::Comma => ",",
            Builtin::Depth => "depth",
            Builtin::Allot => "allot",
        }
    }

    pub fn call(&self, state: &mut InterpState) {
        let stack = &mut state.stack;

        /// Allows defining a builtin using closure-like syntax. Note that the
        /// arguments are in reverse order, as that is how the stack is laid out.
        macro_rules! stackexpr {
            ( | $($aname:ident : $atype:ty),+ | $value:expr ) => {
                {
                    $(
                        let $aname: $atype = stack.pop();
                    )+
                    stack.push($value)
                }
            }
        }

        use self::Builtin::*;
        match *self {
            Dot => print!("{}", stack.pop::<i32>()),
            Plus => stackexpr!(|n2: i32, n1: i32| n1 + n2),
            Minus => stackexpr!(|n2: i32, n1: i32| n1 - n2),
            Star => stackexpr!(|n2: i32, n1: i32| n1 * n2),
            Slash => stackexpr!(|n2: i32, n1: i32| n1 / n2),
            Abs => stackexpr!(|n: i32| n.abs()),
            And => stackexpr!(|n2: i32, n1: i32| n1 & n2),
            Or => stackexpr!(|n2: i32, n1: i32| n1 | n2),
            Xor => stackexpr!(|n2: i32, n1: i32| n1 ^ n2),
            LShift => stackexpr!(|u: i32, n: i32| n << u),
            RShift => stackexpr!(|u: i32, n: i32| n >> u),
            Equals => stackexpr!(|n2: i32, n1: i32| n1 == n2),
            NotEquals => stackexpr!(|n2: i32, n1: i32| n1 != n2),
            LessThan => stackexpr!(|n2: i32, n1: i32| n1 < n2),
            GreaterThan => stackexpr!(|n2: i32, n1: i32| n1 > n2),
            LessEqual => stackexpr!(|n2: i32, n1: i32| n1 <= n2),
            GreaterEqual => stackexpr!(|n2: i32, n1: i32| n1 >= n2),
            Emit => print!("{}", stack.pop::<char>()),
            Dup => {
                let n: i32 = stack.peak();
                stack.push(n);
            }
            Swap => {
                let n2: i32 = stack.pop();
                let n1: i32 = stack.pop();
                stack.push(n2);
                stack.push(n1);
            }
            Over => {
                let n2: i32 = stack.pop();
                let n1: i32 = stack.pop();
                stack.push(n1);
                stack.push(n2);
                stack.push(n1);
            }
            Rot => {
                let n3: i32 = stack.pop();
                let n2: i32 = stack.pop();
                let n1: i32 = stack.pop();
                stack.push(n2);
                stack.push(n3);
                stack.push(n1);
            }
            Tuck => {
                let n2: i32 = stack.pop();
                let n1: i32 = stack.pop();
                stack.push(n2);
                stack.push(n1);
                stack.push(n2);
            }
            Drop => {
                stack.pop::<i32>();
            }
            Fetch => stackexpr!(|addr: i32| state.memory.get::<i32>(addr)),
            Store => {
                let addr: i32 = stack.pop();
                let n: i32 = stack.pop();
                state.memory.set(addr, n);
            }
            Here => stack.push(state.memory.here()),
            Comma => {
                state.memory.new(stack.pop::<i32>());
            }
            Depth => {
                let len = stack.len();
                stack.push(len);
            }
            Allot => {
                let n = stack.pop();
                if n < 0 {
                    // TODO Free memory
                } else {
                    for _ in 0..n {
                        state.memory.new(0);
                    }
                }
            }
        }
    }
}
<commit_msg>Extend functionality of stackexpr!, reducing code duplication<commit_after>use state::InterpState;

/// Represents a builtin function
#[derive(Debug, Clone, PartialEq, Variants)]
pub enum Builtin {
    Dot,
    Plus,
    Minus,
    Star,
    Slash,
    Abs,
    And,
    Or,
    Xor,
    LShift,
    RShift,
    Equals,
    NotEquals,
    LessThan,
    GreaterThan,
    LessEqual,
    GreaterEqual,
    Emit,
    Dup,
    Swap,
    Over,
    Rot,
    Tuck,
    Drop,
    Fetch,
    Store,
    Here,
    Comma,
    Depth,
    Allot,
}

impl Builtin {
    pub fn word(&self) -> &'static str {
        match *self {
            Builtin::Dot => ".",
            Builtin::Plus => "+",
            Builtin::Minus => "-",
            Builtin::Star => "*",
            Builtin::Slash => "/",
            Builtin::Abs => "abs",
            Builtin::And => "and",
            Builtin::Or => "or",
            Builtin::Xor => "xor",
            Builtin::LShift => "lshift",
            Builtin::RShift => "rshift",
            Builtin::Equals => "=",
            Builtin::NotEquals => "<>",
            Builtin::LessThan => "<",
            Builtin::GreaterThan => ">",
            Builtin::LessEqual => "<=",
            Builtin::GreaterEqual => ">=",
            Builtin::Emit => "emit",
            Builtin::Dup => "dup",
            Builtin::Swap => "swap",
            Builtin::Over => "over",
            Builtin::Rot => "rot",
            Builtin::Tuck => "tuck",
            Builtin::Drop => "drop",
            Builtin::Here => "here",
            Builtin::Fetch => "@",
            Builtin::Store => "!",
            Builtin::Comma => ",",
            Builtin::Depth => "depth",
            Builtin::Allot => "allot",
        }
    }

    pub fn call(&self, state: &mut InterpState) {
        let stack = &mut state.stack;

        /// Allows defining a builtin using closure-like syntax. Note that the
        /// arguments are in reverse order, as that is how the stack is laid out.
        macro_rules! stackexpr {
            ( | $($aname:ident : $atype:ty),+ | $($value:expr),+ ) => {
                {
                    $(
                        let $aname: $atype = stack.pop();
                    )+
                    $(
                        stack.push($value);
                    )+
                }
            }
        }

        use self::Builtin::*;
        match *self {
            Dot => print!("{}", stack.pop::<i32>()),
            Plus => stackexpr!(|n2: i32, n1: i32| n1 + n2),
            Minus => stackexpr!(|n2: i32, n1: i32| n1 - n2),
            Star => stackexpr!(|n2: i32, n1: i32| n1 * n2),
            Slash => stackexpr!(|n2: i32, n1: i32| n1 / n2),
            Abs => stackexpr!(|n: i32| n.abs()),
            And => stackexpr!(|n2: i32, n1: i32| n1 & n2),
            Or => stackexpr!(|n2: i32, n1: i32| n1 | n2),
            Xor => stackexpr!(|n2: i32, n1: i32| n1 ^ n2),
            LShift => stackexpr!(|u: i32, n: i32| n << u),
            RShift => stackexpr!(|u: i32, n: i32| n >> u),
            Equals => stackexpr!(|n2: i32, n1: i32| n1 == n2),
            NotEquals => stackexpr!(|n2: i32, n1: i32| n1 != n2),
            LessThan => stackexpr!(|n2: i32, n1: i32| n1 < n2),
            GreaterThan => stackexpr!(|n2: i32, n1: i32| n1 > n2),
            LessEqual => stackexpr!(|n2: i32, n1: i32| n1 <= n2),
            GreaterEqual => stackexpr!(|n2: i32, n1: i32| n1 >= n2),
            Emit => print!("{}", stack.pop::<char>()),
            Dup => {
                let n: i32 = stack.peak();
                stack.push(n);
            }
            Swap => stackexpr!(|n2: i32, n1: i32| n2, n1),
            Over => stackexpr!(|n2: i32, n1: i32| n1, n2, n1),
            Rot => stackexpr!(|n3: i32, n2: i32, n1: i32| n2, n3, n1),
            Tuck => stackexpr!(|n2: i32, n1: i32| n2, n1, n2),
            Drop => {
                stack.pop::<i32>();
            }
            Fetch => stackexpr!(|addr: i32| state.memory.get::<i32>(addr)),
            Store => {
                let addr: i32 = stack.pop();
                let n: i32 = stack.pop();
                state.memory.set(addr, n);
            }
            Here => stack.push(state.memory.here()),
            Comma => {
                state.memory.new(stack.pop::<i32>());
            }
            Depth => {
                let len = stack.len();
                stack.push(len);
            }
            Allot => {
                let n = stack.pop();
                if n < 0 {
                    // TODO Free memory
                } else {
                    for _ in 0..n {
                        state.memory.new(0);
                    }
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    Eof,
    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

#[derive(Debug)]
pub struct TokenWithContext {
    pub token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    lexeme: String,
    /// Position of the first character of the token
    position: Position,
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        match next {
            Some(c) => {
                self.current_lexeme.push(c);
                if c == '\n' {
                    self.current_position.increment_line();
                } else {
                    self.current_position.increment_column();;
                }
            }
            None => (),
        }
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn advance_while(&mut self, condition: &Fn(char) -> bool) -> () {
        while self.peek_check(condition) {
            self.advance();
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, String> {
        let initial_line = self.current_position.line;
        self.advance_while(&|c| c != '"');
        if !self.advance_if_match('"') {
            return Err(format!("Unterminated string at line {}", initial_line));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        self.advance_while(&is_digit);
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            self.advance_while(&is_digit);
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        self.advance_while(&is_alphanumeric);
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Result<TokenWithContext, String> {
        let initial_position = self.current_position;
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => {
                return Ok(TokenWithContext {
                    token: Token::Eof,
                    // Not very meaningful. Is Eof needed at all?
                    lexeme: "".into(),
                    position: initial_position,
                });
            }
        };
        let token = match next_char {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.advance_if_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    self.advance_while(&|c| c != '\n');
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => try!(self.string()),
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Err(format!("Unexpected character {} at line {}, column {}",
                                   c,
                                   self.current_position.line,
                                   self.current_position.column - 1));
            }
        };
        Ok(self.add_context(token, initial_position))
    }
}

pub fn scan(source: &String) -> Result<Vec<TokenWithContext>, String> {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut eof = false;
    while !eof {
        let token_with_context = try!(scanner.scan_next());
        match token_with_context.token {
            // Ignoring tokens we don't care about
            Token::Comment => {}
            Token::Whitespace => {}
            Token::Eof => {
                eof = true;
                tokens.push(token_with_context);
            }
            _ => tokens.push(token_with_context),
        }
    }
    Ok(tokens)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let tokens = scan(&"+".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let tokens = scan(&"1+2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn expression_with_whitespaces() {
        let tokens = scan(&"1 + 2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn assignement_with_comment() {
        let tokens = scan(&"var a = 1.0; // A comment".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Eof);
    }

    #[test]
    fn multiline_statements() {
        let tokens = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
                .into())
            .unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[10].token, Token::Eof);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }
}
<commit_msg>Initial removal of EOF token<commit_after>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    Eof,
    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

#[derive(Debug)]
pub struct TokenWithContext {
    pub token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    lexeme: String,
    /// Position of the first character of the token
    position: Position,
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        match next {
            Some(c) => {
                self.current_lexeme.push(c);
                if c == '\n' {
                    self.current_position.increment_line();
                } else {
                    self.current_position.increment_column();;
                }
            }
            None => (),
        }
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn advance_while(&mut self, condition: &Fn(char) -> bool) -> () {
        while self.peek_check(condition) {
            self.advance();
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, String> {
        let initial_line = self.current_position.line;
        self.advance_while(&|c| c != '"');
        if !self.advance_if_match('"') {
            return Err(format!("Unterminated string at line {}", initial_line));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        self.advance_while(&is_digit);
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            self.advance_while(&is_digit);
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        self.advance_while(&is_alphanumeric);
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Result<TokenWithContext, String> {
        let initial_position = self.current_position;
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => {
                return Ok(TokenWithContext {
                    token: Token::Eof,
                    // Not very meaningful. Is Eof needed at all?
                    lexeme: "".into(),
                    position: initial_position,
                });
            }
        };
        let token = match next_char {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.advance_if_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    self.advance_while(&|c| c != '\n');
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => try!(self.string()),
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Err(format!("Unexpected character {} at line {}, column {}",
                                   c,
                                   self.current_position.line,
                                   self.current_position.column - 1));
            }
        };
        Ok(self.add_context(token, initial_position))
    }
}

pub fn scan(source: &String) -> Result<Vec<TokenWithContext>, String> {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut eof = false;
    while !eof {
        let token_with_context = try!(scanner.scan_next());
        match token_with_context.token {
            // Ignoring tokens we don't care about
            Token::Comment => {}
            Token::Whitespace => {}
            Token::Eof => {
                eof = true;
            }
            _ => tokens.push(token_with_context),
        }
    }
    Ok(tokens)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let tokens = scan(&"+".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let tokens = scan(&"1+2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn expression_with_whitespaces() {
        let tokens = scan(&"1 + 2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn assignement_with_comment() {
        let tokens = scan(&"var a = 1.0; // A comment".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
    }

    #[test]
    fn multiline_statements() {
        let tokens = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
                .into())
            .unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }
}
<|endoftext|>