<commit_before>//! Implements basic arithmetic operations in GF(2^8).

/// Multiply two elements of GF(2^8).
pub fn mul(e: u8, a: u8) -> u8 {
    // This algorithm is constant-time, allowing us to perform GF256 arithmetic over secret values
    // without leaking information about the values via timing.
    let mut aa = e;
    let mut bb = a;
    let mut r = 0;
    // Loop over all 8 bits, regardless of whether or not it's required. aa != 0 is the usual loop
    // condition, but here it's folded into the per-round bitmask.
    for _ in 0..8 {
        // Perform r ^= bb iff aa & 1 == 0 && aa != 0.
        r ^= bb & ((aa & 1 != 0) as u8 * 0xff) & ((aa != 0) as u8 * 0xff);
        let t = bb & 0x80;
        bb <<= 1;
        // Perform bb ^= 0x1b iff t != 0 && aa != 0.
        bb ^= 0x1b & ((t != 0) as u8 * 0xff) & ((aa != 0) as u8 * 0xff);
        aa >>= 1;
    }
    r
}

/// Divide one element of GF(2^8) by another.
pub fn div(e: u8, a: u8) -> u8 {
    if a == 0 {
        panic!("Divide by zero: {} / {}", e, a)
    }

    // Again, this algorithm is constant-time. First, we find the multiplicative inverse of `a` by
    // iterating over all possible values, and using a bitmask to accumulate only that value which,
    // when multiplied by `a`, is `1`.
    let mut inv = 0;
    for i in 0x00..=0xff {
        inv += i & ((mul(i, a) == 1) as u8 * 0xff);
    }

    // Finally, we multiply `e` by the multiplicative inverse of `a`.
    mul(inv, e)
}

#[cfg(test)]
mod test {
    extern crate proptest;

    use super::*;

    use self::proptest::prelude::*;

    proptest! {
        #[test]
        fn div_is_inverse_of_mul(a in 0u8..=255, b in 1u8..=255) {
            assert_eq!(mul(div(a, b), b), a);
        }

        #[test]
        fn mul_is_inverse_of_div(a in 0u8..=255, b in 1u8..=255) {
            assert_eq!(div(mul(a, b), b), a);
        }

        #[test]
        fn mul_is_commutative(a in 0u8..=255, b in 0u8..=255) {
            assert_eq!(mul(a, b), mul(b, a));
        }
    }

    #[test]
    fn test_mul() {
        assert_eq!(mul(90, 21), 254);
        assert_eq!(mul(133, 5), 167);
    }

    #[test]
    fn test_mul_zero() {
        assert_eq!(mul(0, 21), 0);
    }

    #[test]
    fn test_div() {
        assert_eq!(div(90, 21), 189);
        assert_eq!(div(6, 55), 151);
        assert_eq!(div(22, 192), 138);
    }

    #[test]
    fn test_div_zero() {
        assert_eq!(div(0, 21), 0);
    }
}
<commit_msg>Name parameters a little better.<commit_after>//! Implements basic arithmetic operations in GF(2^8).

/// Multiply two elements of GF(2^8).
pub fn mul(lhs: u8, rhs: u8) -> u8 {
    // This algorithm is constant-time, allowing us to perform GF256 arithmetic over secret values
    // without leaking information about the values via timing.
    let mut aa = lhs;
    let mut bb = rhs;
    let mut r = 0;
    // Loop over all 8 bits, regardless of whether or not it's required. aa != 0 is the usual loop
    // condition, but here it's folded into the per-round bitmask.
    for _ in 0..8 {
        // Perform r ^= bb iff aa & 1 == 0 && aa != 0.
        r ^= bb & ((aa & 1 != 0) as u8 * 0xff) & ((aa != 0) as u8 * 0xff);
        let t = bb & 0x80;
        bb <<= 1;
        // Perform bb ^= 0x1b iff t != 0 && aa != 0.
        bb ^= 0x1b & ((t != 0) as u8 * 0xff) & ((aa != 0) as u8 * 0xff);
        aa >>= 1;
    }
    r
}

/// Divide one element of GF(2^8) by another.
pub fn div(lhs: u8, rhs: u8) -> u8 {
    if rhs == 0 {
        panic!("Divide by zero: {} / {}", lhs, rhs)
    }

    // Again, this algorithm is constant-time. First, we find the multiplicative inverse of `a` by
    // iterating over all possible values, and using a bitmask to accumulate only that value which,
    // when multiplied by `a`, is `1`.
    let mut inv = 0;
    for i in 0x00..=0xff {
        inv += i & ((mul(i, rhs) == 1) as u8 * 0xff);
    }

    // Finally, we multiply `e` by the multiplicative inverse of `a`.
    mul(inv, lhs)
}

#[cfg(test)]
mod test {
    extern crate proptest;

    use super::*;

    use self::proptest::prelude::*;

    proptest! {
        #[test]
        fn div_is_inverse_of_mul(a in 0u8..=255, b in 1u8..=255) {
            assert_eq!(mul(div(a, b), b), a);
        }

        #[test]
        fn mul_is_inverse_of_div(a in 0u8..=255, b in 1u8..=255) {
            assert_eq!(div(mul(a, b), b), a);
        }

        #[test]
        fn mul_is_commutative(a in 0u8..=255, b in 0u8..=255) {
            assert_eq!(mul(a, b), mul(b, a));
        }
    }

    #[test]
    fn test_mul() {
        assert_eq!(mul(90, 21), 254);
        assert_eq!(mul(133, 5), 167);
    }

    #[test]
    fn test_mul_zero() {
        assert_eq!(mul(0, 21), 0);
    }

    #[test]
    fn test_div() {
        assert_eq!(div(90, 21), 189);
        assert_eq!(div(6, 55), 151);
        assert_eq!(div(22, 192), 138);
    }

    #[test]
    fn test_div_zero() {
        assert_eq!(div(0, 21), 0);
    }
}
<|endoftext|>


<commit_before>
use gfx;
use gfx::DeviceHelper;
use graphics::{
    BackEnd,
};

use Texture;

static BUFFER_SIZE: uint = 1024;

static VERTEX_SHADER: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
attribute vec2 pos;
attribute vec4 color;
varying vec4 v_Color;
void main() {
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
    GLSL_150: b"
#version 150 core
in vec2 pos;
in vec4 color;
out vec4 v_Color;
void main() {
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
};

static FRAGMENT_SHADER: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
varying vec4 v_Color;
void main() {
    gl_FragColor = v_Color;
}
"
    GLSL_150: b"
#version 150 core
in vec4 v_Color;
out vec4 o_Color;
void main() {
    o_Color = v_Color;
}
"
};

static VERTEX_SHADER_UV: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
attribute vec2 pos;
attribute vec4 color;
attribute vec2 uv;
uniform sampler2D s_texture;
varying vec4 v_Color;
varying vec2 v_UV;
void main() {
    v_UV = uv;
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
    GLSL_150: b"
#version 150 core
in vec2 pos;
in vec4 color;
in vec2 uv;
uniform sampler2D s_texture;
out vec4 v_Color;
out vec2 v_UV;
void main() {
    v_UV = uv;
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
};

static FRAGMENT_SHADER_UV: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
uniform sampler2D s_texture;
varying vec2 v_UV;
varying vec4 v_Color;
void main()
{
    gl_FragColor = texture2D(s_texture, v_UV) * v_Color;
}
"
    GLSL_150: b"
#version 150 core
out vec4 o_Color;
uniform sampler2D s_texture;
in vec2 v_UV;
in vec4 v_Color;
void main()
{
    o_Color = texture(s_texture, v_UV) * v_Color;
}
"
};

#[vertex_format]
struct Vertex {
    pos: [f32, ..2],
    color: [f32, ..4],
}

impl Vertex {
    fn new(pos: [f32, ..2], color: [f32, ..4]) -> Vertex {
        Vertex {
            pos: pos,
            color: color,
        }
    }
}

#[vertex_format]
struct VertexUV {
    pos: [f32, ..2],
    color: [f32, ..4],
    uv: [f32, ..2],
}

impl VertexUV {
    fn new(pos: [f32, ..2], color: [f32, ..4], uv: [f32, ..2]) -> VertexUV {
        VertexUV {
            pos: pos,
            color: color,
            uv: uv,
        }
    }
}

#[shader_param(BatchUV, OwnedBatchUV)]
struct ParamsUV {
    s_texture: gfx::shade::TextureParam,
}

/// The graphics back-end.
pub struct Gfx2d<C: gfx::CommandBuffer> {
    buffer: gfx::BufferHandle<Vertex>,
    buffer_uv: gfx::BufferHandle<VertexUV>,
    batch: gfx::batch::OwnedBatch<(), ()>,
    batch_uv: OwnedBatchUV,
}

impl<C: gfx::CommandBuffer> Gfx2d<C> {
    /// Creates a new Gfx2d object.
    pub fn new<D: gfx::Device<C>>(device: &mut D) -> Gfx2d<C> {
        let program = device.link_program(
                VERTEX_SHADER.clone(),
                FRAGMENT_SHADER.clone()
            ).unwrap();
        let program_uv = device.link_program(
                VERTEX_SHADER_UV.clone(),
                FRAGMENT_SHADER_UV.clone()
            ).unwrap();
        let buffer = device.create_buffer(BUFFER_SIZE, gfx::UsageDynamic);
        let buffer_uv = device.create_buffer(BUFFER_SIZE, gfx::UsageDynamic);
        let mesh = gfx::Mesh::from_format(buffer, BUFFER_SIZE as u32);
        let mesh_uv = gfx::Mesh::from_format(buffer_uv, BUFFER_SIZE as u32);
        let batch = gfx::batch::OwnedBatch::new(mesh, program, ()).unwrap();

        let sampler = device.create_sampler(
                gfx::tex::SamplerInfo::new(gfx::tex::Trilinear,
                                           gfx::tex::Clamp)
            );

        // Create a dummy texture
        let texture_info = gfx::tex::TextureInfo {
            width: 1,
            height: 1,
            depth: 1,
            levels: 1,
            kind: gfx::tex::Texture2D,
            format: gfx::tex::RGBA8,
        };
        let image_info = texture_info.to_image_info();
        let texture = device.create_texture(texture_info).unwrap();
        device.update_texture(&texture, &image_info,
                &vec![0x20u8, 0xA0u8, 0xC0u8, 0x00u8].as_slice())
            .unwrap();
        let params_uv = ParamsUV {
            s_texture: (texture, Some(sampler))
        };

        let batch_uv = gfx::batch::OwnedBatch::new(
            mesh_uv, program_uv, params_uv).unwrap();

        Gfx2d {
            buffer: buffer,
            buffer_uv: buffer_uv,
            batch: batch,
            batch_uv: batch_uv,
        }
    }
}

/// Used for rendering 2D graphics.
pub struct RenderContext<'a, C: 'a + gfx::CommandBuffer> {
    renderer: &'a mut gfx::Renderer<C>,
    frame: &'a gfx::Frame,
    gfx2d: &'a mut Gfx2d<C>,
}

impl<'a, C: gfx::CommandBuffer> RenderContext<'a, C> {
    /// Creates a new object for rendering 2D graphics.
    pub fn new(renderer: &'a mut gfx::Renderer<C>,
               frame: &'a gfx::Frame,
               gfx2d: &'a mut Gfx2d<C>) -> RenderContext<'a, C> {
        RenderContext {
            renderer: renderer,
            frame: frame,
            gfx2d: gfx2d
        }
    }
}

impl<'a, C: gfx::CommandBuffer> BackEnd<Texture>
for RenderContext<'a, C> {
    fn supports_clear_rgba(&self) -> bool { true }

    fn clear_rgba(&mut self, r: f32, g: f32, b: f32, a: f32) {
        let &RenderContext {
            ref mut renderer,
            frame,
            ..
        } = self;
        renderer.clear(
                gfx::ClearData {
                    color: [r, g, b, a],
                    depth: 0.0,
                    stencil: 0,
                },
                gfx::Color,
                frame
            );
    }

    fn enable_alpha_blend(&mut self) {
        use std::default::Default;
        use gfx::state::{Normal, Inverse, Factor};

        self.gfx2d.batch.state.blend = Some(gfx::state::Blend {
                value: [1.0, 1.0, 1.0, 1.0],
                color: gfx::state::BlendChannel {
                        equation: gfx::state::FuncAdd,
                        source: Factor(Normal, gfx::state::SourceAlpha),
                        destination: Factor(Inverse, gfx::state::SourceAlpha)
                    },
                alpha: Default::default()
            })
    }

    fn disable_alpha_blend(&mut self) {
        self.gfx2d.batch.state.blend = None;
    }

    fn supports_tri_list_xy_f32_rgba_f32(&self) -> bool { true }

    fn tri_list_xy_f32_rgba_f32(
        &mut self,
        vertices: &[f32],
        colors: &[f32]
    ) {
        let &RenderContext {
            ref mut renderer,
            ref frame,
            gfx2d: &Gfx2d {
                ref mut buffer,
                ref mut batch,
                ..
            }
        } = self;
        let n = vertices.len() / 2;
        let mut vertex_data = Vec::with_capacity(n);
        for i in range(0, n) {
            vertex_data.push(
                Vertex::new(
                    [vertices[2 * i], vertices[2 * i + 1]],
                    [
                        colors[4 * i],
                        colors[4 * i + 1],
                        colors[4 * i + 2],
                        colors[4 * i + 3]
                    ]
                )
            );
        }

        let n = vertex_data.len();
        renderer.update_buffer_vec(*buffer, vertex_data, 0);
        batch.slice = gfx::VertexSlice(gfx::TriangleList, 0, n as u32);
        renderer.draw(&*batch, *frame);
    }

    fn supports_single_texture(&self) -> bool { true }

    fn enable_single_texture(&mut self, texture: &Texture) {
        let ParamsUV {
            s_texture: (ref mut s_texture, _)
        } = self.gfx2d.batch_uv.param;
        *s_texture = texture.handle;
    }

    fn disable_single_texture(&mut self) {}

    // Assume all textures has alpha channel for now.
    fn has_texture_alpha(&self, _texture: &Texture) -> bool { true }

    fn supports_tri_list_xy_f32_rgba_f32_uv_f32(&self) -> bool { true }

    fn tri_list_xy_f32_rgba_f32_uv_f32(
        &mut self,
        vertices: &[f32],
        colors: &[f32],
        texture_coords: &[f32]
    ) {
        let &RenderContext {
            ref mut renderer,
            ref frame,
            gfx2d: &Gfx2d {
                ref mut buffer_uv,
                ref mut batch_uv,
                ..
            }
        } = self;
        let n = vertices.len() / 2;
        let mut vertex_data = Vec::with_capacity(n);
        for i in range(0, n) {
            vertex_data.push(
                VertexUV::new(
                    [vertices[2 * i], vertices[2 * i + 1]],
                    [
                        colors[4 * i],
                        colors[4 * i + 1],
                        colors[4 * i + 2],
                        colors[4 * i + 3]
                    ],
                    [
                        texture_coords[2 * i],
                        texture_coords[2 * i + 1]
                    ]
                )
            );
        }

        let n = vertex_data.len();
        renderer.update_buffer_vec(*buffer_uv, vertex_data, 0);
        batch_uv.slice = gfx::VertexSlice(gfx::TriangleList, 0, n as u32);
        renderer.draw(&*batch_uv, *frame);
    }
}
<commit_msg>Using max vertex count from rust-graphics<commit_after>
use gfx;
use gfx::DeviceHelper;
use graphics::BackEnd;
use graphics::BACK_END_MAX_VERTEX_COUNT as BUFFER_SIZE;

use Texture;

static VERTEX_SHADER: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
attribute vec2 pos;
attribute vec4 color;
varying vec4 v_Color;
void main() {
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
    GLSL_150: b"
#version 150 core
in vec2 pos;
in vec4 color;
out vec4 v_Color;
void main() {
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
};

static FRAGMENT_SHADER: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
varying vec4 v_Color;
void main() {
    gl_FragColor = v_Color;
}
"
    GLSL_150: b"
#version 150 core
in vec4 v_Color;
out vec4 o_Color;
void main() {
    o_Color = v_Color;
}
"
};

static VERTEX_SHADER_UV: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
attribute vec2 pos;
attribute vec4 color;
attribute vec2 uv;
uniform sampler2D s_texture;
varying vec4 v_Color;
varying vec2 v_UV;
void main() {
    v_UV = uv;
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
    GLSL_150: b"
#version 150 core
in vec2 pos;
in vec4 color;
in vec2 uv;
uniform sampler2D s_texture;
out vec4 v_Color;
out vec2 v_UV;
void main() {
    v_UV = uv;
    v_Color = color;
    gl_Position = vec4(pos, 0.0, 1.0);
}
"
};

static FRAGMENT_SHADER_UV: gfx::ShaderSource = shaders!{
    GLSL_120: b"
#version 120
uniform sampler2D s_texture;
varying vec2 v_UV;
varying vec4 v_Color;
void main()
{
    gl_FragColor = texture2D(s_texture, v_UV) * v_Color;
}
"
    GLSL_150: b"
#version 150 core
out vec4 o_Color;
uniform sampler2D s_texture;
in vec2 v_UV;
in vec4 v_Color;
void main()
{
    o_Color = texture(s_texture, v_UV) * v_Color;
}
"
};

#[vertex_format]
struct Vertex {
    pos: [f32, ..2],
    color: [f32, ..4],
}

impl Vertex {
    fn new(pos: [f32, ..2], color: [f32, ..4]) -> Vertex {
        Vertex {
            pos: pos,
            color: color,
        }
    }
}

#[vertex_format]
struct VertexUV {
    pos: [f32, ..2],
    color: [f32, ..4],
    uv: [f32, ..2],
}

impl VertexUV {
    fn new(pos: [f32, ..2], color: [f32, ..4], uv: [f32, ..2]) -> VertexUV {
        VertexUV {
            pos: pos,
            color: color,
            uv: uv,
        }
    }
}

#[allow(unused_imports)]
#[shader_param(BatchUV, OwnedBatchUV)]
struct ParamsUV {
    s_texture: gfx::shade::TextureParam,
}

/// The graphics back-end.
pub struct Gfx2d<C: gfx::CommandBuffer> {
    buffer: gfx::BufferHandle<Vertex>,
    buffer_uv: gfx::BufferHandle<VertexUV>,
    batch: gfx::batch::OwnedBatch<(), ()>,
    batch_uv: OwnedBatchUV,
}

impl<C: gfx::CommandBuffer> Gfx2d<C> {
    /// Creates a new Gfx2d object.
    pub fn new<D: gfx::Device<C>>(device: &mut D) -> Gfx2d<C> {
        let program = device.link_program(
                VERTEX_SHADER.clone(),
                FRAGMENT_SHADER.clone()
            ).unwrap();
        let program_uv = device.link_program(
                VERTEX_SHADER_UV.clone(),
                FRAGMENT_SHADER_UV.clone()
            ).unwrap();
        let buffer = device.create_buffer(BUFFER_SIZE, gfx::UsageDynamic);
        let buffer_uv = device.create_buffer(BUFFER_SIZE, gfx::UsageDynamic);
        let mesh = gfx::Mesh::from_format(buffer, BUFFER_SIZE as u32);
        let mesh_uv = gfx::Mesh::from_format(buffer_uv, BUFFER_SIZE as u32);
        let batch = gfx::batch::OwnedBatch::new(mesh, program, ()).unwrap();

        let sampler = device.create_sampler(
                gfx::tex::SamplerInfo::new(gfx::tex::Trilinear,
                                           gfx::tex::Clamp)
            );

        // Create a dummy texture
        let texture_info = gfx::tex::TextureInfo {
            width: 1,
            height: 1,
            depth: 1,
            levels: 1,
            kind: gfx::tex::Texture2D,
            format: gfx::tex::RGBA8,
        };
        let image_info = texture_info.to_image_info();
        let texture = device.create_texture(texture_info).unwrap();
        device.update_texture(&texture, &image_info,
                &vec![0x20u8, 0xA0u8, 0xC0u8, 0x00u8].as_slice())
            .unwrap();
        let params_uv = ParamsUV {
            s_texture: (texture, Some(sampler))
        };

        let batch_uv = gfx::batch::OwnedBatch::new(
            mesh_uv, program_uv, params_uv).unwrap();

        Gfx2d {
            buffer: buffer,
            buffer_uv: buffer_uv,
            batch: batch,
            batch_uv: batch_uv,
        }
    }
}

/// Used for rendering 2D graphics.
pub struct RenderContext<'a, C: 'a + gfx::CommandBuffer> {
    renderer: &'a mut gfx::Renderer<C>,
    frame: &'a gfx::Frame,
    gfx2d: &'a mut Gfx2d<C>,
}

impl<'a, C: gfx::CommandBuffer> RenderContext<'a, C> {
    /// Creates a new object for rendering 2D graphics.
    pub fn new(renderer: &'a mut gfx::Renderer<C>,
               frame: &'a gfx::Frame,
               gfx2d: &'a mut Gfx2d<C>) -> RenderContext<'a, C> {
        RenderContext {
            renderer: renderer,
            frame: frame,
            gfx2d: gfx2d
        }
    }
}

impl<'a, C: gfx::CommandBuffer> BackEnd<Texture>
for RenderContext<'a, C> {
    fn supports_clear_rgba(&self) -> bool { true }

    fn clear_rgba(&mut self, r: f32, g: f32, b: f32, a: f32) {
        let &RenderContext {
            ref mut renderer,
            frame,
            ..
        } = self;
        renderer.clear(
                gfx::ClearData {
                    color: [r, g, b, a],
                    depth: 0.0,
                    stencil: 0,
                },
                gfx::Color,
                frame
            );
    }

    fn enable_alpha_blend(&mut self) {
        use std::default::Default;
        use gfx::state::{Normal, Inverse, Factor};

        self.gfx2d.batch.state.blend = Some(gfx::state::Blend {
                value: [1.0, 1.0, 1.0, 1.0],
                color: gfx::state::BlendChannel {
                        equation: gfx::state::FuncAdd,
                        source: Factor(Normal, gfx::state::SourceAlpha),
                        destination: Factor(Inverse, gfx::state::SourceAlpha)
                    },
                alpha: Default::default()
            })
    }

    fn disable_alpha_blend(&mut self) {
        self.gfx2d.batch.state.blend = None;
    }

    fn supports_tri_list_xy_f32_rgba_f32(&self) -> bool { true }

    fn tri_list_xy_f32_rgba_f32(
        &mut self,
        vertices: &[f32],
        colors: &[f32]
    ) {
        let &RenderContext {
            ref mut renderer,
            ref frame,
            gfx2d: &Gfx2d {
                ref mut buffer,
                ref mut batch,
                ..
            }
        } = self;
        let n = vertices.len() / 2;
        let mut vertex_data = Vec::with_capacity(n);
        for i in range(0, n) {
            vertex_data.push(
                Vertex::new(
                    [vertices[2 * i], vertices[2 * i + 1]],
                    [
                        colors[4 * i],
                        colors[4 * i + 1],
                        colors[4 * i + 2],
                        colors[4 * i + 3]
                    ]
                )
            );
        }

        let n = vertex_data.len();
        renderer.update_buffer_vec(*buffer, vertex_data, 0);
        batch.slice = gfx::VertexSlice(gfx::TriangleList, 0, n as u32);
        renderer.draw(&*batch, *frame);
    }

    fn supports_single_texture(&self) -> bool { true }

    fn enable_single_texture(&mut self, texture: &Texture) {
        let ParamsUV {
            s_texture: (ref mut s_texture, _)
        } = self.gfx2d.batch_uv.param;
        *s_texture = texture.handle;
    }

    fn disable_single_texture(&mut self) {}

    // Assume all textures has alpha channel for now.
    fn has_texture_alpha(&self, _texture: &Texture) -> bool { true }

    fn supports_tri_list_xy_f32_rgba_f32_uv_f32(&self) -> bool { true }

    fn tri_list_xy_f32_rgba_f32_uv_f32(
        &mut self,
        vertices: &[f32],
        colors: &[f32],
        texture_coords: &[f32]
    ) {
        let &RenderContext {
            ref mut renderer,
            ref frame,
            gfx2d: &Gfx2d {
                ref mut buffer_uv,
                ref mut batch_uv,
                ..
            }
        } = self;
        let n = vertices.len() / 2;
        let mut vertex_data = Vec::with_capacity(n);
        for i in range(0, n) {
            vertex_data.push(
                VertexUV::new(
                    [vertices[2 * i], vertices[2 * i + 1]],
                    [
                        colors[4 * i],
                        colors[4 * i + 1],
                        colors[4 * i + 2],
                        colors[4 * i + 3]
                    ],
                    [
                        texture_coords[2 * i],
                        texture_coords[2 * i + 1]
                    ]
                )
            );
        }

        let n = vertex_data.len();
        renderer.update_buffer_vec(*buffer_uv, vertex_data, 0);
        batch_uv.slice = gfx::VertexSlice(gfx::TriangleList, 0, n as u32);
        renderer.draw(&*batch_uv, *frame);
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;

use matrix::{self, ClMatrix, ClMatrixMode};

use super::operation::Operation;
use super::var_store::{VarIndex, VarStore};

#[derive(Clone)]
pub struct OutGrad {
    gradient: Option<VarIndex>, // The gradient or sum of gradients
    gradients: Vec<VarIndex>,
}

impl OutGrad {
    pub fn new() -> Self {
        OutGrad {
            gradient: None,
            gradients: vec![],
        }
    }

    pub fn gradient(&self) -> VarIndex {
        self.gradient.unwrap()
    }

    pub fn try_gradient(&self) -> Option<VarIndex> {
        self.gradient
    }

    fn maybe_sum(&self, ctx: &matrix::Context, var_store: &mut VarStore) {
        if self.gradients.len() > 0 {
            if let Some(sum) = self.gradient {
                var_store.get(self.gradients[0]).copy_to(ctx, &mut var_store.get_mut(sum));
                for grad in &self.gradients[1..] {
                    var_store.get(sum).add(ctx, &var_store.get(*grad), &mut var_store.get_mut(sum));
                }
            }
        }
    }

    fn fork(&mut self, ctx: &matrix::Context, var_store: &mut VarStore, v: VarIndex) {
        if self.gradients.len() > 0 {
            // There are multiple gradients already, just add the new one to the list
            self.gradients.push(v);
        } else if let Some(gradient) = self.gradient {
            // There is still only one gradient, switch it to a fork
            let (rows, columns) = {
                let grad = var_store.get(gradient);
                (grad.rows(), grad.columns())
            };
            // Create variable for gradient sum
            self.gradient = Some(var_store.add(ClMatrix::new(ctx, rows, columns, ClMatrixMode::Mut)));
            self.gradients.push(gradient);
            self.gradients.push(v);
        } else {
            // This is the only gradient so far, so we don't need to some anything
            self.gradient = Some(v);
        }
    }
}

pub struct Node {
    pub inputs: Vec<VarIndex>,
    pub outputs: Vec<VarIndex>,
    pub in_grad: Vec<VarIndex>, // gradients on inputs
    pub out_grad: Vec<OutGrad>, // gradients on outputs
}

pub struct Graph {
    nodes: Vec<Node>,
    node_ops: Vec<Box<Operation>>,
    pub var_store: VarStore,
    out_var_map: HashMap<VarIndex, (NodeIndex, usize)>, // Maps output variable to it's node and index within node
    // Gradients on variables that are inputs to the graph - they have no corresponding node
    in_var_grad: HashMap<VarIndex, OutGrad>,
}

impl Graph {
    pub fn new() -> Self {
        Graph {
            nodes: vec![],
            node_ops: vec![],
            var_store: VarStore::new(),
            out_var_map: HashMap::new(),
            in_var_grad: HashMap::new(),
        }
    }

    pub fn add_node(&mut self,
                    ctx: &matrix::Context,
                    op: Box<Operation>,
                    inputs: Vec<VarIndex>,
                    out_shapes: &[(u64, u64)])
                    -> NodeIndex {
        let node_index = NodeIndex(self.nodes.len());

        // Create output variables
        let mut outputs = vec![];
        for (i, &(rows, cols)) in out_shapes.iter().enumerate() {
            let var_index = self.var_store.add(ClMatrix::new(ctx, rows as usize, cols as usize, ClMatrixMode::Mut));
            outputs.push(var_index);
            self.out_var_map.insert(var_index, (node_index, i));
        }
        // Create input gradient variables and set up gradient back flow
        let mut in_grad = vec![];
        for input in &inputs {
            // Create input gradient variables
            let (rows, cols) = (input.get(self).rows(), input.get(self).columns());
            let var_index = self.var_store.add(ClMatrix::new(ctx, rows as usize, cols as usize, ClMatrixMode::Mut));
            in_grad.push(var_index);

            // Set up gradient back flow
            match self.out_var_map.get(input).map(|x| *x) {
                Some((in_node, out_index)) => {
                    self.nodes[in_node.0].out_grad[out_index].fork(ctx, &mut self.var_store, var_index);
                },
                None => {
                    // This input doesn't come from a node's output. It is an input to the graph.
                    self.in_var_grad.get_mut(input).unwrap()
                        .fork(ctx, &mut self.var_store, var_index);
                },
            }
        }
        // Create the node
        self.nodes.push(Node { inputs: inputs,
                               outputs: outputs,
                               in_grad: in_grad,
                               out_grad: vec![OutGrad::new(); out_shapes.len()] });
        // Add the corresponding node op
        self.node_ops.push(op);
        node_index
    }

    pub fn add_variable(&mut self, ctx: &matrix::Context, shape: (u64, u64)) -> VarIndex {
        let v = self.var_store.add(ClMatrix::new(ctx, shape.0 as usize, shape.1 as usize, ClMatrixMode::Mut));
        self.in_var_grad.insert(v, OutGrad::new());
        v
    }

    pub fn add_gradient(&mut self, ctx: &matrix::Context, n: NodeIndex, out_index: usize) -> VarIndex {
        let (rows, cols) = {
            let grad = n.get(self).outputs[out_index];
            let grad = grad.get(self);
            (grad.rows(), grad.columns())
        };
        let v = self.var_store.add(ClMatrix::new(ctx, rows as usize, cols as usize, ClMatrixMode::Mut));
        self.nodes[n.0].out_grad[out_index].fork(ctx, &mut self.var_store, v);
        v
    }

    pub fn run(&mut self, ctx: &matrix::Context) {
        // Forward pass
        //
        // NOTE: We just execute the nodes in order. We can do this because of the way the graph is
        // built. When a user wants to add a node, he/she must also supply the inputs. This means
        // any dependencies must already be added before the node can be added. Therefore, we can
        // assert that all dependents come after their dependencies in the `self.nodes` array.
        for (node, op) in self.nodes.iter_mut().zip(&mut self.node_ops) {
            op.forward(ctx, &mut self.var_store, node);
        }

        // Backward pass
        for (node, op) in self.nodes.iter_mut().rev().zip(self.node_ops.iter_mut().rev()) {
            // Sum the gradients on each output if there are multiple gradients
            for out_grad in &node.out_grad {
                out_grad.maybe_sum(ctx, &mut self.var_store);
            }
            op.backward(ctx, &mut self.var_store, node);
        }
    }
}

////////////////////////////////////////////////////////////////////////////////////////////////////

#[derive(Copy, Clone)]
pub struct NodeIndex(usize);

impl NodeIndex {
    pub fn get<'a>(&self, g: &'a Graph) -> &'a Node {
        &g.nodes[self.0]
    }

    pub fn get_mut<'a>(&self, g: &'a mut Graph) -> &'a mut Node {
        &mut g.nodes[self.0]
    }
}

#[test]
fn it_works() {
    use super::operation::MatMul;

    let ctx = matrix::Context::new();

    // Setup the graph
    let mut graph = Graph::new();
    let a = graph.add_variable(&ctx, (1, 2));
    let wa = graph.add_variable(&ctx, (2, 3));
    let node = graph.add_node(&ctx,
                              Box::new(MatMul::new(&ctx, (1, 2), (2, 3))),
                              vec![a, wa],
                              &[(1, 3)]);
    let node_g = graph.add_gradient(&ctx, node, 0);

    // Send some input data
    let a_cpu = matrix::Matrix::from_vec(1, 2, vec![1.0, 1.0]);
    let wa_cpu = matrix::Matrix::from_vec(2, 3, vec![0.5, 0.3, 0.2,
                                                     0.6, 0.7, 0.7]);
    let node_g_cpu = matrix::Matrix::from_vec(1, 3, vec![1.0, -1.0, 0.5]);
    a.get(&graph).set(&ctx, &a_cpu);
    wa.get(&graph).set(&ctx, &wa_cpu);
    node_g.get(&graph).set(&ctx, &node_g_cpu);

    // Run the network
    graph.run(&ctx);
    let out = node.get(&graph).outputs[0].get(&graph).get(&ctx);
    println!("{:?}", out);
    assert!(false);
}
<commit_msg>Fixed up unit test<commit_after>use std::collections::HashMap;

use matrix::{self, ClMatrix, ClMatrixMode};

use super::operation::Operation;
use super::var_store::{VarIndex, VarStore};

#[derive(Clone)]
pub struct OutGrad {
    gradient: Option<VarIndex>, // The gradient or sum of gradients
    gradients: Vec<VarIndex>,
}

impl OutGrad {
    pub fn new() -> Self {
        OutGrad {
            gradient: None,
            gradients: vec![],
        }
    }

    pub fn gradient(&self) -> VarIndex {
        self.gradient.unwrap()
    }

    pub fn try_gradient(&self) -> Option<VarIndex> {
        self.gradient
    }

    fn maybe_sum(&self, ctx: &matrix::Context, var_store: &mut VarStore) {
        if self.gradients.len() > 0 {
            if let Some(sum) = self.gradient {
                var_store.get(self.gradients[0]).copy_to(ctx, &mut var_store.get_mut(sum));
                for grad in &self.gradients[1..] {
                    var_store.get(sum).add(ctx, &var_store.get(*grad), &mut var_store.get_mut(sum));
                }
            }
        }
    }

    fn fork(&mut self, ctx: &matrix::Context, var_store: &mut VarStore, v: VarIndex) {
        if self.gradients.len() > 0 {
            // There are multiple gradients already, just add the new one to the list
            self.gradients.push(v);
        } else if let Some(gradient) = self.gradient {
            // There is still only one gradient, switch it to a fork
            let (rows, columns) = {
                let grad = var_store.get(gradient);
                (grad.rows(), grad.columns())
            };
            // Create variable for gradient sum
            self.gradient = Some(var_store.add(ClMatrix::new(ctx, rows, columns, ClMatrixMode::Mut)));
            self.gradients.push(gradient);
            self.gradients.push(v);
        } else {
            // This is the only gradient so far, so we don't need to some anything
            self.gradient = Some(v);
        }
    }
}

pub struct Node {
    pub inputs: Vec<VarIndex>,
    pub outputs: Vec<VarIndex>,
    pub in_grad: Vec<VarIndex>, // gradients on inputs
    pub out_grad: Vec<OutGrad>, // gradients on outputs
}

pub struct Graph {
    nodes: Vec<Node>,
    node_ops: Vec<Box<Operation>>,
    pub var_store: VarStore,
    out_var_map: HashMap<VarIndex, (NodeIndex, usize)>, // Maps output variable to it's node and index within node
    // Gradients on variables that are inputs to the graph - they have no corresponding node
    in_var_grad: HashMap<VarIndex, OutGrad>,
}

impl Graph {
    pub fn new() -> Self {
        Graph {
            nodes: vec![],
            node_ops: vec![],
            var_store: VarStore::new(),
            out_var_map: HashMap::new(),
            in_var_grad: HashMap::new(),
        }
    }

    pub fn add_node(&mut self,
                    ctx: &matrix::Context,
                    op: Box<Operation>,
                    inputs: Vec<VarIndex>,
                    out_shapes: &[(u64, u64)])
                    -> NodeIndex {
        let node_index = NodeIndex(self.nodes.len());

        // Create output variables
        let mut outputs = vec![];
        for (i, &(rows, cols)) in out_shapes.iter().enumerate() {
            let var_index = self.var_store.add(ClMatrix::new(ctx, rows as usize, cols as usize, ClMatrixMode::Mut));
            outputs.push(var_index);
            self.out_var_map.insert(var_index, (node_index, i));
        }
        // Create input gradient variables and set up gradient back flow
        let mut in_grad = vec![];
        for input in &inputs {
            // Create input gradient variables
            let (rows, cols) = (input.get(self).rows(), input.get(self).columns());
            let var_index = self.var_store.add(ClMatrix::new(ctx, rows as usize, cols as usize, ClMatrixMode::Mut));
            in_grad.push(var_index);

            // Set up gradient back flow
            match self.out_var_map.get(input).map(|x| *x) {
                Some((in_node, out_index)) => {
                    self.nodes[in_node.0].out_grad[out_index].fork(ctx, &mut self.var_store, var_index);
                },
                None => {
                    // This input doesn't come from a node's output. It is an input to the graph.
                    self.in_var_grad.get_mut(input).unwrap()
                        .fork(ctx, &mut self.var_store, var_index);
                },
            }
        }
        // Create the node
        self.nodes.push(Node { inputs: inputs,
                               outputs: outputs,
                               in_grad: in_grad,
                               out_grad: vec![OutGrad::new(); out_shapes.len()] });
        // Add the corresponding node op
        self.node_ops.push(op);
        node_index
    }

    pub fn add_variable(&mut self, ctx: &matrix::Context, shape: (u64, u64)) -> VarIndex {
        let v = self.var_store.add(ClMatrix::new(ctx, shape.0 as usize, shape.1 as usize, ClMatrixMode::Mut));
        self.in_var_grad.insert(v, OutGrad::new());
        v
    }

    pub fn get_input_gradient<'a>(&'a self, v: VarIndex) -> Option<VarIndex> {
        self.in_var_grad.get(&v).and_then(|x| x.try_gradient())
    }

    pub fn add_gradient(&mut self, ctx: &matrix::Context, n: NodeIndex, out_index: usize) -> VarIndex {
        let (rows, cols) = {
            let grad = n.get(self).outputs[out_index];
            let grad = grad.get(self);
            (grad.rows(), grad.columns())
        };
        let v = self.var_store.add(ClMatrix::new(ctx, rows as usize, cols as usize, ClMatrixMode::Mut));
        self.nodes[n.0].out_grad[out_index].fork(ctx, &mut self.var_store, v);
        v
    }

    pub fn run(&mut self, ctx: &matrix::Context) {
        // Forward pass
        //
        // NOTE: We just execute the nodes in order. We can do this because of the way the graph is
        // built. When a user wants to add a node, he/she must also supply the inputs. This means
        // any dependencies must already be added before the node can be added. Therefore, we can
        // assert that all dependents come after their dependencies in the `self.nodes` array.
        for (node, op) in self.nodes.iter_mut().zip(&mut self.node_ops) {
            op.forward(ctx, &mut self.var_store, node);
        }

        // Backward pass
        for (node, op) in self.nodes.iter_mut().rev().zip(self.node_ops.iter_mut().rev()) {
            // Sum the gradients on each output if there are multiple gradients
            for out_grad in &node.out_grad {
                out_grad.maybe_sum(ctx, &mut self.var_store);
            }
            op.backward(ctx, &mut self.var_store, node);
        }
    }
}

////////////////////////////////////////////////////////////////////////////////////////////////////

#[derive(Copy, Clone)]
pub struct NodeIndex(usize);

impl NodeIndex {
    pub fn get<'a>(&self, g: &'a Graph) -> &'a Node {
        &g.nodes[self.0]
    }

    pub fn get_mut<'a>(&self, g: &'a mut Graph) -> &'a mut Node {
        &mut g.nodes[self.0]
    }
}

#[test]
fn it_works() {
    use super::operation::MatMul;

    let ctx = matrix::Context::new();

    // Setup the graph
    let mut graph = Graph::new();
    let a = graph.add_variable(&ctx, (1, 2));
    let wa = graph.add_variable(&ctx, (2, 3));
    let node = graph.add_node(&ctx,
                              Box::new(MatMul::new(&ctx, (1, 2), (2, 3))),
                              vec![a, wa],
                              &[(1, 3)]);
    let node_g = graph.add_gradient(&ctx, node, 0);

    // Send some input data
    let a_cpu = matrix::Matrix::from_vec(1, 2, vec![1.4, 0.3]);
    let wa_cpu = matrix::Matrix::from_vec(2, 3, vec![0.5, 0.3, 0.2,
                                                     0.6, 0.7, 0.7]);
    let node_g_cpu = matrix::Matrix::from_vec(1, 3, vec![1.0, -1.0, 0.5]);
    a.get(&graph).set(&ctx, &a_cpu);
    wa.get(&graph).set(&ctx, &wa_cpu);
    node_g.get(&graph).set(&ctx, &node_g_cpu);

    // Run the network
    graph.run(&ctx);
    let out = node.get(&graph).outputs[0].get(&graph).get(&ctx);
    let wa_g = graph.get_input_gradient(wa).unwrap().get(&graph).get(&ctx);
    println!("{:?}", out);
    println!("{:?}", wa_g);
    assert!(out.buffer() == &[0.88, 0.63, 0.49]);
    assert!(wa_g.buffer() == &[1.4, -1.4, 0.7,
                               0.3, -0.3, 0.15]);
}
<|endoftext|>


<commit_before>use std::collections::LinkedList;
use std::borrow::Borrow;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
    fn is_es_identifier(self) -> bool;
    fn is_es_identifier_start(self) -> bool;
    fn is_es_identifier_continue(self) -> bool;
}

/*
// https://github.com/ariya/esprima/blob/master/tools/generate-identifier-regex.js
// https://gist.github.com/mathiasbynens/6334847
// http://unicode.org/reports/tr31/

pub trait UnicodeCharExt {
    fn is_L(self) -> bool;
    fn is_Lu(self) -> bool;
    fn is_Ll(self) -> bool;
    fn is_Lt(self) -> bool;
    fn is_Lm(self) -> bool;
    fn is_Lo(self) -> bool;
    fn is_Nl(self) -> bool;
    fn is_OtherID_Start(self) -> bool;
}

impl UnicodeCharExt for char {
    fn is_L(self) -> bool {
        self.is_Lu() ||
        self.is_Ll() ||
        self.is_Lt() ||
        self.is_Lm() ||
        self.is_Lo()
    }

    fn is_Lu(self) -> bool {
        false
    }

    fn is_Ll(self) -> bool {
        false
    }

    fn is_Lt(self) -> bool {
        false
    }

    fn is_Lm(self) -> bool {
        false
    }

    fn is_Lo(self) -> bool {
        false
    }

    fn is_Nl(self) -> bool {
        false
    }

    fn is_OtherID_Start(self) -> bool {
        false
    }
}
*/

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }

    fn is_es_identifier(self) -> bool {
        self.is_es_identifier_continue()
    }

    fn is_es_identifier_start(self) -> bool {
        self == '$' ||
        self == '_' ||
        self.is_alphabetic()
    }

    fn is_es_identifier_continue(self) -> bool {
        self.is_es_identifier_start() ||
        self.is_numeric()
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }
    pub fn next_char(&mut self) -> Option<char> { self.next_char }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

// test case: x=0;y=g=1;alert(eval("while(x)break\n/y/g.exec('y')"))
//       see: https://groups.google.com/d/msg/mozilla.dev.tech.js-engine.internals/2JLH5jRcr7E/Mxc7ZKc5r6sJ

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn is_empty(&mut self) -> bool {
        self.tokens.len() == 0
    }

    fn push_token(&mut self, token: Token) {
        assert!(self.tokens.len() == 0);
        self.tokens.push_back(token);
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }

    fn peek_token(&mut self) -> &Token {
        assert!(self.tokens.len() > 0);
        self.tokens.front().unwrap()
    }

    fn unread_token(&mut self, token: Token) {
        assert!(self.tokens.len() >= 0);
        assert!(self.tokens.len() < 3);
        self.tokens.push_front(token);
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn peek_token(&mut self) -> &Token {
        if self.lookahead.is_empty() {
            let token = self.read_next_token();
            self.lookahead.push_token(token);
        }
        self.lookahead.peek_token()
    }

    pub fn read_token(&mut self) -> Token {
        if self.lookahead.is_empty() {
            self.read_next_token()
        } else {
            self.lookahead.read_token()
        }
    }

    pub fn unread_token(&mut self, token: Token) {
        self.lookahead.unread_token(token);
    }

    // private methods

    fn bump_until<F: Fn(char) -> bool>(&mut self, pred: F) {
        loop {
            match self.reader.curr_char() {
                Some(ch) if pred(ch) => return,
                None => return,
                _ => ()
            }
            self.bump();
        }
    }

    fn skip_line_comment(&mut self) {
        self.bump();
        self.bump();
        self.bump_until(|ch| ch.is_es_newline());
    }

    fn skip_block_comment(&mut self) -> bool {
        self.bump();
        self.bump();
        self.bump_until(|ch| ch == '*');
        if self.reader.curr_char() == None {
            return false;
        }
        self.bump();
        self.bump();
        true
    }

    fn div_or_regexp(&mut self) -> Token {
        if self.cx.get().is_operator() {
            self.reader.bump();
            Token::Slash
        } else {
            unimplemented!()
        }
    }

    fn if_assign(&mut self, cons: Token, alt: Token) -> Token {
        self.reader.bump();
        if self.reader.curr_char() == Some('=') { self.reader.bump(); cons } else { alt }
    }

    fn if_equality(&mut self, zero: Token, one: Token, two: Token) -> Token {
        self.bump();
        if self.reader.curr_char() == Some('=') {
            self.bump();
            if self.reader.curr_char() == Some('=') {
                self.bump();
                two
            } else {
                one
            }
        } else {
            zero
        }
    }

    fn lt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('<') => {
                self.bump();
                if self.reader.curr_char() == Some('=') {
                    self.bump();
                    Token::LShiftAssign
                } else {
                    Token::LShift
                }
            },
            Some('=') => { self.bump(); Token::LEq }
            _ => Token::LAngle
        }
    }

    fn gt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('>') => {
                self.bump();
                match self.reader.curr_char() {
                    Some('>') => {
                        self.bump();
                        if self.reader.curr_char() == Some('=') {
                            self.bump();
                            Token::URShiftAssign
                        } else {
                            Token::URShift
                        }
                    },
                    Some('=') => { self.bump(); Token::RShiftAssign },
                    _ => Token::RShift
                }
            },
            Some('=') => { self.bump(); Token::GEq },
            _ => Token::RAngle
        }
    }

    fn plus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('+') => { self.bump(); Token::Inc },
            Some('=') => { self.bump(); Token::PlusAssign },
            _ => Token::Plus
        }
    }

    fn minus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('-') => { self.bump(); Token::Dec },
            Some('=') => { self.bump(); Token::MinusAssign },
            _ => Token::Minus
        }
    }

    fn number(&mut self) -> Token {
        if self.reader.curr_char() == Some('1') {
            self.bump();
            return Token::DecimalInt(String::from_str("1"));
        }
        unimplemented!()
    }

    fn string(&mut self) -> Token { unimplemented!() }
    fn word(&mut self) -> Token { unimplemented!() }

    fn read_next_token(&mut self) -> Token {
        self.skip_whitespace();
        println!("inspecting {:?}", self.reader.curr_char());
        loop {
            match self.reader.curr_char() {
                Some('/') => {
                    match self.reader.next_char() {
                        Some('/') => self.skip_line_comment(),
                        Some('*') => if !self.skip_block_comment() { return Token::Error(None) },
                        _ => return self.div_or_regexp()
                    }
                },
                Some('.') => {
                    match self.reader.next_char() {
                        Some(ch) if ch.is_digit(10) => { return self.number() },
                        _ => { self.reader.bump(); return Token::Dot }
                    }
                }
                Some('{') => { self.bump(); return Token::LBrace },
                Some('}') => { self.bump(); return Token::RBrace },
                Some('[') => { self.bump(); return Token::LBrack },
                Some(']') => { self.bump(); return Token::RBrack },
                Some('(') => { self.bump(); return Token::LParen },
                Some(')') => { self.bump(); return Token::RParen },
                Some(';') => { self.bump(); return Token::Semi },
                Some(':') => { self.bump(); return Token::Colon },
                Some(',') => { self.bump(); return Token::Comma },
                Some('<') => return self.lt(),
                Some('>') => return self.gt(),
                Some('=') => return self.if_equality(Token::Assign, Token::Eq, Token::StrictEq),
                Some('+') => return self.plus(),
                Some('-') => return self.minus(),
                Some('*') => return self.if_assign(Token::StarAssign, Token::Star),
                Some('%') => return self.if_assign(Token::ModAssign, Token::Mod),
                Some('^') => return self.if_assign(Token::BitXorAssign, Token::BitXor),
                Some('&') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('&') => { self.bump(); return Token::LogicalAnd },
                        _ => return Token::BitAnd
                    }
                },
                Some('|') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('|') => { self.bump(); return Token::LogicalOr },
                        _ => return Token::BitOr
                    }
                },
                Some('~') => { self.bump(); return Token::Tilde },
                Some('!') => return self.if_equality(Token::Bang, Token::NEq, Token::StrictNEq),
                Some('?') => { self.bump(); return Token::Question },
                Some('"') => return self.string(),
                Some('\'') => return self.string(),
                Some('\r') => {
                    self.bump();
                    if self.reader.curr_char() == Some('\n') {
                        self.bump();
                    }
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                },
                Some(ch) if ch.is_es_newline() => {
                    self.bump();
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                }
                Some(ch) if ch.is_digit(10) => return self.number(),
                Some(ch) if ch.is_es_identifier() => return self.word(),
                Some(ch) => return Token::Error(Some(ch)),
                None => return Token::EOF
            }
        }
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn bump(&mut self) {
        self.reader.bump();
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.read_token() {
            Token::EOF => None,
            t => Some(t)
        }
    }
}
<commit_msg>take_until and string (string_escape stubbed)<commit_after>use std::collections::LinkedList;
use std::borrow::Borrow;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
    fn is_es_identifier(self) -> bool;
    fn is_es_identifier_start(self) -> bool;
    fn is_es_identifier_continue(self) -> bool;
}

/*
// https://github.com/ariya/esprima/blob/master/tools/generate-identifier-regex.js
// https://gist.github.com/mathiasbynens/6334847
// http://unicode.org/reports/tr31/

pub trait UnicodeCharExt {
    fn is_L(self) -> bool;
    fn is_Lu(self) -> bool;
    fn is_Ll(self) -> bool;
    fn is_Lt(self) -> bool;
    fn is_Lm(self) -> bool;
    fn is_Lo(self) -> bool;
    fn is_Nl(self) -> bool;
    fn is_OtherID_Start(self) -> bool;
}

impl UnicodeCharExt for char {
    fn is_L(self) -> bool {
        self.is_Lu() ||
        self.is_Ll() ||
        self.is_Lt() ||
        self.is_Lm() ||
        self.is_Lo()
    }

    fn is_Lu(self) -> bool {
        false
    }

    fn is_Ll(self) -> bool {
        false
    }

    fn is_Lt(self) -> bool {
        false
    }

    fn is_Lm(self) -> bool {
        false
    }

    fn is_Lo(self) -> bool {
        false
    }

    fn is_Nl(self) -> bool {
        false
    }

    fn is_OtherID_Start(self) -> bool {
        false
    }
}
*/

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }

    fn is_es_identifier(self) -> bool {
        self.is_es_identifier_continue()
    }

    fn is_es_identifier_start(self) -> bool {
        self == '$' ||
        self == '_' ||
        self.is_alphabetic()
    }

    fn is_es_identifier_continue(self) -> bool {
        self.is_es_identifier_start() ||
        self.is_numeric()
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }
    pub fn next_char(&mut self) -> Option<char> { self.next_char }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

// test case: x=0;y=g=1;alert(eval("while(x)break\n/y/g.exec('y')"))
//       see: https://groups.google.com/d/msg/mozilla.dev.tech.js-engine.internals/2JLH5jRcr7E/Mxc7ZKc5r6sJ

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn is_empty(&mut self) -> bool {
        self.tokens.len() == 0
    }

    fn push_token(&mut self, token: Token) {
        assert!(self.tokens.len() == 0);
        self.tokens.push_back(token);
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }

    fn peek_token(&mut self) -> &Token {
        assert!(self.tokens.len() > 0);
        self.tokens.front().unwrap()
    }

    fn unread_token(&mut self, token: Token) {
        assert!(self.tokens.len() >= 0);
        assert!(self.tokens.len() < 3);
        self.tokens.push_front(token);
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn peek_token(&mut self) -> &Token {
        if self.lookahead.is_empty() {
            let token = self.read_next_token();
            self.lookahead.push_token(token);
        }
        self.lookahead.peek_token()
    }

    pub fn read_token(&mut self) -> Token {
        if self.lookahead.is_empty() {
            self.read_next_token()
        } else {
            self.lookahead.read_token()
        }
    }

    pub fn unread_token(&mut self, token: Token) {
        self.lookahead.unread_token(token);
    }

    // private methods

    fn bump_until<F: Fn(char) -> bool>(&mut self, pred: F) {
        loop {
            match self.reader.curr_char() {
                Some(ch) if pred(ch) => return,
                None => return,
                _ => ()
            }
            self.bump();
        }
    }

    fn take_until<F: Fn(char) -> bool>(&mut self, s: &mut String, pred: F) {
        loop {
            match self.reader.curr_char() {
                Some(ch) if pred(ch) => return,
                Some(ch) => { s.push(ch); }
                None => return,
            }
        }
    }

    fn skip_line_comment(&mut self) {
        self.bump();
        self.bump();
        self.bump_until(|ch| ch.is_es_newline());
    }

    fn skip_block_comment(&mut self) -> bool {
        self.bump();
        self.bump();
        self.bump_until(|ch| ch == '*');
        if self.reader.curr_char() == None {
            return false;
        }
        self.bump();
        self.bump();
        true
    }

    fn div_or_regexp(&mut self) -> Token {
        if self.cx.get().is_operator() {
            self.reader.bump();
            Token::Slash
        } else {
            unimplemented!()
        }
    }

    fn if_assign(&mut self, cons: Token, alt: Token) -> Token {
        self.reader.bump();
        if self.reader.curr_char() == Some('=') { self.reader.bump(); cons } else { alt }
    }

    fn if_equality(&mut self, zero: Token, one: Token, two: Token) -> Token {
        self.bump();
        if self.reader.curr_char() == Some('=') {
            self.bump();
            if self.reader.curr_char() == Some('=') {
                self.bump();
                two
            } else {
                one
            }
        } else {
            zero
        }
    }

    fn lt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('<') => {
                self.bump();
                if self.reader.curr_char() == Some('=') {
                    self.bump();
                    Token::LShiftAssign
                } else {
                    Token::LShift
                }
            },
            Some('=') => { self.bump(); Token::LEq }
            _ => Token::LAngle
        }
    }

    fn gt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('>') => {
                self.bump();
                match self.reader.curr_char() {
                    Some('>') => {
                        self.bump();
                        if self.reader.curr_char() == Some('=') {
                            self.bump();
                            Token::URShiftAssign
                        } else {
                            Token::URShift
                        }
                    },
                    Some('=') => { self.bump(); Token::RShiftAssign },
                    _ => Token::RShift
                }
            },
            Some('=') => { self.bump(); Token::GEq },
            _ => Token::RAngle
        }
    }

    fn plus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('+') => { self.bump(); Token::Inc },
            Some('=') => { self.bump(); Token::PlusAssign },
            _ => Token::Plus
        }
    }

    fn minus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('-') => { self.bump(); Token::Dec },
            Some('=') => { self.bump(); Token::MinusAssign },
            _ => Token::Minus
        }
    }

    fn number(&mut self) -> Token {
        if self.reader.curr_char() == Some('1') {
            self.bump();
            return Token::DecimalInt(String::from_str("1"));
        }
        unimplemented!()
    }

    fn string(&mut self) -> Token {
        let mut s = String::new();
        loop {
            assert!(self.reader.curr_char().is_some());
            let quote = self.eat().unwrap();
            self.take_until(&mut s, |ch| ch == quote || ch == '\\');
            match self.reader.curr_char() {
                Some('\\') => self.string_escape(&mut s),
                Some(_) => { self.bump(); }
                None => return Token::Error(None)
            }
        }
        Token::String(s)
    }

    fn string_escape(&mut self, s: &mut String) {
        self.bump();
        unimplemented!()
    }

    fn word(&mut self) -> Token { unimplemented!() }

    fn read_next_token(&mut self) -> Token {
        self.skip_whitespace();
        println!("inspecting {:?}", self.reader.curr_char());
        loop {
            match self.reader.curr_char() {
                Some('/') => {
                    match self.reader.next_char() {
                        Some('/') => self.skip_line_comment(),
                        Some('*') => if !self.skip_block_comment() { return Token::Error(None) },
                        _ => return self.div_or_regexp()
                    }
                },
                Some('.') => {
                    match self.reader.next_char() {
                        Some(ch) if ch.is_digit(10) => { return self.number() },
                        _ => { self.reader.bump(); return Token::Dot }
                    }
                }
                Some('{') => { self.bump(); return Token::LBrace },
                Some('}') => { self.bump(); return Token::RBrace },
                Some('[') => { self.bump(); return Token::LBrack },
                Some(']') => { self.bump(); return Token::RBrack },
                Some('(') => { self.bump(); return Token::LParen },
                Some(')') => { self.bump(); return Token::RParen },
                Some(';') => { self.bump(); return Token::Semi },
                Some(':') => { self.bump(); return Token::Colon },
                Some(',') => { self.bump(); return Token::Comma },
                Some('<') => return self.lt(),
                Some('>') => return self.gt(),
                Some('=') => return self.if_equality(Token::Assign, Token::Eq, Token::StrictEq),
                Some('+') => return self.plus(),
                Some('-') => return self.minus(),
                Some('*') => return self.if_assign(Token::StarAssign, Token::Star),
                Some('%') => return self.if_assign(Token::ModAssign, Token::Mod),
                Some('^') => return self.if_assign(Token::BitXorAssign, Token::BitXor),
                Some('&') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('&') => { self.bump(); return Token::LogicalAnd },
                        _ => return Token::BitAnd
                    }
                },
                Some('|') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('|') => { self.bump(); return Token::LogicalOr },
                        _ => return Token::BitOr
                    }
                },
                Some('~') => { self.bump(); return Token::Tilde },
                Some('!') => return self.if_equality(Token::Bang, Token::NEq, Token::StrictNEq),
                Some('?') => { self.bump(); return Token::Question },
                Some('"') => return self.string(),
                Some('\'') => return self.string(),
                Some('\r') => {
                    self.bump();
                    if self.reader.curr_char() == Some('\n') {
                        self.bump();
                    }
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                },
                Some(ch) if ch.is_es_newline() => {
                    self.bump();
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                }
                Some(ch) if ch.is_digit(10) => return self.number(),
                Some(ch) if ch.is_es_identifier() => return self.word(),
                Some(ch) => return Token::Error(Some(ch)),
                None => return Token::EOF
            }
        }
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn bump(&mut self) {
        self.reader.bump();
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.read_token() {
            Token::EOF => None,
            t => Some(t)
        }
    }
}
<|endoftext|>


<commit_before>use std::str::FromStr;
use common::Atom;

#[derive(Debug, PartialEq)]
enum LexerErrorCode {
    InvalidSyntax,
    TrailingCharacters,
    EOFWhileReadingToken,
}

#[derive(Debug, PartialEq)]
enum LexerError {
    SyntaxError {
        code: LexerErrorCode,
        line: usize,
        col: usize,
    }
}

#[derive(Debug, PartialEq)]
enum ParseAtomError {
    IncorrectSymbolName
}

impl FromStr for Atom {
    type Err = ParseAtomError;

    fn from_str(s: &str) -> Result<Atom, ParseAtomError> {
        match s.parse::<f64>() {
            Ok(f) => {
                Ok(Atom::Number(f))
            },
            _ => {
                match s.chars().next() {
                    Some(c) if !c.is_numeric() => {
                        Ok(Atom::Symbol(s.to_string()))
                    },
                    _ => {
                        Err(ParseAtomError::IncorrectSymbolName)
                    }
                }
            }
        }
    }
}

#[derive(Debug, PartialEq)]
enum Token {
    Number(f64),
    String(String),
    Symbol(String),
    Error(LexerError),
}

enum LexerState {
    Start,
    BeforeFinish,
    Finish,
}

pub struct Lexer<T> {
    reader: T,
    cur_char: Option<char>,
    line: usize,
    col: usize,
    state: LexerState
}

impl<T: Iterator<Item=char>> Iterator for Lexer<T> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.state {
            LexerState::Finish => {
                None
            },
            LexerState::BeforeFinish => {
                self.read_whitespaces();
                self.cur_char
                    .and(Some(self.error_token(LexerErrorCode::TrailingCharacters)))
                    .or_else(|| { self.state = LexerState::Finish; None })
            },
            _ => {
                Some(self.read())
            }
        }
    }
}

impl<T: Iterator<Item=char>> Lexer<T> {
    fn new(reader: T) -> Lexer<T> {
        let mut l = Lexer {
            reader: reader,
            cur_char: None,
            line: 1,
            col: 0,
            state: LexerState::Start
        };
        l.bump();
        l
    }

    fn read(&mut self) -> Token {
        loop {
            self.read_whitespaces();
            match self.state {
                LexerState::Start => {
                    return self.read_start()
                },
                _ => {
                    return self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        }
    }

    fn read_start(&mut self) -> Token {
        let token = self.read_token();
        self.state = match token {
            Token::Error(_) => {
                LexerState::Finish
            },
            _ => LexerState::BeforeFinish,
        };
        token
    }

    fn read_token(&mut self) -> Token {
        if let Some(c) = self.cur_char {
            match c {
                'a' ... 'z' | 'A' ... 'Z' => {
                    self.read_symbol()
                },
                '0' ... '9' | '-' => {
                    self.read_number()
                },
                '"' => {
                    self.read_string()
                },
                _ => {
                    self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        } else {
            self.error_token(LexerErrorCode::EOFWhileReadingToken)
        }
    }

    fn read_symbol(&mut self) -> Token {
        let mut res = String::new();

        while let Some(c) = self.cur_char {
            match c {
                ' ' | '\t' | '\r' | '\n' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        Token::Symbol(res)
    }

    fn read_number(&mut self) -> Token {
        let mut neg = false;

        if let Some('-') = self.cur_char {
            neg = true;
            self.bump();
        }

        let mut accum = 0_f64;

        while let Some(c) = self.cur_char {
            match c {
                '0' ... '9' => {
                    let c_as_usize = (c as usize) - ('0' as usize);
                    accum *= 10_f64;
                    accum += c_as_usize as f64;
                    self.bump()
                },
                '.' => {
                    self.bump();

                    let mut dec = 1.0;
                    while let Some(c) = self.cur_char {
                        match c {
                            '0' ... '9' => {
                                let c_as_usize = (c as usize) - ('0' as usize);
                                dec /= 10.0;
                                accum += (c_as_usize as f64) * dec;
                                self.bump();
                            },
                            _ => {
                                break
                            }
                        }
                    }
                }
                _ => {
                    break
                }
            }
        }

        if neg {
            accum *= -1_f64;
        }

        Token::Number(accum)
    }

    fn read_string(&mut self) -> Token {
        let mut res = String::new();

        self.bump();

        while let Some(c) = self.cur_char {
            match c {
                '"' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        self.bump();

        Token::String(res)
    }

    fn read_whitespaces(&mut self) {
        while let Some(ch) = self.cur_char {
            if [' ', '\n', '\r', '\t'].contains(&ch) {
                self.bump()
            } else {
                break
            }
        }
    }

    fn bump(&mut self) {
        self.cur_char = self.reader.next();
        if Some('\n') == self.cur_char {
            self.line += 1;
            self.col = 1;
        } else {
            self.col += 1;
        }
    }

    fn error_token(&mut self, ec: LexerErrorCode) -> Token {
        self.state = LexerState::Finish;
        Token::Error(LexerError::SyntaxError {
            code: ec,
            line: self.line,
            col: self.col
        })
    }
}

fn tokenize(s: &str) -> Vec<String> {
    s.replace("("," ( ")
        .replace(")", " ) ")
        .replace("\n", " ")
        .split(' ')
        .filter(|s| { !s.is_empty() })
        .map(|s| { s.to_string() })
        .collect()
}

#[cfg(test)]
mod tests {
    use common::Atom::{self, Symbol, Number};
    use super::{Lexer, Token};
    use super::ParseAtomError::IncorrectSymbolName;
    use super::tokenize;

    #[test]
    fn test_read_nil() {
        let sym_name = "nil".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        assert_eq!(Some(Token::Symbol(sym_name.clone())), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_integer_as_float() {
        let mut lexer = Lexer::new("64".chars());
        assert_eq!(Some(Token::Number(64_f64)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_float() {
        let mut lexer = Lexer::new("64.5".chars());
        assert_eq!(Some(Token::Number(64.5)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_string() {
        let s = "rust is beautiful".to_string();
        let actual_input = format!(r#""{}""#, s);
        let mut lexer = Lexer::new(actual_input.chars());
        assert_eq!(Some(Token::String(s)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_symbol() {
        let sym_name = "my-symbol".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        assert_eq!(Some(Token::Symbol(sym_name.clone())), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_incorrect_symbol_starting_with_digit() {
        assert_eq!(IncorrectSymbolName, "6name".parse::<Atom>().err().unwrap())
    }

    #[test]
    fn test_read_dense_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize("(def a 1)"))
    }

    #[test]
    fn test_read_sparse_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize(" ( \n def a\n1)   \n"))
    }
}
<commit_msg>Add negative test case for symbol starting with digit<commit_after>use std::str::FromStr;
use common::Atom;

#[derive(Debug, PartialEq)]
enum LexerErrorCode {
    InvalidSyntax,
    TrailingCharacters,
    EOFWhileReadingToken,
}

#[derive(Debug, PartialEq)]
enum LexerError {
    SyntaxError {
        code: LexerErrorCode,
        line: usize,
        col: usize,
    }
}

#[derive(Debug, PartialEq)]
enum ParseAtomError {
    IncorrectSymbolName
}

impl FromStr for Atom {
    type Err = ParseAtomError;

    fn from_str(s: &str) -> Result<Atom, ParseAtomError> {
        match s.parse::<f64>() {
            Ok(f) => {
                Ok(Atom::Number(f))
            },
            _ => {
                match s.chars().next() {
                    Some(c) if !c.is_numeric() => {
                        Ok(Atom::Symbol(s.to_string()))
                    },
                    _ => {
                        Err(ParseAtomError::IncorrectSymbolName)
                    }
                }
            }
        }
    }
}

#[derive(Debug, PartialEq)]
enum Token {
    Number(f64),
    String(String),
    Symbol(String),
    Error(LexerError),
}

enum LexerState {
    Start,
    BeforeFinish,
    Finish,
}

pub struct Lexer<T> {
    reader: T,
    cur_char: Option<char>,
    line: usize,
    col: usize,
    state: LexerState
}

impl<T: Iterator<Item=char>> Iterator for Lexer<T> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.state {
            LexerState::Finish => {
                None
            },
            LexerState::BeforeFinish => {
                self.read_whitespaces();
                self.cur_char
                    .and(Some(self.error_token(LexerErrorCode::TrailingCharacters)))
                    .or_else(|| { self.state = LexerState::Finish; None })
            },
            _ => {
                Some(self.read())
            }
        }
    }
}

impl<T: Iterator<Item=char>> Lexer<T> {
    fn new(reader: T) -> Lexer<T> {
        let mut l = Lexer {
            reader: reader,
            cur_char: None,
            line: 1,
            col: 0,
            state: LexerState::Start
        };
        l.bump();
        l
    }

    fn read(&mut self) -> Token {
        loop {
            self.read_whitespaces();
            match self.state {
                LexerState::Start => {
                    return self.read_start()
                },
                _ => {
                    return self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        }
    }

    fn read_start(&mut self) -> Token {
        let token = self.read_token();
        self.state = match token {
            Token::Error(_) => {
                LexerState::Finish
            },
            _ => LexerState::BeforeFinish,
        };
        token
    }

    fn read_token(&mut self) -> Token {
        if let Some(c) = self.cur_char {
            match c {
                'a' ... 'z' | 'A' ... 'Z' => {
                    self.read_symbol()
                },
                '0' ... '9' | '-' => {
                    self.read_number()
                },
                '"' => {
                    self.read_string()
                },
                _ => {
                    self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        } else {
            self.error_token(LexerErrorCode::EOFWhileReadingToken)
        }
    }

    fn read_symbol(&mut self) -> Token {
        let mut res = String::new();

        while let Some(c) = self.cur_char {
            match c {
                ' ' | '\t' | '\r' | '\n' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        Token::Symbol(res)
    }

    fn read_number(&mut self) -> Token {
        let mut neg = false;

        if let Some('-') = self.cur_char {
            neg = true;
            self.bump();
        }

        let mut accum = 0_f64;

        while let Some(c) = self.cur_char {
            match c {
                '0' ... '9' => {
                    let c_as_usize = (c as usize) - ('0' as usize);
                    accum *= 10_f64;
                    accum += c_as_usize as f64;
                    self.bump()
                },
                '.' => {
                    self.bump();

                    let mut dec = 1.0;
                    while let Some(c) = self.cur_char {
                        match c {
                            '0' ... '9' => {
                                let c_as_usize = (c as usize) - ('0' as usize);
                                dec /= 10.0;
                                accum += (c_as_usize as f64) * dec;
                                self.bump();
                            },
                            _ => {
                                break
                            }
                        }
                    }
                },
                ' ' | '\t' | '\r' | '\n' => {
                    break
                },
                _ => {
                    return self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        }

        if neg {
            accum *= -1_f64;
        }

        Token::Number(accum)
    }

    fn read_string(&mut self) -> Token {
        let mut res = String::new();

        self.bump();

        while let Some(c) = self.cur_char {
            match c {
                '"' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        self.bump();

        Token::String(res)
    }

    fn read_whitespaces(&mut self) {
        while let Some(c) = self.cur_char {
            match c {
                ' ' | '\t' | '\r' | '\n' => {
                    self.bump();
                },
                _ => {
                    break;
                }
            }
        }
    }

    fn bump(&mut self) {
        self.cur_char = self.reader.next();
        if Some('\n') == self.cur_char {
            self.line += 1;
            self.col = 1;
        } else {
            self.col += 1;
        }
    }

    fn error_token(&mut self, ec: LexerErrorCode) -> Token {
        self.state = LexerState::Finish;
        Token::Error(LexerError::SyntaxError {
            code: ec,
            line: self.line,
            col: self.col
        })
    }
}

fn tokenize(s: &str) -> Vec<String> {
    s.replace("("," ( ")
        .replace(")", " ) ")
        .replace("\n", " ")
        .split(' ')
        .filter(|s| { !s.is_empty() })
        .map(|s| { s.to_string() })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::{Lexer, Token, LexerError, LexerErrorCode};
    use super::tokenize;

    #[test]
    fn test_read_nil() {
        let sym_name = "nil".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        assert_eq!(Some(Token::Symbol(sym_name.clone())), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_integer_as_float() {
        let mut lexer = Lexer::new("64".chars());
        assert_eq!(Some(Token::Number(64_f64)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_float() {
        let mut lexer = Lexer::new("64.5".chars());
        assert_eq!(Some(Token::Number(64.5)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_string() {
        let s = "rust is beautiful".to_string();
        let actual_input = format!(r#""{}""#, s);
        let mut lexer = Lexer::new(actual_input.chars());
        assert_eq!(Some(Token::String(s)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_symbol() {
        let sym_name = "my-symbol".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        assert_eq!(Some(Token::Symbol(sym_name.clone())), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_incorrect_symbol_starting_with_digit() {
        let sym_name = "6-my-incorrect-symbol".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        let expected_result = Some(Token::Error(LexerError::SyntaxError {
            code: LexerErrorCode::InvalidSyntax,
            line: 1,
            col: 2
        }));
        assert_eq!(expected_result, lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_dense_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize("(def a 1)"))
    }

    #[test]
    fn test_read_sparse_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize(" ( \n def a\n1)   \n"))
    }
}
<|endoftext|>