<commit_before>use std::iter::Iterator;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }
}

impl Iterator for Tokenizer {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn parenOpen() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn parenClose() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
    }

    #[test]
    fn ignores_leading_whitespace() {
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<commit_msg>Implemented a couple more of the tests<commit_after>use std::iter::Iterator;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }
}

impl Iterator for Tokenizer {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn parenOpen() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn parenClose() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!".to_string();
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=";

        // Make sure leading whitespace does not effect the output
        let tokens = tokenizer_for("  " + symbol);

        assert!(tokenizer.next().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
    }

    #[test]
    fn ignores_leading_whitespace() {
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<|endoftext|>


<commit_before>use std::io::{TcpListener, TcpStream};
use std::io::{Acceptor, Listener};
use std::string::String;

pub struct Server
{
    port: String,
    ip: String
}

pub trait ServerFunction
{
    fn new(new_port: String, new_ip: String) -> Server;
    fn start_server(&self);
    fn format_ip(&self) -> String;
    fn get_port(&self) -> String;
    fn get_ip(&self) -> String;
}

impl ServerFunction for Server
{
    fn new(new_port: String, new_ip: String) -> Server
    {
        return Server{port: new_port, ip: new_ip};
    }

    fn start_server(&self)
    {
        let ip = self.format_ip();
        let listener = TcpListener::bind(ip.as_slice());
        let mut acceptor = listener.listen();
    }

    fn format_ip(&self) -> String
    {
        let mut complete_ip = String::new();
        complete_ip.push_str(self.ip.as_slice());
        complete_ip.push_str(":");
        complete_ip.push_str(self.port.as_slice());
        return complete_ip;
    }

    fn get_port(&self) -> String
    {
        let mut port = String::new();
        port.push_str(self.port.as_slice());
        return port;
    }

    fn get_ip(&self) -> String
    {
        let mut ip = String::new();
        ip.push_str(self.ip.as_slice());
        return ip;
    }
}
<commit_msg>Removed unnecessary things<commit_after>use std::io::{TcpListener, TcpStream};
use std::io::{Acceptor, Listener};
use std::string::String;

pub struct Server
{
    port: String,
    ip: String
}

pub trait ServerFunction
{
    fn new(new_port: String) -> Server;
    fn start_server(&self);
    fn format_ip(&self) -> String;
    fn get_port(&self) -> String;
    fn get_ip(&self) -> String;
}

impl ServerFunction for Server
{
    fn new(new_port: String) -> Server
    {
        return Server{port: new_port, ip: String::from_str("localhost")};
    }

    fn start_server(&self)
    {
        let ip = self.format_ip();
        let listener = TcpListener::bind(ip.as_slice());
        let mut acceptor = listener.listen();
    }

    fn format_ip(&self) -> String
    {
        let mut complete_ip = String::new();
        complete_ip.push_str(self.ip.as_slice());
        complete_ip.push_str(":");
        complete_ip.push_str(self.port.as_slice());
        return complete_ip;
    }

    fn get_port(&self) -> String
    {
        let mut port = String::new();
        port.push_str(self.port.as_slice());
        return port;
    }

    fn get_ip(&self) -> String
    {
        let mut ip = String::new();
        ip.push_str(self.ip.as_slice());
        return ip;
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 by Stacy Prowell.  All rights reserved.
//
// Licensed under the BSD 2-Clause license.  See the file LICENSE
// that is part of this distribution.  This file may not be copied,
// modified, or distributed except according to those terms.extern

//! Provide basic definitions of terms.

/// Represents root terms.
struct ERoot ( pub () );
struct EAny ( pub () );
struct ENone ( pub () );
#[derive(Debug)]
struct EString(pub String);
struct Symbol(pub String);
#[derive(Debug)]
struct Integer(pub isize);
struct Float(pub f64);
struct BitString(pub Vec<i8>);
struct Boolean(pub bool);

#[derive(Debug)]
struct Term<T> {
    value: T,
}

impl<T> Term<T> {
    pub fn new(t: T) -> Term<T> {
        Term { value: t }
    }
}

#[test]
fn term_type_check_test() -> (){
    let stringterm = Term::new(EString("test".to_string()));
    let nestedterm = Term::new(Term::new(Integer(3)));

    //let rootkind = rootterm.get_kind();
    //println!("{:?}", rootkind);
    panic!("Intential fail. {:?}", nestedterm);
}

<commit_msg>Added a term access example. The acessor is kind of kludgy.<commit_after>// Copyright (c) 2015 by Stacy Prowell.  All rights reserved.
//
// Licensed under the BSD 2-Clause license.  See the file LICENSE
// that is part of this distribution.  This file may not be copied,
// modified, or distributed except according to those terms.extern

//! Provide basic definitions of terms.

use std::collections::BitVec;
//use std::tuple::Tuple1;

/// Represents root terms.
struct Root ( () );
struct Any ( () );
struct ENone ( () );
#[derive(Debug)]
struct EString( String ) ;
struct Symbol( String );
#[derive(Debug)]
struct Integer( isize );
struct Float( f64 );
struct BitString( BitVec );
struct Boolean( bool );

#[derive(Debug)]
struct Term<T> {
    value: T,
}

impl<T> Term<T>
//where T: Tuple1
{
    pub fn new(t: T) -> Term<T> {
        Term { value: t }
    }
    pub fn unwrap(&self) -> &T {
        &self.value
    }
}

#[test]
fn term_type_check_test() -> (){
    let stringterm = Term::new(EString("test".to_string()));
    let nestedterm = Term::new(Term::new(Integer(3)));
    //how to make this access not require .0?
    let stringval = &stringterm.unwrap().0;
    panic!("Intentionial panic. {:?}", stringval);
}

<|endoftext|>


<commit_before>// Copyright (c) 2019, Ben Boeckel
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//       this list of conditions and the following disclaimer in the documentation
//       and/or other materials provided with the distribution.
//     * Neither the name of this project nor the names of its contributors
//       may be used to endorse or promote products derived from this software
//       without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

use std::iter;

use crate::keytypes::User;
use crate::{Keyring, SpecialKeyring};

use super::utils;
use super::utils::kernel::*;
use super::utils::keys::*;

#[test]
fn empty_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<EmptyKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn unsupported_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<UnsupportedKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::ENODEV));
}

#[test]
fn invalid_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<InvalidKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::EPERM));
}

#[test]
fn maxlen_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<MaxLenKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::ENODEV));
}

#[test]
fn overlong_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<OverlongKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn keyring_with_payload() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring
        .add_key::<KeyringShadow, _, _>("", "payload")
        .unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn max_user_description() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    // Subtract one because the NUL is added in the kernel API.
    let maxdesc: String = iter::repeat('a').take(*PAGE_SIZE - 1).collect();
    let res = keyring.add_key::<User, _, _>(maxdesc.as_ref(), "payload".as_bytes());
    // If the user's quota is smaller than this, it's an error.
    if KEY_INFO.maxbytes < *PAGE_SIZE {
        assert_eq!(res.unwrap_err(), errno::Errno(libc::EDQUOT));
    } else {
        let key = res.unwrap();
        assert_eq!(key.description().unwrap().description, maxdesc);
        key.invalidate().unwrap();
    }
}

#[test]
fn overlong_user_description() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    // On MIPS with < 3.19, there is a bug where this is allowed. 3.19 was released in Feb 2015,
    // so this is being ignored here.
    let toolarge: String = iter::repeat('a').take(*PAGE_SIZE).collect();
    let err = keyring
        .add_key::<User, _, _>(toolarge.as_ref(), "payload".as_bytes())
        .unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn invalid_keyring() {
    let mut keyring = utils::invalid_keyring();
    let err = keyring
        .add_key::<User, _, _>("desc", "payload".as_bytes())
        .unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn add_key_to_session() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Session).unwrap();
    let expected = "stuff".as_bytes();
    let mut key = keyring.add_key::<User, _, _>("wibble", expected).unwrap();
    let payload = key.read().unwrap();
    assert_eq!(payload, expected);
    let new_expected = "lizard".as_bytes();
    key.update(new_expected).unwrap();
    let new_payload = key.read().unwrap();
    assert_eq!(new_payload, new_expected);
    keyring.unlink_key(&key).unwrap();
}
<commit_msg>tests: use test names as key descriptions<commit_after>// Copyright (c) 2019, Ben Boeckel
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//       this list of conditions and the following disclaimer in the documentation
//       and/or other materials provided with the distribution.
//     * Neither the name of this project nor the names of its contributors
//       may be used to endorse or promote products derived from this software
//       without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

use std::iter;

use crate::keytypes::User;
use crate::{Keyring, SpecialKeyring};

use super::utils;
use super::utils::kernel::*;
use super::utils::keys::*;

#[test]
fn empty_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<EmptyKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn unsupported_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<UnsupportedKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::ENODEV));
}

#[test]
fn invalid_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<InvalidKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::EPERM));
}

#[test]
fn maxlen_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<MaxLenKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::ENODEV));
}

#[test]
fn overlong_key_type() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring.add_key::<OverlongKey, _, _>("", ()).unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn keyring_with_payload() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    let err = keyring
        .add_key::<KeyringShadow, _, _>("", "payload")
        .unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn max_user_description() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    // Subtract one because the NUL is added in the kernel API.
    let maxdesc: String = iter::repeat('a').take(*PAGE_SIZE - 1).collect();
    let res = keyring.add_key::<User, _, _>(maxdesc.as_ref(), "payload".as_bytes());
    // If the user's quota is smaller than this, it's an error.
    if KEY_INFO.maxbytes < *PAGE_SIZE {
        assert_eq!(res.unwrap_err(), errno::Errno(libc::EDQUOT));
    } else {
        let key = res.unwrap();
        assert_eq!(key.description().unwrap().description, maxdesc);
        key.invalidate().unwrap();
    }
}

#[test]
fn overlong_user_description() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Process).unwrap();
    // On MIPS with < 3.19, there is a bug where this is allowed. 3.19 was released in Feb 2015,
    // so this is being ignored here.
    let toolarge: String = iter::repeat('a').take(*PAGE_SIZE).collect();
    let err = keyring
        .add_key::<User, _, _>(toolarge.as_ref(), "payload".as_bytes())
        .unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn invalid_keyring() {
    let mut keyring = utils::invalid_keyring();
    let err = keyring
        .add_key::<User, _, _>("invalid_keyring", "payload".as_bytes())
        .unwrap_err();
    assert_eq!(err, errno::Errno(libc::EINVAL));
}

#[test]
fn add_key_to_session() {
    let mut keyring = Keyring::attach_or_create(SpecialKeyring::Session).unwrap();
    let expected = "stuff".as_bytes();
    let mut key = keyring
        .add_key::<User, _, _>("add_key_to_session", expected)
        .unwrap();
    let payload = key.read().unwrap();
    assert_eq!(payload, expected);
    let new_expected = "lizard".as_bytes();
    key.update(new_expected).unwrap();
    let new_payload = key.read().unwrap();
    assert_eq!(new_payload, new_expected);
    keyring.unlink_key(&key).unwrap();
}
<|endoftext|>


<commit_before>#![feature(slice_concat_ext)]
#![feature(integer_atomics)]

extern crate imageflow_helpers;
extern crate imageflow_core as fc;
extern crate imageflow_types as s;
use ::imageflow_helpers::preludes::from_std::*;
extern crate hyper;

use std::slice::SliceConcatExt;

extern crate wait_timeout;
use wait_timeout::ChildExt;
use std::time::Duration;
use std::process::{Command, Stdio, Child,Output};

use ::imageflow_helpers::process_testing::*;
use fc::test_helpers::*;
use fc::test_helpers::process_testing::ProcTestContextExtras;
use fc::test_helpers::process_testing::ProcOutputExtras;
use ::imageflow_helpers::fetching::FetchError;
use ::imageflow_helpers::fetching::{fetch_bytes,get_status_code_for};
fn server_path() -> PathBuf{
    let self_path = std::env::current_exe().expect("For --self-test to work, we need to know the binary's location. env::current_exe failed");
    self_path.parent().unwrap().join("imageflow_server")
}

fn get_next_port() -> u16{
    use std::sync::atomic::{AtomicU16, Ordering, ATOMIC_U16_INIT};
    static NEXT_PORT: AtomicU16 = ATOMIC_U16_INIT;

    NEXT_PORT.compare_and_swap(0, 36203, Ordering::SeqCst);
    NEXT_PORT.fetch_add(1, Ordering::SeqCst)
}

struct ServerInstance{
    port: u16,
}

type CallbackResult = std::result::Result<(), ::imageflow_helpers::fetching::FetchError>;
impl ServerInstance{

    fn speaks_http(&self) -> bool{
        self.hello().is_ok()
    }

    fn hello(&self) -> std::result::Result<hyper::status::StatusCode, FetchError> {
        get_status_code_for(&self.url_for("/hello/are/you/running?"))
    }

    fn url_for(&self, rel_path: &str) -> String{
        format!("http://localhost:{}{}", self.port,rel_path)
    }

    fn get_status(&self, rel_path: &str) -> std::result::Result<hyper::status::StatusCode, FetchError> {
        get_status_code_for(&self.url_for(rel_path))
    }

    fn request_stop(&self) -> std::result::Result<hyper::status::StatusCode, FetchError> {
        get_status_code_for(&self.url_for("/test/shutdown"))
    }

    fn run<F>(c: &ProcTestContext, args: Vec<&str>, callback: F) -> (ProcOutput, CallbackResult)
    where F: Fn(&ServerInstance) -> CallbackResult {
        let instance = ServerInstance {
            port: get_next_port()
        };
        // NOTE --bind=localhost::{} (two colons) causes a generic "error:",exit code 1, and no other output. This is bad UX.
        let test_arg = "--integration-test";
        let port_arg = format!("--port={}", instance.port);
        let mut all_args = args.clone();
        all_args.insert(0, test_arg);
        all_args.insert(0, &port_arg);
        all_args.insert(0, "start");

        c.execute_callback(all_args, false,
                           |child: &mut std::process::Child| -> std::result::Result<(), ::imageflow_helpers::fetching::FetchError> {

                               ::std::thread::sleep_ms(500);
                               // Server may not be running
                               instance.hello()?;

                               let r = callback(&instance);

                               let _ = instance.request_stop();
                               r
                           })
        //po.expect_status_code(Some(0));
    }
}

// ports 36,000 to 39,999 seem the safest.
#[test]
fn run_server_test_i4(){

    let context = ProcTestContext::create_timestamp_subdir_within("server_tests", Some(server_path()));

    {
        let c = context.subfolder_context("basics");
        c.exec("diagnose --show-compilation-info").expect_status_code(Some(0));
        c.exec("--version").expect_status_code(Some(0));
        c.exec("-V").expect_status_code(Some(0));

        //TODO: test diagnose --call-panic (xplat hard)

        //Test incorrect args
        c.execute(vec!["demo"], false, |child: &mut std::process::Child| {
        }).expect_status_code(Some(1));

    }

    {
        let c = context.subfolder_context("demo"); //stuck on port 39876
        c.subfolder_context("demo");
        let (po, callback_result) = ServerInstance::run(&c, vec!["--demo", "--data-dir=."], | server | {
            fetch_bytes(&server.url_for("/ir4/proxy_unsplash/photo-1422493757035-1e5e03968f95?width=100"))?;
            assert_eq!(server.get_status("/ir4/proxy_unsplash/notthere.jpg")?, hyper::status::StatusCode::NotFound);
            Ok(())
        });

        //po.expect_status_code(Some(0));

        callback_result.unwrap();
    }
    {
        let c = context.subfolder_context("mount_local"); //stuck on port 39876
        c.create_blank_image_here("eh", 100,100, s::EncoderPreset::libpng32());
        let a = c.subfolder_context("a"); //stuck on port 39876
        a.create_blank_image_here("eh2", 100,100, s::EncoderPreset::libpng32());

        let mut params = vec!["--data-dir=.", "--mount=/local/:ir4_local:./",
                              "--mount=/local_1/:ir4_local:./a",
                              "--mount=/local_2/:ir4_local:./a/",
                              "--mount=/local_3/:ir4_local:a"];
        if std::path::MAIN_SEPARATOR == '\\'{
            params.push(r"--mount=/local_4/:ir4_local:.\a");
            params.push(r"--mount=/local_5/:ir4_local:.\a/");
            params.push(r"--mount=/local_6/:ir4_local:.\a\");
        }

        let last_mount = params.len() - 2;

        let (po, callback_result) = ServerInstance::run(&c, params , | server | {
            let bytes = fetch_bytes(&server.url_for("/local/eh.png?width=100")).unwrap();

            let info = fc::clients::stateless::LibClient {}.get_image_info(&bytes).expect("Image response should be valid");

            for ix in 1..last_mount + 1{

                let url = format!("/local_{ix}/eh2.png?w=1", ix=ix);
                println!("Testing {}", &url);
                let bytes = fetch_bytes(&server.url_for(&url)).unwrap();

                let info = fc::clients::stateless::LibClient {}.get_image_info(&bytes).expect("Image response should be valid");

                assert_eq!(server.get_status(&url)?, hyper::status::StatusCode::Ok);
            }

            assert_eq!(server.get_status("/local/notthere.jpg")?, hyper::status::StatusCode::NotFound);
            assert_eq!(server.get_status("/notrouted")?, hyper::status::StatusCode::NotFound);
            Ok(())
        });
        //po.expect_status_code(Some(0));

        callback_result.unwrap();
    }
}

trait ProcTestContextHttp{
 fn execute_callback<F,T>(&self, args_vec: Vec<&str>, valgrind_on_signal_death: bool, callback: F) -> (ProcOutput, T)
where F: Fn(&mut std::process::Child) -> T;

}
impl ProcTestContextHttp for ProcTestContext{

    ///
    /// Pass false for valgrind_on_signal_death if your callback might kill the child
    fn execute_callback<F,T>(&self, args_vec: Vec<&str>, valgrind_on_signal_death: bool, callback: F) -> (ProcOutput, T)
        where F: Fn(&mut std::process::Child) -> T {
        //TODO: serialize in a safer way - this isn't correct
        let full_invocation = format!("{} {}", &self.bin_location().to_str().unwrap(), args_vec.join(" "));

        let dir = self.working_dir();
        let exe = self.bin_location();

        let valgrind_copy_result = self.create_valgrind_suppressions();
        let _ = writeln!(&mut std::io::stderr(),
                         "Executing from folder {} with valgrind_suppressions {:?}\n{}",
                         dir.to_str().unwrap(),
                         valgrind_copy_result,
                         full_invocation);
        // change working dir to dir
        let mut cmd = Command::new(exe);
        cmd.args(args_vec.as_slice()).current_dir(dir).env("RUST_BACKTRACE", "1");

        //cmd.stderr(Stdio::piped()).stdout(Stdio::piped());
        cmd.stderr(Stdio::inherit()).stdout(Stdio::inherit());

        let mut child = cmd.spawn().expect("Failed to start?");

        let result = callback(&mut child);

        ///child.kill().unwrap();
        let timeout = Some(Duration::from_secs(1));

        let (status_code, output) = match timeout {
            Some(timeout) => {
                match child.wait_timeout(timeout).unwrap() {
                    Some(status) => (status.code(), None),
                    None => {
                        // child hasn't exited yet
                        child.kill().unwrap();
                        (child.wait().unwrap().code(), None)
                    }
                }
            }
            None => {
                let output: Output = child.wait_with_output().unwrap();
                (output.status.code(), Some(output))
            }
        };

        let _ = writeln!(&mut std::io::stderr(),
                         "exit code {:?}", status_code);

        // Double check we dumped output on segfault
        if status_code == None {
            if let Some(ref out) = output {
                std::io::stderr().write(&out.stderr).unwrap();
                std::io::stdout().write(&out.stdout).unwrap();
            }
            let _ = writeln!(&mut std::io::stderr(),
                             "exit code {:?}", status_code);
        }
        // Killed by signal.
        // 11 Segmentation fault
        // 4 illegal instruction 6 abort 8 floating point error
        if status_code == None && valgrind_on_signal_death {
            if std::env::var("VALGRIND_RUNNING").is_ok() {
                let _ = writeln!(&mut std::io::stderr(),
                                 "VALGRIND_RUNNING defined; skipping valgrind pass");
            } else {
                //ALLOW TO FAIL; valgrind may not be present
                let _ = writeln!(&mut std::io::stderr(),
                                 "Starting valgrind from within self-test:");
                let mut cmd = Command::new("valgrind");
                cmd.arg("-q").arg("--error-exitcode=9").arg(exe);
                cmd.args(args_vec.as_slice()).current_dir(dir).env("RUST_BACKTRACE", "1").env("VALGRIND_RUNNING", "1");

                let _ = writeln!(&mut std::io::stderr(),
                                 "{:?}", cmd);

                let _ = cmd.status(); //.expect("Failed to start valgrind?");
            }
        }

        match output {
            Some(out) => (ProcOutput::from(out), result),
            None => (ProcOutput::from_code(status_code), result)
        }
    }

}<commit_msg>tests: add a heavier server test<commit_after>#![feature(slice_concat_ext)]
#![feature(integer_atomics)]

extern crate imageflow_helpers;
extern crate imageflow_core as fc;
extern crate imageflow_types as s;
use ::imageflow_helpers::preludes::from_std::*;
extern crate hyper;

use std::slice::SliceConcatExt;

extern crate wait_timeout;
use wait_timeout::ChildExt;
use std::time::Duration;
use std::process::{Command, Stdio, Child,Output};

use ::imageflow_helpers::process_testing::*;
use fc::test_helpers::*;
use fc::test_helpers::process_testing::ProcTestContextExtras;
use fc::test_helpers::process_testing::ProcOutputExtras;
use ::imageflow_helpers::fetching::FetchError;
use ::imageflow_helpers::fetching::{fetch_bytes,get_status_code_for};
fn server_path() -> PathBuf{
    let self_path = std::env::current_exe().expect("For --self-test to work, we need to know the binary's location. env::current_exe failed");
    self_path.parent().unwrap().join("imageflow_server")
}

fn get_next_port() -> u16{
    use std::sync::atomic::{AtomicU16, Ordering, ATOMIC_U16_INIT};
    static NEXT_PORT: AtomicU16 = ATOMIC_U16_INIT;

    NEXT_PORT.compare_and_swap(0, 36203, Ordering::SeqCst);
    NEXT_PORT.fetch_add(1, Ordering::SeqCst)
}

struct ServerInstance{
    port: u16,
}

type CallbackResult = std::result::Result<(), ::imageflow_helpers::fetching::FetchError>;
impl ServerInstance{

    fn speaks_http(&self) -> bool{
        self.hello().is_ok()
    }

    fn hello(&self) -> std::result::Result<hyper::status::StatusCode, FetchError> {
        get_status_code_for(&self.url_for("/hello/are/you/running?"))
    }

    fn url_for(&self, rel_path: &str) -> String{
        format!("http://localhost:{}{}", self.port,rel_path)
    }

    fn get_status(&self, rel_path: &str) -> std::result::Result<hyper::status::StatusCode, FetchError> {
        get_status_code_for(&self.url_for(rel_path))
    }

    fn request_stop(&self) -> std::result::Result<hyper::status::StatusCode, FetchError> {
        get_status_code_for(&self.url_for("/test/shutdown"))
    }

    fn run<F>(c: &ProcTestContext, args: Vec<&str>, callback: F) -> (ProcOutput, CallbackResult)
    where F: Fn(&ServerInstance) -> CallbackResult {
        let instance = ServerInstance {
            port: get_next_port()
        };
        // NOTE --bind=localhost::{} (two colons) causes a generic "error:",exit code 1, and no other output. This is bad UX.
        let test_arg = "--integration-test";
        let port_arg = format!("--port={}", instance.port);
        let mut all_args = args.clone();
        all_args.insert(0, test_arg);
        all_args.insert(0, &port_arg);
        all_args.insert(0, "start");

        c.execute_callback(all_args, false,
                           |child: &mut std::process::Child| -> std::result::Result<(), ::imageflow_helpers::fetching::FetchError> {

                               ::std::thread::sleep_ms(500);
                               // Server may not be running
                               instance.hello()?;

                               let r = callback(&instance);

                               let _ = instance.request_stop();
                               r
                           })
        //po.expect_status_code(Some(0));
    }
}

// ports 36,000 to 39,999 seem the safest.
#[test]
fn run_server_test_i4(){

    let context = ProcTestContext::create_timestamp_subdir_within("server_tests", Some(server_path()));

    {
        let c = context.subfolder_context("basics");
        c.exec("diagnose --show-compilation-info").expect_status_code(Some(0));
        c.exec("--version").expect_status_code(Some(0));
        c.exec("-V").expect_status_code(Some(0));

        //TODO: test diagnose --call-panic (xplat hard)

        //Test incorrect args
        c.execute(vec!["demo"], false, |child: &mut std::process::Child| {
        }).expect_status_code(Some(1));

    }

    {
        let c = context.subfolder_context("demo"); //stuck on port 39876
        c.subfolder_context("demo");
        let (po, callback_result) = ServerInstance::run(&c, vec!["--demo", "--data-dir=."], | server | {
            fetch_bytes(&server.url_for("/ir4/proxy_unsplash/photo-1422493757035-1e5e03968f95?width=100"))?;
            assert_eq!(server.get_status("/ir4/proxy_unsplash/notthere.jpg")?, hyper::status::StatusCode::NotFound);
            Ok(())
        });

        //po.expect_status_code(Some(0));

        callback_result.unwrap();
    }
    {
        let c = context.subfolder_context("mount_local"); //stuck on port 39876
        c.create_blank_image_here("eh", 100,100, s::EncoderPreset::libpng32());
        let a = c.subfolder_context("a"); //stuck on port 39876
        a.create_blank_image_here("eh2", 100,100, s::EncoderPreset::libpng32());

        let mut params = vec!["--data-dir=.", "--mount=/local/:ir4_local:./",
                              "--mount=/local_1/:ir4_local:./a",
                              "--mount=/local_2/:ir4_local:./a/",
                              "--mount=/local_3/:ir4_local:a"];
        if std::path::MAIN_SEPARATOR == '\\'{
            params.push(r"--mount=/local_4/:ir4_local:.\a");
            params.push(r"--mount=/local_5/:ir4_local:.\a/");
            params.push(r"--mount=/local_6/:ir4_local:.\a\");
        }

        let last_mount = params.len() - 2;

        let (po, callback_result) = ServerInstance::run(&c, params , | server | {
            let bytes = fetch_bytes(&server.url_for("/local/eh.png?width=100")).unwrap();

            let info = fc::clients::stateless::LibClient {}.get_image_info(&bytes).expect("Image response should be valid");

            for ix in 1..last_mount + 1{

                let url = format!("/local_{ix}/eh2.png?w=1", ix=ix);
                println!("Testing {}", &url);
                let bytes = fetch_bytes(&server.url_for(&url)).unwrap();

                let info = fc::clients::stateless::LibClient {}.get_image_info(&bytes).expect("Image response should be valid");

                assert_eq!(server.get_status(&url)?, hyper::status::StatusCode::Ok);
            }

            assert_eq!(server.get_status("/local/notthere.jpg")?, hyper::status::StatusCode::NotFound);
            assert_eq!(server.get_status("/notrouted")?, hyper::status::StatusCode::NotFound);
            Ok(())
        });
        //po.expect_status_code(Some(0));

        callback_result.unwrap();
    }
}
#[test]
fn run_server_test_ir4_heavy(){
    let context = ProcTestContext::create_timestamp_subdir_within("server_tests_heavy", Some(server_path()));
    {
        let c = context.subfolder_context("mount_local_test"); //stuck on port 39876
        c.exec("diagnose --show-compilation-info").expect_status_code(Some(0));
        c.create_blank_image_here("eh", 100,100, s::EncoderPreset::libpng32());

        let mut params = vec!["--data-dir=.", "--mount=/local/:ir4_local:./"];
        let (po, callback_result) = ServerInstance::run(&c, params , | server | {
            for ix in 1..20{
                let bytes = fetch_bytes(&server.url_for("/local/eh.png?width=100")).unwrap();
                let info = fc::clients::stateless::LibClient {}.get_image_info(&bytes).expect("Image response should be valid");
            }
            Ok(())
        });
        callback_result.unwrap();
    }
}

trait ProcTestContextHttp{
 fn execute_callback<F,T>(&self, args_vec: Vec<&str>, valgrind_on_signal_death: bool, callback: F) -> (ProcOutput, T)
where F: Fn(&mut std::process::Child) -> T;

}
impl ProcTestContextHttp for ProcTestContext{

    ///
    /// Pass false for valgrind_on_signal_death if your callback might kill the child
    fn execute_callback<F,T>(&self, args_vec: Vec<&str>, valgrind_on_signal_death: bool, callback: F) -> (ProcOutput, T)
        where F: Fn(&mut std::process::Child) -> T {
        //TODO: serialize in a safer way - this isn't correct
        let full_invocation = format!("{} {}", &self.bin_location().to_str().unwrap(), args_vec.join(" "));

        let dir = self.working_dir();
        let exe = self.bin_location();

        let valgrind_copy_result = self.create_valgrind_suppressions();
        let _ = writeln!(&mut std::io::stderr(),
                         "Executing from folder {} with valgrind_suppressions {:?}\n{}",
                         dir.to_str().unwrap(),
                         valgrind_copy_result,
                         full_invocation);
        // change working dir to dir
        let mut cmd = Command::new(exe);
        cmd.args(args_vec.as_slice()).current_dir(dir).env("RUST_BACKTRACE", "1");

        //cmd.stderr(Stdio::piped()).stdout(Stdio::piped());
        cmd.stderr(Stdio::inherit()).stdout(Stdio::inherit());

        let mut child = cmd.spawn().expect("Failed to start?");

        let result = callback(&mut child);

        ///child.kill().unwrap();
        let timeout = Some(Duration::from_secs(1));

        let (status_code, output) = match timeout {
            Some(timeout) => {
                match child.wait_timeout(timeout).unwrap() {
                    Some(status) => (status.code(), None),
                    None => {
                        // child hasn't exited yet
                        child.kill().unwrap();
                        (child.wait().unwrap().code(), None)
                    }
                }
            }
            None => {
                let output: Output = child.wait_with_output().unwrap();
                (output.status.code(), Some(output))
            }
        };

        let _ = writeln!(&mut std::io::stderr(),
                         "exit code {:?}", status_code);

        // Double check we dumped output on segfault
        if status_code == None {
            if let Some(ref out) = output {
                std::io::stderr().write(&out.stderr).unwrap();
                std::io::stdout().write(&out.stdout).unwrap();
            }
            let _ = writeln!(&mut std::io::stderr(),
                             "exit code {:?}", status_code);
        }
        // Killed by signal.
        // 11 Segmentation fault
        // 4 illegal instruction 6 abort 8 floating point error
        if status_code == None && valgrind_on_signal_death {
            if std::env::var("VALGRIND_RUNNING").is_ok() {
                let _ = writeln!(&mut std::io::stderr(),
                                 "VALGRIND_RUNNING defined; skipping valgrind pass");
            } else {
                //ALLOW TO FAIL; valgrind may not be present
                let _ = writeln!(&mut std::io::stderr(),
                                 "Starting valgrind from within self-test:");
                let mut cmd = Command::new("valgrind");
                cmd.arg("-q").arg("--error-exitcode=9").arg(exe);
                cmd.args(args_vec.as_slice()).current_dir(dir).env("RUST_BACKTRACE", "1").env("VALGRIND_RUNNING", "1");

                let _ = writeln!(&mut std::io::stderr(),
                                 "{:?}", cmd);

                let _ = cmd.status(); //.expect("Failed to start valgrind?");
            }
        }

        match output {
            Some(out) => (ProcOutput::from(out), result),
            None => (ProcOutput::from_code(status_code), result)
        }
    }

}<|endoftext|>


<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';
const BACKSLASH_CHAR: char = '\\';
const PAREN_OPEN_CHAR: char = '(';
const PAREN_CLOSE_CHAR: char = ')';
const STRING_BOUNDARY_CHAR: char = '"';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '=',
    ].into_iter().collect();

    static ref TOKEN_BOUNDARY_CHARS: HashSet<char> = vec![
        EOL_CHAR,
        BACKSLASH_CHAR,
        PAREN_OPEN_CHAR,
        PAREN_CLOSE_CHAR,
        STRING_BOUNDARY_CHAR,
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    InvalidNumericLiteral,
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                self.scanner.unget_char();
                break;
            }
        }
    }

    fn match_advanced_token(&mut self, start: char) -> TokenResult {
        if self.must_match_numeric(start) {
            return self.match_numeric(start);
        }

        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::UnrecognizedCharacter(c));
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn must_match_numeric(&mut self, start: char) -> bool {
        // Text must match a numeric literal if it could possibly
        // be interpreted as a number.
        // More formally, text must match a number if any of the
        // following conditions are met:
        // 1. start is a digit
        // 2. start is either '-' or '.' and the second character is a digit
        // 3. start is '-', the second character is '.' and the third character is a digit
        //TODO: There must be some way to optimize this significantly
        if start.is_digit(10) {
            return true;
        }

        let start_dash = start == '-';
        let start_dot = start == '.';

        let second = self.scanner.get_char();
        // Cannot undo this mutation right away just in case we need to get a third character after this

        let second_is_digit = second.map_or(false, |c| c.is_digit(10));

        if (start_dash || start_dot) && second_is_digit {
            // Undo any extra mutations that occurred
            if !second.is_none() { self.scanner.unget_char(); }
            return true;
        }

        let third = self.scanner.get_char();
        // Undo any extra mutations that occurred
        if !third.is_none() { self.scanner.unget_char(); }
        if !second.is_none() { self.scanner.unget_char(); }

        let second_is_dot = second.map_or(false, |c| c == '.');
        let third_is_digit = third.map_or(false, |c| c.is_digit(10));

        if start_dash && second_is_dot && third_is_digit {
            return true;
        }

        false
    }

    fn match_numeric(&mut self, start: char) -> TokenResult {
        let mut literal = String::new();
        literal.push(start);

        // This is quite naive because parse() will do most of the work
        loop {
            let next_char = self.scanner.get_char();
            if next_char.is_none() {
                break;
            }
            let c = next_char.unwrap();
            if c.is_digit(10) || c == '.' {
                literal.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::InvalidNumericLiteral);
            }
        }

        literal.parse::<f64>()
            .map(|value| Token::Number(value))
            .map_err(|_| TokenError::InvalidNumericLiteral)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }

    fn is_token_boundary(&self, c: &char) -> bool {
        TOKEN_BOUNDARY_CHARS.contains(&c) || c.is_whitespace()
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        self.ignore_whitespace();

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            BACKSLASH_CHAR => Token::Backslash,
            PAREN_OPEN_CHAR => Token::ParenOpen,
            PAREN_CLOSE_CHAR => Token::ParenClose,
            STRING_BOUNDARY_CHAR => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.match_advanced_token(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);
        test_number("0.5", 0.5f64);
        test_number(".5", 0.5f64);
        test_number("-.5", -0.5f64);
        test_number("1233.14", 1233.14f64);
        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        let symbol = "-abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_only_minus_in_symbol() {
        // This test is important because it may fail if numeric parsing changes
        let mut tokenizer = tokenizer_for("x - 1");
        let expected = [
            Symbol("x".to_owned()),
            Symbol("-".to_owned()),
            Number(1f64),
        ];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_only_minus_dot_in_symbol() {
        // This test is important because it may fail if numeric parsing changes since -.5 is a valid numeric literal
        let mut tokenizer = tokenizer_for("x -. 1");
        let expected = [
            Symbol("x".to_owned()),
            Symbol("-.".to_owned()),
            Number(1f64),
        ];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        let symbol = "-123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_numbers() {
        let symbol = "192.168.0.1".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));
        assert_eq!(tokenizer.next(), None);

        // Make sure invalid character is caught among valid characters too
        let symbol = "a[$abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_symbols_containing_numbers() {
        let symbol = "ms^-2";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<commit_msg>Fixing remaining tests<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';
const BACKSLASH_CHAR: char = '\\';
const PAREN_OPEN_CHAR: char = '(';
const PAREN_CLOSE_CHAR: char = ')';
const STRING_BOUNDARY_CHAR: char = '"';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '=',
    ].into_iter().collect();

    static ref TOKEN_BOUNDARY_CHARS: HashSet<char> = vec![
        EOL_CHAR,
        BACKSLASH_CHAR,
        PAREN_OPEN_CHAR,
        PAREN_CLOSE_CHAR,
        STRING_BOUNDARY_CHAR,
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    InvalidNumericLiteral,
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                self.scanner.unget_char();
                break;
            }
        }
    }

    fn match_advanced_token(&mut self, start: char) -> TokenResult {
        if self.must_match_numeric(start) {
            return self.match_numeric(start);
        }

        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::UnrecognizedCharacter(c));
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn must_match_numeric(&mut self, start: char) -> bool {
        // Text must match a numeric literal if it could possibly
        // be interpreted as a number.
        // More formally, text must match a number if any of the
        // following conditions are met:
        // 1. start is a digit
        // 2. start is either '-' or '.' and the second character is a digit
        // 3. start is '-', the second character is '.' and the third character is a digit
        //TODO: There must be some way to optimize this significantly
        if start.is_digit(10) {
            return true;
        }

        let start_dash = start == '-';
        let start_dot = start == '.';

        let second = self.scanner.get_char();
        // Cannot undo this mutation right away just in case we need to get a third character after this

        let second_is_digit = second.map_or(false, |c| c.is_digit(10));

        if (start_dash || start_dot) && second_is_digit {
            // Undo any extra mutations that occurred
            if !second.is_none() { self.scanner.unget_char(); }
            return true;
        }

        let third = self.scanner.get_char();
        // Undo any extra mutations that occurred
        if !third.is_none() { self.scanner.unget_char(); }
        if !second.is_none() { self.scanner.unget_char(); }

        let second_is_dot = second.map_or(false, |c| c == '.');
        let third_is_digit = third.map_or(false, |c| c.is_digit(10));

        if start_dash && second_is_dot && third_is_digit {
            return true;
        }

        false
    }

    fn match_numeric(&mut self, start: char) -> TokenResult {
        let mut literal = String::new();
        literal.push(start);

        loop {
            let next_char = self.scanner.get_char();
            if next_char.is_none() {
                break;
            }

            let c = next_char.unwrap();
            // This is quite naive because parse() will do most of the work
            if c.is_digit(10) || c == '.' || c == 'e' || c == '-' {
                literal.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::InvalidNumericLiteral);
            }
        }

        literal.parse::<f64>()
            .map(|value| Token::Number(value))
            .map_err(|_| TokenError::InvalidNumericLiteral)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }

    fn is_token_boundary(&self, c: &char) -> bool {
        TOKEN_BOUNDARY_CHARS.contains(&c) || c.is_whitespace()
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        self.ignore_whitespace();

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            BACKSLASH_CHAR => Token::Backslash,
            PAREN_OPEN_CHAR => Token::ParenOpen,
            PAREN_CLOSE_CHAR => Token::ParenClose,
            STRING_BOUNDARY_CHAR => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.match_advanced_token(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);
        test_number("0.5", 0.5f64);
        test_number(".5", 0.5f64);
        test_number("-.5", -0.5f64);
        test_number("1233.14", 1233.14f64);
        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        let symbol = "-abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_only_minus_in_symbol() {
        // This test is important because it may fail if numeric parsing changes
        let mut tokenizer = tokenizer_for("x - 1");
        let expected = [
            Symbol("x".to_owned()),
            Symbol("-".to_owned()),
            Number(1f64),
        ];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_only_minus_dot_in_symbol() {
        // This test is important because it may fail if numeric parsing changes since -.5 is a valid numeric literal
        let mut tokenizer = tokenizer_for("x -. 1");
        let expected = [
            Symbol("x".to_owned()),
            Symbol("-.".to_owned()),
            Number(1f64),
        ];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        let symbol = "-123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_invalid_numbers() {
        let symbol = "192.168.0.1".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));

        // Make sure invalid character is caught among valid characters too
        let symbol = "a[$abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_symbols_containing_numbers() {
        let symbol = "ms^-2";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<|endoftext|>