<commit_before>use std::ops::FnMut;

use ansi_term::{Colour, Style};
use ansi_term::Colour::*;

pub struct LSColors<'var>(pub &'var str);

impl<'var> LSColors<'var> {
    pub fn each_pair<C>(&mut self, mut callback: C) where C: FnMut(Pair<'var>) -> () {
        for next in self.0.split(":") {
            let bits = next.split("=")
                           .take(3)
                           .collect::<Vec<_>>();

            if bits.len() == 2 && !bits[0].is_empty() && !bits[1].is_empty() {
                callback(Pair { key: bits[0], value: bits[1] });
            }
        }
    }
}

pub struct Pair<'var> {
    pub key: &'var str,
    pub value: &'var str,
}

use std::iter::Peekable;
fn parse_into_high_colour<'a, I>(iter: &mut Peekable<I>) -> Option<Colour>
where I: Iterator<Item=&'a str> {
    match iter.peek() {
        Some(&"5") => {
            let _5 = iter.next();
            if let Some(byte) = iter.next() {
                if let Ok(num) = byte.parse() {
                    return Some(Fixed(num));
                }
            }
        }
        _ => {},
    }

    None
}

impl<'var> Pair<'var> {
    pub fn to_style(&self) -> Style {
        let mut style = Style::default();
        let mut iter = self.value.split(";").peekable();

        while let Some(num) = iter.next() {
            match num {

                // Bold and italic
                "1" | "01" => style = style.bold(),
                "4" | "04" => style = style.underline(),

                // Foreground colours
                "30" => style = style.fg(Black),
                "31" => style = style.fg(Red),
                "32" => style = style.fg(Green),
                "33" => style = style.fg(Yellow),
                "34" => style = style.fg(Blue),
                "35" => style = style.fg(Purple),
                "36" => style = style.fg(Cyan),
                "37" => style = style.fg(White),
                "38" => if let Some(c) = parse_into_high_colour(&mut iter) { style = style.fg(c) },

                // Background colours
                "40" => style = style.on(Black),
                "41" => style = style.on(Red),
                "42" => style = style.on(Green),
                "43" => style = style.on(Yellow),
                "44" => style = style.on(Blue),
                "45" => style = style.on(Purple),
                "46" => style = style.on(Cyan),
                "47" => style = style.on(White),
                "48" => if let Some(c) = parse_into_high_colour(&mut iter) { style = style.on(c) },

                 _    => {/* ignore the error and do nothing */},
            }
        }

        style
    }
}

#[cfg(test)]
mod ansi_test {
    use super::*;
    use ansi_term::Style;

    macro_rules! test {
        ($name:ident: $input:expr => $result:expr) => {
            #[test]
            fn $name() {
                assert_eq!(Pair { key: "", value: $input }.to_style(), $result);
            }
        };
    }

    // Styles
    test!(bold:  "1"         => Style::default().bold());
    test!(under: "4"         => Style::default().underline());
    test!(both:  "1;4"       => Style::default().bold().underline());
    test!(fg:    "31"        => Red.normal());
    test!(bg:    "43"        => Style::default().on(Yellow));
    test!(bfg:   "31;43"     => Red.on(Yellow));
    test!(all:   "43;31;1;4" => Red.on(Yellow).bold().underline());
    test!(again: "1;1;1;1;1" => Style::default().bold());

    // Failure cases
    test!(empty: ""          => Style::default());
    test!(semis: ";;;;;;"    => Style::default());
    test!(nines: "99999999"  => Style::default());
    test!(word:  "GREEN"     => Style::default());

    // Higher colours
    test!(hifg:  "38;5;149"  => Fixed(149).normal());
    test!(hibg:  "48;5;1"    => Style::default().on(Fixed(1)));
    test!(hibo:  "48;5;1;1"  => Style::default().on(Fixed(1)).bold());
    test!(hiund: "4;48;5;1"  => Style::default().on(Fixed(1)).underline());

    test!(fgbg:  "38;5;121;48;5;212"  => Fixed(121).on(Fixed(212)));
    test!(bgfg:  "48;5;121;38;5;212"  => Fixed(212).on(Fixed(121)));
    test!(toohi: "48;5;999"           => Style::default());
}

#[cfg(test)]
mod test {
    use super::*;

    macro_rules! test {
        ($name:ident: $input:expr => $result:expr) => {
            #[test]
            fn $name() {
                let mut lscs = Vec::new();
                LSColors($input).each_pair(|p| lscs.push( (p.key.clone(), p.to_style()) ));
                assert_eq!(lscs, $result.to_vec());
            }
        };
    }

    // Bad parses
    test!(empty:    ""       => []);
    test!(jibber:   "blah"   => []);

    test!(equals:     "="    => []);
    test!(starts:     "=di"  => []);
    test!(ends:     "id="    => []);

    // Foreground colours
    test!(green:   "cb=32"   => [ ("cb", Green.normal()) ]);
    test!(red:     "di=31"   => [ ("di", Red.normal()) ]);
    test!(blue:    "la=34"   => [ ("la", Blue.normal()) ]);

    // Background colours
    test!(yellow:  "do=43"   => [ ("do", Style::default().on(Yellow)) ]);
    test!(purple:  "re=45"   => [ ("re", Style::default().on(Purple)) ]);
    test!(cyan:    "mi=46"   => [ ("mi", Style::default().on(Cyan)) ]);

    // Bold and underline
    test!(bold:    "fa=1"    => [ ("fa", Style::default().bold()) ]);
    test!(under:   "so=4"    => [ ("so", Style::default().underline()) ]);
    test!(both:    "la=1;4"  => [ ("la", Style::default().bold().underline()) ]);

    // More and many
    test!(more:  "me=43;21;55;34:yu=1;4;1"  => [ ("me", Blue.on(Yellow)), ("yu", Style::default().bold().underline()) ]);
    test!(many:  "red=31:green=32:blue=34"  => [ ("red", Red.normal()), ("green", Green.normal()), ("blue", Blue.normal()) ]);
}
<commit_msg>Allow more zero padding in ANSI code parsing<commit_after>use std::ops::FnMut;

use ansi_term::{Colour, Style};
use ansi_term::Colour::*;

pub struct LSColors<'var>(pub &'var str);

impl<'var> LSColors<'var> {
    pub fn each_pair<C>(&mut self, mut callback: C) where C: FnMut(Pair<'var>) -> () {
        for next in self.0.split(":") {
            let bits = next.split("=")
                           .take(3)
                           .collect::<Vec<_>>();

            if bits.len() == 2 && !bits[0].is_empty() && !bits[1].is_empty() {
                callback(Pair { key: bits[0], value: bits[1] });
            }
        }
    }
}

pub struct Pair<'var> {
    pub key: &'var str,
    pub value: &'var str,
}

use std::iter::Peekable;
fn parse_into_high_colour<'a, I>(iter: &mut Peekable<I>) -> Option<Colour>
where I: Iterator<Item=&'a str> {
    match iter.peek() {
        Some(&"5") => {
            let _5 = iter.next();
            if let Some(byte) = iter.next() {
                if let Ok(num) = byte.parse() {
                    return Some(Fixed(num));
                }
            }
        }
        _ => {},
    }

    None
}

impl<'var> Pair<'var> {
    pub fn to_style(&self) -> Style {
        let mut style = Style::default();
        let mut iter = self.value.split(";").peekable();

        while let Some(num) = iter.next() {
            match num.trim_left_matches('0') {

                // Bold and italic
                "1" => style = style.bold(),
                "4" => style = style.underline(),

                // Foreground colours
                "30" => style = style.fg(Black),
                "31" => style = style.fg(Red),
                "32" => style = style.fg(Green),
                "33" => style = style.fg(Yellow),
                "34" => style = style.fg(Blue),
                "35" => style = style.fg(Purple),
                "36" => style = style.fg(Cyan),
                "37" => style = style.fg(White),
                "38" => if let Some(c) = parse_into_high_colour(&mut iter) { style = style.fg(c) },

                // Background colours
                "40" => style = style.on(Black),
                "41" => style = style.on(Red),
                "42" => style = style.on(Green),
                "43" => style = style.on(Yellow),
                "44" => style = style.on(Blue),
                "45" => style = style.on(Purple),
                "46" => style = style.on(Cyan),
                "47" => style = style.on(White),
                "48" => if let Some(c) = parse_into_high_colour(&mut iter) { style = style.on(c) },

                 _    => {/* ignore the error and do nothing */},
            }
        }

        style
    }
}

#[cfg(test)]
mod ansi_test {
    use super::*;
    use ansi_term::Style;

    macro_rules! test {
        ($name:ident: $input:expr => $result:expr) => {
            #[test]
            fn $name() {
                assert_eq!(Pair { key: "", value: $input }.to_style(), $result);
            }
        };
    }

    // Styles
    test!(bold:  "1"         => Style::default().bold());
    test!(bold2: "01"        => Style::default().bold());
    test!(under: "4"         => Style::default().underline());
    test!(unde2: "04"        => Style::default().underline());
    test!(both:  "1;4"       => Style::default().bold().underline());
    test!(both2: "01;04"     => Style::default().bold().underline());
    test!(fg:    "31"        => Red.normal());
    test!(bg:    "43"        => Style::default().on(Yellow));
    test!(bfg:   "31;43"     => Red.on(Yellow));
    test!(bfg2:  "0031;0043" => Red.on(Yellow));
    test!(all:   "43;31;1;4" => Red.on(Yellow).bold().underline());
    test!(again: "1;1;1;1;1" => Style::default().bold());

    // Failure cases
    test!(empty: ""          => Style::default());
    test!(semis: ";;;;;;"    => Style::default());
    test!(nines: "99999999"  => Style::default());
    test!(word:  "GREEN"     => Style::default());

    // Higher colours
    test!(hifg:  "38;5;149"  => Fixed(149).normal());
    test!(hibg:  "48;5;1"    => Style::default().on(Fixed(1)));
    test!(hibo:  "48;5;1;1"  => Style::default().on(Fixed(1)).bold());
    test!(hiund: "4;48;5;1"  => Style::default().on(Fixed(1)).underline());

    test!(fgbg:  "38;5;121;48;5;212"  => Fixed(121).on(Fixed(212)));
    test!(bgfg:  "48;5;121;38;5;212"  => Fixed(212).on(Fixed(121)));
    test!(toohi: "48;5;999"           => Style::default());
}

#[cfg(test)]
mod test {
    use super::*;

    macro_rules! test {
        ($name:ident: $input:expr => $result:expr) => {
            #[test]
            fn $name() {
                let mut lscs = Vec::new();
                LSColors($input).each_pair(|p| lscs.push( (p.key.clone(), p.to_style()) ));
                assert_eq!(lscs, $result.to_vec());
            }
        };
    }

    // Bad parses
    test!(empty:    ""       => []);
    test!(jibber:   "blah"   => []);

    test!(equals:     "="    => []);
    test!(starts:     "=di"  => []);
    test!(ends:     "id="    => []);

    // Foreground colours
    test!(green:   "cb=32"   => [ ("cb", Green.normal()) ]);
    test!(red:     "di=31"   => [ ("di", Red.normal()) ]);
    test!(blue:    "la=34"   => [ ("la", Blue.normal()) ]);

    // Background colours
    test!(yellow:  "do=43"   => [ ("do", Style::default().on(Yellow)) ]);
    test!(purple:  "re=45"   => [ ("re", Style::default().on(Purple)) ]);
    test!(cyan:    "mi=46"   => [ ("mi", Style::default().on(Cyan)) ]);

    // Bold and underline
    test!(bold:    "fa=1"    => [ ("fa", Style::default().bold()) ]);
    test!(under:   "so=4"    => [ ("so", Style::default().underline()) ]);
    test!(both:    "la=1;4"  => [ ("la", Style::default().bold().underline()) ]);

    // More and many
    test!(more:  "me=43;21;55;34:yu=1;4;1"  => [ ("me", Blue.on(Yellow)), ("yu", Style::default().bold().underline()) ]);
    test!(many:  "red=31:green=32:blue=34"  => [ ("red", Red.normal()), ("green", Green.normal()), ("blue", Blue.normal()) ]);
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 by Stacy Prowell.  All rights reserved.
//
// Licensed under the BSD 2-Clause license.  See the file LICENSE
// that is part of this distribution.  This file may not be copied,
// modified, or distributed except according to those terms.extern

//! Provide basic definitions of terms.

use std::collections::BitVec;
//use std::tuple::Tuple1;

/// Represents root terms.
struct Root ( () );
struct Any ( () );
struct ENone ( () );
#[derive(Debug)]
struct EString( String ) ;
struct Symbol( String );
#[derive(Debug)]
struct Integer( isize );
struct Float( f64 );
struct BitString( BitVec );
struct Boolean( bool );
#[derive(Debug)]
struct Term<T> ( T );

trait Termable<T>{
    fn new(value: T) -> Self;
    fn native(&self) -> &T;
}

impl Termable<String> for EString{
    fn new(value: String) -> EString{
        EString(value)
    }
    fn native(&self) -> &String {
        &self.0
    }
}

impl Termable<isize> for Integer{
    fn new(value: isize) -> Integer{
        Integer(value)
    }
    fn native(&self) -> &isize {
        &self.0
    }
}

impl<T> Termable<T> for Term<T>{
    fn new(t: T) -> Term<T> {
        Term ( t )
    }
    fn native(&self) -> &T { &self.0 }
}

impl<T> Term<T>
{
    pub fn new<U>(t: U) -> Term<U> where U: Termable<T> {
        Term ( t )
    }
    pub fn unwrap(&self) -> &T {
        &self.0
    }
}

#[test]
fn term_type_check_test() -> (){
    let termable = EString::new("Test".to_string());
    let mut stringterm = Term::new(termable);
    let integerterm = Term::new(Integer(3));
    // Uncomment the following line to see the term type checker assist in action
    // stringterm = integerterm;
    // The following line shoudl fail to compile because i32 does not have the Termable trait.
    // let failterm = Term::new(3);

    //Nested terms don't work right now
    let nestedterm = Term::new(Term::new(Integer(3)));
    let stringval = &stringterm.unwrap().native();
    panic!("Intentionial panic. {:?}", stringval);
}

<commit_msg>Got rid of several generics. Term is now a trait. new() is implemented on structs. trait RootType has come into existence.<commit_after>// Copyright (c) 2015 by Stacy Prowell.  All rights reserved.
//
// Licensed under the BSD 2-Clause license.  See the file LICENSE
// that is part of this distribution.  This file may not be copied,
// modified, or distributed except according to those terms.extern

//! Provide basic definitions of terms.

use std::collections::BitVec;
//use std::tuple::Tuple1;

/// Represents root terms.
struct Root ( () );
struct Any ( () );
struct ENone ( () );
#[derive(Debug)]
struct EString( String ) ;
impl EString{
    fn new(val: &str) -> Self { EString(val.to_string()) }
}

struct Symbol( String );
#[derive(Debug)]
struct Integer( isize );
impl Integer{
    fn new(val: isize) -> Self { Integer(val) }
}

struct Float( f64 );
struct BitString( BitVec );
struct Boolean( bool );
//#[derive(Debug)]
//struct Term<T> ( T );

trait RootType<T>{
    fn native(&self) -> &T;
}
impl RootType<String> for EString{
    fn native(&self) -> &String { &self.0 }
}
impl RootType<isize> for Integer{
    fn native(&self) -> &isize { &self.0 }
}

trait Term{
    fn to_string(&self) -> String { "This is a Term.".to_string() }
}

impl Term for EString{
    fn to_string(&self) -> String { "This is an EString".to_string() }
}

impl Term for Integer{
    fn to_string(&self) -> String {"This is an Integer".to_string() }
}

#[test]
fn term_type_check_test() -> (){
    let mut estring = EString("Test".to_string());
    let integer = Integer(3);
    // Uncomment the following line to see the term type checker assist in action
    // stringterm = integerterm;
    // The following line shoudl fail to compile because i32 does not have the Termable trait.
    //estring = integer;

    panic!("Intentionial panic. {:?}", estring.to_string());
}

<|endoftext|>


<commit_before>// Copyright (c) 2019, Ben Boeckel
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//       this list of conditions and the following disclaimer in the documentation
//       and/or other materials provided with the distribution.
//     * Neither the name of this project nor the names of its contributors
//       may be used to endorse or promote products derived from this software
//       without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

//! The test structure here comes from the structure in libkeyutils.

pub(crate) mod utils;

mod add;
mod clear;
mod describe;
mod instantiate;
mod invalidate;
mod link;
mod newring;
mod permitting;
mod reading;
mod revoke;
mod search;
mod session;
mod timeout;
mod unlink;
mod update;
<commit_msg>tests: disable session tests (for now)<commit_after>// Copyright (c) 2019, Ben Boeckel
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//       this list of conditions and the following disclaimer in the documentation
//       and/or other materials provided with the distribution.
//     * Neither the name of this project nor the names of its contributors
//       may be used to endorse or promote products derived from this software
//       without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

//! The test structure here comes from the structure in libkeyutils.

pub(crate) mod utils;

mod add;
mod clear;
mod describe;
mod instantiate;
mod invalidate;
mod link;
mod newring;
mod permitting;
mod reading;
mod revoke;
mod search;
// FIXME(#39): These tests fail when run in the same process. Something should be done about this.
// mod session;
mod timeout;
mod unlink;
mod update;
<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Token {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d*\]$").unwrap();
        }

        match RE.captures(s) {
            None => {
                Token::Identifier { ident: String::from(s), next: None }
            },
            Some(captures) => {
                match captures.get(0) {
                    None => Token::Identifier { ident: String::from(s), next: None },
                    Some(mtch) => {
                        let mtch = mtch.as_str().replace("[","").replace("]","");
                        println!("{}", mtch);
                        let i : i64 = FromStr::from_str(&mtch).unwrap();
                        Token::Index {
                            idx: i,
                            next: None,
                        }
                    }
                }
            }
        }
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let mut token = mk_token_object(token);
                build_token_tree(split, &mut token);
                last.set_next(token);
            }
        }
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = mk_token_object(token);
            build_token_tree(&mut tokens, &mut tok);
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Impl tokenize_with_seperator to fix test_tokenize_array_brackets_only<commit_after>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d+\]$").unwrap();
        }

        if s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']' {
            match RE.captures(s) {
                None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
                Some(captures) => {
                    match captures.get(0) {
                        None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                        Some(mtch) => {
                            let mtch = mtch.as_str().replace("[","").replace("]","");
                            let i : i64 = FromStr::from_str(&mtch).unwrap(); // save because regex
                            Ok(Token::Index {
                                idx: i,
                                next: None,
                            })
                        }
                    }
                }
            }
        } else {
            Ok(Token::Identifier { ident: String::from(s), next: None })
        }

    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let mut token = try!(mk_token_object(token));
                build_token_tree(split, &mut token);
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Box<Token>>
    },

    Index {
        idx: usize,
        next: Option<Box<Token>>
    }
}

impl Token {

    pub fn next(&self) -> Option<&Box<Token>> {
        match self {
            &Token::Identifier { ref next, .. } => next.as_ref(),
            &Token::Index { ref next, .. }      => next.as_ref(),
        }
    }

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { ref mut next, .. } => *next = Some(Box::new(token)),
            &mut Token::Index { ref mut next, .. }      => *next = Some(Box::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::ErrorKind;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { ref ident, next: None } => {
                assert_eq!("example", ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Box::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Box::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Box::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index { next: None, ..  }) => true,
                _                                    => false,
            }
        }
    }

    #[test]
    fn test_pop_last_token_from_single_identifier_token_is_none() {
        let mut token = Token::Identifier {
            ident: String::from("something"),
            next: None,
        };

        let last = token.pop_last();
        assert!(last.is_none());
    }

    #[test]
    fn test_pop_last_token_from_single_index_token_is_none() {
        let mut token = Token::Index {
            idx: 0,
            next: None,
        };

        let last = token.pop_last();
        assert!(last.is_none());
    }

    #[test]
    fn test_pop_last_token_from_single_identifier_token_is_one() {
        let mut token = Token::Identifier {
            ident: String::from("some"),
            next: Some(Box::new(Token::Identifier {
                ident: String::from("thing"),
                next: None,
            })),
        };

        let last = token.pop_last();

        assert!(last.is_some());
        let last = last.unwrap();

        assert!(is_match!(*last, Token::Identifier { .. }));
        match *last {
            Token::Identifier { ident: ident, .. } => {
                assert_eq!("thing", ident);
            }
            _ => panic!("What just happened?"),
        }
    }

    #[test]
    fn test_pop_last_token_from_single_index_token_is_one() {
        let mut token = Token::Index {
            idx: 0,
            next: Some(Box::new(Token::Index {
                idx: 1,
                next: None,
            })),
        };

        let last = token.pop_last();

        assert!(last.is_some());
        let last = last.unwrap();

        assert!(is_match!(*last, Token::Index { idx: 1, .. }));
    }

    #[test]
    fn test_pop_last_token_from_identifier_chain() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.d.e.f"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!("f", last.unwrap().identifier());
    }

    #[test]
    fn test_pop_last_token_from_mixed_chain() {
        let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!("f", last.unwrap().identifier());
    }

    #[test]
    fn test_pop_last_token_from_identifier_chain_is_array() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.d.e.f.[1000]"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!(1000, last.unwrap().idx());
    }

    #[test]
    fn test_pop_last_token_from_mixed_chain_is_array() {
        let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f.[1000]"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!(1000, last.unwrap().idx());
    }

    #[test]
    fn test_pop_last_token_from_one_token() {
        let tokens = tokenize_with_seperator(&String::from("a"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_none());
    }

}
<commit_msg>Add Token::pop_last() stub to get code compiling<commit_after>/// The tokenizer for the query interpreter

use error::*;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Box<Token>>
    },

    Index {
        idx: usize,
        next: Option<Box<Token>>
    }
}

impl Token {

    pub fn next(&self) -> Option<&Box<Token>> {
        match self {
            &Token::Identifier { ref next, .. } => next.as_ref(),
            &Token::Index { ref next, .. }      => next.as_ref(),
        }
    }

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { ref mut next, .. } => *next = Some(Box::new(token)),
            &mut Token::Index { ref mut next, .. }      => *next = Some(Box::new(token)),
        }
    }

    /// Pop the last token from the chain of tokens
    ///
    /// Returns None if the current Token has no next token
    pub fn pop_last(&mut self) -> Option<Box<Token>> {
        unimplemented!()
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::ErrorKind;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { ref ident, next: None } => {
                assert_eq!("example", ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Box::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Box::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Box::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index { next: None, ..  }) => true,
                _                                    => false,
            }
        }
    }

    #[test]
    fn test_pop_last_token_from_single_identifier_token_is_none() {
        let mut token = Token::Identifier {
            ident: String::from("something"),
            next: None,
        };

        let last = token.pop_last();
        assert!(last.is_none());
    }

    #[test]
    fn test_pop_last_token_from_single_index_token_is_none() {
        let mut token = Token::Index {
            idx: 0,
            next: None,
        };

        let last = token.pop_last();
        assert!(last.is_none());
    }

    #[test]
    fn test_pop_last_token_from_single_identifier_token_is_one() {
        let mut token = Token::Identifier {
            ident: String::from("some"),
            next: Some(Box::new(Token::Identifier {
                ident: String::from("thing"),
                next: None,
            })),
        };

        let last = token.pop_last();

        assert!(last.is_some());
        let last = last.unwrap();

        assert!(is_match!(*last, Token::Identifier { .. }));
        match *last {
            Token::Identifier { ident: ident, .. } => {
                assert_eq!("thing", ident);
            }
            _ => panic!("What just happened?"),
        }
    }

    #[test]
    fn test_pop_last_token_from_single_index_token_is_one() {
        let mut token = Token::Index {
            idx: 0,
            next: Some(Box::new(Token::Index {
                idx: 1,
                next: None,
            })),
        };

        let last = token.pop_last();

        assert!(last.is_some());
        let last = last.unwrap();

        assert!(is_match!(*last, Token::Index { idx: 1, .. }));
    }

    #[test]
    fn test_pop_last_token_from_identifier_chain() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.d.e.f"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!("f", last.unwrap().identifier());
    }

    #[test]
    fn test_pop_last_token_from_mixed_chain() {
        let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!("f", last.unwrap().identifier());
    }

    #[test]
    fn test_pop_last_token_from_identifier_chain_is_array() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.d.e.f.[1000]"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!(1000, last.unwrap().idx());
    }

    #[test]
    fn test_pop_last_token_from_mixed_chain_is_array() {
        let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f.[1000]"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_some());
        assert_eq!(1000, last.unwrap().idx());
    }

    #[test]
    fn test_pop_last_token_from_one_token() {
        let tokens = tokenize_with_seperator(&String::from("a"), '.');
        assert!(tokens.is_ok());
        let mut tokens = tokens.unwrap();

        let last = tokens.pop_last();
        assert!(last.is_none());
    }

}
<|endoftext|>


<commit_before>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! let tokens = tokenize("word");
//! assert_eq!(tokens.len(), 1);
//!
//! let tokens = tokenize("show interface");
//! assert_eq!(tokens.len(), 3);
//!
//! let tokens = tokenize("echo -n \"a b c\"");
//! assert_eq!(tokens.len(), 5);
//! ```

/// A position within a string.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the string.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the string.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a string.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the string. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' {
            self.special(offset);
        } else if c == '?' {
            self.special(offset);
        } else if c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Special => {
                    panic!("Special not yet implemented.");
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word => self.reduce(),
            State::Whitespace => self.reduce(),
            State::WordBackslash => panic!("Escaping backslash at end of input"),
            State::Doublequote => panic!("Unclosed double quote at end of input"),
            State::DoublequoteBackslash => panic!("Escaped doublequote at end of input"),
            State::Special => panic!("Special not yet implemented"),
        }
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Vec<Token> {
    let mut tokenizer = Tokenizer::new(text);
    tokenizer.tokenize();
    tokenizer.tokens
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        let ts = tokenize("");
        assert_eq!(ts.len(), 0);
    }

    #[test]
    fn single_word() {
        let ts = tokenize("a");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
    }

    #[test]
    fn multiple_words() {
        let ts = tokenize(" aa bb  ccc ");
        assert_eq!(ts.len(), 7);
        assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
        assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
        assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
        assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
        assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
        assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
        assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
    }

    #[test]
    fn quoted_text() {
        let ts = tokenize("a \"b c\"");
        assert_eq!(ts.len(), 3);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
        assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
        assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
    }
}
<commit_msg>Fix backslash-in-word tokenization.<commit_after>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! let tokens = tokenize("word");
//! assert_eq!(tokens.len(), 1);
//!
//! let tokens = tokenize("show interface");
//! assert_eq!(tokens.len(), 3);
//!
//! let tokens = tokenize("echo -n \"a b c\"");
//! assert_eq!(tokens.len(), 5);
//! ```

/// A position within a string.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the string.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the string.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a string.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the string. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' {
            self.special(offset);
        } else if c == '?' {
            self.special(offset);
        } else if c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Special => {
                    panic!("Special not yet implemented.");
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word => self.reduce(),
            State::Whitespace => self.reduce(),
            State::WordBackslash => panic!("Escaping backslash at end of input"),
            State::Doublequote => panic!("Unclosed double quote at end of input"),
            State::DoublequoteBackslash => panic!("Escaped doublequote at end of input"),
            State::Special => panic!("Special not yet implemented"),
        }
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Vec<Token> {
    let mut tokenizer = Tokenizer::new(text);
    tokenizer.tokenize();
    tokenizer.tokens
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        let ts = tokenize("");
        assert_eq!(ts.len(), 0);
    }

    #[test]
    fn single_word() {
        let ts = tokenize("a");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
    }

    #[test]
    fn multiple_words() {
        let ts = tokenize(" aa bb  ccc ");
        assert_eq!(ts.len(), 7);
        assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
        assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
        assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
        assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
        assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
        assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
        assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
    }

    #[test]
    fn quoted_text() {
        let ts = tokenize("a \"b c\"");
        assert_eq!(ts.len(), 3);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
        assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
        assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
    }

    #[test]
    fn escaped_whitespace_in_word() {
        let ts = tokenize("a\\ b");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
    }
}
<|endoftext|>


<commit_before>//! A type is a classification which determines how data is used.

#[deny(missing_docs)]
mod array_type;
mod enums;
#[deny(missing_docs)]
mod float_type;
#[deny(missing_docs)]
mod fn_type;
#[deny(missing_docs)]
mod int_type;
#[deny(missing_docs)]
mod ptr_type;
#[deny(missing_docs)]
mod struct_type;
#[deny(missing_docs)]
mod traits;
#[deny(missing_docs)]
mod vec_type;
#[deny(missing_docs)]
mod void_type;

pub use crate::types::array_type::ArrayType;
pub use crate::types::enums::{AnyTypeEnum, BasicTypeEnum};
pub use crate::types::float_type::FloatType;
pub use crate::types::fn_type::FunctionType;
pub use crate::types::int_type::{IntType, StringRadix};
pub use crate::types::ptr_type::PointerType;
pub use crate::types::struct_type::StructType;
pub use crate::types::traits::{AnyType, BasicType, IntMathType, FloatMathType, PointerMathType};
pub use crate::types::vec_type::VectorType;
pub use crate::types::void_type::VoidType;
pub(crate) use crate::types::traits::AsTypeRef;

#[llvm_versions(3.7..=4.0)]
use llvm_sys::core::LLVMDumpType;
use llvm_sys::core::{LLVMAlignOf, LLVMGetTypeContext, LLVMFunctionType, LLVMArrayType, LLVMGetUndef, LLVMPointerType, LLVMPrintTypeToString, LLVMTypeIsSized, LLVMSizeOf, LLVMVectorType, LLVMGetElementType, LLVMConstNull};
use llvm_sys::prelude::{LLVMTypeRef, LLVMValueRef};
#[cfg(feature = "experimental")]
use static_alloc::Slab;

use std::fmt;
use std::marker::PhantomData;

use crate::AddressSpace;
use crate::context::ContextRef;
use crate::support::LLVMString;
use crate::values::IntValue;

// Worth noting that types seem to be singletons. At the very least, primitives are.
// Though this is likely only true per thread since LLVM claims to not be very thread-safe.
#[derive(PartialEq, Eq, Clone, Copy)]
struct Type<'ctx> {
    ty: LLVMTypeRef,
    _marker: PhantomData<&'ctx ()>,
}

impl<'ctx> Type<'ctx> {
    fn new(ty: LLVMTypeRef) -> Self {
        assert!(!ty.is_null());

        Type {
            ty,
            _marker: PhantomData,
        }
    }

    // REVIEW: LLVM 5.0+ seems to have removed this function in release mode
    // and so will fail to link when used. I've decided to remove it from 5.0+
    // for now. We should consider removing it altogether since print_to_string
    // could be used and manually written to stderr in rust...
    #[llvm_versions(3.7..=4.0)]
    fn print_to_stderr(&self) {
        unsafe {
            LLVMDumpType(self.ty);
        }
    }

    fn const_zero(&self) -> LLVMValueRef {
        unsafe {
            LLVMConstNull(self.ty)
        }
    }

    fn ptr_type(&self, address_space: AddressSpace) -> PointerType<'ctx> {
        let ptr_type = unsafe {
            LLVMPointerType(self.ty, address_space as u32)
        };

        PointerType::new(ptr_type)
    }

    fn vec_type(&self, size: u32) -> VectorType<'ctx> {
        assert!(size != 0, "Vectors of size zero are not allowed.");
        // -- https://llvm.org/docs/LangRef.html#vector-type

        let vec_type = unsafe {
            LLVMVectorType(self.ty, size)
        };

        VectorType::new(vec_type)
    }

    #[cfg(not(feature = "experimental"))]
    fn fn_type(&self, param_types: &[BasicTypeEnum<'ctx>], is_var_args: bool) -> FunctionType<'ctx> {
        let mut param_types: Vec<LLVMTypeRef> = param_types.iter()
                                                           .map(|val| val.as_type_ref())
                                                           .collect();
        let fn_type = unsafe {
            LLVMFunctionType(self.ty, param_types.as_mut_ptr(), param_types.len() as u32, is_var_args as i32)
        };

        FunctionType::new(fn_type)
    }

    #[cfg(feature = "experimental")]
    fn fn_type(&self, param_types: &[BasicTypeEnum<'ctx>], is_var_args: bool) -> FunctionType<'ctx> {
        let pool: Slab<[usize; 16]> = Slab::uninit();
        let mut fixed_vec = pool.fixed_vec(param_types.len()).expect("Found more than 16 params");

        for param_type in param_types {
            fixed_vec.push(param_type.as_type_ref()).expect("Unexpected error");
        }

        let fn_type = unsafe {
            LLVMFunctionType(self.ty, fixed_vec.as_mut_ptr(), fixed_vec.len() as u32, is_var_args as i32)
        };

        FunctionType::new(fn_type)
    }

    fn array_type(&self, size: u32) -> ArrayType<'ctx> {
        let ty = unsafe {
            LLVMArrayType(self.ty, size)
        };

        ArrayType::new(ty)
    }

    fn get_undef(&self) -> LLVMValueRef {
        unsafe {
            LLVMGetUndef(self.ty)
        }
    }

    fn get_alignment(&self) -> IntValue<'ctx> {
        let val = unsafe {
            LLVMAlignOf(self.ty)
        };

        IntValue::new(val)
    }

    fn get_context(&self) -> ContextRef<'ctx> {
        let context = unsafe {
            LLVMGetTypeContext(self.ty)
        };

        ContextRef::new(context)
    }

    // REVIEW: This should be known at compile time, maybe as a const fn?
    // On an enum or trait, this would not be known at compile time (unless
    // enum has only sized types for example)
    fn is_sized(&self) -> bool {
        unsafe {
            LLVMTypeIsSized(self.ty) == 1
        }
    }

    // REVIEW: Option<IntValue>? If we want to provide it on enums that
    // contain unsized types
    fn size_of(&self) -> IntValue<'ctx> {
        debug_assert!(self.is_sized());

        let int_value = unsafe {
            LLVMSizeOf(self.ty)
        };

        IntValue::new(int_value)
    }

    fn print_to_string(&self) -> LLVMString {
        let c_string_ptr = unsafe {
            LLVMPrintTypeToString(self.ty)
        };

        LLVMString::new(c_string_ptr)
    }

    pub fn get_element_type(&self) -> AnyTypeEnum<'ctx> {
        let ptr = unsafe {
            LLVMGetElementType(self.ty)
        };

        AnyTypeEnum::new(ptr)
    }

}

impl fmt::Debug for Type<'_> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let llvm_type = self.print_to_string();

        f.debug_struct("Type")
            .field("address", &self.ty)
            .field("llvm_type", &llvm_type)
            .finish()
    }
}
<commit_msg>Fixed experimental usage of static-alloc<commit_after>//! A type is a classification which determines how data is used.

#[deny(missing_docs)]
mod array_type;
mod enums;
#[deny(missing_docs)]
mod float_type;
#[deny(missing_docs)]
mod fn_type;
#[deny(missing_docs)]
mod int_type;
#[deny(missing_docs)]
mod ptr_type;
#[deny(missing_docs)]
mod struct_type;
#[deny(missing_docs)]
mod traits;
#[deny(missing_docs)]
mod vec_type;
#[deny(missing_docs)]
mod void_type;

pub use crate::types::array_type::ArrayType;
pub use crate::types::enums::{AnyTypeEnum, BasicTypeEnum};
pub use crate::types::float_type::FloatType;
pub use crate::types::fn_type::FunctionType;
pub use crate::types::int_type::{IntType, StringRadix};
pub use crate::types::ptr_type::PointerType;
pub use crate::types::struct_type::StructType;
pub use crate::types::traits::{AnyType, BasicType, IntMathType, FloatMathType, PointerMathType};
pub use crate::types::vec_type::VectorType;
pub use crate::types::void_type::VoidType;
pub(crate) use crate::types::traits::AsTypeRef;

#[llvm_versions(3.7..=4.0)]
use llvm_sys::core::LLVMDumpType;
use llvm_sys::core::{LLVMAlignOf, LLVMGetTypeContext, LLVMFunctionType, LLVMArrayType, LLVMGetUndef, LLVMPointerType, LLVMPrintTypeToString, LLVMTypeIsSized, LLVMSizeOf, LLVMVectorType, LLVMGetElementType, LLVMConstNull};
use llvm_sys::prelude::{LLVMTypeRef, LLVMValueRef};
#[cfg(feature = "experimental")]
use static_alloc::Bump;

use std::fmt;
use std::marker::PhantomData;

use crate::AddressSpace;
use crate::context::ContextRef;
use crate::support::LLVMString;
use crate::values::IntValue;

// Worth noting that types seem to be singletons. At the very least, primitives are.
// Though this is likely only true per thread since LLVM claims to not be very thread-safe.
#[derive(PartialEq, Eq, Clone, Copy)]
struct Type<'ctx> {
    ty: LLVMTypeRef,
    _marker: PhantomData<&'ctx ()>,
}

impl<'ctx> Type<'ctx> {
    fn new(ty: LLVMTypeRef) -> Self {
        assert!(!ty.is_null());

        Type {
            ty,
            _marker: PhantomData,
        }
    }

    // REVIEW: LLVM 5.0+ seems to have removed this function in release mode
    // and so will fail to link when used. I've decided to remove it from 5.0+
    // for now. We should consider removing it altogether since print_to_string
    // could be used and manually written to stderr in rust...
    #[llvm_versions(3.7..=4.0)]
    fn print_to_stderr(&self) {
        unsafe {
            LLVMDumpType(self.ty);
        }
    }

    fn const_zero(&self) -> LLVMValueRef {
        unsafe {
            LLVMConstNull(self.ty)
        }
    }

    fn ptr_type(&self, address_space: AddressSpace) -> PointerType<'ctx> {
        let ptr_type = unsafe {
            LLVMPointerType(self.ty, address_space as u32)
        };

        PointerType::new(ptr_type)
    }

    fn vec_type(&self, size: u32) -> VectorType<'ctx> {
        assert!(size != 0, "Vectors of size zero are not allowed.");
        // -- https://llvm.org/docs/LangRef.html#vector-type

        let vec_type = unsafe {
            LLVMVectorType(self.ty, size)
        };

        VectorType::new(vec_type)
    }

    #[cfg(not(feature = "experimental"))]
    fn fn_type(&self, param_types: &[BasicTypeEnum<'ctx>], is_var_args: bool) -> FunctionType<'ctx> {
        let mut param_types: Vec<LLVMTypeRef> = param_types.iter()
                                                           .map(|val| val.as_type_ref())
                                                           .collect();
        let fn_type = unsafe {
            LLVMFunctionType(self.ty, param_types.as_mut_ptr(), param_types.len() as u32, is_var_args as i32)
        };

        FunctionType::new(fn_type)
    }

    #[cfg(feature = "experimental")]
    fn fn_type(&self, param_types: &[BasicTypeEnum<'ctx>], is_var_args: bool) -> FunctionType<'ctx> {
        let pool: Bump<[usize; 16]> = Bump::uninit();
        let mut pool_start = None;

        for (i, param_type) in param_types.iter().enumerate() {
            let addr = pool.leak(param_type.as_type_ref()).expect("Found more than 16 params");

            if i == 0 {
                pool_start = Some(addr as *mut _);
            }
        }

        let fn_type = unsafe {
            LLVMFunctionType(self.ty, pool_start.unwrap_or(std::ptr::null_mut()), param_types.len() as u32, is_var_args as i32)
        };

        FunctionType::new(fn_type)
    }

    fn array_type(&self, size: u32) -> ArrayType<'ctx> {
        let ty = unsafe {
            LLVMArrayType(self.ty, size)
        };

        ArrayType::new(ty)
    }

    fn get_undef(&self) -> LLVMValueRef {
        unsafe {
            LLVMGetUndef(self.ty)
        }
    }

    fn get_alignment(&self) -> IntValue<'ctx> {
        let val = unsafe {
            LLVMAlignOf(self.ty)
        };

        IntValue::new(val)
    }

    fn get_context(&self) -> ContextRef<'ctx> {
        let context = unsafe {
            LLVMGetTypeContext(self.ty)
        };

        ContextRef::new(context)
    }

    // REVIEW: This should be known at compile time, maybe as a const fn?
    // On an enum or trait, this would not be known at compile time (unless
    // enum has only sized types for example)
    fn is_sized(&self) -> bool {
        unsafe {
            LLVMTypeIsSized(self.ty) == 1
        }
    }

    // REVIEW: Option<IntValue>? If we want to provide it on enums that
    // contain unsized types
    fn size_of(&self) -> IntValue<'ctx> {
        debug_assert!(self.is_sized());

        let int_value = unsafe {
            LLVMSizeOf(self.ty)
        };

        IntValue::new(int_value)
    }

    fn print_to_string(&self) -> LLVMString {
        let c_string_ptr = unsafe {
            LLVMPrintTypeToString(self.ty)
        };

        LLVMString::new(c_string_ptr)
    }

    pub fn get_element_type(&self) -> AnyTypeEnum<'ctx> {
        let ptr = unsafe {
            LLVMGetElementType(self.ty)
        };

        AnyTypeEnum::new(ptr)
    }

}

impl fmt::Debug for Type<'_> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let llvm_type = self.print_to_string();

        f.debug_struct("Type")
            .field("address", &self.ty)
            .field("llvm_type", &llvm_type)
            .finish()
    }
}
<|endoftext|>