<commit_before>use regex::Regex;
use regex::Captures;
use string::constants::UNACCONTABLE_WORDS;

const SINGULAR_RULES: [&'static str; 27] = ["ews", "", "um", "sis", "sis", "fe", "", "", "f", "y",
                                            "eries", "ovie", "", "ouse", "", "", "is", "xis",
                                            "us", "", "", "ex", "ix", "", "", "", ""];

/// Converts a `String` to singularized `String`
///
/// #Examples
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "foo_bars".to_string();
///     let expected_string: String = "foo_bar".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "oxen".to_string();
///     let expected_string: String = "ox".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "crates".to_string();
///     let expected_string: String = "crate".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "boxes".to_string();
///     let expected_string: String = "box".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "vengeance".to_string();
///     let expected_string: String = "vengeance".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "yoga".to_string();
///     let expected_string: String = "yoga".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
///
pub fn to_singular(non_singular_string: String) -> String {
    if UNACCONTABLE_WORDS.contains(&non_singular_string.as_ref()) {
        non_singular_string
    } else {
        let safe_out_string: String = non_singular_string.to_string();
        let maybe_regex_match: Option<String> = singularize_string(non_singular_string);
        maybe_regex_match.unwrap_or(safe_out_string)
    }
}

fn singularize_string(non_singular_string: String) -> Option<String> {
    let singular_regexes: [Regex; 27] = singular_rules_regexes();
    for singular_index in 0..singular_regexes.len() {
        let maybe_match: Option<String> =
            singularize_string_from_capture_groups_if_match(non_singular_string.to_string(),
                                                            singular_index);

        if maybe_match.is_some() {
            return Some(maybe_match.unwrap_or(non_singular_string));
        }
    }
    return None;
}

fn singularize_string_from_capture_groups_if_match(non_singular_string: String,
                                                   singular_index: usize)
                                                   -> Option<String> {
    let singular_regexes: [Regex; 27] = singular_rules_regexes();

    if singular_regexes[singular_index].is_match(&non_singular_string) {

        let cap: Captures = singular_regexes[singular_index]
            .captures(&non_singular_string)
            .unwrap();

        Some(singularize_string_from_capture_groups(cap, singular_index))
    } else {
        None
    }
}

fn singularize_string_from_capture_groups(capture_groups: Captures,
                                          singular_index: usize)
                                          -> String {
    if capture_groups.len() > 2 {

        return format!("{}{}{}",
                       capture_groups.at(1).unwrap_or(""),
                       capture_groups.at(2).unwrap_or(""),
                       SINGULAR_RULES[singular_index]);
    } else {

        return format!("{}{}",
                       capture_groups.at(1).unwrap_or(""),
                       SINGULAR_RULES[singular_index]);
    }
}

fn singular_rules_regexes() -> [Regex; 27] {
    [Regex::new(r"(n)ews$").unwrap(),
     Regex::new(r"(\w*)(o)es$").unwrap(),
     Regex::new(r"(\w*)([ti])a$").unwrap(),
     Regex::new(r"((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)(sis|ses)$").unwrap(),
     Regex::new(r"(^analy)(sis|ses)$").unwrap(),
     Regex::new(r"(\w*)([^f])ves$").unwrap(),
     Regex::new(r"(\w*)(hive)s$").unwrap(),
     Regex::new(r"(\w*)(tive)s$").unwrap(),
     Regex::new(r"(\w*)([lr])ves$").unwrap(),
     Regex::new(r"(\w*)([^aeiouy]|qu)ies$").unwrap(),
     Regex::new(r"(s)eries$").unwrap(),
     Regex::new(r"(m)ovies$").unwrap(),
     Regex::new(r"(\w*)(x|ch|ss|sh)es$").unwrap(),
     Regex::new(r"(m|l)ice$").unwrap(),
     Regex::new(r"(bus)(es)?$").unwrap(),
     Regex::new(r"(shoe)s$").unwrap(),
     Regex::new(r"(cris|test)(is|es)$").unwrap(),
     Regex::new(r"^(a)x[ie]s$").unwrap(),
     Regex::new(r"(octop|vir)(us|i)$").unwrap(),
     Regex::new(r"(alias|status)(es)?$").unwrap(),
     Regex::new(r"^(ox)en").unwrap(),
     Regex::new(r"(vert|ind)ices$").unwrap(),
     Regex::new(r"(matr)ices$").unwrap(),
     Regex::new(r"(quiz)zes$").unwrap(),
     Regex::new(r"(database)s$").unwrap(),
     Regex::new(r"(\w*)s$").unwrap(),
     Regex::new(r"(\w*)(ss)$").unwrap()]

}

#[test]
fn singularize_string_if_a_regex_will_match() {
    let expected_string: String = "oxen".to_string();
    let asserted_string: Option<String> = singularize_string(expected_string);

    assert!(asserted_string.is_some());

}

#[test]
fn singularize_string_returns_none_option_if_no_match() {
    let expected_string: String = "bacon".to_string();
    let asserted_string: Option<String> = singularize_string(expected_string);

    assert!(asserted_string.is_none());

}

#[test]
fn singularize_string_from_capture_groups_if_match_should_return_a_string_if_a_match() {
    let expected_string: String = "oxen".to_string();
    let singular_index: usize = 20;
    let asserted_string: Option<String> =
        singularize_string_from_capture_groups_if_match(expected_string, singular_index);

    assert!(asserted_string.is_some());

}

#[test]
fn singularize_string_from_capture_groups_if_match_should_return_an_option_none_if_no_match() {
    let expected_string: String = "bacon".to_string();
    let singular_index: usize = 20;
    let asserted_string: Option<String> =
        singularize_string_from_capture_groups_if_match(expected_string, singular_index);

    assert!(asserted_string.is_none());

}

#[test]
fn singularize_string_from_capture_groups_should_return_a_singularized_string() {
    let expected_string: String = "ox".to_string();
    let test_string: String = "oxen".to_string();
    let singular_regexes: [Regex; 27] = singular_rules_regexes();
    let singular_index: usize = 20;
    let cap: Captures = singular_regexes[singular_index]
        .captures(&test_string)
        .unwrap();
    let asserted_string: String = singularize_string_from_capture_groups(cap, singular_index);
    assert!(expected_string == asserted_string);

}

#[test]
fn singular_rules_regexes_should_return_27_regex() {
    let regexes: [Regex; 27] = singular_rules_regexes();
    assert!(regexes.len() == 27);
}
<commit_msg>Mirrored pluralize with singularize<commit_after>use regex::Regex;
use regex::Captures;
use string::constants::UNACCONTABLE_WORDS;
/// Converts a `String` to singularized `String`
///
/// #Examples
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "foo_bars".to_string();
///     let expected_string: String = "foo_bar".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "oxen".to_string();
///     let expected_string: String = "ox".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "crates".to_string();
///     let expected_string: String = "crate".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "boxes".to_string();
///     let expected_string: String = "box".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "vengeance".to_string();
///     let expected_string: String = "vengeance".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "yoga".to_string();
///     let expected_string: String = "yoga".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
///
macro_rules! special_cases{
    ($s:ident, $($singular: expr => $plural:expr), *) => {
        match &$s[..] {
            $(
                $singular => {
                    return $plural.to_string();
                },
            )*
            _ => ()
        }
    }
}
pub fn to_singular(non_singular_string: String) -> String {
    if UNACCONTABLE_WORDS.contains(&non_singular_string.as_ref()) {
        non_singular_string
    } else {
        special_cases![non_singular_string,
            "oxen" => "ox",
            "men" => "man",
            "women" => "woman",
            "dice" => "die",
            "yeses" => "yes",
            "feet" => "foot",
            "eaves" => "eave",
            "geese" => "goose",
            "teeth" => "tooth",
            "quizzes" => "quiz"
        ];
        for &(ref rule, replace) in RULES.iter().rev() {
            if let Some(c) = rule.captures(&non_singular_string) {
                if let Some(c) = c.at(1) {
                    return format!("{}{}", c, replace);
                }
            }
        }

        format!("{}", non_singular_string)
    }
}

macro_rules! add_rule{
    ($r:ident, $rule:expr => $replace:expr) => {
        $r.push((Regex::new($rule).unwrap(), $replace));
    }
}

macro_rules! rules{
    ($r:ident; $($rule:expr => $replace:expr), *) => {
        $(
            add_rule!{$r, $rule => $replace}
        )*
    }
}

lazy_static!{
    static ref RULES: Vec<(Regex, &'static str)> = {
    let mut r = Vec::new();
    rules![r;
     r"(n)ews$" => "ews",
     r"(\w*)(o)es$" => "",
     r"(\w*)([ti])a$" => "um",
     r"((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)(sis|ses)$" => "sis",
     r"(^analy)(sis|ses)$" => "sis",
     r"(\w*)([^f])ves$" => "fe",
     r"(\w*)(hive)s$" => "",
     r"(\w*)(tive)s$" => "",
     r"(\w*)([lr])ves$" => "f",
     r"(\w*)([^aeiouy]|qu)ies$" => "y",
     r"(s)eries$" => "eries",
     r"(m)ovies$" => "ovie",
     r"(\w*)(x|ch|ss|sh)es$" => "",
     r"(m|l)ice$" => "ouse",
     r"(bus)(es)?$" => "",
     r"(shoe)s$" => "",
     r"(cris|test)(is|es)$" => "is",
     r"^(a)x[ie]s$" => "xis",
     r"(octop|vir)(us|i)$" => "us",
     r"(alias|status)(es)?$" => "",
     r"^(ox)en" => "",
     r"(vert|ind)ices$" => "ex",
     r"(matr)ices$" => "ix",
     r"(quiz)zes$" => "",
     r"(database)s$" => "",
     r"(\w*)s$" => "",
     r"(\w*)(ss)$" => ""
         ];
     r
    };
}

#[test]
fn singularize_string_if_a_regex_will_match() {
    let expected_string: String = "ox".to_string();
    let asserted_string: String = to_singular("oxen".to_string());
    assert!(expected_string == asserted_string);

}

#[test]
fn singularize_string_returns_none_option_if_no_match() {
    let expected_string: String = "bacon".to_string();
    let asserted_string: String = to_singular("bacon".to_string());

    assert!(expected_string == asserted_string);

}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use super::SubTok;

use std::char::{to_digit, from_u32};

mod data;

pub struct CharRef {
    chars: [char, ..2],
    num_chars: u8,
    parse_error: bool,
}

pub enum Status {
    Stuck,
    Progress,
    Done,
}

enum State {
    Begin,
    Octothorpe,
    Numeric(u32), // base
    NumericSemicolon,
    Named,
}

pub struct CharRefTokenizer {
    priv state: State,
    priv addnl_allowed: Option<char>,
    priv result: Option<CharRef>,

    priv num: u32,
    priv num_too_big: bool,
    priv seen_digit: bool,
    priv hex_marker: Option<char>,

    priv name_buf_opt: Option<~str>,
    priv name_match: Option<&'static [char, ..2]>,
    priv name_len: uint,
}

impl CharRefTokenizer {
    pub fn new(addnl_allowed: Option<char>) -> CharRefTokenizer {
        CharRefTokenizer {
            state: Begin,
            addnl_allowed: addnl_allowed,
            result: None,
            num: 0,
            num_too_big: false,
            seen_digit: false,
            hex_marker: None,
            name_buf_opt: None,
            name_match: None,
            name_len: 0,
        }
    }

    pub fn step<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        if self.result.is_some() {
            return Done;
        }

        match self.state {
            Begin => self.do_begin(tokenizer),
            Octothorpe => self.do_octothorpe(tokenizer),
            Numeric(base) => self.do_numeric(tokenizer, base),
            NumericSemicolon => self.do_numeric_semicolon(tokenizer),
            Named => self.do_named(tokenizer),
        }
    }

    // A CharRefTokenizer can only tokenize one character reference,
    // so this method consumes the tokenizer.
    pub fn get_result(self) -> CharRef {
        self.result.expect("get_result called before done")
    }

    fn name_buf<'t>(&'t mut self) -> &'t mut ~str {
        self.name_buf_opt.as_mut()
            .expect("name_buf missing in named character reference")
    }

    fn finish_none(&mut self, error: bool) -> Status {
        self.result = Some(CharRef {
            chars: ['\0', '\0'],
            num_chars: 0,
            parse_error: error,
        });
        Done
    }

    fn finish_one(&mut self, c: char, error: bool) -> Status {
        self.result = Some(CharRef {
            chars: [c, '\0'],
            num_chars: 1,
            parse_error: error,
        });
        Done
    }

    fn do_begin<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        match unwrap_or_return!(tokenizer.peek(), Stuck) {
            '\t' | '\n' | '\x0C' | ' ' | '<' | '&'
                => self.finish_none(false),
            c if Some(c) == self.addnl_allowed
                => self.finish_none(false),

            '#' => {
                tokenizer.discard_char();
                self.state = Octothorpe;
                Progress
            }

            _ => {
                self.state = Named;
                self.name_buf_opt = Some(~"");
                Progress
            }
        }
    }

    fn do_octothorpe<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        let c = unwrap_or_return!(tokenizer.peek(), Stuck);
        match c {
            'x' | 'X' => {
                tokenizer.discard_char();
                self.hex_marker = Some(c);
                self.state = Numeric(16);
            }

            _ => {
                self.hex_marker = None;
                self.state = Numeric(10);
            }
        }
        Progress
    }

    fn do_numeric<T: SubTok>(&mut self, tokenizer: &mut T, base: u32) -> Status {
        let c = unwrap_or_return!(tokenizer.peek(), Stuck);
        match to_digit(c, base as uint) {
            Some(n) => {
                tokenizer.discard_char();
                self.num *= base;
                if self.num > 0x10FFFF {
                    // We might overflow, and the character is definitely invalid.
                    // We still parse digits and semicolon, but don't use the result.
                    self.num_too_big = true;
                }
                self.num += n as u32;
                self.seen_digit = true;
                Progress
            }

            None if !self.seen_digit => {
                let mut unconsume = ~"#";
                match self.hex_marker {
                    Some(c) => unconsume.push_char(c),
                    None => (),
                }

                tokenizer.unconsume(unconsume);
                self.finish_none(true)
            }

            None => {
                self.state = NumericSemicolon;
                Progress
            }
        }
    }

    fn do_numeric_semicolon<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        fn conv(n: u32) -> char {
            from_u32(n).expect("invalid char missed by error handling cases")
        }

        let semi_missing = match unwrap_or_return!(tokenizer.peek(), Stuck) {
            ';' => { tokenizer.discard_char(); false }
            _   => true
        };

        match self.num {
            n if (n > 0x10FFFF) || self.num_too_big => self.finish_one('\ufffd', true),
            0x00 | 0xD800..0xDFFF => self.finish_one('\ufffd', true),

            0x80..0x9F => match data::c1_replacements[self.num - 0x80] {
                Some(c) => self.finish_one(c, true),
                None => self.finish_one(conv(self.num), semi_missing),
            },

            0x01..0x08 | 0x0D..0x1F | 0x7F..0x9F | 0xFDD0..0xFDEF | 0x0B
                => self.finish_one(conv(self.num), true),

            n if (n & 0xFFFE) == 0xFFFE
                => self.finish_one(conv(n), true),

            n => self.finish_one(conv(n), semi_missing),
        }
    }

    fn do_named<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        let c = unwrap_or_return!(tokenizer.peek(), Stuck);
        tokenizer.discard_char();
        self.name_buf().push_char(c);
        match data::named_entities.find(&self.name_buf().as_slice()) {
            // We have either a full match or a prefix of one.
            Some(m) => {
                if m[0] != '\0' {
                    // We have a full match, but there might be a longer one to come.
                    self.name_match = Some(m);
                    self.name_len = self.name_buf().len();
                }
                // Otherwise we just have a prefix match.
                Progress
            }

            // Can't continue the match.
            None => match self.name_match {
                None => {
                    // FIXME: "if the characters after the U+0026 AMPERSAND
                    // character (&) consist of a sequence of one or more
                    // alphanumeric ASCII characters followed by a U+003B
                    // SEMICOLON character (;), then this is a parse error".

                    tokenizer.unconsume(self.name_buf_opt.take_unwrap());
                    self.finish_none(false)
                }
                Some(m) => {
                    // We have a complete match, but we've consumed at least
                    // one additional character into self.name_buf, and more
                    // in cases like
                    //
                    //     &not    => match for U+00AC
                    //     &noti   => valid prefix for &notin
                    //     &notit  => can't continue match

                    assert!(self.name_len > 0);
                    assert!(self.name_len < self.name_buf().len());
                    tokenizer.unconsume(self.name_buf().slice_from(self.name_len).to_owned());
                    let missing_semi = ';' != self.name_buf().char_at(self.name_len-1);

                    self.result = Some(CharRef {
                        chars: *m,
                        num_chars: if m[1] == '\0' { 1 } else { 2 },
                        parse_error: missing_semi,
                    });
                    Done
                }
            }
        }
    }
}
<commit_msg>Correctly flag an error for char refs in 0x80..0x9F<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use super::SubTok;

use std::char::{to_digit, from_u32};

mod data;

pub struct CharRef {
    chars: [char, ..2],
    num_chars: u8,
    parse_error: bool,
}

pub enum Status {
    Stuck,
    Progress,
    Done,
}

enum State {
    Begin,
    Octothorpe,
    Numeric(u32), // base
    NumericSemicolon,
    Named,
}

pub struct CharRefTokenizer {
    priv state: State,
    priv addnl_allowed: Option<char>,
    priv result: Option<CharRef>,

    priv num: u32,
    priv num_too_big: bool,
    priv seen_digit: bool,
    priv hex_marker: Option<char>,

    priv name_buf_opt: Option<~str>,
    priv name_match: Option<&'static [char, ..2]>,
    priv name_len: uint,
}

impl CharRefTokenizer {
    pub fn new(addnl_allowed: Option<char>) -> CharRefTokenizer {
        CharRefTokenizer {
            state: Begin,
            addnl_allowed: addnl_allowed,
            result: None,
            num: 0,
            num_too_big: false,
            seen_digit: false,
            hex_marker: None,
            name_buf_opt: None,
            name_match: None,
            name_len: 0,
        }
    }

    pub fn step<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        if self.result.is_some() {
            return Done;
        }

        match self.state {
            Begin => self.do_begin(tokenizer),
            Octothorpe => self.do_octothorpe(tokenizer),
            Numeric(base) => self.do_numeric(tokenizer, base),
            NumericSemicolon => self.do_numeric_semicolon(tokenizer),
            Named => self.do_named(tokenizer),
        }
    }

    // A CharRefTokenizer can only tokenize one character reference,
    // so this method consumes the tokenizer.
    pub fn get_result(self) -> CharRef {
        self.result.expect("get_result called before done")
    }

    fn name_buf<'t>(&'t mut self) -> &'t mut ~str {
        self.name_buf_opt.as_mut()
            .expect("name_buf missing in named character reference")
    }

    fn finish_none(&mut self, error: bool) -> Status {
        self.result = Some(CharRef {
            chars: ['\0', '\0'],
            num_chars: 0,
            parse_error: error,
        });
        Done
    }

    fn finish_one(&mut self, c: char, error: bool) -> Status {
        self.result = Some(CharRef {
            chars: [c, '\0'],
            num_chars: 1,
            parse_error: error,
        });
        Done
    }

    fn do_begin<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        match unwrap_or_return!(tokenizer.peek(), Stuck) {
            '\t' | '\n' | '\x0C' | ' ' | '<' | '&'
                => self.finish_none(false),
            c if Some(c) == self.addnl_allowed
                => self.finish_none(false),

            '#' => {
                tokenizer.discard_char();
                self.state = Octothorpe;
                Progress
            }

            _ => {
                self.state = Named;
                self.name_buf_opt = Some(~"");
                Progress
            }
        }
    }

    fn do_octothorpe<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        let c = unwrap_or_return!(tokenizer.peek(), Stuck);
        match c {
            'x' | 'X' => {
                tokenizer.discard_char();
                self.hex_marker = Some(c);
                self.state = Numeric(16);
            }

            _ => {
                self.hex_marker = None;
                self.state = Numeric(10);
            }
        }
        Progress
    }

    fn do_numeric<T: SubTok>(&mut self, tokenizer: &mut T, base: u32) -> Status {
        let c = unwrap_or_return!(tokenizer.peek(), Stuck);
        match to_digit(c, base as uint) {
            Some(n) => {
                tokenizer.discard_char();
                self.num *= base;
                if self.num > 0x10FFFF {
                    // We might overflow, and the character is definitely invalid.
                    // We still parse digits and semicolon, but don't use the result.
                    self.num_too_big = true;
                }
                self.num += n as u32;
                self.seen_digit = true;
                Progress
            }

            None if !self.seen_digit => {
                let mut unconsume = ~"#";
                match self.hex_marker {
                    Some(c) => unconsume.push_char(c),
                    None => (),
                }

                tokenizer.unconsume(unconsume);
                self.finish_none(true)
            }

            None => {
                self.state = NumericSemicolon;
                Progress
            }
        }
    }

    fn do_numeric_semicolon<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        fn conv(n: u32) -> char {
            from_u32(n).expect("invalid char missed by error handling cases")
        }

        let semi_missing = match unwrap_or_return!(tokenizer.peek(), Stuck) {
            ';' => { tokenizer.discard_char(); false }
            _   => true
        };

        match self.num {
            n if (n > 0x10FFFF) || self.num_too_big => self.finish_one('\ufffd', true),
            0x00 | 0xD800..0xDFFF => self.finish_one('\ufffd', true),

            0x80..0x9F => match data::c1_replacements[self.num - 0x80] {
                Some(c) => self.finish_one(c, true),
                None => self.finish_one(conv(self.num), true),
            },

            0x01..0x08 | 0x0B | 0x0D..0x1F | 0x7F | 0xFDD0..0xFDEF
                => self.finish_one(conv(self.num), true),

            n if (n & 0xFFFE) == 0xFFFE
                => self.finish_one(conv(n), true),

            n => self.finish_one(conv(n), semi_missing),
        }
    }

    fn do_named<T: SubTok>(&mut self, tokenizer: &mut T) -> Status {
        let c = unwrap_or_return!(tokenizer.peek(), Stuck);
        tokenizer.discard_char();
        self.name_buf().push_char(c);
        match data::named_entities.find(&self.name_buf().as_slice()) {
            // We have either a full match or a prefix of one.
            Some(m) => {
                if m[0] != '\0' {
                    // We have a full match, but there might be a longer one to come.
                    self.name_match = Some(m);
                    self.name_len = self.name_buf().len();
                }
                // Otherwise we just have a prefix match.
                Progress
            }

            // Can't continue the match.
            None => match self.name_match {
                None => {
                    // FIXME: "if the characters after the U+0026 AMPERSAND
                    // character (&) consist of a sequence of one or more
                    // alphanumeric ASCII characters followed by a U+003B
                    // SEMICOLON character (;), then this is a parse error".

                    tokenizer.unconsume(self.name_buf_opt.take_unwrap());
                    self.finish_none(false)
                }
                Some(m) => {
                    // We have a complete match, but we've consumed at least
                    // one additional character into self.name_buf, and more
                    // in cases like
                    //
                    //     &not    => match for U+00AC
                    //     &noti   => valid prefix for &notin
                    //     &notit  => can't continue match

                    assert!(self.name_len > 0);
                    assert!(self.name_len < self.name_buf().len());
                    tokenizer.unconsume(self.name_buf().slice_from(self.name_len).to_owned());
                    let missing_semi = ';' != self.name_buf().char_at(self.name_len-1);

                    self.result = Some(CharRef {
                        chars: *m,
                        num_chars: if m[1] == '\0' { 1 } else { 2 },
                        parse_error: missing_semi,
                    });
                    Done
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>//! The `screener` module checks the inputstream before the lexer will get it

use std::error::Error;
use std::io::{BufReader,Cursor,Read};

/// checks the input if there is an bom, if true it will delete it
pub fn handle_bom_encoding<'a, R: Read>(input: &'a mut R) -> Cursor<Vec<u8>> {
    let mut reader = BufReader::new(input);
    let mut content = String::new();
    match reader.read_to_string(&mut content) {
        Err(why) => error!("Couldn't read {}", Error::description(&why)),
        Ok(_) => (),
    };

    let mut v: Vec<u8> = content.bytes().collect();
    if v.len() < 5 {
        error!("The file is to short for a valid twee File");
    }
    let has_bom = if &v[0..3] == [0xef, 0xbb, 0xbf] {
        true
    } else {
        false
    };
    if has_bom {
        debug!("File has Byte order mark");
        v.remove(0);
        v.remove(0);
        v.remove(0);
    }

    let cursor: Cursor<Vec<u8>> = Cursor::new(v);

    cursor
}<commit_msg>renamed variable<commit_after>//! The `screener` module checks the inputstream before the lexer will get it

use std::error::Error;
use std::io::{BufReader,Cursor,Read};

/// checks the input if there is an BOM, if true it will delete it
pub fn handle_bom_encoding<'a, R: Read>(input: &'a mut R) -> Cursor<Vec<u8>> {
    let mut reader = BufReader::new(input);
    let mut content = String::new();
    match reader.read_to_string(&mut content) {
        Err(why) => error!("Couldn't read {}", Error::description(&why)),
        Ok(_) => (),
    };

    let mut bytes: Vec<u8> = content.bytes().collect();
    if bytes.len() < 5 {
        error!("The file is to short for a valid twee File");
    }
    let has_bom = if &bytes[0..3] == [0xef, 0xbb, 0xbf] {
        true
    } else {
        false
    };
    if has_bom {
        debug!("File has Byte order mark");
        bytes.remove(0);
        bytes.remove(0);
        bytes.remove(0);
    }

    let cursor: Cursor<Vec<u8>> = Cursor::new(bytes);

    cursor
}<|endoftext|>


<commit_before>
  fn take_n(v: &r/[u8], n: uint) -> (&r/[u8], &r/[u8]) {
    (vec::view(v, 0, n), vec::view(v, n, v.len()))
  }

  fn take_atmost_n(v: &r/[u8], n: uint) -> (&r/[u8], &r/[u8]) {
    take_n(v, uint::min(n, v.len()))
  }

  fn take_head(v: &r/[u8]) -> (u8, &r/[u8]) {
    assert v.is_not_empty();
    (v.head(), vec::view(v, 1, v.len()))
  }

  // in_integer
  enum RedisType {
    TyNone,
    TyList,
    TyData,
    TyInt
  }

  enum Inside {
    in_nothing,
    in_number,
    in_number_digits_only,
    in_number_done,
    in_number_need_newline,
    in_data,
    //in_data_need_newline
  }

  //in_number(TYPE),
  //in_number_done(TYPE),
  //in_number_need_newline(TYPE)

  struct RedisState {
    mut in: Inside,
    mut typ: RedisType,
    mut data_size: uint,
    mut data_have: uint,
    mut number: uint,
    mut negative_number: bool,
    mut stack: ~[(uint, uint)] // (len, idx)
  }

  enum RedisResult {
    Finished,
    NeedMore,
    Error
  }

  // returns true if we are finished parsing a redis request
  fn element_done(st: &mut RedisState) -> bool {
    if st.stack.is_empty() {
      true
    }
    else {
      let (len, idx) = st.stack.last();
      assert idx+1 <= len;
      st.stack[st.stack.len()-1] = (len, idx+1);
      if idx + 1 == len {
        error!("Finsihed list");
        let _ = vec::pop(&mut st.stack);
        element_done(st)
      }
      else {
        false
      }
    }
  }

  trait Visitor {
    fn on_list(rec: int, len: uint);
    //fn on_list_elt(rec: int, len: uint);
  }

// keep recursion in visitor?
  struct MyVisitor {
    i: int
  }

  impl MyVisitor : Visitor {
    fn on_list(rec: int, len: uint) {
      error!("on_list(len: %u)", len)
    }
  }

  fn parse_redis(st: &mut RedisState, buf: &r/[u8]) -> (RedisResult, &r/[u8]) {
    let mut buf = buf;

    loop {
      match st.in {
        in_nothing => {
          if (buf.is_empty()) {break}

          let (c, b) = take_head(buf);
          buf = b;

          match c as char {
            '\r' | '\n' | ' ' =>  {
                error!("ignoring carriage return/new/whitespace line")
              }
            '*' => {
                st.typ = TyList;
                st.in = in_number;
                st.number = 0;
                st.negative_number = false;
              }
            '$' => {
                st.typ = TyData;
                st.in = in_number;
                st.number = 0;
                st.negative_number = false;
             }
            ':' => {
                st.typ = TyInt;
                st.in = in_number;
                st.number = 0;
                st.negative_number = false;
             }

             _ => {
                return (Error, buf) // XXX: return a slice including the character that failed
             }
          }

        }

        in_data => {
          if (buf.is_empty()) {break}
          let (data, b) = take_atmost_n(buf, st.data_size - st.data_have);
          buf = b;
          error!("GOT data: %?", data);
          st.data_have += data.len();
          assert st.data_have <= st.data_size;
          if st.data_have == st.data_size {
            error!("GOT DATA COMPLETE");
            //st.in = in_data_need_newline;
            st.in = in_nothing;
            if element_done(st) {
              return (Finished, buf)
            }
          }
        }

        // we could treat the newline after data differently.
/*
        in_data_need_newline => {
          if (buf.is_empty()) {break}
          let (c, b) = take_head(buf);
          buf = b;
          if c == ('\n' as u8) {
            st.in = in_nothing;
          }
          else if c == ('\r' as u8) || c == (' ' as u8) {
            error("Consume whitespace");
          }
          else {
            return (Error, buf)
          }
        }
*/

        in_number_need_newline => {
          if (buf.is_empty()) {break}
          let (c, b) = take_head(buf);
          buf = b;
          if c == ('\n' as u8) {
            st.in = in_number_done;
          }
          else {
            return (Error, buf)
          }
        }

        // XXX: make a function instead of a STATE
        in_number_done => {
          match st.typ {
            TyData => {
              if st.negative_number {
                if st.number == 1 {
                  error!("GOT NIL VALUE");
                  st.in = in_nothing;
                  if element_done(st) {
                    return (Finished, buf)
                  }
                }
                else {
                  return (Error, buf)
                }
              }
              else {
                st.data_size = st.number;
                st.data_have = 0;
                st.in = in_data;
              }
            }
            TyList => {
              // XXX:
              // push current recursion level and index on stack
              st.in = in_nothing;
              if st.negative_number {
                if (st.number == 1) {
                  // NIL
                  if element_done(st) {
                    return (Finished, buf)
                  }
                }
                else {
                  return (Error, buf)
                }
              }
              else {
                if st.number > 0 {
                  vec::push(&mut st.stack, (st.number, 0));
                }
                else {
                  error!("GOT EMPTY LIST");
                  if element_done(st) {
                    return (Finished, buf)
                  }
                }
              }
            }

            TyInt => {
              error!("GOT INTEGER: %?", st.number);
              st.in = in_nothing;
              if element_done(st) {
                return (Finished, buf)
              }
            }
            _ => {
              fail ~"THIS SHOULD NEVER HAPPEN"
            }
          }
        }

        in_number | in_number_digits_only => {
          if (buf.is_empty()) {break}
          let (c, b) = take_head(buf);
          buf = b;

          if c >= ('0' as u8) && c <= ('9' as u8) {
            st.number *= 10;
            st.number += (c - ('0' as u8)) as uint;
            error!("number: %?", st.number);
            st.in = in_number_digits_only;
          }
          else if c as char == '-' {
            error!("NEGATIVE NUMBER");
            match st.in {
              in_number_digits_only => {
                return (Error, buf)
              }
              _ => {
                st.negative_number = true;
                st.in = in_number_digits_only;
              }
            }
          }
          else if c as char == '\r' || c as char == ' ' {
            error!("number need newline");
            st.in = in_number_need_newline;
          }
          else if c as char == '\n' {
            error!("number finsihed");
            st.in = in_number_done;
          }
          else {
            return (Error, buf)
          }

        }
      }
    }
    (NeedMore, buf)
  }

fn main() {

  let mut st = RedisState {
    in: in_nothing,
    typ: TyNone,
    data_size: 0,
    data_have: 0,
    number: 0,
    negative_number: false,
    stack: ~[]
  };

  error!("%?", st);

  let s = ~"*4\r\n$3\r\nabc\r\n:123\n:1\n$-1\n";
  do str::as_bytes(&s) |v| {
    let x = parse_redis(&mut st, vec::view(*v, 0, (*v).len() - 1));
    error!("%?", st);
    error!("%?", x);
  }

}
<commit_msg>Introduce visitor<commit_after>
  fn take_n(v: &r/[u8], n: uint) -> (&r/[u8], &r/[u8]) {
    (vec::view(v, 0, n), vec::view(v, n, v.len()))
  }

  fn take_atmost_n(v: &r/[u8], n: uint) -> (&r/[u8], &r/[u8]) {
    take_n(v, uint::min(n, v.len()))
  }

  fn take_head(v: &r/[u8]) -> (u8, &r/[u8]) {
    assert v.is_not_empty();
    (v.head(), vec::view(v, 1, v.len()))
  }

  // in_integer
  enum RedisType {
    TyNone,
    TyList,
    TyData,
    TyInt
  }

  enum Inside {
    in_nothing,
    in_number,
    in_number_digits_only,
    in_number_done,
    in_number_need_newline,
    in_data,
    //in_data_need_newline
  }

  struct RedisState {
    mut in: Inside,
    mut typ: RedisType,
    mut data_size: uint,
    mut data_have: uint,
    mut number: uint,
    mut negative_number: bool,
  }

  enum RedisResult {
    Finished,
    NeedMore,
    Error
  }

  // Return
  trait Visitor {
    fn on_list(&mut self, len: uint) -> bool;

    fn on_data_beg(&mut self, len: uint);
    fn on_data(&mut self, data: &[u8]);
    fn on_data_end(&mut self) -> bool;

    fn on_integer(&mut self, num: uint, sign: bool) -> bool;

    fn on_nil(&mut self) -> bool;
  }

// keep recursion in visitor?
  struct MyVisitor {
    mut stack: ~[(uint, uint)] // (len, idx)
  }

  // returns true if we are finished parsing a redis request
  fn element_done(visitor: &mut MyVisitor) -> bool {
    if visitor.stack.is_empty() {
      true
    }
    else {
      let (len, idx) = visitor.stack.last();
      assert idx+1 <= len;
      visitor.stack[visitor.stack.len()-1] = (len, idx+1);
      if idx + 1 == len {
        error!("Finsihed list");
        let _ = vec::pop(&mut visitor.stack);
        element_done(visitor)
      }
      else {
        false
      }
    }
  }

  impl MyVisitor : Visitor {
    fn on_list(&mut self, len: uint) -> bool {
      error!("on_list(len: %u)", len);
      if len == 0 {
        element_done(self)
      }
      else {
        vec::push(&mut self.stack, (len, 0));
        false
      }
    }

    fn on_data_beg(&mut self, len: uint) {
      error!("on_data_beg(len: %u)", len)
    }

    fn on_data(&mut self, data: &[u8]) {
      error!("on_data(%?)", data)
    }

    fn on_data_end(&mut self) -> bool {
      error!("on_data_end()");
      element_done(self)
    }

    fn on_integer(&mut self, num: uint, sign: bool) -> bool {
      error!("on_integer(%?, %?)", num, sign);
      element_done(self)
    }

    fn on_nil(&mut self) -> bool {
      error!("on_nil");
      element_done(self)
    }
  }

  fn parse_redis<V: Visitor>(st: &mut RedisState, buf: &r/[u8], visitor: &mut V) -> (RedisResult, &r/[u8]) {
    let mut buf = buf;

    loop {
      match st.in {
        in_nothing => {
          if (buf.is_empty()) {break}

          let (c, b) = take_head(buf);
          buf = b;

          match c as char {
            '\r' | '\n' | ' ' =>  {
                error!("ignoring carriage return/new/whitespace line")
              }
            '*' => {
                st.typ = TyList;
                st.in = in_number;
                st.number = 0;
                st.negative_number = false;
              }
            '$' => {
                st.typ = TyData;
                st.in = in_number;
                st.number = 0;
                st.negative_number = false;
             }
            ':' => {
                st.typ = TyInt;
                st.in = in_number;
                st.number = 0;
                st.negative_number = false;
             }

             _ => {
                return (Error, buf) // XXX: return a slice including the character that failed
             }
          }

        }

        in_data => {
          if (buf.is_empty()) {break}
          let (data, b) = take_atmost_n(buf, st.data_size - st.data_have);
          buf = b;
          visitor.on_data(data);
          st.data_have += data.len();
          assert st.data_have <= st.data_size;
          if st.data_have == st.data_size {
            //XXX: st.in = in_data_need_newline;
            st.in = in_nothing;
            if visitor.on_data_end() {
              return (Finished, buf)
            }
          }
        }

        // we could treat the newline after data differently.
/*
        in_data_need_newline => {
          if (buf.is_empty()) {break}
          let (c, b) = take_head(buf);
          buf = b;
          if c == ('\n' as u8) {
            st.in = in_nothing;
          }
          else if c == ('\r' as u8) || c == (' ' as u8) {
            error("Consume whitespace");
          }
          else {
            return (Error, buf)
          }
        }
*/

        in_number_need_newline => {
          if (buf.is_empty()) {break}
          let (c, b) = take_head(buf);
          buf = b;
          if c == ('\n' as u8) {
            st.in = in_number_done;
          }
          else {
            return (Error, buf)
          }
        }

        // XXX: make a function instead of a STATE
        in_number_done => {
          match st.typ {
            TyData => {
              if st.negative_number {
                if st.number == 1 {
                  st.in = in_nothing;
                  if visitor.on_nil() {
                    return (Finished, buf)
                  }
                }
                else {
                  return (Error, buf)
                }
              }
              else {
                st.data_size = st.number;
                st.data_have = 0;
                st.in = in_data;
                visitor.on_data_beg(st.data_size);
              }
            }
            TyList => {
              // XXX:
              // push current recursion level and index on stack
              st.in = in_nothing;
              if st.negative_number {
                if (st.number == 1) {
                  if visitor.on_nil() {
                    return (Finished, buf)
                  }
                }
                else {
                  return (Error, buf)
                }
              }
              else {
                if visitor.on_list(st.number) {
                  return (Finished, buf)
                }
              }
            }

            TyInt => {
              st.in = in_nothing;
              if visitor.on_integer(st.number, st.negative_number) {
                return (Finished, buf)
              }
            }
            _ => {
              fail ~"THIS SHOULD NEVER HAPPEN"
            }
          }
        }

        in_number | in_number_digits_only => {
          if (buf.is_empty()) {break}
          let (c, b) = take_head(buf);
          buf = b;

          if c >= ('0' as u8) && c <= ('9' as u8) {
            st.number *= 10;
            st.number += (c - ('0' as u8)) as uint;
            st.in = in_number_digits_only;
          }
          else if c as char == '-' {
            match st.in {
              in_number_digits_only => {
                return (Error, buf)
              }
              _ => {
                st.negative_number = true;
                st.in = in_number_digits_only;
              }
            }
          }
          else if c as char == '\r' || c as char == ' ' {
            st.in = in_number_need_newline;
          }
          else if c as char == '\n' {
            st.in = in_number_done;
          }
          else {
            return (Error, buf)
          }

        }
      }
    }
    (NeedMore, buf)
  }

fn main() {

  let mut st = RedisState {
    in: in_nothing,
    typ: TyNone,
    data_size: 0,
    data_have: 0,
    number: 0,
    negative_number: false,
  };

  error!("%?", st);

  let mut visitor = MyVisitor {stack: ~[]};

  let s = ~"*4\r\n$3\r\nabc\r\n:123\n:1\n$-1\n";
  do str::as_bytes(&s) |v| {
    let x = parse_redis(&mut st, vec::view(*v, 0, (*v).len() - 1), &mut visitor);
    error!("%?", st);
    error!("%?", x);
  }

}
<|endoftext|>


<commit_before>//
// rslex - a lexer generator for rust
//
// parser.rs
// Input file parsing
//
// Andrei de A. Formiga, 2013-08-04
//

extern mod std;

// --- utility functions -------------------------------------------

#[inline]
fn is_eof(c: char) -> bool {
    c == (-1 as char)
}

// --- the lookahead buffer ----------------------------------------
struct LookaheadBuffer {
    c: char,
    valid: bool,
    reader: @Reader
}

impl LookaheadBuffer {
    fn new(r: @Reader) -> LookaheadBuffer {
        LookaheadBuffer { c: ' ', valid: false, reader: r}
    }

    // FIX: should be read_char but doesn't work due to method resolution bug in rustc
    fn next_char(&mut self) -> char {
        if self.valid {
            self.valid = false;
            self.c
        }
        else {
            self.reader.read_char()
        }
    }

    fn return_char(&mut self, c: char) {
        // assumes at most one char of lookahead
        assert!(self.valid == false);
        self.c = c;
        self.valid = true;
    }
}

#[cfg(test)]
mod buffer_tests {
    use super::*;
    use std::io::with_str_reader;

    #[test]
    fn no_buffer_use() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(!buffer.valid);
    }

    #[test]
    fn buffer_use() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        buffer.return_char('a');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'a');
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
    }

    #[test]
    fn return_other_char() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        buffer.return_char('x');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'x');
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(!buffer.valid);
        buffer.return_char('c');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'c');
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'd');
        assert!(buffer.next_char() == 'e');
    }

    #[test]
    #[should_fail]
    fn return_other_char_expect_same() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        buffer.return_char('x');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'a');
    }

    #[test]
    fn read_until_eof() {
        let mut buffer = with_str_reader("abc", LookaheadBuffer::new);
        assert!(buffer.next_char() == 'a');
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(buffer.next_char() == -1 as char);
    }

    #[test]
    fn return_char_eof() {
        let mut buffer = with_str_reader("abc", LookaheadBuffer::new);
        assert!(buffer.next_char() == 'a');
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(buffer.next_char() == -1 as char);
        buffer.return_char(-1 as char);
        assert!(buffer.valid);
        assert!(buffer.next_char() == -1 as char);
    }
}

fn skip_whitespace(buffer: &mut LookaheadBuffer) {
    let mut c = buffer.next_char();
    while (!is_eof(c) && std::char::is_whitespace(c)) {
        c = buffer.next_char();
    }
    buffer.return_char(c);
}

#[test]
fn test_skip_ws() {
    let mut buffer = std::io::with_str_reader("   abc   ", LookaheadBuffer::new);
    skip_whitespace(&mut buffer);
    assert!(buffer.next_char() == 'a');
    skip_whitespace(&mut buffer);
    assert!(buffer.next_char() == 'b');
    skip_whitespace(&mut buffer);
    assert!(buffer.next_char() == 'c');
    skip_whitespace(&mut buffer);
    assert!(is_eof(buffer.next_char()));
}

// token types
enum Token { LBrace, RBrace, Equals, Comma, Semicolon,
             LParen, RParen, Star, Plus, Bar,
             Defs, Rules, Code, Id(~str), RegExp }

// fn match_next_token() -> bool {
// }

// fn get_next_token(reader: @Reader) -> ~Token {
// }

// fn parse_regexp() {
// }

pub fn parse_lexer_spec(reader: @Reader) {
    let c : char = reader.read_char();
    println("Parsing")
}

<commit_msg>Some parsing.<commit_after>//
// rslex - a lexer generator for rust
//
// parser.rs
// Input file parsing
//
// Andrei de A. Formiga, 2013-08-04
//

extern mod std;

// --- utility functions -------------------------------------------

#[inline]
fn is_eof(c: char) -> bool {
    c == (-1 as char)
}

// --- the lookahead buffer ----------------------------------------
struct LookaheadBuffer {
    c: char,
    valid: bool,
    reader: @Reader
}

impl LookaheadBuffer {
    fn new(r: @Reader) -> LookaheadBuffer {
        LookaheadBuffer { c: ' ', valid: false, reader: r}
    }

    // FIX: should be read_char but doesn't work due to method resolution bug in rustc
    fn next_char(&mut self) -> char {
        if self.valid {
            self.valid = false;
            self.c
        }
        else {
            self.reader.read_char()
        }
    }

    fn return_char(&mut self, c: char) {
        // assumes at most one char of lookahead
        assert!(self.valid == false);
        self.c = c;
        self.valid = true;
    }
}

#[cfg(test)]
mod buffer_tests {
    use super::*;
    use std::io::with_str_reader;

    #[test]
    fn no_buffer_use() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(!buffer.valid);
    }

    #[test]
    fn buffer_use() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        buffer.return_char('a');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'a');
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
    }

    #[test]
    fn return_other_char() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        buffer.return_char('x');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'x');
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(!buffer.valid);
        buffer.return_char('c');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'c');
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'd');
        assert!(buffer.next_char() == 'e');
    }

    #[test]
    #[should_fail]
    fn return_other_char_expect_same() {
        let mut buffer = with_str_reader("abcdef", LookaheadBuffer::new);
        assert!(!buffer.valid);
        assert!(buffer.next_char() == 'a');
        buffer.return_char('x');
        assert!(buffer.valid);
        assert!(buffer.next_char() == 'a');
    }

    #[test]
    fn read_until_eof() {
        let mut buffer = with_str_reader("abc", LookaheadBuffer::new);
        assert!(buffer.next_char() == 'a');
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(buffer.next_char() == -1 as char);
    }

    #[test]
    fn return_char_eof() {
        let mut buffer = with_str_reader("abc", LookaheadBuffer::new);
        assert!(buffer.next_char() == 'a');
        assert!(buffer.next_char() == 'b');
        assert!(buffer.next_char() == 'c');
        assert!(buffer.next_char() == -1 as char);
        buffer.return_char(-1 as char);
        assert!(buffer.valid);
        assert!(buffer.next_char() == -1 as char);
    }
}

fn skip_whitespace(buffer: &mut LookaheadBuffer) {
    let mut c = buffer.next_char();
    while (!is_eof(c) && std::char::is_whitespace(c)) {
        c = buffer.next_char();
    }
    buffer.return_char(c);
}

#[test]
fn test_skip_ws() {
    let mut buffer = std::io::with_str_reader("   abc   ", LookaheadBuffer::new);
    skip_whitespace(&mut buffer);
    assert!(buffer.next_char() == 'a');
    skip_whitespace(&mut buffer);
    assert!(buffer.next_char() == 'b');
    skip_whitespace(&mut buffer);
    assert!(buffer.next_char() == 'c');
    skip_whitespace(&mut buffer);
    assert!(is_eof(buffer.next_char()));
}

// token types
enum Token { LBrace, RBrace, Equals, Comma, Semicolon,
             LParen, RParen, Star, Plus, Bar,
             DefsId, RulesId, CodeId, Id(~str), RegExp, Eof }

enum Block { Rules, Defs, Code }

// fn match_next_token() -> bool {
// }

fn get_next_token(buffer: &mut LookaheadBuffer) -> ~Token {
    let mut c = buffer.next_char();
    let mut done = false;
    let mut res = ~Eof;
    while !done {
        skip_whitespace(buffer);
        if is_eof(c) {
            done = true;
        }
        else {
            match c {
                '{' => { res = ~LBrace; done = true; },
                _   => { done = true; }   // FIX
            }
        }
    }
    res
}

fn parse_id(buffer: &mut LookaheadBuffer, first: char) -> ~str {
    let mut res : ~str = ~"";
    let mut c = buffer.next_char();
    res.push_char(first);
    while !is_eof(c) && !std::char::is_whitespace(c) {
        res.push_char(c);
        c = buffer.next_char();
    }
    buffer.return_char(c);
    res
}

#[test]
fn test_parse_id() {
    let mut buffer = std::io::with_str_reader("hombas   ", LookaheadBuffer::new);
    assert!(std::str::eq(&parse_id(&mut buffer, 's'), &~"shombas"));
}

fn parse_toplevel_block(buffer: &mut LookaheadBuffer) {
}

// fn parse_regexp() {
// }

pub fn parse_lexer_spec(reader: @Reader) {
    let mut buffer = LookaheadBuffer::new(reader);
    parse_toplevel_block(&mut buffer);
}

<|endoftext|>


<commit_before>use redox::*;

pub struct Editor {
    url: String,
    file: Option<File>,
    string: String,
    offset: usize,
    scroll_x: isize,
    scroll_y: isize,
}

impl Editor {
    #[inline(never)]
    pub fn new() -> Self {
        Editor {
            url: String::new(),
            file: None,
            string: String::new(),
            offset: 0,
            scroll_x: 0,
            scroll_y: 0,
        }
    }

    fn reload(&mut self) {
        self.offset = 0;
        self.scroll_x = 0;
        self.scroll_y = 0;

        match self.file {
            Some(ref mut file) => {
                file.seek(SeekFrom::Start(0));
                let mut string = String::new();
                file.read_to_string(&mut string);
                self.string = string;
            }
            None => self.string = String::new(),
        }
    }

    fn save(&mut self) {
        match self.file {
            Some(ref mut file) => {
                file.seek(SeekFrom::Start(0));
                file.write(&self.string.as_bytes());
                file.sync();
            }
            None => {
                //TODO: Ask for file to save to
            }
        }
    }

    fn draw_content(&mut self, window: &mut Window) {
        let mut redraw = false;

        {
            window.set([255, 255, 255, 255]);

            let scroll_x = self.scroll_x;
            let scroll_y = self.scroll_y;

            let mut offset = 0;

            let mut col = -scroll_x;
            let cols = window.width() as isize / 8;

            let mut row = -scroll_y;
            let rows = window.height() as isize / 16;

            for c in self.string.chars() {
                if offset == self.offset {
                    if col >= 0 && col < cols && row >= 0 && row < rows {
                        window.char(8 * col, 16 * row, '_', [128, 128, 128, 255]);
                    } else {
                        if col < 0 { //Too far to the left
                            self.scroll_x += col;
                        } else if col >= cols { //Too far to the right
                            self.scroll_x += cols - col + 1;
                        }
                        if row < 0 { //Too far up
                            self.scroll_y += row;
                        } else if row >= rows { //Too far down
                            self.scroll_y += rows - row + 1;
                        }

                        redraw = true;
                    }
                }

                if c == '\n' {
                    col = -scroll_x;
                    row += 1;
                } else if c == '\t' {
                    col += 8 - col % 8;
                } else {
                    if col >= 0 && col < cols && row >= 0 && row < rows {
                        window.char(8 * col, 16 * row, c, [0, 0, 0, 255]);
                    }
                    col += 1;
                }

                offset += 1;
            }

            if offset == self.offset {
                if col >= 0 && col < cols && row >= 0 && row < rows {
                    window.char(8 * col, 16 * row, '_', [128, 128, 128, 255]);
                } else {
                    if col < 0 { //Too far to the left
                        self.scroll_x += col;
                    } else if col >= cols { //Too far to the right
                        self.scroll_x += cols - col + 1;
                    }
                    if row < 0 { //Too far up
                        self.scroll_y += row;
                    } else if row >= rows { //Too far down
                        self.scroll_y += rows - row + 1;
                    }

                    redraw = true;
                }
            }

            window.sync();
        }

        if redraw {
            self.draw_content(window);
        }
    }

    fn main(&mut self, url: &str) {
        let mut window = Window::new((rand() % 400 + 50) as isize,
                                     (rand() % 300 + 50) as isize,
                                     576,
                                     400,
                                      &("Editor (".to_string() + url + ")")).unwrap();

        self.url = url.to_string();
        self.file = File::open(&self.url);

        self.reload();
        self.draw_content(&mut window);

        while let Some(event) = window.poll() {
            match event.to_option() {
                EventOption::Key(key_event) => {
                    if key_event.pressed {
                        match key_event.scancode {
                            K_ESC => break,
                            K_BKSP => if self.offset > 0 {
                                self.string = self.string[0 .. self.offset - 1].to_string() +
                                              &self.string[self.offset .. self.string.len()];
                                self.offset -= 1;
                            },
                            K_DEL => if self.offset < self.string.len() {
                                self.string = self.string[0 .. self.offset].to_string() +
                                              &self.string[self.offset + 1 .. self.string.len() - 1];
                            },
                            K_F5 => self.reload(),
                            K_F6 => self.save(),
                            K_HOME => self.offset = 0,
                            K_UP => {
                                let mut new_offset = 0;
                                for i in 2..self.offset {
                                    match self.string.as_bytes()[self.offset - i] {
                                        0 => break,
                                        10 => {
                                            new_offset = self.offset - i + 1;
                                            break;
                                        }
                                        _ => (),
                                    }
                                }
                                self.offset = new_offset;
                            }
                            K_LEFT => if self.offset > 0 {
                                self.offset -= 1;
                            },
                            K_RIGHT => if self.offset < self.string.len() {
                                self.offset += 1;
                            },
                            K_END => self.offset = self.string.len(),
                            K_DOWN => {
                                let mut new_offset = self.string.len();
                                for i in self.offset..self.string.len() {
                                    match self.string.as_bytes()[i] {
                                        0 => break,
                                        10 => {
                                            new_offset = i + 1;
                                            break;
                                        }
                                        _ => (),
                                    }
                                }
                                self.offset = new_offset;
                            }
                            _ => match key_event.character {
                                '\0' => (),
                                _ => {
                                    self.string = self.string[0 .. self.offset].to_string() +
                                                  &key_event.character.to_string() +
                                                  &self.string[self.offset .. self.string.len()];
                                    self.offset += 1;
                                }
                            },
                        }

                        self.draw_content(&mut window);
                    }
                }
                _ => (),
            }
        }
    }
}

pub fn main() {
    match args().get(1) {
        Some(arg) => Editor::new().main(&arg),
        None => Editor::new().main("none://"),
    }
}
<commit_msg>Started implementing Save window for editor as a freebie<commit_after>use redox::*;

pub struct Editor {
    url: String,
    file: Option<File>,
    string: String,
    offset: usize,
    scroll_x: isize,
    scroll_y: isize,
}

impl Editor {
    #[inline(never)]
    pub fn new() -> Self {
        Editor {
            url: String::new(),
            file: None,
            string: String::new(),
            offset: 0,
            scroll_x: 0,
            scroll_y: 0,
        }
    }

    fn reload(&mut self) {
        self.offset = 0;
        self.scroll_x = 0;
        self.scroll_y = 0;

        match self.file {
            Some(ref mut file) => {
                file.seek(SeekFrom::Start(0));
                let mut string = String::new();
                file.read_to_string(&mut string);
                self.string = string;
            }
            None => self.string = String::new(),
        }
    }

    fn save(&mut self, window: &Window) {
        match self.file {
            Some(ref mut file) => {
                file.seek(SeekFrom::Start(0));
                file.write(&self.string.as_bytes());
                file.sync();
            }
            None => {
                let mut save_window = {
                    const width: usize = 400;
                    const height: usize = 200;
                    Window::new((window.x() + (window.width()/2 - width/2) as isize),
                                (window.y() + (window.height()/2 - height/2) as isize),
                                width,
                                height,
                                "Save As").unwrap()
                };
                if let Some(event) = save_window.poll() {
                    //TODO: Create a Save/Cancel button for file saving
                    // and prompt the user for asking to save
                }
            }
        }
    }

    fn draw_content(&mut self, window: &mut Window) {
        let mut redraw = false;

        {
            window.set([255, 255, 255, 255]);

            let scroll_x = self.scroll_x;
            let scroll_y = self.scroll_y;

            let mut offset = 0;

            let mut col = -scroll_x;
            let cols = window.width() as isize / 8;

            let mut row = -scroll_y;
            let rows = window.height() as isize / 16;

            for c in self.string.chars() {
                if offset == self.offset {
                    if col >= 0 && col < cols && row >= 0 && row < rows {
                        window.char(8 * col, 16 * row, '_', [128, 128, 128, 255]);
                    } else {
                        if col < 0 { //Too far to the left
                            self.scroll_x += col;
                        } else if col >= cols { //Too far to the right
                            self.scroll_x += cols - col + 1;
                        }
                        if row < 0 { //Too far up
                            self.scroll_y += row;
                        } else if row >= rows { //Too far down
                            self.scroll_y += rows - row + 1;
                        }

                        redraw = true;
                    }
                }

                if c == '\n' {
                    col = -scroll_x;
                    row += 1;
                } else if c == '\t' {
                    col += 8 - col % 8;
                } else {
                    if col >= 0 && col < cols && row >= 0 && row < rows {
                        window.char(8 * col, 16 * row, c, [0, 0, 0, 255]);
                    }
                    col += 1;
                }

                offset += 1;
            }

            if offset == self.offset {
                if col >= 0 && col < cols && row >= 0 && row < rows {
                    window.char(8 * col, 16 * row, '_', [128, 128, 128, 255]);
                } else {
                    if col < 0 { //Too far to the left
                        self.scroll_x += col;
                    } else if col >= cols { //Too far to the right
                        self.scroll_x += cols - col + 1;
                    }
                    if row < 0 { //Too far up
                        self.scroll_y += row;
                    } else if row >= rows { //Too far down
                        self.scroll_y += rows - row + 1;
                    }

                    redraw = true;
                }
            }

            window.sync();
        }

        if redraw {
            self.draw_content(window);
        }
    }

    fn main(&mut self, url: &str) {
        let mut window = Window::new((rand() % 400 + 50) as isize,
                                     (rand() % 300 + 50) as isize,
                                     576,
                                     400,
                                     &("Editor (".to_string() + url + ")")).unwrap();

        self.url = url.to_string();
        self.file = File::open(&self.url);

        self.reload();
        self.draw_content(&mut window);

        while let Some(event) = window.poll() {
            match event.to_option() {
                EventOption::Key(key_event) => {
                    if key_event.pressed {
                        match key_event.scancode {
                            K_ESC => break,
                            K_BKSP => if self.offset > 0 {
                                self.string = self.string[0 .. self.offset - 1].to_string() +
                                              &self.string[self.offset .. self.string.len()];
                                self.offset -= 1;
                            },
                            K_DEL => if self.offset < self.string.len() {
                                self.string = self.string[0 .. self.offset].to_string() +
                                              &self.string[self.offset + 1 .. self.string.len() - 1];
                            },
                            K_F5 => self.reload(),
                            K_F6 => self.save(&window),
                            K_HOME => self.offset = 0,
                            K_UP => {
                                let mut new_offset = 0;
                                for i in 2..self.offset {
                                    match self.string.as_bytes()[self.offset - i] {
                                        0 => break,
                                        10 => {
                                            new_offset = self.offset - i + 1;
                                            break;
                                        }
                                        _ => (),
                                    }
                                }
                                self.offset = new_offset;
                            }
                            K_LEFT => if self.offset > 0 {
                                self.offset -= 1;
                            },
                            K_RIGHT => if self.offset < self.string.len() {
                                self.offset += 1;
                            },
                            K_END => self.offset = self.string.len(),
                            K_DOWN => {
                                let mut new_offset = self.string.len();
                                for i in self.offset..self.string.len() {
                                    match self.string.as_bytes()[i] {
                                        0 => break,
                                        10 => {
                                            new_offset = i + 1;
                                            break;
                                        }
                                        _ => (),
                                    }
                                }
                                self.offset = new_offset;
                            }
                            _ => match key_event.character {
                                '\0' => (),
                                _ => {
                                    self.string = self.string[0 .. self.offset].to_string() +
                                                  &key_event.character.to_string() +
                                                  &self.string[self.offset .. self.string.len()];
                                    self.offset += 1;
                                }
                            },
                        }

                        self.draw_content(&mut window);
                    }
                }
                _ => (),
            }
        }
    }
}

pub fn main() {
    match args().get(1) {
        Some(arg) => Editor::new().main(&arg),
        None => Editor::new().main("none://"),
    }
}
<|endoftext|>


<commit_before>#![feature(test)]
#![feature(conservative_impl_trait)]

#[macro_use]
extern crate slog;
extern crate slog_json;
extern crate slog_stream;

extern crate test;

use std::io;
use test::Bencher;
use slog::*;
use slog_stream::*;

const LONG_STRING : &'static str = "A long string that would take some time to allocate";

struct BlackBoxDrain;

impl Drain for BlackBoxDrain {
    type Error = ();
    fn log(&self, ri: &Record, o : &OwnedKeyValueList) -> std::result::Result<(), ()> {

        test::black_box((ri, o));
        Ok(())
    }
}

struct BlackBoxWriter;

impl io::Write for BlackBoxWriter {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        test::black_box(buf);
        Ok(buf.len())
    }

    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

fn empty_json_blackbox() -> impl Drain<Error=()> {
    stream(BlackBoxWriter, slog_json::build().build()).ignore_err()
}

fn json_blackbox() -> impl Drain<Error=()> {
    stream(BlackBoxWriter, slog_json::new()).ignore_err()
}

#[bench]
fn log_filter_out_empty(b: &mut Bencher) {
    let log = Logger::root(LevelFilter::new(BlackBoxDrain, Level::Warning), o!());

    b.iter(|| {
        info!(log, "");
    });
}

#[bench]
fn log_discard_empty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        info!(log, "");
    });
}

#[bench]
fn log_discard_nonempty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        info!(log, ""; "what" => "write");
    });
}

#[bench]
fn logger_clone_empty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        log.clone()
    });
}

#[bench]
fn logger_clone_nonempty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        log.clone()
    });
}

#[bench]
fn logger_new_empty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        log.new(o!())
    });
}

#[bench]
fn logger_new_nonempty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        log.new(o!("what" => "write"));
    });
}

#[bench]
fn log_discard_i32val(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        info!(log, ""; "i32" => 5);
    });
}

#[bench]
fn log_discard_i32closure(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        info!(log, ""; "i32" => |_:&Record|{5});
    });
}

#[bench]
fn log_stream_empty_json_blackbox_i32val(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, "";  "i32" => 5);
    });
}

#[bench]
fn log_stream_empty_json_blackbox_i32closure(b: &mut Bencher) {

    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => |_:&Record|{5});
    });
}

#[bench]
fn log_stream_empty_json_blackbox_i32pushclosure(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(5)
        }));
    });
}

#[bench]
fn log_stream_empty_json_blackbox_strclosure(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => |_:&Record| {
            String::from(LONG_STRING)
        });
    });
}

#[bench]
fn log_stream_empty_json_blackbox_strpushclosure(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(LONG_STRING)
        }));
    });
}

#[bench]
fn log_stream_json_blackbox_i32val(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, "";  "i32" => 5);
    });
}

#[bench]
fn log_stream_json_blackbox_i32closure(b: &mut Bencher) {

    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => |_:&Record|{5});
    });
}

#[bench]
fn log_stream_json_blackbox_i32pushclosure(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(5)
        }));
    });
}

#[bench]
fn log_stream_json_blackbox_strclosure(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => |_:&Record| {
            String::from(LONG_STRING)
        });
    });
}

#[bench]
fn log_stream_json_blackbox_strpushclosure(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(LONG_STRING)
        }));
    });
}

<commit_msg>feat(bench): More tests.<commit_after>#![feature(test)]
#![feature(conservative_impl_trait)]

#[macro_use]
extern crate slog;
extern crate slog_json;
extern crate slog_stream;

extern crate test;

use std::io;
use test::Bencher;
use slog::*;
use slog_stream::*;

const LONG_STRING : &'static str = "A long string that would take some time to allocate";

struct BlackBoxDrain;

impl Drain for BlackBoxDrain {
    type Error = ();
    fn log(&self, ri: &Record, o : &OwnedKeyValueList) -> std::result::Result<(), ()> {

        test::black_box((ri, o));
        Ok(())
    }
}

struct BlackBoxWriter;

impl io::Write for BlackBoxWriter {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        test::black_box(buf);
        Ok(buf.len())
    }

    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

fn empty_json_blackbox() -> impl Drain<Error=()> {
    stream(BlackBoxWriter, slog_json::build().build()).ignore_err()
}

fn json_blackbox() -> impl Drain<Error=()> {
    stream(BlackBoxWriter, slog_json::new()).ignore_err()
}

#[bench]
fn log_filter_out_empty(b: &mut Bencher) {
    let log = Logger::root(LevelFilter::new(BlackBoxDrain, Level::Warning), o!());

    b.iter(|| {
        info!(log, "");
    });
}

#[bench]
fn log_discard_empty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        info!(log, "");
    });
}

#[bench]
fn log_discard_nonempty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        info!(log, ""; "what" => "write");
    });
}

#[bench]
fn log_discard_nonempty_10(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        info!(log,
              "";
              "u8" => 0u8,
              "u16" => 0u16,
              "u32" => 0u32,
              "u64" => 0u64,
              "bool" => false,
              "str" => "",
              "f32" => 0f32,
              "f64" => 0f64,
              "option" => Some(0),
              "unit" => ()
             );
    });
}

#[bench]
fn logger_clone_empty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        log.clone()
    });
}

#[bench]
fn logger_clone_nonempty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        log.clone()
    });
}

#[bench]
fn logger_new_empty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        log.new(o!())
    });
}

#[bench]
fn logger_new_nonempty(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        log.new(o!("what" => "write"));
    });
}

#[bench]
fn logger_new_nonempty_10(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!("build" => "123456", "id" => 123456));

    b.iter(|| {
        log.new(o!(
                "u8" => 0u8,
                "u16" => 0u16,
                "u32" => 0u32,
                "u64" => 0u64,
                "bool" => false,
                "str" => "",
                "f32" => 0f32,
                "f64" => 0f64,
                "option" => Some(0),
                "unit" => ()
                ));
    });
}

#[bench]
fn log_discard_i32val(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        info!(log, ""; "i32" => 5);
    });
}

#[bench]
fn log_discard_i32closure(b: &mut Bencher) {
    let log = Logger::root(BlackBoxDrain, o!());

    b.iter(|| {
        info!(log, ""; "i32" => |_:&Record|{5});
    });
}

#[bench]
fn log_stream_empty_json_blackbox_i32val(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, "";  "i32" => 5);
    });
}

#[bench]
fn log_stream_empty_json_blackbox_i32closure(b: &mut Bencher) {

    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => |_:&Record|{5});
    });
}

#[bench]
fn log_stream_empty_json_blackbox_i32pushclosure(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(5)
        }));
    });
}

#[bench]
fn log_stream_empty_json_blackbox_strclosure(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => |_:&Record| {
            String::from(LONG_STRING)
        });
    });
}

#[bench]
fn log_stream_empty_json_blackbox_strpushclosure(b: &mut Bencher) {
    let log = Logger::root(empty_json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(LONG_STRING)
        }));
    });
}

#[bench]
fn log_stream_json_blackbox_i32val(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, "";  "i32" => 5);
    });
}

#[bench]
fn log_stream_json_blackbox_i32closure(b: &mut Bencher) {

    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => |_:&Record|{5});
    });
}

#[bench]
fn log_stream_json_blackbox_i32pushclosure(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "i32" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(5)
        }));
    });
}

#[bench]
fn log_stream_json_blackbox_strclosure(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => |_:&Record| {
            String::from(LONG_STRING)
        });
    });
}

#[bench]
fn log_stream_json_blackbox_strpushclosure(b: &mut Bencher) {
    let log = Logger::root(json_blackbox(), o!());

    b.iter(|| {
        info!(log, ""; "str" => PushLazy(|_:&Record, ser : ValueSerializer|{
            ser.serialize(LONG_STRING)
        }));
    });
}

<|endoftext|>


<commit_before>use crate::NewlineStyle;

/// Apply this newline style to the formatted text. When the style is set
/// to `Auto`, the `raw_input_text` is used to detect the existing line
/// endings.
///
/// If the style is set to `Auto` and `raw_input_text` contains no
/// newlines, the `Native` style will be used.
pub(crate) fn apply_newline_style(
    newline_style: NewlineStyle,
    formatted_text: &mut String,
    raw_input_text: &str,
) {
    match effective_newline_style(newline_style, raw_input_text) {
        EffectiveNewlineStyle::Windows => {
            *formatted_text = convert_to_windows_newlines(formatted_text);
        }
        EffectiveNewlineStyle::Unix => {}
    }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum EffectiveNewlineStyle {
    Windows,
    Unix,
}

fn effective_newline_style(
    newline_style: NewlineStyle,
    raw_input_text: &str,
) -> EffectiveNewlineStyle {
    match newline_style {
        NewlineStyle::Auto => auto_detect_newline_style(raw_input_text),
        NewlineStyle::Native => native_newline_style(),
        NewlineStyle::Windows => EffectiveNewlineStyle::Windows,
        NewlineStyle::Unix => EffectiveNewlineStyle::Unix,
    }
}

const LINE_FEED: char = '\n';
const CARRIAGE_RETURN: char = '\r';

fn auto_detect_newline_style(raw_input_text: &str) -> EffectiveNewlineStyle {
    if let Some(pos) = raw_input_text.chars().position(|ch| ch == LINE_FEED) {
        let pos = pos.saturating_sub(1);
        if let Some(CARRIAGE_RETURN) = raw_input_text.chars().nth(pos) {
            EffectiveNewlineStyle::Windows
        } else {
            EffectiveNewlineStyle::Unix
        }
    } else {
        native_newline_style()
    }
}

fn native_newline_style() -> EffectiveNewlineStyle {
    if cfg!(windows) {
        EffectiveNewlineStyle::Windows
    } else {
        EffectiveNewlineStyle::Unix
    }
}

fn convert_to_windows_newlines(formatted_text: &String) -> String {
    let mut transformed = String::with_capacity(2 * formatted_text.capacity());
    for c in formatted_text.chars() {
        const WINDOWS_NEWLINE: &str = "\r\n";
        match c {
            LINE_FEED => transformed.push_str(WINDOWS_NEWLINE),
            CARRIAGE_RETURN => continue,
            c => transformed.push(c),
        }
    }
    transformed
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn auto_detects_unix_newlines() {
        assert_eq!(
            EffectiveNewlineStyle::Unix,
            auto_detect_newline_style("One\nTwo\nThree")
        );
    }

    #[test]
    fn auto_detects_windows_newlines() {
        assert_eq!(
            EffectiveNewlineStyle::Windows,
            auto_detect_newline_style("One\r\nTwo\r\nThree")
        );
    }

    #[test]
    fn auto_detects_windows_newlines_with_multibyte_char_on_first_line() {
        assert_eq!(
            EffectiveNewlineStyle::Windows,
            auto_detect_newline_style("A 🎢 of a first line\r\nTwo\r\nThree")
        );
    }

    #[test]
    fn falls_back_to_native_newlines_if_no_newlines_are_found() {
        let expected_newline_style = if cfg!(windows) {
            EffectiveNewlineStyle::Windows
        } else {
            EffectiveNewlineStyle::Unix
        };
        assert_eq!(
            expected_newline_style,
            auto_detect_newline_style("One Two Three")
        );
    }

    #[test]
    fn test_newline_style_auto_apply() {
        let auto = NewlineStyle::Auto;

        let formatted_text = "One\nTwo\nThree";
        let raw_input_text = "One\nTwo\nThree";

        let mut out = String::from(formatted_text);
        apply_newline_style(auto, &mut out, raw_input_text);
        assert_eq!("One\nTwo\nThree", &out, "auto should detect 'lf'");

        let formatted_text = "One\nTwo\nThree";
        let raw_input_text = "One\r\nTwo\r\nThree";

        let mut out = String::from(formatted_text);
        apply_newline_style(auto, &mut out, raw_input_text);
        assert_eq!("One\r\nTwo\r\nThree", &out, "auto should detect 'crlf'");

        #[cfg(not(windows))]
        {
            let formatted_text = "One\nTwo\nThree";
            let raw_input_text = "One Two Three";

            let mut out = String::from(formatted_text);
            apply_newline_style(auto, &mut out, raw_input_text);
            assert_eq!(
                "One\nTwo\nThree", &out,
                "auto-native-unix should detect 'lf'"
            );
        }

        #[cfg(windows)]
        {
            let formatted_text = "One\nTwo\nThree";
            let raw_input_text = "One Two Three";

            let mut out = String::from(formatted_text);
            apply_newline_style(auto, &mut out, raw_input_text);
            assert_eq!(
                "One\r\nTwo\r\nThree", &out,
                "auto-native-windows should detect 'crlf'"
            );
        }
    }
}
<commit_msg>Split apply with auto detection tests into separate fns<commit_after>use crate::NewlineStyle;

/// Apply this newline style to the formatted text. When the style is set
/// to `Auto`, the `raw_input_text` is used to detect the existing line
/// endings.
///
/// If the style is set to `Auto` and `raw_input_text` contains no
/// newlines, the `Native` style will be used.
pub(crate) fn apply_newline_style(
    newline_style: NewlineStyle,
    formatted_text: &mut String,
    raw_input_text: &str,
) {
    match effective_newline_style(newline_style, raw_input_text) {
        EffectiveNewlineStyle::Windows => {
            *formatted_text = convert_to_windows_newlines(formatted_text);
        }
        EffectiveNewlineStyle::Unix => {}
    }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum EffectiveNewlineStyle {
    Windows,
    Unix,
}

fn effective_newline_style(
    newline_style: NewlineStyle,
    raw_input_text: &str,
) -> EffectiveNewlineStyle {
    match newline_style {
        NewlineStyle::Auto => auto_detect_newline_style(raw_input_text),
        NewlineStyle::Native => native_newline_style(),
        NewlineStyle::Windows => EffectiveNewlineStyle::Windows,
        NewlineStyle::Unix => EffectiveNewlineStyle::Unix,
    }
}

const LINE_FEED: char = '\n';
const CARRIAGE_RETURN: char = '\r';

fn auto_detect_newline_style(raw_input_text: &str) -> EffectiveNewlineStyle {
    if let Some(pos) = raw_input_text.chars().position(|ch| ch == LINE_FEED) {
        let pos = pos.saturating_sub(1);
        if let Some(CARRIAGE_RETURN) = raw_input_text.chars().nth(pos) {
            EffectiveNewlineStyle::Windows
        } else {
            EffectiveNewlineStyle::Unix
        }
    } else {
        native_newline_style()
    }
}

fn native_newline_style() -> EffectiveNewlineStyle {
    if cfg!(windows) {
        EffectiveNewlineStyle::Windows
    } else {
        EffectiveNewlineStyle::Unix
    }
}

fn convert_to_windows_newlines(formatted_text: &String) -> String {
    let mut transformed = String::with_capacity(2 * formatted_text.capacity());
    for c in formatted_text.chars() {
        const WINDOWS_NEWLINE: &str = "\r\n";
        match c {
            LINE_FEED => transformed.push_str(WINDOWS_NEWLINE),
            CARRIAGE_RETURN => continue,
            c => transformed.push(c),
        }
    }
    transformed
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn auto_detects_unix_newlines() {
        assert_eq!(
            EffectiveNewlineStyle::Unix,
            auto_detect_newline_style("One\nTwo\nThree")
        );
    }

    #[test]
    fn auto_detects_windows_newlines() {
        assert_eq!(
            EffectiveNewlineStyle::Windows,
            auto_detect_newline_style("One\r\nTwo\r\nThree")
        );
    }

    #[test]
    fn auto_detects_windows_newlines_with_multibyte_char_on_first_line() {
        assert_eq!(
            EffectiveNewlineStyle::Windows,
            auto_detect_newline_style("A 🎢 of a first line\r\nTwo\r\nThree")
        );
    }

    #[test]
    fn falls_back_to_native_newlines_if_no_newlines_are_found() {
        let expected_newline_style = if cfg!(windows) {
            EffectiveNewlineStyle::Windows
        } else {
            EffectiveNewlineStyle::Unix
        };
        assert_eq!(
            expected_newline_style,
            auto_detect_newline_style("One Two Three")
        );
    }

    #[test]
    fn auto_detects_and_applies_unix_newlines() {
        let formatted_text = "One\nTwo\nThree";
        let raw_input_text = "One\nTwo\nThree";

        let mut out = String::from(formatted_text);
        apply_newline_style(NewlineStyle::Auto, &mut out, raw_input_text);
        assert_eq!("One\nTwo\nThree", &out, "auto should detect 'lf'");
    }

    #[test]
    fn auto_detects_and_applies_windows_newlines() {
        let formatted_text = "One\nTwo\nThree";
        let raw_input_text = "One\r\nTwo\r\nThree";

        let mut out = String::from(formatted_text);
        apply_newline_style(NewlineStyle::Auto, &mut out, raw_input_text);
        assert_eq!("One\r\nTwo\r\nThree", &out, "auto should detect 'crlf'");
    }

    #[test]
    fn auto_detects_and_applies_native_newlines() {
        let formatted_text = "One\nTwo\nThree";
        let raw_input_text = "One Two Three";

        let mut out = String::from(formatted_text);
        apply_newline_style(NewlineStyle::Auto, &mut out, raw_input_text);

        if cfg!(windows) {
            assert_eq!(
                "One\r\nTwo\r\nThree", &out,
                "auto-native-windows should detect 'crlf'"
            );
        } else {
            assert_eq!(
                "One\nTwo\nThree", &out,
                "auto-native-unix should detect 'lf'"
            );
        }
    }
}
<|endoftext|>


<commit_before>extern crate serde;
extern crate serde_xml;

include!(concat!(env!("OUT_DIR"), "/glusterfs_exporter/types.rs"));

mod tests {
    use super::*;
    use serde_xml::de::from_iter;
    use std::fs::File;
    use std::io::Read;

    #[test]
    fn it_can_parse() -> () {
        let test_file = File::open("test_files/cumulative.xml").unwrap();
        let deserialized: Result<CliOutput,_> = from_iter(test_file.bytes());
        println!("{:?}", deserialized);
        assert!(deserialized.is_ok())
    }
}
<commit_msg>hide tests module in non-test builds<commit_after>extern crate serde;
extern crate serde_xml;

include!(concat!(env!("OUT_DIR"), "/glusterfs_exporter/types.rs"));

#[cfg(test)]
mod tests {
    use super::*;
    use serde_xml::de::from_iter;
    use std::fs::File;
    use std::io::Read;

    #[test]
    fn it_can_parse() -> () {
        let test_file = File::open("test_files/cumulative.xml").unwrap();
        let deserialized: Result<CliOutput,_> = from_iter(test_file.bytes());
        println!("{:?}", deserialized);
        assert!(deserialized.is_ok())
    }
}
<|endoftext|>