<commit_before>use nom::{
    branch::alt,
    bytes::streaming::{tag, tag_no_case, take, take_while, take_while1},
    character::streaming::{char, digit1},
    combinator::{map, map_res},
    multi::{separated_list, separated_nonempty_list},
    sequence::{delimited, tuple},
    IResult,
};

use std::str::{from_utf8, FromStr};

// ----- number -----

// number          = 1*DIGIT
//                    ; Unsigned 32-bit integer
//                    ; (0 <= n < 4,294,967,296)
pub fn number(i: &[u8]) -> IResult<&[u8], u32> {
    let (i, bytes) = digit1(i)?;
    match from_utf8(bytes).ok().and_then(|s| u32::from_str(s).ok()) {
        Some(v) => Ok((i, v)),
        None => Err(nom::Err::Error(nom::error::make_error(
            i,
            nom::error::ErrorKind::ParseTo,
        ))),
    }
}

// same as `number` but 64-bit
pub fn number_64(i: &[u8]) -> IResult<&[u8], u64> {
    let (i, bytes) = digit1(i)?;
    match from_utf8(bytes).ok().and_then(|s| u64::from_str(s).ok()) {
        Some(v) => Ok((i, v)),
        None => Err(nom::Err::Error(nom::error::make_error(
            i,
            nom::error::ErrorKind::ParseTo,
        ))),
    }
}

// ----- string -----

// string = quoted / literal
pub fn string(i: &[u8]) -> IResult<&[u8], &[u8]> {
    alt((quoted, literal))(i)
}

// string bytes as utf8
pub fn string_utf8(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(string, from_utf8)(i)
}

// quoted = DQUOTE *QUOTED-CHAR DQUOTE
pub fn quoted(i: &[u8]) -> IResult<&[u8], &[u8]> {
    delimited(char('"'), quoted_data, char('"'))(i)
}

// quoted bytes as utf8
pub fn quoted_utf8(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(quoted, from_utf8)(i)
}

// QUOTED-CHAR = <any TEXT-CHAR except quoted-specials> / "\" quoted-specials
pub fn quoted_data(i: &[u8]) -> IResult<&[u8], &[u8]> {
    // Ideally this should use nom's `escaped` macro, but it suffers from broken
    // type inference unless compiled with the verbose-errors feature enabled.
    let mut escape = false;
    let mut len = 0;
    for c in i {
        if *c == b'"' && !escape {
            break;
        }
        len += 1;
        if *c == b'\\' && !escape {
            escape = true
        } else if escape {
            escape = false;
        }
    }
    Ok((&i[len..], &i[..len]))
}

// quoted-specials = DQUOTE / "\"
pub fn is_quoted_specials(c: u8) -> bool {
    c == b'"' || c == b'\\'
}

/// literal = "{" number "}" CRLF *CHAR8
///            ; Number represents the number of CHAR8s
pub fn literal(input: &[u8]) -> IResult<&[u8], &[u8]> {
    let parser = tuple((tag(b"{"), number, tag(b"}"), tag("\r\n")));

    let (remaining, (_, count, _, _)) = parser(input)?;

    let (remaining, data) = take(count)(remaining)?;

    if !data.iter().all(|byte| is_char8(*byte)) {
        // FIXME: what ErrorKind should this have?
        return Err(nom::Err::Error((remaining, nom::error::ErrorKind::Verify)));
    }

    Ok((remaining, data))
}

/// CHAR8 = %x01-ff ; any OCTET except NUL, %x00
pub fn is_char8(i: u8) -> bool {
    i != 0
}

// ----- astring ----- atom (roughly) or string

// astring = 1*ASTRING-CHAR / string
pub fn astring(i: &[u8]) -> IResult<&[u8], &[u8]> {
    alt((take_while1(is_astring_char), string))(i)
}

// astring bytes as utf8
pub fn astring_utf8(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(astring, from_utf8)(i)
}

// ASTRING-CHAR = ATOM-CHAR / resp-specials
pub fn is_astring_char(c: u8) -> bool {
    is_atom_char(c) || is_resp_specials(c)
}

// ATOM-CHAR = <any CHAR except atom-specials>
pub fn is_atom_char(c: u8) -> bool {
    is_char(c) && !is_atom_specials(c)
}

// atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / quoted-specials / resp-specials
pub fn is_atom_specials(c: u8) -> bool {
    c == b'('
        || c == b')'
        || c == b'{'
        || c == b' '
        || c < 32
        || is_list_wildcards(c)
        || is_quoted_specials(c)
        || is_resp_specials(c)
}

// resp-specials = "]"
pub fn is_resp_specials(c: u8) -> bool {
    c == b']'
}

// atom = 1*ATOM-CHAR
pub fn atom(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(take_while1(is_atom_char), from_utf8)(i)
}

// ----- nstring ----- nil or string

// nstring = string / nil
pub fn nstring(i: &[u8]) -> IResult<&[u8], Option<&[u8]>> {
    alt((map(nil, |_| None), map(string, Some)))(i)
}

// nstring bytes as utf8
pub fn nstring_utf8(i: &[u8]) -> IResult<&[u8], Option<&str>> {
    alt((map(nil, |_| None), map(string_utf8, Some)))(i)
}

// nil = "NIL"
pub fn nil(i: &[u8]) -> IResult<&[u8], &[u8]> {
    tag_no_case("NIL")(i)
}

// ----- text -----

// text = 1*TEXT-CHAR
pub fn text(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(take_while(is_text_char), from_utf8)(i)
}

// TEXT-CHAR = <any CHAR except CR and LF>
pub fn is_text_char(c: u8) -> bool {
    is_char(c) && c != b'\r' && c != b'\n'
}

// CHAR = %x01-7F
//          ; any 7-bit US-ASCII character,
//          ;  excluding NUL
// From RFC5234
pub fn is_char(c: u8) -> bool {
    match c {
        0x01..=0x7F => true,
        _ => false,
    }
}

// ----- others -----

// list-wildcards = "%" / "*"
pub fn is_list_wildcards(c: u8) -> bool {
    c == b'%' || c == b'*'
}

pub fn paren_delimited<'a, F, O, E>(f: F) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], O, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], O, E>,
    E: nom::error::ParseError<&'a [u8]>,
{
    delimited(char('('), f, char(')'))
}

pub fn parenthesized_nonempty_list<'a, F, O, E>(
    f: F,
) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Vec<O>, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], O, E>,
    E: nom::error::ParseError<&'a [u8]>,
{
    delimited(char('('), separated_nonempty_list(char(' '), f), char(')'))
}

pub fn parenthesized_list<'a, F, O, E>(f: F) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Vec<O>, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], O, E>,
    E: nom::error::ParseError<&'a [u8]>,
{
    delimited(char('('), separated_list(char(' '), f), char(')'))
}

pub fn opt_opt<'a, F, O, E>(f: F) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Option<O>, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], Option<O>, E>,
{
    move |i: &[u8]| match f(i) {
        Ok((i, o)) => Ok((i, o)),
        Err(nom::Err::Error(_)) => Ok((i, None)),
        Err(e) => Err(e),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_string_literal() {
        match string(b"{3}\r\nXYZ") {
            Ok((_, value)) => {
                assert_eq!(value, b"XYZ");
            }
            rsp => panic!("unexpected response {:?}", rsp),
        }
    }

    #[test]
    fn test_astring() {
        match astring(b"text ") {
            Ok((_, value)) => {
                assert_eq!(value, b"text");
            }
            rsp => panic!("unexpected response {:?}", rsp),
        }
    }
}
<commit_msg>Add test_quoted<commit_after>use nom::{
    branch::alt,
    bytes::streaming::{tag, tag_no_case, take, take_while, take_while1},
    character::streaming::{char, digit1},
    combinator::{map, map_res},
    multi::{separated_list, separated_nonempty_list},
    sequence::{delimited, tuple},
    IResult,
};

use std::str::{from_utf8, FromStr};

// ----- number -----

// number          = 1*DIGIT
//                    ; Unsigned 32-bit integer
//                    ; (0 <= n < 4,294,967,296)
pub fn number(i: &[u8]) -> IResult<&[u8], u32> {
    let (i, bytes) = digit1(i)?;
    match from_utf8(bytes).ok().and_then(|s| u32::from_str(s).ok()) {
        Some(v) => Ok((i, v)),
        None => Err(nom::Err::Error(nom::error::make_error(
            i,
            nom::error::ErrorKind::ParseTo,
        ))),
    }
}

// same as `number` but 64-bit
pub fn number_64(i: &[u8]) -> IResult<&[u8], u64> {
    let (i, bytes) = digit1(i)?;
    match from_utf8(bytes).ok().and_then(|s| u64::from_str(s).ok()) {
        Some(v) => Ok((i, v)),
        None => Err(nom::Err::Error(nom::error::make_error(
            i,
            nom::error::ErrorKind::ParseTo,
        ))),
    }
}

// ----- string -----

// string = quoted / literal
pub fn string(i: &[u8]) -> IResult<&[u8], &[u8]> {
    alt((quoted, literal))(i)
}

// string bytes as utf8
pub fn string_utf8(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(string, from_utf8)(i)
}

// quoted = DQUOTE *QUOTED-CHAR DQUOTE
pub fn quoted(i: &[u8]) -> IResult<&[u8], &[u8]> {
    delimited(char('"'), quoted_data, char('"'))(i)
}

// quoted bytes as utf8
pub fn quoted_utf8(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(quoted, from_utf8)(i)
}

// QUOTED-CHAR = <any TEXT-CHAR except quoted-specials> / "\" quoted-specials
pub fn quoted_data(i: &[u8]) -> IResult<&[u8], &[u8]> {
    // Ideally this should use nom's `escaped` macro, but it suffers from broken
    // type inference unless compiled with the verbose-errors feature enabled.
    let mut escape = false;
    let mut len = 0;
    for c in i {
        if *c == b'"' && !escape {
            break;
        }
        len += 1;
        if *c == b'\\' && !escape {
            escape = true
        } else if escape {
            escape = false;
        }
    }
    Ok((&i[len..], &i[..len]))
}

// quoted-specials = DQUOTE / "\"
pub fn is_quoted_specials(c: u8) -> bool {
    c == b'"' || c == b'\\'
}

/// literal = "{" number "}" CRLF *CHAR8
///            ; Number represents the number of CHAR8s
pub fn literal(input: &[u8]) -> IResult<&[u8], &[u8]> {
    let parser = tuple((tag(b"{"), number, tag(b"}"), tag("\r\n")));

    let (remaining, (_, count, _, _)) = parser(input)?;

    let (remaining, data) = take(count)(remaining)?;

    if !data.iter().all(|byte| is_char8(*byte)) {
        // FIXME: what ErrorKind should this have?
        return Err(nom::Err::Error((remaining, nom::error::ErrorKind::Verify)));
    }

    Ok((remaining, data))
}

/// CHAR8 = %x01-ff ; any OCTET except NUL, %x00
pub fn is_char8(i: u8) -> bool {
    i != 0
}

// ----- astring ----- atom (roughly) or string

// astring = 1*ASTRING-CHAR / string
pub fn astring(i: &[u8]) -> IResult<&[u8], &[u8]> {
    alt((take_while1(is_astring_char), string))(i)
}

// astring bytes as utf8
pub fn astring_utf8(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(astring, from_utf8)(i)
}

// ASTRING-CHAR = ATOM-CHAR / resp-specials
pub fn is_astring_char(c: u8) -> bool {
    is_atom_char(c) || is_resp_specials(c)
}

// ATOM-CHAR = <any CHAR except atom-specials>
pub fn is_atom_char(c: u8) -> bool {
    is_char(c) && !is_atom_specials(c)
}

// atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / quoted-specials / resp-specials
pub fn is_atom_specials(c: u8) -> bool {
    c == b'('
        || c == b')'
        || c == b'{'
        || c == b' '
        || c < 32
        || is_list_wildcards(c)
        || is_quoted_specials(c)
        || is_resp_specials(c)
}

// resp-specials = "]"
pub fn is_resp_specials(c: u8) -> bool {
    c == b']'
}

// atom = 1*ATOM-CHAR
pub fn atom(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(take_while1(is_atom_char), from_utf8)(i)
}

// ----- nstring ----- nil or string

// nstring = string / nil
pub fn nstring(i: &[u8]) -> IResult<&[u8], Option<&[u8]>> {
    alt((map(nil, |_| None), map(string, Some)))(i)
}

// nstring bytes as utf8
pub fn nstring_utf8(i: &[u8]) -> IResult<&[u8], Option<&str>> {
    alt((map(nil, |_| None), map(string_utf8, Some)))(i)
}

// nil = "NIL"
pub fn nil(i: &[u8]) -> IResult<&[u8], &[u8]> {
    tag_no_case("NIL")(i)
}

// ----- text -----

// text = 1*TEXT-CHAR
pub fn text(i: &[u8]) -> IResult<&[u8], &str> {
    map_res(take_while(is_text_char), from_utf8)(i)
}

// TEXT-CHAR = <any CHAR except CR and LF>
pub fn is_text_char(c: u8) -> bool {
    is_char(c) && c != b'\r' && c != b'\n'
}

// CHAR = %x01-7F
//          ; any 7-bit US-ASCII character,
//          ;  excluding NUL
// From RFC5234
pub fn is_char(c: u8) -> bool {
    match c {
        0x01..=0x7F => true,
        _ => false,
    }
}

// ----- others -----

// list-wildcards = "%" / "*"
pub fn is_list_wildcards(c: u8) -> bool {
    c == b'%' || c == b'*'
}

pub fn paren_delimited<'a, F, O, E>(f: F) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], O, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], O, E>,
    E: nom::error::ParseError<&'a [u8]>,
{
    delimited(char('('), f, char(')'))
}

pub fn parenthesized_nonempty_list<'a, F, O, E>(
    f: F,
) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Vec<O>, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], O, E>,
    E: nom::error::ParseError<&'a [u8]>,
{
    delimited(char('('), separated_nonempty_list(char(' '), f), char(')'))
}

pub fn parenthesized_list<'a, F, O, E>(f: F) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Vec<O>, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], O, E>,
    E: nom::error::ParseError<&'a [u8]>,
{
    delimited(char('('), separated_list(char(' '), f), char(')'))
}

pub fn opt_opt<'a, F, O, E>(f: F) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Option<O>, E>
where
    F: Fn(&'a [u8]) -> IResult<&'a [u8], Option<O>, E>,
{
    move |i: &[u8]| match f(i) {
        Ok((i, o)) => Ok((i, o)),
        Err(nom::Err::Error(_)) => Ok((i, None)),
        Err(e) => Err(e),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use assert_matches::assert_matches;

    #[test]
    fn test_quoted() {
        let (rem, val) = quoted(br#""Hello"???"#).unwrap();
        assert_eq!(rem, b"???");
        assert_eq!(val, b"Hello");

        // Allowed escapes...
        assert!(quoted(br#""Hello \" "???"#).is_ok());
        assert!(quoted(br#""Hello \\ "???"#).is_ok());

        // Not allowed escapes...
        //assert!(quoted(br#""Hello \a "???"#).is_err()); // fails
        //assert!(quoted(br#""Hello \z "???"#).is_err()); // fails
        //assert!(quoted(br#""Hello \? "???"#).is_err()); // fails

        let (rem, val) = quoted(br#""Hello \"World\""???"#).unwrap();
        assert_eq!(rem, br#"???"#);
        // Should it be this (Hello \"World\") ...
        assert_eq!(val, br#"Hello \"World\""#);
        // ... or this (Hello "World")?
        //assert_eq!(val, br#"Hello "World""#); // fails

        // Test Incomplete
        assert_matches!(quoted(br#""#), Err(nom::Err::Incomplete(_)));
        assert_matches!(quoted(br#""\"#), Err(nom::Err::Incomplete(_)));
        assert_matches!(quoted(br#""Hello "#), Err(nom::Err::Incomplete(_)));

        // Test Error
        assert_matches!(quoted(br#"\"#), Err(nom::Err::Error(_)));
    }

    #[test]
    fn test_string_literal() {
        match string(b"{3}\r\nXYZ") {
            Ok((_, value)) => {
                assert_eq!(value, b"XYZ");
            }
            rsp => panic!("unexpected response {:?}", rsp),
        }
    }

    #[test]
    fn test_astring() {
        match astring(b"text ") {
            Ok((_, value)) => {
                assert_eq!(value, b"text");
            }
            rsp => panic!("unexpected response {:?}", rsp),
        }
    }
}
<|endoftext|>


<commit_before>use crate::command::Config;
use crate::error::Error;
use crate::page::{parse_page, DiffOp, Line, Page};
use crate::test::helper::empty_env;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};

fn config(repo: &str, branch: Option<&str>, args: Vec<&str>) -> Config {
    let mut dir = env::current_dir().unwrap();
    dir.push(Path::new(".git"));
    let dir = fs::canonicalize(dir).unwrap();
    let mut a = Vec::new();
    for arg in args {
        a.push(arg.to_string());
    }
    Config {
        repo: repo.to_string(),
        branch: branch.map(|s| s.to_string()),
        git_dir: dir,
        args: a,
        stdout: false,
        pull_request: false,
        env: empty_env(),
    }
}

#[test]
fn parse_empty_args() {
    let c = config("https://github.com/user/repo.git", None, vec![]);
    match parse_page(&c).unwrap() {
        Page::Open => { /* OK */ }
        _ => assert!(false),
    }
}

#[test]
fn parse_file_or_dir() {
    for &(entry, relative) in &[
        (
            Path::new(".").join("README.md").as_path(),
            Path::new("README.md"),
        ),
        (Path::new("src"), Path::new("src")),
        (
            Path::new(".").join("src").join("main.rs").as_path(),
            Path::new("src").join("main.rs").as_path(),
        ),
        // Contains '..'
        (
            Path::new(".")
                .join("src")
                .join("..")
                .join("README.md")
                .as_path(),
            Path::new("README.md"),
        ),
        // Suffix '..'
        (
            Path::new(".").join("src").join("test").join("..").as_path(),
            Path::new("src"),
        ),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![&entry.to_str().unwrap()],
        );
        match parse_page(&c).unwrap() {
            Page::FileOrDir {
                relative_path,
                hash,
                line: None,
            } => {
                assert_eq!(relative_path, relative.to_str().unwrap());
                assert!(!hash.is_empty());
            }
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
fn parse_file_single_line() {
    for &(ref file, ref expected) in &[
        (Path::new(".").join("README.md#21"), Some(Line::At(21))),
        (
            Path::new(".").join("src").join("main.rs#10"),
            Some(Line::At(10)),
        ),
        (PathBuf::from("LICENSE.txt"), None),
        (
            Path::new(".").join("src").join("..").join("README.md#21"),
            Some(Line::At(21)),
        ),
        (Path::new(".").join("README.md#L21"), Some(Line::At(21))),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![&file.to_str().unwrap()],
        );
        match parse_page(&c).unwrap() {
            Page::FileOrDir { ref line, .. } => assert_eq!(line, expected, "input: {:?}", file),
            p => assert!(false, "Unexpected result: {:?}, input: {:?}", p, file),
        }
    }
}

#[test]
fn not_existing_file() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["not/existing/file.txt"],
    );
    assert!(parse_page(&c).is_err());
}

#[test]
fn file_at_specific_commit() {
    let c = config("https://github.com/user/repo.git", None, vec!["README.md"]);
    let p = parse_page(&c).unwrap();
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["README.md", "HEAD^"],
    );
    let p2 = parse_page(&c).unwrap();
    assert!(p != p2, format!("{:?} v.s. {:?}", p, p2));
}

#[test]
fn parse_commit_ref() {
    for &cm in &["HEAD", "HEAD~1", "HEAD^"] {
        let c = config("https://github.com/user/repo.git", None, vec![cm]);
        match parse_page(&c).unwrap() {
            Page::Commit { hash } => assert!(!hash.is_empty()),
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
#[cfg_attr(feature = "on-ci", ignore)]
fn parse_short_commit_hash() {
    for &(cm, expected) in &[
        ("499edbb", "499edbbbad4d8054e4a47e12944e5fb4a2ef7ec5"),
        (
            "bc869a14617a131fefe8fa1a3dcdeba0745880d5",
            "bc869a14617a131fefe8fa1a3dcdeba0745880d5",
        ),
    ] {
        let c = config("https://github.com/user/repo.git", None, vec![cm]);
        match parse_page(&c).unwrap() {
            Page::Commit { hash } => assert_eq!(hash, expected),
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
fn parse_diff_ref_name() {
    for &(arg, expected_op) in &[
        ("HEAD^..HEAD", DiffOp::TwoDots),
        ("HEAD^...HEAD", DiffOp::ThreeDots),
    ] {
        let c = config("https://github.com/user/repo.git", None, vec![arg]);
        match parse_page(&c).unwrap() {
            Page::Diff { lhs, rhs, op } => {
                assert!(!lhs.is_empty());
                assert!(!rhs.is_empty());
                assert_eq!(op, expected_op, "arg is {}", arg);
            }
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
#[cfg_attr(feature = "on-ci", ignore)]
fn parse_diff_revisions() {
    for &(arg, expected_op) in &[
        ("499edbb..bc869a1", DiffOp::TwoDots),
        ("499edbb...bc869a1", DiffOp::ThreeDots),
    ] {
        let c = config("https://github.com/user/repo.git", None, vec![arg]);
        match parse_page(&c).unwrap() {
            Page::Diff { lhs, rhs, op } => {
                assert_eq!(lhs, "499edbbbad4d8054e4a47e12944e5fb4a2ef7ec5");
                assert_eq!(rhs, "bc869a14617a131fefe8fa1a3dcdeba0745880d5");
                assert_eq!(op, expected_op, "arg is {}", arg);
            }
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
fn wrong_num_of_args() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["foo", "bar", "piyo", "blah"],
    );
    match parse_page(&c) {
        Err(Error::PageParseError { attempts, .. }) => {
            assert!(!attempts.is_empty());
            for err in attempts {
                match err {
                    Error::WrongNumberOfArgs { actual, .. } => assert_eq!(actual, 4),
                    err => assert!(false, "Unexpected error: {}", err),
                }
            }
        }
        v => assert!(false, "Unexpected success or error: {:?}", v),
    }
}

#[test]
fn unknown_diff() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["HEAD~114514..HEAD~114513"],
    );
    assert!(parse_page(&c).is_err());
}

#[test]
fn file_for_unknown_commit() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["README.md", "HEAD~114514"],
    );
    assert!(parse_page(&c).is_err());
}

#[test]
fn diff_lhs_or_rhs_empty() {
    for ref path in &[
        Path::new("..").join("foo.txt"),
        Path::new("foo").join(".."),
        PathBuf::from(".."),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![path.to_str().unwrap()],
        );
        match parse_page(&c) {
            Ok(p @ Page::Diff { .. }) => assert!(false, "Unexpectedly parsed as diff: {:?}", p),
            _ => { /*ok*/ }
        }
    }
}

#[test]
fn issue_number() {
    let c = config("https://github.com/user/repo.git", None, vec!["#123"]);
    match parse_page(&c) {
        Ok(Page::Issue { number }) => assert_eq!(number, 123),
        v => assert!(false, "Unexpected result {:?}", v),
    }
}

#[test]
fn line_cannot_be_set_to_dir() {
    for arg in &["src#123", "src#12-23"] {
        let c = config("https://github.com/user/repo.git", None, vec![arg]);
        match parse_page(&c) {
            Err(Error::PageParseError { attempts, .. }) => assert!(
                attempts.iter().any(|err| match err {
                    Error::LineSpecifiedForDir(path) => format!("{:?}", path).contains("src"),
                    _ => false,
                }),
                "{:?} for {}",
                attempts,
                arg,
            ),
            v => assert!(false, "Unexpected result {:?} for {}", v, arg),
        }
    }
}

#[test]
fn parse_file_line_range() {
    for file in &[
        Path::new("README.md#1-2"),
        Path::new("README.md#L1-2"),
        Path::new("README.md#1-L2"),
        Path::new("README.md#L1-L2"),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![&file.to_str().unwrap()],
        );
        match parse_page(&c).unwrap() {
            Page::FileOrDir { line, .. } => {
                assert_eq!(line, Some(Line::Range(1, 2)), "input: {:?}", file)
            }
            p => assert!(false, "Unexpected result: {:?}, input: {:?}", p, file),
        }
    }
}
<commit_msg>add more test cases for ref specs<commit_after>use crate::command::Config;
use crate::error::Error;
use crate::page::{parse_page, DiffOp, Line, Page};
use crate::test::helper::empty_env;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};

fn config(repo: &str, branch: Option<&str>, args: Vec<&str>) -> Config {
    let mut dir = env::current_dir().unwrap();
    dir.push(Path::new(".git"));
    let dir = fs::canonicalize(dir).unwrap();
    let mut a = Vec::new();
    for arg in args {
        a.push(arg.to_string());
    }
    Config {
        repo: repo.to_string(),
        branch: branch.map(|s| s.to_string()),
        git_dir: dir,
        args: a,
        stdout: false,
        pull_request: false,
        env: empty_env(),
    }
}

#[test]
fn parse_empty_args() {
    let c = config("https://github.com/user/repo.git", None, vec![]);
    match parse_page(&c).unwrap() {
        Page::Open => { /* OK */ }
        _ => assert!(false),
    }
}

#[test]
fn parse_file_or_dir() {
    for &(entry, relative) in &[
        (
            Path::new(".").join("README.md").as_path(),
            Path::new("README.md"),
        ),
        (Path::new("src"), Path::new("src")),
        (
            Path::new(".").join("src").join("main.rs").as_path(),
            Path::new("src").join("main.rs").as_path(),
        ),
        // Contains '..'
        (
            Path::new(".")
                .join("src")
                .join("..")
                .join("README.md")
                .as_path(),
            Path::new("README.md"),
        ),
        // Suffix '..'
        (
            Path::new(".").join("src").join("test").join("..").as_path(),
            Path::new("src"),
        ),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![&entry.to_str().unwrap()],
        );
        match parse_page(&c).unwrap() {
            Page::FileOrDir {
                relative_path,
                hash,
                line: None,
            } => {
                assert_eq!(relative_path, relative.to_str().unwrap());
                assert!(!hash.is_empty());
            }
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
fn parse_file_single_line() {
    for &(ref file, ref expected) in &[
        (Path::new(".").join("README.md#21"), Some(Line::At(21))),
        (
            Path::new(".").join("src").join("main.rs#10"),
            Some(Line::At(10)),
        ),
        (PathBuf::from("LICENSE.txt"), None),
        (
            Path::new(".").join("src").join("..").join("README.md#21"),
            Some(Line::At(21)),
        ),
        (Path::new(".").join("README.md#L21"), Some(Line::At(21))),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![&file.to_str().unwrap()],
        );
        match parse_page(&c).unwrap() {
            Page::FileOrDir { ref line, .. } => assert_eq!(line, expected, "input: {:?}", file),
            p => assert!(false, "Unexpected result: {:?}, input: {:?}", p, file),
        }
    }
}

#[test]
fn not_existing_file() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["not/existing/file.txt"],
    );
    assert!(parse_page(&c).is_err());
}

#[test]
fn file_at_specific_commit() {
    let c = config("https://github.com/user/repo.git", None, vec!["README.md"]);
    let p = parse_page(&c).unwrap();
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["README.md", "HEAD^"],
    );
    let p2 = parse_page(&c).unwrap();
    assert!(p != p2, format!("{:?} v.s. {:?}", p, p2));
}

#[test]
fn parse_commit_ref() {
    for &cm in &["HEAD", "HEAD~1", "HEAD^", "HEAD^^"] {
        let c = config("https://github.com/user/repo.git", None, vec![cm]);
        match parse_page(&c).unwrap() {
            Page::Commit { hash } => assert!(!hash.is_empty()),
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
#[cfg_attr(feature = "on-ci", ignore)]
fn parse_short_commit_hash() {
    for &(cm, expected) in &[
        ("499edbb", "499edbbbad4d8054e4a47e12944e5fb4a2ef7ec5"),
        (
            "bc869a14617a131fefe8fa1a3dcdeba0745880d5",
            "bc869a14617a131fefe8fa1a3dcdeba0745880d5",
        ),
    ] {
        let c = config("https://github.com/user/repo.git", None, vec![cm]);
        match parse_page(&c).unwrap() {
            Page::Commit { hash } => assert_eq!(hash, expected),
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
fn parse_diff_ref_name() {
    for &(arg, expected_op) in &[
        ("HEAD^..HEAD", DiffOp::TwoDots),
        ("HEAD^...HEAD", DiffOp::ThreeDots),
        ("master...HEAD", DiffOp::ThreeDots),
        ("master@{1month}..master@{1day}", DiffOp::TwoDots),
    ] {
        let c = config("https://github.com/user/repo.git", None, vec![arg]);
        match parse_page(&c).unwrap() {
            Page::Diff { lhs, rhs, op } => {
                assert!(!lhs.is_empty());
                assert!(!rhs.is_empty());
                assert_eq!(op, expected_op, "arg is {}", arg);
            }
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
#[cfg_attr(feature = "on-ci", ignore)]
fn parse_diff_revisions() {
    for &(arg, expected_op) in &[
        ("499edbb..bc869a1", DiffOp::TwoDots),
        ("499edbb...bc869a1", DiffOp::ThreeDots),
    ] {
        let c = config("https://github.com/user/repo.git", None, vec![arg]);
        match parse_page(&c).unwrap() {
            Page::Diff { lhs, rhs, op } => {
                assert_eq!(lhs, "499edbbbad4d8054e4a47e12944e5fb4a2ef7ec5");
                assert_eq!(rhs, "bc869a14617a131fefe8fa1a3dcdeba0745880d5");
                assert_eq!(op, expected_op, "arg is {}", arg);
            }
            p => assert!(false, "Unexpected result: {:?}", p),
        }
    }
}

#[test]
fn wrong_num_of_args() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["foo", "bar", "piyo", "blah"],
    );
    match parse_page(&c) {
        Err(Error::PageParseError { attempts, .. }) => {
            assert!(!attempts.is_empty());
            for err in attempts {
                match err {
                    Error::WrongNumberOfArgs { actual, .. } => assert_eq!(actual, 4),
                    err => assert!(false, "Unexpected error: {}", err),
                }
            }
        }
        v => assert!(false, "Unexpected success or error: {:?}", v),
    }
}

#[test]
fn unknown_diff() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["HEAD~114514..HEAD~114513"],
    );
    assert!(parse_page(&c).is_err());
}

#[test]
fn file_for_unknown_commit() {
    let c = config(
        "https://github.com/user/repo.git",
        None,
        vec!["README.md", "HEAD~114514"],
    );
    assert!(parse_page(&c).is_err());
}

#[test]
fn diff_lhs_or_rhs_empty() {
    for ref path in &[
        Path::new("..").join("foo.txt"),
        Path::new("foo").join(".."),
        PathBuf::from(".."),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![path.to_str().unwrap()],
        );
        match parse_page(&c) {
            Ok(p @ Page::Diff { .. }) => assert!(false, "Unexpectedly parsed as diff: {:?}", p),
            _ => { /*ok*/ }
        }
    }
}

#[test]
fn issue_number() {
    let c = config("https://github.com/user/repo.git", None, vec!["#123"]);
    match parse_page(&c) {
        Ok(Page::Issue { number }) => assert_eq!(number, 123),
        v => assert!(false, "Unexpected result {:?}", v),
    }
}

#[test]
fn line_cannot_be_set_to_dir() {
    for arg in &["src#123", "src#12-23"] {
        let c = config("https://github.com/user/repo.git", None, vec![arg]);
        match parse_page(&c) {
            Err(Error::PageParseError { attempts, .. }) => assert!(
                attempts.iter().any(|err| match err {
                    Error::LineSpecifiedForDir(path) => format!("{:?}", path).contains("src"),
                    _ => false,
                }),
                "{:?} for {}",
                attempts,
                arg,
            ),
            v => assert!(false, "Unexpected result {:?} for {}", v, arg),
        }
    }
}

#[test]
fn parse_file_line_range() {
    for file in &[
        Path::new("README.md#1-2"),
        Path::new("README.md#L1-2"),
        Path::new("README.md#1-L2"),
        Path::new("README.md#L1-L2"),
    ] {
        let c = config(
            "https://github.com/user/repo.git",
            None,
            vec![&file.to_str().unwrap()],
        );
        match parse_page(&c).unwrap() {
            Page::FileOrDir { line, .. } => {
                assert_eq!(line, Some(Line::Range(1, 2)), "input: {:?}", file)
            }
            p => assert!(false, "Unexpected result: {:?}, input: {:?}", p, file),
        }
    }
}
<|endoftext|>


<commit_before>use super::TokenStreamExt;

use std::borrow::Cow;
use std::iter;
use std::rc::Rc;

use proc_macro2::{Group, Ident, Literal, Punct, Span, TokenStream, TokenTree};

/// Types that can be interpolated inside a `quote!` invocation.
///
/// [`quote!`]: macro.quote.html
pub trait ToTokens {
    /// Write `self` to the given `TokenStream`.
    ///
    /// The token append methods provided by the [`TokenStreamExt`] extension
    /// trait may be useful for implementing `ToTokens`.
    ///
    /// [`TokenStreamExt`]: trait.TokenStreamExt.html
    ///
    /// # Example
    ///
    /// Example implementation for a struct representing Rust paths like
    /// `std::cmp::PartialEq`:
    ///
    /// ```
    /// use proc_macro2::{TokenTree, Spacing, Span, Punct, TokenStream};
    /// use quote::{TokenStreamExt, ToTokens};
    ///
    /// pub struct Path {
    ///     pub global: bool,
    ///     pub segments: Vec<PathSegment>,
    /// }
    ///
    /// impl ToTokens for Path {
    ///     fn to_tokens(&self, tokens: &mut TokenStream) {
    ///         for (i, segment) in self.segments.iter().enumerate() {
    ///             if i > 0 || self.global {
    ///                 // Double colon `::`
    ///                 tokens.append(Punct::new(':', Spacing::Joint));
    ///                 tokens.append(Punct::new(':', Spacing::Alone));
    ///             }
    ///             segment.to_tokens(tokens);
    ///         }
    ///     }
    /// }
    /// #
    /// # pub struct PathSegment;
    /// #
    /// # impl ToTokens for PathSegment {
    /// #     fn to_tokens(&self, tokens: &mut TokenStream) {
    /// #         unimplemented!()
    /// #     }
    /// # }
    /// ```
    fn to_tokens(&self, tokens: &mut TokenStream);

    /// Convert `self` directly into a `TokenStream` object.
    ///
    /// This method is implicitly implemented using `to_tokens`, and acts as a
    /// convenience method for consumers of the `ToTokens` trait.
    fn into_token_stream(self) -> TokenStream
    where
        Self: Sized,
    {
        let mut tokens = TokenStream::new();
        self.to_tokens(&mut tokens);
        tokens
    }
}

impl<'a, T: ?Sized + ToTokens> ToTokens for &'a T {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<'a, T: ?Sized + ToTokens> ToTokens for &'a mut T {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ?Sized + ToTokens> ToTokens for Box<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ?Sized + ToTokens> ToTokens for Rc<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ToTokens> ToTokens for Option<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        if let Some(ref t) = *self {
            t.to_tokens(tokens);
        }
    }
}

impl ToTokens for str {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Literal::string(self));
    }
}

impl ToTokens for String {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        self.as_str().to_tokens(tokens);
    }
}

macro_rules! primitive {
    ($($t:ident => $name:ident)*) => ($(
        impl ToTokens for $t {
            fn to_tokens(&self, tokens: &mut TokenStream) {
                tokens.append(Literal::$name(*self));
            }
        }
    )*)
}

primitive! {
    i8 => i8_suffixed
    i16 => i16_suffixed
    i32 => i32_suffixed
    i64 => i64_suffixed
    i128 => i128_suffixed
    isize => isize_suffixed

    u8 => u8_suffixed
    u16 => u16_suffixed
    u32 => u32_suffixed
    u64 => u64_suffixed
    u128 => u128_suffixed
    usize => usize_suffixed

    f32 => f32_suffixed
    f64 => f64_suffixed
}

impl ToTokens for char {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Literal::character(*self));
    }
}

impl ToTokens for bool {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        let word = if *self { "true" } else { "false" };
        tokens.append(Ident::new(word, Span::call_site()));
    }
}

impl ToTokens for Group {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Ident {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Punct {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Literal {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for TokenTree {
    fn to_tokens(&self, dst: &mut TokenStream) {
        dst.append(self.clone());
    }
}

impl ToTokens for TokenStream {
    fn to_tokens(&self, dst: &mut TokenStream) {
        dst.extend(iter::once(self.clone()));
    }

    fn into_token_stream(self) -> TokenStream {
        self
    }
}
<commit_msg>Add a version of into_token_stream that takes &self<commit_after>use super::TokenStreamExt;

use std::borrow::Cow;
use std::iter;
use std::rc::Rc;

use proc_macro2::{Group, Ident, Literal, Punct, Span, TokenStream, TokenTree};

/// Types that can be interpolated inside a `quote!` invocation.
///
/// [`quote!`]: macro.quote.html
pub trait ToTokens {
    /// Write `self` to the given `TokenStream`.
    ///
    /// The token append methods provided by the [`TokenStreamExt`] extension
    /// trait may be useful for implementing `ToTokens`.
    ///
    /// [`TokenStreamExt`]: trait.TokenStreamExt.html
    ///
    /// # Example
    ///
    /// Example implementation for a struct representing Rust paths like
    /// `std::cmp::PartialEq`:
    ///
    /// ```
    /// use proc_macro2::{TokenTree, Spacing, Span, Punct, TokenStream};
    /// use quote::{TokenStreamExt, ToTokens};
    ///
    /// pub struct Path {
    ///     pub global: bool,
    ///     pub segments: Vec<PathSegment>,
    /// }
    ///
    /// impl ToTokens for Path {
    ///     fn to_tokens(&self, tokens: &mut TokenStream) {
    ///         for (i, segment) in self.segments.iter().enumerate() {
    ///             if i > 0 || self.global {
    ///                 // Double colon `::`
    ///                 tokens.append(Punct::new(':', Spacing::Joint));
    ///                 tokens.append(Punct::new(':', Spacing::Alone));
    ///             }
    ///             segment.to_tokens(tokens);
    ///         }
    ///     }
    /// }
    /// #
    /// # pub struct PathSegment;
    /// #
    /// # impl ToTokens for PathSegment {
    /// #     fn to_tokens(&self, tokens: &mut TokenStream) {
    /// #         unimplemented!()
    /// #     }
    /// # }
    /// ```
    fn to_tokens(&self, tokens: &mut TokenStream);

    /// Convert `self` directly into a `TokenStream` object.
    ///
    /// This method is implicitly implemented using `to_tokens`, and acts as a
    /// convenience method for consumers of the `ToTokens` trait.
    fn to_token_stream(&self) -> TokenStream {
        let mut tokens = TokenStream::new();
        self.to_tokens(&mut tokens);
        tokens
    }

    /// Convert `self` directly into a `TokenStream` object.
    ///
    /// This method is implicitly implemented using `to_tokens`, and acts as a
    /// convenience method for consumers of the `ToTokens` trait.
    fn into_token_stream(self) -> TokenStream
    where
        Self: Sized,
    {
        self.to_token_stream()
    }
}

impl<'a, T: ?Sized + ToTokens> ToTokens for &'a T {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<'a, T: ?Sized + ToTokens> ToTokens for &'a mut T {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ?Sized + ToTokens> ToTokens for Box<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ?Sized + ToTokens> ToTokens for Rc<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        (**self).to_tokens(tokens);
    }
}

impl<T: ToTokens> ToTokens for Option<T> {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        if let Some(ref t) = *self {
            t.to_tokens(tokens);
        }
    }
}

impl ToTokens for str {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Literal::string(self));
    }
}

impl ToTokens for String {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        self.as_str().to_tokens(tokens);
    }
}

macro_rules! primitive {
    ($($t:ident => $name:ident)*) => ($(
        impl ToTokens for $t {
            fn to_tokens(&self, tokens: &mut TokenStream) {
                tokens.append(Literal::$name(*self));
            }
        }
    )*)
}

primitive! {
    i8 => i8_suffixed
    i16 => i16_suffixed
    i32 => i32_suffixed
    i64 => i64_suffixed
    i128 => i128_suffixed
    isize => isize_suffixed

    u8 => u8_suffixed
    u16 => u16_suffixed
    u32 => u32_suffixed
    u64 => u64_suffixed
    u128 => u128_suffixed
    usize => usize_suffixed

    f32 => f32_suffixed
    f64 => f64_suffixed
}

impl ToTokens for char {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Literal::character(*self));
    }
}

impl ToTokens for bool {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        let word = if *self { "true" } else { "false" };
        tokens.append(Ident::new(word, Span::call_site()));
    }
}

impl ToTokens for Group {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Ident {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Punct {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for Literal {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(self.clone());
    }
}

impl ToTokens for TokenTree {
    fn to_tokens(&self, dst: &mut TokenStream) {
        dst.append(self.clone());
    }
}

impl ToTokens for TokenStream {
    fn to_tokens(&self, dst: &mut TokenStream) {
        dst.extend(iter::once(self.clone()));
    }

    fn into_token_stream(self) -> TokenStream {
        self
    }
}
<|endoftext|>


<commit_before>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! Tokens also track their source location within the source
//! text. This allows the parser using the tokenizer to provide
//! better error highlighting and other functionality.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! if let Ok(tokens) = tokenize("word") {
//!     assert_eq!(tokens.len(), 1);
//! }
//!
//! if let Ok(tokens) = tokenize("show interface") {
//!     assert_eq!(tokens.len(), 3);
//! }
//!
//! if let Ok(tokens) = tokenize("echo -n \"a b c\"") {
//!     assert_eq!(tokens.len(), 5);
//! }
//!
//! if let Ok(tokens) = tokenize("ls My\\ Documents") {
//!     assert_eq!(tokens.len(), 3);
//!     assert_eq!(tokens[2].text, "My\\ Documents");
//! }
//! ```

use std::fmt;
use std::error::Error;

/// A position within a body of text.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the body of text.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the body of text.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a body of text.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// Errors
#[derive(Clone,Debug)]
pub enum TokenizerError {
    /// Character not allowed here
    CharacterNotAllowedHere(usize),

    /// Special not yet implemented
    SpecialNotYetImplemented(usize),

    /// Escaping backslash at end of input
    EscapingBackslashAtEndOfInput,

    /// Unclosed double quote at end of input
    UnclosedDoubleQuoteAtEndOfInput,

    /// Escaped double quote at end of input
    EscapedDoubleQuoteAtEndOfInput,
}

impl Error for TokenizerError {
    fn description(&self) -> &str {
        match *self {
            TokenizerError::CharacterNotAllowedHere(_) => "Character not allowed here",
            TokenizerError::SpecialNotYetImplemented(_) => "Special not yet implemented",
            TokenizerError::EscapingBackslashAtEndOfInput => "Escaping backlash at end of input",
            TokenizerError::UnclosedDoubleQuoteAtEndOfInput => {
                "Unclosed double quote at end of input"
            }
            TokenizerError::EscapedDoubleQuoteAtEndOfInput => {
                "Escaped double quote at end of input"
            }
        }
    }
}

impl fmt::Display for TokenizerError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        self.description().fmt(f)
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the body of text. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' || c == '?' || c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) -> Result<(), TokenizerError> {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' || c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        return Err(TokenizerError::CharacterNotAllowedHere(offset));
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        return Err(TokenizerError::CharacterNotAllowedHere(offset));
                    };
                }
                State::Special => {
                    return Err(TokenizerError::SpecialNotYetImplemented(offset));
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word | State::Whitespace => self.reduce(),
            State::WordBackslash => return Err(TokenizerError::EscapingBackslashAtEndOfInput),
            State::Doublequote => return Err(TokenizerError::UnclosedDoubleQuoteAtEndOfInput),
            State::DoublequoteBackslash => {
                return Err(TokenizerError::EscapedDoubleQuoteAtEndOfInput)
            }
            State::Special => {
                return Err(TokenizerError::SpecialNotYetImplemented(self.text.len() - 1))
            }
        }

        Ok(())
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Result<Vec<Token>, TokenizerError> {
    let mut tokenizer = Tokenizer::new(text);
    match tokenizer.tokenize() {
        Ok(_) => Ok(tokenizer.tokens),
        Err(error) => Err(error),
    }
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        match tokenize("") {
            Ok(ts) => assert_eq!(ts.len(), 0),
            _ => {}
        };
    }

    #[test]
    fn single_word() {
        match tokenize("a") {
            Ok(ts) => {
                assert_eq!(ts.len(), 1);
                assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
            }
            _ => {}
        };
    }

    #[test]
    fn multiple_words() {
        match tokenize(" aa bb  ccc ") {
            Ok(ts) => {
                assert_eq!(ts.len(), 7);
                assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
                assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
                assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
                assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
                assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
                assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
                assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
            }
            _ => {}
        };
    }

    #[test]
    fn quoted_text() {
        match tokenize("a \"b c\"") {
            Ok(ts) => {
                assert_eq!(ts.len(), 3);
                assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
                assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
                assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
            }
            _ => {}
        };
    }

    #[test]
    fn escaped_whitespace_in_word() {
        match tokenize("a\\ b") {
            Ok(ts) => {
                assert_eq!(ts.len(), 1);
                assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
            }
            _ => {}
        };
    }
}
<commit_msg>tokenizer: added some TokenizerError tests (#3)<commit_after>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! Tokens also track their source location within the source
//! text. This allows the parser using the tokenizer to provide
//! better error highlighting and other functionality.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! if let Ok(tokens) = tokenize("word") {
//!     assert_eq!(tokens.len(), 1);
//! }
//!
//! if let Ok(tokens) = tokenize("show interface") {
//!     assert_eq!(tokens.len(), 3);
//! }
//!
//! if let Ok(tokens) = tokenize("echo -n \"a b c\"") {
//!     assert_eq!(tokens.len(), 5);
//! }
//!
//! if let Ok(tokens) = tokenize("ls My\\ Documents") {
//!     assert_eq!(tokens.len(), 3);
//!     assert_eq!(tokens[2].text, "My\\ Documents");
//! }
//! ```

use std::fmt;
use std::error::Error;

/// A position within a body of text.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the body of text.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the body of text.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a body of text.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// Errors
#[derive(Clone,Debug)]
pub enum TokenizerError {
    /// Character not allowed here
    CharacterNotAllowedHere(usize),

    /// Special not yet implemented
    SpecialNotYetImplemented(usize),

    /// Escaping backslash at end of input
    EscapingBackslashAtEndOfInput,

    /// Unclosed double quote at end of input
    UnclosedDoubleQuoteAtEndOfInput,

    /// Escaped double quote at end of input
    EscapedDoubleQuoteAtEndOfInput,
}

impl Error for TokenizerError {
    fn description(&self) -> &str {
        match *self {
            TokenizerError::CharacterNotAllowedHere(_) => "Character not allowed here",
            TokenizerError::SpecialNotYetImplemented(_) => "Special not yet implemented",
            TokenizerError::EscapingBackslashAtEndOfInput => "Escaping backlash at end of input",
            TokenizerError::UnclosedDoubleQuoteAtEndOfInput => {
                "Unclosed double quote at end of input"
            }
            TokenizerError::EscapedDoubleQuoteAtEndOfInput => {
                "Escaped double quote at end of input"
            }
        }
    }
}

impl fmt::Display for TokenizerError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        self.description().fmt(f)
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the body of text. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' || c == '?' || c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) -> Result<(), TokenizerError> {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' || c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        return Err(TokenizerError::CharacterNotAllowedHere(offset));
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        return Err(TokenizerError::CharacterNotAllowedHere(offset));
                    };
                }
                State::Special => {
                    return Err(TokenizerError::SpecialNotYetImplemented(offset));
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word | State::Whitespace => self.reduce(),
            State::WordBackslash => return Err(TokenizerError::EscapingBackslashAtEndOfInput),
            State::Doublequote => return Err(TokenizerError::UnclosedDoubleQuoteAtEndOfInput),
            State::DoublequoteBackslash => {
                return Err(TokenizerError::EscapedDoubleQuoteAtEndOfInput)
            }
            State::Special => {
                return Err(TokenizerError::SpecialNotYetImplemented(self.text.len() - 1))
            }
        }

        Ok(())
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Result<Vec<Token>, TokenizerError> {
    let mut tokenizer = Tokenizer::new(text);
    match tokenizer.tokenize() {
        Ok(_) => Ok(tokenizer.tokens),
        Err(error) => Err(error),
    }
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        match tokenize("") {
            Ok(ts) => assert_eq!(ts.len(), 0),
            _ => {}
        };
    }

    #[test]
    fn single_word() {
        match tokenize("a") {
            Ok(ts) => {
                assert_eq!(ts.len(), 1);
                assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
            }
            _ => {}
        };
    }

    #[test]
    fn multiple_words() {
        match tokenize(" aa bb  ccc ") {
            Ok(ts) => {
                assert_eq!(ts.len(), 7);
                assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
                assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
                assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
                assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
                assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
                assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
                assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
            }
            _ => {}
        };
    }

    #[test]
    fn quoted_text() {
        match tokenize("a \"b c\"") {
            Ok(ts) => {
                assert_eq!(ts.len(), 3);
                assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
                assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
                assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
            }
            _ => {}
        };
    }

    #[test]
    fn escaped_whitespace_in_word() {
        match tokenize("a\\ b") {
            Ok(ts) => {
                assert_eq!(ts.len(), 1);
                assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
            }
            _ => {}
        };
    }

    #[test]
    fn character_not_allowed_here() {
        match tokenize("ab \\!") {
            Err(TokenizerError::CharacterNotAllowedHere(_)) => {}
            _ => panic!(),
        };

        match tokenize("ab \"\\ ab") {
            Err(TokenizerError::CharacterNotAllowedHere(_)) => {}
            _ => panic!(),
        };
    }

    // TODO: Test TokenizeError::SpecialNotYetImplemented

    #[test]
    #[should_panic]
    fn escaping_backslash_at_end_of_input() {
        match tokenize("ab \\") {
            Err(TokenizerError::EscapingBackslashAtEndOfInput) => panic!(),
            _ => {}
        }
    }

    #[test]
    #[should_panic]
    fn unclosed_double_quote_at_end_of_input() {
        match tokenize("ab \"") {
            Err(TokenizerError::UnclosedDoubleQuoteAtEndOfInput) => panic!(),
            _ => {}
        }
    }

    #[test]
    #[should_panic]
    fn escaped_double_quote_at_end_of_input() {
        match tokenize("ab \"\\") {
            Err(TokenizerError::EscapedDoubleQuoteAtEndOfInput) => panic!(),
            _ => {}
        }
    }
}
<|endoftext|>


<commit_before>#[deriving(PartialEq, Show)]
pub enum Token {
    Comma,
    Number(f64),
    ParenClose,
    ParenOpen,
    Word(String),
}

pub fn tokenize(input: &str) -> Tokenizer {
    Tokenizer {text: String::from_str(input)}
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\n' | '\r' | '\t' | ' ' => true,
        _                        => false,
    }
}

fn is_numberlike(c: char) -> bool {
    match c {
        c if c.is_numeric() => true,
        '.' | '-'           => true,
        _                   => false,
    }
}

struct Tokenizer {
    text: String
}

impl Iterator<Token> for Tokenizer {
    fn next(&mut self) -> Option<Token> {
        let popped_char = self.pop_char();
        if popped_char.is_none() {
            return None
        }
        let next_char = popped_char.unwrap();

        match next_char {
            '\0' => None,
            '(' => Some(Token::ParenOpen),
            ')' => Some(Token::ParenClose),
            ',' => Some(Token::Comma),
            c if is_whitespace(c) => self.next(),
            c if is_numberlike(c) => {
                let number = c.to_string() + self.read_until_whitespace().as_slice();
                match number.parse::<f64>() {
                    Some(parsed_num) => Some(Token::Number(parsed_num)),
                    None => panic!("Could not parse number: {}", number),
                }
            }
            c => {
                let word = c.to_string() + self.read_until_whitespace().as_slice();
                Some(Token::Word(word))
            }
        }
    }
}

impl Tokenizer {
    fn pop_char(&mut self) -> Option<char> {
        if self.text.is_empty() {
            None
        } else {
            self.text.remove(0)
        }
    }

    fn read_until_whitespace(&mut self) -> String {
        let popped_char = self.pop_char();
        if popped_char.is_none() {
            return "".to_string()
        }
        let next_char = popped_char.unwrap();

        match next_char {
            '\0' | '(' | ')' | ',' => {
                self.text.insert(0, next_char);
                "".to_string()
            }
            c if is_whitespace(c) => "".to_string(),
            _ => next_char.to_string() + self.read_until_whitespace().as_slice(),
        }
    }
}

#[test]
fn test_tokenizer_empty() {
    let test_str = "";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![]);
}

#[test]
fn test_tokenizer_1word() {
    let test_str = "hello";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
    ]);
}

#[test]
fn test_tokenizer_2words() {
    let test_str = "hello world";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
        Token::Word("world".to_string()),
    ]);
}

#[test]
fn test_tokenizer_1number() {
    let test_str = "4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_2numbers() {
    let test_str = ".4 -2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(0.4),
        Token::Number(-2.0),
    ]);
}

#[test]
fn test_tokenizer_point() {
    let test_str = "POINT (10 -20)";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("POINT".to_string()),
        Token::ParenOpen,
        Token::Number(10.0),
        Token::Number(-20.0),
        Token::ParenClose,
    ]);
}
<commit_msg>Ignore leading '+' for numbers<commit_after>#[deriving(PartialEq, Show)]
pub enum Token {
    Comma,
    Number(f64),
    ParenClose,
    ParenOpen,
    Word(String),
}

pub fn tokenize(input: &str) -> Tokenizer {
    Tokenizer {text: String::from_str(input)}
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\n' | '\r' | '\t' | ' ' => true,
        _                        => false,
    }
}

fn is_numberlike(c: char) -> bool {
    match c {
        c if c.is_numeric() => true,
        '.' | '-' | '+'     => true,
        _                   => false,
    }
}

struct Tokenizer {
    text: String
}

impl Iterator<Token> for Tokenizer {
    fn next(&mut self) -> Option<Token> {
        let popped_char = self.pop_char();
        if popped_char.is_none() {
            return None
        }
        let next_char = popped_char.unwrap();

        match next_char {
            '\0' => None,
            '(' => Some(Token::ParenOpen),
            ')' => Some(Token::ParenClose),
            ',' => Some(Token::Comma),
            c if is_whitespace(c) => self.next(),
            c if is_numberlike(c) => {
                let mut number = c.to_string() + self.read_until_whitespace().as_slice();
                number = number.trim_left_chars('+').to_string();
                match number.parse::<f64>() {
                    Some(parsed_num) => Some(Token::Number(parsed_num)),
                    None => panic!("Could not parse number: {}", number),
                }
            }
            c => {
                let word = c.to_string() + self.read_until_whitespace().as_slice();
                Some(Token::Word(word))
            }
        }
    }
}

impl Tokenizer {
    fn pop_char(&mut self) -> Option<char> {
        if self.text.is_empty() {
            None
        } else {
            self.text.remove(0)
        }
    }

    fn read_until_whitespace(&mut self) -> String {
        let popped_char = self.pop_char();
        if popped_char.is_none() {
            return "".to_string()
        }
        let next_char = popped_char.unwrap();

        match next_char {
            '\0' | '(' | ')' | ',' => {
                self.text.insert(0, next_char);
                "".to_string()
            }
            c if is_whitespace(c) => "".to_string(),
            _ => next_char.to_string() + self.read_until_whitespace().as_slice(),
        }
    }
}

#[test]
fn test_tokenizer_empty() {
    let test_str = "";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![]);
}

#[test]
fn test_tokenizer_1word() {
    let test_str = "hello";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
    ]);
}

#[test]
fn test_tokenizer_2words() {
    let test_str = "hello world";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
        Token::Word("world".to_string()),
    ]);
}

#[test]
fn test_tokenizer_1number() {
    let test_str = "4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_1number_plus() {
    let test_str = "+4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_2numbers() {
    let test_str = ".4 -2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(0.4),
        Token::Number(-2.0),
    ]);
}

#[test]
fn test_tokenizer_point() {
    let test_str = "POINT (10 -20)";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("POINT".to_string()),
        Token::ParenOpen,
        Token::Number(10.0),
        Token::Number(-20.0),
        Token::ParenClose,
    ]);
}
<|endoftext|>


<commit_before>use std::alloc;

#[no_mangle]
pub fn call_indirect1(f: fn(u16), x: u16) { f(x); }

#[no_mangle]
pub static mut mem8: *mut u8 = 0 as *mut u8;

#[no_mangle]
pub fn allocate_memory(size: u32) -> u32 {
    let layout = alloc::Layout::from_size_align(size as usize, 0x1000).unwrap();
    let ptr = unsafe { alloc::alloc(layout) as u32 };
    unsafe {
        mem8 = ptr as *mut u8;
    };
    ptr
}
<commit_msg>Log initial guest memory allocation, assert only once<commit_after>use std::alloc;
use std::ptr;

#[no_mangle]
pub fn call_indirect1(f: fn(u16), x: u16) { f(x); }

#[no_mangle]
pub static mut mem8: *mut u8 = ptr::null_mut();

#[no_mangle]
pub fn allocate_memory(size: u32) -> u32 {
    unsafe {
        dbg_assert!(mem8.is_null())
    };
    dbg_log!("Allocate memory size={}m", size >> 20);
    let layout = alloc::Layout::from_size_align(size as usize, 0x1000).unwrap();
    let ptr = unsafe { alloc::alloc(layout) as u32 };
    unsafe {
        mem8 = ptr as *mut u8;
    };
    ptr
}
<|endoftext|>


<commit_before>// Copyright 2014-2016 Johannes Köster, Martin Larralde.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! One-way open reading frame (ORF) finder algorithm.
//!
//! Complexity: O(n).
//!
//! # Example
//!
//! ```
//! use bio::seq_analysis::orf::{Finder, Orf};
//! let start_codons = vec![b"ATG"];
//! let stop_codons = vec![b"TGA", b"TAG", b"TAA"];
//! let min_len = 50;
//! let finder = Finder::new(start_codons, stop_codons, min_len);
//!
//! let sequence = b"ACGGCTAGAAAAGGCTAGAAAA";
//!
//! for Orf { start, end, offset } in finder.find_all(sequence) {
//!     let orf = &sequence[start..end];
//!     //...do something with orf sequence...
//! }
//! ```
//!
//! Right now the only way to check the reverse strand for ORF is to use
//! the `alphabet::dna::RevComp` struct and to check for both sequences.
//! But that's not so performance friendly, as the reverse complementation and the ORF research
//! could go on at the same time.

use std::borrow::Borrow;
use std::collections::VecDeque;
use std::iter;

/// An implementation of a naive algorithm finder
// Implementation note:
//
// VecDeque is used rather than the obvious [u8; 3] to represent
// codons because a VecDeque<u8> is used to represent a sliding codon
// (see: State.codon) window which unfortunately, cannot be compared
// to [u8; 3].
pub struct Finder {
    start_codons: Vec<VecDeque<u8>>,
    stop_codons: Vec<VecDeque<u8>>,
    min_len: usize,
}

impl Finder {
    /// Create a new instance of a finder for the given start and stop codons and the minimum
    /// length of an ORF.
    pub fn new<'a>(
        start_codons: Vec<&'a [u8; 3]>,
        stop_codons: Vec<&'a [u8; 3]>,
        min_len: usize,
    ) -> Self {
        Finder {
            start_codons: start_codons
                .iter()
                .map(|x| x.iter().map(|&x| x as u8).collect::<VecDeque<u8>>())
                .collect(),
            stop_codons: stop_codons
                .iter()
                .map(|x| x.iter().map(|&x| x as u8).collect::<VecDeque<u8>>())
                .collect(),
            min_len,
        }
    }

    /// Find all ORFs in the given sequence
    pub fn find_all<C, T>(&self, seq: T) -> Matches<'_, C, T::IntoIter>
    where
        C: Borrow<u8>,
        T: IntoIterator<Item = C>,
    {
        Matches {
            finder: self,
            state: State::new(),
            seq: seq.into_iter().enumerate(),
        }
    }
}

/// An ORF representation with start and end position of said ORF,
/// as well as offset of the reading frame (1,2,3) and strand location
// (current: +, reverse complementary: -).
#[derive(Debug, PartialEq, Eq)]
pub struct Orf {
    pub start: usize,
    pub end: usize,
    pub offset: i8,
}

/// The current algorithm state.
struct State {
    start_pos: [Option<usize>; 3],
    codon: VecDeque<u8>,
}

impl State {
    /// Create new state.
    pub fn new() -> Self {
        State {
            start_pos: [None, None, None],
            codon: VecDeque::new(),
        }
    }
}

/// Iterator over offset, start position, end position and sequence of matched ORFs.
pub struct Matches<'a, C, T>
where
    C: Borrow<u8>,
    T: Iterator<Item = C>,
{
    finder: &'a Finder,
    state: State,
    seq: iter::Enumerate<T>,
}

impl<'a, C, T> Iterator for Matches<'a, C, T>
where
    C: Borrow<u8>,
    T: Iterator<Item = C>,
{
    type Item = Orf;

    fn next(&mut self) -> Option<Orf> {
        let mut result: Option<Orf> = None;
        let mut offset: usize;

        for (index, nuc) in self.seq.by_ref() {
            // update the codon
            if self.state.codon.len() >= 3 {
                self.state.codon.pop_front();
            }
            self.state.codon.push_back(*nuc.borrow());
            offset = (index + 1) % 3;

            // inside orf
            if self.state.start_pos[offset].is_some() {
                // check if leaving orf
                if self.finder.stop_codons.contains(&self.state.codon) {
                    // check if length is sufficient
                    if index + 1 - self.state.start_pos[offset].unwrap() > self.finder.min_len {
                        // build results
                        result = Some(Orf {
                            start: self.state.start_pos[offset].unwrap() - 2,
                            end: index + 1,
                            offset: offset as i8,
                        });
                    }
                    // reinitialize
                    self.state.start_pos[offset] = None;
                }
            // check if entering orf
            } else if self.finder.start_codons.contains(&self.state.codon) {
                self.state.start_pos[offset] = Some(index);
            }
            if result.is_some() {
                return result;
            }
        }
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn basic_finder() -> Finder {
        let start_codons = vec![b"ATG"];
        let stop_codons = vec![b"TGA", b"TAG", b"TAA"];
        let min_len = 5;
        Finder::new(start_codons, stop_codons, min_len)
    }

    #[test]
    fn test_no_orf() {
        let finder = basic_finder();
        let sequence = b"ACGGCTAGAAAAGGCTAGAAAA";
        assert!(finder.find_all(sequence).next().is_none());
    }

    #[test]
    fn test_one_orf_no_offset() {
        let finder = basic_finder();
        let sequence = b"GGGATGGGGTGAGGG";
        let expected = vec![Orf {
            start: 3,
            end: 12,
            offset: 0,
        }];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }

    #[test]
    fn test_one_orf_with_offset() {
        let finder = basic_finder();
        let sequence = b"AGGGATGGGGTGAGGG";
        let expected = vec![Orf {
            start: 4,
            end: 13,
            offset: 1,
        }];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }

    #[test]
    fn test_two_orfs_different_offsets() {
        let finder = basic_finder();
        let sequence = b"ATGGGGTGAGGGGGATGGAAAAATAAG";
        let expected = vec![
            Orf {
                start: 0,
                end: 9,
                offset: 0,
            },
            Orf {
                start: 14,
                end: 26,
                offset: 2,
            },
        ];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }
}
<commit_msg>feat: detect nested ORFs (#501)<commit_after>// Copyright 2014-2016 Johannes Köster, Martin Larralde.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! One-way open reading frame (ORF) finder algorithm.
//!
//! Complexity: O(n).
//!
//! # Example
//!
//! ```
//! use bio::seq_analysis::orf::{Finder, Orf};
//! let start_codons = vec![b"ATG"];
//! let stop_codons = vec![b"TGA", b"TAG", b"TAA"];
//! let min_len = 50;
//! let finder = Finder::new(start_codons, stop_codons, min_len);
//!
//! let sequence = b"ACGGCTAGAAAAGGCTAGAAAA";
//!
//! for Orf { start, end, offset } in finder.find_all(sequence) {
//!     let orf = &sequence[start..end];
//!     //...do something with orf sequence...
//! }
//! ```
//!
//! Right now the only way to check the reverse strand for ORF is to use
//! the `alphabet::dna::RevComp` struct and to check for both sequences.
//! But that's not so performance friendly, as the reverse complementation and the ORF research
//! could go on at the same time.

use std::borrow::Borrow;
use std::collections::VecDeque;
use std::iter;

/// An implementation of a naive algorithm finder
// Implementation note:
//
// VecDeque is used rather than the obvious [u8; 3] to represent
// codons because a VecDeque<u8> is used to represent a sliding codon
// (see: State.codon) window which unfortunately, cannot be compared
// to [u8; 3].
pub struct Finder {
    start_codons: Vec<VecDeque<u8>>,
    stop_codons: Vec<VecDeque<u8>>,
    min_len: usize,
}

impl Finder {
    /// Create a new instance of a finder for the given start and stop codons and the minimum
    /// length of an ORF.
    pub fn new<'a>(
        start_codons: Vec<&'a [u8; 3]>,
        stop_codons: Vec<&'a [u8; 3]>,
        min_len: usize,
    ) -> Self {
        Finder {
            start_codons: start_codons
                .iter()
                .map(|x| x.iter().map(|&x| x as u8).collect::<VecDeque<u8>>())
                .collect(),
            stop_codons: stop_codons
                .iter()
                .map(|x| x.iter().map(|&x| x as u8).collect::<VecDeque<u8>>())
                .collect(),
            min_len,
        }
    }

    /// Find all ORFs in the given sequence
    pub fn find_all<C, T>(&self, seq: T) -> Matches<'_, C, T::IntoIter>
    where
        C: Borrow<u8>,
        T: IntoIterator<Item = C>,
    {
        Matches {
            finder: self,
            state: State::new(),
            seq: seq.into_iter().enumerate(),
        }
    }
}

/// An ORF representation with start and end position of said ORF,
/// as well as offset of the reading frame (1,2,3) and strand location
// (current: +, reverse complementary: -).
#[derive(Debug, PartialEq, Eq)]
pub struct Orf {
    pub start: usize,
    pub end: usize,
    pub offset: i8,
}

/// The current algorithm state.
struct State {
    start_pos: [Vec<usize>; 3],
    codon: VecDeque<u8>,
    found: VecDeque<Orf>,
}

impl State {
    /// Create new state.
    pub fn new() -> Self {
        State {
            start_pos: [Vec::new(), Vec::new(), Vec::new()],
            codon: VecDeque::new(),
            found: VecDeque::new(),
        }
    }
}

/// Iterator over offset, start position, end position and sequence of matched ORFs.
pub struct Matches<'a, C, T>
where
    C: Borrow<u8>,
    T: Iterator<Item = C>,
{
    finder: &'a Finder,
    state: State,
    seq: iter::Enumerate<T>,
}

impl<'a, C, T> Iterator for Matches<'a, C, T>
where
    C: Borrow<u8>,
    T: Iterator<Item = C>,
{
    type Item = Orf;

    fn next(&mut self) -> Option<Orf> {
        let mut offset: usize;

        // return any orfs already found
        if !self.state.found.is_empty() {
            return self.state.found.pop_front();
        }

        for (index, nuc) in self.seq.by_ref() {
            // update the codon
            if self.state.codon.len() >= 3 {
                self.state.codon.pop_front();
            }
            self.state.codon.push_back(*nuc.borrow());
            offset = (index + 1) % 3;

            // check if entering orf
            if self.finder.start_codons.contains(&self.state.codon) {
                self.state.start_pos[offset].push(index);
            }
            // inside orf
            if !self.state.start_pos[offset].is_empty() {
                // check if leaving orf
                if self.finder.stop_codons.contains(&self.state.codon) {
                    for start_pos in &self.state.start_pos[offset] {
                        // check if length is sufficient
                        if index + 1 - start_pos > self.finder.min_len {
                            // build results
                            self.state.found.push_back(Orf {
                                start: start_pos - 2,
                                end: index + 1,
                                offset: offset as i8,
                            });
                        // if the first orf is too short, so are the others
                        } else {
                            break;
                        }
                    }
                    // reinitialize
                    self.state.start_pos[offset] = Vec::new();
                }
            }
            if !self.state.found.is_empty() {
                return self.state.found.pop_front();
            }
        }
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn basic_finder() -> Finder {
        let start_codons = vec![b"ATG"];
        let stop_codons = vec![b"TGA", b"TAG", b"TAA"];
        let min_len = 5;
        Finder::new(start_codons, stop_codons, min_len)
    }

    #[test]
    fn test_no_orf() {
        let finder = basic_finder();
        let sequence = b"ACGGCTAGAAAAGGCTAGAAAA";
        assert!(finder.find_all(sequence).next().is_none());
    }

    #[test]
    fn test_one_orf_no_offset() {
        let finder = basic_finder();
        let sequence = b"GGGATGGGGTGAGGG";
        let expected = vec![Orf {
            start: 3,
            end: 12,
            offset: 0,
        }];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }

    #[test]
    fn test_one_orf_with_offset() {
        let finder = basic_finder();
        let sequence = b"AGGGATGGGGTGAGGG";
        let expected = vec![Orf {
            start: 4,
            end: 13,
            offset: 1,
        }];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }

    #[test]
    fn test_two_orfs_different_offsets() {
        let finder = basic_finder();
        let sequence = b"ATGGGGTGAGGGGGATGGAAAAATAAG";
        let expected = vec![
            Orf {
                start: 0,
                end: 9,
                offset: 0,
            },
            Orf {
                start: 14,
                end: 26,
                offset: 2,
            },
        ];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }

    #[test]
    fn test_three_nested_and_offset_orfs() {
        let finder = basic_finder();
        let sequence = b"ATGGGGATGGGGGGATGGAAAAATAAGTAG";
        let expected = vec![
            Orf {
                start: 14,
                end: 26,
                offset: 2,
            },
            Orf {
                start: 0,
                end: 30,
                offset: 0,
            },
            Orf {
                start: 6,
                end: 30,
                offset: 0,
            },
        ];
        assert_eq!(expected, finder.find_all(sequence).collect::<Vec<Orf>>());
    }
}
<|endoftext|>