<commit_before>use token::Token;
use token::Token::*;

#[derive(Default)]
struct CompileState {
    output: String,
    loopnum: i32,
    ifnum: i32,
    outbuffsize: i32,
}

/// Takes an offset from the current cell, and returns a string in assembly code
/// representing the register or memory region it is stored in
fn offset_to_operand(offset: i32) -> String {
    if offset == 0 {
        "%r12".to_string()
    } else {
        format!("{}(%rbx)", (offset * 8))
    }
}

fn compile_iter(state: &mut CompileState, tokens: Vec<Token>) {
    let mut outbuffpos = 0;
    for token in tokens {
        match token {
            Add(offset, value) => {
                let dest = offset_to_operand(offset);
                if value == 1 && dest == "%r12" {
                    state.output.push_str("    inc %r12\n");
                } else if value >= 1 {
                    state.output.push_str(&format!("    addq ${}, {}\n", value, dest));
                } else if value == -1 && dest == "%r12" {
                    state.output.push_str("    dec %r12\n");
                } else if value <= -1 {
                    state.output.push_str(&format!("    subq ${}, {}\n", -value, dest));
                }
            }
            MulCopy(src_idx, dest_idx, mul) => {
                let mut src = offset_to_operand(src_idx);
                let dest = offset_to_operand(dest_idx);

                if mul != -1 && mul != 1 {
                    state.output.push_str(&format!(concat!("    movq {}, %rax\n",
                                                           "    movq ${}, %rdx\n",
                                                           "    mulq %rdx\n"),
                                                   src,
                                                   mul.abs()));
                    src = "%rax".to_string();
                } else if src != "%r12" && dest != "%r12" {
                    // x86 cannot move memory to memory
                    state.output.push_str(&format!("    movq {}, %rax\n", src));
                    src = "%rax".to_string();
                }

                if mul > 0 {
                    state.output.push_str(&format!("    addq {}, {}\n", src, dest));
                } else {
                    state.output.push_str(&format!("    subq {}, {}\n", src, dest));
                }
            }
            Set(offset, value) => {
                if offset == 0 && value == 0 {
                    state.output.push_str("    xor %r12, %r12\n");
                } else {
                    state.output.push_str(&format!("    movq ${}, {}\n", value, offset_to_operand(offset)));
                }
            }
            Move(offset) => {
                if offset != 0 {
                    state.output.push_str("    movq %r12, (%rbx)\n");
                    if offset > 0 {
                        state.output.push_str(&format!("    addq ${}, %rbx\n", offset * 8));
                    } else {
                        state.output.push_str(&format!("    subq ${}, %rbx\n", -offset * 8));
                    }
                    state.output.push_str("    movq (%rbx), %r12\n");
                }
            }
            Loop(content) => {
                state.loopnum += 1;
                let curloop = state.loopnum;
                state.output.push_str(&format!(concat!("    jmp endloop{}\n", "    loop{}:\n"),
                                               curloop,
                                               curloop));

                compile_iter(state, content);

                state.output.push_str(&format!(concat!("    endloop{}:\n",
                                                       "    test %r12, %r12\n",
                                                       "    jnz loop{}\n"),
                                               curloop,
                                               curloop))
            }
            If(offset, content) => {
                state.ifnum += 1;
                let curif = state.ifnum;
                if offset == 0 {
                    state.output.push_str("    test %r12, %r12\n");
                } else {
                    state.output.push_str(&format!("    cmpq $0, {}(%rbx)\n", offset * 8));
                }
                state.output.push_str(&format!("    jz endif{}\n", curif));

                compile_iter(state, content);

                state.output.push_str(&format!("    endif{}:\n", curif))
            }
            Scan(offset) => {
                // Slighly more optimal than normal loop and move
                state.loopnum += 1;
                state.output.push_str(&format!(concat!("    movq %r12, (%rbx)\n",
                                                       "    jmp endloop{num}\n",
                                                       "    loop{num}:\n",
                                                       "    {add_sub} ${shift}, %rbx\n",
                                                       "    endloop{num}:\n",
                                                       "    cmp $0, (%rbx)\n",
                                                       "    jnz loop{num}\n",
                                                       "    movq (%rbx), %r12\n"),
                                               num = state.loopnum,
                                               add_sub = if offset > 0 { "addq" } else { "subq" },
                                               shift = offset.abs() * 8));
            }
            Input => {
                state.output.push_str(concat!("\n    xor %rax, %rax\n",
                                              "    xor %rdi, %rdi\n",
                                              "    movq %rbx, %rsi\n",
                                              "    movq $1, %rdx\n",
                                              "    syscall\n",
                                              "    movq (%rbx), %r12\n\n"))
            }
            LoadOut(offset, add) => {
                let outaddr = format!("(strbuff+{})", outbuffpos);
                if offset == 0 {
                    state.output.push_str(&format!("    movq %r12, {}\n", outaddr));
                } else {
                    state.output.push_str(&format!("    movq {}(%rbx), %rax\n", offset * 8));
                    state.output.push_str(&format!("    movq %rax, {}\n", outaddr));
                }
                if add > 0 {
                    state.output.push_str(&format!("    addb ${}, {}\n", add, outaddr));
                } else if add < 0 {
                    state.output.push_str(&format!("    subb ${}, {}\n", -add, outaddr));
                }
                outbuffpos += 1;
            }
            LoadOutSet(value) => {
                let outaddr = format!("(strbuff+{})", outbuffpos);
                state.output.push_str(&format!("    movq ${}, {}\n", value, outaddr));
                outbuffpos += 1;
            }
            Output => {
                state.output.push_str(&format!(concat!("    movq $1, %rax\n",
                                                       "    movq $1, %rdi\n",
                                                       "    movq $strbuff, %rsi\n",
                                                       "    movq ${}, %rdx\n",
                                                       "    syscall\n\n"),
                                               outbuffpos));

                if state.outbuffsize < outbuffpos + 8 {
                    state.outbuffsize = outbuffpos + 8;
                }
                outbuffpos = 0;
            }
        }
    }
}

pub fn compile(tokens: Vec<Token>, tape_size: i32) -> String {
    let mut state = CompileState::default();

    compile_iter(&mut state, tokens);

    format!(concat!(".section .bss\n",
                    "    .lcomm strbuff, {outbuffsize}\n",
                    "    .lcomm mem, {}\n",
                    "    .set startidx, mem + {}\n",
                    ".section .text\n",
                    ".global _start\n",
                    "_start:\n",
                    "    xor %r12, %r12\n",
                    "    movq $startidx, %rbx\n\n",
                    "{}\n",
                    // Exit syscall
                    "    movq $60, %rax\n",
                    "    movq $0, %rdi\n",
                    "    syscall\n"),
            tape_size,
            tape_size / 2,
            state.output,
            outbuffsize = state.outbuffsize)
}
<commit_msg>Simplify handling of Move in compiler<commit_after>use token::Token;
use token::Token::*;

#[derive(Default)]
struct CompileState {
    output: String,
    loopnum: i32,
    ifnum: i32,
    outbuffsize: i32,
}

/// Takes an offset from the current cell, and returns a string in assembly code
/// representing the register or memory region it is stored in
fn offset_to_operand(offset: i32) -> String {
    if offset == 0 {
        "%r12".to_string()
    } else {
        format!("{}(%rbx)", (offset * 8))
    }
}

fn compile_iter(state: &mut CompileState, tokens: Vec<Token>) {
    let mut outbuffpos = 0;
    for token in tokens {
        match token {
            Add(offset, value) => {
                let dest = offset_to_operand(offset);
                if value == 1 && dest == "%r12" {
                    state.output.push_str("    inc %r12\n");
                } else if value >= 1 {
                    state.output.push_str(&format!("    addq ${}, {}\n", value, dest));
                } else if value == -1 && dest == "%r12" {
                    state.output.push_str("    dec %r12\n");
                } else if value <= -1 {
                    state.output.push_str(&format!("    subq ${}, {}\n", -value, dest));
                }
            }
            MulCopy(src_idx, dest_idx, mul) => {
                let mut src = offset_to_operand(src_idx);
                let dest = offset_to_operand(dest_idx);

                if mul != -1 && mul != 1 {
                    state.output.push_str(&format!(concat!("    movq {}, %rax\n",
                                                           "    movq ${}, %rdx\n",
                                                           "    mulq %rdx\n"),
                                                   src,
                                                   mul.abs()));
                    src = "%rax".to_string();
                } else if src != "%r12" && dest != "%r12" {
                    // x86 cannot move memory to memory
                    state.output.push_str(&format!("    movq {}, %rax\n", src));
                    src = "%rax".to_string();
                }

                if mul > 0 {
                    state.output.push_str(&format!("    addq {}, {}\n", src, dest));
                } else {
                    state.output.push_str(&format!("    subq {}, {}\n", src, dest));
                }
            }
            Set(offset, value) => {
                if offset == 0 && value == 0 {
                    state.output.push_str("    xor %r12, %r12\n");
                } else {
                    state.output.push_str(&format!("    movq ${}, {}\n", value, offset_to_operand(offset)));
                }
            }
            Move(offset) => {
                if offset != 0 {
                    state.output.push_str(&format!(concat!("    movq %r12, (%rbx)\n",
                                                           "    {add_sub} ${shift}, %rbx\n",
                                                           "    movq (%rbx), %r12\n"),
                                                   add_sub = if offset > 0 { "addq" } else { "subq" },
                                                   shift = offset.abs() * 8));
                }
            }
            Loop(content) => {
                state.loopnum += 1;
                let curloop = state.loopnum;
                state.output.push_str(&format!(concat!("    jmp endloop{}\n", "    loop{}:\n"),
                                               curloop,
                                               curloop));

                compile_iter(state, content);

                state.output.push_str(&format!(concat!("    endloop{}:\n",
                                                       "    test %r12, %r12\n",
                                                       "    jnz loop{}\n"),
                                               curloop,
                                               curloop))
            }
            If(offset, content) => {
                state.ifnum += 1;
                let curif = state.ifnum;
                if offset == 0 {
                    state.output.push_str("    test %r12, %r12\n");
                } else {
                    state.output.push_str(&format!("    cmpq $0, {}(%rbx)\n", offset * 8));
                }
                state.output.push_str(&format!("    jz endif{}\n", curif));

                compile_iter(state, content);

                state.output.push_str(&format!("    endif{}:\n", curif))
            }
            Scan(offset) => {
                // Slighly more optimal than normal loop and move
                state.loopnum += 1;
                state.output.push_str(&format!(concat!("    movq %r12, (%rbx)\n",
                                                       "    jmp endloop{num}\n",
                                                       "    loop{num}:\n",
                                                       "    {add_sub} ${shift}, %rbx\n",
                                                       "    endloop{num}:\n",
                                                       "    cmp $0, (%rbx)\n",
                                                       "    jnz loop{num}\n",
                                                       "    movq (%rbx), %r12\n"),
                                               num = state.loopnum,
                                               add_sub = if offset > 0 { "addq" } else { "subq" },
                                               shift = offset.abs() * 8));
            }
            Input => {
                state.output.push_str(concat!("\n    xor %rax, %rax\n",
                                              "    xor %rdi, %rdi\n",
                                              "    movq %rbx, %rsi\n",
                                              "    movq $1, %rdx\n",
                                              "    syscall\n",
                                              "    movq (%rbx), %r12\n\n"))
            }
            LoadOut(offset, add) => {
                let outaddr = format!("(strbuff+{})", outbuffpos);
                if offset == 0 {
                    state.output.push_str(&format!("    movq %r12, {}\n", outaddr));
                } else {
                    state.output.push_str(&format!("    movq {}(%rbx), %rax\n", offset * 8));
                    state.output.push_str(&format!("    movq %rax, {}\n", outaddr));
                }
                if add > 0 {
                    state.output.push_str(&format!("    addb ${}, {}\n", add, outaddr));
                } else if add < 0 {
                    state.output.push_str(&format!("    subb ${}, {}\n", -add, outaddr));
                }
                outbuffpos += 1;
            }
            LoadOutSet(value) => {
                let outaddr = format!("(strbuff+{})", outbuffpos);
                state.output.push_str(&format!("    movq ${}, {}\n", value, outaddr));
                outbuffpos += 1;
            }
            Output => {
                state.output.push_str(&format!(concat!("    movq $1, %rax\n",
                                                       "    movq $1, %rdi\n",
                                                       "    movq $strbuff, %rsi\n",
                                                       "    movq ${}, %rdx\n",
                                                       "    syscall\n\n"),
                                               outbuffpos));

                if state.outbuffsize < outbuffpos + 8 {
                    state.outbuffsize = outbuffpos + 8;
                }
                outbuffpos = 0;
            }
        }
    }
}

pub fn compile(tokens: Vec<Token>, tape_size: i32) -> String {
    let mut state = CompileState::default();

    compile_iter(&mut state, tokens);

    format!(concat!(".section .bss\n",
                    "    .lcomm strbuff, {outbuffsize}\n",
                    "    .lcomm mem, {}\n",
                    "    .set startidx, mem + {}\n",
                    ".section .text\n",
                    ".global _start\n",
                    "_start:\n",
                    "    xor %r12, %r12\n",
                    "    movq $startidx, %rbx\n\n",
                    "{}\n",
                    // Exit syscall
                    "    movq $60, %rax\n",
                    "    movq $0, %rdi\n",
                    "    syscall\n"),
            tape_size,
            tape_size / 2,
            state.output,
            outbuffsize = state.outbuffsize)
}
<|endoftext|>


<commit_before>use std::fmt::Debug;
use std::convert::TryFrom;
use regex_syntax;
use cripes::pattern::{Atom, Element};

fn from_regex<T: Atom>(s: &str) -> Element<T>
    where Element<T>: TryFrom<regex_syntax::Expr>,
          <Element<T> as TryFrom<regex_syntax::Expr>>::Err: Debug
{
    TryFrom::try_from(regex_syntax::Expr::parse(s).unwrap()).unwrap()
}

macro_rules! extract_variant {
    ((ref $input: expr) -> $variant: ident) => ( match $input {
        &Element::$variant(ref x) => x,
        _ => unreachable!() });

    (($input: expr) -> $variant: ident) => ( match $input {
        Element::$variant(x) => x,
        _ => unreachable!() });
}

/// Helper for parsing a regex and returning the inner pattern type.
macro_rules! from_regex {
    ($s: tt -> Element<$T: ty>) => ( from_regex::<$T>($s) );
    ($s: tt -> $which: ident < $T: ty>) => (extract_variant!((from_regex::<$T>($s)) -> $which));
}

mod sequence {
    use std::iter::{self, FromIterator};
    use cripes::pattern::{Class, Element, Sequence, Union};
    use cripes::pattern::Reduce;
    use cripes::util::set::IsSubsetOf;

    #[test]
    fn test_reduce() {
        let seq_of_seqs = Sequence::from_iter(["abc", "def", "ghi"].iter().map(|s| Sequence::from_iter(s.chars().map(|c| c.into())).into()));
        let flat_seq = Sequence::<char>::from_iter("abcdefghi".chars().map(|c| c.into()));

        assert!(seq_of_seqs != flat_seq);
        assert_eq!(Some(Element::Sequence(flat_seq)), seq_of_seqs.reduce());

        // Empty sequences should reduce to `None`
        assert_eq!(None, Sequence::<char>::from_iter(iter::empty()).reduce());

        // Unit-length sequences should reduce to their only element.
        assert_eq!(Some(Element::Wildcard), Sequence::<char>::new(vec![Element::Wildcard]).reduce());
    }

    #[test]
    fn test_is_subset_of() {
        let unions = (&["abc", "def", "ghi"]).iter().map(|s| Union::<char>::from_iter(s.chars().map(|c| c.into()))).collect::<Vec<_>>();
        let classes = (&["abc", "def", "ghi"]).iter().map(|s| Class::<char>::from_iter(s.chars())).collect::<Vec<_>>();
        let seqs = (&["adg", "beh", "cfi"]).iter().map(|s| Sequence::<char>::from_iter(s.chars().map(|c| c.into()))).collect::<Vec<_>>();
        let useq = Sequence::from_iter(unions.iter().map(|u| Element::Union(u.clone())));
        let cseq = Sequence::from_iter(classes.iter().map(|c| Element::Class(c.clone())));

        // Every sequence should be a subset of itself.
        assert!(useq.is_subset_of(&useq));
        assert!(cseq.is_subset_of(&cseq));

        // `useq` and `cseq` should be subsets of each other.
        assert!(useq.is_subset_of(&cseq));
        assert!(cseq.is_subset_of(&useq));

        // each sequence in `seqs` should be a subset of both `useq` and
        // `cseq`.
        for seq in seqs.iter() {
            for i in 0..3 {
                assert!(seq[i].is_subset_of(&cseq[i]));
                assert!(seq[i].is_subset_of(&useq[i]));
            }

            assert!(seq.is_subset_of(&cseq));
            assert!(seq.is_subset_of(&useq));
        }
    }
}

mod union {
    use super::from_regex;
    use std::iter::FromIterator;
    use cripes::pattern::{Element, Union, Wildcard};
    use cripes::util::set::{Contains, IsSubsetOf};

    #[test]
    fn test_contains() {
        let u = Union::from_iter("abc".chars().map(|c| c.into()));
        assert!(u.contains('a'));
        assert!(u.contains('b'));
        assert!(u.contains('c'));
    }

    #[test]
    fn test_is_subset_of_union() {
        let u1 = Union::<char>::from_iter("abc".chars().map(|c| c.into()));
        let u2 = Union::from_iter("abcdef".chars().map(|c| c.into()));
        assert!(u1.is_subset_of(&u2));
        assert!(! u2.is_subset_of(&u1));

        assert!(u1.is_subset_of(&Wildcard));
        assert!(u2.is_subset_of(&Wildcard));
    }
    #[test]
    fn test_is_subset_of_sequence() {
        let seq_of_classes = from_regex!("[abc][def][ghi]" -> Sequence<char>);
        let seq_of_unions = from_regex!("(a|b|c)(d|e|f)(g|h|i)" -> Sequence<char>);

        let union_of_seqs = from_regex!("adg|beh|cfi" -> Union<char>);
        println!("{:?}", union_of_seqs);
        assert_eq!(3, union_of_seqs.len());

        // check that the union would match each contained sequence.
        for seq in union_of_seqs.iter() {
            println!("{:?}", seq);
            assert!(seq.is_subset_of(&union_of_seqs));

            for i in 0..3 {
                let s = extract_variant!((ref seq) -> Sequence);
                println!("{:?} <=> {:?}", s[i], seq_of_unions[i]);
                assert!(s[i].is_subset_of(&seq_of_unions[i]));
                assert!(s[i].is_subset_of(&seq_of_classes[i]));
            }

            assert!(seq.is_subset_of(&seq_of_classes));
            assert!(seq.is_subset_of(&seq_of_unions));
        }

        assert!(union_of_seqs.is_subset_of(&seq_of_classes));
        assert!(union_of_seqs.is_subset_of(&seq_of_unions));

        assert!(! seq_of_classes.is_subset_of(&union_of_seqs));
        assert!(! seq_of_unions.is_subset_of(&union_of_seqs));

    }
}

mod wildcard {
    use std::{char, u8};
    use std::iter::FromIterator;
    use char_iter;
    use cripes::pattern::{ByteOrChar, Class, Element, Wildcard};
    use cripes::util::set::{Contains, IsSubsetOf, IsSupersetOf};

    #[test]
    fn test_is_subset_of() {
        let wild = Wildcard::<char>;
        assert!(wild.is_superset_of(&'a'));
        assert!(wild.is_superset_of(&Element::Atom('a')));
        assert!(!wild.is_subset_of(&Element::Atom('a')));
        assert!(wild.is_subset_of(&Wildcard));

        assert!(ByteOrChar::Byte(b'H').is_subset_of(&Wildcard));

        let cls = Class::from_iter("xyz".chars());
        assert!(wild.is_superset_of(&cls));
    }

    #[test]
    fn test_contains() {
        let wildc = Wildcard::<char>;
        for c in char_iter::new(0 as char, char::MAX) {
            assert!(wildc.contains(c));
        }

        let wildbc = Wildcard::<ByteOrChar>;
        for c in char_iter::new(0 as char, char::MAX) {
            assert!(wildbc.contains(c.into()));
        }
        for b in 0...u8::MAX {
            assert!(wildbc.contains(b.into()));
        }
    }
}

mod class {
    use std::char;
    use std::iter::FromIterator;

    use char_iter;
    use cripes::pattern::{Class, ClassMember, Wildcard};
    use cripes::util::set::{Contains, IsSubsetOf};

    #[test]
    fn test_contains() {
        let range = ClassMember::Range('a', 'z');
        panic_unless_eq!(26, range.len());
        assert!(range.contains('a'));
        assert!(range.contains('z'));
        assert!(! range.contains('A'));
        assert!(! range.contains('Z'));

        let atom = ClassMember::Atom('X');
        panic_unless_eq!(1, atom.len());
        assert!(atom.contains('X'));
        assert!(! atom.contains('x'));

        let class_from_members = Class::from_members([range, atom].into_iter().cloned());
        assert!(class_from_members.contains(&range));
        assert!(class_from_members.contains(&atom));
        panic_unless_eq!(27, class_from_members.len());
        for c in char_iter::new('a', 'z') {
            assert!(class_from_members.contains(c));
        }
        for c in char_iter::new('A', 'W') {
            assert!(! class_from_members.contains(c));
        }
        assert!(class_from_members.contains('X'));
        for c in char_iter::new('Y', 'Z') {
            assert!(! class_from_members.contains(c));
        }

        let cls = Class::from_iter(['a', 'b', 'c'].into_iter().cloned());
        assert_eq!(3, cls.len());

        // contains
        assert!(cls.contains('a'));
        assert!(cls.contains('b'));
        assert!(cls.contains('c'));
        for c in char_iter::new(0 as char, char::MAX) {
            if c == 'a' || c == 'b' || c == 'c' {
                assert!(cls.contains(c));
            } else {
                assert!(!cls.contains(c))
            }
        }
    }

    #[test]
    fn test_is_subset_of() {
        let cls1: Class<char> = FromIterator::from_iter(['x', 'y', 'z'].into_iter().cloned());
        assert!(cls1.is_subset_of(&Wildcard));

        let cls2: Class<char> = FromIterator::from_iter(['x', 'y'].into_iter().cloned());
        assert!(cls2.is_subset_of(&cls1));
        assert!(! cls1.is_subset_of(&cls2));
    }
}
<commit_msg>tests/pattern: fix a test for Sequence::reduce<commit_after>use std::fmt::Debug;
use std::convert::TryFrom;
use regex_syntax;
use cripes::pattern::{Atom, Element};

fn from_regex<T: Atom>(s: &str) -> Element<T>
    where Element<T>: TryFrom<regex_syntax::Expr>,
          <Element<T> as TryFrom<regex_syntax::Expr>>::Err: Debug
{
    TryFrom::try_from(regex_syntax::Expr::parse(s).unwrap()).unwrap()
}

macro_rules! extract_variant {
    ((ref $input: expr) -> $variant: ident) => ( match $input {
        &Element::$variant(ref x) => x,
        _ => unreachable!() });

    (($input: expr) -> $variant: ident) => ( match $input {
        Element::$variant(x) => x,
        _ => unreachable!() });
}

/// Helper for parsing a regex and returning the inner pattern type.
macro_rules! from_regex {
    ($s: tt -> Element<$T: ty>) => ( from_regex::<$T>($s) );
    ($s: tt -> $which: ident < $T: ty>) => (extract_variant!((from_regex::<$T>($s)) -> $which));
}

mod sequence {
    use std::iter::{self, FromIterator};
    use cripes::pattern::{Class, Element, Sequence, Union};
    use cripes::pattern::Reduce;
    use cripes::util::set::IsSubsetOf;

    #[test]
    fn test_reduce() {
        let seq_of_seqs = Sequence::from_iter(["abc", "def", "ghi"].iter().map(|s| Sequence::from_iter(s.chars().map(|c| c.into())).into()));
        let flat_seq = Sequence::<char>::from_iter("abcdefghi".chars().map(|c| c.into()));

        assert!(seq_of_seqs != flat_seq);
        assert_eq!(Some(Element::Sequence(flat_seq)), seq_of_seqs.reduce());

        // Empty sequences should reduce to `None`
        assert_eq!(None, Sequence::<char>::from_iter(iter::empty()).reduce());

        // Unit-length sequences should reduce to their only element.
        assert_eq!(Some(Element::Wildcard), Sequence::<char>::from_iter(vec![Element::Wildcard]).reduce());
    }

    #[test]
    fn test_is_subset_of() {
        let unions = (&["abc", "def", "ghi"]).iter().map(|s| Union::<char>::from_iter(s.chars().map(|c| c.into()))).collect::<Vec<_>>();
        let classes = (&["abc", "def", "ghi"]).iter().map(|s| Class::<char>::from_iter(s.chars())).collect::<Vec<_>>();
        let seqs = (&["adg", "beh", "cfi"]).iter().map(|s| Sequence::<char>::from_iter(s.chars().map(|c| c.into()))).collect::<Vec<_>>();
        let useq = Sequence::from_iter(unions.iter().map(|u| Element::Union(u.clone())));
        let cseq = Sequence::from_iter(classes.iter().map(|c| Element::Class(c.clone())));

        // Every sequence should be a subset of itself.
        assert!(useq.is_subset_of(&useq));
        assert!(cseq.is_subset_of(&cseq));

        // `useq` and `cseq` should be subsets of each other.
        assert!(useq.is_subset_of(&cseq));
        assert!(cseq.is_subset_of(&useq));

        // each sequence in `seqs` should be a subset of both `useq` and
        // `cseq`.
        for seq in seqs.iter() {
            for i in 0..3 {
                assert!(seq[i].is_subset_of(&cseq[i]));
                assert!(seq[i].is_subset_of(&useq[i]));
            }

            assert!(seq.is_subset_of(&cseq));
            assert!(seq.is_subset_of(&useq));
        }
    }
}

mod union {
    use super::from_regex;
    use std::iter::FromIterator;
    use cripes::pattern::{Element, Union, Wildcard};
    use cripes::util::set::{Contains, IsSubsetOf};

    #[test]
    fn test_contains() {
        let u = Union::from_iter("abc".chars().map(|c| c.into()));
        assert!(u.contains('a'));
        assert!(u.contains('b'));
        assert!(u.contains('c'));
    }

    #[test]
    fn test_is_subset_of_union() {
        let u1 = Union::<char>::from_iter("abc".chars().map(|c| c.into()));
        let u2 = Union::from_iter("abcdef".chars().map(|c| c.into()));
        assert!(u1.is_subset_of(&u2));
        assert!(! u2.is_subset_of(&u1));

        assert!(u1.is_subset_of(&Wildcard));
        assert!(u2.is_subset_of(&Wildcard));
    }
    #[test]
    fn test_is_subset_of_sequence() {
        let seq_of_classes = from_regex!("[abc][def][ghi]" -> Sequence<char>);
        let seq_of_unions = from_regex!("(a|b|c)(d|e|f)(g|h|i)" -> Sequence<char>);

        let union_of_seqs = from_regex!("adg|beh|cfi" -> Union<char>);
        println!("{:?}", union_of_seqs);
        assert_eq!(3, union_of_seqs.len());

        // check that the union would match each contained sequence.
        for seq in union_of_seqs.iter() {
            println!("{:?}", seq);
            assert!(seq.is_subset_of(&union_of_seqs));

            for i in 0..3 {
                let s = extract_variant!((ref seq) -> Sequence);
                println!("{:?} <=> {:?}", s[i], seq_of_unions[i]);
                assert!(s[i].is_subset_of(&seq_of_unions[i]));
                assert!(s[i].is_subset_of(&seq_of_classes[i]));
            }

            assert!(seq.is_subset_of(&seq_of_classes));
            assert!(seq.is_subset_of(&seq_of_unions));
        }

        assert!(union_of_seqs.is_subset_of(&seq_of_classes));
        assert!(union_of_seqs.is_subset_of(&seq_of_unions));

        assert!(! seq_of_classes.is_subset_of(&union_of_seqs));
        assert!(! seq_of_unions.is_subset_of(&union_of_seqs));

    }
}

mod wildcard {
    use std::{char, u8};
    use std::iter::FromIterator;
    use char_iter;
    use cripes::pattern::{ByteOrChar, Class, Element, Wildcard};
    use cripes::util::set::{Contains, IsSubsetOf, IsSupersetOf};

    #[test]
    fn test_is_subset_of() {
        let wild = Wildcard::<char>;
        assert!(wild.is_superset_of(&'a'));
        assert!(wild.is_superset_of(&Element::Atom('a')));
        assert!(!wild.is_subset_of(&Element::Atom('a')));
        assert!(wild.is_subset_of(&Wildcard));

        assert!(ByteOrChar::Byte(b'H').is_subset_of(&Wildcard));

        let cls = Class::from_iter("xyz".chars());
        assert!(wild.is_superset_of(&cls));
    }

    #[test]
    fn test_contains() {
        let wildc = Wildcard::<char>;
        for c in char_iter::new(0 as char, char::MAX) {
            assert!(wildc.contains(c));
        }

        let wildbc = Wildcard::<ByteOrChar>;
        for c in char_iter::new(0 as char, char::MAX) {
            assert!(wildbc.contains(c.into()));
        }
        for b in 0...u8::MAX {
            assert!(wildbc.contains(b.into()));
        }
    }
}

mod class {
    use std::char;
    use std::iter::FromIterator;

    use char_iter;
    use cripes::pattern::{Class, ClassMember, Wildcard};
    use cripes::util::set::{Contains, IsSubsetOf};

    #[test]
    fn test_contains() {
        let range = ClassMember::Range('a', 'z');
        panic_unless_eq!(26, range.len());
        assert!(range.contains('a'));
        assert!(range.contains('z'));
        assert!(! range.contains('A'));
        assert!(! range.contains('Z'));

        let atom = ClassMember::Atom('X');
        panic_unless_eq!(1, atom.len());
        assert!(atom.contains('X'));
        assert!(! atom.contains('x'));

        let class_from_members = Class::from_members([range, atom].into_iter().cloned());
        assert!(class_from_members.contains(&range));
        assert!(class_from_members.contains(&atom));
        panic_unless_eq!(27, class_from_members.len());
        for c in char_iter::new('a', 'z') {
            assert!(class_from_members.contains(c));
        }
        for c in char_iter::new('A', 'W') {
            assert!(! class_from_members.contains(c));
        }
        assert!(class_from_members.contains('X'));
        for c in char_iter::new('Y', 'Z') {
            assert!(! class_from_members.contains(c));
        }

        let cls = Class::from_iter(['a', 'b', 'c'].into_iter().cloned());
        assert_eq!(3, cls.len());

        // contains
        assert!(cls.contains('a'));
        assert!(cls.contains('b'));
        assert!(cls.contains('c'));
        for c in char_iter::new(0 as char, char::MAX) {
            if c == 'a' || c == 'b' || c == 'c' {
                assert!(cls.contains(c));
            } else {
                assert!(!cls.contains(c))
            }
        }
    }

    #[test]
    fn test_is_subset_of() {
        let cls1: Class<char> = FromIterator::from_iter(['x', 'y', 'z'].into_iter().cloned());
        assert!(cls1.is_subset_of(&Wildcard));

        let cls2: Class<char> = FromIterator::from_iter(['x', 'y'].into_iter().cloned());
        assert!(cls2.is_subset_of(&cls1));
        assert!(! cls1.is_subset_of(&cls2));
    }
}
<|endoftext|>


<commit_before>#![allow(
    clippy::cast_lossless,
    clippy::cast_possible_wrap,
    clippy::derive_partial_eq_without_eq
)]

use indoc::indoc;
use serde_derive::Deserialize;
use serde_yaml::Value;
use std::collections::BTreeMap;
use std::fmt::Debug;

fn test_de<'de, T>(yaml: &'de str, expected: &T)
where
    T: serde::de::Deserialize<'de> + PartialEq + Debug,
{
    let deserialized: T = serde_yaml::from_str(yaml).unwrap();
    assert_eq!(*expected, deserialized);

    serde_yaml::from_str::<serde_yaml::Value>(yaml).unwrap();
    serde_yaml::from_str::<serde::de::IgnoredAny>(yaml).unwrap();
}

fn test_de_seed<'de, T, S>(yaml: &'de str, seed: S, expected: &T)
where
    T: PartialEq + Debug,
    S: serde::de::DeserializeSeed<'de, Value = T>,
{
    let deserialized: T = serde_yaml::seed::from_str_seed(yaml, seed).unwrap();
    assert_eq!(*expected, deserialized);

    serde_yaml::from_str::<serde_yaml::Value>(yaml).unwrap();
    serde_yaml::from_str::<serde::de::IgnoredAny>(yaml).unwrap();
}

#[test]
fn test_borrowed() {
    let yaml = indoc! {"
        - plain nonàscii
        - 'single quoted'
        - \"double quoted\"
    "};
    let expected = vec!["plain nonàscii", "single quoted", "double quoted"];
    test_de(yaml, &expected);
}

#[test]
fn test_alias() {
    let yaml = indoc! {"
        first:
          &alias
          1
        second:
          *alias
        third: 3
    "};
    let mut expected = BTreeMap::new();
    expected.insert("first".to_owned(), 1);
    expected.insert("second".to_owned(), 1);
    expected.insert("third".to_owned(), 3);
    test_de(yaml, &expected);
}

#[test]
fn test_option() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Data {
        a: Option<f64>,
        b: Option<String>,
        c: Option<bool>,
    }
    let yaml = indoc! {"
        b:
        c: true
    "};
    let expected = Data {
        a: None,
        b: None,
        c: Some(true),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_option_alias() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Data {
        a: Option<f64>,
        b: Option<String>,
        c: Option<bool>,
        d: Option<f64>,
        e: Option<String>,
        f: Option<bool>,
    }
    let yaml = indoc! {"
        none_f:
          &none_f
          ~
        none_s:
          &none_s
          ~
        none_b:
          &none_b
          ~

        some_f:
          &some_f
          1.0
        some_s:
          &some_s
          x
        some_b:
          &some_b
          true

        a: *none_f
        b: *none_s
        c: *none_b
        d: *some_f
        e: *some_s
        f: *some_b
    "};
    let expected = Data {
        a: None,
        b: None,
        c: None,
        d: Some(1.0),
        e: Some("x".to_owned()),
        f: Some(true),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_enum_alias() {
    #[derive(Deserialize, PartialEq, Debug)]
    enum E {
        A,
        B(u8, u8),
    }
    #[derive(Deserialize, PartialEq, Debug)]
    struct Data {
        a: E,
        b: E,
    }
    let yaml = indoc! {"
        aref:
          &aref
          A
        bref:
          &bref
          !B
            - 1
            - 2

        a: *aref
        b: *bref
    "};
    let expected = Data {
        a: E::A,
        b: E::B(1, 2),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_enum_representations() {
    #[derive(Deserialize, PartialEq, Debug)]
    enum Enum {
        Unit,
        Tuple(i32, i32),
        Struct { x: i32, y: i32 },
        String(String),
        Number(f64),
    }
    let yaml = indoc! {"
        - Unit
        - 'Unit'
        - !Unit
        - !Unit ~
        - !Unit null
        - !Tuple [0, 0]
        - !Tuple
          - 0
          - 0
        - !Struct {x: 0, y: 0}
        - !Struct
          x: 0
          y: 0
        - !String '...'
        - !String ...
        - !String
        - !Number 0

    "};
    let expected = vec![
        Enum::Unit,
        Enum::Unit,
        Enum::Unit,
        Enum::Unit,
        Enum::Unit,
        Enum::Tuple(0, 0),
        Enum::Tuple(0, 0),
        Enum::Struct { x: 0, y: 0 },
        Enum::Struct { x: 0, y: 0 },
        Enum::String("...".to_owned()),
        Enum::String("...".to_owned()),
        Enum::String(String::new()),
        Enum::Number(0.0),
    ];
    test_de(yaml, &expected);
}

#[test]
fn test_number_as_string() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Num {
        value: String,
    }
    let yaml = indoc! {"
        # Cannot be represented as u128
        value: 340282366920938463463374607431768211457
    "};
    let expected = Num {
        value: "340282366920938463463374607431768211457".to_owned(),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_i128_big() {
    let expected: i128 = i64::MIN as i128 - 1;
    let yaml = indoc! {"
        -9223372036854775809
    "};
    assert_eq!(expected, serde_yaml::from_str::<i128>(yaml).unwrap());

    let octal = indoc! {"
        -0o1000000000000000000001
    "};
    assert_eq!(expected, serde_yaml::from_str::<i128>(octal).unwrap());
}

#[test]
fn test_u128_big() {
    let expected: u128 = u64::MAX as u128 + 1;
    let yaml = indoc! {"
        18446744073709551616
    "};
    assert_eq!(expected, serde_yaml::from_str::<u128>(yaml).unwrap());

    let octal = indoc! {"
        0o2000000000000000000000
    "};
    assert_eq!(expected, serde_yaml::from_str::<u128>(octal).unwrap());
}

#[test]
fn test_number_alias_as_string() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Num {
        version: String,
        value: String,
    }
    let yaml = indoc! {"
        version: &a 1.10
        value: *a
    "};
    let expected = Num {
        version: "1.10".to_owned(),
        value: "1.10".to_owned(),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_de_mapping() {
    #[derive(Debug, Deserialize, PartialEq)]
    struct Data {
        pub substructure: serde_yaml::Mapping,
    }
    let yaml = indoc! {"
        substructure:
          a: 'foo'
          b: 'bar'
    "};

    let mut expected = Data {
        substructure: serde_yaml::Mapping::new(),
    };
    expected.substructure.insert(
        serde_yaml::Value::String("a".to_owned()),
        serde_yaml::Value::String("foo".to_owned()),
    );
    expected.substructure.insert(
        serde_yaml::Value::String("b".to_owned()),
        serde_yaml::Value::String("bar".to_owned()),
    );

    test_de(yaml, &expected);
}

#[test]
fn test_bomb() {
    #[derive(Debug, Deserialize, PartialEq)]
    struct Data {
        expected: String,
    }

    // This would deserialize an astronomical number of elements if we were
    // vulnerable.
    let yaml = indoc! {"
        a: &a ~
        b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]
        c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b]
        d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c]
        e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d]
        f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e]
        g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f]
        h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g]
        i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h]
        j: &j [*i,*i,*i,*i,*i,*i,*i,*i,*i]
        k: &k [*j,*j,*j,*j,*j,*j,*j,*j,*j]
        l: &l [*k,*k,*k,*k,*k,*k,*k,*k,*k]
        m: &m [*l,*l,*l,*l,*l,*l,*l,*l,*l]
        n: &n [*m,*m,*m,*m,*m,*m,*m,*m,*m]
        o: &o [*n,*n,*n,*n,*n,*n,*n,*n,*n]
        p: &p [*o,*o,*o,*o,*o,*o,*o,*o,*o]
        q: &q [*p,*p,*p,*p,*p,*p,*p,*p,*p]
        r: &r [*q,*q,*q,*q,*q,*q,*q,*q,*q]
        s: &s [*r,*r,*r,*r,*r,*r,*r,*r,*r]
        t: &t [*s,*s,*s,*s,*s,*s,*s,*s,*s]
        u: &u [*t,*t,*t,*t,*t,*t,*t,*t,*t]
        v: &v [*u,*u,*u,*u,*u,*u,*u,*u,*u]
        w: &w [*v,*v,*v,*v,*v,*v,*v,*v,*v]
        x: &x [*w,*w,*w,*w,*w,*w,*w,*w,*w]
        y: &y [*x,*x,*x,*x,*x,*x,*x,*x,*x]
        z: &z [*y,*y,*y,*y,*y,*y,*y,*y,*y]
        expected: string
    "};

    let expected = Data {
        expected: "string".to_owned(),
    };

    assert_eq!(expected, serde_yaml::from_str::<Data>(yaml).unwrap());
}

#[test]
fn test_numbers() {
    let cases = [
        ("0xF0", "240"),
        ("+0xF0", "240"),
        ("-0xF0", "-240"),
        ("0o70", "56"),
        ("+0o70", "56"),
        ("-0o70", "-56"),
        ("0b10", "2"),
        ("+0b10", "2"),
        ("-0b10", "-2"),
        ("127", "127"),
        ("+127", "127"),
        ("-127", "-127"),
        (".inf", ".inf"),
        (".Inf", ".inf"),
        (".INF", ".inf"),
        ("-.inf", "-.inf"),
        ("-.Inf", "-.inf"),
        ("-.INF", "-.inf"),
        (".nan", ".nan"),
        (".NaN", ".nan"),
        (".NAN", ".nan"),
        ("0.1", "0.1"),
    ];
    for &(yaml, expected) in &cases {
        let value = serde_yaml::from_str::<Value>(yaml).unwrap();
        match value {
            Value::Number(number) => assert_eq!(number.to_string(), expected),
            _ => panic!("expected number. input={:?}, result={:?}", yaml, value),
        }
    }

    // NOT numbers.
    let cases = [
        "0127", "+0127", "-0127", "++.inf", "+-.inf", "++1", "+-1", "-+1", "--1", "0x+1", "0x-1",
        "-0x+1", "-0x-1", "++0x1", "+-0x1", "-+0x1", "--0x1",
    ];
    for yaml in &cases {
        let value = serde_yaml::from_str::<Value>(yaml).unwrap();
        match value {
            Value::String(string) => assert_eq!(string, *yaml),
            _ => panic!("expected string. input={:?}, result={:?}", yaml, value),
        }
    }
}

#[test]
fn test_stateful() {
    struct Seed(i64);

    impl<'de> serde::de::DeserializeSeed<'de> for Seed {
        type Value = i64;
        fn deserialize<D>(self, deserializer: D) -> Result<i64, D::Error>
        where
            D: serde::de::Deserializer<'de>,
        {
            struct Visitor(i64);
            impl<'de> serde::de::Visitor<'de> for Visitor {
                type Value = i64;

                fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
                    write!(formatter, "an integer")
                }

                fn visit_i64<E: serde::de::Error>(self, v: i64) -> Result<i64, E> {
                    Ok(v * self.0)
                }

                fn visit_u64<E: serde::de::Error>(self, v: u64) -> Result<i64, E> {
                    Ok(v as i64 * self.0)
                }
            }

            deserializer.deserialize_any(Visitor(self.0))
        }
    }

    let cases = [("3", 5, 15), ("6", 7, 42), ("-5", 9, -45)];
    for &(yaml, seed, expected) in &cases {
        test_de_seed(yaml, Seed(seed), &expected);
    }
}
<commit_msg>Add test of deserialization ignoreing tags<commit_after>#![allow(
    clippy::cast_lossless,
    clippy::cast_possible_wrap,
    clippy::derive_partial_eq_without_eq
)]

use indoc::indoc;
use serde_derive::Deserialize;
use serde_yaml::Value;
use std::collections::BTreeMap;
use std::fmt::Debug;

fn test_de<'de, T>(yaml: &'de str, expected: &T)
where
    T: serde::de::Deserialize<'de> + PartialEq + Debug,
{
    let deserialized: T = serde_yaml::from_str(yaml).unwrap();
    assert_eq!(*expected, deserialized);

    serde_yaml::from_str::<serde_yaml::Value>(yaml).unwrap();
    serde_yaml::from_str::<serde::de::IgnoredAny>(yaml).unwrap();
}

fn test_de_seed<'de, T, S>(yaml: &'de str, seed: S, expected: &T)
where
    T: PartialEq + Debug,
    S: serde::de::DeserializeSeed<'de, Value = T>,
{
    let deserialized: T = serde_yaml::seed::from_str_seed(yaml, seed).unwrap();
    assert_eq!(*expected, deserialized);

    serde_yaml::from_str::<serde_yaml::Value>(yaml).unwrap();
    serde_yaml::from_str::<serde::de::IgnoredAny>(yaml).unwrap();
}

#[test]
fn test_borrowed() {
    let yaml = indoc! {"
        - plain nonàscii
        - 'single quoted'
        - \"double quoted\"
    "};
    let expected = vec!["plain nonàscii", "single quoted", "double quoted"];
    test_de(yaml, &expected);
}

#[test]
fn test_alias() {
    let yaml = indoc! {"
        first:
          &alias
          1
        second:
          *alias
        third: 3
    "};
    let mut expected = BTreeMap::new();
    expected.insert("first".to_owned(), 1);
    expected.insert("second".to_owned(), 1);
    expected.insert("third".to_owned(), 3);
    test_de(yaml, &expected);
}

#[test]
fn test_option() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Data {
        a: Option<f64>,
        b: Option<String>,
        c: Option<bool>,
    }
    let yaml = indoc! {"
        b:
        c: true
    "};
    let expected = Data {
        a: None,
        b: None,
        c: Some(true),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_option_alias() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Data {
        a: Option<f64>,
        b: Option<String>,
        c: Option<bool>,
        d: Option<f64>,
        e: Option<String>,
        f: Option<bool>,
    }
    let yaml = indoc! {"
        none_f:
          &none_f
          ~
        none_s:
          &none_s
          ~
        none_b:
          &none_b
          ~

        some_f:
          &some_f
          1.0
        some_s:
          &some_s
          x
        some_b:
          &some_b
          true

        a: *none_f
        b: *none_s
        c: *none_b
        d: *some_f
        e: *some_s
        f: *some_b
    "};
    let expected = Data {
        a: None,
        b: None,
        c: None,
        d: Some(1.0),
        e: Some("x".to_owned()),
        f: Some(true),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_enum_alias() {
    #[derive(Deserialize, PartialEq, Debug)]
    enum E {
        A,
        B(u8, u8),
    }
    #[derive(Deserialize, PartialEq, Debug)]
    struct Data {
        a: E,
        b: E,
    }
    let yaml = indoc! {"
        aref:
          &aref
          A
        bref:
          &bref
          !B
            - 1
            - 2

        a: *aref
        b: *bref
    "};
    let expected = Data {
        a: E::A,
        b: E::B(1, 2),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_enum_representations() {
    #[derive(Deserialize, PartialEq, Debug)]
    enum Enum {
        Unit,
        Tuple(i32, i32),
        Struct { x: i32, y: i32 },
        String(String),
        Number(f64),
    }
    let yaml = indoc! {"
        - Unit
        - 'Unit'
        - !Unit
        - !Unit ~
        - !Unit null
        - !Tuple [0, 0]
        - !Tuple
          - 0
          - 0
        - !Struct {x: 0, y: 0}
        - !Struct
          x: 0
          y: 0
        - !String '...'
        - !String ...
        - !String
        - !Number 0

    "};
    let expected = vec![
        Enum::Unit,
        Enum::Unit,
        Enum::Unit,
        Enum::Unit,
        Enum::Unit,
        Enum::Tuple(0, 0),
        Enum::Tuple(0, 0),
        Enum::Struct { x: 0, y: 0 },
        Enum::Struct { x: 0, y: 0 },
        Enum::String("...".to_owned()),
        Enum::String("...".to_owned()),
        Enum::String(String::new()),
        Enum::Number(0.0),
    ];
    test_de(yaml, &expected);
}

#[test]
fn test_number_as_string() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Num {
        value: String,
    }
    let yaml = indoc! {"
        # Cannot be represented as u128
        value: 340282366920938463463374607431768211457
    "};
    let expected = Num {
        value: "340282366920938463463374607431768211457".to_owned(),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_i128_big() {
    let expected: i128 = i64::MIN as i128 - 1;
    let yaml = indoc! {"
        -9223372036854775809
    "};
    assert_eq!(expected, serde_yaml::from_str::<i128>(yaml).unwrap());

    let octal = indoc! {"
        -0o1000000000000000000001
    "};
    assert_eq!(expected, serde_yaml::from_str::<i128>(octal).unwrap());
}

#[test]
fn test_u128_big() {
    let expected: u128 = u64::MAX as u128 + 1;
    let yaml = indoc! {"
        18446744073709551616
    "};
    assert_eq!(expected, serde_yaml::from_str::<u128>(yaml).unwrap());

    let octal = indoc! {"
        0o2000000000000000000000
    "};
    assert_eq!(expected, serde_yaml::from_str::<u128>(octal).unwrap());
}

#[test]
fn test_number_alias_as_string() {
    #[derive(Deserialize, PartialEq, Debug)]
    struct Num {
        version: String,
        value: String,
    }
    let yaml = indoc! {"
        version: &a 1.10
        value: *a
    "};
    let expected = Num {
        version: "1.10".to_owned(),
        value: "1.10".to_owned(),
    };
    test_de(yaml, &expected);
}

#[test]
fn test_de_mapping() {
    #[derive(Debug, Deserialize, PartialEq)]
    struct Data {
        pub substructure: serde_yaml::Mapping,
    }
    let yaml = indoc! {"
        substructure:
          a: 'foo'
          b: 'bar'
    "};

    let mut expected = Data {
        substructure: serde_yaml::Mapping::new(),
    };
    expected.substructure.insert(
        serde_yaml::Value::String("a".to_owned()),
        serde_yaml::Value::String("foo".to_owned()),
    );
    expected.substructure.insert(
        serde_yaml::Value::String("b".to_owned()),
        serde_yaml::Value::String("bar".to_owned()),
    );

    test_de(yaml, &expected);
}

#[test]
fn test_bomb() {
    #[derive(Debug, Deserialize, PartialEq)]
    struct Data {
        expected: String,
    }

    // This would deserialize an astronomical number of elements if we were
    // vulnerable.
    let yaml = indoc! {"
        a: &a ~
        b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]
        c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b]
        d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c]
        e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d]
        f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e]
        g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f]
        h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g]
        i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h]
        j: &j [*i,*i,*i,*i,*i,*i,*i,*i,*i]
        k: &k [*j,*j,*j,*j,*j,*j,*j,*j,*j]
        l: &l [*k,*k,*k,*k,*k,*k,*k,*k,*k]
        m: &m [*l,*l,*l,*l,*l,*l,*l,*l,*l]
        n: &n [*m,*m,*m,*m,*m,*m,*m,*m,*m]
        o: &o [*n,*n,*n,*n,*n,*n,*n,*n,*n]
        p: &p [*o,*o,*o,*o,*o,*o,*o,*o,*o]
        q: &q [*p,*p,*p,*p,*p,*p,*p,*p,*p]
        r: &r [*q,*q,*q,*q,*q,*q,*q,*q,*q]
        s: &s [*r,*r,*r,*r,*r,*r,*r,*r,*r]
        t: &t [*s,*s,*s,*s,*s,*s,*s,*s,*s]
        u: &u [*t,*t,*t,*t,*t,*t,*t,*t,*t]
        v: &v [*u,*u,*u,*u,*u,*u,*u,*u,*u]
        w: &w [*v,*v,*v,*v,*v,*v,*v,*v,*v]
        x: &x [*w,*w,*w,*w,*w,*w,*w,*w,*w]
        y: &y [*x,*x,*x,*x,*x,*x,*x,*x,*x]
        z: &z [*y,*y,*y,*y,*y,*y,*y,*y,*y]
        expected: string
    "};

    let expected = Data {
        expected: "string".to_owned(),
    };

    assert_eq!(expected, serde_yaml::from_str::<Data>(yaml).unwrap());
}

#[test]
fn test_numbers() {
    let cases = [
        ("0xF0", "240"),
        ("+0xF0", "240"),
        ("-0xF0", "-240"),
        ("0o70", "56"),
        ("+0o70", "56"),
        ("-0o70", "-56"),
        ("0b10", "2"),
        ("+0b10", "2"),
        ("-0b10", "-2"),
        ("127", "127"),
        ("+127", "127"),
        ("-127", "-127"),
        (".inf", ".inf"),
        (".Inf", ".inf"),
        (".INF", ".inf"),
        ("-.inf", "-.inf"),
        ("-.Inf", "-.inf"),
        ("-.INF", "-.inf"),
        (".nan", ".nan"),
        (".NaN", ".nan"),
        (".NAN", ".nan"),
        ("0.1", "0.1"),
    ];
    for &(yaml, expected) in &cases {
        let value = serde_yaml::from_str::<Value>(yaml).unwrap();
        match value {
            Value::Number(number) => assert_eq!(number.to_string(), expected),
            _ => panic!("expected number. input={:?}, result={:?}", yaml, value),
        }
    }

    // NOT numbers.
    let cases = [
        "0127", "+0127", "-0127", "++.inf", "+-.inf", "++1", "+-1", "-+1", "--1", "0x+1", "0x-1",
        "-0x+1", "-0x-1", "++0x1", "+-0x1", "-+0x1", "--0x1",
    ];
    for yaml in &cases {
        let value = serde_yaml::from_str::<Value>(yaml).unwrap();
        match value {
            Value::String(string) => assert_eq!(string, *yaml),
            _ => panic!("expected string. input={:?}, result={:?}", yaml, value),
        }
    }
}

#[test]
fn test_stateful() {
    struct Seed(i64);

    impl<'de> serde::de::DeserializeSeed<'de> for Seed {
        type Value = i64;
        fn deserialize<D>(self, deserializer: D) -> Result<i64, D::Error>
        where
            D: serde::de::Deserializer<'de>,
        {
            struct Visitor(i64);
            impl<'de> serde::de::Visitor<'de> for Visitor {
                type Value = i64;

                fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
                    write!(formatter, "an integer")
                }

                fn visit_i64<E: serde::de::Error>(self, v: i64) -> Result<i64, E> {
                    Ok(v * self.0)
                }

                fn visit_u64<E: serde::de::Error>(self, v: u64) -> Result<i64, E> {
                    Ok(v as i64 * self.0)
                }
            }

            deserializer.deserialize_any(Visitor(self.0))
        }
    }

    let cases = [("3", 5, 15), ("6", 7, 42), ("-5", 9, -45)];
    for &(yaml, seed, expected) in &cases {
        test_de_seed(yaml, Seed(seed), &expected);
    }
}

#[test]
fn test_ignore_tag() {
    #[derive(Deserialize, Debug, PartialEq)]
    struct Data {
        struc: Struc,
        tuple: Tuple,
        newtype: Newtype,
        map: BTreeMap<char, usize>,
        vec: Vec<usize>,
    }

    #[derive(Deserialize, Debug, PartialEq)]
    struct Struc {
        x: usize,
    }

    #[derive(Deserialize, Debug, PartialEq)]
    struct Tuple(usize, usize);

    #[derive(Deserialize, Debug, PartialEq)]
    struct Newtype(usize);

    let yaml = indoc! {"
        struc: !wat
          x: 0
        tuple: !wat
          - 0
          - 0
        newtype: !wat 0
        map: !wat
          x: 0
        vec: !wat
          - 0
    "};

    let expected = Data {
        struc: Struc { x: 0 },
        tuple: Tuple(0, 0),
        newtype: Newtype(0),
        map: {
            let mut map = BTreeMap::new();
            map.insert('x', 0);
            map
        },
        vec: vec![0],
    };

    test_de(yaml, &expected);
}
<|endoftext|>


<commit_before>//! A stably addressed token buffer supporting efficient traversal based on a
//! cheaply copyable cursor.
//!
//! *This module is available only if Syn is built with the `"parsing"` feature.*

// This module is heavily commented as it contains most of the unsafe code in
// Syn, and caution should be used when editing it. The public-facing interface
// is 100% safe but the implementation is fragile internally.

#[cfg(all(
    not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
    feature = "proc-macro"
))]
use crate::proc_macro as pm;
use crate::Lifetime;
use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use std::marker::PhantomData;
use std::ptr;
use std::slice;

/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
enum Entry {
    // Mimicking types from proc-macro.
    Group(Group, TokenBuffer),
    Ident(Ident),
    Punct(Punct),
    Literal(Literal),
    // End entries contain a raw pointer to the entry from the containing
    // token tree, or null if this is the outermost level.
    End(*const Entry),
}

/// A buffer that can be efficiently traversed multiple times, unlike
/// `TokenStream` which requires a deep copy in order to traverse more than
/// once.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct TokenBuffer {
    // NOTE: Do not implement clone on this - there are raw pointers inside
    // these entries which will be messed up. Moving the `TokenBuffer` itself is
    // safe as the data pointed to won't be moved.
    ptr: *const Entry,
    len: usize,
}

impl Drop for TokenBuffer {
    fn drop(&mut self) {
        unsafe {
            let slice = slice::from_raw_parts_mut(self.ptr as *mut Entry, self.len);
            let _ = Box::from_raw(slice);
        }
    }
}

impl TokenBuffer {
    // NOTE: Do not mutate the Vec returned from this function once it returns;
    // the address of its backing memory must remain stable.
    fn inner_new(stream: TokenStream, up: *const Entry) -> TokenBuffer {
        // Build up the entries list, recording the locations of any Groups
        // in the list to be processed later.
        let mut entries = Vec::new();
        let mut groups = Vec::new();
        for tt in stream {
            match tt {
                TokenTree::Ident(sym) => {
                    entries.push(Entry::Ident(sym));
                }
                TokenTree::Punct(op) => {
                    entries.push(Entry::Punct(op));
                }
                TokenTree::Literal(l) => {
                    entries.push(Entry::Literal(l));
                }
                TokenTree::Group(g) => {
                    // Record the index of the interesting entry, and store an
                    // `End(null)` there temporarily.
                    groups.push((entries.len(), g));
                    entries.push(Entry::End(ptr::null()));
                }
            }
        }
        // Add an `End` entry to the end with a reference to the enclosing token
        // stream which was passed in.
        entries.push(Entry::End(up));

        // NOTE: This is done to ensure that we don't accidentally modify the
        // length of the backing buffer. The backing buffer must remain at a
        // constant address after this point, as we are going to store a raw
        // pointer into it.
        let mut entries = entries.into_boxed_slice();
        for (idx, group) in groups {
            // We know that this index refers to one of the temporary
            // `End(null)` entries, and we know that the last entry is
            // `End(up)`, so the next index is also valid.
            let group_up = unsafe { entries.as_ptr().add(idx + 1) };

            // The end entry stored at the end of this Entry::Group should
            // point to the Entry which follows the Group in the list.
            let inner = Self::inner_new(group.stream(), group_up);
            entries[idx] = Entry::Group(group, inner);
        }

        let len = entries.len();
        let ptr = Box::into_raw(entries);
        TokenBuffer {
            ptr: ptr as *const Entry,
            len,
        }
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `proc_macro::TokenStream`.
    ///
    /// *This method is available only if Syn is built with both the `"parsing"` and
    /// `"proc-macro"` features.*
    #[cfg(all(
        not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
        feature = "proc-macro"
    ))]
    pub fn new(stream: pm::TokenStream) -> Self {
        Self::new2(stream.into())
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `proc_macro2::TokenStream`.
    pub fn new2(stream: TokenStream) -> Self {
        Self::inner_new(stream, ptr::null())
    }

    /// Creates a cursor referencing the first token in the buffer and able to
    /// traverse until the end of the buffer.
    pub fn begin(&self) -> Cursor {
        unsafe { Cursor::create(self.ptr, self.ptr.add(self.len - 1)) }
    }
}

/// A cheaply copyable cursor into a `TokenBuffer`.
///
/// This cursor holds a shared reference into the immutable data which is used
/// internally to represent a `TokenStream`, and can be efficiently manipulated
/// and copied around.
///
/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer`
/// object and get a cursor to its first token with `begin()`.
///
/// Two cursors are equal if they have the same location in the same input
/// stream, and have the same scope.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct Cursor<'a> {
    // The current entry which the `Cursor` is pointing at.
    ptr: *const Entry,
    // This is the only `Entry::End(..)` object which this cursor is allowed to
    // point at. All other `End` objects are skipped over in `Cursor::create`.
    scope: *const Entry,
    // Cursor is covariant in 'a. This field ensures that our pointers are still
    // valid.
    marker: PhantomData<&'a Entry>,
}

impl<'a> Cursor<'a> {
    /// Creates a cursor referencing a static empty TokenStream.
    pub fn empty() -> Self {
        // It's safe in this situation for us to put an `Entry` object in global
        // storage, despite it not actually being safe to send across threads
        // (`Ident` is a reference into a thread-local table). This is because
        // this entry never includes a `Ident` object.
        //
        // This wrapper struct allows us to break the rules and put a `Sync`
        // object in global storage.
        struct UnsafeSyncEntry(Entry);
        unsafe impl Sync for UnsafeSyncEntry {}
        static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0 as *const Entry));

        Cursor {
            ptr: &EMPTY_ENTRY.0,
            scope: &EMPTY_ENTRY.0,
            marker: PhantomData,
        }
    }

    /// This create method intelligently exits non-explicitly-entered
    /// `None`-delimited scopes when the cursor reaches the end of them,
    /// allowing for them to be treated transparently.
    unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self {
        // NOTE: If we're looking at a `End(..)`, we want to advance the cursor
        // past it, unless `ptr == scope`, which means that we're at the edge of
        // our cursor's scope. We should only have `ptr != scope` at the exit
        // from None-delimited groups entered with `ignore_none`.
        while let Entry::End(exit) = *ptr {
            if ptr == scope {
                break;
            }
            ptr = exit;
        }

        Cursor {
            ptr,
            scope,
            marker: PhantomData,
        }
    }

    /// Get the current entry.
    fn entry(self) -> &'a Entry {
        unsafe { &*self.ptr }
    }

    /// Bump the cursor to point at the next token after the current one. This
    /// is undefined behavior if the cursor is currently looking at an
    /// `Entry::End`.
    unsafe fn bump(self) -> Cursor<'a> {
        Cursor::create(self.ptr.offset(1), self.scope)
    }

    /// While the cursor is looking at a `None`-delimited group, move it to look
    /// at the first token inside instead. If the group is empty, this will move
    /// the cursor past the `None`-delimited group.
    ///
    /// WARNING: This mutates its argument.
    fn ignore_none(&mut self) {
        while let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == Delimiter::None {
                // NOTE: We call `Cursor::create` here to make sure that
                // situations where we should immediately exit the span after
                // entering it are handled correctly.
                unsafe {
                    *self = Cursor::create(buf.ptr, self.scope);
                }
            } else {
                break;
            }
        }
    }

    /// Checks whether the cursor is currently pointing at the end of its valid
    /// scope.
    pub fn eof(self) -> bool {
        // We're at eof if we're at the end of our scope.
        self.ptr == self.scope
    }

    /// If the cursor is pointing at a `Group` with the given delimiter, returns
    /// a cursor into that group and one pointing to the next `TokenTree`.
    pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> {
        // If we're not trying to enter a none-delimited group, we want to
        // ignore them. We have to make sure to _not_ ignore them when we want
        // to enter them, of course. For obvious reasons.
        if delim != Delimiter::None {
            self.ignore_none();
        }

        if let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == delim {
                return Some((buf.begin(), group.span(), unsafe { self.bump() }));
            }
        }

        None
    }

    /// If the cursor is pointing at a `Ident`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at an `Punct`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() != '\'' => Some((op.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Literal`, return it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Literal(lit) => Some((lit.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Lifetime`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.ident() {
                    Some((ident, rest)) => {
                        let lifetime = Lifetime {
                            apostrophe: op.span(),
                            ident,
                        };
                        Some((lifetime, rest))
                    }
                    None => None,
                }
            }
            _ => None,
        }
    }

    /// Copies all remaining tokens visible from this cursor into a
    /// `TokenStream`.
    pub fn token_stream(self) -> TokenStream {
        let mut tts = Vec::new();
        let mut cursor = self;
        while let Some((tt, rest)) = cursor.token_tree() {
            tts.push(tt);
            cursor = rest;
        }
        tts.into_iter().collect()
    }

    /// If the cursor is pointing at a `TokenTree`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    ///
    /// Returns `None` if the cursor has reached the end of its stream.
    ///
    /// This method does not treat `None`-delimited groups as transparent, and
    /// will return a `Group(None, ..)` if the cursor is looking at one.
    pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> {
        let tree = match self.entry() {
            Entry::Group(group, _) => group.clone().into(),
            Entry::Literal(lit) => lit.clone().into(),
            Entry::Ident(ident) => ident.clone().into(),
            Entry::Punct(op) => op.clone().into(),
            Entry::End(..) => return None,
        };

        Some((tree, unsafe { self.bump() }))
    }

    /// Returns the `Span` of the current token, or `Span::call_site()` if this
    /// cursor points to eof.
    pub fn span(self) -> Span {
        match self.entry() {
            Entry::Group(group, _) => group.span(),
            Entry::Literal(l) => l.span(),
            Entry::Ident(t) => t.span(),
            Entry::Punct(o) => o.span(),
            Entry::End(..) => Span::call_site(),
        }
    }

    /// Skip over the next token without cloning it. Returns `None` if this
    /// cursor points to eof.
    ///
    /// This method treats `'lifetimes` as a single token.
    pub(crate) fn skip(self) -> Option<Cursor<'a>> {
        match self.entry() {
            Entry::End(..) => None,

            // Treat lifetimes as a single tt for the purposes of 'skip'.
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.entry() {
                    Entry::Ident(_) => Some(unsafe { next.bump() }),
                    _ => Some(next),
                }
            }
            _ => Some(unsafe { self.bump() }),
        }
    }
}

impl<'a> Copy for Cursor<'a> {}

impl<'a> Clone for Cursor<'a> {
    fn clone(&self) -> Self {
        *self
    }
}

impl<'a> Eq for Cursor<'a> {}

impl<'a> PartialEq for Cursor<'a> {
    fn eq(&self, other: &Self) -> bool {
        let Cursor { ptr, scope, marker } = self;
        let _ = marker;
        *ptr == other.ptr && *scope == other.scope
    }
}

pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool {
    a.scope == b.scope
}

pub(crate) fn open_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_open(),
        _ => cursor.span(),
    }
}

pub(crate) fn close_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_close(),
        _ => cursor.span(),
    }
}
<commit_msg>Fix typo in documentation of Cursor::punct<commit_after>//! A stably addressed token buffer supporting efficient traversal based on a
//! cheaply copyable cursor.
//!
//! *This module is available only if Syn is built with the `"parsing"` feature.*

// This module is heavily commented as it contains most of the unsafe code in
// Syn, and caution should be used when editing it. The public-facing interface
// is 100% safe but the implementation is fragile internally.

#[cfg(all(
    not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
    feature = "proc-macro"
))]
use crate::proc_macro as pm;
use crate::Lifetime;
use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use std::marker::PhantomData;
use std::ptr;
use std::slice;

/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
enum Entry {
    // Mimicking types from proc-macro.
    Group(Group, TokenBuffer),
    Ident(Ident),
    Punct(Punct),
    Literal(Literal),
    // End entries contain a raw pointer to the entry from the containing
    // token tree, or null if this is the outermost level.
    End(*const Entry),
}

/// A buffer that can be efficiently traversed multiple times, unlike
/// `TokenStream` which requires a deep copy in order to traverse more than
/// once.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct TokenBuffer {
    // NOTE: Do not implement clone on this - there are raw pointers inside
    // these entries which will be messed up. Moving the `TokenBuffer` itself is
    // safe as the data pointed to won't be moved.
    ptr: *const Entry,
    len: usize,
}

impl Drop for TokenBuffer {
    fn drop(&mut self) {
        unsafe {
            let slice = slice::from_raw_parts_mut(self.ptr as *mut Entry, self.len);
            let _ = Box::from_raw(slice);
        }
    }
}

impl TokenBuffer {
    // NOTE: Do not mutate the Vec returned from this function once it returns;
    // the address of its backing memory must remain stable.
    fn inner_new(stream: TokenStream, up: *const Entry) -> TokenBuffer {
        // Build up the entries list, recording the locations of any Groups
        // in the list to be processed later.
        let mut entries = Vec::new();
        let mut groups = Vec::new();
        for tt in stream {
            match tt {
                TokenTree::Ident(sym) => {
                    entries.push(Entry::Ident(sym));
                }
                TokenTree::Punct(op) => {
                    entries.push(Entry::Punct(op));
                }
                TokenTree::Literal(l) => {
                    entries.push(Entry::Literal(l));
                }
                TokenTree::Group(g) => {
                    // Record the index of the interesting entry, and store an
                    // `End(null)` there temporarily.
                    groups.push((entries.len(), g));
                    entries.push(Entry::End(ptr::null()));
                }
            }
        }
        // Add an `End` entry to the end with a reference to the enclosing token
        // stream which was passed in.
        entries.push(Entry::End(up));

        // NOTE: This is done to ensure that we don't accidentally modify the
        // length of the backing buffer. The backing buffer must remain at a
        // constant address after this point, as we are going to store a raw
        // pointer into it.
        let mut entries = entries.into_boxed_slice();
        for (idx, group) in groups {
            // We know that this index refers to one of the temporary
            // `End(null)` entries, and we know that the last entry is
            // `End(up)`, so the next index is also valid.
            let group_up = unsafe { entries.as_ptr().add(idx + 1) };

            // The end entry stored at the end of this Entry::Group should
            // point to the Entry which follows the Group in the list.
            let inner = Self::inner_new(group.stream(), group_up);
            entries[idx] = Entry::Group(group, inner);
        }

        let len = entries.len();
        let ptr = Box::into_raw(entries);
        TokenBuffer {
            ptr: ptr as *const Entry,
            len,
        }
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `proc_macro::TokenStream`.
    ///
    /// *This method is available only if Syn is built with both the `"parsing"` and
    /// `"proc-macro"` features.*
    #[cfg(all(
        not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
        feature = "proc-macro"
    ))]
    pub fn new(stream: pm::TokenStream) -> Self {
        Self::new2(stream.into())
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `proc_macro2::TokenStream`.
    pub fn new2(stream: TokenStream) -> Self {
        Self::inner_new(stream, ptr::null())
    }

    /// Creates a cursor referencing the first token in the buffer and able to
    /// traverse until the end of the buffer.
    pub fn begin(&self) -> Cursor {
        unsafe { Cursor::create(self.ptr, self.ptr.add(self.len - 1)) }
    }
}

/// A cheaply copyable cursor into a `TokenBuffer`.
///
/// This cursor holds a shared reference into the immutable data which is used
/// internally to represent a `TokenStream`, and can be efficiently manipulated
/// and copied around.
///
/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer`
/// object and get a cursor to its first token with `begin()`.
///
/// Two cursors are equal if they have the same location in the same input
/// stream, and have the same scope.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct Cursor<'a> {
    // The current entry which the `Cursor` is pointing at.
    ptr: *const Entry,
    // This is the only `Entry::End(..)` object which this cursor is allowed to
    // point at. All other `End` objects are skipped over in `Cursor::create`.
    scope: *const Entry,
    // Cursor is covariant in 'a. This field ensures that our pointers are still
    // valid.
    marker: PhantomData<&'a Entry>,
}

impl<'a> Cursor<'a> {
    /// Creates a cursor referencing a static empty TokenStream.
    pub fn empty() -> Self {
        // It's safe in this situation for us to put an `Entry` object in global
        // storage, despite it not actually being safe to send across threads
        // (`Ident` is a reference into a thread-local table). This is because
        // this entry never includes a `Ident` object.
        //
        // This wrapper struct allows us to break the rules and put a `Sync`
        // object in global storage.
        struct UnsafeSyncEntry(Entry);
        unsafe impl Sync for UnsafeSyncEntry {}
        static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0 as *const Entry));

        Cursor {
            ptr: &EMPTY_ENTRY.0,
            scope: &EMPTY_ENTRY.0,
            marker: PhantomData,
        }
    }

    /// This create method intelligently exits non-explicitly-entered
    /// `None`-delimited scopes when the cursor reaches the end of them,
    /// allowing for them to be treated transparently.
    unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self {
        // NOTE: If we're looking at a `End(..)`, we want to advance the cursor
        // past it, unless `ptr == scope`, which means that we're at the edge of
        // our cursor's scope. We should only have `ptr != scope` at the exit
        // from None-delimited groups entered with `ignore_none`.
        while let Entry::End(exit) = *ptr {
            if ptr == scope {
                break;
            }
            ptr = exit;
        }

        Cursor {
            ptr,
            scope,
            marker: PhantomData,
        }
    }

    /// Get the current entry.
    fn entry(self) -> &'a Entry {
        unsafe { &*self.ptr }
    }

    /// Bump the cursor to point at the next token after the current one. This
    /// is undefined behavior if the cursor is currently looking at an
    /// `Entry::End`.
    unsafe fn bump(self) -> Cursor<'a> {
        Cursor::create(self.ptr.offset(1), self.scope)
    }

    /// While the cursor is looking at a `None`-delimited group, move it to look
    /// at the first token inside instead. If the group is empty, this will move
    /// the cursor past the `None`-delimited group.
    ///
    /// WARNING: This mutates its argument.
    fn ignore_none(&mut self) {
        while let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == Delimiter::None {
                // NOTE: We call `Cursor::create` here to make sure that
                // situations where we should immediately exit the span after
                // entering it are handled correctly.
                unsafe {
                    *self = Cursor::create(buf.ptr, self.scope);
                }
            } else {
                break;
            }
        }
    }

    /// Checks whether the cursor is currently pointing at the end of its valid
    /// scope.
    pub fn eof(self) -> bool {
        // We're at eof if we're at the end of our scope.
        self.ptr == self.scope
    }

    /// If the cursor is pointing at a `Group` with the given delimiter, returns
    /// a cursor into that group and one pointing to the next `TokenTree`.
    pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> {
        // If we're not trying to enter a none-delimited group, we want to
        // ignore them. We have to make sure to _not_ ignore them when we want
        // to enter them, of course. For obvious reasons.
        if delim != Delimiter::None {
            self.ignore_none();
        }

        if let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == delim {
                return Some((buf.begin(), group.span(), unsafe { self.bump() }));
            }
        }

        None
    }

    /// If the cursor is pointing at a `Ident`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Punct`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() != '\'' => Some((op.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Literal`, return it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Literal(lit) => Some((lit.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Lifetime`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.ident() {
                    Some((ident, rest)) => {
                        let lifetime = Lifetime {
                            apostrophe: op.span(),
                            ident,
                        };
                        Some((lifetime, rest))
                    }
                    None => None,
                }
            }
            _ => None,
        }
    }

    /// Copies all remaining tokens visible from this cursor into a
    /// `TokenStream`.
    pub fn token_stream(self) -> TokenStream {
        let mut tts = Vec::new();
        let mut cursor = self;
        while let Some((tt, rest)) = cursor.token_tree() {
            tts.push(tt);
            cursor = rest;
        }
        tts.into_iter().collect()
    }

    /// If the cursor is pointing at a `TokenTree`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    ///
    /// Returns `None` if the cursor has reached the end of its stream.
    ///
    /// This method does not treat `None`-delimited groups as transparent, and
    /// will return a `Group(None, ..)` if the cursor is looking at one.
    pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> {
        let tree = match self.entry() {
            Entry::Group(group, _) => group.clone().into(),
            Entry::Literal(lit) => lit.clone().into(),
            Entry::Ident(ident) => ident.clone().into(),
            Entry::Punct(op) => op.clone().into(),
            Entry::End(..) => return None,
        };

        Some((tree, unsafe { self.bump() }))
    }

    /// Returns the `Span` of the current token, or `Span::call_site()` if this
    /// cursor points to eof.
    pub fn span(self) -> Span {
        match self.entry() {
            Entry::Group(group, _) => group.span(),
            Entry::Literal(l) => l.span(),
            Entry::Ident(t) => t.span(),
            Entry::Punct(o) => o.span(),
            Entry::End(..) => Span::call_site(),
        }
    }

    /// Skip over the next token without cloning it. Returns `None` if this
    /// cursor points to eof.
    ///
    /// This method treats `'lifetimes` as a single token.
    pub(crate) fn skip(self) -> Option<Cursor<'a>> {
        match self.entry() {
            Entry::End(..) => None,

            // Treat lifetimes as a single tt for the purposes of 'skip'.
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.entry() {
                    Entry::Ident(_) => Some(unsafe { next.bump() }),
                    _ => Some(next),
                }
            }
            _ => Some(unsafe { self.bump() }),
        }
    }
}

impl<'a> Copy for Cursor<'a> {}

impl<'a> Clone for Cursor<'a> {
    fn clone(&self) -> Self {
        *self
    }
}

impl<'a> Eq for Cursor<'a> {}

impl<'a> PartialEq for Cursor<'a> {
    fn eq(&self, other: &Self) -> bool {
        let Cursor { ptr, scope, marker } = self;
        let _ = marker;
        *ptr == other.ptr && *scope == other.scope
    }
}

pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool {
    a.scope == b.scope
}

pub(crate) fn open_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_open(),
        _ => cursor.span(),
    }
}

pub(crate) fn close_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_close(),
        _ => cursor.span(),
    }
}
<|endoftext|>


<commit_before>use std::ops;

use crate::document::Document;
use crate::key::Key;
use crate::table::TableKeyValue;
use crate::{value, InlineTable, InternalString, Item, Table, Value};

// copied from
// https://github.com/serde-rs/json/blob/master/src/value/index.rs

pub trait Index: crate::private::Sealed {
    #[doc(hidden)]
    fn index<'v>(&self, val: &'v Item) -> Option<&'v Item>;
    #[doc(hidden)]
    fn index_mut<'v>(&self, val: &'v mut Item) -> Option<&'v mut Item>;
}

impl Index for usize {
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        match *v {
            Item::ArrayOfTables(ref aot) => aot.values.get(*self),
            Item::Value(ref a) if a.is_array() => a.as_array().and_then(|a| a.values.get(*self)),
            _ => None,
        }
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        match *v {
            Item::ArrayOfTables(ref mut vec) => vec.values.get_mut(*self),
            Item::Value(ref mut a) => a.as_array_mut().and_then(|a| a.values.get_mut(*self)),
            _ => None,
        }
    }
}

impl Index for str {
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        match *v {
            Item::Table(ref t) => t.get(self),
            Item::Value(ref v) => v
                .as_inline_table()
                .and_then(|t| t.items.get(self).map(|kv| &kv.value)),
            _ => None,
        }
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        if let Item::None = *v {
            let mut t = InlineTable::default();
            t.items.insert(
                InternalString::from(self),
                TableKeyValue::new(Key::new(self), Item::None),
            );
            *v = value(Value::InlineTable(t));
        }
        match *v {
            Item::Table(ref mut t) => Some(t.entry(self).or_insert(Item::None)),
            Item::Value(ref mut v) => v.as_inline_table_mut().map(|t| {
                &mut t
                    .items
                    .entry(InternalString::from(self))
                    .or_insert_with(|| TableKeyValue::new(Key::new(self), Item::None))
                    .value
            }),
            _ => None,
        }
    }
}

impl Index for String {
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        self[..].index(v)
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        self[..].index_mut(v)
    }
}

impl<'a, T: ?Sized> Index for &'a T
where
    T: Index,
{
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        (**self).index(v)
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        (**self).index_mut(v)
    }
}

impl<I> ops::Index<I> for Item
where
    I: Index,
{
    type Output = Item;

    fn index(&self, index: I) -> &Item {
        index.index(self).expect("index not found")
    }
}

impl<I> ops::IndexMut<I> for Item
where
    I: Index,
{
    fn index_mut(&mut self, index: I) -> &mut Item {
        index.index_mut(self).expect("index not found")
    }
}

impl<'s> ops::Index<&'s str> for Table {
    type Output = Item;

    fn index(&self, key: &'s str) -> &Item {
        self.get(key).expect("index not found")
    }
}

impl<'s> ops::IndexMut<&'s str> for Table {
    fn index_mut(&mut self, key: &'s str) -> &mut Item {
        self.entry(key).or_insert(Item::None)
    }
}

impl<'s> ops::Index<&'s str> for InlineTable {
    type Output = Value;

    fn index(&self, key: &'s str) -> &Value {
        self.get(key).expect("index not found")
    }
}

impl<'s> ops::IndexMut<&'s str> for InlineTable {
    fn index_mut(&mut self, key: &'s str) -> &mut Value {
        self.get_mut(key).expect("index not found")
    }
}

impl<'s> ops::Index<&'s str> for Document {
    type Output = Item;

    fn index(&self, key: &'s str) -> &Item {
        self.root.index(key)
    }
}

impl<'s> ops::IndexMut<&'s str> for Document {
    fn index_mut(&mut self, key: &'s str) -> &mut Item {
        self.root.index_mut(key)
    }
}
<commit_msg>fix: Match Item Indexing with InlineTable::get<commit_after>use std::ops;

use crate::document::Document;
use crate::key::Key;
use crate::table::TableKeyValue;
use crate::{value, InlineTable, InternalString, Item, Table, Value};

// copied from
// https://github.com/serde-rs/json/blob/master/src/value/index.rs

pub trait Index: crate::private::Sealed {
    #[doc(hidden)]
    fn index<'v>(&self, val: &'v Item) -> Option<&'v Item>;
    #[doc(hidden)]
    fn index_mut<'v>(&self, val: &'v mut Item) -> Option<&'v mut Item>;
}

impl Index for usize {
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        match *v {
            Item::ArrayOfTables(ref aot) => aot.values.get(*self),
            Item::Value(ref a) if a.is_array() => a.as_array().and_then(|a| a.values.get(*self)),
            _ => None,
        }
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        match *v {
            Item::ArrayOfTables(ref mut vec) => vec.values.get_mut(*self),
            Item::Value(ref mut a) => a.as_array_mut().and_then(|a| a.values.get_mut(*self)),
            _ => None,
        }
    }
}

impl Index for str {
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        match *v {
            Item::Table(ref t) => t.get(self),
            Item::Value(ref v) => v
                .as_inline_table()
                .and_then(|t| t.items.get(self))
                .and_then(|kv| {
                    if !kv.value.is_none() {
                        Some(&kv.value)
                    } else {
                        None
                    }
                }),
            _ => None,
        }
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        if let Item::None = *v {
            let mut t = InlineTable::default();
            t.items.insert(
                InternalString::from(self),
                TableKeyValue::new(Key::new(self), Item::None),
            );
            *v = value(Value::InlineTable(t));
        }
        match *v {
            Item::Table(ref mut t) => Some(t.entry(self).or_insert(Item::None)),
            Item::Value(ref mut v) => v.as_inline_table_mut().map(|t| {
                &mut t
                    .items
                    .entry(InternalString::from(self))
                    .or_insert_with(|| TableKeyValue::new(Key::new(self), Item::None))
                    .value
            }),
            _ => None,
        }
    }
}

impl Index for String {
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        self[..].index(v)
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        self[..].index_mut(v)
    }
}

impl<'a, T: ?Sized> Index for &'a T
where
    T: Index,
{
    fn index<'v>(&self, v: &'v Item) -> Option<&'v Item> {
        (**self).index(v)
    }
    fn index_mut<'v>(&self, v: &'v mut Item) -> Option<&'v mut Item> {
        (**self).index_mut(v)
    }
}

impl<I> ops::Index<I> for Item
where
    I: Index,
{
    type Output = Item;

    fn index(&self, index: I) -> &Item {
        index.index(self).expect("index not found")
    }
}

impl<I> ops::IndexMut<I> for Item
where
    I: Index,
{
    fn index_mut(&mut self, index: I) -> &mut Item {
        index.index_mut(self).expect("index not found")
    }
}

impl<'s> ops::Index<&'s str> for Table {
    type Output = Item;

    fn index(&self, key: &'s str) -> &Item {
        self.get(key).expect("index not found")
    }
}

impl<'s> ops::IndexMut<&'s str> for Table {
    fn index_mut(&mut self, key: &'s str) -> &mut Item {
        self.entry(key).or_insert(Item::None)
    }
}

impl<'s> ops::Index<&'s str> for InlineTable {
    type Output = Value;

    fn index(&self, key: &'s str) -> &Value {
        self.get(key).expect("index not found")
    }
}

impl<'s> ops::IndexMut<&'s str> for InlineTable {
    fn index_mut(&mut self, key: &'s str) -> &mut Value {
        self.get_mut(key).expect("index not found")
    }
}

impl<'s> ops::Index<&'s str> for Document {
    type Output = Item;

    fn index(&self, key: &'s str) -> &Item {
        self.root.index(key)
    }
}

impl<'s> ops::IndexMut<&'s str> for Document {
    fn index_mut(&mut self, key: &'s str) -> &mut Item {
        self.root.index_mut(key)
    }
}
<|endoftext|>


<commit_before>use std::f32::consts::PI;

use na::Norm;

use {Vector, Dim, Point, Point2f};
use ray::Ray;

pub struct Camera {
    pub origin: Point,
    pub dimension: Dim,
    pub fov: f32,
    inv_width: f32,
    inv_height: f32,
    aspect_ratio: f32,
    angle: f32,
}

impl Camera {
    pub fn new(origin: Point, dims: Dim, fov: f32) -> Camera {
        let (w, h) = dims;
        Camera {
            origin: origin,
            dimension: dims,
            fov: fov,
            inv_width: 1.0 / w as f32,
            inv_height: 1.0 / h as f32,
            aspect_ratio: w as f32 / h as f32,
            angle: (PI * 0.5 * fov / 180.0).tan(),
        }
    }

    pub fn ray_for(&self, p: &Point2f) -> Ray {
        let xx = (2.0 * p.x * self.inv_width - 1.0) * self.angle * self.aspect_ratio;
        let yy = (1.0 - 2.0 * p.y * self.inv_height) * self.angle;
        let raydir = Vector::new(xx, yy, -1.0).normalize();
        Ray::new(self.origin, raydir)
    }
}
<commit_msg>WIP Start refactoring Camera<commit_after>use std::f32::consts::PI;

use na::{Norm, Inverse};

use {Vector, Dim, Point, Point2f, Transform};
use ray::Ray;

/// Projective pinhole camera.
/// TODO: abstract the Camera interface and handle multiple camera types.
pub struct Camera {
    camera_to_world: Transform,
    camera_to_screen: Transform,
    raster_to_camera: Transform, /* pub origin: Point,
                                  * pub dimension: Dim,
                                  * pub fov: f32,
                                  * inv_width: f32,
                                  * inv_height: f32,
                                  * aspect_ratio: f32,
                                  * angle: f32, */
}

impl Camera {
    pub fn new(camera_to_world: Transform,
               camera_to_screen: Transform,
               screen_window: Point2f,
               lensr: f32,
               focald: f32)
               -> Camera {

       let screen_to_raster = ...;
       let raster_to_screen = screen_to_raster.inverse().unwrap();
        let raster_to_camera = camera_to_screen.inverse().unwrap() * raster_to_screen;
        Camera {
            camera_to_world: camera_to_world,
            camera_to_screen: camera_to_screen,
        }
    }

    pub fn generate_ray(&self, sample: CameraSample) -> Ray {
        unimplemented!();
    }
    // pub fn new(origin: Point, dims: Dim, fov: f32) -> Camera {
    //     let (w, h) = dims;
    //     Camera {
    //         origin: origin,
    //         dimension: dims,
    //         fov: fov,
    //         inv_width: 1.0 / w as f32,
    //         inv_height: 1.0 / h as f32,
    //         aspect_ratio: w as f32 / h as f32,
    //         angle: (PI * 0.5 * fov / 180.0).tan(),
    //     }
    // }

    // pub fn ray_for(&self, p: &Point2f) -> Ray {
    //     let xx = (2.0 * p.x * self.inv_width - 1.0) * self.angle * self.aspect_ratio;
    //     let yy = (1.0 - 2.0 * p.y * self.inv_height) * self.angle;
    //     let raydir = Vector::new(xx, yy, 1.0).normalize();
    //     Ray::new(self.origin, raydir)
    // }
}

pub struct CameraSample {
    pub p_film: Point2f,
    pub p_lens: Point2f,
    pub time: f32,
}
<|endoftext|>