<commit_before>//! Code for generating action code.

use grammar::repr as r;
use rust::RustWrite;
use std::io::{self, Write};

pub fn emit_action_code<W: Write>(grammar: &r::Grammar, rust: &mut RustWrite<W>) -> io::Result<()> {
    for (i, defn) in grammar.action_fn_defns.iter().enumerate() {
        rust!(rust, "");

        match defn.kind {
            r::ActionFnDefnKind::User(ref data) => {
                try!(emit_user_action_code(grammar, rust, i, defn, data))
            }
            r::ActionFnDefnKind::Lookaround(ref variant) => {
                try!(emit_lookaround_action_code(grammar, rust, i, defn, variant))
            }
            r::ActionFnDefnKind::Inline(ref data) => {
                try!(emit_inline_action_code(grammar, rust, i, defn, data))
            }
        }
    }

    Ok(())
}

fn ret_type_string(grammar: &r::Grammar, defn: &r::ActionFnDefn) -> String {
    if defn.fallible {
        format!("Result<{},{}ParseError<{},{},{}>>",
                defn.ret_type,
                grammar.prefix,
                grammar.types.terminal_loc_type(),
                grammar.types.terminal_token_type(),
                grammar.types.error_type())
    } else {
        format!("{}", defn.ret_type)
    }
}

fn emit_user_action_code<W: Write>(grammar: &r::Grammar,
                                   rust: &mut RustWrite<W>,
                                   index: usize,
                                   defn: &r::ActionFnDefn,
                                   data: &r::UserActionFnDefn)
                                   -> io::Result<()> {
    let ret_type = ret_type_string(grammar, defn);

    // For each symbol to be reduced, we will receive
    // a (L, T, L) triple where the Ls are locations and
    // the T is the data. Ignore the locations and bind
    // the data to the name the user gave.
    let mut arguments: Vec<String> = data.arg_patterns
                                         .iter()
                                         .zip(data.arg_types
                                                  .iter()
                                                  .cloned()
                                                  .map(|t| grammar.types.spanned_type(t)))
                                         .map(|(p, t)| format!("(_, {}, _): {}", p, t))
                                         .collect();

    // If this is a reduce of an empty production, we will
    // automatically add position information in the form of
    // lookbehind/lookahead values. Otherwise, those values would be
    // determined from the arguments themselves.
    if data.arg_patterns.is_empty() {
        arguments.extend(vec![format!("{}lookbehind: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type()),
                              format!("{}lookahead: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type())]);
    }

    try!(rust.write_pub_fn_header(grammar,
                                  format!("{}action{}", grammar.prefix, index),
                                  vec![],
                                  arguments,
                                  ret_type,
                                  vec![]));
    rust!(rust, "{{");
    rust!(rust, "{}", data.code);
    rust!(rust, "}}");
    Ok(())
}

fn emit_lookaround_action_code<W: Write>(grammar: &r::Grammar,
                                         rust: &mut RustWrite<W>,
                                         index: usize,
                                         _defn: &r::ActionFnDefn,
                                         data: &r::LookaroundActionFnDefn)
                                         -> io::Result<()> {
    try!(rust.write_pub_fn_header(grammar,
                                  format!("{}action{}", grammar.prefix, index),
                                  vec![],
                                  vec![format!("{}lookbehind: &{}",
                                               grammar.prefix,
                                               grammar.types.terminal_loc_type()),
                                       format!("{}lookahead: &{}",
                                               grammar.prefix,
                                               grammar.types.terminal_loc_type())],
                                  format!("{}", grammar.types.terminal_loc_type()),
                                  vec![]));

    rust!(rust, "{{");
    match *data {
        r::LookaroundActionFnDefn::Lookahead => {
            // take the lookahead, if any; otherwise, we are
            // at EOF, so taker the lookbehind (end of last
            // pushed token); if that is missing too, then
            // supply default.
            rust!(rust, "{}lookahead.clone()", grammar.prefix);
        }
        r::LookaroundActionFnDefn::Lookbehind => {
            // take lookbehind or supply default
            rust!(rust, "{}lookbehind.clone()", grammar.prefix);
        }
    }
    rust!(rust, "}}");
    Ok(())
}

fn emit_inline_action_code<W: Write>(grammar: &r::Grammar,
                                     rust: &mut RustWrite<W>,
                                     index: usize,
                                     defn: &r::ActionFnDefn,
                                     data: &r::InlineActionFnDefn)
                                     -> io::Result<()> {
    let ret_type = ret_type_string(grammar, defn);

    let arg_types: Vec<_> = data.symbols
                                .iter()
                                .flat_map(|sym| {
                                    match *sym {
                                        r::InlinedSymbol::Original(s) => vec![s],
                                        r::InlinedSymbol::Inlined(_, ref syms) => syms.clone(),
                                    }
                                })
                                .map(|s| s.ty(&grammar.types))
                                .collect();

    // this is the number of symbols we expect to be passed in; it is
    // distinct from data.symbols.len(), because sometimes we have
    // inlined actions with no input symbols
    let num_flat_args = arg_types.len();

    let mut arguments: Vec<_> = arg_types.iter()
                                         .map(|&t| grammar.types.spanned_type(t.clone()))
                                         .enumerate()
                                         .map(|(i, t)| format!("{}{}: {}", grammar.prefix, i, t))
                                         .collect();

    // If no symbols are being reduced, add in the
    // lookbehind/lookahead.
    if arguments.len() == 0 {
        arguments.extend(vec![format!("{}lookbehind: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type()),
                              format!("{}lookahead: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type())]);
    }

    try!(rust.write_pub_fn_header(grammar,
                                  format!("{}action{}", grammar.prefix, index),
                                  vec![],
                                  arguments,
                                  ret_type,
                                  vec![]));
    rust!(rust, "{{");

    // For each inlined thing, compute the start/end locations.
    // Do this first so that none of the arguments have been moved
    // yet and we can easily access their locations.
    let mut arg_counter = 0;
    let mut temp_counter = 0;
    for symbol in &data.symbols {
        match *symbol {
            r::InlinedSymbol::Original(_) => {
                arg_counter += 1;
            }
            r::InlinedSymbol::Inlined(_, ref syms) => {
                if syms.len() > 0 {
                    // If we are reducing symbols, then start and end
                    // can be the start/end location of the first/last
                    // symbol respectively. Easy peezy.

                    rust!(rust,
                          "let {}start{} = {}{}.0.clone();",
                          grammar.prefix,
                          temp_counter,
                          grammar.prefix,
                          arg_counter);

                    let last_arg_index = arg_counter + syms.len() - 1;
                    rust!(rust,
                          "let {}end{} = {}{}.2.clone();",
                          grammar.prefix,
                          temp_counter,
                          grammar.prefix,
                          last_arg_index);
                } else {
                    // If we have no symbols, then `arg_counter`
                    // represents index of the first symbol after this
                    // inlined item (if any), and `arg_counter-1`
                    // represents index of the symbol before this
                    // item.

                    if arg_counter > 0 {
                        rust!(rust,
                              "let {}start{} = {}{}.2.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              arg_counter - 1);
                    } else if num_flat_args > 0 {
                        rust!(rust,
                              "let {}start{} = {}{}.0.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              arg_counter);
                    } else {
                        rust!(rust,
                              "let {}start{} = {}lookbehind.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix);
                    }

                    if arg_counter < num_flat_args {
                        rust!(rust,
                              "let {}end{} = {}{}.0.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              arg_counter);
                    } else if num_flat_args > 0 {
                        rust!(rust,
                              "let {}end{} = {}{}.2.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              num_flat_args - 1);
                    } else {
                        rust!(rust,
                              "let {}end{} = {}lookahead.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix);
                    }
                }

                temp_counter += 1;
                arg_counter += syms.len();
            }
        }
    }

    // Now create temporaries for the inlined things
    let mut arg_counter = 0;
    let mut temp_counter = 0;
    for symbol in &data.symbols {
        match *symbol {
            r::InlinedSymbol::Original(_) => {
                arg_counter += 1;
            }
            r::InlinedSymbol::Inlined(inlined_action, ref syms) => {
                // execute the inlined reduce action
                rust!(rust,
                      "let {}temp{} = {}action{}(",
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      inlined_action.index());
                for parameter in &grammar.parameters {
                    rust!(rust, "{},", parameter.name);
                }
                for i in 0..syms.len() {
                    rust!(rust, "{}{},", grammar.prefix, arg_counter + i);
                }
                if syms.len() == 0 {
                    rust!(rust, "&{}start{},", grammar.prefix, temp_counter);
                    rust!(rust, "&{}end{},", grammar.prefix, temp_counter);
                }
                rust!(rust, ");");

                // wrap up the inlined value along with its span
                rust!(rust,
                      "let {}temp{} = ({}start{}, {}temp{}, {}end{});",
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      temp_counter);

                temp_counter += 1;
                arg_counter += syms.len();
            }
        }
    }

    rust!(rust, "{}action{}(", grammar.prefix, data.action.index());
    for parameter in &grammar.parameters {
        rust!(rust, "{},", parameter.name);
    }
    let mut arg_counter = 0;
    let mut temp_counter = 0;
    for symbol in &data.symbols {
        match *symbol {
            r::InlinedSymbol::Original(_) => {
                rust!(rust, "{}{},", grammar.prefix, arg_counter);
                arg_counter += 1;
            }
            r::InlinedSymbol::Inlined(_, ref syms) => {
                rust!(rust, "{}temp{},", grammar.prefix, temp_counter);
                temp_counter += 1;
                arg_counter += syms.len();
            }
        }
    }
    assert!(data.symbols.len() > 0);
    rust!(rust, ")");

    rust!(rust, "}}");
    Ok(())
}
<commit_msg>add a comment explaining how action code looks<commit_after>//! Code for generating action code.
//!
//! From the outside, action fns have one of two forms. If they take
//! symbols as input, e.g. from a production like `X = Y Z => ...`
//! (which takes Y and Z as input), they have this form:
//!
//! ```
//! pub fn __action17<
//!     'input,                       // user-declared type parameters (*)
//! >(
//!     input: &'input str,           // user-declared parameters
//!     __0: (usize, usize, usize),   // symbols being reduced, if any
//!     ...
//!     __N: (usize, Foo, usize),     // each has a type (L, T, L)
//! ) -> Box<Expr<'input>>
//! ```
//!
//! Otherwise, they have this form:
//!
//! ```
//! pub fn __action17<
//!     'input,                       // user-declared type parameters (*)
//! >(
//!     input: &'input str,           // user-declared parameters
//!    __lookbehind: &usize,          // value for @R -- "end of previous token"
//!    __lookahead: &usize,           // value for @L -- "start of next token"
//! ) -> Box<Expr<'input>>
//! ```
//!
//! * -- in this case, those "user-declared" parameters are inserted by
//! the "internal tokenizer".

use grammar::repr as r;
use rust::RustWrite;
use std::io::{self, Write};

pub fn emit_action_code<W: Write>(grammar: &r::Grammar, rust: &mut RustWrite<W>) -> io::Result<()> {
    for (i, defn) in grammar.action_fn_defns.iter().enumerate() {
        rust!(rust, "");

        match defn.kind {
            r::ActionFnDefnKind::User(ref data) => {
                try!(emit_user_action_code(grammar, rust, i, defn, data))
            }
            r::ActionFnDefnKind::Lookaround(ref variant) => {
                try!(emit_lookaround_action_code(grammar, rust, i, defn, variant))
            }
            r::ActionFnDefnKind::Inline(ref data) => {
                try!(emit_inline_action_code(grammar, rust, i, defn, data))
            }
        }
    }

    Ok(())
}

fn ret_type_string(grammar: &r::Grammar, defn: &r::ActionFnDefn) -> String {
    if defn.fallible {
        format!("Result<{},{}ParseError<{},{},{}>>",
                defn.ret_type,
                grammar.prefix,
                grammar.types.terminal_loc_type(),
                grammar.types.terminal_token_type(),
                grammar.types.error_type())
    } else {
        format!("{}", defn.ret_type)
    }
}

fn emit_user_action_code<W: Write>(grammar: &r::Grammar,
                                   rust: &mut RustWrite<W>,
                                   index: usize,
                                   defn: &r::ActionFnDefn,
                                   data: &r::UserActionFnDefn)
                                   -> io::Result<()> {
    let ret_type = ret_type_string(grammar, defn);

    // For each symbol to be reduced, we will receive
    // a (L, T, L) triple where the Ls are locations and
    // the T is the data. Ignore the locations and bind
    // the data to the name the user gave.
    let mut arguments: Vec<String> = data.arg_patterns
                                         .iter()
                                         .zip(data.arg_types
                                                  .iter()
                                                  .cloned()
                                                  .map(|t| grammar.types.spanned_type(t)))
                                         .map(|(p, t)| format!("(_, {}, _): {}", p, t))
                                         .collect();

    // If this is a reduce of an empty production, we will
    // automatically add position information in the form of
    // lookbehind/lookahead values. Otherwise, those values would be
    // determined from the arguments themselves.
    if data.arg_patterns.is_empty() {
        arguments.extend(vec![format!("{}lookbehind: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type()),
                              format!("{}lookahead: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type())]);
    }

    try!(rust.write_pub_fn_header(grammar,
                                  format!("{}action{}", grammar.prefix, index),
                                  vec![],
                                  arguments,
                                  ret_type,
                                  vec![]));
    rust!(rust, "{{");
    rust!(rust, "{}", data.code);
    rust!(rust, "}}");
    Ok(())
}

fn emit_lookaround_action_code<W: Write>(grammar: &r::Grammar,
                                         rust: &mut RustWrite<W>,
                                         index: usize,
                                         _defn: &r::ActionFnDefn,
                                         data: &r::LookaroundActionFnDefn)
                                         -> io::Result<()> {
    try!(rust.write_pub_fn_header(grammar,
                                  format!("{}action{}", grammar.prefix, index),
                                  vec![],
                                  vec![format!("{}lookbehind: &{}",
                                               grammar.prefix,
                                               grammar.types.terminal_loc_type()),
                                       format!("{}lookahead: &{}",
                                               grammar.prefix,
                                               grammar.types.terminal_loc_type())],
                                  format!("{}", grammar.types.terminal_loc_type()),
                                  vec![]));

    rust!(rust, "{{");
    match *data {
        r::LookaroundActionFnDefn::Lookahead => {
            // take the lookahead, if any; otherwise, we are
            // at EOF, so taker the lookbehind (end of last
            // pushed token); if that is missing too, then
            // supply default.
            rust!(rust, "{}lookahead.clone()", grammar.prefix);
        }
        r::LookaroundActionFnDefn::Lookbehind => {
            // take lookbehind or supply default
            rust!(rust, "{}lookbehind.clone()", grammar.prefix);
        }
    }
    rust!(rust, "}}");
    Ok(())
}

fn emit_inline_action_code<W: Write>(grammar: &r::Grammar,
                                     rust: &mut RustWrite<W>,
                                     index: usize,
                                     defn: &r::ActionFnDefn,
                                     data: &r::InlineActionFnDefn)
                                     -> io::Result<()> {
    let ret_type = ret_type_string(grammar, defn);

    let arg_types: Vec<_> = data.symbols
                                .iter()
                                .flat_map(|sym| {
                                    match *sym {
                                        r::InlinedSymbol::Original(s) => vec![s],
                                        r::InlinedSymbol::Inlined(_, ref syms) => syms.clone(),
                                    }
                                })
                                .map(|s| s.ty(&grammar.types))
                                .collect();

    // this is the number of symbols we expect to be passed in; it is
    // distinct from data.symbols.len(), because sometimes we have
    // inlined actions with no input symbols
    let num_flat_args = arg_types.len();

    let mut arguments: Vec<_> = arg_types.iter()
                                         .map(|&t| grammar.types.spanned_type(t.clone()))
                                         .enumerate()
                                         .map(|(i, t)| format!("{}{}: {}", grammar.prefix, i, t))
                                         .collect();

    // If no symbols are being reduced, add in the
    // lookbehind/lookahead.
    if arguments.len() == 0 {
        arguments.extend(vec![format!("{}lookbehind: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type()),
                              format!("{}lookahead: &{}",
                                      grammar.prefix,
                                      grammar.types.terminal_loc_type())]);
    }

    try!(rust.write_pub_fn_header(grammar,
                                  format!("{}action{}", grammar.prefix, index),
                                  vec![],
                                  arguments,
                                  ret_type,
                                  vec![]));
    rust!(rust, "{{");

    // For each inlined thing, compute the start/end locations.
    // Do this first so that none of the arguments have been moved
    // yet and we can easily access their locations.
    let mut arg_counter = 0;
    let mut temp_counter = 0;
    for symbol in &data.symbols {
        match *symbol {
            r::InlinedSymbol::Original(_) => {
                arg_counter += 1;
            }
            r::InlinedSymbol::Inlined(_, ref syms) => {
                if syms.len() > 0 {
                    // If we are reducing symbols, then start and end
                    // can be the start/end location of the first/last
                    // symbol respectively. Easy peezy.

                    rust!(rust,
                          "let {}start{} = {}{}.0.clone();",
                          grammar.prefix,
                          temp_counter,
                          grammar.prefix,
                          arg_counter);

                    let last_arg_index = arg_counter + syms.len() - 1;
                    rust!(rust,
                          "let {}end{} = {}{}.2.clone();",
                          grammar.prefix,
                          temp_counter,
                          grammar.prefix,
                          last_arg_index);
                } else {
                    // If we have no symbols, then `arg_counter`
                    // represents index of the first symbol after this
                    // inlined item (if any), and `arg_counter-1`
                    // represents index of the symbol before this
                    // item.

                    if arg_counter > 0 {
                        rust!(rust,
                              "let {}start{} = {}{}.2.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              arg_counter - 1);
                    } else if num_flat_args > 0 {
                        rust!(rust,
                              "let {}start{} = {}{}.0.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              arg_counter);
                    } else {
                        rust!(rust,
                              "let {}start{} = {}lookbehind.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix);
                    }

                    if arg_counter < num_flat_args {
                        rust!(rust,
                              "let {}end{} = {}{}.0.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              arg_counter);
                    } else if num_flat_args > 0 {
                        rust!(rust,
                              "let {}end{} = {}{}.2.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix,
                              num_flat_args - 1);
                    } else {
                        rust!(rust,
                              "let {}end{} = {}lookahead.clone();",
                              grammar.prefix,
                              temp_counter,
                              grammar.prefix);
                    }
                }

                temp_counter += 1;
                arg_counter += syms.len();
            }
        }
    }

    // Now create temporaries for the inlined things
    let mut arg_counter = 0;
    let mut temp_counter = 0;
    for symbol in &data.symbols {
        match *symbol {
            r::InlinedSymbol::Original(_) => {
                arg_counter += 1;
            }
            r::InlinedSymbol::Inlined(inlined_action, ref syms) => {
                // execute the inlined reduce action
                rust!(rust,
                      "let {}temp{} = {}action{}(",
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      inlined_action.index());
                for parameter in &grammar.parameters {
                    rust!(rust, "{},", parameter.name);
                }
                for i in 0..syms.len() {
                    rust!(rust, "{}{},", grammar.prefix, arg_counter + i);
                }
                if syms.len() == 0 {
                    rust!(rust, "&{}start{},", grammar.prefix, temp_counter);
                    rust!(rust, "&{}end{},", grammar.prefix, temp_counter);
                }
                rust!(rust, ");");

                // wrap up the inlined value along with its span
                rust!(rust,
                      "let {}temp{} = ({}start{}, {}temp{}, {}end{});",
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      temp_counter,
                      grammar.prefix,
                      temp_counter);

                temp_counter += 1;
                arg_counter += syms.len();
            }
        }
    }

    rust!(rust, "{}action{}(", grammar.prefix, data.action.index());
    for parameter in &grammar.parameters {
        rust!(rust, "{},", parameter.name);
    }
    let mut arg_counter = 0;
    let mut temp_counter = 0;
    for symbol in &data.symbols {
        match *symbol {
            r::InlinedSymbol::Original(_) => {
                rust!(rust, "{}{},", grammar.prefix, arg_counter);
                arg_counter += 1;
            }
            r::InlinedSymbol::Inlined(_, ref syms) => {
                rust!(rust, "{}temp{},", grammar.prefix, temp_counter);
                temp_counter += 1;
                arg_counter += syms.len();
            }
        }
    }
    assert!(data.symbols.len() > 0);
    rust!(rust, ")");

    rust!(rust, "}}");
    Ok(())
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#[prelude_import] use base::prelude::*;
use arch_fns::{memchr, all_bytes};
use core::{mem};
use base::{error};
use rmo::{Rmo, ToOwned};
use str_one::{CStr, NoNullStr, ByteStr, AsByteStr};
use str_two::{CString, NoNullString, ByteString};
use alloc::{Allocator};

/// Objects that can be turned into a `CString` by allocating.
pub trait ToCString {
    /// Converts the object into an allocated `CString`.
    fn to_cstring<H>(&self) -> Result<CString<H>> where H: Allocator;

    /// Tries to convert the object into an `Rmo<CStr>` with or without allocating.
    ///
    /// [argument, _buf]
    /// Scratch space.
    ///
    /// = Remarks
    ///
    /// The general strategy is as follows: If the object can be interpreted as a `CStr`,
    /// a borrowed `Rmo` is returned. If it can be interpreted as a `CStr` by copying it
    /// into the provided buffer, it's copied and returned as a borrowed `Rmo`. Otherwise,
    /// a `CString` is allocated.
    ///
    /// The default implementation simply calls `to_cstring`.
    fn rmo_cstr<'a, H>(&'a self, _buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        self.to_cstring().map(|r| Rmo::Owned(r))
    }
}

impl<'b, T: ToCString+?Sized> ToCString for &'b T {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        (**self).to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        (**self).rmo_cstr(buf)
    }
}

impl<'b, T: ToCString+?Sized> ToCString for &'b mut T {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        (**self).to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        (**self).rmo_cstr(buf)
    }
}

impl ToCString for CStr {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.bytes_with_null();
        bytes.to_owned().map(|o| unsafe { CString::from_bytes_unchecked(o) })
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.bytes_with_null();
        if bytes.len() <= buf.len() {
            mem::copy(buf, bytes);
            Ok(Rmo::Ref(unsafe { CStr::from_bytes_unchecked(&buf[..bytes.len()]) }))
        } else {
            self.to_cstring().map(|o| Rmo::Owned(o))
        }
    }
}

impl ToCString for [u8] {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        if let Some(idx) = memchr(self, 0) {
            if idx == self.len() - 1 || all_bytes(&self[idx+1..], 0) {
                self[..idx+1].to_owned().map(|o| unsafe { CString::from_bytes_unchecked(o) })
            } else {
                Err(error::InvalidArgument)
            }
        } else {
            let mut vec = try!(self.to_owned());
            try!(vec.reserve(1));
            vec.push(0);
            Ok(unsafe { CString::from_bytes_unchecked(vec) })
        }
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        if let Some(idx) = memchr(self, 0) {
            if idx == self.len() - 1 || all_bytes(&self[idx+1..], 0) {
                Ok(unsafe { Rmo::Ref(CStr::from_bytes_unchecked(&self[..idx+1])) })
            } else {
                Err(error::InvalidArgument)
            }
        } else {
            if self.len() >= buf.len() {
                let mut vec = try!(self.to_owned());
                try!(vec.reserve(1));
                vec.push(0);
                Ok(Rmo::Owned(unsafe { CString::from_bytes_unchecked(vec) }))
            } else {
                mem::copy(buf, self);
                buf[self.len()] = 0;
                Ok(unsafe { Rmo::Ref(CStr::from_bytes_unchecked(&buf[..self.len()+1])) })
            }
        }
    }
}

impl ToCString for [i8] {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.rmo_cstr(buf)
    }
}

impl ToCString for str {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.rmo_cstr(buf)
    }
}

impl ToCString for NoNullStr {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.as_ref();
        match bytes.to_owned() {
            Ok(mut o) => {
                try!(o.reserve(1));
                o.push(0);
                Ok(unsafe { CString::from_bytes_unchecked(o) })
            },
            Err(e) => Err(e),
        }
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        let bytes: &[u8] = self.as_ref();
        if bytes.len() < buf.len() {
            mem::copy(buf, bytes);
            buf[bytes.len()] = 0;
            Ok(Rmo::Ref(unsafe { CStr::from_bytes_unchecked(&buf[..bytes.len()]) }))
        } else {
            self.to_cstring().map(|o| Rmo::Owned(o))
        }
    }
}

impl ToCString for NoNullString
    where A: Allocator,
{
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        self.deref().to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        self.deref().rmo_cstr(buf)
    }
}

impl ToCString for ByteStr {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        self.as_ref().to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        self.as_ref().rmo_cstr(buf)
    }
}

impl ToCString for ByteString
    where A: Allocator,
{
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
    {
        self.as_byte_str().to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
    {
        self.as_byte_str().rmo_cstr(buf)
    }
}
<commit_msg>autocommit 2015-05-22 14:21:15 CEST<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#[prelude_import] use base::prelude::*;
use arch_fns::{memchr, all_bytes};
use core::{mem};
use base::{error};
use base::default::{Default};
use rmo::{Rmo, ToOwned};
use str_one::{CStr, NoNullStr, ByteStr, AsByteStr};
use str_two::{CString, NoNullString, ByteString};
use alloc::{Allocator};

/// Objects that can be turned into a `CString` by allocating.
pub trait ToCString {
    /// Converts the object into an allocated `CString`.
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default;

    /// Tries to convert the object into an `Rmo<CStr>` with or without allocating.
    ///
    /// [argument, _buf]
    /// Scratch space.
    ///
    /// = Remarks
    ///
    /// The general strategy is as follows: If the object can be interpreted as a `CStr`,
    /// a borrowed `Rmo` is returned. If it can be interpreted as a `CStr` by copying it
    /// into the provided buffer, it's copied and returned as a borrowed `Rmo`. Otherwise,
    /// a `CString` is allocated.
    ///
    /// The default implementation simply calls `to_cstring`.
    fn rmo_cstr<'a, H>(&'a self, _buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.to_cstring().map(|r| Rmo::Owned(r))
    }
}

impl<'b, T: ToCString+?Sized> ToCString for &'b T {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        (**self).to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        (**self).rmo_cstr(buf)
    }
}

impl<'b, T: ToCString+?Sized> ToCString for &'b mut T {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        (**self).to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        (**self).rmo_cstr(buf)
    }
}

impl ToCString for CStr {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.bytes_with_null();
        bytes.to_owned().map(|o| unsafe { CString::from_bytes_unchecked(o) })
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.bytes_with_null();
        if bytes.len() <= buf.len() {
            mem::copy(buf, bytes);
            Ok(Rmo::Ref(unsafe { CStr::from_bytes_unchecked(&buf[..bytes.len()]) }))
        } else {
            self.to_cstring().map(|o| Rmo::Owned(o))
        }
    }
}

impl ToCString for [u8] {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        if let Some(idx) = memchr(self, 0) {
            if idx == self.len() - 1 || all_bytes(&self[idx+1..], 0) {
                self[..idx+1].to_owned().map(|o| unsafe { CString::from_bytes_unchecked(o) })
            } else {
                Err(error::InvalidArgument)
            }
        } else {
            let mut vec = try!(self.to_owned());
            try!(vec.reserve(1));
            vec.push(0);
            Ok(unsafe { CString::from_bytes_unchecked(vec) })
        }
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        if let Some(idx) = memchr(self, 0) {
            if idx == self.len() - 1 || all_bytes(&self[idx+1..], 0) {
                Ok(unsafe { Rmo::Ref(CStr::from_bytes_unchecked(&self[..idx+1])) })
            } else {
                Err(error::InvalidArgument)
            }
        } else {
            if self.len() >= buf.len() {
                let mut vec = try!(self.to_owned());
                try!(vec.reserve(1));
                vec.push(0);
                Ok(Rmo::Owned(unsafe { CString::from_bytes_unchecked(vec) }))
            } else {
                mem::copy(buf, self);
                buf[self.len()] = 0;
                Ok(unsafe { Rmo::Ref(CStr::from_bytes_unchecked(&buf[..self.len()+1])) })
            }
        }
    }
}

impl ToCString for [i8] {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.rmo_cstr(buf)
    }
}

impl ToCString for str {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.as_ref();
        bytes.rmo_cstr(buf)
    }
}

impl ToCString for NoNullStr {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.as_ref();
        match bytes.to_owned() {
            Ok(mut o) => {
                try!(o.reserve(1));
                o.push(0);
                Ok(unsafe { CString::from_bytes_unchecked(o) })
            },
            Err(e) => Err(e),
        }
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        let bytes: &[u8] = self.as_ref();
        if bytes.len() < buf.len() {
            mem::copy(buf, bytes);
            buf[bytes.len()] = 0;
            Ok(Rmo::Ref(unsafe { CStr::from_bytes_unchecked(&buf[..bytes.len()]) }))
        } else {
            self.to_cstring().map(|o| Rmo::Owned(o))
        }
    }
}

impl ToCString for NoNullString
    where A: Allocator,
{
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.deref().to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.deref().rmo_cstr(buf)
    }
}

impl ToCString for ByteStr {
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.as_ref().to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.as_ref().rmo_cstr(buf)
    }
}

impl ToCString for ByteString
    where A: Allocator,
{
    fn to_cstring<H>(&self) -> Result<CString<H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.as_byte_str().to_cstring()
    }

    fn rmo_cstr<'a, H>(&'a self, buf: &'a mut [u8]) -> Result<Rmo<'a, CStr, H>>
        where H: Allocator,
              H::Pool: Default,
    {
        self.as_byte_str().rmo_cstr(buf)
    }
}
<|endoftext|>


<commit_before>use syntax::core::keywords::Keywords;
use syntax::core::punctuation::Punctuation;

pub enum Token {
    // Just for initializing the lexer
    Start,

    // True, False
    BooleanLiteral(bool),

    // Variables, fn names
    Identifier(String),

    // Keep track of whitespace
    Indent,

    // Reserved words
    Keyword(Keywords),

    // (,),[,],:,:,>,<, ...
    Punctuation(Punctuation),

    // >> Comments. Eventually multiline as well
    //Comment(String),

    // End of file
    EOF
}
<commit_msg>Updated token types to newest design specifications.<commit_after>use syntax::core::keywords::Keywords;
use syntax::core::punctuation::Punctuation;

pub enum Token {
    // Begin a line with this token
    LineBegin,

    // True, False
    BooleanLiteral(bool),

    // Variables, fn names
    Identifier(String),

    // Keep track of scope via indentation
    Indent,
    Dedent(size),

    // Reserved words
    Keyword(Keywords),

    // (,),[,],:,:,>,<, ...
    Punctuation(Punctuation),

    // >> Singleline and >>> \nMultiline comments\n <<<
    CommentStart(String),
    CommentEnd,

    // End of File
    EOF
}
<|endoftext|>


<commit_before>use crate::Msg;
use chrono::{NaiveDate};
use seed::prelude::*;

pub fn render() -> seed::dom_types::Node<Msg> {
    div![
        h1!["News"],
        br![],
        br![],
        br![],
        El::from_html("

Date	News

28th July 2019	bigprimes.net has been rewritten from the ground up using rust!

22nd January 2019	Ethereum bounty contracts have now been added for anyone that would like to try to push the boundaries of human knowledge. The first contract is a competition for the correct solution to a³+b³+c³ = 33. Tweet me if you need assistance using these contracts!

10th March 2018	Added torrents for the first million primes and first billion prime numbers.

12th November 2017	Site is now running on AWS Lambda, but can still fallback to regular VM hosting if ever needed. Site is now ~40% covered by unit tests.

13th November 2016	Site source code has been converted to PHP7 with Silex framework. Some methods have also been moved to cpp from PHP https://github.com/craigmayhew/bigprimes.net/

28th February 2014	You can now find the websites php source code here https://github.com/craigmayhew/bigprimes.net/

19th November 2011	Thank you to Yu Kwong Yiu Wilson for the donation!

13th July 2011	We are retiring the distributed computing client. Thank you all for your help and processing cycles.
        Watch this space for an open source GPU client coming soon...

10th February 2010	We have added a page that shows the sum of the digits of prime numbers. Click here

1st February 2010	Thanks to everyone for donating computer time, we are now upto 1.4 billion primes. We now also have a facebook group where you get news updates and show your support for the project!

28th September 2009	Thanks for all your help! We made it to one billion prime numbers! Please keep on donating processor time as we still have plenty of storage for the prime numbers. here. Our database is now more than double the size (300 million primes) and we've put in the capacity to keep growing. To view our current progress please visit the status page.

18th August 2009	It's been a while but we have been very busy. We've put together a distributed computing client written in javascript. If you wish to donate computing time and process some primes then please go here. Our database is now more than double the size (300 million primes) and we've put in the capacity to keep growing. To view our current progress please visit the status page.

17th July 2008	We've now added a Forum which can be found on the navigation links on the left.

14th July 2008	Someone named Kirk just emailed me to say the download links wern't working. I've now fixed this. Thanks Kirk!

4th July 2008	Added much faster code to the cruncher so that we can handle more of your page loads! Lots of little updates and some reorganizing of the site.

23rd June 2007	Added the Downloads section.

11th February 2007	Added the Fibonacci number archive containing the first 70331 Fibonacci numbers.

16th January 2007	Added the 44th Mersenne Prime number to the archive.

12th January 2007	The Archives are now more organized.

28th September 2005	We have managed to archive the first 100 million prime numbers.

20th May 2005	We have managed to archive the first 75 million prime numbers.

16th May 2005	We have managed to archive the first 50 million prime numbers.

14th May 2005	We have managed to archive the first 10 million prime numbers.

13th May 2005	We have managed to archive the first 1 million prime numbers.

"),
    ]
}<commit_msg>Retired old links.<commit_after>use crate::Msg;
use chrono::{NaiveDate};
use seed::prelude::*;

pub fn render() -> seed::dom_types::Node<Msg> {
    div![
        h1!["News"],
        br![],
        br![],
        br![],
        El::from_html("

Date	News

28th July 2019	bigprimes.net has been rewritten from the ground up using rust!

22nd January 2019	Ethereum bounty contracts have now been added for anyone that would like to try to push the boundaries of human knowledge. The first contract is a competition for the correct solution to a³+b³+c³ = 33. Tweet me if you need assistance using these contracts!

10th March 2018	Added torrents for the first million primes and first billion prime numbers.

12th November 2017	Site is now running on AWS Lambda, but can still fallback to regular VM hosting if ever needed. Site is now ~40% covered by unit tests.

13th November 2016	Site source code has been converted to PHP7 with Silex framework. Some methods have also been moved to cpp from PHP https://github.com/craigmayhew/bigprimes.net/

28th February 2014	You can now find the websites php source code here https://github.com/craigmayhew/bigprimes.net/

19th November 2011	Thank you to Yu Kwong Yiu Wilson for the donation!

13th July 2011	We are retiring the distributed computing client. Thank you all for your help and processing cycles.
        Watch this space for an open source GPU client coming soon...

10th February 2010	We have added a page that shows the sum of the digits of prime numbers (now retired).

1st February 2010	Thanks to everyone for donating computer time, we are now upto 1.4 billion primes. We now also have a facebook group where you get news updates and show your support for the project!

28th September 2009	Thanks for all your help! We made it to one billion prime numbers! Please keep on donating processor time as we still have plenty of storage for the prime numbers. Our database is now more than double the size (300 million primes) and we've put in the capacity to keep growing. To view our current progress please visit the status page.

18th August 2009	It's been a while but we have been very busy. We've put together a distributed computing client written in javascript. Our database is now more than double the size (300 million primes) and we've put in the capacity to keep growing. To view our current progress please visit the status page.

17th July 2008	We've now added a Forum (now retired) which can be found on the navigation links on the left.

14th July 2008	Someone named Kirk just emailed me to say the download links wern't working. I've now fixed this. Thanks Kirk!

4th July 2008	Added much faster code to the cruncher so that we can handle more of your page loads! Lots of little updates and some reorganizing of the site.

23rd June 2007	Added the Downloads section.

11th February 2007	Added the Fibonacci number archive containing the first 70331 Fibonacci numbers.

16th January 2007	Added the 44th Mersenne Prime number to the archive.

12th January 2007	The Archives are now more organized.

28th September 2005	We have managed to archive the first 100 million prime numbers.

20th May 2005	We have managed to archive the first 75 million prime numbers.

16th May 2005	We have managed to archive the first 50 million prime numbers.

14th May 2005	We have managed to archive the first 10 million prime numbers.

13th May 2005	We have managed to archive the first 1 million prime numbers.

"),
    ]
}<|endoftext|>


<commit_before>//! Parser module is responsible for parsing FITS files.

named!(fits<&[u8], (Vec<&[u8]>, Vec<&[u8]>) >,
       pair!(primary_header, many0!( take!(2880) )));

named!(primary_header<&[u8], Vec<&[u8]> >, count!(take!(2880), 2));

named!(end_header<&[u8],(&[u8], Vec<&[u8]>)>, pair!(tag!("END"), count!(tag!(" "), 77)));

named!(blank_header<&[u8],Vec<&[u8]> >, count!(tag!(" "), 80));

#[cfg(test)]
mod tests {
    use nom::{IResult};
    use super::{fits, end_header, blank_header};

    #[test]
    fn it_should_parse_a_fits_file(){
        let data = include_bytes!("../../assets/images/k2-trappist1-unofficial-tpf-long-cadence.fits");

        let result = fits(data);

        match result {
            IResult::Done(_, (header, blocks)) => {
                assert_eq!(header.len(), 2);
                assert_eq!(blocks.len(), 3675);
            },
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }

    #[test]
    fn end_header_should_parse_an_END_header(){
        let data = "END                                                                             "
            .as_bytes();

        let result = end_header(data);

        match result {
            IResult::Done(_,_) => assert!(true),
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }

    #[test]
    fn blank_header_should_parse_a_BLANK_header(){
        let data = "                                                                                "
            .as_bytes();

        let result = blank_header(data);

        match result {
            IResult::Done(_,_) => assert!(true),
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }
}
<commit_msg>Parse a header<commit_after>//! Parser module is responsible for parsing FITS files.

named!(fits<&[u8], (Vec<&[u8]>, Vec<&[u8]>) >,
       pair!(primary_header, many0!( take!(2880) )));

named!(primary_header<&[u8], Vec<&[u8]> >, count!(take!(2880), 2));

named!(a_header<&[u8], &[u8]>, take!(80));

named!(end_header<&[u8],(&[u8], Vec<&[u8]>)>, pair!(tag!("END"), count!(tag!(" "), 77)));

named!(blank_header<&[u8],Vec<&[u8]> >, count!(tag!(" "), 80));

#[cfg(test)]
mod tests {
    use nom::{IResult};
    use super::{fits, end_header, blank_header, a_header};

    #[test]
    fn it_should_parse_a_fits_file(){
        let data = include_bytes!("../../assets/images/k2-trappist1-unofficial-tpf-long-cadence.fits");

        let result = fits(data);

        match result {
            IResult::Done(_, (header, blocks)) => {
                assert_eq!(header.len(), 2);
                assert_eq!(blocks.len(), 3675);
            },
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }

    #[test]
    fn a_header_should_parse_a_header(){
        let data = "OBJECT  = 'EPIC 200164267'     / string version of target id                    "
            .as_bytes();

        let result = a_header(data);

        match result {
            IResult::Done(_,_) => assert!(true),
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }

    #[test]
    fn end_header_should_parse_an_END_header(){
        let data = "END                                                                             "
            .as_bytes();

        let result = end_header(data);

        match result {
            IResult::Done(_,_) => assert!(true),
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }

    #[test]
    fn blank_header_should_parse_a_BLANK_header(){
        let data = "                                                                                "
            .as_bytes();

        let result = blank_header(data);

        match result {
            IResult::Done(_,_) => assert!(true),
            IResult::Error(_) => panic!("Did not expect an error"),
            IResult::Incomplete(_) => panic!("Did not expect to be incomplete")
        }
    }
}
<|endoftext|>


<commit_before>use combine::{eof, satisfy, skip_many, none_of, token, between, optional, many, many1, choice,
              try, Parser, Stream, ParseError};
use combine::char::{string, spaces, digit, char};

#[derive(Clone, Debug, PartialEq)]
pub enum Tokens {
    ACCELERATOR,
    ACTIVETRANSFORM,
    ALL,
    AREALIGHTSOURCE,
    ATTRIBUTEBEGIN,
    ATTRIBUTEEND,
    CAMERA,
    CONCATTRANSFORM,
    COORDINATESYSTEM,
    COORDSYSTRANSFORM,
    ENDTIME,
    FILM,
    IDENTITY,
    INCLUDE,
    LIGHTSOURCE,
    LOOKAT,
    MAKENAMEDMEDIUM,
    MAKENAMEDMATERIAL,
    MATERIAL,
    MEDIUMINTERFACE,
    NAMEDMATERIAL,
    OBJECTBEGIN,
    OBJECTEND,
    OBJECTINSTANCE,
    PIXELFILTER,
    REVERSEORIENTATION,
    ROTATE,
    SAMPLER,
    SCALE,
    SHAPE,
    STARTTIME,
    INTEGRATOR,
    TEXTURE,
    TRANSFORMBEGIN,
    TRANSFORMEND,
    TRANSFORMTIMES,
    TRANSFORM,
    TRANSLATE,
    WORLDBEGIN,
    WORLDEND,
    STR(String),
    NUMBER(f32),
    LBRACK,
    RBRACK,
    COMMENT,
}

pub fn tokenize(input: &str) -> Result<(Vec<Tokens>, &str), ParseError<&str>> {

    // parsers for keywords
    let mut parsers = vec![token_parser("Accelerator", Tokens::ACCELERATOR),
                           token_parser("ActiveTransform", Tokens::ACTIVETRANSFORM),
                           token_parser("All", Tokens::ALL),
                           token_parser("AreaLightSource", Tokens::AREALIGHTSOURCE),
                           token_parser("AttributeBegin", Tokens::ATTRIBUTEBEGIN),
                           token_parser("AttributeEnd", Tokens::ATTRIBUTEEND),
                           token_parser("Camera", Tokens::CAMERA),
                           token_parser("ConcatTransform", Tokens::CONCATTRANSFORM),
                           token_parser("CoordinateSystem", Tokens::COORDINATESYSTEM),
                           token_parser("CoordSysTransform", Tokens::COORDSYSTRANSFORM),
                           token_parser("EndTime", Tokens::ENDTIME),
                           token_parser("Film", Tokens::FILM),
                           token_parser("Identity", Tokens::IDENTITY),
                           token_parser("Include", Tokens::INCLUDE),
                           token_parser("LightSource", Tokens::LIGHTSOURCE),
                           token_parser("LookAt", Tokens::LOOKAT),
                           token_parser("MakeNamedMedium", Tokens::MAKENAMEDMEDIUM),
                           token_parser("MakeNamedMaterial", Tokens::MAKENAMEDMATERIAL),
                           token_parser("Material", Tokens::MATERIAL),
                           token_parser("MediumInterface", Tokens::MEDIUMINTERFACE),
                           token_parser("NamedMaterial", Tokens::NAMEDMATERIAL),
                           token_parser("ObjectBegin", Tokens::OBJECTBEGIN),
                           token_parser("ObjectEnd", Tokens::OBJECTEND),
                           token_parser("ObjectInstance", Tokens::OBJECTINSTANCE),
                           token_parser("PixelFilter", Tokens::PIXELFILTER),
                           token_parser("ReverseOrientation", Tokens::REVERSEORIENTATION),
                           token_parser("Rotate", Tokens::ROTATE),
                           token_parser("Sampler", Tokens::SAMPLER),
                           token_parser("Scale", Tokens::SCALE),
                           token_parser("Shape", Tokens::SHAPE),
                           token_parser("StartTime", Tokens::STARTTIME),
                           token_parser("Integrator", Tokens::INTEGRATOR),
                           token_parser("Texture", Tokens::TEXTURE),
                           token_parser("TransformBegin", Tokens::TRANSFORMBEGIN),
                           token_parser("TransformEnd", Tokens::TRANSFORMEND),
                           token_parser("TransformTimes", Tokens::TRANSFORMTIMES),
                           token_parser("Transform", Tokens::TRANSFORM),
                           token_parser("Translate", Tokens::TRANSLATE),
                           token_parser("WorldBegin", Tokens::WORLDBEGIN),
                           token_parser("WorldEnd", Tokens::WORLDEND),
                           token_parser("[", Tokens::LBRACK),
                           token_parser("]", Tokens::RBRACK)]
            .into_iter()
            .map(try)
            .collect::<Vec<_>>();

    // Add parsers from num, strings, etc...
    parsers.push(try(float_parser()));
    parsers.push(try(string_parser()));
    parsers.push(try(comment_parser()));

    (spaces().with(many::<Vec<_>, _>(choice(parsers))), eof())
        .map(|(res, _)| res)
        .parse(input)
}

fn token_parser<'a, I: Stream<Item = char> + 'a>
    (s: &'static str,
     t: Tokens)
     -> Box<Parser<Input = I, Output = Tokens> + 'a> {
    string(s).skip(spaces()).map(move |_| t.clone()).boxed()
}

fn float_parser<'a, I: Stream<Item = char> + 'a>
    ()
    -> Box<Parser<Input = I, Output = Tokens> + 'a>
{

    (optional(char('-').or(char('+'))),
     many1::<Vec<_>, _>(digit()),
     optional(char('.').with(many::<Vec<_>, _>(digit()))),
     optional(char('e')
                  .or(char('E'))
                  .with((optional(char('-').or(char('+'))), many1::<Vec<_>, _>(digit())))))
            .skip(spaces())
            .and_then(|(sign, int_part, frac_part, mantissa)| {
                let mut buf = String::new();
                if let Some(s) = sign {
                    buf.push(s);
                }
                for c in int_part {
                    buf.push(c);
                }
                if let Some(frac) = frac_part {
                    buf.push('.');
                    for c in frac {
                        buf.push(c);
                    }
                }
                if let Some((mant_sign, mant)) = mantissa {
                    buf.push('e');
                    buf.push(mant_sign.unwrap_or('+'));
                    for c in mant {
                        buf.push(c);
                    }
                }

                buf.parse::<f32>().map(Tokens::NUMBER)
            })
            .boxed()
}

fn string_parser<'a, I: Stream<Item = char> + 'a>
    ()
    -> Box<Parser<Input = I, Output = Tokens> + 'a>
{
    between(token('"'),
            token('"'),
            many::<Vec<_>, _>(none_of("\"".chars())))
            .skip(spaces())
            .map(|chars| {
                     let mut buf = String::new();
                     for c in chars {
                         buf.push(c);
                     }
                     Tokens::STR(buf)
                 })
            .boxed()
}

fn comment_parser<'a, I: Stream<Item = char> + 'a>
    ()
    -> Box<Parser<Input = I, Output = Tokens> + 'a>
{
    token('#')
        .and(skip_many(satisfy(|c| c != '\n')))
        .skip(spaces())
        .map(|_| Tokens::COMMENT)
        .boxed()
}

#[test]
fn test_tokenize() {
    let scene = r##"
LookAt 0 0 5 0 0 0 0 1 0
Camera "perspective" "float fov" [50]

Film "image" "integer xresolution" [800] "integer yresolution" [600]
    "string filename" "test-whitted.tga"

Integrator "whitted"
#Integrator "directlighting"

WorldBegin
  LightSource "distant" "point from" [0 1 5] "point to" [0 0 0]

  #Material "matte" "rgb Kd" [1.0 0.0 0.0] "float sigma" [20]
  AttributeBegin
    #Material "matte" "rgb Kd" [1.0 0.0 0.0]
    Material "plastic" "rgb Kd" [1.0 0.0 0.0] "rgb Ks" [1.0 1.0 1.0]

    Shape "sphere"
  AttributeEnd

  AttributeBegin
    Rotate -90 1 0 0
    Material "matte" "rgb Kd" [1.0 1.0 1.0]
    Shape "disk" "float radius" [20] "float height" [-1]
  AttributeEnd

  AttributeBegin
    AreaLightSource "diffuse" "rgb L" [2.0 2.0 2.0]
    Rotate 90 1 0 0
    Shape "disk" "float height" [-2] "float radius" [0.5]
  AttributeEnd
WorldEnd
        "##;
    let result = tokenize(scene);
    assert!(result.is_ok());
    let (_tokens, rest) = result.unwrap();
    assert_eq!(rest, "");
}

#[test]
fn test_string_parser() {
    let s = "\"this is a string\"";
    let result = string_parser().parse(s);
    assert_eq!(result,
               Ok((Tokens::STR("this is a string".to_string()), "")))
}
<commit_msg>Try to allow parsing floats like -.5<commit_after>use combine::{eof, satisfy, skip_many, none_of, token, between, optional, many, many1, choice,
              try, Parser, Stream, ParseError};
use combine::char::{string, spaces, digit, char};

#[derive(Clone, Debug, PartialEq)]
pub enum Tokens {
    ACCELERATOR,
    ACTIVETRANSFORM,
    ALL,
    AREALIGHTSOURCE,
    ATTRIBUTEBEGIN,
    ATTRIBUTEEND,
    CAMERA,
    CONCATTRANSFORM,
    COORDINATESYSTEM,
    COORDSYSTRANSFORM,
    ENDTIME,
    FILM,
    IDENTITY,
    INCLUDE,
    LIGHTSOURCE,
    LOOKAT,
    MAKENAMEDMEDIUM,
    MAKENAMEDMATERIAL,
    MATERIAL,
    MEDIUMINTERFACE,
    NAMEDMATERIAL,
    OBJECTBEGIN,
    OBJECTEND,
    OBJECTINSTANCE,
    PIXELFILTER,
    REVERSEORIENTATION,
    ROTATE,
    SAMPLER,
    SCALE,
    SHAPE,
    STARTTIME,
    INTEGRATOR,
    TEXTURE,
    TRANSFORMBEGIN,
    TRANSFORMEND,
    TRANSFORMTIMES,
    TRANSFORM,
    TRANSLATE,
    WORLDBEGIN,
    WORLDEND,
    STR(String),
    NUMBER(f32),
    LBRACK,
    RBRACK,
    COMMENT,
}

pub fn tokenize(input: &str) -> Result<(Vec<Tokens>, &str), ParseError<&str>> {

    // parsers for keywords
    let mut parsers = vec![token_parser("Accelerator", Tokens::ACCELERATOR),
                           token_parser("ActiveTransform", Tokens::ACTIVETRANSFORM),
                           token_parser("All", Tokens::ALL),
                           token_parser("AreaLightSource", Tokens::AREALIGHTSOURCE),
                           token_parser("AttributeBegin", Tokens::ATTRIBUTEBEGIN),
                           token_parser("AttributeEnd", Tokens::ATTRIBUTEEND),
                           token_parser("Camera", Tokens::CAMERA),
                           token_parser("ConcatTransform", Tokens::CONCATTRANSFORM),
                           token_parser("CoordinateSystem", Tokens::COORDINATESYSTEM),
                           token_parser("CoordSysTransform", Tokens::COORDSYSTRANSFORM),
                           token_parser("EndTime", Tokens::ENDTIME),
                           token_parser("Film", Tokens::FILM),
                           token_parser("Identity", Tokens::IDENTITY),
                           token_parser("Include", Tokens::INCLUDE),
                           token_parser("LightSource", Tokens::LIGHTSOURCE),
                           token_parser("LookAt", Tokens::LOOKAT),
                           token_parser("MakeNamedMedium", Tokens::MAKENAMEDMEDIUM),
                           token_parser("MakeNamedMaterial", Tokens::MAKENAMEDMATERIAL),
                           token_parser("Material", Tokens::MATERIAL),
                           token_parser("MediumInterface", Tokens::MEDIUMINTERFACE),
                           token_parser("NamedMaterial", Tokens::NAMEDMATERIAL),
                           token_parser("ObjectBegin", Tokens::OBJECTBEGIN),
                           token_parser("ObjectEnd", Tokens::OBJECTEND),
                           token_parser("ObjectInstance", Tokens::OBJECTINSTANCE),
                           token_parser("PixelFilter", Tokens::PIXELFILTER),
                           token_parser("ReverseOrientation", Tokens::REVERSEORIENTATION),
                           token_parser("Rotate", Tokens::ROTATE),
                           token_parser("Sampler", Tokens::SAMPLER),
                           token_parser("Scale", Tokens::SCALE),
                           token_parser("Shape", Tokens::SHAPE),
                           token_parser("StartTime", Tokens::STARTTIME),
                           token_parser("Integrator", Tokens::INTEGRATOR),
                           token_parser("Texture", Tokens::TEXTURE),
                           token_parser("TransformBegin", Tokens::TRANSFORMBEGIN),
                           token_parser("TransformEnd", Tokens::TRANSFORMEND),
                           token_parser("TransformTimes", Tokens::TRANSFORMTIMES),
                           token_parser("Transform", Tokens::TRANSFORM),
                           token_parser("Translate", Tokens::TRANSLATE),
                           token_parser("WorldBegin", Tokens::WORLDBEGIN),
                           token_parser("WorldEnd", Tokens::WORLDEND),
                           token_parser("[", Tokens::LBRACK),
                           token_parser("]", Tokens::RBRACK)]
            .into_iter()
            .map(try)
            .collect::<Vec<_>>();

    // Add parsers from num, strings, etc...
    parsers.push(try(float_parser()));
    parsers.push(try(string_parser()));
    parsers.push(try(comment_parser()));

    (spaces().with(many::<Vec<_>, _>(choice(parsers))), eof())
        .map(|(res, _)| res)
        .parse(input)
}

fn token_parser<'a, I: Stream<Item = char> + 'a>
    (s: &'static str,
     t: Tokens)
     -> Box<Parser<Input = I, Output = Tokens> + 'a> {
    string(s).skip(spaces()).map(move |_| t.clone()).boxed()
}

fn float_parser<'a, I: Stream<Item = char> + 'a>
    ()
    -> Box<Parser<Input = I, Output = Tokens> + 'a>
{

    (optional(char('-').or(char('+'))),
     optional(many::<Vec<_>, _>(digit())),
     optional(char('.').with(many::<Vec<_>, _>(digit()))),
     optional(char('e')
                  .or(char('E'))
                  .with((optional(char('-').or(char('+'))), many1::<Vec<_>, _>(digit())))))
            .skip(spaces())
            .and_then(|(sign, int_part, frac_part, mantissa)| {
                let mut buf = String::new();
                if let Some(s) = sign {
                    buf.push(s);
                }
                if let Some(intpart) = int_part {
                    for c in intpart {
                        buf.push(c);
                    }
                } else {
                    buf.push('0');
                }
                if let Some(frac) = frac_part {
                    buf.push('.');
                    for c in frac {
                        buf.push(c);
                    }
                }
                if let Some((mant_sign, mant)) = mantissa {
                    buf.push('e');
                    buf.push(mant_sign.unwrap_or('+'));
                    for c in mant {
                        buf.push(c);
                    }
                }

                buf.parse::<f32>().map(Tokens::NUMBER)
            })
            .boxed()
}

fn string_parser<'a, I: Stream<Item = char> + 'a>
    ()
    -> Box<Parser<Input = I, Output = Tokens> + 'a>
{
    between(token('"'),
            token('"'),
            many::<Vec<_>, _>(none_of("\"".chars())))
            .skip(spaces())
            .map(|chars| {
                     let mut buf = String::new();
                     for c in chars {
                         buf.push(c);
                     }
                     Tokens::STR(buf)
                 })
            .boxed()
}

fn comment_parser<'a, I: Stream<Item = char> + 'a>
    ()
    -> Box<Parser<Input = I, Output = Tokens> + 'a>
{
    token('#')
        .and(skip_many(satisfy(|c| c != '\n')))
        .skip(spaces())
        .map(|_| Tokens::COMMENT)
        .boxed()
}

#[test]
fn test_tokenize() {
    let scene = r##"
LookAt 0 0 5 0 0 0 0 1 0
Camera "perspective" "float fov" [50]

Film "image" "integer xresolution" [800] "integer yresolution" [600]
    "string filename" "test-whitted.tga"

Integrator "whitted"
#Integrator "directlighting"

WorldBegin
  LightSource "distant" "point from" [0 1 5] "point to" [0 0 0]

  #Material "matte" "rgb Kd" [1.0 0.0 0.0] "float sigma" [20]
  AttributeBegin
    #Material "matte" "rgb Kd" [1.0 0.0 0.0]
    Material "plastic" "rgb Kd" [1.0 0.0 0.0] "rgb Ks" [1.0 1.0 1.0]

    Shape "sphere"
  AttributeEnd

  AttributeBegin
    Rotate -90 1 0 0
    Material "matte" "rgb Kd" [1.0 1.0 1.0]
    Shape "disk" "float radius" [20] "float height" [-1]
  AttributeEnd

  AttributeBegin
    AreaLightSource "diffuse" "rgb L" [2.0 2.0 2.0]
    Rotate 90 1 0 0
    Shape "disk" "float height" [-2] "float radius" [0.5]
  AttributeEnd
WorldEnd
        "##;
    let result = tokenize(scene);
    assert!(result.is_ok());
    let (_tokens, rest) = result.unwrap();
    assert_eq!(rest, "");
}

#[test]
fn test_string_parser() {
    let s = "\"this is a string\"";
    let result = string_parser().parse(s);
    assert_eq!(result,
               Ok((Tokens::STR("this is a string".to_string()), "")))
}
<|endoftext|>


<commit_before>#![cfg_attr(feature = "cargo-clippy", allow(blacklisted_name))]

use std::borrow::Cow;
use std::collections::BTreeSet;

use proc_macro2::{Ident, Span, TokenStream};
use quote::{format_ident, quote, TokenStreamExt};

struct X;

impl quote::ToTokens for X {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Ident::new("X", Span::call_site()));
    }
}

#[test]
fn test_quote_impl() {
    let tokens = quote! {
        impl<'a, T: ToTokens> ToTokens for &'a T {
            fn to_tokens(&self, tokens: &mut TokenStream) {
                (**self).to_tokens(tokens)
            }
        }
    };

    let expected = concat!(
        "impl < 'a , T : ToTokens > ToTokens for & 'a T { ",
        "fn to_tokens ( & self , tokens : & mut TokenStream ) { ",
        "( * * self ) . to_tokens ( tokens ) ",
        "} ",
        "}"
    );

    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_substitution() {
    let x = X;
    let tokens = quote!(#x <#x> (#x) [#x] {#x});

    let expected = "X < X > ( X ) [ X ] { X }";

    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_iter() {
    let primes = &[X, X, X, X];

    assert_eq!("X X X X", quote!(#(#primes)*).to_string());

    assert_eq!("X , X , X , X ,", quote!(#(#primes,)*).to_string());

    assert_eq!("X , X , X , X", quote!(#(#primes),*).to_string());
}

#[test]
fn test_advanced() {
    let generics = quote!( <'a, T> );

    let where_clause = quote!( where T: Serialize );

    let field_ty = quote!(String);

    let item_ty = quote!(Cow<'a, str>);

    let path = quote!(SomeTrait::serialize_with);

    let value = quote!(self.x);

    let tokens = quote! {
        struct SerializeWith #generics #where_clause {
            value: &'a #field_ty,
            phantom: ::std::marker::PhantomData<#item_ty>,
        }

        impl #generics ::serde::Serialize for SerializeWith #generics #where_clause {
            fn serialize<S>(&self, s: &mut S) -> Result<(), S::Error>
                where S: ::serde::Serializer
            {
                #path(self.value, s)
            }
        }

        SerializeWith {
            value: #value,
            phantom: ::std::marker::PhantomData::<#item_ty>,
        }
    };

    let expected = concat!(
        "struct SerializeWith < 'a , T > where T : Serialize { ",
        "value : & 'a String , ",
        "phantom : :: std :: marker :: PhantomData < Cow < 'a , str > > , ",
        "} ",
        "impl < 'a , T > :: serde :: Serialize for SerializeWith < 'a , T > where T : Serialize { ",
        "fn serialize < S > ( & self , s : & mut S ) -> Result < ( ) , S :: Error > ",
        "where S : :: serde :: Serializer ",
        "{ ",
        "SomeTrait :: serialize_with ( self . value , s ) ",
        "} ",
        "} ",
        "SerializeWith { ",
        "value : self . x , ",
        "phantom : :: std :: marker :: PhantomData :: < Cow < 'a , str > > , ",
        "}"
    );

    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_integer() {
    let ii8 = -1i8;
    let ii16 = -1i16;
    let ii32 = -1i32;
    let ii64 = -1i64;
    let ii128 = -1i128;
    let iisize = -1isize;
    let uu8 = 1u8;
    let uu16 = 1u16;
    let uu32 = 1u32;
    let uu64 = 1u64;
    let uu128 = 1u128;
    let uusize = 1usize;

    let tokens = quote! {
        #ii8 #ii16 #ii32 #ii64 #ii128 #iisize
        #uu8 #uu16 #uu32 #uu64 #uu128 #uusize
    };
    let expected = "-1i8 -1i16 -1i32 -1i64 -1i128 -1isize 1u8 1u16 1u32 1u64 1u128 1usize";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_floating() {
    let e32 = 2.345f32;

    let e64 = 2.345f64;

    let tokens = quote! {
        #e32
        #e64
    };
    let expected = concat!("2.345f32 2.345f64");
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_char() {
    let zero = '\0';
    let pound = '#';
    let quote = '"';
    let apost = '\'';
    let newline = '\n';
    let heart = '\u{2764}';

    let tokens = quote! {
        #zero #pound #quote #apost #newline #heart
    };
    let expected = "'\\u{0}' '#' '\"' '\\'' '\\n' '\\u{2764}'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_str() {
    let s = "\0 a 'b \" c";
    let tokens = quote!(#s);
    let expected = "\"\\u{0} a 'b \\\" c\"";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_string() {
    let s = "\0 a 'b \" c".to_string();
    let tokens = quote!(#s);
    let expected = "\"\\u{0} a 'b \\\" c\"";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_ident() {
    let foo = Ident::new("Foo", Span::call_site());
    let bar = Ident::new(&format!("Bar{}", 7), Span::call_site());
    let tokens = quote!(struct #foo; enum #bar {});
    let expected = "struct Foo ; enum Bar7 { }";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_duplicate() {
    let ch = 'x';

    let tokens = quote!(#ch #ch);

    let expected = "'x' 'x'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_fancy_repetition() {
    let foo = vec!["a", "b"];
    let bar = vec![true, false];

    let tokens = quote! {
        #(#foo: #bar),*
    };

    let expected = r#""a" : true , "b" : false"#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_nested_fancy_repetition() {
    let nested = vec![vec!['a', 'b', 'c'], vec!['x', 'y', 'z']];

    let tokens = quote! {
        #(
            #(#nested)*
        ),*
    };

    let expected = "'a' 'b' 'c' , 'x' 'y' 'z'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_duplicate_name_repetition() {
    let foo = &["a", "b"];

    let tokens = quote! {
        #(#foo: #foo),*
        #(#foo: #foo),*
    };

    let expected = r#""a" : "a" , "b" : "b" "a" : "a" , "b" : "b""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_duplicate_name_repetition_no_copy() {
    let foo = vec!["a".to_owned(), "b".to_owned()];

    let tokens = quote! {
        #(#foo: #foo),*
    };

    let expected = r#""a" : "a" , "b" : "b""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_btreeset_repetition() {
    let mut set = BTreeSet::new();
    set.insert("a".to_owned());
    set.insert("b".to_owned());

    let tokens = quote! {
        #(#set: #set),*
    };

    let expected = r#""a" : "a" , "b" : "b""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_variable_name_conflict() {
    // The implementation of `#(...),*` uses the variable `_i` but it should be
    // fine, if a little confusing when debugging.
    let _i = vec!['a', 'b'];
    let tokens = quote! { #(#_i),* };
    let expected = "'a' , 'b'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_nonrep_in_repetition() {
    let rep = vec!["a", "b"];
    let nonrep = "c";

    let tokens = quote! {
        #(#rep #rep : #nonrep #nonrep),*
    };

    let expected = r#""a" "a" : "c" "c" , "b" "b" : "c" "c""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_empty_quote() {
    let tokens = quote!();
    assert_eq!("", tokens.to_string());
}

#[test]
fn test_box_str() {
    let b = "str".to_owned().into_boxed_str();
    let tokens = quote! { #b };
    assert_eq!("\"str\"", tokens.to_string());
}

#[test]
fn test_cow() {
    let owned: Cow<Ident> = Cow::Owned(Ident::new("owned", Span::call_site()));

    let ident = Ident::new("borrowed", Span::call_site());
    let borrowed = Cow::Borrowed(&ident);

    let tokens = quote! { #owned #borrowed };
    assert_eq!("owned borrowed", tokens.to_string());
}

#[test]
fn test_closure() {
    fn field_i(i: usize) -> Ident {
        format_ident!("__field{}", i)
    }

    let fields = (0usize..3)
        .map(field_i as fn(_) -> _)
        .map(|var| quote! { #var });

    let tokens = quote! { #(#fields)* };
    assert_eq!("__field0 __field1 __field2", tokens.to_string());
}

#[test]
fn test_append_tokens() {
    let mut a = quote!(a);
    let b = quote!(b);
    a.append_all(b);
    assert_eq!("a b", a.to_string());
}

#[test]
fn test_format_ident() {
    let id0 = format_ident!("Aa");
    let id1 = format_ident!("Hello{x}", x = id0);
    let id2 = format_ident!("Hello{x}", x = 5usize);
    let id3 = format_ident!("Hello{}_{x}", id0, x = 10usize);
    let id4 = format_ident!("Aa", span = Span::call_site());

    assert_eq!(id0, "Aa");
    assert_eq!(id1, "HelloAa");
    assert_eq!(id2, "Hello5");
    assert_eq!(id3, "HelloAa_10");
    assert_eq!(id4, "Aa");
}

#[test]
fn test_format_ident_strip_raw() {
    let id = format_ident!("r#struct");
    let my_id = format_ident!("MyId{}", id);
    let raw_my_id = format_ident!("r#MyId{}", id);

    assert_eq!(id, "r#struct");
    assert_eq!(my_id, "MyIdstruct");
    assert_eq!(raw_my_id, "r#MyIdstruct");
}

#[test]
fn test_outer_line_comment() {
    let tokens = quote! {
        /// doc
    };
    let expected = "# [ doc = r\" doc\" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_inner_line_comment() {
    let tokens = quote! {
        //! doc
    };
    let expected = "# ! [ doc = r\" doc\" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_outer_block_comment() {
    let tokens = quote! {
        /** doc */
    };
    let expected = "# [ doc = r\" doc \" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_inner_block_comment() {
    let tokens = quote! {
        /*! doc */
    };
    let expected = "# ! [ doc = r\" doc \" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_outer_attr() {
    let tokens = quote! {
        #[inline]
    };
    let expected = "# [ inline ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_inner_attr() {
    let tokens = quote! {
        #![no_std]
    };
    let expected = "# ! [ no_std ]";
    assert_eq!(expected, tokens.to_string());
}
<commit_msg>Add regression test for issue 130<commit_after>#![cfg_attr(feature = "cargo-clippy", allow(blacklisted_name))]

use std::borrow::Cow;
use std::collections::BTreeSet;

use proc_macro2::{Ident, Span, TokenStream};
use quote::{format_ident, quote, TokenStreamExt};

struct X;

impl quote::ToTokens for X {
    fn to_tokens(&self, tokens: &mut TokenStream) {
        tokens.append(Ident::new("X", Span::call_site()));
    }
}

#[test]
fn test_quote_impl() {
    let tokens = quote! {
        impl<'a, T: ToTokens> ToTokens for &'a T {
            fn to_tokens(&self, tokens: &mut TokenStream) {
                (**self).to_tokens(tokens)
            }
        }
    };

    let expected = concat!(
        "impl < 'a , T : ToTokens > ToTokens for & 'a T { ",
        "fn to_tokens ( & self , tokens : & mut TokenStream ) { ",
        "( * * self ) . to_tokens ( tokens ) ",
        "} ",
        "}"
    );

    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_substitution() {
    let x = X;
    let tokens = quote!(#x <#x> (#x) [#x] {#x});

    let expected = "X < X > ( X ) [ X ] { X }";

    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_iter() {
    let primes = &[X, X, X, X];

    assert_eq!("X X X X", quote!(#(#primes)*).to_string());

    assert_eq!("X , X , X , X ,", quote!(#(#primes,)*).to_string());

    assert_eq!("X , X , X , X", quote!(#(#primes),*).to_string());
}

#[test]
fn test_advanced() {
    let generics = quote!( <'a, T> );

    let where_clause = quote!( where T: Serialize );

    let field_ty = quote!(String);

    let item_ty = quote!(Cow<'a, str>);

    let path = quote!(SomeTrait::serialize_with);

    let value = quote!(self.x);

    let tokens = quote! {
        struct SerializeWith #generics #where_clause {
            value: &'a #field_ty,
            phantom: ::std::marker::PhantomData<#item_ty>,
        }

        impl #generics ::serde::Serialize for SerializeWith #generics #where_clause {
            fn serialize<S>(&self, s: &mut S) -> Result<(), S::Error>
                where S: ::serde::Serializer
            {
                #path(self.value, s)
            }
        }

        SerializeWith {
            value: #value,
            phantom: ::std::marker::PhantomData::<#item_ty>,
        }
    };

    let expected = concat!(
        "struct SerializeWith < 'a , T > where T : Serialize { ",
        "value : & 'a String , ",
        "phantom : :: std :: marker :: PhantomData < Cow < 'a , str > > , ",
        "} ",
        "impl < 'a , T > :: serde :: Serialize for SerializeWith < 'a , T > where T : Serialize { ",
        "fn serialize < S > ( & self , s : & mut S ) -> Result < ( ) , S :: Error > ",
        "where S : :: serde :: Serializer ",
        "{ ",
        "SomeTrait :: serialize_with ( self . value , s ) ",
        "} ",
        "} ",
        "SerializeWith { ",
        "value : self . x , ",
        "phantom : :: std :: marker :: PhantomData :: < Cow < 'a , str > > , ",
        "}"
    );

    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_integer() {
    let ii8 = -1i8;
    let ii16 = -1i16;
    let ii32 = -1i32;
    let ii64 = -1i64;
    let ii128 = -1i128;
    let iisize = -1isize;
    let uu8 = 1u8;
    let uu16 = 1u16;
    let uu32 = 1u32;
    let uu64 = 1u64;
    let uu128 = 1u128;
    let uusize = 1usize;

    let tokens = quote! {
        #ii8 #ii16 #ii32 #ii64 #ii128 #iisize
        #uu8 #uu16 #uu32 #uu64 #uu128 #uusize
    };
    let expected = "-1i8 -1i16 -1i32 -1i64 -1i128 -1isize 1u8 1u16 1u32 1u64 1u128 1usize";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_floating() {
    let e32 = 2.345f32;

    let e64 = 2.345f64;

    let tokens = quote! {
        #e32
        #e64
    };
    let expected = concat!("2.345f32 2.345f64");
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_char() {
    let zero = '\0';
    let pound = '#';
    let quote = '"';
    let apost = '\'';
    let newline = '\n';
    let heart = '\u{2764}';

    let tokens = quote! {
        #zero #pound #quote #apost #newline #heart
    };
    let expected = "'\\u{0}' '#' '\"' '\\'' '\\n' '\\u{2764}'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_str() {
    let s = "\0 a 'b \" c";
    let tokens = quote!(#s);
    let expected = "\"\\u{0} a 'b \\\" c\"";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_string() {
    let s = "\0 a 'b \" c".to_string();
    let tokens = quote!(#s);
    let expected = "\"\\u{0} a 'b \\\" c\"";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_ident() {
    let foo = Ident::new("Foo", Span::call_site());
    let bar = Ident::new(&format!("Bar{}", 7), Span::call_site());
    let tokens = quote!(struct #foo; enum #bar {});
    let expected = "struct Foo ; enum Bar7 { }";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_duplicate() {
    let ch = 'x';

    let tokens = quote!(#ch #ch);

    let expected = "'x' 'x'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_fancy_repetition() {
    let foo = vec!["a", "b"];
    let bar = vec![true, false];

    let tokens = quote! {
        #(#foo: #bar),*
    };

    let expected = r#""a" : true , "b" : false"#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_nested_fancy_repetition() {
    let nested = vec![vec!['a', 'b', 'c'], vec!['x', 'y', 'z']];

    let tokens = quote! {
        #(
            #(#nested)*
        ),*
    };

    let expected = "'a' 'b' 'c' , 'x' 'y' 'z'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_duplicate_name_repetition() {
    let foo = &["a", "b"];

    let tokens = quote! {
        #(#foo: #foo),*
        #(#foo: #foo),*
    };

    let expected = r#""a" : "a" , "b" : "b" "a" : "a" , "b" : "b""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_duplicate_name_repetition_no_copy() {
    let foo = vec!["a".to_owned(), "b".to_owned()];

    let tokens = quote! {
        #(#foo: #foo),*
    };

    let expected = r#""a" : "a" , "b" : "b""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_btreeset_repetition() {
    let mut set = BTreeSet::new();
    set.insert("a".to_owned());
    set.insert("b".to_owned());

    let tokens = quote! {
        #(#set: #set),*
    };

    let expected = r#""a" : "a" , "b" : "b""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_variable_name_conflict() {
    // The implementation of `#(...),*` uses the variable `_i` but it should be
    // fine, if a little confusing when debugging.
    let _i = vec!['a', 'b'];
    let tokens = quote! { #(#_i),* };
    let expected = "'a' , 'b'";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_nonrep_in_repetition() {
    let rep = vec!["a", "b"];
    let nonrep = "c";

    let tokens = quote! {
        #(#rep #rep : #nonrep #nonrep),*
    };

    let expected = r#""a" "a" : "c" "c" , "b" "b" : "c" "c""#;
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_empty_quote() {
    let tokens = quote!();
    assert_eq!("", tokens.to_string());
}

#[test]
fn test_box_str() {
    let b = "str".to_owned().into_boxed_str();
    let tokens = quote! { #b };
    assert_eq!("\"str\"", tokens.to_string());
}

#[test]
fn test_cow() {
    let owned: Cow<Ident> = Cow::Owned(Ident::new("owned", Span::call_site()));

    let ident = Ident::new("borrowed", Span::call_site());
    let borrowed = Cow::Borrowed(&ident);

    let tokens = quote! { #owned #borrowed };
    assert_eq!("owned borrowed", tokens.to_string());
}

#[test]
fn test_closure() {
    fn field_i(i: usize) -> Ident {
        format_ident!("__field{}", i)
    }

    let fields = (0usize..3)
        .map(field_i as fn(_) -> _)
        .map(|var| quote! { #var });

    let tokens = quote! { #(#fields)* };
    assert_eq!("__field0 __field1 __field2", tokens.to_string());
}

#[test]
fn test_append_tokens() {
    let mut a = quote!(a);
    let b = quote!(b);
    a.append_all(b);
    assert_eq!("a b", a.to_string());
}

#[test]
fn test_format_ident() {
    let id0 = format_ident!("Aa");
    let id1 = format_ident!("Hello{x}", x = id0);
    let id2 = format_ident!("Hello{x}", x = 5usize);
    let id3 = format_ident!("Hello{}_{x}", id0, x = 10usize);
    let id4 = format_ident!("Aa", span = Span::call_site());

    assert_eq!(id0, "Aa");
    assert_eq!(id1, "HelloAa");
    assert_eq!(id2, "Hello5");
    assert_eq!(id3, "HelloAa_10");
    assert_eq!(id4, "Aa");
}

#[test]
fn test_format_ident_strip_raw() {
    let id = format_ident!("r#struct");
    let my_id = format_ident!("MyId{}", id);
    let raw_my_id = format_ident!("r#MyId{}", id);

    assert_eq!(id, "r#struct");
    assert_eq!(my_id, "MyIdstruct");
    assert_eq!(raw_my_id, "r#MyIdstruct");
}

#[test]
fn test_outer_line_comment() {
    let tokens = quote! {
        /// doc
    };
    let expected = "# [ doc = r\" doc\" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_inner_line_comment() {
    let tokens = quote! {
        //! doc
    };
    let expected = "# ! [ doc = r\" doc\" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_outer_block_comment() {
    let tokens = quote! {
        /** doc */
    };
    let expected = "# [ doc = r\" doc \" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_inner_block_comment() {
    let tokens = quote! {
        /*! doc */
    };
    let expected = "# ! [ doc = r\" doc \" ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_outer_attr() {
    let tokens = quote! {
        #[inline]
    };
    let expected = "# [ inline ]";
    assert_eq!(expected, tokens.to_string());
}

#[test]
fn test_inner_attr() {
    let tokens = quote! {
        #![no_std]
    };
    let expected = "# ! [ no_std ]";
    assert_eq!(expected, tokens.to_string());
}

// https://github.com/dtolnay/quote/issues/130
#[test]
fn test_star_after_repetition() {
    let c = vec!['0', '1'];
    let tokens = quote! {
        #(
            f(#c);
        )*
        *out = None;
    };
    let expected = "f ( '0' ) ; f ( '1' ) ; * out = None ;";
    assert_eq!(expected, tokens.to_string());
}
<|endoftext|>