<commit_before>#![crate_name="json_macros"]
#![crate_type="dylib"]
#![feature(phase, plugin_registrar, macro_rules, quote)]

#[phase(plugin, link)] extern crate log;
extern crate rustc;
extern crate syntax;
extern crate serialize;

use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast;
use syntax::ext::base::{ExtCtxt, MacResult, MacExpr, DummyResult};
use syntax::parse::token;
use syntax::print::pprust;
use rustc::plugin::Registry;

type PExpr = P<ast::Expr>;

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("json", expand);
}

fn expand<'cx>(cx: &'cx mut ExtCtxt, sp: Span,
               tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    debug!("JSON token tree {}", tts);

    let tt = match tts.get(0) {
        Some(tt) => tt,
        None => {
            cx.span_err(sp, "expected JSON literal");
            return DummyResult::expr(sp);
        }
    };

    let expr = match tt_to_expr(cx, tt) {
        Some(e) => e,
        None => return DummyResult::expr(sp)
    };
    MacExpr::new(expr)
}

fn tt_to_expr(cx: &ExtCtxt, tt: &TokenTree) -> Option<PExpr> {
    use syntax::ext::build::AstBuilder;

    match *tt {
        ast::TtToken(sp, ref tok) => token_to_expr(cx, sp, tok),
        ast::TtDelimited(sp, ref d) => {
            let ref toks = d.tts;
            match d.delim {
                // array
                token::Bracket => {
                    let exprs = match parse_array(cx, sp, toks.as_slice()) {
                        Some(e) => e,
                        None => return None
                    };
                    let exprs = cx.expr_vec(sp, exprs);
                    Some(quote_expr!(cx, {
                        {
                            use std::slice::BoxedSlicePrelude;
                            let xs: ::std::boxed::Box<[_]> = box $exprs;
                            ::serialize::json::List(xs.into_vec())
                        }
                    }))
                }
                // object
                token::Brace => {
                    let items = match parse_object(cx, sp, toks.as_slice()) {
                        Some(i) => i,
                        None => return None
                    };
                    let ob = quote_expr!(cx, _ob);
                    let mut stmts = vec![];
                    for &(ref key, ref value) in items.iter() {
                        stmts.push(quote_stmt!(cx, $ob.insert($key, $value)));
                    }

                    Some(quote_expr!(cx, {
                        {
                            let mut $ob = ::std::collections::TreeMap::new();
                            $stmts;
                            ::serialize::json::Object($ob)
                        }
                    }))
                }
                token::Paren => {
                    let mut parser = cx.new_parser_from_tts(toks.as_slice());
                    let expr = parser.parse_expr();

                    Some(quote_expr!(cx, {
                        use serialize::json::ToJson;
                        ($expr).to_json()
                    }))
                }
            }
        }
        ast::TtSequence(sp, _) => {
            cx.span_err(sp, "`json!` unexpected TtSequence");
            None
        }
    }
}

fn parse_array(cx: &ExtCtxt, sp: Span, tts: &[TokenTree]) -> Option<Vec<PExpr>> {
    let mut exprs = Vec::with_capacity(tts.len() / 2);
    for (i, tt) in tts.iter().enumerate() {
        if i % 2 == 1 {
            match tt {
                &ast::TtToken(_, token::Comma) => {
                    continue;
                }
                _ => {
                    expected_but_found(cx, sp, "`,`", tt);
                    return None;
                }
            }
        }
        let expr = tt_to_expr(cx, tt);
        let expr = match expr {
            Some(e) => e,
            None => return None,
        };
        exprs.push(expr);
    }
    Some(exprs)
}

fn parse_object(cx: &ExtCtxt, sp: Span, tts: &[TokenTree]) -> Option<Vec<(PExpr, PExpr)>> {
    use syntax::ast::TtToken;
    use syntax::parse::token as tok;

    macro_rules! comma {
        () => {
            ::syntax::ast::TtToken(_, ::syntax::parse::token::Comma)
        }
    }

    macro_rules! colon {
        () => {
            ::syntax::ast::TtToken(_, ::syntax::parse::token::Colon)
        };
        ($sp:ident) => {
            ::syntax::ast::TtToken($sp, ::syntax::parse::token::Colon)
        }
    }

    let mut items = Vec::with_capacity(tts.len() / 4);
    if tts.len() == 0 {
        return Some(items);
    }

    // horrible
    for entry in tts.chunks(4) {
        let item = match entry {
            // "foo": VALUE | "foo": VALUE,
            [TtToken(_, tok::LitStr(ref n)), colon!(), ref v] |
            [TtToken(_, tok::LitStr(ref n)), colon!(), ref v, comma!()] => {
                let k = n.as_str();
                let v = tt_to_expr(cx, v);
                if v.is_none() {
                    return None;
                }
                let k = quote_expr!(cx, $k.to_string());
                let v = quote_expr!(cx, $v);
                (k, v)
            }
            // "foo": VALUE X
            [TtToken(_, tok::LitStr(_)), colon!(), _, ref tt] => {
                expected_but_found(cx, sp, "`,`", tt);
                return None;
            }
            [TtToken(_, tok::LitStr(_)), colon!(sp)] => {
                cx.span_err(sp, "found `:` but no value afterwards");
                return None;
            }
            [TtToken(_, tok::LitStr(_)), ref tt, ..] => {
                expected_but_found(cx, sp, "`:`", tt);
                return None;
            }
            [TtToken(sp, tok::LitStr(_))] => {
                cx.span_err(sp, "found name but no colon-value afterwards");
                return None;
            }
            [ref tt, ..] => {
                expected_but_found(cx, sp, "string literal", tt);
                return None;
            }
            [] => unreachable!(), // chunks() never returns an empty slice
        };
        items.push(item);
    }

    Some(items)
}

fn token_to_expr(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> Option<PExpr> {
    use syntax::print::pprust;

    match *tok {
        token::LitStr(ref n) => {
            let s = n.as_str();
            Some(quote_expr!(cx, {
                ::serialize::json::String($s.into_string())
            }))
        }
        // FIXME: handle suffixed literals (i.e. u64) correctly
        // FIXME: handle negative numbers
        token::LitInteger(_) => {
            let tt = ast::TtToken(sp, tok.clone());
            Some(quote_expr!(cx, {
                ::serialize::json::I64($tt as i64)
            }))
        }
        token::LitFloat(_) => {
            let tt = ast::TtToken(sp, tok.clone());
            Some(quote_expr!(cx, {
                ::serialize::json::F64($tt)
            }))
        }
        token::Ident(ref id, token::Plain) if id.as_str() == "null" => {
            Some(quote_expr!(cx, {
                ::serialize::json::Null
            }))
        }
        ref t @ token::Ident(..) if t.is_keyword(token::keywords::True) => {
            Some(quote_expr!(cx, { ::serialize::json::Boolean(true) }))
        }
        ref t @ token::Ident(..) if t.is_keyword(token::keywords::False) => {
            Some(quote_expr!(cx, { ::serialize::json::Boolean(false) }))
        }
        _ => {
            let tt = ast::TtToken(sp, tok.clone());
            let s = pprust::tt_to_string(&tt);
            cx.span_err(sp, format!("unexpected `{}` in JSON", s).as_slice());
            None
        }
    }
}

fn best_span(sp: Span, tt: &TokenTree) -> Span {
    let sp = match *tt {
        ast::TtToken(tok_sp, _) => tok_sp,
        _ => sp // the span passed into the function!
    };
    sp
}

fn expected_but_found(cx: &ExtCtxt, sp: Span, expected: &str, found: &TokenTree) {
    let pp = pprust::tt_to_string(found);
    let err = format!("expected {} but found: `{}`", expected, pp);
    cx.span_err(best_span(sp, found), err.as_slice());
}
<commit_msg>Use `into_string` for keys<commit_after>#![crate_name="json_macros"]
#![crate_type="dylib"]
#![feature(phase, plugin_registrar, macro_rules, quote)]

#[phase(plugin, link)] extern crate log;
extern crate rustc;
extern crate syntax;
extern crate serialize;

use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast;
use syntax::ext::base::{ExtCtxt, MacResult, MacExpr, DummyResult};
use syntax::parse::token;
use syntax::print::pprust;
use rustc::plugin::Registry;

type PExpr = P<ast::Expr>;

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("json", expand);
}

fn expand<'cx>(cx: &'cx mut ExtCtxt, sp: Span,
               tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    debug!("JSON token tree {}", tts);

    let tt = match tts.get(0) {
        Some(tt) => tt,
        None => {
            cx.span_err(sp, "expected JSON literal");
            return DummyResult::expr(sp);
        }
    };

    let expr = match tt_to_expr(cx, tt) {
        Some(e) => e,
        None => return DummyResult::expr(sp)
    };
    MacExpr::new(expr)
}

fn tt_to_expr(cx: &ExtCtxt, tt: &TokenTree) -> Option<PExpr> {
    use syntax::ext::build::AstBuilder;

    match *tt {
        ast::TtToken(sp, ref tok) => token_to_expr(cx, sp, tok),
        ast::TtDelimited(sp, ref d) => {
            let ref toks = d.tts;
            match d.delim {
                // array
                token::Bracket => {
                    let exprs = match parse_array(cx, sp, toks.as_slice()) {
                        Some(e) => e,
                        None => return None
                    };
                    let exprs = cx.expr_vec(sp, exprs);
                    Some(quote_expr!(cx, {
                        {
                            use std::slice::BoxedSlicePrelude;
                            let xs: ::std::boxed::Box<[_]> = box $exprs;
                            ::serialize::json::List(xs.into_vec())
                        }
                    }))
                }
                // object
                token::Brace => {
                    let items = match parse_object(cx, sp, toks.as_slice()) {
                        Some(i) => i,
                        None => return None
                    };
                    let ob = quote_expr!(cx, _ob);
                    let mut stmts = vec![];
                    for &(ref key, ref value) in items.iter() {
                        stmts.push(quote_stmt!(cx, $ob.insert($key, $value)));
                    }

                    Some(quote_expr!(cx, {
                        {
                            let mut $ob = ::std::collections::TreeMap::new();
                            $stmts;
                            ::serialize::json::Object($ob)
                        }
                    }))
                }
                token::Paren => {
                    let mut parser = cx.new_parser_from_tts(toks.as_slice());
                    let expr = parser.parse_expr();

                    Some(quote_expr!(cx, {
                        use serialize::json::ToJson;
                        ($expr).to_json()
                    }))
                }
            }
        }
        ast::TtSequence(sp, _) => {
            cx.span_err(sp, "`json!` unexpected TtSequence");
            None
        }
    }
}

fn parse_array(cx: &ExtCtxt, sp: Span, tts: &[TokenTree]) -> Option<Vec<PExpr>> {
    let mut exprs = Vec::with_capacity(tts.len() / 2);
    for (i, tt) in tts.iter().enumerate() {
        if i % 2 == 1 {
            match tt {
                &ast::TtToken(_, token::Comma) => {
                    continue;
                }
                _ => {
                    expected_but_found(cx, sp, "`,`", tt);
                    return None;
                }
            }
        }
        let expr = tt_to_expr(cx, tt);
        let expr = match expr {
            Some(e) => e,
            None => return None,
        };
        exprs.push(expr);
    }
    Some(exprs)
}

fn parse_object(cx: &ExtCtxt, sp: Span, tts: &[TokenTree]) -> Option<Vec<(PExpr, PExpr)>> {
    use syntax::ast::TtToken;
    use syntax::parse::token as tok;

    macro_rules! comma {
        () => {
            ::syntax::ast::TtToken(_, ::syntax::parse::token::Comma)
        }
    }

    macro_rules! colon {
        () => {
            ::syntax::ast::TtToken(_, ::syntax::parse::token::Colon)
        };
        ($sp:ident) => {
            ::syntax::ast::TtToken($sp, ::syntax::parse::token::Colon)
        }
    }

    let mut items = Vec::with_capacity(tts.len() / 4);
    if tts.len() == 0 {
        return Some(items);
    }

    // horrible
    for entry in tts.chunks(4) {
        let item = match entry {
            // "foo": VALUE | "foo": VALUE,
            [TtToken(_, tok::LitStr(ref n)), colon!(), ref v] |
            [TtToken(_, tok::LitStr(ref n)), colon!(), ref v, comma!()] => {
                let k = n.as_str();
                let v = tt_to_expr(cx, v);
                if v.is_none() {
                    return None;
                }
                let k = quote_expr!(cx, $k.into_string());
                let v = quote_expr!(cx, $v);
                (k, v)
            }
            // "foo": VALUE X
            [TtToken(_, tok::LitStr(_)), colon!(), _, ref tt] => {
                expected_but_found(cx, sp, "`,`", tt);
                return None;
            }
            [TtToken(_, tok::LitStr(_)), colon!(sp)] => {
                cx.span_err(sp, "found `:` but no value afterwards");
                return None;
            }
            [TtToken(_, tok::LitStr(_)), ref tt, ..] => {
                expected_but_found(cx, sp, "`:`", tt);
                return None;
            }
            [TtToken(sp, tok::LitStr(_))] => {
                cx.span_err(sp, "found name but no colon-value afterwards");
                return None;
            }
            [ref tt, ..] => {
                expected_but_found(cx, sp, "string literal", tt);
                return None;
            }
            [] => unreachable!(), // chunks() never returns an empty slice
        };
        items.push(item);
    }

    Some(items)
}

fn token_to_expr(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> Option<PExpr> {
    use syntax::print::pprust;

    match *tok {
        token::LitStr(ref n) => {
            let s = n.as_str();
            Some(quote_expr!(cx, {
                ::serialize::json::String($s.into_string())
            }))
        }
        // FIXME: handle suffixed literals (i.e. u64) correctly
        // FIXME: handle negative numbers
        token::LitInteger(_) => {
            let tt = ast::TtToken(sp, tok.clone());
            Some(quote_expr!(cx, {
                ::serialize::json::I64($tt as i64)
            }))
        }
        token::LitFloat(_) => {
            let tt = ast::TtToken(sp, tok.clone());
            Some(quote_expr!(cx, {
                ::serialize::json::F64($tt)
            }))
        }
        token::Ident(ref id, token::Plain) if id.as_str() == "null" => {
            Some(quote_expr!(cx, {
                ::serialize::json::Null
            }))
        }
        ref t @ token::Ident(..) if t.is_keyword(token::keywords::True) => {
            Some(quote_expr!(cx, { ::serialize::json::Boolean(true) }))
        }
        ref t @ token::Ident(..) if t.is_keyword(token::keywords::False) => {
            Some(quote_expr!(cx, { ::serialize::json::Boolean(false) }))
        }
        _ => {
            let tt = ast::TtToken(sp, tok.clone());
            let s = pprust::tt_to_string(&tt);
            cx.span_err(sp, format!("unexpected `{}` in JSON", s).as_slice());
            None
        }
    }
}

fn best_span(sp: Span, tt: &TokenTree) -> Span {
    let sp = match *tt {
        ast::TtToken(tok_sp, _) => tok_sp,
        _ => sp // the span passed into the function!
    };
    sp
}

fn expected_but_found(cx: &ExtCtxt, sp: Span, expected: &str, found: &TokenTree) {
    let pp = pprust::tt_to_string(found);
    let err = format!("expected {} but found: `{}`", expected, pp);
    cx.span_err(best_span(sp, found), err.as_slice());
}
<|endoftext|>


<commit_before>use ::{Guard, Nav};
use ::util::{ChildIndex, SiblingIndex};

use std::clone::Clone;
use std::iter::Iterator;

/// Single-ownership trees wherein a parent owns its children.
///
/// This tree structure keeps its children in a heap-allocated array, so
/// appending children is a cheap operation. References into the tree cannot be
/// retained when modifying it, however, and subtrees cannot be shared between
/// parents.
pub struct Tree<T> {
    data: T, children: Vec<Tree<T>>,
}

impl<T> Tree<T> {
    pub fn new(data: T, children: Vec<Tree<T>>) -> Self {
        Tree { data: data, children: children, }
    }

    pub fn leaf(data: T) -> Self {
        Tree { data: data, children: Vec::new(), }
    }

    // TODO: loop instead of recurring to avoid blowing the stack.
    pub fn from_traversal<I: Iterator<Item=(T, I)>>(data: T, children: I) -> Self {
        let mut t = Tree { data: data, children: vec![], };
        for (child_data, child_children) in children {
            t.children.push(Tree::from_traversal(child_data, child_children));
        }
        t.children.shrink_to_fit();
        return t;
    }

    pub fn nav<'s>(&'s self) -> Navigator<'s, T> {
        Navigator::new(self)
    }
}

pub struct DataGuard<'a, T: 'a> {
    tree: &'a Tree<T>,
}

impl<'a, T: 'a> Guard<'a, T> for DataGuard<'a, T> {
    fn super_deref<'s>(&'s self) -> &'a T {
        &self.tree.data
    }
}

pub struct Navigator<'a, T: 'a> {
    here: &'a Tree<T>,
    path: Vec<(&'a Tree<T>, usize)>,
}

impl<'a, T: 'a> Navigator<'a, T> {
    fn new(tree: &'a Tree<T>) -> Self {
        Navigator { here: tree, path: Vec::new(), }
    }
}

impl<'a, T: 'a> Clone for Navigator<'a, T> {
    fn clone(&self) -> Self {
        Navigator { here: self.here, path: self.path.clone(), }
    }
}

impl<'a, T: 'a> Nav<'a> for Navigator<'a, T> {
    type Data = T;
    type DataGuard = DataGuard<'a, T>;

    fn seek_sibling(&mut self, offset: isize) {
        let new_index = {
            if self.at_root() {
                SiblingIndex::Root
            } else {
                let (parent, here_index) = self.path[self.path.len() - 1];
                SiblingIndex::compute(parent.children.len(),
                                      here_index,
                                      offset)
            }
        }.unwrap();
        let (parent, _) = self.path.pop().unwrap();
        self.path.push((parent, new_index));
        self.here = &parent.children[new_index];
    }

    fn seek_child(&mut self, index: usize) {
        let new_index =
            ChildIndex::compute(self.here.children.len(), index).unwrap();
        self.path.push((self.here, new_index));
        self.here = &self.here.children[new_index];
    }

    fn child_count(&self) -> usize {
        self.here.children.len()
    }

    fn at_root(&self) -> bool {
        self.path.is_empty()
    }

    fn to_parent(&mut self) {
        let (parent, _) = self.path.pop().expect("already at root");
        self.here = parent;
    }

    fn to_root(&mut self) {
        if ! self.at_root() {
            let (parent, _) = self.path[0];
            self.here = parent;
            self.path.clear();
        }
    }

    fn data(&self) -> DataGuard<'a, T> {
        DataGuard { tree: self.here, }
    }
}
<commit_msg>Add some tree modification methods to owned trees.<commit_after>use ::{Guard, Nav};
use ::util::{ChildIndex, SiblingIndex};

use std::clone::Clone;
use std::iter::Iterator;

/// Single-ownership trees wherein a parent owns its children.
///
/// This tree structure keeps its children in a heap-allocated array, so
/// appending children is a cheap operation. References into the tree cannot be
/// retained when modifying it, however, and subtrees cannot be shared between
/// parents.
pub struct Tree<T> {
    data: T, children: Vec<Tree<T>>,
}

impl<T> Tree<T> {
    pub fn new(data: T, children: Vec<Tree<T>>) -> Self {
        Tree { data: data, children: children, }
    }

    pub fn leaf(data: T) -> Self {
        Tree { data: data, children: Vec::new(), }
    }

    pub fn push_child(&mut self, child: Tree<T>) {
        self.children.push(child);
    }

    pub fn remove_child(&mut self, index: usize) {
        assert![index < self.children.len(),
                "cannot remove child at index {} (only {} children)", index, self.children.len()];
        self.children.remove(index);
    }

    pub fn insert_child(&mut self, index: usize, child: Tree<T>) {
        self.children.insert(index, child);
    }

    // TODO: loop instead of recurring to avoid blowing the stack.
    pub fn from_traversal<I: Iterator<Item=(T, I)>>(data: T, children: I) -> Self {
        let mut t = Tree { data: data, children: vec![], };
        for (child_data, child_children) in children {
            t.children.push(Tree::from_traversal(child_data, child_children));
        }
        t.children.shrink_to_fit();
        return t;
    }

    pub fn nav<'s>(&'s self) -> Navigator<'s, T> {
        Navigator::new(self)
    }
}

pub struct DataGuard<'a, T: 'a> {
    tree: &'a Tree<T>,
}

impl<'a, T: 'a> Guard<'a, T> for DataGuard<'a, T> {
    fn super_deref<'s>(&'s self) -> &'a T {
        &self.tree.data
    }
}

pub struct Navigator<'a, T: 'a> {
    here: &'a Tree<T>,
    path: Vec<(&'a Tree<T>, usize)>,
}

impl<'a, T: 'a> Navigator<'a, T> {
    fn new(tree: &'a Tree<T>) -> Self {
        Navigator { here: tree, path: Vec::new(), }
    }
}

impl<'a, T: 'a> Clone for Navigator<'a, T> {
    fn clone(&self) -> Self {
        Navigator { here: self.here, path: self.path.clone(), }
    }
}

impl<'a, T: 'a> Nav<'a> for Navigator<'a, T> {
    type Data = T;
    type DataGuard = DataGuard<'a, T>;

    fn seek_sibling(&mut self, offset: isize) {
        let new_index = {
            if self.at_root() {
                SiblingIndex::Root
            } else {
                let (parent, here_index) = self.path[self.path.len() - 1];
                SiblingIndex::compute(parent.children.len(),
                                      here_index,
                                      offset)
            }
        }.unwrap();
        let (parent, _) = self.path.pop().unwrap();
        self.path.push((parent, new_index));
        self.here = &parent.children[new_index];
    }

    fn seek_child(&mut self, index: usize) {
        let new_index =
            ChildIndex::compute(self.here.children.len(), index).unwrap();
        self.path.push((self.here, new_index));
        self.here = &self.here.children[new_index];
    }

    fn child_count(&self) -> usize {
        self.here.children.len()
    }

    fn at_root(&self) -> bool {
        self.path.is_empty()
    }

    fn to_parent(&mut self) {
        let (parent, _) = self.path.pop().expect("already at root");
        self.here = parent;
    }

    fn to_root(&mut self) {
        if ! self.at_root() {
            let (parent, _) = self.path[0];
            self.here = parent;
            self.path.clear();
        }
    }

    fn data(&self) -> DataGuard<'a, T> {
        DataGuard { tree: self.here, }
    }
}
<|endoftext|>


<commit_before>
use regex::Regex;

use std::collections::VecDeque;
use std::fmt;

use eval::FuncError;
use value::Value;

#[derive(Debug, Clone)]
pub enum ParseError {
    UnclosedList,
    InvalidListDelimitter,
    UnreadableSourceCode,
}

#[derive(Debug, Clone)]
pub struct FilePos {
    line: usize,
    col: usize,
}

impl FilePos {
    fn from_offset(text: &str, mut offset: usize) -> FilePos {
        let mut line_no = 1;

        for (cur_offset, ch) in text.bytes().enumerate() {
            if cur_offset >= offset {
                break;
            }

            if ch == '\n' as u8 {
                line_no += 1;
            }
        }

        FilePos {
            line: line_no,
            col: 0,
        }
    }
}

pub type TokenizeResult = Result<Vec<Value>, ParseError>; // result of tokenzing a collection of tokens
pub type ParseResult = Result<Value, ParseError>; // result of tokenizing a single token

const LIST_OPEN: &'static str = "(";
const LIST_CLOSE: &'static str = ")";

const QUOTE_OPEN: &'static str = "{";
const QUOTE_CLOSE: &'static str = "}";

pub fn write_list<T>(fmt: &mut fmt::Formatter, list: &Vec<T>, start: &str, end: &str) -> fmt::Result where T: fmt::Debug {
    write!(fmt, "{}{:?}", start, list[0]);

    for token in &list[1..] {
        write!(fmt, " {:?}", token);
    }

    write!(fmt, "{}", end)
}

fn preprocess(code: &str) -> VecDeque<String> {
    let string_re = r#""[^"]*""#;
    let sym_re = r"[-!?#\w]+";
    let num_re = r"\d+\.?\d*e?\d*";
    let list_re = r"[(){}\[\]]";
    let op_re = r"\+|-|\*|/|\^|&|\||=";
	let quote_re = r"'";

    let regex = format!("{}|{}|{}|{}|{}|{}", string_re, num_re, sym_re, list_re, op_re, quote_re);

    let re = match Regex::new(&regex) {
        Ok(re) => re,
        Err(_) => return VecDeque::new(),
    };

    let spaced_code = code.to_string()
        .replace("(", " ( ")
        .replace(")", " ) ");

    let mut token_strs = VecDeque::new();

    for cap in re.captures_iter(&spaced_code) {
        let (offset, _) = cap.pos(0).unwrap_or((0, 0));
        let match_str = cap.at(0).unwrap_or("");

        token_strs.push_back(match_str.to_string());
    }

    return token_strs;
}

pub fn tokenize_str(code: &str) -> TokenizeResult {
    let mut tokens = Vec::new();

    let mut seperated_code = preprocess(code);

    while seperated_code.len() > 0 {
        tokens.push(try!(tokenize(&mut seperated_code)));
    }

    Ok(tokens)
}

/// Extracts a single token out of a list of strings, which may contain multiple tokens
fn tokenize(list: &mut VecDeque<String>) -> ParseResult {
    let token_str = match list.pop_front() {
        Some(string) => string,
        None => return Err(ParseError::UnreadableSourceCode),
    };

    match &token_str[..] {
        LIST_OPEN => {
            let tokens = try!(tokenize_list(list, LIST_CLOSE));
            Ok(Value::List(tokens))
        },
        QUOTE_OPEN => {
            let tokens = try!(tokenize_list(list, QUOTE_CLOSE));
            Ok(Value::Quote(box Value::List(tokens)))
        },
        LIST_CLOSE | QUOTE_CLOSE => Err(ParseError::InvalidListDelimitter),
		r"'" => {
			let token = try!(tokenize(list));
			Ok(Value::Quote(box token))
		},
        atom => Ok(tokenize_atom(atom.to_string())),
    }
}

fn tokenize_atom(atom: String) -> Value {
    if let Ok(n) = atom.parse() {
        Value::Number(n)
    } else if let Some(lit) = string_lit(&atom[..]) {
        Value::Str(lit)
    } else {
        Value::Symbol(atom.to_string())
    }
}

fn tokenize_list(list: &mut VecDeque<String>, delimit: &str) -> Result<Vec<Value>, ParseError> {
    let mut tokens = Vec::new();

    while let Some(item_str) = list.pop_front() {
        if &item_str[..] == delimit {
            return Ok(tokens);
        }

        // If not the end of the list, push the item back and continue to process the list
        list.push_front(item_str);
        tokens.push(try!(tokenize(list)));
    }

    Err(ParseError::UnclosedList)
}

fn string_lit(slice: &str) -> Option<String> {
    let len = slice.len();
    let end = len - 1;

    if &slice[0..1] == "\"" && &slice[end .. len] == "\"" {
        let lit = &slice[1..end];

        let lit_str = lit.to_string()
            .replace(r"\n", "\n")
            .replace(r"\t", "\t")
            .replace(r"\r", "\r");

        return Some(lit_str);
    }

    return None;
}
<commit_msg>Fix out of bounds error when printing empty lists<commit_after>
use regex::Regex;

use std::collections::VecDeque;
use std::fmt;

use eval::FuncError;
use value::Value;

#[derive(Debug, Clone)]
pub enum ParseError {
    UnclosedList,
    InvalidListDelimitter,
    UnreadableSourceCode,
}

#[derive(Debug, Clone)]
pub struct FilePos {
    line: usize,
    col: usize,
}

impl FilePos {
    fn from_offset(text: &str, mut offset: usize) -> FilePos {
        let mut line_no = 1;

        for (cur_offset, ch) in text.bytes().enumerate() {
            if cur_offset >= offset {
                break;
            }

            if ch == '\n' as u8 {
                line_no += 1;
            }
        }

        FilePos {
            line: line_no,
            col: 0,
        }
    }
}

pub type TokenizeResult = Result<Vec<Value>, ParseError>; // result of tokenzing a collection of tokens
pub type ParseResult = Result<Value, ParseError>; // result of tokenizing a single token

const LIST_OPEN: &'static str = "(";
const LIST_CLOSE: &'static str = ")";

const QUOTE_OPEN: &'static str = "{";
const QUOTE_CLOSE: &'static str = "}";

pub fn write_list<T>(fmt: &mut fmt::Formatter, list: &Vec<T>, start: &str, end: &str) -> fmt::Result where T: fmt::Debug {
    if list.len() == 0 {
        return write!(fmt, "{}{}", start, end);
    }

    write!(fmt, "{}{:?}", start, list[0]);

    for token in &list[1..] {
        write!(fmt, " {:?}", token);
    }

    write!(fmt, "{}", end)
}

fn preprocess(code: &str) -> VecDeque<String> {
    let string_re = r#""[^"]*""#;
    let sym_re = r"[-!?#\w]+";
    let num_re = r"\d+\.?\d*e?\d*";
    let list_re = r"[(){}\[\]]";
    let op_re = r"\+|-|\*|/|\^|&|\||=";
	let quote_re = r"'";

    let regex = format!("{}|{}|{}|{}|{}|{}", string_re, num_re, sym_re, list_re, op_re, quote_re);

    let re = match Regex::new(&regex) {
        Ok(re) => re,
        Err(_) => return VecDeque::new(),
    };

    let spaced_code = code.to_string()
        .replace("(", " ( ")
        .replace(")", " ) ");

    let mut token_strs = VecDeque::new();

    for cap in re.captures_iter(&spaced_code) {
        let (offset, _) = cap.pos(0).unwrap_or((0, 0));
        let match_str = cap.at(0).unwrap_or("");

        token_strs.push_back(match_str.to_string());
    }

    return token_strs;
}

pub fn tokenize_str(code: &str) -> TokenizeResult {
    let mut tokens = Vec::new();

    let mut seperated_code = preprocess(code);

    while seperated_code.len() > 0 {
        tokens.push(try!(tokenize(&mut seperated_code)));
    }

    Ok(tokens)
}

/// Extracts a single token out of a list of strings, which may contain multiple tokens
fn tokenize(list: &mut VecDeque<String>) -> ParseResult {
    let token_str = match list.pop_front() {
        Some(string) => string,
        None => return Err(ParseError::UnreadableSourceCode),
    };

    match &token_str[..] {
        LIST_OPEN => {
            let tokens = try!(tokenize_list(list, LIST_CLOSE));
            Ok(Value::List(tokens))
        },
        QUOTE_OPEN => {
            let tokens = try!(tokenize_list(list, QUOTE_CLOSE));
            Ok(Value::Quote(box Value::List(tokens)))
        },
        LIST_CLOSE | QUOTE_CLOSE => Err(ParseError::InvalidListDelimitter),
		r"'" => {
			let token = try!(tokenize(list));
			Ok(Value::Quote(box token))
		},
        atom => Ok(tokenize_atom(atom.to_string())),
    }
}

fn tokenize_atom(atom: String) -> Value {
    if let Ok(n) = atom.parse() {
        Value::Number(n)
    } else if let Some(lit) = string_lit(&atom[..]) {
        Value::Str(lit)
    } else {
        Value::Symbol(atom.to_string())
    }
}

fn tokenize_list(list: &mut VecDeque<String>, delimit: &str) -> Result<Vec<Value>, ParseError> {
    let mut tokens = Vec::new();

    while let Some(item_str) = list.pop_front() {
        if &item_str[..] == delimit {
            return Ok(tokens);
        }

        // If not the end of the list, push the item back and continue to process the list
        list.push_front(item_str);
        tokens.push(try!(tokenize(list)));
    }

    Err(ParseError::UnclosedList)
}

fn string_lit(slice: &str) -> Option<String> {
    let len = slice.len();
    let end = len - 1;

    if &slice[0..1] == "\"" && &slice[end .. len] == "\"" {
        let lit = &slice[1..end];

        let lit_str = lit.to_string()
            .replace(r"\n", "\n")
            .replace(r"\t", "\t")
            .replace(r"\r", "\r");

        return Some(lit_str);
    }

    return None;
}
<|endoftext|>


<commit_before>
#[test]
fn test_tokenize() {
    assert tokenize( ~"(1 2 3)" ) == ~[~"(", ~"1", ~"2", ~"3", ~")"];
}

#[test]
fn test_tokenize_empty() {
    assert tokenize( ~"" ) == ~[];
}

#[test]
fn test_tokenize_no_spaces() {
    assert tokenize( ~"one" ) == ~[~"one"];
}

#[test]
fn test_pad_parentheses() {
    assert pad_parentheses( ~"(1 2 3)" ) == ~" ( 1 2 3 ) "
}

#[test]
fn test_pad_parentheses_empty() {
    assert pad_parentheses( ~"" ) == ~""
}

#[test]
fn test_pad_parentheses_one() {
    assert pad_parentheses( ~"(" ) == ~" ( ";
    assert pad_parentheses( ~")" ) == ~" ) ";
}

fn pad_parentheses( input:&str ) -> ~str {
    str::replace(str::replace(input, ")", " ) "), "(", " ( ")
}

fn tokenize( input:&str ) -> ~[~str] {
    str::words(pad_parentheses(input))
}

#[test]
fn test_that_atom_can_read_a_symbol() {
    match atom(~"hello") {
        Symbol(~"hello") => (),
            _ => fail
    }
}

#[test]
fn test_that_atom_can_read_an_int() {
    match atom(~"10") {
        Int(10) => (),
            _ => fail
    }
}

#[test]
fn test_that_atom_can_read_a_float() {
    match atom(~"10.1") {
        Float(10.1) => (),
            _ => fail
    }
}

fn atom( input:~str ) -> Expression {
    match int::from_str(input) {
        Some(number) => Int(number),
        None => match float::from_str(input) {
            Some(number) => Float(number),
            None => Symbol(input)
        }
    }
}

#[test]
fn test_that_read_can_read_an_atom() {
    let atom = read( ~[~"12"] );
    match atom {
        Int(12) => (),
            _ => fail
    }
}

#[test]
fn test_that_read_can_read_a_list() {
    let list = read( ~[ ~"(", ~"1", ~")" ] );
    match list {
        List([Int(1)]) => (),
            _ => fail ~"not a list"
    }
}

#[test]
fn test_that_read_can_read_a_nested_list() {
    let list = read( ~[ ~"(", ~"1", ~"(", ~"2", ~")", ~"3", ~")" ] );
    match list {
        List([Int(1), List([Int(2)]), Int(3)]) => (),
        _ => fail
    }
}

fn read( tokens:~[~str] ) -> Expression {
    fn subexpression( tokens:~[~str] ) -> (Expression, ~[~str]) {
        let mut remainder = copy tokens;
        let token = remainder.remove(0);
        match token {
            ~"(" => {
                let mut accumulator:~[Expression] = ~[];
                while remainder.len() > 0 && remainder[0] != ~")" {
                    let (expr, new_remainder) = subexpression( remainder );
                    accumulator.push(expr);
                    remainder = new_remainder
                }
                // remove the final close paren, this will fail if the parens
                // aren't closed properly due to an assert in remove
                remainder.remove(0);
                (List(accumulator), remainder)
            }
            ~")" => fail,
                _ => (atom(token), remainder)
        }
    }

    let (expression, _remainder) = subexpression( tokens );
    expression
}

pub fn parse( program:&str ) -> Expression {
    read( tokenize( program ) )
}
<commit_msg>Workaround for float::from_str accepting + and - as floats...<commit_after>
#[test]
fn test_tokenize() {
    assert tokenize( ~"(1 2 3)" ) == ~[~"(", ~"1", ~"2", ~"3", ~")"];
}

#[test]
fn test_tokenize_empty() {
    assert tokenize( ~"" ) == ~[];
}

#[test]
fn test_tokenize_no_spaces() {
    assert tokenize( ~"one" ) == ~[~"one"];
}

#[test]
fn test_pad_parentheses() {
    assert pad_parentheses( ~"(1 2 3)" ) == ~" ( 1 2 3 ) "
}

#[test]
fn test_pad_parentheses_empty() {
    assert pad_parentheses( ~"" ) == ~""
}

#[test]
fn test_pad_parentheses_one() {
    assert pad_parentheses( ~"(" ) == ~" ( ";
    assert pad_parentheses( ~")" ) == ~" ) ";
}

fn pad_parentheses( input:&str ) -> ~str {
    str::replace(str::replace(input, ")", " ) "), "(", " ( ")
}

fn tokenize( input:&str ) -> ~[~str] {
    str::words(pad_parentheses(input))
}

#[test]
fn test_that_atom_can_read_a_symbol() {
    match atom(~"hello") {
        Symbol(~"hello") => (),
            _ => fail
    }
}

#[test]
fn test_that_a_plus_sign_becomes_a_symbol() {
    match atom(~"+") {
        Symbol(~"+") => (),
        _ => fail fmt!("+ became: %s", atom(~"+").to_str())
    }
}

#[test]
fn test_that_a_minus_sign_becomes_a_symbol() {
    match atom(~"-") {
        Symbol(~"-") => (),
        _ => fail fmt!("- became: %s", atom(~"-").to_str())
    }
}

#[test]
fn test_that_atom_can_read_an_int() {
    match atom(~"10") {
        Int(10) => (),
            _ => fail
    }
}

#[test]
fn test_that_atom_can_read_a_float() {
    match atom(~"10.1") {
        Float(10.1) => (),
            _ => fail
    }
}

fn atom( input:~str ) -> Expression {
    match input {
        ~"+" => Symbol( ~"+" ),
        ~"-" => Symbol( ~"-" ),
        _ => match int::from_str(input) {
            Some(number) => Int(number),
            None => match float::from_str(input) {
                Some(number) => Float(number),
                None => Symbol(input)
            }
        }
    }
}

#[test]
fn test_that_read_can_read_an_atom() {
    let atom = read( ~[~"12"] );
    match atom {
        Int(12) => (),
            _ => fail
    }
}

#[test]
fn test_that_read_can_read_a_list() {
    let list = read( ~[ ~"(", ~"1", ~")" ] );
    match list {
        List([Int(1)]) => (),
            _ => fail ~"not a list"
    }
}

#[test]
fn test_that_read_can_read_a_nested_list() {
    let list = read( ~[ ~"(", ~"1", ~"(", ~"2", ~")", ~"3", ~")" ] );
    match list {
        List([Int(1), List([Int(2)]), Int(3)]) => (),
        _ => fail
    }
}

fn read( tokens:~[~str] ) -> Expression {
    fn subexpression( tokens:~[~str] ) -> (Expression, ~[~str]) {
        let mut remainder = copy tokens;
        let token = remainder.remove(0);
        match token {
            ~"(" => {
                let mut accumulator:~[Expression] = ~[];
                while remainder.len() > 0 && remainder[0] != ~")" {
                    let (expr, new_remainder) = subexpression( remainder );
                    accumulator.push(expr);
                    remainder = new_remainder
                }
                // remove the final close paren, this will fail if the parens
                // aren't closed properly due to an assert in remove
                remainder.remove(0);
                (List(accumulator), remainder)
            }
            ~")" => fail,
                _ => (atom(token), remainder)
        }
    }

    let (expression, _remainder) = subexpression( tokens );
    expression
}

pub fn parse( program:&str ) -> Expression {
    read( tokenize( program ) )
}
<|endoftext|>


<commit_before>//! Parallel iterator types for [ranges][std::range],
//! the type for values created by `a..b` expressions
//!
//! You will rarely need to interact with this module directly unless you have
//! need to name one of the iterator types.
//!
//! ```
//! use rayon::prelude::*;
//!
//! let r = (0..100u64).into_par_iter()
//!                    .sum();
//!
//! // compare result with sequential calculation
//! assert_eq!((0..100).sum::<u64>(), r);
//! ```
//!
//! [std::range]: https://doc.rust-lang.org/core/ops/struct.Range.html

use iter::*;
use iter::plumbing::*;
use std::ops::Range;
use std::usize;

/// Parallel iterator over a range, implemented for all integer types.
///
/// **Note:** The `zip` operation requires `IndexedParallelIterator`
/// which is not implemented for `u64`, `i64`, `u128`, or `i128`.
///
/// ```
/// use rayon::prelude::*;
///
/// let p = (0..25usize).into_par_iter()
///                   .zip(0..25usize)
///                   .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0)
///                   .map(|(x, y)| x * y)
///                   .sum::<usize>();
///
/// let s = (0..25usize).zip(0..25)
///                   .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0)
///                   .map(|(x, y)| x * y)
///                   .sum();
///
/// assert_eq!(p, s);
/// ```
#[derive(Debug, Clone)]
pub struct Iter<T> {
    range: Range<T>,
}

impl<T> IntoParallelIterator for Range<T>
    where Iter<T>: ParallelIterator
{
    type Item = <Iter<T> as ParallelIterator>::Item;
    type Iter = Iter<T>;

    fn into_par_iter(self) -> Self::Iter {
        Iter { range: self }
    }
}

struct IterProducer<T> {
    range: Range<T>,
}

impl<T> IntoIterator for IterProducer<T>
    where Range<T>: Iterator
{
    type Item = <Range<T> as Iterator>::Item;
    type IntoIter = Range<T>;

    fn into_iter(self) -> Self::IntoIter {
        self.range
    }
}

macro_rules! indexed_range_impl {
    ( $t:ty ) => {
        impl ParallelIterator for Iter<$t> {
            type Item = $t;

            fn drive_unindexed<C>(self, consumer: C) -> C::Result
                where C: UnindexedConsumer<Self::Item>
            {
                bridge(self, consumer)
            }

            fn opt_len(&self) -> Option<usize> {
                Some(self.len())
            }
        }

        impl IndexedParallelIterator for Iter<$t> {
            fn drive<C>(self, consumer: C) -> C::Result
                where C: Consumer<Self::Item>
            {
                bridge(self, consumer)
            }

            fn len(&self) -> usize {
                self.range.len()
            }

            fn with_producer<CB>(self, callback: CB) -> CB::Output
                where CB: ProducerCallback<Self::Item>
            {
                callback.callback(IterProducer { range: self.range })
            }
        }

        impl Producer for IterProducer<$t> {

            type Item = <Range<$t> as Iterator>::Item;
            type IntoIter = Range<$t>;
            fn into_iter(self) -> Self::IntoIter {
                self.range
            }

            fn split_at(self, index: usize) -> (Self, Self) {
                assert!(index <= self.range.len());
                // For signed $t, the length and requested index could be greater than $t::MAX, and
                // then `index as $t` could wrap to negative, so wrapping_add is necessary.
                let mid = self.range.start.wrapping_add(index as $t);
                let left = self.range.start .. mid;
                let right = mid .. self.range.end;
                (IterProducer { range: left }, IterProducer { range: right })
            }
        }
    }
}

trait UnindexedRangeLen<L> {
    fn len(&self) -> L;
}

macro_rules! unindexed_range_impl {
    ( $t:ty, $len_t:ty ) => {
        impl UnindexedRangeLen<$len_t> for Range<$t> {
            fn len(&self) -> $len_t {
                let &Range { start, end } = self;
                if end > start {
                    end.wrapping_sub(start) as $len_t
                } else {
                    0
                }
            }
        }

        impl ParallelIterator for Iter<$t> {
            type Item = $t;

            fn drive_unindexed<C>(self, consumer: C) -> C::Result
                where C: UnindexedConsumer<Self::Item>
            {
                if let Some(len) = self.opt_len() {
                    // Drive this in indexed mode for better `collect`.
                    (0..len).into_par_iter()
                        .map(|i| self.range.start.wrapping_add(i as $t))
                        .drive(consumer)
                } else {
                    bridge_unindexed(IterProducer { range: self.range }, consumer)
                }
            }

            fn opt_len(&self) -> Option<usize> {
                let len = self.range.len();
                if len <= usize::MAX as $len_t {
                    Some(len as usize)
                } else {
                    None
                }
            }
        }

        impl UnindexedProducer for IterProducer<$t> {
            type Item = $t;

            fn split(mut self) -> (Self, Option<Self>) {
                let index = self.range.len() / 2;
                if index > 0 {
                    let mid = self.range.start.wrapping_add(index as $t);
                    let right = mid .. self.range.end;
                    self.range.end = mid;
                    (self, Some(IterProducer { range: right }))
                } else {
                    (self, None)
                }
            }

            fn fold_with<F>(self, folder: F) -> F
                where F: Folder<Self::Item>
            {
                folder.consume_iter(self)
            }
        }
    }
}

// all Range<T> with ExactSizeIterator
indexed_range_impl!{u8}
indexed_range_impl!{u16}
indexed_range_impl!{u32}
indexed_range_impl!{usize}
indexed_range_impl!{i8}
indexed_range_impl!{i16}
indexed_range_impl!{i32}
indexed_range_impl!{isize}

// other Range<T> with just Iterator
unindexed_range_impl!{u64, u64}
unindexed_range_impl!{i64, u64}
#[cfg(has_i128)] unindexed_range_impl!{u128, u128}
#[cfg(has_i128)] unindexed_range_impl!{i128, u128}

#[test]
fn check_range_split_at_overflow() {
    // Note, this split index overflows i8!
    let producer = IterProducer { range: -100i8..100 };
    let (left, right) = producer.split_at(150);
    let r1: i32 = left.range.map(|i| i as i32).sum();
    let r2: i32 = right.range.map(|i| i as i32).sum();
    assert_eq!(r1 + r2, -100);
}

#[cfg(has_i128)]
#[test]
fn test_i128_len_doesnt_overflow() {
    use std::{i128, u128};

    // Using parse because some versions of rust don't allow long literals
    let octillion: i128 = "1000000000000000000000000000".parse().unwrap();
    let producer = IterProducer { range: 0..octillion };

    assert_eq!(octillion as u128, producer.range.len());
    assert_eq!(octillion as u128, (0..octillion).len());
    assert_eq!(2 * octillion as u128, (-octillion..octillion).len());

    assert_eq!(u128::MAX, (i128::MIN..i128::MAX).len());

}

#[test]
fn test_u64_opt_len() {
    use std::{u64, usize};
    assert_eq!(Some(100), (0..100u64).into_par_iter().opt_len());
    assert_eq!(Some(usize::MAX), (0..usize::MAX as u64).into_par_iter().opt_len());
    if (usize::MAX as u64) < u64::MAX {
        assert_eq!(None, (0..1 + usize::MAX as u64).into_par_iter().opt_len());
        assert_eq!(None, (0..u64::MAX).into_par_iter().opt_len());
    }
}

#[cfg(has_i128)]
#[test]
fn test_u128_opt_len() {
    use std::{u128, usize};
    assert_eq!(Some(100), (0..100u128).into_par_iter().opt_len());
    assert_eq!(Some(usize::MAX), (0..usize::MAX as u128).into_par_iter().opt_len());
    assert_eq!(None, (0..1 + usize::MAX as u128).into_par_iter().opt_len());
    assert_eq!(None, (0..u128::MAX).into_par_iter().opt_len());
}

// `usize as i64` can overflow, so make sure to wrap it appropriately
// when using the `opt_len` "indexed" mode.
#[test]
#[cfg(target_pointer_width = "64")]
fn test_usize_i64_overflow() {
    use std::i64;
    use ThreadPoolBuilder;

    let iter = (-2..i64::MAX).into_par_iter();
    assert_eq!(iter.opt_len(), Some(i64::MAX as usize + 2));

    // always run with multiple threads to split into, or this will take forever...
    let pool = ThreadPoolBuilder::new().num_threads(8).build().unwrap();
    pool.install(|| assert_eq!(iter.find_last(|_| true), Some(i64::MAX - 1)));
}
<commit_msg>Add a workaround for rust#52966<commit_after>//! Parallel iterator types for [ranges][std::range],
//! the type for values created by `a..b` expressions
//!
//! You will rarely need to interact with this module directly unless you have
//! need to name one of the iterator types.
//!
//! ```
//! use rayon::prelude::*;
//!
//! let r = (0..100u64).into_par_iter()
//!                    .sum();
//!
//! // compare result with sequential calculation
//! assert_eq!((0..100).sum::<u64>(), r);
//! ```
//!
//! [std::range]: https://doc.rust-lang.org/core/ops/struct.Range.html

use iter::*;
use iter::plumbing::*;
use std::ops::Range;
use std::usize;

/// Parallel iterator over a range, implemented for all integer types.
///
/// **Note:** The `zip` operation requires `IndexedParallelIterator`
/// which is not implemented for `u64`, `i64`, `u128`, or `i128`.
///
/// ```
/// use rayon::prelude::*;
///
/// let p = (0..25usize).into_par_iter()
///                   .zip(0..25usize)
///                   .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0)
///                   .map(|(x, y)| x * y)
///                   .sum::<usize>();
///
/// let s = (0..25usize).zip(0..25)
///                   .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0)
///                   .map(|(x, y)| x * y)
///                   .sum();
///
/// assert_eq!(p, s);
/// ```
#[derive(Debug, Clone)]
pub struct Iter<T> {
    range: Range<T>,
}

impl<T> IntoParallelIterator for Range<T>
    where Iter<T>: ParallelIterator
{
    type Item = <Iter<T> as ParallelIterator>::Item;
    type Iter = Iter<T>;

    fn into_par_iter(self) -> Self::Iter {
        Iter { range: self }
    }
}

struct IterProducer<T> {
    range: Range<T>,
}

impl<T> IntoIterator for IterProducer<T>
    where Range<T>: Iterator
{
    type Item = <Range<T> as Iterator>::Item;
    type IntoIter = Range<T>;

    fn into_iter(self) -> Self::IntoIter {
        self.range
    }
}

macro_rules! indexed_range_impl {
    ( $t:ty ) => {
        impl ParallelIterator for Iter<$t> {
            type Item = $t;

            fn drive_unindexed<C>(self, consumer: C) -> C::Result
                where C: UnindexedConsumer<Self::Item>
            {
                bridge(self, consumer)
            }

            fn opt_len(&self) -> Option<usize> {
                Some(self.len())
            }
        }

        impl IndexedParallelIterator for Iter<$t> {
            fn drive<C>(self, consumer: C) -> C::Result
                where C: Consumer<Self::Item>
            {
                bridge(self, consumer)
            }

            fn len(&self) -> usize {
                self.range.len()
            }

            fn with_producer<CB>(self, callback: CB) -> CB::Output
                where CB: ProducerCallback<Self::Item>
            {
                callback.callback(IterProducer { range: self.range })
            }
        }

        impl Producer for IterProducer<$t> {

            type Item = <Range<$t> as Iterator>::Item;
            type IntoIter = Range<$t>;
            fn into_iter(self) -> Self::IntoIter {
                self.range
            }

            fn split_at(self, index: usize) -> (Self, Self) {
                assert!(index <= self.range.len());
                // For signed $t, the length and requested index could be greater than $t::MAX, and
                // then `index as $t` could wrap to negative, so wrapping_add is necessary.
                let mid = self.range.start.wrapping_add(index as $t);
                let left = self.range.start .. mid;
                let right = mid .. self.range.end;
                (IterProducer { range: left }, IterProducer { range: right })
            }
        }
    }
}

trait UnindexedRangeLen<L> {
    fn len(&self) -> L;
}

macro_rules! unindexed_range_impl {
    ( $t:ty, $len_t:ty ) => {
        impl UnindexedRangeLen<$len_t> for Range<$t> {
            fn len(&self) -> $len_t {
                let &Range { start, end } = self;
                if end > start {
                    end.wrapping_sub(start) as $len_t
                } else {
                    0
                }
            }
        }

        impl ParallelIterator for Iter<$t> {
            type Item = $t;

            fn drive_unindexed<C>(self, consumer: C) -> C::Result
                where C: UnindexedConsumer<Self::Item>
            {
                if let Some(len) = self.opt_len() {
                    // Drive this in indexed mode for better `collect`.
                    (0..len).into_par_iter()
                        .map(|i| self.range.start.wrapping_add(i as $t))
                        .drive(consumer)
                } else {
                    bridge_unindexed(IterProducer { range: self.range }, consumer)
                }
            }

            fn opt_len(&self) -> Option<usize> {
                let len = self.range.len();
                if len <= usize::MAX as $len_t {
                    Some(len as usize)
                } else {
                    None
                }
            }
        }

        impl UnindexedProducer for IterProducer<$t> {
            type Item = $t;

            fn split(mut self) -> (Self, Option<Self>) {
                let index = self.range.len() / 2;
                if index > 0 {
                    let mid = self.range.start.wrapping_add(index as $t);
                    let right = mid .. self.range.end;
                    self.range.end = mid;
                    (self, Some(IterProducer { range: right }))
                } else {
                    (self, None)
                }
            }

            fn fold_with<F>(self, folder: F) -> F
                where F: Folder<Self::Item>
            {
                folder.consume_iter(self)
            }
        }
    }
}

// all Range<T> with ExactSizeIterator
indexed_range_impl!{u8}
indexed_range_impl!{u16}
indexed_range_impl!{u32}
indexed_range_impl!{usize}
indexed_range_impl!{i8}
indexed_range_impl!{i16}
indexed_range_impl!{i32}
indexed_range_impl!{isize}

// other Range<T> with just Iterator
unindexed_range_impl!{u64, u64}
unindexed_range_impl!{i64, u64}
#[cfg(has_i128)] unindexed_range_impl!{u128, u128}
#[cfg(has_i128)] unindexed_range_impl!{i128, u128}

#[test]
fn check_range_split_at_overflow() {
    // Note, this split index overflows i8!
    let producer = IterProducer { range: -100i8..100 };
    let (left, right) = producer.split_at(150);
    let r1: i32 = left.range.map(|i| i as i32).sum();
    let r2: i32 = right.range.map(|i| i as i32).sum();
    assert_eq!(r1 + r2, -100);
}

#[cfg(has_i128)]
#[test]
fn test_i128_len_doesnt_overflow() {
    use std::{i128, u128};

    // Using parse because some versions of rust don't allow long literals
    let octillion: i128 = "1000000000000000000000000000".parse().unwrap();
    let producer = IterProducer { range: 0..octillion };

    assert_eq!(octillion as u128, producer.range.len());
    assert_eq!(octillion as u128, (0..octillion).len());
    assert_eq!(2 * octillion as u128, (-octillion..octillion).len());

    assert_eq!(u128::MAX, (i128::MIN..i128::MAX).len());

}

#[test]
fn test_u64_opt_len() {
    use std::{u64, usize};
    assert_eq!(Some(100), (0..100u64).into_par_iter().opt_len());
    assert_eq!(Some(usize::MAX), (0..usize::MAX as u64).into_par_iter().opt_len());
    if (usize::MAX as u64) < u64::MAX {
        assert_eq!(None, (0..(usize::MAX as u64).wrapping_add(1)).into_par_iter().opt_len());
        assert_eq!(None, (0..u64::MAX).into_par_iter().opt_len());
    }
}

#[cfg(has_i128)]
#[test]
fn test_u128_opt_len() {
    use std::{u128, usize};
    assert_eq!(Some(100), (0..100u128).into_par_iter().opt_len());
    assert_eq!(Some(usize::MAX), (0..usize::MAX as u128).into_par_iter().opt_len());
    assert_eq!(None, (0..1 + usize::MAX as u128).into_par_iter().opt_len());
    assert_eq!(None, (0..u128::MAX).into_par_iter().opt_len());
}

// `usize as i64` can overflow, so make sure to wrap it appropriately
// when using the `opt_len` "indexed" mode.
#[test]
#[cfg(target_pointer_width = "64")]
fn test_usize_i64_overflow() {
    use std::i64;
    use ThreadPoolBuilder;

    let iter = (-2..i64::MAX).into_par_iter();
    assert_eq!(iter.opt_len(), Some(i64::MAX as usize + 2));

    // always run with multiple threads to split into, or this will take forever...
    let pool = ThreadPoolBuilder::new().num_threads(8).build().unwrap();
    pool.install(|| assert_eq!(iter.find_last(|_| true), Some(i64::MAX - 1)));
}
<|endoftext|>


<commit_before>use std::io::Read;
use std::io::Write;

struct Status {
    low: u64,
    high: u64,
    range: u64,
    pending: u8,
    freq: [u64; 258],
    buf: [u8; 65536],
    len: usize
}

impl Status {
    fn init() -> Status {
        let mut res = Status {
            low: 0x00000000_00000000u64,
            high: 0xFFFFFFFF_FFFFFFFFu64,
            range: 0u64,
            pending: 0u8,
            freq: [0u64; 258],
            buf: [0u8; 65536],
            len: 0us
        };
        for i in 1..258 {
            res.freq[i] = i as u64;
        }
        return res;
    }
}

pub fn compress(input: &mut Read, output: &mut Write) {
    let st = Status::init();
}

<commit_msg>Started implementing buffered io<commit_after>use std::io::Read;
use std::io::Write;
use std::result::Result;

struct Buffer {
    data: [u8; 16384],
    len: usize,
    cur: usize,
    bit: u8
}

impl Buffer {
    fn create() -> Buffer {
        return Buffer { data: [0u8; 16384], len: 0us, cur: 0us, bit: 0u8 };
    }

    fn fill(&mut self, istream: &mut Read) -> bool {
        if self.len > 0 {
            true
        } else {
            match istream.read(&mut self.data) {
                Ok(0) => false,
                Ok(n) => { self.len = n; true },
                Err(_) => false
            }
        }
    }
}

struct Status {
    low: u64,
    high: u64,
    range: u64,
    pending: u8,
    freq: [u64; 258],
    input: Buffer,
    output: Buffer
}

impl Status {
    fn init() -> Status {
        let mut res = Status {
            low: 0x00000000_00000000u64,
            high: 0xFFFFFFFF_FFFFFFFFu64,
            range: 0u64,
            pending: 0u8,
            freq: [0u64; 258],
            input: Buffer::create(),
            output: Buffer::create()
        };
        for i in 1..258 {
            res.freq[i] = i as u64;
        }
        return res;
    }
}

pub fn compress(istream: &mut Read, ostream: &mut Write) {
    let mut st = Status::init();
    while st.input.fill(istream) {
        println!("length: {}", st.input.len);
        st.input.len = 0;
    }
}

<|endoftext|>


<commit_before>// Copyright 2018 Syn Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Parsing interface for parsing a token stream into a syntax tree node.
//!
//! Parsing in Syn is built on parser functions that take in a [`Cursor`] and
//! produce a [`PResult<T>`] where `T` is some syntax tree node. `Cursor` is a
//! cheaply copyable cursor over a range of tokens in a token stream, and
//! `PResult` is a result that packages together a parsed syntax tree node `T`
//! with a stream of remaining unparsed tokens after `T` represented as another
//! `Cursor`, or an [`Error`] if parsing failed.
//!
//! [`Cursor`]: ../buffer/index.html
//! [`PResult<T>`]: type.PResult.html
//! [`Error`]: struct.Error.html
//!
//! This `Cursor`- and `PResult`-based interface is convenient for parser
//! combinators and parser implementations, but not necessarily when you just
//! have some tokens that you want to parse. For that we expose the following
//! two entry points.
//!
//! ## The `syn::parse*` functions
//!
//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
//! as an entry point for parsing syntax tree nodes that can be parsed in an
//! obvious default way. These functions can return any syntax tree node that
//! implements the [`Synom`] trait, which includes most types in Syn.
//!
//! [`syn::parse`]: ../fn.parse.html
//! [`syn::parse2`]: ../fn.parse2.html
//! [`syn::parse_str`]: ../fn.parse_str.html
//! [`Synom`]: trait.Synom.html
//!
//! ```
//! use syn::Type;
//!
//! # fn run_parser() -> Result<(), syn::synom::Error> {
//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
//! #     Ok(())
//! # }
//! #
//! # fn main() {
//! #     run_parser().unwrap();
//! # }
//! ```
//!
//! The [`parse_quote!`] macro also uses this approach.
//!
//! [`parse_quote!`]: ../macro.parse_quote.html
//!
//! ## The `Parser` trait
//!
//! Some types can be parsed in several ways depending on context. For example
//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
//! may or may not allow trailing punctuation, and parsing it the wrong way
//! would either reject valid input or accept invalid input.
//!
//! [`Attribute`]: ../struct.Attribute.html
//! [`Punctuated`]: ../punctuated/index.html
//!
//! The `Synom` trait is not implemented in these cases because there is no good
//! behavior to consider the default.
//!
//! ```ignore
//! // Can't parse `Punctuated` without knowing whether trailing punctuation
//! // should be allowed in this context.
//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
//! ```
//!
//! In these cases the types provide a choice of parser functions rather than a
//! single `Synom` implementation, and those parser functions can be invoked
//! through the [`Parser`] trait.
//!
//! [`Parser`]: trait.Parser.html
//!
//! ```
//! # #[macro_use]
//! # extern crate syn;
//! #
//! # extern crate proc_macro2;
//! # use proc_macro2::TokenStream;
//! #
//! use syn::synom::Parser;
//! use syn::punctuated::Punctuated;
//! use syn::{PathSegment, Expr, Attribute};
//!
//! # fn run_parsers() -> Result<(), syn::synom::Error> {
//! #     let tokens = TokenStream::new().into();
//! // Parse a nonempty sequence of path segments separated by `::` punctuation
//! // with no trailing punctuation.
//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
//! let path = parser.parse(tokens)?;
//!
//! #     let tokens = TokenStream::new().into();
//! // Parse a possibly empty sequence of expressions terminated by commas with
//! // an optional trailing punctuation.
//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
//! let args = parser.parse(tokens)?;
//!
//! #     let tokens = TokenStream::new().into();
//! // Parse zero or more outer attributes but not inner attributes.
//! named!(outer_attrs -> Vec<Attribute>, many0!(Attribute::parse_outer));
//! let attrs = outer_attrs.parse(tokens)?;
//! #
//! #     Ok(())
//! # }
//! #
//! # fn main() {}
//! ```
//!
//! # Implementing a parser function
//!
//! Parser functions are usually implemented using the [`nom`]-style parser
//! combinator macros provided by Syn, but may also be implemented without
//! macros be using the low-level [`Cursor`] API directly.
//!
//! [`nom`]: https://github.com/Geal/nom
//!
//! The following parser combinator macros are available and a `Synom` parsing
//! example is provided for each one.
//!
//! - [`alt!`](../macro.alt.html)
//! - [`braces!`](../macro.braces.html)
//! - [`brackets!`](../macro.brackets.html)
//! - [`call!`](../macro.call.html)
//! - [`cond!`](../macro.cond.html)
//! - [`cond_reduce!`](../macro.cond_reduce.html)
//! - [`custom_keyword!`](../macro.custom_keyword.html)
//! - [`do_parse!`](../macro.do_parse.html)
//! - [`epsilon!`](../macro.epsilon.html)
//! - [`input_end!`](../macro.input_end.html)
//! - [`keyword!`](../macro.keyword.html)
//! - [`many0!`](../macro.many0.html)
//! - [`map!`](../macro.map.html)
//! - [`not!`](../macro.not.html)
//! - [`option!`](../macro.option.html)
//! - [`parens!`](../macro.parens.html)
//! - [`punct!`](../macro.punct.html)
//! - [`reject!`](../macro.reject.html)
//! - [`switch!`](../macro.switch.html)
//! - [`syn!`](../macro.syn.html)
//! - [`tuple!`](../macro.tuple.html)
//! - [`value!`](../macro.value.html)
//!
//! *This module is available if Syn is built with the `"parsing"` feature.*

#[cfg(all(
    not(all(target_arch = "wasm32", target_os = "unknown")),
    feature = "proc-macro"
))]
use proc_macro;
use proc_macro2::{Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};

use error::parse_error;
pub use error::{Error, PResult};

use buffer::{Cursor, TokenBuffer};
use parse;

/// Parsing interface implemented by all types that can be parsed in a default
/// way from a token stream.
///
/// Refer to the [module documentation] for details about parsing in Syn.
///
/// [module documentation]: index.html
///
/// *This trait is available if Syn is built with the `"parsing"` feature.*
pub trait Synom: Sized {
    fn parse(input: Cursor) -> PResult<Self>;

    /// A short name of the type being parsed.
    ///
    /// The description should only be used for a simple name.  It should not
    /// contain newlines or sentence-ending punctuation, to facilitate embedding in
    /// larger user-facing strings.  Syn will use this description when building
    /// error messages about parse failures.
    ///
    /// # Examples
    ///
    /// ```
    /// # use syn::buffer::Cursor;
    /// # use syn::synom::{Synom, PResult};
    /// #
    /// struct ExprMacro {
    ///     // ...
    /// }
    ///
    /// impl Synom for ExprMacro {
    /// #   fn parse(input: Cursor) -> PResult<Self> { unimplemented!() }
    ///     // fn parse(...) -> ... { ... }
    ///
    ///     fn description() -> Option<&'static str> {
    ///         // Will result in messages like
    ///         //
    ///         //     "failed to parse macro invocation expression: $reason"
    ///         Some("macro invocation expression")
    ///     }
    /// }
    /// ```
    fn description() -> Option<&'static str> {
        None
    }
}

impl<T> Synom for T
where
    T: parse::Parse,
{
    fn parse(input: Cursor) -> PResult<Self> {
        let state = parse::ParseBuffer::new(Span::call_site(), input);
        match <T as parse::Parse>::parse(&state) {
            Ok(node) => Ok((node, state.cursor())),
            Err(err) => Err(err),
        }
    }
}

impl Synom for TokenStream {
    fn parse(input: Cursor) -> PResult<Self> {
        Ok((input.token_stream(), Cursor::empty()))
    }

    fn description() -> Option<&'static str> {
        Some("arbitrary token stream")
    }
}

impl Synom for TokenTree {
    fn parse(input: Cursor) -> PResult<Self> {
        match input.token_tree() {
            Some((tt, rest)) => Ok((tt, rest)),
            None => parse_error(),
        }
    }

    fn description() -> Option<&'static str> {
        Some("token tree")
    }
}

impl Synom for Group {
    fn parse(input: Cursor) -> PResult<Self> {
        for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
            if let Some((inside, span, rest)) = input.group(*delim) {
                let mut group = Group::new(*delim, inside.token_stream());
                group.set_span(span);
                return Ok((group, rest));
            }
        }
        parse_error()
    }

    fn description() -> Option<&'static str> {
        Some("group token")
    }
}

impl Synom for Punct {
    fn parse(input: Cursor) -> PResult<Self> {
        match input.punct() {
            Some((punct, rest)) => Ok((punct, rest)),
            None => parse_error(),
        }
    }

    fn description() -> Option<&'static str> {
        Some("punctuation token")
    }
}

impl Synom for Literal {
    fn parse(input: Cursor) -> PResult<Self> {
        match input.literal() {
            Some((literal, rest)) => Ok((literal, rest)),
            None => parse_error(),
        }
    }

    fn description() -> Option<&'static str> {
        Some("literal token")
    }
}

/// Parser that can parse Rust tokens into a particular syntax tree node.
///
/// Refer to the [module documentation] for details about parsing in Syn.
///
/// [module documentation]: index.html
///
/// *This trait is available if Syn is built with the `"parsing"` feature.*
pub trait Parser: Sized {
    type Output;

    /// Parse a proc-macro2 token stream into the chosen syntax tree node.
    fn parse2(self, tokens: TokenStream) -> Result<Self::Output, Error>;

    /// Parse tokens of source code into the chosen syntax tree node.
    ///
    /// *This method is available if Syn is built with both the `"parsing"` and
    /// `"proc-macro"` features.*
    #[cfg(all(
        not(all(target_arch = "wasm32", target_os = "unknown")),
        feature = "proc-macro"
    ))]
    fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output, Error> {
        self.parse2(tokens.into())
    }

    /// Parse a string of Rust code into the chosen syntax tree node.
    ///
    /// # Hygiene
    ///
    /// Every span in the resulting syntax tree will be set to resolve at the
    /// macro call site.
    fn parse_str(self, s: &str) -> Result<Self::Output, Error> {
        match s.parse() {
            Ok(tts) => self.parse2(tts),
            Err(_) => Err(Error::new(
                Span::call_site(),
                "error while lexing input string",
            )),
        }
    }
}

impl<F, T> Parser for F
where
    F: FnOnce(Cursor) -> PResult<T>,
{
    type Output = T;

    fn parse2(self, tokens: TokenStream) -> Result<T, Error> {
        let buf = TokenBuffer::new2(tokens);
        let (t, rest) = self(buf.begin())?;
        if rest.eof() {
            Ok(t)
        } else if rest == buf.begin() {
            // parsed nothing
            Err(Error::new(Span::call_site(), "failed to parse anything"))
        } else {
            Err(Error::new(Span::call_site(), "failed to parse all tokens"))
        }
    }
}

/// Extension traits that are made available within the `call!` parser.
///
/// *This module is available if Syn is built with the `"parsing"` feature.*
pub mod ext {
    use super::*;
    use proc_macro2::Ident;

    /// Additional parsing methods for `Ident`.
    ///
    /// This trait is sealed and cannot be implemented for types outside of Syn.
    ///
    /// *This trait is available if Syn is built with the `"parsing"` feature.*
    pub trait IdentExt: Sized + private::Sealed {
        /// Parses any identifier including keywords.
        ///
        /// This is useful when parsing a DSL which allows Rust keywords as
        /// identifiers.
        ///
        /// ```rust
        /// #[macro_use]
        /// extern crate syn;
        ///
        /// use syn::Ident;
        ///
        /// // Parses input that looks like `name = NAME` where `NAME` can be
        /// // any identifier.
        /// //
        /// // Examples:
        /// //
        /// //     name = anything
        /// //     name = impl
        /// named!(parse_dsl -> Ident, do_parse!(
        ///     custom_keyword!(name) >>
        ///     punct!(=) >>
        ///     name: call!(Ident::parse_any) >>
        ///     (name)
        /// ));
        /// #
        /// # fn main() {}
        /// ```
        fn parse_any(input: Cursor) -> PResult<Self>;
    }

    impl IdentExt for Ident {
        fn parse_any(input: Cursor) -> PResult<Self> {
            input.ident().map_or_else(parse_error, Ok)
        }
    }

    mod private {
        use proc_macro2::Ident;

        pub trait Sealed {}

        impl Sealed for Ident {}
    }
}
<commit_msg>Simplify signature of Synom blanket impl<commit_after>// Copyright 2018 Syn Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Parsing interface for parsing a token stream into a syntax tree node.
//!
//! Parsing in Syn is built on parser functions that take in a [`Cursor`] and
//! produce a [`PResult<T>`] where `T` is some syntax tree node. `Cursor` is a
//! cheaply copyable cursor over a range of tokens in a token stream, and
//! `PResult` is a result that packages together a parsed syntax tree node `T`
//! with a stream of remaining unparsed tokens after `T` represented as another
//! `Cursor`, or an [`Error`] if parsing failed.
//!
//! [`Cursor`]: ../buffer/index.html
//! [`PResult<T>`]: type.PResult.html
//! [`Error`]: struct.Error.html
//!
//! This `Cursor`- and `PResult`-based interface is convenient for parser
//! combinators and parser implementations, but not necessarily when you just
//! have some tokens that you want to parse. For that we expose the following
//! two entry points.
//!
//! ## The `syn::parse*` functions
//!
//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
//! as an entry point for parsing syntax tree nodes that can be parsed in an
//! obvious default way. These functions can return any syntax tree node that
//! implements the [`Synom`] trait, which includes most types in Syn.
//!
//! [`syn::parse`]: ../fn.parse.html
//! [`syn::parse2`]: ../fn.parse2.html
//! [`syn::parse_str`]: ../fn.parse_str.html
//! [`Synom`]: trait.Synom.html
//!
//! ```
//! use syn::Type;
//!
//! # fn run_parser() -> Result<(), syn::synom::Error> {
//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
//! #     Ok(())
//! # }
//! #
//! # fn main() {
//! #     run_parser().unwrap();
//! # }
//! ```
//!
//! The [`parse_quote!`] macro also uses this approach.
//!
//! [`parse_quote!`]: ../macro.parse_quote.html
//!
//! ## The `Parser` trait
//!
//! Some types can be parsed in several ways depending on context. For example
//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
//! may or may not allow trailing punctuation, and parsing it the wrong way
//! would either reject valid input or accept invalid input.
//!
//! [`Attribute`]: ../struct.Attribute.html
//! [`Punctuated`]: ../punctuated/index.html
//!
//! The `Synom` trait is not implemented in these cases because there is no good
//! behavior to consider the default.
//!
//! ```ignore
//! // Can't parse `Punctuated` without knowing whether trailing punctuation
//! // should be allowed in this context.
//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
//! ```
//!
//! In these cases the types provide a choice of parser functions rather than a
//! single `Synom` implementation, and those parser functions can be invoked
//! through the [`Parser`] trait.
//!
//! [`Parser`]: trait.Parser.html
//!
//! ```
//! # #[macro_use]
//! # extern crate syn;
//! #
//! # extern crate proc_macro2;
//! # use proc_macro2::TokenStream;
//! #
//! use syn::synom::Parser;
//! use syn::punctuated::Punctuated;
//! use syn::{PathSegment, Expr, Attribute};
//!
//! # fn run_parsers() -> Result<(), syn::synom::Error> {
//! #     let tokens = TokenStream::new().into();
//! // Parse a nonempty sequence of path segments separated by `::` punctuation
//! // with no trailing punctuation.
//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
//! let path = parser.parse(tokens)?;
//!
//! #     let tokens = TokenStream::new().into();
//! // Parse a possibly empty sequence of expressions terminated by commas with
//! // an optional trailing punctuation.
//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
//! let args = parser.parse(tokens)?;
//!
//! #     let tokens = TokenStream::new().into();
//! // Parse zero or more outer attributes but not inner attributes.
//! named!(outer_attrs -> Vec<Attribute>, many0!(Attribute::parse_outer));
//! let attrs = outer_attrs.parse(tokens)?;
//! #
//! #     Ok(())
//! # }
//! #
//! # fn main() {}
//! ```
//!
//! # Implementing a parser function
//!
//! Parser functions are usually implemented using the [`nom`]-style parser
//! combinator macros provided by Syn, but may also be implemented without
//! macros be using the low-level [`Cursor`] API directly.
//!
//! [`nom`]: https://github.com/Geal/nom
//!
//! The following parser combinator macros are available and a `Synom` parsing
//! example is provided for each one.
//!
//! - [`alt!`](../macro.alt.html)
//! - [`braces!`](../macro.braces.html)
//! - [`brackets!`](../macro.brackets.html)
//! - [`call!`](../macro.call.html)
//! - [`cond!`](../macro.cond.html)
//! - [`cond_reduce!`](../macro.cond_reduce.html)
//! - [`custom_keyword!`](../macro.custom_keyword.html)
//! - [`do_parse!`](../macro.do_parse.html)
//! - [`epsilon!`](../macro.epsilon.html)
//! - [`input_end!`](../macro.input_end.html)
//! - [`keyword!`](../macro.keyword.html)
//! - [`many0!`](../macro.many0.html)
//! - [`map!`](../macro.map.html)
//! - [`not!`](../macro.not.html)
//! - [`option!`](../macro.option.html)
//! - [`parens!`](../macro.parens.html)
//! - [`punct!`](../macro.punct.html)
//! - [`reject!`](../macro.reject.html)
//! - [`switch!`](../macro.switch.html)
//! - [`syn!`](../macro.syn.html)
//! - [`tuple!`](../macro.tuple.html)
//! - [`value!`](../macro.value.html)
//!
//! *This module is available if Syn is built with the `"parsing"` feature.*

#[cfg(all(
    not(all(target_arch = "wasm32", target_os = "unknown")),
    feature = "proc-macro"
))]
use proc_macro;
use proc_macro2::{Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};

use error::parse_error;
pub use error::{Error, PResult};

use buffer::{Cursor, TokenBuffer};
use parse::{Parse, ParseBuffer};

/// Parsing interface implemented by all types that can be parsed in a default
/// way from a token stream.
///
/// Refer to the [module documentation] for details about parsing in Syn.
///
/// [module documentation]: index.html
///
/// *This trait is available if Syn is built with the `"parsing"` feature.*
pub trait Synom: Sized {
    fn parse(input: Cursor) -> PResult<Self>;

    /// A short name of the type being parsed.
    ///
    /// The description should only be used for a simple name.  It should not
    /// contain newlines or sentence-ending punctuation, to facilitate embedding in
    /// larger user-facing strings.  Syn will use this description when building
    /// error messages about parse failures.
    ///
    /// # Examples
    ///
    /// ```
    /// # use syn::buffer::Cursor;
    /// # use syn::synom::{Synom, PResult};
    /// #
    /// struct ExprMacro {
    ///     // ...
    /// }
    ///
    /// impl Synom for ExprMacro {
    /// #   fn parse(input: Cursor) -> PResult<Self> { unimplemented!() }
    ///     // fn parse(...) -> ... { ... }
    ///
    ///     fn description() -> Option<&'static str> {
    ///         // Will result in messages like
    ///         //
    ///         //     "failed to parse macro invocation expression: $reason"
    ///         Some("macro invocation expression")
    ///     }
    /// }
    /// ```
    fn description() -> Option<&'static str> {
        None
    }
}

impl<T: Parse> Synom for T {
    fn parse(input: Cursor) -> PResult<Self> {
        let state = ParseBuffer::new(Span::call_site(), input);
        match <T as Parse>::parse(&state) {
            Ok(node) => Ok((node, state.cursor())),
            Err(err) => Err(err),
        }
    }
}

impl Synom for TokenStream {
    fn parse(input: Cursor) -> PResult<Self> {
        Ok((input.token_stream(), Cursor::empty()))
    }

    fn description() -> Option<&'static str> {
        Some("arbitrary token stream")
    }
}

impl Synom for TokenTree {
    fn parse(input: Cursor) -> PResult<Self> {
        match input.token_tree() {
            Some((tt, rest)) => Ok((tt, rest)),
            None => parse_error(),
        }
    }

    fn description() -> Option<&'static str> {
        Some("token tree")
    }
}

impl Synom for Group {
    fn parse(input: Cursor) -> PResult<Self> {
        for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
            if let Some((inside, span, rest)) = input.group(*delim) {
                let mut group = Group::new(*delim, inside.token_stream());
                group.set_span(span);
                return Ok((group, rest));
            }
        }
        parse_error()
    }

    fn description() -> Option<&'static str> {
        Some("group token")
    }
}

impl Synom for Punct {
    fn parse(input: Cursor) -> PResult<Self> {
        match input.punct() {
            Some((punct, rest)) => Ok((punct, rest)),
            None => parse_error(),
        }
    }

    fn description() -> Option<&'static str> {
        Some("punctuation token")
    }
}

impl Synom for Literal {
    fn parse(input: Cursor) -> PResult<Self> {
        match input.literal() {
            Some((literal, rest)) => Ok((literal, rest)),
            None => parse_error(),
        }
    }

    fn description() -> Option<&'static str> {
        Some("literal token")
    }
}

/// Parser that can parse Rust tokens into a particular syntax tree node.
///
/// Refer to the [module documentation] for details about parsing in Syn.
///
/// [module documentation]: index.html
///
/// *This trait is available if Syn is built with the `"parsing"` feature.*
pub trait Parser: Sized {
    type Output;

    /// Parse a proc-macro2 token stream into the chosen syntax tree node.
    fn parse2(self, tokens: TokenStream) -> Result<Self::Output, Error>;

    /// Parse tokens of source code into the chosen syntax tree node.
    ///
    /// *This method is available if Syn is built with both the `"parsing"` and
    /// `"proc-macro"` features.*
    #[cfg(all(
        not(all(target_arch = "wasm32", target_os = "unknown")),
        feature = "proc-macro"
    ))]
    fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output, Error> {
        self.parse2(tokens.into())
    }

    /// Parse a string of Rust code into the chosen syntax tree node.
    ///
    /// # Hygiene
    ///
    /// Every span in the resulting syntax tree will be set to resolve at the
    /// macro call site.
    fn parse_str(self, s: &str) -> Result<Self::Output, Error> {
        match s.parse() {
            Ok(tts) => self.parse2(tts),
            Err(_) => Err(Error::new(
                Span::call_site(),
                "error while lexing input string",
            )),
        }
    }
}

impl<F, T> Parser for F
where
    F: FnOnce(Cursor) -> PResult<T>,
{
    type Output = T;

    fn parse2(self, tokens: TokenStream) -> Result<T, Error> {
        let buf = TokenBuffer::new2(tokens);
        let (t, rest) = self(buf.begin())?;
        if rest.eof() {
            Ok(t)
        } else if rest == buf.begin() {
            // parsed nothing
            Err(Error::new(Span::call_site(), "failed to parse anything"))
        } else {
            Err(Error::new(Span::call_site(), "failed to parse all tokens"))
        }
    }
}

/// Extension traits that are made available within the `call!` parser.
///
/// *This module is available if Syn is built with the `"parsing"` feature.*
pub mod ext {
    use super::*;
    use proc_macro2::Ident;

    /// Additional parsing methods for `Ident`.
    ///
    /// This trait is sealed and cannot be implemented for types outside of Syn.
    ///
    /// *This trait is available if Syn is built with the `"parsing"` feature.*
    pub trait IdentExt: Sized + private::Sealed {
        /// Parses any identifier including keywords.
        ///
        /// This is useful when parsing a DSL which allows Rust keywords as
        /// identifiers.
        ///
        /// ```rust
        /// #[macro_use]
        /// extern crate syn;
        ///
        /// use syn::Ident;
        ///
        /// // Parses input that looks like `name = NAME` where `NAME` can be
        /// // any identifier.
        /// //
        /// // Examples:
        /// //
        /// //     name = anything
        /// //     name = impl
        /// named!(parse_dsl -> Ident, do_parse!(
        ///     custom_keyword!(name) >>
        ///     punct!(=) >>
        ///     name: call!(Ident::parse_any) >>
        ///     (name)
        /// ));
        /// #
        /// # fn main() {}
        /// ```
        fn parse_any(input: Cursor) -> PResult<Self>;
    }

    impl IdentExt for Ident {
        fn parse_any(input: Cursor) -> PResult<Self> {
            input.ident().map_or_else(parse_error, Ok)
        }
    }

    mod private {
        use proc_macro2::Ident;

        pub trait Sealed {}

        impl Sealed for Ident {}
    }
}
<|endoftext|>