<commit_before>/// This module takes a (parsed) definition of `macro_rules` invocation, a
/// `tt::TokenTree` representing an argument of macro invocation, and produces a
/// `tt::TokenTree` for the result of the expansion.
use rustc_hash::FxHashMap;
use ra_syntax::SmolStr;
use tt::TokenId;

use crate::ExpandError;
use crate::tt_cursor::TtCursor;

pub(crate) fn expand(
    rules: &crate::MacroRules,
    input: &tt::Subtree,
) -> Result<tt::Subtree, ExpandError> {
    rules.rules.iter().find_map(|it| expand_rule(it, input).ok()).ok_or(ExpandError::NoMatchingRule)
}

fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
    let mut input = TtCursor::new(input);
    let bindings = match_lhs(&rule.lhs, &mut input)?;
    if !input.is_eof() {
        return Err(ExpandError::UnexpectedToken);
    }
    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
}

/// The actual algorithm for expansion is not too hard, but is pretty tricky.
/// `Bindings` structure is the key to understanding what we are doing here.
///
/// On the high level, it stores mapping from meta variables to the bits of
/// syntax it should be substituted with. For example, if `$e:expr` is matched
/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
///
/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
///
/// ```not_rust
/// macro_rules! foo {
///     ($($ i:ident $($ e:expr),*);*) => {
///         $(fn $ i() { $($ e);*; })*
///     }
/// }
/// foo! { foo 1,2,3; bar 4,5,6 }
/// ```
///
/// Here, the `$i` meta variable is matched first with `foo` and then with
/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
///
/// To represent such "multi-mappings", we use a recursive structures: we map
/// variables not to values, but to *lists* of values or other lists (that is,
/// to the trees).
///
/// For the above example, the bindings would store
///
/// ```not_rust
/// i -> [foo, bar]
/// e -> [[1, 2, 3], [4, 5, 6]]
/// ```
///
/// We construct `Bindings` in the `match_lhs`. The interesting case is
/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
/// nesting structure.
///
/// The other side of the puzzle is `expand_subtree`, where we use the bindings
/// to substitute meta variables in the output template. When expanding, we
/// maintain a `nesting` stack of indices which tells us which occurrence from
/// the `Bindings` we should take. We push to the stack when we enter a
/// repetition.
///
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but an `&[usize]`.
#[derive(Debug, Default)]
struct Bindings {
    inner: FxHashMap<SmolStr, Binding>,
}

#[derive(Debug)]
enum Binding {
    Simple(tt::TokenTree),
    Nested(Vec<Binding>),
}

impl Bindings {
    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree, ExpandError> {
        let mut b = self
            .inner
            .get(name)
            .ok_or(ExpandError::BindingError(format!("could not find binding `{}`", name)))?;
        for &idx in nesting.iter() {
            b = match b {
                Binding::Simple(_) => break,
                Binding::Nested(bs) => bs.get(idx).ok_or(ExpandError::BindingError(format!(
                    "could not find nested binding `{}`",
                    name
                )))?,
            };
        }
        match b {
            Binding::Simple(it) => Ok(it),
            Binding::Nested(_) => Err(ExpandError::BindingError(format!(
                "expected simple binding, found nested binding `{}`",
                name
            ))),
        }
    }

    fn push_nested(&mut self, nested: Bindings) -> Result<(), ExpandError> {
        for (key, value) in nested.inner {
            if !self.inner.contains_key(&key) {
                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
            }
            match self.inner.get_mut(&key) {
                Some(Binding::Nested(it)) => it.push(value),
                _ => {
                    return Err(ExpandError::BindingError(format!(
                        "could not find binding `{}`",
                        key
                    )));
                }
            }
        }
        Ok(())
    }

    fn merge(&mut self, nested: Bindings) {
        self.inner.extend(nested.inner);
    }
}

fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
    let mut res = Bindings::default();
    for pat in pattern.token_trees.iter() {
        match pat {
            crate::TokenTree::Leaf(leaf) => match leaf {
                crate::Leaf::Var(crate::Var { text, kind }) => {
                    let kind = kind.clone().ok_or(ExpandError::UnexpectedToken)?;
                    match kind.as_str() {
                        "ident" => {
                            let ident =
                                input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(ident).into()),
                            );
                        }
                        "path" => {
                            let path =
                                input.eat_path().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(path.into()));
                        }
                        "expr" => {
                            let expr =
                                input.eat_expr().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(expr.into()));
                        }
                        "ty" => {
                            let ty = input.eat_ty().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(ty.into()));
                        }
                        "pat" => {
                            let pat = input.eat_pat().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(pat.into()));
                        }
                        "stmt" => {
                            let pat = input.eat_stmt().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(pat.into()));
                        }
                        "block" => {
                            let block =
                                input.eat_block().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(block.into()));
                        }
                        "meta" => {
                            let meta =
                                input.eat_meta().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(meta.into()));
                        }
                        // FIXME:
                        // Enable followiing code when everything is fixed
                        // At least we can dogfood itself to not stackoverflow
                        //
                        // "tt" => {
                        //     let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
                        //     res.inner.insert(text.clone(), Binding::Simple(token.into()));
                        // }
                        "item" => {
                            let item =
                                input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(item.into()));
                        }
                        "lifetime" => {
                            let lifetime =
                                input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(lifetime.into()));
                        }
                        "literal" => {
                            let literal =
                                input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(literal).into()),
                            );
                        }
                        "vis" => {
                            let vis = input.eat_vis().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(vis.into()));
                        }

                        _ => return Err(ExpandError::UnexpectedToken),
                    }
                }
                crate::Leaf::Punct(punct) => {
                    if input.eat_punct() != Some(punct) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                crate::Leaf::Ident(ident) => {
                    if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                _ => return Err(ExpandError::UnexpectedToken),
            },
            crate::TokenTree::Repeat(crate::Repeat { subtree, kind: _, separator }) => {
                // Dirty hack to make macro-expansion terminate.
                // This should be replaced by a propper macro-by-example implementation
                let mut limit = 128;
                while let Ok(nested) = match_lhs(subtree, input) {
                    limit -= 1;
                    if limit == 0 {
                        break;
                    }
                    res.push_nested(nested)?;
                    if let Some(separator) = *separator {
                        if !input.is_eof() {
                            if input.eat_punct().map(|p| p.char) != Some(separator) {
                                return Err(ExpandError::UnexpectedToken);
                            }
                        }
                    }
                }
            }
            crate::TokenTree::Subtree(subtree) => {
                let input_subtree =
                    input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
                if subtree.delimiter != input_subtree.delimiter {
                    return Err(ExpandError::UnexpectedToken);
                }

                let mut input = TtCursor::new(input_subtree);
                let bindings = match_lhs(&subtree, &mut input)?;
                if !input.is_eof() {
                    return Err(ExpandError::UnexpectedToken);
                }

                res.merge(bindings);
            }
        }
    }
    Ok(res)
}

fn expand_subtree(
    template: &crate::Subtree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::Subtree, ExpandError> {
    let token_trees = template
        .token_trees
        .iter()
        .map(|it| expand_tt(it, bindings, nesting))
        .collect::<Result<Vec<_>, ExpandError>>()?;

    Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
}

fn expand_tt(
    template: &crate::TokenTree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::TokenTree, ExpandError> {
    let res: tt::TokenTree = match template {
        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
        crate::TokenTree::Repeat(repeat) => {
            let mut token_trees = Vec::new();
            nesting.push(0);
            // Dirty hack to make macro-expansion terminate.
            // This should be replaced by a propper macro-by-example implementation
            let mut limit = 128;
            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
                limit -= 1;
                if limit == 0 {
                    break;
                }
                let idx = nesting.pop().unwrap();
                nesting.push(idx + 1);
                token_trees.push(t.into())
            }
            nesting.pop().unwrap();
            tt::Subtree { token_trees, delimiter: tt::Delimiter::None }.into()
        }
        crate::TokenTree::Leaf(leaf) => match leaf {
            crate::Leaf::Ident(ident) => {
                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
                    .into()
            }
            crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
            crate::Leaf::Var(v) => {
                if v.text == "crate" {
                    // FIXME: Properly handle $crate token
                    tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
                        .into()
                } else {
                    bindings.get(&v.text, nesting)?.clone()
                }
            }
            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
        },
    };
    Ok(res)
}

#[cfg(test)]
mod tests {
    use ra_syntax::{ast, AstNode};

    use super::*;
    use crate::ast_to_token_tree;

    #[test]
    fn test_expand_rule() {
        assert_err(
            "($i:ident) => ($j)",
            "foo!{a}",
            ExpandError::BindingError(String::from("could not find binding `j`")),
        );

        assert_err(
            "($($i:ident);*) => ($i)",
            "foo!{a}",
            ExpandError::BindingError(String::from(
                "expected simple binding, found nested binding `i`",
            )),
        );

        assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
        assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);

        // FIXME:
        // Add an err test case for ($($i:ident)) => ($())
    }

    fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) {
        assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation), Err(err));
    }

    fn format_macro(macro_body: &str) -> String {
        format!(
            "
        macro_rules! foo {{
            {}
        }}
",
            macro_body
        )
    }

    fn create_rules(macro_definition: &str) -> crate::MacroRules {
        let source_file = ast::SourceFile::parse(macro_definition);
        let macro_definition =
            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();

        let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
        crate::MacroRules::parse(&definition_tt).unwrap()
    }

    fn expand_first(
        rules: &crate::MacroRules,
        invocation: &str,
    ) -> Result<tt::Subtree, ExpandError> {
        let source_file = ast::SourceFile::parse(invocation);
        let macro_invocation =
            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();

        let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();

        expand_rule(&rules.rules[0], &invocation_tt)
    }
}
<commit_msg>Handle `*+` case and single token case<commit_after>/// This module takes a (parsed) definition of `macro_rules` invocation, a
/// `tt::TokenTree` representing an argument of macro invocation, and produces a
/// `tt::TokenTree` for the result of the expansion.
use rustc_hash::FxHashMap;
use ra_syntax::SmolStr;
use tt::TokenId;

use crate::ExpandError;
use crate::tt_cursor::TtCursor;

pub(crate) fn expand(
    rules: &crate::MacroRules,
    input: &tt::Subtree,
) -> Result<tt::Subtree, ExpandError> {
    rules.rules.iter().find_map(|it| expand_rule(it, input).ok()).ok_or(ExpandError::NoMatchingRule)
}

fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
    let mut input = TtCursor::new(input);
    let bindings = match_lhs(&rule.lhs, &mut input)?;
    if !input.is_eof() {
        return Err(ExpandError::UnexpectedToken);
    }
    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
}

/// The actual algorithm for expansion is not too hard, but is pretty tricky.
/// `Bindings` structure is the key to understanding what we are doing here.
///
/// On the high level, it stores mapping from meta variables to the bits of
/// syntax it should be substituted with. For example, if `$e:expr` is matched
/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
///
/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
///
/// ```not_rust
/// macro_rules! foo {
///     ($($ i:ident $($ e:expr),*);*) => {
///         $(fn $ i() { $($ e);*; })*
///     }
/// }
/// foo! { foo 1,2,3; bar 4,5,6 }
/// ```
///
/// Here, the `$i` meta variable is matched first with `foo` and then with
/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
///
/// To represent such "multi-mappings", we use a recursive structures: we map
/// variables not to values, but to *lists* of values or other lists (that is,
/// to the trees).
///
/// For the above example, the bindings would store
///
/// ```not_rust
/// i -> [foo, bar]
/// e -> [[1, 2, 3], [4, 5, 6]]
/// ```
///
/// We construct `Bindings` in the `match_lhs`. The interesting case is
/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
/// nesting structure.
///
/// The other side of the puzzle is `expand_subtree`, where we use the bindings
/// to substitute meta variables in the output template. When expanding, we
/// maintain a `nesting` stack of indices which tells us which occurrence from
/// the `Bindings` we should take. We push to the stack when we enter a
/// repetition.
///
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but an `&[usize]`.
#[derive(Debug, Default)]
struct Bindings {
    inner: FxHashMap<SmolStr, Binding>,
}

#[derive(Debug)]
enum Binding {
    Simple(tt::TokenTree),
    Nested(Vec<Binding>),
}

impl Bindings {
    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree, ExpandError> {
        let mut b = self
            .inner
            .get(name)
            .ok_or(ExpandError::BindingError(format!("could not find binding `{}`", name)))?;
        for &idx in nesting.iter() {
            b = match b {
                Binding::Simple(_) => break,
                Binding::Nested(bs) => bs.get(idx).ok_or(ExpandError::BindingError(format!(
                    "could not find nested binding `{}`",
                    name
                )))?,
            };
        }
        match b {
            Binding::Simple(it) => Ok(it),
            Binding::Nested(_) => Err(ExpandError::BindingError(format!(
                "expected simple binding, found nested binding `{}`",
                name
            ))),
        }
    }

    fn push_nested(&mut self, nested: Bindings) -> Result<(), ExpandError> {
        for (key, value) in nested.inner {
            if !self.inner.contains_key(&key) {
                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
            }
            match self.inner.get_mut(&key) {
                Some(Binding::Nested(it)) => it.push(value),
                _ => {
                    return Err(ExpandError::BindingError(format!(
                        "could not find binding `{}`",
                        key
                    )));
                }
            }
        }
        Ok(())
    }

    fn merge(&mut self, nested: Bindings) {
        self.inner.extend(nested.inner);
    }
}

fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
    let mut res = Bindings::default();
    for pat in pattern.token_trees.iter() {
        match pat {
            crate::TokenTree::Leaf(leaf) => match leaf {
                crate::Leaf::Var(crate::Var { text, kind }) => {
                    let kind = kind.clone().ok_or(ExpandError::UnexpectedToken)?;
                    match kind.as_str() {
                        "ident" => {
                            let ident =
                                input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(ident).into()),
                            );
                        }
                        "path" => {
                            let path =
                                input.eat_path().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(path.into()));
                        }
                        "expr" => {
                            let expr =
                                input.eat_expr().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(expr.into()));
                        }
                        "ty" => {
                            let ty = input.eat_ty().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(ty.into()));
                        }
                        "pat" => {
                            let pat = input.eat_pat().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(pat.into()));
                        }
                        "stmt" => {
                            let pat = input.eat_stmt().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(pat.into()));
                        }
                        "block" => {
                            let block =
                                input.eat_block().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(block.into()));
                        }
                        "meta" => {
                            let meta =
                                input.eat_meta().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(meta.into()));
                        }
                        // FIXME:
                        // Enable followiing code when everything is fixed
                        // At least we can dogfood itself to not stackoverflow
                        //
                        // "tt" => {
                        //     let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
                        //     res.inner.insert(text.clone(), Binding::Simple(token.into()));
                        // }
                        "item" => {
                            let item =
                                input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(item.into()));
                        }
                        "lifetime" => {
                            let lifetime =
                                input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(lifetime.into()));
                        }
                        "literal" => {
                            let literal =
                                input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(literal).into()),
                            );
                        }
                        "vis" => {
                            let vis = input.eat_vis().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(text.clone(), Binding::Simple(vis.into()));
                        }

                        _ => return Err(ExpandError::UnexpectedToken),
                    }
                }
                crate::Leaf::Punct(punct) => {
                    if input.eat_punct() != Some(punct) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                crate::Leaf::Ident(ident) => {
                    if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                _ => return Err(ExpandError::UnexpectedToken),
            },
            crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
                // Dirty hack to make macro-expansion terminate.
                // This should be replaced by a propper macro-by-example implementation
                let mut limit = 128;
                let mut counter = 0;
                while let Ok(nested) = match_lhs(subtree, input) {
                    counter += 1;
                    limit -= 1;
                    if limit == 0 {
                        break;
                    }
                    res.push_nested(nested)?;
                    if let Some(separator) = *separator {
                        if !input.is_eof() {
                            if input.eat_punct().map(|p| p.char) != Some(separator) {
                                return Err(ExpandError::UnexpectedToken);
                            }
                        }
                    }
                }

                match kind {
                    crate::RepeatKind::OneOrMore if counter == 0 => {
                        return Err(ExpandError::UnexpectedToken);
                    }
                    crate::RepeatKind::ZeroOrOne if counter > 1 => {
                        return Err(ExpandError::UnexpectedToken);
                    }

                    _ => {}
                }
            }
            crate::TokenTree::Subtree(subtree) => {
                let input_subtree =
                    input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
                if subtree.delimiter != input_subtree.delimiter {
                    return Err(ExpandError::UnexpectedToken);
                }

                let mut input = TtCursor::new(input_subtree);
                let bindings = match_lhs(&subtree, &mut input)?;
                if !input.is_eof() {
                    return Err(ExpandError::UnexpectedToken);
                }

                res.merge(bindings);
            }
        }
    }
    Ok(res)
}

fn expand_subtree(
    template: &crate::Subtree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::Subtree, ExpandError> {
    let token_trees = template
        .token_trees
        .iter()
        .map(|it| expand_tt(it, bindings, nesting))
        .collect::<Result<Vec<_>, ExpandError>>()?;

    Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
}

/// Reduce single token subtree to single token
/// In `tt` matcher case, all tt tokens will be braced by a Delimiter::None
/// which makes all sort of problems.
fn reduce_single_token(mut subtree: tt::Subtree) -> tt::TokenTree {
    if subtree.delimiter != tt::Delimiter::None || subtree.token_trees.len() != 1 {
        return subtree.into();
    }

    match subtree.token_trees.pop().unwrap() {
        tt::TokenTree::Subtree(subtree) => reduce_single_token(subtree),
        tt::TokenTree::Leaf(token) => token.into(),
    }
}

fn expand_tt(
    template: &crate::TokenTree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::TokenTree, ExpandError> {
    let res: tt::TokenTree = match template {
        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
        crate::TokenTree::Repeat(repeat) => {
            let mut token_trees: Vec<tt::TokenTree> = Vec::new();
            nesting.push(0);
            // Dirty hack to make macro-expansion terminate.
            // This should be replaced by a propper macro-by-example implementation
            let mut limit = 128;
            let mut has_sep = false;

            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
                limit -= 1;
                if limit == 0 {
                    break;
                }
                let idx = nesting.pop().unwrap();
                nesting.push(idx + 1);
                token_trees.push(reduce_single_token(t).into());

                if let Some(sep) = repeat.separator {
                    let punct =
                        tt::Leaf::from(tt::Punct { char: sep, spacing: tt::Spacing::Alone });
                    token_trees.push(punct.into());
                    has_sep = true;
                }
            }
            nesting.pop().unwrap();

            // Dirty hack for remove the last sep
            // if it is a "," undo the push
            if has_sep && repeat.separator.unwrap() == ',' {
                token_trees.pop();
            }

            // Check if it is a singel token subtree without any delimiter
            // e.g {Delimiter:None> ['>'] /Delimiter:None>}
            reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None })
        }
        crate::TokenTree::Leaf(leaf) => match leaf {
            crate::Leaf::Ident(ident) => {
                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
                    .into()
            }
            crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
            crate::Leaf::Var(v) => {
                if v.text == "crate" {
                    // FIXME: Properly handle $crate token
                    tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
                        .into()
                } else {
                    let tkn = bindings.get(&v.text, nesting)?.clone();

                    if let tt::TokenTree::Subtree(subtree) = tkn {
                        reduce_single_token(subtree)
                    } else {
                        tkn
                    }
                }
            }
            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
        },
    };
    Ok(res)
}

#[cfg(test)]
mod tests {
    use ra_syntax::{ast, AstNode};

    use super::*;
    use crate::ast_to_token_tree;

    #[test]
    fn test_expand_rule() {
        assert_err(
            "($i:ident) => ($j)",
            "foo!{a}",
            ExpandError::BindingError(String::from("could not find binding `j`")),
        );

        assert_err(
            "($($i:ident);*) => ($i)",
            "foo!{a}",
            ExpandError::BindingError(String::from(
                "expected simple binding, found nested binding `i`",
            )),
        );

        assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
        assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);

        // FIXME:
        // Add an err test case for ($($i:ident)) => ($())
    }

    fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) {
        assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation), Err(err));
    }

    fn format_macro(macro_body: &str) -> String {
        format!(
            "
        macro_rules! foo {{
            {}
        }}
",
            macro_body
        )
    }

    fn create_rules(macro_definition: &str) -> crate::MacroRules {
        let source_file = ast::SourceFile::parse(macro_definition);
        let macro_definition =
            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();

        let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
        crate::MacroRules::parse(&definition_tt).unwrap()
    }

    fn expand_first(
        rules: &crate::MacroRules,
        invocation: &str,
    ) -> Result<tt::Subtree, ExpandError> {
        let source_file = ast::SourceFile::parse(invocation);
        let macro_invocation =
            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();

        let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();

        expand_rule(&rules.rules[0], &invocation_tt)
    }
}
<|endoftext|>


<commit_before>use bson::Bson;
use mongodb::{CommandType, Client, ThreadedClient};
use mongodb::db::ThreadedDatabase;

#[test]
fn auth() {
    let client = Client::connect("localhost", 27017).unwrap();
    let db = client.db("auth");

    let _ = db.drop_all_users(None).unwrap();

    let doc = doc! { "connectionStatus" => 1};
    let before = db.command(doc.clone(), CommandType::Suppressed).unwrap();

    let info = match before.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for initial connectionStatus command")
    };

    match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => assert!(vec.is_empty()),
        _ => panic!("Invalid array of authenticatedUsers for initial connectionStatus command")
    };

    db.create_user("saghm", "such_secure_password", None).unwrap();
    db.auth("saghm", "such_secure_password").unwrap();

    let after = db.command(doc, CommandType::Suppressed).unwrap();

    let info = match after.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for initial connectionStatus command")
    };

    let authed_users = match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => vec.clone(),
        _ => panic!("Invalid array of authenticatedUsers for initial connectionStatus command")
    };

    assert_eq!(authed_users.len(), 1);

    let user = match authed_users[0] {
        Bson::Document(ref doc) => doc.clone(),
        _ => panic!("Invalid auth'd user in second connectionStatus response")
    };

    match user.get("user") {
        Some(&Bson::String(ref s)) => assert_eq!(s, "saghm"),
        _ => panic!("Invalid `user` field of auth'd user")
    };

    match user.get("db") {
        Some(&Bson::String(ref s)) => assert_eq!(s, "auth"),
        _ => panic!("Invalid `db` field of auth'd user")
    };
}
<commit_msg>added tests for invalid authentication<commit_after>use bson::Bson;
use mongodb::{CommandType, Client, ThreadedClient};
use mongodb::db::ThreadedDatabase;
use mongodb::error::Error::OperationError;

#[test]
fn invalid_user() {
    let client = Client::connect("localhost", 27017).unwrap();
    let db = client.db("auth");
    let _ = db.drop_all_users(None).unwrap();
    let doc = doc! { "connectionStatus" => 1};
    let before = db.command(doc.clone(), CommandType::Suppressed).unwrap();

    let info = match before.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for initial connectionStatus command")
    };

    match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => assert!(vec.is_empty()),
        _ => panic!("Invalid array of authenticatedUsers for initial connectionStatus command")
    };

    match db.auth("invalid_user", "some_password") {
        Err(OperationError(_)) => (),
        Err(_) => panic!("Expected OperationError for invalid authentication, but got some other error instead"),
        _ => panic!("Authentication succeeded despite invalid credentials")
    };

    let after = db.command(doc, CommandType::Suppressed).unwrap();

    let info = match after.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for subsequent connectionStatus command")
    };

    match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => assert!(vec.is_empty()),
        _ => panic!("Invalid array of authenticatedUsers for subsequent connectionStatus command")
    };
}

#[test]
fn invalid_password() {
    let client = Client::connect("localhost", 27017).unwrap();
    let db = client.db("auth");
    let _ = db.drop_all_users(None).unwrap();
    let doc = doc! { "connectionStatus" => 1};
    let before = db.command(doc.clone(), CommandType::Suppressed).unwrap();

    let info = match before.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for initial connectionStatus command")
    };

    match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => assert!(vec.is_empty()),
        _ => panic!("Invalid array of authenticatedUsers for initial connectionStatus command")
    };

    db.create_user("saghm", "such_secure_password", None).unwrap();

    match db.auth("saghm", "wrong_password") {
        Err(OperationError(_)) => (),
        Err(_) => panic!("Expected OperationError for invalid authentication, but got some other error instead"),
        _ => panic!("Authentication succeeded despite invalid credentials")
    };

    let after = db.command(doc, CommandType::Suppressed).unwrap();

    let info = match after.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for subsequent connectionStatus command")
    };

    match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => assert!(vec.is_empty()),
        _ => panic!("Invalid array of authenticatedUsers for subsequent connectionStatus command")
    };
}

#[test]
fn successful_login() {
    let client = Client::connect("localhost", 27017).unwrap();
    let db = client.db("auth");
    let _ = db.drop_all_users(None).unwrap();
    let doc = doc! { "connectionStatus" => 1};
    let before = db.command(doc.clone(), CommandType::Suppressed).unwrap();

    let info = match before.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for initial connectionStatus command")
    };

    match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => assert!(vec.is_empty()),
        _ => panic!("Invalid array of authenticatedUsers for initial connectionStatus command")
    };

    db.create_user("saghm", "such_secure_password", None).unwrap();
    db.auth("saghm", "such_secure_password").unwrap();

    let after = db.command(doc, CommandType::Suppressed).unwrap();

    let info = match after.get("authInfo") {
        Some(&Bson::Document(ref doc)) => doc.clone(),
        _ => panic!("Invalid response for subsequent connectionStatus command")
    };

    let authed_users = match info.get("authenticatedUsers") {
        Some(&Bson::Array(ref vec)) => vec.clone(),
        _ => panic!("Invalid array of authenticatedUsers for subsequent connectionStatus command")
    };

    assert_eq!(authed_users.len(), 1);

    let user = match authed_users[0] {
        Bson::Document(ref doc) => doc.clone(),
        _ => panic!("Invalid auth'd user in subsequent connectionStatus response")
    };

    match user.get("user") {
        Some(&Bson::String(ref s)) => assert_eq!(s, "saghm"),
        _ => panic!("Invalid `user` field of auth'd user")
    };

    match user.get("db") {
        Some(&Bson::String(ref s)) => assert_eq!(s, "auth"),
        _ => panic!("Invalid `db` field of auth'd user")
    };
}
<|endoftext|>


<commit_before>#![allow(clippy::manual_assert)]

mod progress;

use self::progress::Progress;
use anyhow::Result;
use flate2::read::GzDecoder;
use std::fs;
use std::path::Path;
use tar::Archive;
use walkdir::DirEntry;

const REVISION: &str = "e95b10ba4ac4564ed25f7eef143e3182c33b3902";

#[rustfmt::skip]
static EXCLUDE: &[&str] = &[
    // TODO
    "src/test/ui/associated-consts/assoc-const.rs",
    "src/test/ui/generic-associated-types/collections.rs",
    "src/test/ui/generic-associated-types/construct_with_other_type.rs",
    "src/test/ui/generic-associated-types/impl_bounds_ok.rs",
    "src/test/ui/generic-associated-types/issue-76826.rs",
    "src/test/ui/generic-associated-types/issue-88287.rs",
    "src/test/ui/generic-associated-types/issue-91139.rs",
    "src/test/ui/generic-associated-types/issue-92280.rs",
    "src/test/ui/generic-associated-types/iterable.rs",
    "src/test/ui/generic-associated-types/streaming_iterator.rs",
    "src/test/ui/symbol-names/const-generics-demangling.rs",

    // TODO: impl ~const T {}
    // https://github.com/dtolnay/syn/issues/1051
    "src/test/ui/rfc-2632-const-trait-impl/syntax.rs",

    // Compile-fail expr parameter in const generic position: f::<1 + 2>()
    "src/test/ui/const-generics/early/closing-args-token.rs",
    "src/test/ui/const-generics/early/const-expression-parameter.rs",

    // Deprecated anonymous parameter syntax in traits
    "src/test/ui/issues/issue-13105.rs",
    "src/test/ui/issues/issue-13775.rs",
    "src/test/ui/issues/issue-34074.rs",
    "src/test/ui/proc-macro/trait-fn-args-2015.rs",
    "src/tools/rustfmt/tests/source/trait.rs",
    "src/tools/rustfmt/tests/target/trait.rs",

    // Excessive nesting
    "src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs",

    // Testing rustfmt on invalid syntax
    "src/tools/rustfmt/tests/coverage/target/comments.rs",
    "src/tools/rustfmt/tests/parser/issue-4126/invalid.rs",
    "src/tools/rustfmt/tests/parser/issue_4418.rs",
    "src/tools/rustfmt/tests/parser/unclosed-delims/issue_4466.rs",
    "src/tools/rustfmt/tests/source/configs/disable_all_formatting/true.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/source/type.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/target/type.rs",

    // Clippy lint lists represented as expressions
    "src/tools/clippy/clippy_lints/src/lib.deprecated.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_all.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_cargo.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_complexity.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_correctness.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_internal.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_lints.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_nursery.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_pedantic.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_perf.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_restriction.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_style.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_suspicious.rs",

    // Not actually test cases
    "src/test/rustdoc-ui/test-compile-fail2.rs",
    "src/test/rustdoc-ui/test-compile-fail3.rs",
    "src/test/ui/json-bom-plus-crlf-multifile-aux.rs",
    "src/test/ui/lint/expansion-time-include.rs",
    "src/test/ui/macros/auxiliary/macro-comma-support.rs",
    "src/test/ui/macros/auxiliary/macro-include-items-expr.rs",
    "src/test/ui/macros/include-single-expr-helper.rs",
    "src/test/ui/macros/include-single-expr-helper-1.rs",
    "src/test/ui/parser/issues/auxiliary/issue-21146-inc.rs",
];

pub fn base_dir_filter(entry: &DirEntry) -> bool {
    let path = entry.path();
    if path.is_dir() {
        return true; // otherwise walkdir does not visit the files
    }
    if path.extension().map_or(true, |e| e != "rs") {
        return false;
    }

    let mut path_string = path.to_string_lossy();
    if cfg!(windows) {
        path_string = path_string.replace('\\', "/").into();
    }
    let path = if let Some(path) = path_string.strip_prefix("tests/rust/") {
        path
    } else {
        panic!("unexpected path in Rust dist: {}", path_string);
    };

    if path.starts_with("src/test/compile-fail") || path.starts_with("src/test/rustfix") {
        return false;
    }

    if path.starts_with("src/test/ui") {
        let stderr_path = entry.path().with_extension("stderr");
        if stderr_path.exists() {
            // Expected to fail in some way
            return false;
        }
    }

    !EXCLUDE.contains(&path)
}

#[allow(dead_code)]
pub fn edition(path: &Path) -> &'static str {
    if path.ends_with("dyn-2015-no-warnings-without-lints.rs") {
        "2015"
    } else {
        "2018"
    }
}

pub fn clone_rust() {
    let needs_clone = match fs::read_to_string("tests/rust/COMMIT") {
        Err(_) => true,
        Ok(contents) => contents.trim() != REVISION,
    };
    if needs_clone {
        download_and_unpack().unwrap();
    }
    let mut missing = String::new();
    let test_src = Path::new("tests/rust");
    for exclude in EXCLUDE {
        if !test_src.join(exclude).exists() {
            missing += "\ntests/rust/";
            missing += exclude;
        }
    }
    if !missing.is_empty() {
        panic!("excluded test file does not exist:{}\n", missing);
    }
}

fn download_and_unpack() -> Result<()> {
    let url = format!(
        "https://github.com/rust-lang/rust/archive/{}.tar.gz",
        REVISION
    );
    let response = reqwest::blocking::get(&url)?.error_for_status()?;
    let progress = Progress::new(response);
    let decoder = GzDecoder::new(progress);
    let mut archive = Archive::new(decoder);
    let prefix = format!("rust-{}", REVISION);

    let tests_rust = Path::new("tests/rust");
    if tests_rust.exists() {
        fs::remove_dir_all(tests_rust)?;
    }

    for entry in archive.entries()? {
        let mut entry = entry?;
        let path = entry.path()?;
        if path == Path::new("pax_global_header") {
            continue;
        }
        let relative = path.strip_prefix(&prefix)?;
        let out = tests_rust.join(relative);
        entry.unpack(&out)?;
    }

    fs::write("tests/rust/COMMIT", REVISION)?;
    Ok(())
}
<commit_msg>Categorize new round of rust repo source file failures<commit_after>#![allow(clippy::manual_assert)]

mod progress;

use self::progress::Progress;
use anyhow::Result;
use flate2::read::GzDecoder;
use std::fs;
use std::path::Path;
use tar::Archive;
use walkdir::DirEntry;

const REVISION: &str = "e95b10ba4ac4564ed25f7eef143e3182c33b3902";

#[rustfmt::skip]
static EXCLUDE: &[&str] = &[
    // TODO: trait bound with associated constant constraint:
    // where T: Trait<N = 3usize>
    "src/test/ui/associated-consts/assoc-const.rs",

    // TODO: trailing where-clause on impl associated type:
    // impl<T> Trait for Ty<T> { type Assoc<'a> = T where T: 'a; }
    "src/test/ui/generic-associated-types/collections.rs",
    "src/test/ui/generic-associated-types/construct_with_other_type.rs",
    "src/test/ui/generic-associated-types/impl_bounds_ok.rs",
    "src/test/ui/generic-associated-types/issue-76826.rs",
    "src/test/ui/generic-associated-types/issue-88287.rs",
    "src/test/ui/generic-associated-types/issue-91139.rs",
    "src/test/ui/generic-associated-types/issue-92280.rs",
    "src/test/ui/generic-associated-types/iterable.rs",
    "src/test/ui/generic-associated-types/streaming_iterator.rs",

    // TODO: negative literal const generic:
    // impl Signed<-152> {…}
    "src/test/ui/symbol-names/const-generics-demangling.rs",

    // TODO: impl ~const T {}
    // https://github.com/dtolnay/syn/issues/1051
    "src/test/ui/rfc-2632-const-trait-impl/syntax.rs",

    // Compile-fail expr parameter in const generic position: f::<1 + 2>()
    "src/test/ui/const-generics/early/closing-args-token.rs",
    "src/test/ui/const-generics/early/const-expression-parameter.rs",

    // Deprecated anonymous parameter syntax in traits
    "src/test/ui/issues/issue-13105.rs",
    "src/test/ui/issues/issue-13775.rs",
    "src/test/ui/issues/issue-34074.rs",
    "src/test/ui/proc-macro/trait-fn-args-2015.rs",
    "src/tools/rustfmt/tests/source/trait.rs",
    "src/tools/rustfmt/tests/target/trait.rs",

    // Excessive nesting
    "src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs",

    // Testing rustfmt on invalid syntax
    "src/tools/rustfmt/tests/coverage/target/comments.rs",
    "src/tools/rustfmt/tests/parser/issue-4126/invalid.rs",
    "src/tools/rustfmt/tests/parser/issue_4418.rs",
    "src/tools/rustfmt/tests/parser/unclosed-delims/issue_4466.rs",
    "src/tools/rustfmt/tests/source/configs/disable_all_formatting/true.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/source/type.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/target/type.rs",

    // Clippy lint lists represented as expressions
    "src/tools/clippy/clippy_lints/src/lib.deprecated.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_all.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_cargo.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_complexity.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_correctness.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_internal.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_lints.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_nursery.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_pedantic.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_perf.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_restriction.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_style.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_suspicious.rs",

    // Not actually test cases
    "src/test/rustdoc-ui/test-compile-fail2.rs",
    "src/test/rustdoc-ui/test-compile-fail3.rs",
    "src/test/ui/json-bom-plus-crlf-multifile-aux.rs",
    "src/test/ui/lint/expansion-time-include.rs",
    "src/test/ui/macros/auxiliary/macro-comma-support.rs",
    "src/test/ui/macros/auxiliary/macro-include-items-expr.rs",
    "src/test/ui/macros/include-single-expr-helper.rs",
    "src/test/ui/macros/include-single-expr-helper-1.rs",
    "src/test/ui/parser/issues/auxiliary/issue-21146-inc.rs",
];

pub fn base_dir_filter(entry: &DirEntry) -> bool {
    let path = entry.path();
    if path.is_dir() {
        return true; // otherwise walkdir does not visit the files
    }
    if path.extension().map_or(true, |e| e != "rs") {
        return false;
    }

    let mut path_string = path.to_string_lossy();
    if cfg!(windows) {
        path_string = path_string.replace('\\', "/").into();
    }
    let path = if let Some(path) = path_string.strip_prefix("tests/rust/") {
        path
    } else {
        panic!("unexpected path in Rust dist: {}", path_string);
    };

    if path.starts_with("src/test/compile-fail") || path.starts_with("src/test/rustfix") {
        return false;
    }

    if path.starts_with("src/test/ui") {
        let stderr_path = entry.path().with_extension("stderr");
        if stderr_path.exists() {
            // Expected to fail in some way
            return false;
        }
    }

    !EXCLUDE.contains(&path)
}

#[allow(dead_code)]
pub fn edition(path: &Path) -> &'static str {
    if path.ends_with("dyn-2015-no-warnings-without-lints.rs") {
        "2015"
    } else {
        "2018"
    }
}

pub fn clone_rust() {
    let needs_clone = match fs::read_to_string("tests/rust/COMMIT") {
        Err(_) => true,
        Ok(contents) => contents.trim() != REVISION,
    };
    if needs_clone {
        download_and_unpack().unwrap();
    }
    let mut missing = String::new();
    let test_src = Path::new("tests/rust");
    for exclude in EXCLUDE {
        if !test_src.join(exclude).exists() {
            missing += "\ntests/rust/";
            missing += exclude;
        }
    }
    if !missing.is_empty() {
        panic!("excluded test file does not exist:{}\n", missing);
    }
}

fn download_and_unpack() -> Result<()> {
    let url = format!(
        "https://github.com/rust-lang/rust/archive/{}.tar.gz",
        REVISION
    );
    let response = reqwest::blocking::get(&url)?.error_for_status()?;
    let progress = Progress::new(response);
    let decoder = GzDecoder::new(progress);
    let mut archive = Archive::new(decoder);
    let prefix = format!("rust-{}", REVISION);

    let tests_rust = Path::new("tests/rust");
    if tests_rust.exists() {
        fs::remove_dir_all(tests_rust)?;
    }

    for entry in archive.entries()? {
        let mut entry = entry?;
        let path = entry.path()?;
        if path == Path::new("pax_global_header") {
            continue;
        }
        let relative = path.strip_prefix(&prefix)?;
        let out = tests_rust.join(relative);
        entry.unpack(&out)?;
    }

    fs::write("tests/rust/COMMIT", REVISION)?;
    Ok(())
}
<|endoftext|>


<commit_before>extern crate rz80;
extern crate time;

#[cfg(test)]
mod test_zex {
    use time::PreciseTime;
    use rz80;
    use rz80::RegT;

    static ZEXDOC: &'static [u8] = include_bytes!("zexdoc.com");
    static ZEXALL: &'static [u8] = include_bytes!("zexall.com");

    // emulates a CP/M BDOS call, only what's needed by ZEX
    fn cpm_bdos<I,O>(cpu: &mut rz80::CPU<I,O>)
        where I: FnMut(RegT)->RegT,
              O: FnMut(RegT, RegT) {

        match cpu.reg.c() {
            2 => {
                // output a character
                print!("{}", cpu.reg.e() as u8 as char);
            },
            9 => {
                // output a string
                let mut addr = cpu.reg.de();
                loop {
                    let c = cpu.mem.r8(addr) as u8 as char;
                    addr = (addr + 1) & 0xFFFF;
                    if c != '$' {
                        print!("{}", c);
                    }
                    else {
                        break;
                    }
                }
            },
            _ => {
                panic!("Unknown CP/M call {}!", cpu.reg.c());
            }
        }
        // emulate a RET
        let sp = cpu.reg.sp();
        cpu.reg.set_pc(cpu.mem.r16(sp));
        cpu.reg.set_sp(sp + 2);
    }

    fn dummy_in(_: RegT) -> RegT { 0 }
    fn dummy_out(_: RegT, _: RegT) { }

    fn run_test(prog: &[u8]) -> (i64, i64) {
        let mut num_ops = 0;
        let mut num_cycles = 0;
        let mut cpu = rz80::CPU::new(dummy_in, dummy_out);
        cpu.mem.write(0x0100, prog);
        cpu.reg.set_sp(0xF000);
        cpu.reg.set_pc(0x0100);
        loop {
            num_ops += 1;
            num_cycles += cpu.step();
            match cpu.reg.pc() {
                0x0005 => { cpm_bdos(&mut cpu); },  // emulated CP/M BDOS call
                0x0000 => { break; },
                _ => { },
            }
        }
        (num_ops, num_cycles)
    }

    fn test_zexdoc() {
        println!(">>> RUNNING ZEXDOC");

        let start = PreciseTime::now();
        let (num_ops, num_cycles) = run_test(&ZEXDOC);
        let end = PreciseTime::now();
        let ms = start.to(end).num_milliseconds();
        let mips = (num_ops / ms)/1000;
        let mhz  = num_cycles / ms;

        println!("\n\nops: {}, cycles: {}, duration: {}ms", num_ops, num_cycles, ms);
        println!("mips: {}, MHz: {}\n\n", mips, mhz);
    }

    fn test_zexall() {
        println!(">>> RUNNING ZEXALL");

        let start = PreciseTime::now();
        let (num_ops, num_cycles) = run_test(&ZEXALL);
        let end = PreciseTime::now();
        let ms = start.to(end).num_milliseconds();
        let mips = (num_ops / ms)/1000;
        let mhz  = num_cycles / ms;

        println!("\n\nops: {}, cycles: {}, duration: {}ms", num_ops, num_cycles, ms);
        println!("mips: {}, MHz: {}", mips, mhz);
    }

    #[test]
    #[ignore]
    fn test_zex() {
        // have 1 test function run both sub-tests, we don't want to
        // run them in parallel
        test_zexdoc();
        test_zexall();
    }
}

<commit_msg>Fixed the MHz display in the ZEX tests<commit_after>extern crate rz80;
extern crate time;

#[cfg(test)]
mod test_zex {
    use time::PreciseTime;
    use rz80;
    use rz80::RegT;

    static ZEXDOC: &'static [u8] = include_bytes!("zexdoc.com");
    static ZEXALL: &'static [u8] = include_bytes!("zexall.com");

    // emulates a CP/M BDOS call, only what's needed by ZEX
    fn cpm_bdos<I,O>(cpu: &mut rz80::CPU<I,O>)
        where I: FnMut(RegT)->RegT,
              O: FnMut(RegT, RegT) {

        match cpu.reg.c() {
            2 => {
                // output a character
                print!("{}", cpu.reg.e() as u8 as char);
            },
            9 => {
                // output a string
                let mut addr = cpu.reg.de();
                loop {
                    let c = cpu.mem.r8(addr) as u8 as char;
                    addr = (addr + 1) & 0xFFFF;
                    if c != '$' {
                        print!("{}", c);
                    }
                    else {
                        break;
                    }
                }
            },
            _ => {
                panic!("Unknown CP/M call {}!", cpu.reg.c());
            }
        }
        // emulate a RET
        let sp = cpu.reg.sp();
        cpu.reg.set_pc(cpu.mem.r16(sp));
        cpu.reg.set_sp(sp + 2);
    }

    fn dummy_in(_: RegT) -> RegT { 0 }
    fn dummy_out(_: RegT, _: RegT) { }

    fn run_test(prog: &[u8]) -> (i64, i64) {
        let mut num_ops = 0;
        let mut num_cycles = 0;
        let mut cpu = rz80::CPU::new(dummy_in, dummy_out);
        cpu.mem.write(0x0100, prog);
        cpu.reg.set_sp(0xF000);
        cpu.reg.set_pc(0x0100);
        loop {
            num_ops += 1;
            num_cycles += cpu.step();
            match cpu.reg.pc() {
                0x0005 => { cpm_bdos(&mut cpu); },  // emulated CP/M BDOS call
                0x0000 => { break; },
                _ => { },
            }
        }
        (num_ops, num_cycles)
    }

    fn test_zexdoc() {
        println!(">>> RUNNING ZEXDOC");

        let start = PreciseTime::now();
        let (num_ops, num_cycles) = run_test(&ZEXDOC);
        let end = PreciseTime::now();
        let ms = start.to(end).num_milliseconds();
        let mips = (num_ops / ms)/1000;
        let mhz  = num_cycles / ms;

        println!("\n\nops: {}, cycles: {}, duration: {}ms", num_ops, num_cycles, ms);
        println!("mips: {}, MHz: {}\n\n", mips, mhz);
    }

    fn test_zexall() {
        println!(">>> RUNNING ZEXALL");

        let start = PreciseTime::now();
        let (num_ops, num_cycles) = run_test(&ZEXALL);
        let end = PreciseTime::now();
        let ms = start.to(end).num_milliseconds();
        let mips = (num_ops / ms)/1000;
        let mhz  = (num_cycles / ms)/1000;

        println!("\n\nops: {}, cycles: {}, duration: {}ms", num_ops, num_cycles, ms);
        println!("mips: {}, MHz: {}", mips, mhz);
    }

    #[test]
    #[ignore]
    fn test_zex() {
        // have 1 test function run both sub-tests, we don't want to
        // run them in parallel
        test_zexdoc();
        test_zexall();
    }
}

<|endoftext|>


<commit_before>// Copyright (C) 2019 Inderjit Gill

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::colour::Colour;
use crate::matrix::Matrix;

pub mod bezier;
pub mod bezier_bulging;
pub mod circle;
pub mod circle_slice;
pub mod line;
pub mod poly;
pub mod quadratic;
pub mod rect;
pub mod stroked_bezier;
pub mod stroked_bezier_rect;

// todo: work out reasonable defaults
const RENDER_PACKET_MAX_SIZE: usize = 40096;
pub const RENDER_PACKET_FLOAT_PER_VERTEX: usize = 8;

pub struct RenderPacket {
    pub geo: Vec<f32>,
}

impl Default for RenderPacket {
    fn default() -> RenderPacket {
        RenderPacket {
            geo: Vec::with_capacity(RENDER_PACKET_MAX_SIZE),
        }
    }
}

impl RenderPacket {
    pub fn get_geo_len(&self) -> usize {
        self.geo.len()
    }

    pub fn get_geo_ptr(&self) -> *const f32 {
        self.geo.as_ptr() as *const f32
    }

    pub fn add_vertex(&mut self, matrix: &Matrix, x: f32, y: f32, col: &Colour, u: f32, v: f32) {
        // assuming that col is ColourFormat::Rgb

        let (nx, ny) = matrix.transform_vec2(x, y);

        // pre-multiply the alpha
        // see http://www.realtimerendering.com/blog/gpus-prefer-premultiplication/
        self.geo.append(&mut vec![
            nx,
            ny,
            col.e0 * col.e3,
            col.e1 * col.e3,
            col.e2 * col.e3,
            col.e3,
            u,
            v,
        ]);
    }

    pub fn form_degenerate_triangle(&mut self, matrix: &Matrix, x: f32, y: f32) {
        // just copy the previous entries
        self.dup();

        // add the new vertex to complete the degenerate triangle
        self.add_vertex(matrix, x, y, &Colour::default(), 0.0, 0.0);

        // Note: still need to call addVertex on the first
        // vertex when we 'really' render the strip
    }

    // duplicate the last geometry point
    pub fn dup(&mut self) {
        let len = self.geo.len();

        let x: f32;
        let y: f32;
        let r: f32;
        let g: f32;
        let b: f32;
        let a: f32;
        let u: f32;
        let v: f32;
        {
            x = self.geo[len - 8];
            y = self.geo[len - 7];
            r = self.geo[len - 6];
            g = self.geo[len - 5];
            b = self.geo[len - 4];
            a = self.geo[len - 3];
            u = self.geo[len - 2];
            v = self.geo[len - 1];
        }

        self.geo.append(&mut vec![x, y, r, g, b, a, u, v]);
    }

    pub fn can_vertices_fit(&self, num: usize) -> bool {
        self.geo.len() < (RENDER_PACKET_MAX_SIZE - (num * RENDER_PACKET_FLOAT_PER_VERTEX))
    }

    pub fn is_empty(&self) -> bool {
        self.geo.is_empty()
    }
}

pub struct Geometry {
    pub render_packets: Vec<RenderPacket>,
}

impl Default for Geometry {
    fn default() -> Geometry {
        let mut render_packets: Vec<RenderPacket> = Vec::new();
        render_packets.push(Default::default());

        Geometry { render_packets }
    }
}

impl Geometry {
    pub fn reset(&mut self) {
        self.render_packets.clear();
        self.render_packets.push(Default::default())
    }

    pub fn get_render_packet_geo_len(&self, packet_number: usize) -> usize {
        let rp = &self.render_packets[packet_number];
        rp.geo.len()
    }

    pub fn get_render_packet_geo_ptr(&self, packet_number: usize) -> *const f32 {
        let rp = &self.render_packets[packet_number];
        rp.geo.as_ptr() as *const f32
    }

    pub fn get_num_render_packets(&self) -> usize {
        self.render_packets.len()
    }

    pub fn prepare_to_add_triangle_strip(
        &mut self,
        matrix: &Matrix,
        num_vertices: usize,
        x: f32,
        y: f32,
    ) {
        let mut last = self.render_packets.len() - 1;
        let mut rp = &mut self.render_packets[last];
        if !rp.can_vertices_fit(num_vertices) {
            self.render_packets.push(Default::default());
            last += 1;
        }

        rp = &mut self.render_packets[last];
        if !rp.is_empty() {
            rp.form_degenerate_triangle(matrix, x, y);
        }
    }
}
<commit_msg>larger RENDER_PACKET_MAX_SIZE<commit_after>// Copyright (C) 2019 Inderjit Gill

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::colour::Colour;
use crate::matrix::Matrix;

pub mod bezier;
pub mod bezier_bulging;
pub mod circle;
pub mod circle_slice;
pub mod line;
pub mod poly;
pub mod quadratic;
pub mod rect;
pub mod stroked_bezier;
pub mod stroked_bezier_rect;

const RENDER_PACKET_MAX_SIZE: usize = 262144;
pub const RENDER_PACKET_FLOAT_PER_VERTEX: usize = 8;
// 262144 * 4 == 1MB per render packet
// 262144 / 8 == 32768 vertices per render packet

pub struct RenderPacket {
    pub geo: Vec<f32>,
}

impl Default for RenderPacket {
    fn default() -> RenderPacket {
        RenderPacket {
            geo: Vec::with_capacity(RENDER_PACKET_MAX_SIZE),
        }
    }
}

impl RenderPacket {
    pub fn get_geo_len(&self) -> usize {
        self.geo.len()
    }

    pub fn get_geo_ptr(&self) -> *const f32 {
        self.geo.as_ptr() as *const f32
    }

    pub fn add_vertex(&mut self, matrix: &Matrix, x: f32, y: f32, col: &Colour, u: f32, v: f32) {
        // assuming that col is ColourFormat::Rgb

        let (nx, ny) = matrix.transform_vec2(x, y);

        // pre-multiply the alpha
        // see http://www.realtimerendering.com/blog/gpus-prefer-premultiplication/
        self.geo.append(&mut vec![
            nx,
            ny,
            col.e0 * col.e3,
            col.e1 * col.e3,
            col.e2 * col.e3,
            col.e3,
            u,
            v,
        ]);
    }

    pub fn form_degenerate_triangle(&mut self, matrix: &Matrix, x: f32, y: f32) {
        // just copy the previous entries
        self.dup();

        // add the new vertex to complete the degenerate triangle
        self.add_vertex(matrix, x, y, &Colour::default(), 0.0, 0.0);

        // Note: still need to call addVertex on the first
        // vertex when we 'really' render the strip
    }

    // duplicate the last geometry point
    pub fn dup(&mut self) {
        let len = self.geo.len();

        let x: f32;
        let y: f32;
        let r: f32;
        let g: f32;
        let b: f32;
        let a: f32;
        let u: f32;
        let v: f32;
        {
            x = self.geo[len - 8];
            y = self.geo[len - 7];
            r = self.geo[len - 6];
            g = self.geo[len - 5];
            b = self.geo[len - 4];
            a = self.geo[len - 3];
            u = self.geo[len - 2];
            v = self.geo[len - 1];
        }

        self.geo.append(&mut vec![x, y, r, g, b, a, u, v]);
    }

    pub fn can_vertices_fit(&self, num: usize) -> bool {
        self.geo.len() < (RENDER_PACKET_MAX_SIZE - (num * RENDER_PACKET_FLOAT_PER_VERTEX))
    }

    pub fn is_empty(&self) -> bool {
        self.geo.is_empty()
    }
}

pub struct Geometry {
    pub render_packets: Vec<RenderPacket>,
}

impl Default for Geometry {
    fn default() -> Geometry {
        let mut render_packets: Vec<RenderPacket> = Vec::new();
        render_packets.push(Default::default());

        Geometry { render_packets }
    }
}

impl Geometry {
    pub fn reset(&mut self) {
        self.render_packets.clear();
        self.render_packets.push(Default::default())
    }

    pub fn get_render_packet_geo_len(&self, packet_number: usize) -> usize {
        let rp = &self.render_packets[packet_number];
        rp.geo.len()
    }

    pub fn get_render_packet_geo_ptr(&self, packet_number: usize) -> *const f32 {
        let rp = &self.render_packets[packet_number];
        rp.geo.as_ptr() as *const f32
    }

    pub fn get_num_render_packets(&self) -> usize {
        self.render_packets.len()
    }

    pub fn prepare_to_add_triangle_strip(
        &mut self,
        matrix: &Matrix,
        num_vertices: usize,
        x: f32,
        y: f32,
    ) {
        let mut last = self.render_packets.len() - 1;
        let mut rp = &mut self.render_packets[last];
        if !rp.can_vertices_fit(num_vertices) {
            self.render_packets.push(Default::default());
            last += 1;
        }

        rp = &mut self.render_packets[last];
        if !rp.is_empty() {
            rp.form_degenerate_triangle(matrix, x, y);
        }
    }
}
<|endoftext|>


<commit_before>#[feature(globs)];
#[feature(struct_variant)];
#[feature(macro_rules)];
extern mod std;
extern mod extra;
use extra::getopts;
use basics::*;
use std::io::File;

mod basics;
mod policies;
mod raft;
mod sim;

/* features to support selectmatch */

/* SelectBox is a wrapper to allow calling select() on pointers */
struct SelectBox<'self> {
    inner: &'self mut std::select::Select,
}

impl<'self> std::rt::shouldnt_be_public::SelectInner for SelectBox<'self> {
    fn optimistic_check(&mut self) -> bool {
        self.inner.optimistic_check()
    }
    fn block_on(&mut self, sched: &mut std::rt::sched::Scheduler, task: std::rt::BlockedTask) -> bool {
        self.inner.block_on(sched, task)
    }
    fn unblock_from(&mut self) -> bool {
        self.inner.unblock_from()
    }
}

impl<'self> std::select::Select for SelectBox<'self> {
}

/* recursive helper for selectmatch macro */
macro_rules! selectmatch_rec(
    ($index:expr,
     $port:expr => $action:expr,
     $( $port_rest:expr => $action_rest:expr ),+) => (
        if $index == 0 {
            $action
        } else {
            $index -= 1;
            selectmatch_rec!($index, $( $port_rest => $action_rest ),*)
        }
    );
    ($index:expr, $port:expr => $action:expr) => (
        if $index == 0 {
            $action
        } else {
            fail!("Bogus return value from select()");
        }
    );
)

/* bearable syntax for calling select with SelectBox */
macro_rules! selectmatch(
     ($( $port:expr => $action:expr ),+) => (
        {
            let mut selectmatch_i : uint = std::select::select([$( SelectBox{ inner: &mut $port as &mut std::select::Select } ),+]);
            selectmatch_rec!(selectmatch_i, $( $port => $action ),+)
        }
    );
)

/* end selectmatch */

fn main() {
    let args = std::os::args();
    let opts = [
        getopts::optflag("h"),
        getopts::optflag("help"),
        getopts::optopt("algorithm"),
        getopts::optopt("cluster"),
        getopts::optopt("heartbeats"),
        getopts::optopt("logs"),
        getopts::optopt("maxticks"),
        getopts::optopt("samples"),
        getopts::optopt("tasks"),
        getopts::optopt("terms"),
        getopts::optopt("timeout"),
        getopts::optopt("timing"),
        getopts::optopt("trace"),
    ];
    let matches = match getopts::getopts(args.tail(), opts) {
        Ok(m) => { m },
        Err(f) => { fail!(f.to_err_msg()) },
    };
    let usage = || {
        println!("Usage: {} [options]", args[0]);
        println!("--algorithm=ALGO Algorithm variant (default submission)");
        println!("--cluster=POLICY Type/size of cluster (default 5)");
        println!("-h, --help       Print this help message");
        println!("--heartbeats=N   Number of heartbeats till leader ");
        println!("                 considered stable (default 16)");
        println!("--logs=POLICY    Log length policy (default same)");
        println!("--maxticks=N     Number of simulation ticks after which ");
        println!("                 to stop a sample (default 5,000,000)");
        println!("--samples=N      Number of simulations (default 10,000)");
        println!("--tasks=N        Number of parallel jobs (default {})",
                 std::rt::default_sched_threads() - 1);
        println!("--terms=POLICY   Initial terms policy (default same)");
        println!("--timeout=MS     Milliseconds after which to stop");
        println!("                 (default 0 meaning infinity)");
        println!("--timing=POLICY  Network timing policy (default LAN)");
        println!("--trace=N         Dump N simulation traces to files (default 0)");
    };
    if matches.opt_present("h") || matches.opt_present("help") {
        usage();
        return;
    }
    if !matches.free.is_empty() {
        usage();
        fail!(format!("Extra arguments: {}", matches.free.len()));
    }
    let algorithm : ~str = match matches.opt_str("algorithm") {
        Some(s) => { s },
        None => { ~"submission" },
    };
    let cluster : ~str = match matches.opt_str("cluster") {
        Some(s) => { s },
        None => { ~"5" },
    };
    let heartbeats : uint = match matches.opt_str("heartbeats") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of heartbeats from '{}'", s));
            },
        }},
        None => { 16 },
    };
    let log_length : ~str = match matches.opt_str("logs") {
        Some(s) => { s },
        None => { ~"same" },
    };
    let max_ticks : Time = match matches.opt_str("maxticks") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { Time(i) },
            None => {
                usage();
                fail!(format!("Couldn't parse max number of ticks from '{}'", s));
            },
        }},
        None => { Time(5000000) },
    };
    let num_samples : uint = match matches.opt_str("samples") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of samples from '{}'", s));
            },
        }},
        None => { 10000 },
    };
    let num_tasks : uint = match matches.opt_str("tasks") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of tasks from '{}'", s));
            },
        }},
        None => { std::rt::default_sched_threads() - 1 },
    };
    let terms : ~str = match matches.opt_str("terms") {
        Some(s) => { s },
        None => { ~"same" },
    };
    let timeout : uint = match matches.opt_str("timeout") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse timeout from '{}'", s));
            },
        }},
        None => { 0 },
    };
    let timeout = if timeout == 0 {
        !0
    } else {
        timeout
    };
    let timing : ~str = match matches.opt_str("timing") {
        Some(s) => { s },
        None => { ~"LAN" },
    };
    let trace : uint = match matches.opt_str("trace") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of traces from '{}'", s));
            },
        }},
        None => { 0 },
    };

    let mut meta : ~[(~str, ~str)] = ~[];
    meta.push((~"algorithm",  algorithm.clone()));
    meta.push((~"log_length", log_length.clone()));
    meta.push((~"cluster",    cluster.clone()));
    meta.push((~"heartbeats", format!("{}", heartbeats)));
    meta.push((~"maxticks",   format!("{}", max_ticks)));
    meta.push((~"tasks",      format!("{}", num_tasks)));
    meta.push((~"terms",      terms.clone()));
    meta.push((~"timing",     timing.clone()));
    meta.push((~"trials_req", format!("{}", num_samples)));
    meta.push((~"timeout",    format!("{}", timeout)));

    println!("Algorithm:  {}", algorithm)
    println!("Cluster:    {}", cluster)
    println!("Heartbeats: {}", heartbeats)
    println!("Log length: {}", log_length)
    println!("Max ticks:  {}", max_ticks)
    println!("Tasks:      {}", num_tasks)
    println!("Terms:      {}", terms)
    println!("Timing:     {}", timing)
    println!("Req. trials:{}", num_samples)
    println!("Timeout:    {} ms", timeout)

    let start_ns = extra::time::precise_time_ns();

    let mut signals = std::io::signal::Listener::new();
    signals.register(std::io::signal::Interrupt);

    let sim_opts = sim::SimOpts {
       cluster: cluster,
       heartbeats: heartbeats,
       timing: timing,
       log_length: log_length,
       algorithm: algorithm,
       terms: terms,
       max_ticks: max_ticks,
       trace: trace,
    };
    let samples = if num_tasks <= 1 {
        run_task(std::iter::range_step(0, num_samples, 1), &sim_opts, &signals.port)
    } else {
        let (mut port, chan): (std::comm::Port<~[Sample]>, std::comm::Chan<~[Sample]>) = stream();
        let chan = std::comm::SharedChan::new(chan);
        let mut exit_chans : ~[std::comm::Chan<()>] = ~[];
        let exit = || {
            for c in exit_chans.iter() {
                c.send(());
            }
        };
        for tid in range(0, num_tasks) {
            let (exit_port, exit_chan): (std::comm::Port<()>, std::comm::Chan<()>) = stream();
            exit_chans.push(exit_chan);
            let child_chan = chan.clone();
            let child_opts = sim_opts.clone();
            do spawn {
                let runs = std::iter::range_step(tid, num_samples, num_tasks);
                child_chan.send(run_task(runs, &child_opts, &exit_port));
            }
        }

        let mut samples = ~[];
        samples.reserve(num_samples);
        let mut tasks_outstanding = num_tasks;
        let mut timeout_timer = std::io::timer::Timer::new().unwrap();
        // simulate a one_shot timer with a periodic timer and a boolean, since
        // I can't get the types to work otherwise
        let mut timeout_port = timeout_timer.periodic(timeout as u64);
        let mut timed_out = false;
        while tasks_outstanding > 0 {
            selectmatch!(
                signals.port.x => {
                    match signals.port.recv() {
                        std::io::signal::Interrupt => {
                            println!("\nCtrl-C: exiting");
                        },
                        _ => fail!("Unexpected signal"),
                    }
                },
                timeout_port.x => {
                    timeout_port.recv();
                    if !timed_out {
                        println!("Timeout");
                        timed_out = true;
                        exit();
                    }
                },
                port.x => {
                    tasks_outstanding -= 1;
                    samples.push_all(port.recv());
                }
            )
        }
        sort_samples(&mut samples);
        samples
    };

    let end_ns = extra::time::precise_time_ns();
    let elapsed_ns = end_ns - start_ns;
    meta.push((~"wall",   format!("{}", elapsed_ns as f64 / 1e9)));
    meta.push((~"trials", format!("{}", samples.len())));
    meta.push((~"min",    format!("{}", samples[0].ticks)));
    meta.push((~"median", format!("{}", samples[samples.len()/2].ticks)));
    meta.push((~"max",    format!("{}", samples[samples.len()-1].ticks)));

    println!("Trials:     {}",       samples.len());
    println!("Min:        {} ticks", samples[0].ticks);
    println!("Median:     {} ticks", samples[samples.len()/2].ticks);
    println!("Max:        {} ticks", samples[samples.len()-1].ticks);
    println!("Wall:       {:.2f} s", elapsed_ns as f64 / 1e9);

    let metaf = &mut File::create(&Path::new("meta.csv")).unwrap() as &mut Writer;
    for &(ref k, ref _v) in meta.iter() {
        write!(metaf, "{}, ", *k);
    }
    write!(metaf, "\n");
    for &(ref _k, ref v) in meta.iter() {
        write!(metaf, "{}, ", *v);
    }
    write!(metaf, "\n");

    let runf = &mut File::create(&Path::new("samples.csv")).unwrap() as &mut Writer;
    let runf = &mut std::io::buffered::BufferedWriter::new(runf) as &mut Writer;
    writeln!(runf, "election_time,run");
    for sample in samples.iter() {
        writeln!(runf, "{},{}", sample.ticks, sample.run);
    }
    runf.flush();
}

#[deriving(Clone)]
struct Sample {
    run: uint,
    ticks: Time,
}

fn sort_samples(samples: &mut ~[Sample]) {
    // merge sort seems to greatly outperform quicksort for about 2500 samples
    *samples = extra::sort::merge_sort(*samples, |x,y| x.ticks <= y.ticks);
}

fn run_task<T: Send>(mut runs: std::iter::RangeStep<uint>, opts: &sim::SimOpts, exit_port: &Port<T>) -> ~[Sample] {
    let mut samples = ~[];
    samples.reserve(runs.size_hint().first());
    for run in runs {
        if exit_port.peek() {
            return samples;
        }
        let sample = Sample {
            run: run,
            ticks: sim::simulate(run, opts),
        };
        samples.push(sample);
    }
    return samples;
}
<commit_msg>Add mean to simulator output<commit_after>#[feature(globs)];
#[feature(struct_variant)];
#[feature(macro_rules)];
extern mod std;
extern mod extra;
use extra::getopts;
use basics::*;
use std::io::File;

mod basics;
mod policies;
mod raft;
mod sim;

/* features to support selectmatch */

/* SelectBox is a wrapper to allow calling select() on pointers */
struct SelectBox<'self> {
    inner: &'self mut std::select::Select,
}

impl<'self> std::rt::shouldnt_be_public::SelectInner for SelectBox<'self> {
    fn optimistic_check(&mut self) -> bool {
        self.inner.optimistic_check()
    }
    fn block_on(&mut self, sched: &mut std::rt::sched::Scheduler, task: std::rt::BlockedTask) -> bool {
        self.inner.block_on(sched, task)
    }
    fn unblock_from(&mut self) -> bool {
        self.inner.unblock_from()
    }
}

impl<'self> std::select::Select for SelectBox<'self> {
}

/* recursive helper for selectmatch macro */
macro_rules! selectmatch_rec(
    ($index:expr,
     $port:expr => $action:expr,
     $( $port_rest:expr => $action_rest:expr ),+) => (
        if $index == 0 {
            $action
        } else {
            $index -= 1;
            selectmatch_rec!($index, $( $port_rest => $action_rest ),*)
        }
    );
    ($index:expr, $port:expr => $action:expr) => (
        if $index == 0 {
            $action
        } else {
            fail!("Bogus return value from select()");
        }
    );
)

/* bearable syntax for calling select with SelectBox */
macro_rules! selectmatch(
     ($( $port:expr => $action:expr ),+) => (
        {
            let mut selectmatch_i : uint = std::select::select([$( SelectBox{ inner: &mut $port as &mut std::select::Select } ),+]);
            selectmatch_rec!(selectmatch_i, $( $port => $action ),+)
        }
    );
)

/* end selectmatch */

fn main() {
    let args = std::os::args();
    let opts = [
        getopts::optflag("h"),
        getopts::optflag("help"),
        getopts::optopt("algorithm"),
        getopts::optopt("cluster"),
        getopts::optopt("heartbeats"),
        getopts::optopt("logs"),
        getopts::optopt("maxticks"),
        getopts::optopt("samples"),
        getopts::optopt("tasks"),
        getopts::optopt("terms"),
        getopts::optopt("timeout"),
        getopts::optopt("timing"),
        getopts::optopt("trace"),
    ];
    let matches = match getopts::getopts(args.tail(), opts) {
        Ok(m) => { m },
        Err(f) => { fail!(f.to_err_msg()) },
    };
    let usage = || {
        println!("Usage: {} [options]", args[0]);
        println!("--algorithm=ALGO Algorithm variant (default submission)");
        println!("--cluster=POLICY Type/size of cluster (default 5)");
        println!("-h, --help       Print this help message");
        println!("--heartbeats=N   Number of heartbeats till leader ");
        println!("                 considered stable (default 16)");
        println!("--logs=POLICY    Log length policy (default same)");
        println!("--maxticks=N     Number of simulation ticks after which ");
        println!("                 to stop a sample (default 5,000,000)");
        println!("--samples=N      Number of simulations (default 10,000)");
        println!("--tasks=N        Number of parallel jobs (default {})",
                 std::rt::default_sched_threads() - 1);
        println!("--terms=POLICY   Initial terms policy (default same)");
        println!("--timeout=MS     Milliseconds after which to stop");
        println!("                 (default 0 meaning infinity)");
        println!("--timing=POLICY  Network timing policy (default LAN)");
        println!("--trace=N         Dump N simulation traces to files (default 0)");
    };
    if matches.opt_present("h") || matches.opt_present("help") {
        usage();
        return;
    }
    if !matches.free.is_empty() {
        usage();
        fail!(format!("Extra arguments: {}", matches.free.len()));
    }
    let algorithm : ~str = match matches.opt_str("algorithm") {
        Some(s) => { s },
        None => { ~"submission" },
    };
    let cluster : ~str = match matches.opt_str("cluster") {
        Some(s) => { s },
        None => { ~"5" },
    };
    let heartbeats : uint = match matches.opt_str("heartbeats") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of heartbeats from '{}'", s));
            },
        }},
        None => { 16 },
    };
    let log_length : ~str = match matches.opt_str("logs") {
        Some(s) => { s },
        None => { ~"same" },
    };
    let max_ticks : Time = match matches.opt_str("maxticks") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { Time(i) },
            None => {
                usage();
                fail!(format!("Couldn't parse max number of ticks from '{}'", s));
            },
        }},
        None => { Time(5000000) },
    };
    let num_samples : uint = match matches.opt_str("samples") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of samples from '{}'", s));
            },
        }},
        None => { 10000 },
    };
    let num_tasks : uint = match matches.opt_str("tasks") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of tasks from '{}'", s));
            },
        }},
        None => { std::rt::default_sched_threads() - 1 },
    };
    let terms : ~str = match matches.opt_str("terms") {
        Some(s) => { s },
        None => { ~"same" },
    };
    let timeout : uint = match matches.opt_str("timeout") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse timeout from '{}'", s));
            },
        }},
        None => { 0 },
    };
    let timeout = if timeout == 0 {
        !0
    } else {
        timeout
    };
    let timing : ~str = match matches.opt_str("timing") {
        Some(s) => { s },
        None => { ~"LAN" },
    };
    let trace : uint = match matches.opt_str("trace") {
        Some(s) => { match std::from_str::from_str(s) {
            Some(i) => { i },
            None => {
                usage();
                fail!(format!("Couldn't parse number of traces from '{}'", s));
            },
        }},
        None => { 0 },
    };

    let mut meta : ~[(~str, ~str)] = ~[];
    meta.push((~"algorithm",  algorithm.clone()));
    meta.push((~"log_length", log_length.clone()));
    meta.push((~"cluster",    cluster.clone()));
    meta.push((~"heartbeats", format!("{}", heartbeats)));
    meta.push((~"maxticks",   format!("{}", max_ticks)));
    meta.push((~"tasks",      format!("{}", num_tasks)));
    meta.push((~"terms",      terms.clone()));
    meta.push((~"timing",     timing.clone()));
    meta.push((~"trials_req", format!("{}", num_samples)));
    meta.push((~"timeout",    format!("{}", timeout)));

    println!("Algorithm:  {}", algorithm)
    println!("Cluster:    {}", cluster)
    println!("Heartbeats: {}", heartbeats)
    println!("Log length: {}", log_length)
    println!("Max ticks:  {}", max_ticks)
    println!("Tasks:      {}", num_tasks)
    println!("Terms:      {}", terms)
    println!("Timing:     {}", timing)
    println!("Req. trials:{}", num_samples)
    println!("Timeout:    {} ms", timeout)

    let start_ns = extra::time::precise_time_ns();

    let mut signals = std::io::signal::Listener::new();
    signals.register(std::io::signal::Interrupt);

    let sim_opts = sim::SimOpts {
       cluster: cluster,
       heartbeats: heartbeats,
       timing: timing,
       log_length: log_length,
       algorithm: algorithm,
       terms: terms,
       max_ticks: max_ticks,
       trace: trace,
    };
    let samples = if num_tasks <= 1 {
        run_task(std::iter::range_step(0, num_samples, 1), &sim_opts, &signals.port)
    } else {
        let (mut port, chan): (std::comm::Port<~[Sample]>, std::comm::Chan<~[Sample]>) = stream();
        let chan = std::comm::SharedChan::new(chan);
        let mut exit_chans : ~[std::comm::Chan<()>] = ~[];
        let exit = || {
            for c in exit_chans.iter() {
                c.send(());
            }
        };
        for tid in range(0, num_tasks) {
            let (exit_port, exit_chan): (std::comm::Port<()>, std::comm::Chan<()>) = stream();
            exit_chans.push(exit_chan);
            let child_chan = chan.clone();
            let child_opts = sim_opts.clone();
            do spawn {
                let runs = std::iter::range_step(tid, num_samples, num_tasks);
                child_chan.send(run_task(runs, &child_opts, &exit_port));
            }
        }

        let mut samples = ~[];
        samples.reserve(num_samples);
        let mut tasks_outstanding = num_tasks;
        let mut timeout_timer = std::io::timer::Timer::new().unwrap();
        // simulate a one_shot timer with a periodic timer and a boolean, since
        // I can't get the types to work otherwise
        let mut timeout_port = timeout_timer.periodic(timeout as u64);
        let mut timed_out = false;
        while tasks_outstanding > 0 {
            selectmatch!(
                signals.port.x => {
                    match signals.port.recv() {
                        std::io::signal::Interrupt => {
                            println!("\nCtrl-C: exiting");
                        },
                        _ => fail!("Unexpected signal"),
                    }
                },
                timeout_port.x => {
                    timeout_port.recv();
                    if !timed_out {
                        println!("Timeout");
                        timed_out = true;
                        exit();
                    }
                },
                port.x => {
                    tasks_outstanding -= 1;
                    samples.push_all(port.recv());
                }
            )
        }
        sort_samples(&mut samples);
        samples
    };

    let end_ns = extra::time::precise_time_ns();
    let elapsed_ns = end_ns - start_ns;
    let mean_ticks = {
        let mut total = 0;
        for sample in samples.iter() {
            total += *sample.ticks;
        }
        total / samples.len()
    };

    meta.push((~"wall",   format!("{}", elapsed_ns as f64 / 1e9)));
    meta.push((~"trials", format!("{}", samples.len())));
    meta.push((~"min",    format!("{}", samples[0].ticks)));
    meta.push((~"median", format!("{}", samples[samples.len()/2].ticks)));
    meta.push((~"mean",   format!("{}", mean_ticks)));
    meta.push((~"max",    format!("{}", samples[samples.len()-1].ticks)));

    println!("Trials:     {}",       samples.len());
    println!("Min:        {} ticks", samples[0].ticks);
    println!("Median:     {} ticks", samples[samples.len()/2].ticks);
    println!("Mean:       {} ticks", mean_ticks);
    println!("Max:        {} ticks", samples[samples.len()-1].ticks);
    println!("Wall:       {:.2f} s", elapsed_ns as f64 / 1e9);

    let metaf = &mut File::create(&Path::new("meta.csv")).unwrap() as &mut Writer;
    for &(ref k, ref _v) in meta.iter() {
        write!(metaf, "{}, ", *k);
    }
    write!(metaf, "\n");
    for &(ref _k, ref v) in meta.iter() {
        write!(metaf, "{}, ", *v);
    }
    write!(metaf, "\n");

    let runf = &mut File::create(&Path::new("samples.csv")).unwrap() as &mut Writer;
    let runf = &mut std::io::buffered::BufferedWriter::new(runf) as &mut Writer;
    writeln!(runf, "election_time,run");
    for sample in samples.iter() {
        writeln!(runf, "{},{}", sample.ticks, sample.run);
    }
    runf.flush();
}

#[deriving(Clone)]
struct Sample {
    run: uint,
    ticks: Time,
}

fn sort_samples(samples: &mut ~[Sample]) {
    // merge sort seems to greatly outperform quicksort for about 2500 samples
    *samples = extra::sort::merge_sort(*samples, |x,y| x.ticks <= y.ticks);
}

fn run_task<T: Send>(mut runs: std::iter::RangeStep<uint>, opts: &sim::SimOpts, exit_port: &Port<T>) -> ~[Sample] {
    let mut samples = ~[];
    samples.reserve(runs.size_hint().first());
    for run in runs {
        if exit_port.peek() {
            return samples;
        }
        let sample = Sample {
            run: run,
            ticks: sim::simulate(run, opts),
        };
        samples.push(sample);
    }
    return samples;
}
<|endoftext|>


<commit_before>generate_error_module!(
    generate_error_types!(RefError, RefErrorKind,
        StoreReadError          => "Store read error",
        StoreWriteError         => "Store write error",
        IOError                 => "IO Error",
        HeaderTypeError         => "Header type error",
        HeaderFieldMissingError => "Header field missing error",
        HeaderFieldWriteError   => "Header field cannot be written",
        HeaderFieldReadError    => "Header field cannot be read",
        HeaderFieldAlreadyExistsError => "Header field already exists, cannot override",
        PathUTF8Error => "Path cannot be converted because of UTF8 Error",
        PathHashingError => "Path cannot be hashed",
        PathCanonicalizationError => "Path cannot be canonicalized",

        TypeConversionError => "Couldn't convert types",

        RefNotInStore => "Ref/StoreId does not exist in store",

        RefTargetDoesNotExist       => "Ref Target does not exist",
        RefTargetPermissionError    => "Ref Target permissions insufficient for referencing",
        RefTargetCannotBeHashed     => "Ref Target cannot be hashed (is it a directory?)",
        RefTargetFileCannotBeOpened => "Ref Target File cannot be open()ed",
        RefTargetCannotReadPermissions => "Ref Target: Cannot read permissions"
    );
);

pub use self::error::RefError;
pub use self::error::RefErrorKind;

<commit_msg>Add RefToDisplayError kind<commit_after>generate_error_module!(
    generate_error_types!(RefError, RefErrorKind,
        StoreReadError          => "Store read error",
        StoreWriteError         => "Store write error",
        IOError                 => "IO Error",
        HeaderTypeError         => "Header type error",
        HeaderFieldMissingError => "Header field missing error",
        HeaderFieldWriteError   => "Header field cannot be written",
        HeaderFieldReadError    => "Header field cannot be read",
        HeaderFieldAlreadyExistsError => "Header field already exists, cannot override",
        PathUTF8Error => "Path cannot be converted because of UTF8 Error",
        PathHashingError => "Path cannot be hashed",
        PathCanonicalizationError => "Path cannot be canonicalized",

        TypeConversionError => "Couldn't convert types",
        RefToDisplayError => "Cannot convert Ref to string to show it to user",

        RefNotInStore => "Ref/StoreId does not exist in store",

        RefTargetDoesNotExist       => "Ref Target does not exist",
        RefTargetPermissionError    => "Ref Target permissions insufficient for referencing",
        RefTargetCannotBeHashed     => "Ref Target cannot be hashed (is it a directory?)",
        RefTargetFileCannotBeOpened => "Ref Target File cannot be open()ed",
        RefTargetCannotReadPermissions => "Ref Target: Cannot read permissions"
    );
);

pub use self::error::RefError;
pub use self::error::RefErrorKind;
pub use self::error::MapErrInto;

<|endoftext|>