<commit_before>/// This module takes a (parsed) definition of `macro_rules` invocation, a
/// `tt::TokenTree` representing an argument of macro invocation, and produces a
/// `tt::TokenTree` for the result of the expansion.
use rustc_hash::FxHashMap;
use ra_syntax::SmolStr;
use tt::TokenId;

use crate::ExpandError;
use crate::tt_cursor::TtCursor;

pub(crate) fn expand(
    rules: &crate::MacroRules,
    input: &tt::Subtree,
) -> Result<tt::Subtree, ExpandError> {
    rules.rules.iter().find_map(|it| expand_rule(it, input).ok()).ok_or(ExpandError::NoMatchingRule)
}

fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
    let mut input = TtCursor::new(input);
    let bindings = match_lhs(&rule.lhs, &mut input)?;
    if !input.is_eof() {
        return Err(ExpandError::UnexpectedToken);
    }
    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
}

/// The actual algorithm for expansion is not too hard, but is pretty tricky.
/// `Bindings` structure is the key to understanding what we are doing here.
///
/// On the high level, it stores mapping from meta variables to the bits of
/// syntax it should be substituted with. For example, if `$e:expr` is matched
/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
///
/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
///
/// ```not_rust
/// macro_rules! foo {
///     ($($ i:ident $($ e:expr),*);*) => {
///         $(fn $ i() { $($ e);*; })*
///     }
/// }
/// foo! { foo 1,2,3; bar 4,5,6 }
/// ```
///
/// Here, the `$i` meta variable is matched first with `foo` and then with
/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
///
/// To represent such "multi-mappings", we use a recursive structures: we map
/// variables not to values, but to *lists* of values or other lists (that is,
/// to the trees).
///
/// For the above example, the bindings would store
///
/// ```not_rust
/// i -> [foo, bar]
/// e -> [[1, 2, 3], [4, 5, 6]]
/// ```
///
/// We construct `Bindings` in the `match_lhs`. The interesting case is
/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
/// nesting structure.
///
/// The other side of the puzzle is `expand_subtree`, where we use the bindings
/// to substitute meta variables in the output template. When expanding, we
/// maintain a `nesting` stack of indices which tells us which occurrence from
/// the `Bindings` we should take. We push to the stack when we enter a
/// repetition.
///
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but an `&[usize]`.
#[derive(Debug, Default)]
struct Bindings {
    inner: FxHashMap<SmolStr, Binding>,
}

#[derive(Debug)]
enum Binding {
    Simple(tt::TokenTree),
    Nested(Vec<Binding>),
}

impl Bindings {
    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree, ExpandError> {
        let mut b = self
            .inner
            .get(name)
            .ok_or(ExpandError::BindingError(format!("could not find binding {}", name)))?;
        for &idx in nesting.iter() {
            b = match b {
                Binding::Simple(_) => break,
                Binding::Nested(bs) => bs.get(idx).ok_or(ExpandError::BindingError(format!(
                    "could not find nested binding {}",
                    name
                )))?,
            };
        }
        match b {
            Binding::Simple(it) => Ok(it),
            Binding::Nested(_) => Err(ExpandError::BindingError(format!(
                "expected simple binding, found nested binding {}",
                name
            ))),
        }
    }

    fn push_nested(&mut self, nested: Bindings) -> Result<(), ExpandError> {
        for (key, value) in nested.inner {
            if !self.inner.contains_key(&key) {
                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
            }
            match self.inner.get_mut(&key) {
                Some(Binding::Nested(it)) => it.push(value),
                _ => {
                    return Err(ExpandError::BindingError(format!(
                        "nested binding for {} not found",
                        key
                    )));
                }
            }
        }
        Ok(())
    }
}

fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
    let mut res = Bindings::default();
    for pat in pattern.token_trees.iter() {
        match pat {
            crate::TokenTree::Leaf(leaf) => match leaf {
                crate::Leaf::Var(crate::Var { text, kind }) => {
                    let kind = kind.clone().ok_or(ExpandError::UnexpectedToken)?;
                    match kind.as_str() {
                        "ident" => {
                            let ident =
                                input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(ident).into()),
                            );
                        }
                        _ => return Err(ExpandError::UnexpectedToken),
                    }
                }
                crate::Leaf::Punct(punct) => {
                    if input.eat_punct() != Some(punct) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                crate::Leaf::Ident(ident) => {
                    if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                _ => return Err(ExpandError::UnexpectedToken),
            },
            crate::TokenTree::Repeat(crate::Repeat { subtree, kind: _, separator }) => {
                while let Ok(nested) = match_lhs(subtree, input) {
                    res.push_nested(nested)?;
                    if let Some(separator) = *separator {
                        if !input.is_eof() {
                            if input.eat_punct().map(|p| p.char) != Some(separator) {
                                return Err(ExpandError::UnexpectedToken);
                            }
                        }
                    }
                }
            }
            _ => {}
        }
    }
    Ok(res)
}

fn expand_subtree(
    template: &crate::Subtree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::Subtree, ExpandError> {
    let token_trees = template
        .token_trees
        .iter()
        .map(|it| expand_tt(it, bindings, nesting))
        .collect::<Result<Vec<_>, ExpandError>>()?;

    Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
}

fn expand_tt(
    template: &crate::TokenTree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::TokenTree, ExpandError> {
    let res: tt::TokenTree = match template {
        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
        crate::TokenTree::Repeat(repeat) => {
            let mut token_trees = Vec::new();
            nesting.push(0);
            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
                let idx = nesting.pop().unwrap();
                nesting.push(idx + 1);
                token_trees.push(t.into())
            }
            nesting.pop().unwrap();
            tt::Subtree { token_trees, delimiter: tt::Delimiter::None }.into()
        }
        crate::TokenTree::Leaf(leaf) => match leaf {
            crate::Leaf::Ident(ident) => {
                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
                    .into()
            }
            crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
            crate::Leaf::Var(v) => bindings.get(&v.text, nesting)?.clone(),
            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
        },
    };
    Ok(res)
}
<commit_msg>Add expander unit tests<commit_after>/// This module takes a (parsed) definition of `macro_rules` invocation, a
/// `tt::TokenTree` representing an argument of macro invocation, and produces a
/// `tt::TokenTree` for the result of the expansion.
use rustc_hash::FxHashMap;
use ra_syntax::SmolStr;
use tt::TokenId;

use crate::ExpandError;
use crate::tt_cursor::TtCursor;

pub(crate) fn expand(
    rules: &crate::MacroRules,
    input: &tt::Subtree,
) -> Result<tt::Subtree, ExpandError> {
    rules.rules.iter().find_map(|it| expand_rule(it, input).ok()).ok_or(ExpandError::NoMatchingRule)
}

fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
    let mut input = TtCursor::new(input);
    let bindings = match_lhs(&rule.lhs, &mut input)?;
    if !input.is_eof() {
        return Err(ExpandError::UnexpectedToken);
    }
    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
}

/// The actual algorithm for expansion is not too hard, but is pretty tricky.
/// `Bindings` structure is the key to understanding what we are doing here.
///
/// On the high level, it stores mapping from meta variables to the bits of
/// syntax it should be substituted with. For example, if `$e:expr` is matched
/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
///
/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
///
/// ```not_rust
/// macro_rules! foo {
///     ($($ i:ident $($ e:expr),*);*) => {
///         $(fn $ i() { $($ e);*; })*
///     }
/// }
/// foo! { foo 1,2,3; bar 4,5,6 }
/// ```
///
/// Here, the `$i` meta variable is matched first with `foo` and then with
/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
///
/// To represent such "multi-mappings", we use a recursive structures: we map
/// variables not to values, but to *lists* of values or other lists (that is,
/// to the trees).
///
/// For the above example, the bindings would store
///
/// ```not_rust
/// i -> [foo, bar]
/// e -> [[1, 2, 3], [4, 5, 6]]
/// ```
///
/// We construct `Bindings` in the `match_lhs`. The interesting case is
/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
/// nesting structure.
///
/// The other side of the puzzle is `expand_subtree`, where we use the bindings
/// to substitute meta variables in the output template. When expanding, we
/// maintain a `nesting` stack of indices which tells us which occurrence from
/// the `Bindings` we should take. We push to the stack when we enter a
/// repetition.
///
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but an `&[usize]`.
#[derive(Debug, Default)]
struct Bindings {
    inner: FxHashMap<SmolStr, Binding>,
}

#[derive(Debug)]
enum Binding {
    Simple(tt::TokenTree),
    Nested(Vec<Binding>),
}

impl Bindings {
    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree, ExpandError> {
        let mut b = self
            .inner
            .get(name)
            .ok_or(ExpandError::BindingError(format!("could not find binding `{}`", name)))?;
        for &idx in nesting.iter() {
            b = match b {
                Binding::Simple(_) => break,
                Binding::Nested(bs) => bs.get(idx).ok_or(ExpandError::BindingError(format!(
                    "could not find nested binding `{}`",
                    name
                )))?,
            };
        }
        match b {
            Binding::Simple(it) => Ok(it),
            Binding::Nested(_) => Err(ExpandError::BindingError(format!(
                "expected simple binding, found nested binding `{}`",
                name
            ))),
        }
    }

    fn push_nested(&mut self, nested: Bindings) -> Result<(), ExpandError> {
        for (key, value) in nested.inner {
            if !self.inner.contains_key(&key) {
                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
            }
            match self.inner.get_mut(&key) {
                Some(Binding::Nested(it)) => it.push(value),
                _ => {
                    return Err(ExpandError::BindingError(format!(
                        "could not find binding `{}`",
                        key
                    )));
                }
            }
        }
        Ok(())
    }
}

fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
    let mut res = Bindings::default();
    for pat in pattern.token_trees.iter() {
        match pat {
            crate::TokenTree::Leaf(leaf) => match leaf {
                crate::Leaf::Var(crate::Var { text, kind }) => {
                    let kind = kind.clone().ok_or(ExpandError::UnexpectedToken)?;
                    match kind.as_str() {
                        "ident" => {
                            let ident =
                                input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(ident).into()),
                            );
                        }
                        _ => return Err(ExpandError::UnexpectedToken),
                    }
                }
                crate::Leaf::Punct(punct) => {
                    if input.eat_punct() != Some(punct) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                crate::Leaf::Ident(ident) => {
                    if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
                        return Err(ExpandError::UnexpectedToken);
                    }
                }
                _ => return Err(ExpandError::UnexpectedToken),
            },
            crate::TokenTree::Repeat(crate::Repeat { subtree, kind: _, separator }) => {
                while let Ok(nested) = match_lhs(subtree, input) {
                    res.push_nested(nested)?;
                    if let Some(separator) = *separator {
                        if !input.is_eof() {
                            if input.eat_punct().map(|p| p.char) != Some(separator) {
                                return Err(ExpandError::UnexpectedToken);
                            }
                        }
                    }
                }
            }
            _ => {}
        }
    }
    Ok(res)
}

fn expand_subtree(
    template: &crate::Subtree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::Subtree, ExpandError> {
    let token_trees = template
        .token_trees
        .iter()
        .map(|it| expand_tt(it, bindings, nesting))
        .collect::<Result<Vec<_>, ExpandError>>()?;

    Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
}

fn expand_tt(
    template: &crate::TokenTree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::TokenTree, ExpandError> {
    let res: tt::TokenTree = match template {
        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
        crate::TokenTree::Repeat(repeat) => {
            let mut token_trees = Vec::new();
            nesting.push(0);
            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
                let idx = nesting.pop().unwrap();
                nesting.push(idx + 1);
                token_trees.push(t.into())
            }
            nesting.pop().unwrap();
            tt::Subtree { token_trees, delimiter: tt::Delimiter::None }.into()
        }
        crate::TokenTree::Leaf(leaf) => match leaf {
            crate::Leaf::Ident(ident) => {
                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
                    .into()
            }
            crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
            crate::Leaf::Var(v) => bindings.get(&v.text, nesting)?.clone(),
            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
        },
    };
    Ok(res)
}

#[cfg(test)]
mod tests {
    use ra_syntax::{ast, AstNode};

    use super::*;
    use crate::ast_to_token_tree;

    #[test]
    fn test_expand_rule() {
        assert_err(
            "($i:ident) => ($j)",
            "foo!{a}",
            ExpandError::BindingError(String::from("could not find binding `j`")),
        );

        assert_err(
            "($($i:ident);*) => ($i)",
            "foo!{a}",
            ExpandError::BindingError(String::from(
                "expected simple binding, found nested binding `i`",
            )),
        );

        assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
        assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
    }

    fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) {
        assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation), Err(err));
    }

    fn format_macro(macro_body: &str) -> String {
        format!(
            "
        macro_rules! foo {{
            {}
        }}
",
            macro_body
        )
    }

    fn create_rules(macro_definition: &str) -> crate::MacroRules {
        let source_file = ast::SourceFile::parse(macro_definition);
        let macro_definition =
            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();

        let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
        crate::MacroRules::parse(&definition_tt).unwrap()
    }

    fn expand_first(
        rules: &crate::MacroRules,
        invocation: &str,
    ) -> Result<tt::Subtree, ExpandError> {
        let source_file = ast::SourceFile::parse(invocation);
        let macro_invocation =
            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();

        let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();

        expand_rule(&rules.rules[0], &invocation_tt)
    }
}
<|endoftext|>


<commit_before>/// This module takes a (parsed) definition of `macro_rules` invocation, a
/// `tt::TokenTree` representing an argument of macro invocation, and produces a
/// `tt::TokenTree` for the result of the expansion.
use rustc_hash::FxHashMap;
use ra_syntax::SmolStr;
use tt::TokenId;

use crate::{MacroRulesError, Result};
use crate::tt_cursor::TtCursor;

pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> Result<tt::Subtree> {
    rules.rules.iter().find_map(|it| expand_rule(it, input).ok())
        .ok_or(MacroRulesError::NoMatchingRule)
}

fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree> {
    let mut input = TtCursor::new(input);
    let bindings = match_lhs(&rule.lhs, &mut input)?;
    if !input.is_eof() {
        return Err(MacroRulesError::UnexpectedToken);
    }
    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
}

/// The actual algorithm for expansion is not too hard, but is pretty tricky.
/// `Bindings` structure is the key to understanding what we are doing here.
///
/// On the high level, it stores mapping from meta variables to the bits of
/// syntax it should be substituted with. For example, if `$e:expr` is matched
/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
///
/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
///
/// ```not_rust
/// macro_rules! foo {
///     ($($ i:ident $($ e:expr),*);*) => {
///         $(fn $ i() { $($ e);*; })*
///     }
/// }
/// foo! { foo 1,2,3; bar 4,5,6 }
/// ```
///
/// Here, the `$i` meta variable is matched first with `foo` and then with
/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
///
/// To represent such "multi-mappings", we use a recursive structures: we map
/// variables not to values, but to *lists* of values or other lists (that is,
/// to the trees).
///
/// For the above example, the bindings would store
///
/// ```not_rust
/// i -> [foo, bar]
/// e -> [[1, 2, 3], [4, 5, 6]]
/// ```
///
/// We construct `Bindings` in the `match_lhs`. The interesting case is
/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
/// nesting structure.
///
/// The other side of the puzzle is `expand_subtree`, where we use the bindings
/// to substitute meta variables in the output template. When expanding, we
/// maintain a `nesting` stack of indices which tells us which occurrence from
/// the `Bindings` we should take. We push to the stack when we enter a
/// repetition.
///
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but an `&[usize]`.
#[derive(Debug, Default)]
struct Bindings {
    inner: FxHashMap<SmolStr, Binding>,
}

#[derive(Debug)]
enum Binding {
    Simple(tt::TokenTree),
    Nested(Vec<Binding>),
}

impl Bindings {
    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree> {
        let mut b = self.inner.get(name).ok_or(MacroRulesError::BindingError(
            format!("could not find binding {}", name)
        ))?;
        for &idx in nesting.iter() {
            b = match b {
                Binding::Simple(_) => break,
                Binding::Nested(bs) => bs.get(idx).ok_or(MacroRulesError::BindingError(
                    format!("could not find nested binding {}", name))
                )?,
            };
        }
        match b {
            Binding::Simple(it) => Ok(it),
            Binding::Nested(_) => Err(MacroRulesError::BindingError(
                    format!("expected simple binding, found nested binding {}", name))),
        }
    }

    fn push_nested(&mut self, nested: Bindings) -> Result<()> {
        for (key, value) in nested.inner {
            if !self.inner.contains_key(&key) {
                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
            }
            match self.inner.get_mut(&key) {
                Some(Binding::Nested(it)) => it.push(value),
                _ => return Err(MacroRulesError::BindingError(
                    format!("nested binding for {} not found", key))),
            }
        }
        Ok(())
    }
}

fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings> {
    let mut res = Bindings::default();
    for pat in pattern.token_trees.iter() {
        match pat {
            crate::TokenTree::Leaf(leaf) => match leaf {
                crate::Leaf::Var(crate::Var { text, kind }) => {
                    let kind = kind.clone().ok_or(MacroRulesError::ParseError)?;
                    match kind.as_str() {
                        "ident" => {
                            let ident = input.eat_ident()?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(ident).into()),
                            );
                        }
                        _ => return Err(MacroRulesError::UnexpectedToken),
                    }
                }
                crate::Leaf::Punct(punct) => {
                    if input.eat_punct()? != punct {
                        return Err(MacroRulesError::UnexpectedToken);
                    }
                }
                crate::Leaf::Ident(ident) => {
                    if input.eat_ident()?.text != ident.text {
                        return Err(MacroRulesError::UnexpectedToken);
                    }
                }
                _ => return Err(MacroRulesError::UnexpectedToken),
            },
            crate::TokenTree::Repeat(crate::Repeat { subtree, kind: _, separator }) => {
                while let Ok(nested) = match_lhs(subtree, input) {
                    res.push_nested(nested)?;
                    if let Some(separator) = *separator {
                        if !input.is_eof() {
                            if input.eat_punct()?.char != separator {
                                return Err(MacroRulesError::UnexpectedToken);
                            }
                        }
                    }
                }
            }
            _ => {}
        }
    }
    Ok(res)
}

fn expand_subtree(
    template: &crate::Subtree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::Subtree> {
    let token_trees = template
        .token_trees
        .iter()
        .map(|it| expand_tt(it, bindings, nesting))
        .collect::<Result<Vec<_>>>()?;

    Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
}

fn expand_tt(
    template: &crate::TokenTree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::TokenTree> {
    let res: tt::TokenTree = match template {
        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
        crate::TokenTree::Repeat(repeat) => {
            let mut token_trees = Vec::new();
            nesting.push(0);
            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
                let idx = nesting.pop().unwrap();
                nesting.push(idx + 1);
                token_trees.push(t.into())
            }
            nesting.pop().unwrap();
            tt::Subtree { token_trees, delimiter: tt::Delimiter::None }.into()
        }
        crate::TokenTree::Leaf(leaf) => match leaf {
            crate::Leaf::Ident(ident) => {
                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
                    .into()
            }
            crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
            crate::Leaf::Var(v) => bindings.get(&v.text, nesting)?.clone(),
            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
        },
    };
    Ok(res)
}
<commit_msg>Formatting<commit_after>/// This module takes a (parsed) definition of `macro_rules` invocation, a
/// `tt::TokenTree` representing an argument of macro invocation, and produces a
/// `tt::TokenTree` for the result of the expansion.
use rustc_hash::FxHashMap;
use ra_syntax::SmolStr;
use tt::TokenId;

use crate::{MacroRulesError, Result};
use crate::tt_cursor::TtCursor;

pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> Result<tt::Subtree> {
    rules
        .rules
        .iter()
        .find_map(|it| expand_rule(it, input).ok())
        .ok_or(MacroRulesError::NoMatchingRule)
}

fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree> {
    let mut input = TtCursor::new(input);
    let bindings = match_lhs(&rule.lhs, &mut input)?;
    if !input.is_eof() {
        return Err(MacroRulesError::UnexpectedToken);
    }
    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
}

/// The actual algorithm for expansion is not too hard, but is pretty tricky.
/// `Bindings` structure is the key to understanding what we are doing here.
///
/// On the high level, it stores mapping from meta variables to the bits of
/// syntax it should be substituted with. For example, if `$e:expr` is matched
/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
///
/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
///
/// ```not_rust
/// macro_rules! foo {
///     ($($ i:ident $($ e:expr),*);*) => {
///         $(fn $ i() { $($ e);*; })*
///     }
/// }
/// foo! { foo 1,2,3; bar 4,5,6 }
/// ```
///
/// Here, the `$i` meta variable is matched first with `foo` and then with
/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
///
/// To represent such "multi-mappings", we use a recursive structures: we map
/// variables not to values, but to *lists* of values or other lists (that is,
/// to the trees).
///
/// For the above example, the bindings would store
///
/// ```not_rust
/// i -> [foo, bar]
/// e -> [[1, 2, 3], [4, 5, 6]]
/// ```
///
/// We construct `Bindings` in the `match_lhs`. The interesting case is
/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
/// nesting structure.
///
/// The other side of the puzzle is `expand_subtree`, where we use the bindings
/// to substitute meta variables in the output template. When expanding, we
/// maintain a `nesting` stack of indices which tells us which occurrence from
/// the `Bindings` we should take. We push to the stack when we enter a
/// repetition.
///
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but an `&[usize]`.
#[derive(Debug, Default)]
struct Bindings {
    inner: FxHashMap<SmolStr, Binding>,
}

#[derive(Debug)]
enum Binding {
    Simple(tt::TokenTree),
    Nested(Vec<Binding>),
}

impl Bindings {
    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree> {
        let mut b = self
            .inner
            .get(name)
            .ok_or(MacroRulesError::BindingError(format!("could not find binding {}", name)))?;
        for &idx in nesting.iter() {
            b = match b {
                Binding::Simple(_) => break,
                Binding::Nested(bs) => bs.get(idx).ok_or(MacroRulesError::BindingError(
                    format!("could not find nested binding {}", name),
                ))?,
            };
        }
        match b {
            Binding::Simple(it) => Ok(it),
            Binding::Nested(_) => Err(MacroRulesError::BindingError(format!(
                "expected simple binding, found nested binding {}",
                name
            ))),
        }
    }

    fn push_nested(&mut self, nested: Bindings) -> Result<()> {
        for (key, value) in nested.inner {
            if !self.inner.contains_key(&key) {
                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
            }
            match self.inner.get_mut(&key) {
                Some(Binding::Nested(it)) => it.push(value),
                _ => {
                    return Err(MacroRulesError::BindingError(format!(
                        "nested binding for {} not found",
                        key
                    )))
                }
            }
        }
        Ok(())
    }
}

fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings> {
    let mut res = Bindings::default();
    for pat in pattern.token_trees.iter() {
        match pat {
            crate::TokenTree::Leaf(leaf) => match leaf {
                crate::Leaf::Var(crate::Var { text, kind }) => {
                    let kind = kind.clone().ok_or(MacroRulesError::ParseError)?;
                    match kind.as_str() {
                        "ident" => {
                            let ident = input.eat_ident()?.clone();
                            res.inner.insert(
                                text.clone(),
                                Binding::Simple(tt::Leaf::from(ident).into()),
                            );
                        }
                        _ => return Err(MacroRulesError::UnexpectedToken),
                    }
                }
                crate::Leaf::Punct(punct) => {
                    if input.eat_punct()? != punct {
                        return Err(MacroRulesError::UnexpectedToken);
                    }
                }
                crate::Leaf::Ident(ident) => {
                    if input.eat_ident()?.text != ident.text {
                        return Err(MacroRulesError::UnexpectedToken);
                    }
                }
                _ => return Err(MacroRulesError::UnexpectedToken),
            },
            crate::TokenTree::Repeat(crate::Repeat { subtree, kind: _, separator }) => {
                while let Ok(nested) = match_lhs(subtree, input) {
                    res.push_nested(nested)?;
                    if let Some(separator) = *separator {
                        if !input.is_eof() {
                            if input.eat_punct()?.char != separator {
                                return Err(MacroRulesError::UnexpectedToken);
                            }
                        }
                    }
                }
            }
            _ => {}
        }
    }
    Ok(res)
}

fn expand_subtree(
    template: &crate::Subtree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::Subtree> {
    let token_trees = template
        .token_trees
        .iter()
        .map(|it| expand_tt(it, bindings, nesting))
        .collect::<Result<Vec<_>>>()?;

    Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
}

fn expand_tt(
    template: &crate::TokenTree,
    bindings: &Bindings,
    nesting: &mut Vec<usize>,
) -> Result<tt::TokenTree> {
    let res: tt::TokenTree = match template {
        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
        crate::TokenTree::Repeat(repeat) => {
            let mut token_trees = Vec::new();
            nesting.push(0);
            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
                let idx = nesting.pop().unwrap();
                nesting.push(idx + 1);
                token_trees.push(t.into())
            }
            nesting.pop().unwrap();
            tt::Subtree { token_trees, delimiter: tt::Delimiter::None }.into()
        }
        crate::TokenTree::Leaf(leaf) => match leaf {
            crate::Leaf::Ident(ident) => {
                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
                    .into()
            }
            crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
            crate::Leaf::Var(v) => bindings.get(&v.text, nesting)?.clone(),
            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
        },
    };
    Ok(res)
}
<|endoftext|>


<commit_before>mod math;

use super::{ RawLink, Framesque, Allocator };
use self::math::PowersOf2;

use core::mem;
use core::cmp::{max, min};

pub struct Free { next: RawLink<Free> }

impl Framesque for Free {
    #[inline] fn as_ptr(&self) -> *mut u8 {
        unsafe { mem::transmute(self) } // HOPEFULLY this is good
    }
}

// Variadic macro for taking the maximum of n > 2 numbers.
// because I'm lazy.
macro_rules! max {
    ($x:expr) => ($x);
    ($x:expr, $($xs:expr),+) => (max($x, max!($($xs),+)));
}

pub struct FreeList<'a> {
    /// A pointer to the head of the free list
    head: Option<&'a mut Free>
  , /// Number of blocks in the free list
    pub length: usize
}

impl<'a> FreeList<'a> {

    /// Create a new empty `FreeList`
    const fn new() -> FreeList<'a> {
        FreeList { head: None, length: 0 }
    }

    /// Push a new block onto the free list
    ///
    /// # Unsafe due to
    ///   - `mem::transmute()`
    ///   - Dereferencing a raw pointer
    unsafe fn push(&mut self, block: *mut u8) {
        let block_ptr = block as *mut Free;
        // be nice if rawlink was kinder to pattern-matching but whatever
        *block_ptr = if let Some(head) = self.head.take() {
            // if the head block is defined, set that block to point to The
            // head block
            Free { next: RawLink::some(head) }
        } else {
            // if the head block is not defined, set this block to point to
            // an empty block
            Free { next: RawLink::none() }
        };
        // replace the head-block pointer with the pushed block
        self.head = Some(mem::transmute(block_ptr));
        // the list is now one block longer
        self.length += 1;
    }
    /// Pop the head block off of the free list.
    ///
    /// # Returns
    ///   - `Some(*mut u8)` if the free list has blocks left
    ///   - `None` if the free list is empty
    ///
    /// # Unsafe due to
    ///   - `mem::transmute()`
    ///   - Dereferencing a raw pointer
    unsafe fn pop(&mut self) -> Option<*mut u8> {
        self.head.take()
            .map(|head| {
                let popped_block
                    = mem::replace(&mut self.head, head.next.resolve_mut());
                let block_ptr: *mut u8
                    = mem::transmute(popped_block);
                block_ptr
            })
    }

    /// Returns true if this `FreeList` has free blocks remaining
    #[inline] fn has_free_blocks(&self) -> bool { self.head.is_some() }

    /// Attempt to remove a block from the free list.
    ///
    /// This function searches the free list for the specified block, and
    /// removes it if it is found, returning whether or not the block was
    /// removed.
    ///
    /// This is quite slow; with a worst-case time complexity of O(log n),
    /// this function is a major bottleneck in our allocator implementation.
    /// By maintaining sorted free lists, we could perhaps improve performance
    /// somewhat.
    ///
    /// # Returns
    ///   - `true` if the block was removed from the free list
    ///   - `false` if the block was not present in the free list
    unsafe fn remove(&mut self, target_block: *mut u8) -> bool {
        let target_ptr = target_block as *mut Free;
        // loop through the free list's iterator to find the target block.
        // this is gross but hopefully much faster than the much prettier
        // recursive strategy.
        for block in self.iter_mut() {
            let block_ptr: *mut Free = block;
            if block_ptr == target_ptr {
                // if we've found the target block, remove it and break
                *block_ptr = Free { next: block.next.take() };
                return true;
            }
        }
        false
    }

    /// Returns an iterator over the blocks in this free list
    fn iter<'b>(&'b self) -> FreeListIter<'b> {
        // FreeListIter { current: self.head.map(|c| c.borrow())
        //                             .as_ref()
        //              }
        match self.head {
            Some(ref head) => FreeListIter { current: Some(head) }
          , None           => FreeListIter { current: None }
        }
        // unimplemented!()
    }

    /// Returns a mutable iterator over the blocks in this free list.
    fn iter_mut<'b>(&'b mut self) -> FreeListIterMut<'b> {
        // FreeListIterMut { current: self.head.map(|c| *c ).as_mut() }
        match self.head {
            Some(ref mut head) => FreeListIterMut { current: Some(head) }
          , None               => FreeListIterMut { current: None }
        }
    }

}

struct FreeListIter<'a> {
    current: Option<&'a Free>
}

impl<'a> Iterator for FreeListIter<'a> {
    type Item = &'a Free;

    fn next(&mut self) -> Option<&'a Free> {
        self.current
            .map(|c| {
                self.current = unsafe { c.next.resolve() };
                c
            })
    }
}

struct FreeListIterMut<'a> {
    current: Option<&'a mut Free>
}

impl<'a> Iterator for FreeListIterMut<'a> {
    type Item = &'a mut Free;

    fn next(&mut self) -> Option<&'a mut Free> {
        self.current.take()
            .map(|c| {
                self.current = unsafe { c.next.resolve_mut() };
                c
            })
    }
}

pub struct BuddyHeapAllocator<'a> {
    /// Address of the base of the heap. This must be aligned
    /// on a `MIN_ALIGN` boundary.
    start_addr: *mut u8
  , /// The allocator's free list
    free_lists: &'a mut [FreeList<'a>]
  , /// Number of blocks in the heap (must be a power of 2)
    heap_size: usize
  , /// Minimum block size
    min_block_size: usize
}

impl<'a> BuddyHeapAllocator<'a> {
    pub unsafe fn new( start_addr: *mut u8
                     , free_lists: &'a mut [FreeList<'a>]
                     , heap_size: usize) -> BuddyHeapAllocator<'a> {
        let n_free_lists = free_lists.len();

        assert!( !start_addr.is_null()
                , "Heap start address cannot be null." );
        assert!( n_free_lists > 0
               , "Allocator must have at least one free list.");
        assert!( start_addr as usize & (::PAGE_SIZE-1) == 0
               , "Heap start address must be aligned on a 4k boundary.");

        let min_block_size = heap_size >> (n_free_lists - 1);

        assert!( heap_size >= min_block_size
               , "Heap must be large enough to contain at least one block.");
        assert!( min_block_size >= mem::size_of::<Free>()
               , "Minimum block size must be large enough to contain \
                  the free block header.");

        let mut heap
            = BuddyHeapAllocator { start_addr: start_addr
                                 , free_lists: free_lists
                                 , heap_size: heap_size
                                 , min_block_size: min_block_size
                                 };
        // TODO: put first head block on appropriately-sized freelist
        heap
    }

    /// Computes the size of an allocation request.
    ///
    /// # Arguments
    ///   - `size`: A `usize` containing the size of the request
    ///   - `align`: A `usize` containing the alignment of the request
    ///
    /// # Returns
    ///   - `None` if the request is invalid
    ///   - `Some(usize)` containing the size needed if the request is valid
    #[inline]
    pub fn alloc_size(&self, size: usize, align: usize) -> Option<usize> {
        // Pre-check if this is a valid allocation request:
        //  - allocations must be aligned on power of 2 boundaries
        //  - we cannot allocate requests with alignments greater than the
        //    base alignment of the heap without jumping through a bunch of
        //    hoops.
        if !align.is_pow2() || align > ::PAGE_SIZE {
            None
        // If the request is valid, compute the size we need to allocate
        } else {
            let alloc_size
                // the allocation size for the request is the next power of 2
                // after the size of the request, the alignment of the request,
                // or the minimum block size (whichever is greatest).
                = max!( size
                        // we can't allocate less than the minimum block size
                      , self.min_block_size
                        // we can't allocate less than the alignment, either
                      , align )
                    .next_pow2();

            if alloc_size > self.heap_size {
                // if the calculated size is greater than the size of the heap,
                // we (obviously) cannot allocate this request.
                None
            } else {
                // otherwise, return the calculated size.
                Some(alloc_size)
            }
        }
    }

    /// Computes the order of an allocation request.
    ///
    /// The "order" of an allocation refers to the number of times we need to
    /// double the minimum block size to get a large enough block for that
    /// allocation.
    #[inline]
    pub fn alloc_order(&self, size: usize, align: usize) -> Option<usize> {
        self.alloc_size(size, align)
            .map(|s| // the order of the allocation is the base-2 log of the
                     // allocation size minus the base-2 log of the minimum
                     // block size
                s.log2() - self.min_block_size.log2() // TODO: cache this?
            )
    }

    /// Computes the size  allocated for a request of the given order.
    #[inline]
    fn order_alloc_size(&self, order: usize) -> usize {
        1 << (self.min_block_size.log2() + order)
    }

    /// Splits a block
    unsafe fn split_block( &mut self, block: *mut u8
                         , order: usize, new_order: usize )
    {
        assert!( new_order < order
               , "Cannot split a block larger than its current order!");
        assert!( order <= self.free_lists.len()
               , "Order is too large for this allocator!");

        let mut split_size = self.order_alloc_size(order);
        let mut curr_order = order;

        while curr_order > new_order {
            split_size >>= 1;
            curr_order -= 1;

            self.free_lists[curr_order]
                .push(block.offset(split_size as isize))
        }

    }

    pub fn get_buddy(&self, order: usize, block: *mut u8) -> Option<*mut u8> {
        // Determine the size of the block allocated for the given order
        let block_size = self.order_alloc_size(order);
        if block_size < self.heap_size {
            // In the non-degenerate case, the block is not the same size
            // as the entire heap.

        } else {
            // If the block is the size of the entire heap, it (obviously)
            // cannot have a buddy block.
            None
        }
    }
}

impl<'a> Allocator for BuddyHeapAllocator<'a> {
    // type *mut u8 = Free;

    unsafe fn allocate(&mut self, size: usize, align: usize) -> Option<*mut u8> {
        // First, compute the allocation order for this request
        self.alloc_order(size, align)
            // If the allocation order is defined, then we try to allocate
            // a block of that order. Otherwise, the request is invalid.
            .and_then(|min_order| {
                // Starting at the minimum possible order...
                // TODO: this is ugly and not FP, rewrite.
                let mut result = None;
                for order in min_order..self.free_lists.len() {
                    if let Some(block) = self.free_lists[order].pop() {
                        if order > min_order {
                            self.split_block(block, order, min_order);
                        }
                        result = Some(block); break;
                    }
                }
                result
                // self.free_lists[min_order..].iter().enumerate()
                // ...find the first free list that has free blocks left.
                    // .find(|&(_, ref f)| f.has_free_blocks())
                    // .and_then(|(order, f)| unsafe {
                    //     let block = f.pop();
                    //     if order > min_order {
                    //         block.map(|b| {
                    //             self.split_block(b, order, min_order);
                    //             block
                    //         });
                    //     }
                    //     block
                    // })
            })
    }

    unsafe fn deallocate( &mut self, block: *mut u8
                        , old_size: usize, align: usize )
    {
        let min_order = self.alloc_order(old_size, align)
                            .unwrap();

        // Check if the deallocated block's buddy block is also free.
        // If it is, merge the two blocks.
        let mut new_block = block;
        for order in min_order..self.free_lists.len() {

        }
    }
}
<commit_msg>Added deallocate implementation for BuddyHeapAllocator<commit_after>mod math;

use super::{ RawLink, Framesque, Allocator };
use self::math::PowersOf2;

use core::mem;
use core::cmp::{max, min};

pub struct Free { next: RawLink<Free> }

impl Framesque for Free {
    #[inline] fn as_ptr(&self) -> *mut u8 {
        unsafe { mem::transmute(self) } // HOPEFULLY this is good
    }
}

// Variadic macro for taking the maximum of n > 2 numbers.
// because I'm lazy.
macro_rules! max {
    ($x:expr) => ($x);
    ($x:expr, $($xs:expr),+) => (max($x, max!($($xs),+)));
}

pub struct FreeList<'a> {
    /// A pointer to the head of the free list
    head: Option<&'a mut Free>
  , /// Number of blocks in the free list
    pub length: usize
}

impl<'a> FreeList<'a> {

    /// Create a new empty `FreeList`
    const fn new() -> FreeList<'a> {
        FreeList { head: None, length: 0 }
    }

    /// Push a new block onto the free list
    ///
    /// # Unsafe due to
    ///   - `mem::transmute()`
    ///   - Dereferencing a raw pointer
    unsafe fn push(&mut self, block: *mut u8) {
        let block_ptr = block as *mut Free;
        // be nice if rawlink was kinder to pattern-matching but whatever
        *block_ptr = if let Some(head) = self.head.take() {
            // if the head block is defined, set that block to point to The
            // head block
            Free { next: RawLink::some(head) }
        } else {
            // if the head block is not defined, set this block to point to
            // an empty block
            Free { next: RawLink::none() }
        };
        // replace the head-block pointer with the pushed block
        self.head = Some(mem::transmute(block_ptr));
        // the list is now one block longer
        self.length += 1;
    }
    /// Pop the head block off of the free list.
    ///
    /// # Returns
    ///   - `Some(*mut u8)` if the free list has blocks left
    ///   - `None` if the free list is empty
    ///
    /// # Unsafe due to
    ///   - `mem::transmute()`
    ///   - Dereferencing a raw pointer
    unsafe fn pop(&mut self) -> Option<*mut u8> {
        self.head.take()
            .map(|head| {
                let popped_block
                    = mem::replace(&mut self.head, head.next.resolve_mut());
                let block_ptr: *mut u8
                    = mem::transmute(popped_block);
                block_ptr
            })
    }

    /// Returns true if this `FreeList` has free blocks remaining
    #[inline] fn has_free_blocks(&self) -> bool { self.head.is_some() }

    /// Attempt to remove a block from the free list.
    ///
    /// This function searches the free list for the specified block, and
    /// removes it if it is found, returning whether or not the block was
    /// removed.
    ///
    /// This is quite slow; with a worst-case time complexity of O(log n),
    /// this function is a major bottleneck in our allocator implementation.
    /// By maintaining sorted free lists, we could perhaps improve performance
    /// somewhat.
    ///
    /// # Returns
    ///   - `true` if the block was removed from the free list
    ///   - `false` if the block was not present in the free list
    unsafe fn remove(&mut self, target_block: *mut u8) -> bool {
        let target_ptr = target_block as *mut Free;
        // loop through the free list's iterator to find the target block.
        // this is gross but hopefully much faster than the much prettier
        // recursive strategy.
        for block in self.iter_mut() {
            let block_ptr: *mut Free = block;
            if block_ptr == target_ptr {
                // if we've found the target block, remove it and break
                *block_ptr = Free { next: block.next.take() };
                return true;
            }
        }
        false
    }

    /// Returns an iterator over the blocks in this free list
    fn iter<'b>(&'b self) -> FreeListIter<'b> {
        // FreeListIter { current: self.head.map(|c| c.borrow())
        //                             .as_ref()
        //              }
        match self.head {
            Some(ref head) => FreeListIter { current: Some(head) }
          , None           => FreeListIter { current: None }
        }
        // unimplemented!()
    }

    /// Returns a mutable iterator over the blocks in this free list.
    fn iter_mut<'b>(&'b mut self) -> FreeListIterMut<'b> {
        // FreeListIterMut { current: self.head.map(|c| *c ).as_mut() }
        match self.head {
            Some(ref mut head) => FreeListIterMut { current: Some(head) }
          , None               => FreeListIterMut { current: None }
        }
    }

}

struct FreeListIter<'a> {
    current: Option<&'a Free>
}

impl<'a> Iterator for FreeListIter<'a> {
    type Item = &'a Free;

    fn next(&mut self) -> Option<&'a Free> {
        self.current
            .map(|c| {
                self.current = unsafe { c.next.resolve() };
                c
            })
    }
}

struct FreeListIterMut<'a> {
    current: Option<&'a mut Free>
}

impl<'a> Iterator for FreeListIterMut<'a> {
    type Item = &'a mut Free;

    fn next(&mut self) -> Option<&'a mut Free> {
        self.current.take()
            .map(|c| {
                self.current = unsafe { c.next.resolve_mut() };
                c
            })
    }
}

pub struct BuddyHeapAllocator<'a> {
    /// Address of the base of the heap. This must be aligned
    /// on a `MIN_ALIGN` boundary.
    start_addr: *mut u8
  , /// The allocator's free list
    free_lists: &'a mut [FreeList<'a>]
  , /// Number of blocks in the heap (must be a power of 2)
    heap_size: usize
  , /// Minimum block size
    min_block_size: usize
}

impl<'a> BuddyHeapAllocator<'a> {
    pub unsafe fn new( start_addr: *mut u8
                     , free_lists: &'a mut [FreeList<'a>]
                     , heap_size: usize) -> BuddyHeapAllocator<'a> {
        let n_free_lists = free_lists.len();

        assert!( !start_addr.is_null()
                , "Heap start address cannot be null." );
        assert!( n_free_lists > 0
               , "Allocator must have at least one free list.");
        assert!( start_addr as usize & (::PAGE_SIZE-1) == 0
               , "Heap start address must be aligned on a 4k boundary.");

        let min_block_size = heap_size >> (n_free_lists - 1);

        assert!( heap_size >= min_block_size
               , "Heap must be large enough to contain at least one block.");
        assert!( min_block_size >= mem::size_of::<Free>()
               , "Minimum block size must be large enough to contain \
                  the free block header.");

        let mut heap
            = BuddyHeapAllocator { start_addr: start_addr
                                 , free_lists: free_lists
                                 , heap_size: heap_size
                                 , min_block_size: min_block_size
                                 };
        // TODO: put first head block on appropriately-sized freelist
        heap
    }

    /// Computes the size of an allocation request.
    ///
    /// # Arguments
    ///   - `size`: A `usize` containing the size of the request
    ///   - `align`: A `usize` containing the alignment of the request
    ///
    /// # Returns
    ///   - `None` if the request is invalid
    ///   - `Some(usize)` containing the size needed if the request is valid
    #[inline]
    pub fn alloc_size(&self, size: usize, align: usize) -> Option<usize> {
        // Pre-check if this is a valid allocation request:
        //  - allocations must be aligned on power of 2 boundaries
        //  - we cannot allocate requests with alignments greater than the
        //    base alignment of the heap without jumping through a bunch of
        //    hoops.
        if !align.is_pow2() || align > ::PAGE_SIZE {
            None
        // If the request is valid, compute the size we need to allocate
        } else {
            let alloc_size
                // the allocation size for the request is the next power of 2
                // after the size of the request, the alignment of the request,
                // or the minimum block size (whichever is greatest).
                = max!( size
                        // we can't allocate less than the minimum block size
                      , self.min_block_size
                        // we can't allocate less than the alignment, either
                      , align )
                    .next_pow2();

            if alloc_size > self.heap_size {
                // if the calculated size is greater than the size of the heap,
                // we (obviously) cannot allocate this request.
                None
            } else {
                // otherwise, return the calculated size.
                Some(alloc_size)
            }
        }
    }

    /// Computes the order of an allocation request.
    ///
    /// The "order" of an allocation refers to the number of times we need to
    /// double the minimum block size to get a large enough block for that
    /// allocation.
    #[inline]
    pub fn alloc_order(&self, size: usize, align: usize) -> Option<usize> {
        self.alloc_size(size, align)
            .map(|s| // the order of the allocation is the base-2 log of the
                     // allocation size minus the base-2 log of the minimum
                     // block size
                s.log2() - self.min_block_size.log2() // TODO: cache this?
            )
    }

    /// Computes the size  allocated for a request of the given order.
    #[inline]
    fn order_alloc_size(&self, order: usize) -> usize {
        1 << (self.min_block_size.log2() + order)
    }

    /// Splits a block
    unsafe fn split_block( &mut self, block: *mut u8
                         , order: usize, new_order: usize )
    {
        assert!( new_order < order
               , "Cannot split a block larger than its current order!");
        assert!( order <= self.free_lists.len()
               , "Order is too large for this allocator!");

        let mut split_size = self.order_alloc_size(order);
        let mut curr_order = order;

        while curr_order > new_order {
            split_size >>= 1;
            curr_order -= 1;

            self.free_lists[curr_order]
                .push(block.offset(split_size as isize))
        }

    }

    pub unsafe fn get_buddy(&self, order: usize, block: *mut u8)
                           -> Option<*mut u8>
    {
        // Determine the size of the block allocated for the given order
        let block_size = self.order_alloc_size(order);
        if block_size < self.heap_size {
            // Determine the block's position in the heap.
            let block_pos = (block as usize) - (self.start_addr as usize);
            // Calculate the block's buddy by XORing the block's position
            // in the heap with its size.
            Some(self.start_addr.offset((block_pos ^ block_size) as isize))
        } else {
            // If the block is the size of the entire heap, it (obviously)
            // cannot have a buddy block.
            None
        }
    }
}

impl<'a> Allocator for BuddyHeapAllocator<'a> {
    // type *mut u8 = Free;

    unsafe fn allocate(&mut self, size: usize, align: usize) -> Option<*mut u8> {
        // First, compute the allocation order for this request
        self.alloc_order(size, align)
            // If the allocation order is defined, then we try to allocate
            // a block of that order. Otherwise, the request is invalid.
            .and_then(|min_order| {
                // Starting at the minimum possible order...
                // TODO: this is ugly and not FP, rewrite.
                let mut result = None;
                for order in min_order..self.free_lists.len() {
                    if let Some(block) = self.free_lists[order].pop() {
                        if order > min_order {
                            self.split_block(block, order, min_order);
                        }
                        result = Some(block); break;
                    }
                }
                result
                // self.free_lists[min_order..].iter().enumerate()
                // ...find the first free list that has free blocks left.
                    // .find(|&(_, ref f)| f.has_free_blocks())
                    // .and_then(|(order, f)| unsafe {
                    //     let block = f.pop();
                    //     if order > min_order {
                    //         block.map(|b| {
                    //             self.split_block(b, order, min_order);
                    //             block
                    //         });
                    //     }
                    //     block
                    // })
            })
    }

    unsafe fn deallocate( &mut self, block: *mut u8
                        , old_size: usize, align: usize )
    {
        let min_order = self.alloc_order(old_size, align)
                            .unwrap();

        // Check if the deallocated block's buddy block is also free.
        // If it is, merge the two blocks.
        let mut new_block = block;
        for order in min_order..self.free_lists.len() {
            // If there is a buddy for this block of the given order...
            if let Some(buddy) = self.get_buddy(order, block) {
                // ...and if the buddy was free...
                if self.free_lists[order].remove(buddy) {
                    // ...merge the buddy with the new block (just use
                    // the lower address), and keep going.
                    new_block = min(new_block, buddy);
                    continue;
                }
            }
            // Otherwise, if we've run out of free buddies, push the new
            // merged block onto the free lsit and return.
            self.free_lists[order]
                .push(new_block);
            return;
        }
    }
}
<|endoftext|>


<commit_before>#[cfg(feature = "binaries")]
mod binary {
  use assert_cmd::Command;
  use rand::distributions::Alphanumeric;
  use rand::{thread_rng, Rng};
  use std::env::temp_dir;
  use std::fs::File;
  use std::io::Read;
  use std::path::PathBuf;

  fn get_y4m_input() -> Vec<u8> {
    let mut input = File::open(&format!(
      "{}/tests/small_input.y4m",
      env!("CARGO_MANIFEST_DIR")
    ))
    .unwrap();
    let mut data = Vec::new();
    input.read_to_end(&mut data).unwrap();
    data
  }

  fn get_tempfile_path(extension: &str) -> PathBuf {
    let mut path = temp_dir();
    let filename =
      thread_rng().sample_iter(&Alphanumeric).take(12).collect::<String>();
    path.push(format!("{}.{}", filename, extension));
    path
  }

  #[cfg(not(windows))]
  fn get_rav1e_command() -> Command {
    let mut cmd = Command::cargo_bin("rav1e").unwrap();
    cmd.env_clear();
    cmd
  }

  #[cfg(windows)]
  // `env_clear` doesn't work on Windows: https://github.com/rust-lang/rust/issues/31259
  fn get_rav1e_command() -> Command {
    Command::cargo_bin("rav1e").unwrap()
  }

  fn get_common_cmd(outfile: &PathBuf) -> Command {
    let mut cmd = get_rav1e_command();
    cmd.args(&["--bitrate", "1000"]).arg("-o").arg(outfile);
    cmd
  }

  #[test]
  fn one_pass_qp_based() {
    let outfile = get_tempfile_path("ivf");

    get_rav1e_command()
      .args(&["--quantizer", "100"])
      .arg("-o")
      .arg(&outfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }

  #[test]
  fn one_pass_bitrate_based() {
    let outfile = get_tempfile_path("ivf");

    get_common_cmd(&outfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }

  #[test]
  fn two_pass_bitrate_based() {
    let outfile = get_tempfile_path("ivf");
    let passfile = get_tempfile_path("pass");

    get_common_cmd(&outfile)
      .arg("--first-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();

    get_common_cmd(&outfile)
      .arg("--second-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }
  #[test]
  fn two_pass_bitrate_based_constrained() {
    let outfile = get_tempfile_path("ivf");
    let passfile = get_tempfile_path("pass");

    get_common_cmd(&outfile)
      .args(&["--reservoir-frame-delay", "14"])
      .arg("--first-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();

    get_common_cmd(&outfile)
      .args(&["--reservoir-frame-delay", "14"])
      .arg("--second-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }
}
<commit_msg>Unbreak some cli tests<commit_after>#[cfg(feature = "binaries")]
mod binary {
  use assert_cmd::Command;
  use rand::distributions::Alphanumeric;
  use rand::{thread_rng, Rng};
  use std::env::temp_dir;
  use std::fs::File;
  use std::io::Read;
  use std::path::PathBuf;

  fn get_y4m_input() -> Vec<u8> {
    let mut input = File::open(&format!(
      "{}/tests/small_input.y4m",
      env!("CARGO_MANIFEST_DIR")
    ))
    .unwrap();
    let mut data = Vec::new();
    input.read_to_end(&mut data).unwrap();
    data
  }

  fn get_tempfile_path(extension: &str) -> PathBuf {
    let mut path = temp_dir();
    let filename =
      thread_rng().sample_iter(&Alphanumeric).take(12).collect::<String>();
    path.push(format!("{}.{}", filename, extension));
    path
  }

  #[cfg(not(windows))]
  fn get_rav1e_command() -> Command {
    let mut cmd = Command::cargo_bin("rav1e").unwrap();
    cmd.env_clear();
    cmd
  }

  #[cfg(windows)]
  // `env_clear` doesn't work on Windows: https://github.com/rust-lang/rust/issues/31259
  fn get_rav1e_command() -> Command {
    Command::cargo_bin("rav1e").unwrap()
  }

  fn get_common_cmd(outfile: &PathBuf) -> Command {
    let mut cmd = get_rav1e_command();
    cmd.args(&["--bitrate", "1000"]).arg("-o").arg(outfile).arg("-y");
    cmd
  }

  #[test]
  fn one_pass_qp_based() {
    let outfile = get_tempfile_path("ivf");

    get_rav1e_command()
      .args(&["--quantizer", "100"])
      .arg("-o")
      .arg(&outfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }

  #[test]
  fn one_pass_bitrate_based() {
    let outfile = get_tempfile_path("ivf");

    get_common_cmd(&outfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }

  #[test]
  fn two_pass_bitrate_based() {
    let outfile = get_tempfile_path("ivf");
    let passfile = get_tempfile_path("pass");

    get_common_cmd(&outfile)
      .arg("--first-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();

    get_common_cmd(&outfile)
      .arg("--second-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }
  #[test]
  fn two_pass_bitrate_based_constrained() {
    let outfile = get_tempfile_path("ivf");
    let passfile = get_tempfile_path("pass");

    get_common_cmd(&outfile)
      .args(&["--reservoir-frame-delay", "14"])
      .arg("--first-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();

    get_common_cmd(&outfile)
      .args(&["--reservoir-frame-delay", "14"])
      .arg("--second-pass")
      .arg(&passfile)
      .arg("-")
      .write_stdin(get_y4m_input())
      .assert()
      .success();
  }
}
<|endoftext|>


<commit_before>//#![feature(trace_macros)]
#![allow(dead_code)]
#![cfg_attr(feature = "cargo-clippy", allow(redundant_closure))]

#[macro_use]
extern crate nom;

use nom::{space, Err, IResult, Needed, ErrorKind, le_u64, is_digit};
use nom::types::{CompleteStr, CompleteByteSlice};

#[allow(dead_code)]
struct Range {
  start: char,
  end: char,
}

pub fn take_char(input: &[u8]) -> IResult<&[u8], char> {
  if !input.is_empty() {
    Ok((&input[1..], input[0] as char))
  } else {
    Err(Err::Incomplete(Needed::Size(1)))
  }
}

//trace_macros!(true);

#[allow(dead_code)]
named!(range<&[u8], Range>,
    alt!(
        do_parse!(
            start: take_char >>
            tag!("-")        >>
            end: take_char   >>
            (Range {
                start: start,
                end:   end,
            })
        ) |
        map!(
            take_char,
            |c| {
                Range {
                    start: c,
                    end:   c,
                }
            }
        )
    )
);

#[allow(dead_code)]
named!(literal<&[u8], Vec<char> >,
    map!(
        many1!(take_char),
        |cs| {
          cs
        }
    )
);

#[test]
fn issue_58() {
  let _ = range(&b"abcd"[..]);
  let _ = literal(&b"abcd"[..]);
}

//trace_macros!(false);

#[cfg(feature = "std")]
mod parse_int {
  use nom::HexDisplay;
  use nom::{digit, space, IResult};
  use std::str;

  named!(parse_ints<Vec<i32>>, many0!(spaces_or_int));

  fn spaces_or_int(input: &[u8]) -> IResult<&[u8], i32> {
    println!("{}", input.to_hex(8));
    do_parse!(
      input,
      opt!(complete!(space)) >> res: map!(complete!(digit), |x| {
        println!("x: {:?}", x);
        let result = str::from_utf8(x).unwrap();
        println!("Result: {}", result);
        println!("int is empty?: {}", x.is_empty());
        match result.parse() {
          Ok(i) => i,
          Err(e) => panic!("UH OH! NOT A DIGIT! {:?}", e),
        }
      }) >> (res)
    )
  }

  #[test]
  fn issue_142() {
    let subject = parse_ints(&b"12 34 5689a"[..]);
    let expected = Ok((&b"a"[..], vec![12, 34, 5689]));
    assert_eq!(subject, expected);

    let subject = parse_ints(&b"12 34 5689 "[..]);
    let expected = Ok((&b" "[..], vec![12, 34, 5689]));
    assert_eq!(subject, expected)
  }
}

#[test]
fn usize_length_bytes_issue() {
  use nom::be_u16;
  let _: IResult<&[u8], &[u8], u32> = length_bytes!(b"012346", be_u16);
}

/*
 DOES NOT COMPILE
#[test]
fn issue_152() {
  named!(take4, take!(4));
  named!(xyz, tag!("XYZ"));
  named!(abc, tag!("abc"));

  named!(sw,
    switch!(take4,
      b"abcd" => xyz |
      b"efgh" => abc
    )
  );
}
*/

#[test]
fn take_till_issue() {
  named!(nothing, take_till!(call!(|_| true)));

  assert_eq!(nothing(b""), Err(Err::Incomplete(Needed::Size(1))));
  assert_eq!(nothing(b"abc"), Ok((&b"abc"[..], &b""[..])));
}

named!(
  issue_498<Vec<&[u8]>>,
  separated_nonempty_list!(opt!(space), tag!("abcd"))
);

named!(issue_308(&str) -> bool,
    do_parse! (
        tag! ("foo") >>
        b: alt_complete! (
            map! (tag! ("1"), |_: &str|->bool {true}) |
            value! (false)
        ) >>
        (b) ));

#[cfg(feature = "alloc")]
fn issue_302(input: &[u8]) -> IResult<&[u8], Option<Vec<u64>>> {
  do_parse!(input, entries: cond!(true, count!(le_u64, 3)) >> (entries))
}

#[test]
fn issue_655() {
  use nom::{line_ending, not_line_ending};
  named!(twolines(&str) -> (&str, &str),
    do_parse!(
      l1 : not_line_ending >>
           line_ending >>
      l2 : not_line_ending >>
           line_ending >>
      ((l1, l2))
    )
  );

  assert_eq!(twolines("foo\nbar\n"), Ok(("", ("foo", "bar"))));
  assert_eq!(twolines("féo\nbar\n"), Ok(("", ("féo", "bar"))));
  assert_eq!(twolines("foé\nbar\n"), Ok(("", ("foé", "bar"))));
  assert_eq!(twolines("foé\r\nbar\n"), Ok(("", ("foé", "bar"))));
}

#[cfg(feature = "std")]
named!(issue_666 <CompleteByteSlice, CompleteByteSlice>, dbg_dmp!(tag!("abc")));

#[test]
fn issue_667() {
  use nom::alpha;

  named!(foo <CompleteByteSlice, Vec<CompleteByteSlice>>,
    many0!(
      alt!(alpha | is_a!("_"))
    )
  );
  assert_eq!(
    foo(CompleteByteSlice(b"")),
    Ok((CompleteByteSlice(b""), vec![]))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"loremipsum")),
    Ok((
      CompleteByteSlice(b""),
      vec![CompleteByteSlice(b"loremipsum")]
    ))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"lorem_ipsum")),
    Ok((
      CompleteByteSlice(b""),
      vec![
        CompleteByteSlice(b"lorem"),
        CompleteByteSlice(b"_"),
        CompleteByteSlice(b"ipsum"),
      ]
    ))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"_lorem_ipsum")),
    Ok((
      CompleteByteSlice(b""),
      vec![
        CompleteByteSlice(b"_"),
        CompleteByteSlice(b"lorem"),
        CompleteByteSlice(b"_"),
        CompleteByteSlice(b"ipsum"),
      ]
    ))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"!@#$")),
    Ok((CompleteByteSlice(b"!@#$"), vec![]))
  );
}

#[test]
fn issue_721() {
  assert_eq!(parse_to!("1234", u16), Ok(("", 1234)));
  assert_eq!(parse_to!("foo", String), Ok(("", "foo".to_string())));
}

#[cfg(feature = "alloc")]
named!(issue_717<&[u8], Vec<&[u8]> >,
  separated_list!(tag!([0x0]), is_not!([0x0u8]))
);

struct NoPartialEq {
  value: i32,
}

named!(issue_724<&str, i32>,
  do_parse!(
    metadata: permutation!(
      map!(tag!("hello"), |_| NoPartialEq { value: 1 }),
      map!(tag!("world"), |_| NoPartialEq { value: 2 })
    ) >>
    (metadata.0.value + metadata.1.value)
  )
);

named!(issue_741_str<CompleteStr, CompleteStr>, re_match!(r"^_?[A-Za-z][0-9A-Z_a-z-]*"));
named!(issue_741_bytes<CompleteByteSlice, CompleteByteSlice>, re_bytes_match!(r"^_?[A-Za-z][0-9A-Z_a-z-]*"));

#[test]
fn issue_752() {
    assert_eq!(
        Err::Error(nom::Context::Code("ab", nom::ErrorKind::ParseTo)),
        parse_to!("ab", usize).unwrap_err()
    )
}

fn atom_specials(c: u8) -> bool {
    c == b'q'
}

named!(
    capability<&str>,
    do_parse!(tag!(" ") >> _atom: map_res!(take_till1!(atom_specials), std::str::from_utf8) >> ("a"))
);

#[test]
fn issue_759() {
    assert_eq!(capability(b" abcqd"), Ok((&b"qd"[..], "a")));
}

named_args!(issue_771(count: usize)<Vec<u32>>,
  length_count!(value!(count), call!(nom::be_u32))
);

#[test]
fn issue_768() {
  named!(bit_vec8<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 8))));
  named!(bit_vec4<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 4))));
  named!(bit_vec3<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 3))));
  named!(bit_vec11<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 11))));

  let m: Vec<u8> = vec![70, 97, 106, 121, 86, 66, 105, 98, 86, 106, 101];
  assert_eq!(
    bit_vec8(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![70, 97, 106, 121, 86, 66, 105, 98, 86, 106, 101]))
  );
  assert_eq!(
    bit_vec4(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![4, 6, 6, 1, 6, 10, 7, 9, 5, 6, 4, 2, 6, 9, 6, 2, 5, 6, 6, 10, 6, 5]))
  );
  assert_eq!(
    bit_vec3(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![2, 1, 4, 6, 0, 5, 5, 2, 3, 6, 2, 5, 3, 1, 0, 2, 3, 2, 2, 6, 1, 1, 2, 6, 3, 2, 4, 6, 2]))
  );
  assert_eq!(
    bit_vec11(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![563, 90, 1266, 1380, 308, 1417, 717, 613]))
  );
}

/// This test is in a separate module to check that all required symbols are imported in
/// `escaped_transform!()`. Without the module, the `use`-es of the current module would
/// mask the error ('"Use of undeclared type or module `Needed`" in escaped_transform!').
mod issue_780 {
  named!(issue_780<&str, String>,
    escaped_transform!(call!(::nom::alpha), '\\', tag!("n"))
  );
}

// issue 617
named!(digits, take_while1!( is_digit ));
named!(multi_617<&[u8], () >, fold_many0!( digits, (), |_, _| {}));

// Sad :(
named!(multi_617_fails<&[u8], () >, fold_many0!( take_while1!( is_digit ), (), |_, _| {}));

mod issue_647 {
  use nom::{Err,be_f64};
  pub type Input<'a> = &'a [u8];

  #[derive(PartialEq, Debug, Clone)]
  struct Data {
      c: f64,
      v: Vec<f64>
  }

  fn list<'a,'b>(input: Input<'a>, _cs: &'b f64) -> Result<(Input<'a>,Vec<f64>), Err<&'a [u8]>> {
      separated_list_complete!(input, tag!(","),be_f64)
  }

  named!(data<Input,Data>, map!(
      do_parse!(
          c: be_f64 >>
          tag!("\n") >>
          v: call!(list,&c) >>
          (c,v)
      ), |(c,v)| {
          Data {
              c: c,
              v: v
          }
      }
  ));
}

named!(issue_775, take_till1!(|_| true));

#[test]
fn issue_848_overflow_incomplete_bits_to_bytes() {
  named!(parser<&[u8], &[u8]>, bits!(bytes!(take!(0x2000000000000000))));
  assert_eq!(parser(&b""[..]), Err(Err::Failure(error_position!(&b""[..], ErrorKind::TooLarge))));
}
<commit_msg>fix unit tests<commit_after>//#![feature(trace_macros)]
#![allow(dead_code)]
#![cfg_attr(feature = "cargo-clippy", allow(redundant_closure))]

#[macro_use]
extern crate nom;

use nom::{space, Err, IResult, Needed, ErrorKind, le_u64, is_digit};
use nom::types::{CompleteStr, CompleteByteSlice};

#[allow(dead_code)]
struct Range {
  start: char,
  end: char,
}

pub fn take_char(input: &[u8]) -> IResult<&[u8], char> {
  if !input.is_empty() {
    Ok((&input[1..], input[0] as char))
  } else {
    Err(Err::Incomplete(Needed::Size(1)))
  }
}

//trace_macros!(true);

#[allow(dead_code)]
named!(range<&[u8], Range>,
    alt!(
        do_parse!(
            start: take_char >>
            tag!("-")        >>
            end: take_char   >>
            (Range {
                start: start,
                end:   end,
            })
        ) |
        map!(
            take_char,
            |c| {
                Range {
                    start: c,
                    end:   c,
                }
            }
        )
    )
);

#[allow(dead_code)]
named!(literal<&[u8], Vec<char> >,
    map!(
        many1!(take_char),
        |cs| {
          cs
        }
    )
);

#[test]
fn issue_58() {
  let _ = range(&b"abcd"[..]);
  let _ = literal(&b"abcd"[..]);
}

//trace_macros!(false);

#[cfg(feature = "std")]
mod parse_int {
  use nom::HexDisplay;
  use nom::{digit, space, IResult};
  use std::str;

  named!(parse_ints<Vec<i32>>, many0!(spaces_or_int));

  fn spaces_or_int(input: &[u8]) -> IResult<&[u8], i32> {
    println!("{}", input.to_hex(8));
    do_parse!(
      input,
      opt!(complete!(space)) >> res: map!(complete!(digit), |x| {
        println!("x: {:?}", x);
        let result = str::from_utf8(x).unwrap();
        println!("Result: {}", result);
        println!("int is empty?: {}", x.is_empty());
        match result.parse() {
          Ok(i) => i,
          Err(e) => panic!("UH OH! NOT A DIGIT! {:?}", e),
        }
      }) >> (res)
    )
  }

  #[test]
  fn issue_142() {
    let subject = parse_ints(&b"12 34 5689a"[..]);
    let expected = Ok((&b"a"[..], vec![12, 34, 5689]));
    assert_eq!(subject, expected);

    let subject = parse_ints(&b"12 34 5689 "[..]);
    let expected = Ok((&b" "[..], vec![12, 34, 5689]));
    assert_eq!(subject, expected)
  }
}

#[test]
fn usize_length_bytes_issue() {
  use nom::be_u16;
  let _: IResult<&[u8], &[u8], (&[u8], ErrorKind)> = length_bytes!(b"012346", be_u16);
}

/*
 DOES NOT COMPILE
#[test]
fn issue_152() {
  named!(take4, take!(4));
  named!(xyz, tag!("XYZ"));
  named!(abc, tag!("abc"));

  named!(sw,
    switch!(take4,
      b"abcd" => xyz |
      b"efgh" => abc
    )
  );
}
*/

#[test]
fn take_till_issue() {
  named!(nothing, take_till!(call!(|_| true)));

  assert_eq!(nothing(b""), Err(Err::Incomplete(Needed::Size(1))));
  assert_eq!(nothing(b"abc"), Ok((&b"abc"[..], &b""[..])));
}

named!(
  issue_498<Vec<&[u8]>>,
  separated_nonempty_list!(opt!(space), tag!("abcd"))
);

named!(issue_308(&str) -> bool,
    do_parse! (
        tag! ("foo") >>
        b: alt_complete! (
            map! (tag! ("1"), |_: &str|->bool {true}) |
            value! (false)
        ) >>
        (b) ));

#[cfg(feature = "alloc")]
fn issue_302(input: &[u8]) -> IResult<&[u8], Option<Vec<u64>>> {
  do_parse!(input, entries: cond!(true, count!(le_u64, 3)) >> (entries))
}

#[test]
fn issue_655() {
  use nom::{line_ending, not_line_ending};
  named!(twolines(&str) -> (&str, &str),
    do_parse!(
      l1 : not_line_ending >>
           line_ending >>
      l2 : not_line_ending >>
           line_ending >>
      ((l1, l2))
    )
  );

  assert_eq!(twolines("foo\nbar\n"), Ok(("", ("foo", "bar"))));
  assert_eq!(twolines("féo\nbar\n"), Ok(("", ("féo", "bar"))));
  assert_eq!(twolines("foé\nbar\n"), Ok(("", ("foé", "bar"))));
  assert_eq!(twolines("foé\r\nbar\n"), Ok(("", ("foé", "bar"))));
}

#[cfg(feature = "std")]
named!(issue_666 <CompleteByteSlice, CompleteByteSlice>, dbg_dmp!(tag!("abc")));

#[test]
fn issue_667() {
  use nom::alpha;

  named!(foo <CompleteByteSlice, Vec<CompleteByteSlice>>,
    many0!(
      alt!(alpha | is_a!("_"))
    )
  );
  assert_eq!(
    foo(CompleteByteSlice(b"")),
    Ok((CompleteByteSlice(b""), vec![]))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"loremipsum")),
    Ok((
      CompleteByteSlice(b""),
      vec![CompleteByteSlice(b"loremipsum")]
    ))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"lorem_ipsum")),
    Ok((
      CompleteByteSlice(b""),
      vec![
        CompleteByteSlice(b"lorem"),
        CompleteByteSlice(b"_"),
        CompleteByteSlice(b"ipsum"),
      ]
    ))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"_lorem_ipsum")),
    Ok((
      CompleteByteSlice(b""),
      vec![
        CompleteByteSlice(b"_"),
        CompleteByteSlice(b"lorem"),
        CompleteByteSlice(b"_"),
        CompleteByteSlice(b"ipsum"),
      ]
    ))
  );
  assert_eq!(
    foo(CompleteByteSlice(b"!@#$")),
    Ok((CompleteByteSlice(b"!@#$"), vec![]))
  );
}

#[test]
fn issue_721() {
  named!(f1<&str, u16>, parse_to!(u16));
  named!(f2<&str, String>, parse_to!(String));
  assert_eq!(f1("1234"), Ok(("", 1234)));
  assert_eq!(f2("foo"), Ok(("", "foo".to_string())));
  //assert_eq!(parse_to!("1234", u16), Ok(("", 1234)));
  //assert_eq!(parse_to!("foo", String), Ok(("", "foo".to_string())));
}

#[cfg(feature = "alloc")]
named!(issue_717<&[u8], Vec<&[u8]> >,
  separated_list!(tag!([0x0]), is_not!([0x0u8]))
);

struct NoPartialEq {
  value: i32,
}

named!(issue_724<&str, i32>,
  do_parse!(
    metadata: permutation!(
      map!(tag!("hello"), |_| NoPartialEq { value: 1 }),
      map!(tag!("world"), |_| NoPartialEq { value: 2 })
    ) >>
    (metadata.0.value + metadata.1.value)
  )
);

named!(issue_741_str<CompleteStr, CompleteStr>, re_match!(r"^_?[A-Za-z][0-9A-Z_a-z-]*"));
named!(issue_741_bytes<CompleteByteSlice, CompleteByteSlice>, re_bytes_match!(r"^_?[A-Za-z][0-9A-Z_a-z-]*"));

#[test]
fn issue_752() {
    assert_eq!(
        Err::Error(("ab", nom::ErrorKind::ParseTo)),
        parse_to!("ab", usize).unwrap_err()
    )
}

fn atom_specials(c: u8) -> bool {
    c == b'q'
}

named!(
    capability<&str>,
    do_parse!(tag!(" ") >> _atom: map_res!(take_till1!(atom_specials), std::str::from_utf8) >> ("a"))
);

#[test]
fn issue_759() {
    assert_eq!(capability(b" abcqd"), Ok((&b"qd"[..], "a")));
}

named_args!(issue_771(count: usize)<Vec<u32>>,
  length_count!(value!(count), call!(nom::be_u32))
);

#[test]
fn issue_768() {
  named!(bit_vec8<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 8))));
  named!(bit_vec4<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 4))));
  named!(bit_vec3<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 3))));
  named!(bit_vec11<CompleteByteSlice, Vec<u16>>, bits!(many0!(take_bits!(u16, 11))));

  let m: Vec<u8> = vec![70, 97, 106, 121, 86, 66, 105, 98, 86, 106, 101];
  assert_eq!(
    bit_vec8(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![70, 97, 106, 121, 86, 66, 105, 98, 86, 106, 101]))
  );
  assert_eq!(
    bit_vec4(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![4, 6, 6, 1, 6, 10, 7, 9, 5, 6, 4, 2, 6, 9, 6, 2, 5, 6, 6, 10, 6, 5]))
  );
  assert_eq!(
    bit_vec3(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![2, 1, 4, 6, 0, 5, 5, 2, 3, 6, 2, 5, 3, 1, 0, 2, 3, 2, 2, 6, 1, 1, 2, 6, 3, 2, 4, 6, 2]))
  );
  assert_eq!(
    bit_vec11(CompleteByteSlice(m.as_slice())),
    Ok((CompleteByteSlice(&[]), vec![563, 90, 1266, 1380, 308, 1417, 717, 613]))
  );
}

/// This test is in a separate module to check that all required symbols are imported in
/// `escaped_transform!()`. Without the module, the `use`-es of the current module would
/// mask the error ('"Use of undeclared type or module `Needed`" in escaped_transform!').
mod issue_780 {
  named!(issue_780<&str, String>,
    escaped_transform!(call!(::nom::alpha), '\\', tag!("n"))
  );
}

// issue 617
named!(digits, take_while1!( is_digit ));
named!(multi_617<&[u8], () >, fold_many0!( digits, (), |_, _| {}));

// Sad :(
named!(multi_617_fails<&[u8], () >, fold_many0!( take_while1!( is_digit ), (), |_, _| {}));

mod issue_647 {
  use nom::{Err,be_f64, ErrorKind};
  pub type Input<'a> = &'a [u8];

  #[derive(PartialEq, Debug, Clone)]
  struct Data {
      c: f64,
      v: Vec<f64>
  }

  fn list<'a,'b>(input: Input<'a>, _cs: &'b f64) -> Result<(Input<'a>,Vec<f64>), Err<(&'a [u8], ErrorKind)>> {
      separated_list_complete!(input, tag!(","),be_f64)
  }

  named!(data<Input,Data>, map!(
      do_parse!(
          c: be_f64 >>
          tag!("\n") >>
          v: call!(list,&c) >>
          (c,v)
      ), |(c,v)| {
          Data {
              c: c,
              v: v
          }
      }
  ));
}

named!(issue_775, take_till1!(|_| true));

#[test]
fn issue_848_overflow_incomplete_bits_to_bytes() {
  named!(parser<&[u8], &[u8]>, bits!(bytes!(take!(0x2000000000000000))));
  assert_eq!(parser(&b""[..]), Err(Err::Failure(error_position!(&b""[..], ErrorKind::TooLarge))));
}
<|endoftext|>


<commit_before>#![feature(custom_attribute)]

#[macro_use]
extern crate optional_struct;

#[derive(OptionalStruct)]
#[opt_nested_original(LogConfig)]
#[opt_nested_generated(OptionalLogConfig)]
struct Config {
    timeout: Option<u32>,
    log_config: LogConfig,
}

#[derive(OptionalStruct)]
struct LogConfig {
    log_file: String,
    log_level: usize,
}

#[test]
fn test_apply_options() {
    let mut config = Config {
        timeout: Some(2),
        log_config: LogConfig {
            log_file: "/var/log/foobar.log".to_owned(),
            log_level: 3,
        },
    };

    let opt_config = OptionalConfig {
        timeout: None,
        log_config: OptionalLogConfig {
            log_file: Some("/tmp/bar.log".to_owned()),
            log_level: None,
        },
    };

    config.apply_options(opt_config);

    assert_eq!(config.timeout, None);
    assert_eq!(config.log_config.log_file, "/tmp/bar.log");
    assert_eq!(config.log_config.log_level, 3);
}
<commit_msg>remove unstable feature macro from example/test<commit_after>#[macro_use]
extern crate optional_struct;

#[derive(OptionalStruct)]
#[opt_nested_original(LogConfig)]
#[opt_nested_generated(OptionalLogConfig)]
struct Config {
    timeout: Option<u32>,
    log_config: LogConfig,
}

#[derive(OptionalStruct)]
struct LogConfig {
    log_file: String,
    log_level: usize,
}

#[test]
fn test_apply_options() {
    let mut config = Config {
        timeout: Some(2),
        log_config: LogConfig {
            log_file: "/var/log/foobar.log".to_owned(),
            log_level: 3,
        },
    };

    let opt_config = OptionalConfig {
        timeout: None,
        log_config: OptionalLogConfig {
            log_file: Some("/tmp/bar.log".to_owned()),
            log_level: None,
        },
    };

    config.apply_options(opt_config);

    assert_eq!(config.timeout, None);
    assert_eq!(config.log_config.log_file, "/tmp/bar.log");
    assert_eq!(config.log_config.log_level, 3);
}
<|endoftext|>


<commit_before>#![feature(test)]
extern crate test;

#[macro_use]
extern crate lambda_calculus as lambda;

use test::Bencher;
use lambda::reduction::*;
use lambda::church::numerals::*;

#[bench]
fn successor(b: &mut Bencher) {
    b.iter(|| { beta(app!(succ(), 1.into()), HAP, 0, false) } );
}

#[bench]
fn predecessor(b: &mut Bencher) {
    b.iter(|| { beta(app!(pred(), 1.into()), HAP, 0, false) } );
}

#[bench]
fn subtraction(b: &mut Bencher) {
    b.iter(|| { beta(app!(sub(), 2.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn addition(b: &mut Bencher) {
    b.iter(|| { beta(app!(plus(), 2.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn multiplication(b: &mut Bencher) {
    b.iter(|| { beta(app!(mult(), 2.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn exponentiation(b: &mut Bencher) {
    b.iter(|| { beta(app!(pow(), 2.into(), 2.into()), NOR, 0, false) } );
}

#[bench]
fn division(b: &mut Bencher) {
    b.iter(|| { beta(app!(div(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn quotient(b: &mut Bencher) {
    b.iter(|| { beta(app!(quot(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn remainder(b: &mut Bencher) {
    b.iter(|| { beta(app!(rem(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn factorial(b: &mut Bencher) {
    b.iter(|| { beta(app!(fac(), 3.into()), HAP, 0, false) } );
}
<commit_msg>add more benches<commit_after>#![feature(test)]
extern crate test;

#[macro_use]
extern crate lambda_calculus as lambda;

use test::Bencher;
use lambda::reduction::*;
use lambda::church::numerals::*;

#[bench]
fn successor(b: &mut Bencher) {
    b.iter(|| { beta(app!(succ(), 1.into()), HAP, 0, false) } );
}

#[bench]
fn predecessor(b: &mut Bencher) {
    b.iter(|| { beta(app!(pred(), 1.into()), HAP, 0, false) } );
}

#[bench]
fn subtraction(b: &mut Bencher) {
    b.iter(|| { beta(app!(sub(), 2.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn addition(b: &mut Bencher) {
    b.iter(|| { beta(app!(plus(), 2.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn multiplication(b: &mut Bencher) {
    b.iter(|| { beta(app!(mult(), 2.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn exponentiation(b: &mut Bencher) {
    b.iter(|| { beta(app!(pow(), 2.into(), 2.into()), NOR, 0, false) } );
}

#[bench]
fn division(b: &mut Bencher) {
    b.iter(|| { beta(app!(div(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn quotient(b: &mut Bencher) {
    b.iter(|| { beta(app!(quot(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn remainder(b: &mut Bencher) {
    b.iter(|| { beta(app!(rem(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn compare_eq(b: &mut Bencher) {
    b.iter(|| { beta(app!(eq(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn compare_neq(b: &mut Bencher) {
    b.iter(|| { beta(app!(neq(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn compare_gt(b: &mut Bencher) {
    b.iter(|| { beta(app!(gt(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn compare_lt(b: &mut Bencher) {
    b.iter(|| { beta(app!(lt(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn compare_geq(b: &mut Bencher) {
    b.iter(|| { beta(app!(geq(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn compare_leq(b: &mut Bencher) {
    b.iter(|| { beta(app!(leq(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn minimum(b: &mut Bencher) {
    b.iter(|| { beta(app!(min(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn maximum(b: &mut Bencher) {
    b.iter(|| { beta(app!(max(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn left_shift(b: &mut Bencher) {
    b.iter(|| { beta(app!(lshift(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn right_shift(b: &mut Bencher) {
    b.iter(|| { beta(app!(rshift(), 3.into(), 2.into()), HAP, 0, false) } );
}

#[bench]
fn factorial(b: &mut Bencher) {
    b.iter(|| { beta(app!(fac(), 3.into()), HAP, 0, false) } );
}
<|endoftext|>