<commit_before>use crate::utils::{differing_macro_contexts, in_macro, snippet_opt, span_note_and_lint};
use rustc::lint::{in_external_macro, EarlyContext, EarlyLintPass, LintArray, LintPass};
use rustc::{declare_tool_lint, lint_array};
use syntax::ast;
use syntax::ptr::P;

declare_clippy_lint! {
    /// **What it does:** Checks for use of the non-existent `=*`, `=!` and `=-`
    /// operators.
    ///
    /// **Why is this bad?** This is either a typo of `*=`, `!=` or `-=` or
    /// confusing.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust,ignore
    /// a =- 42; // confusing, should it be `a -= 42` or `a = -42`?
    /// ```
    pub SUSPICIOUS_ASSIGNMENT_FORMATTING,
    style,
    "suspicious formatting of `*=`, `-=` or `!=`"
}

declare_clippy_lint! {
    /// **What it does:** Checks for formatting of `else`. It lints if the `else`
    /// is followed immediately by a newline or the `else` seems to be missing.
    ///
    /// **Why is this bad?** This is probably some refactoring remnant, even if the
    /// code is correct, it might look confusing.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust,ignore
    /// if foo {
    /// } { // looks like an `else` is missing here
    /// }
    ///
    /// if foo {
    /// } if bar { // looks like an `else` is missing here
    /// }
    ///
    /// if foo {
    /// } else
    ///
    /// { // this is the `else` block of the previous `if`, but should it be?
    /// }
    ///
    /// if foo {
    /// } else
    ///
    /// if bar { // this is the `else` block of the previous `if`, but should it be?
    /// }
    /// ```
    pub SUSPICIOUS_ELSE_FORMATTING,
    style,
    "suspicious formatting of `else`"
}

declare_clippy_lint! {
    /// **What it does:** Checks for possible missing comma in an array. It lints if
    /// an array element is a binary operator expression and it lies on two lines.
    ///
    /// **Why is this bad?** This could lead to unexpected results.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust,ignore
    /// let a = &[
    ///     -1, -2, -3 // <= no comma here
    ///     -4, -5, -6
    /// ];
    /// ```
    pub POSSIBLE_MISSING_COMMA,
    correctness,
    "possible missing comma in array"
}

#[derive(Copy, Clone)]
pub struct Formatting;

impl LintPass for Formatting {
    fn get_lints(&self) -> LintArray {
        lint_array!(
            SUSPICIOUS_ASSIGNMENT_FORMATTING,
            SUSPICIOUS_ELSE_FORMATTING,
            POSSIBLE_MISSING_COMMA
        )
    }

    fn name(&self) -> &'static str {
        "Formatting"
    }
}

impl EarlyLintPass for Formatting {
    fn check_block(&mut self, cx: &EarlyContext<'_>, block: &ast::Block) {
        for w in block.stmts.windows(2) {
            match (&w[0].node, &w[1].node) {
                (&ast::StmtKind::Expr(ref first), &ast::StmtKind::Expr(ref second))
                | (&ast::StmtKind::Expr(ref first), &ast::StmtKind::Semi(ref second)) => {
                    check_missing_else(cx, first, second);
                },
                _ => (),
            }
        }
    }

    fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &ast::Expr) {
        check_assign(cx, expr);
        check_else(cx, expr);
        check_array(cx, expr);
    }
}

/// Implementation of the `SUSPICIOUS_ASSIGNMENT_FORMATTING` lint.
fn check_assign(cx: &EarlyContext<'_>, expr: &ast::Expr) {
    if let ast::ExprKind::Assign(ref lhs, ref rhs) = expr.node {
        if !differing_macro_contexts(lhs.span, rhs.span) && !in_macro(lhs.span) {
            let eq_span = lhs.span.between(rhs.span);
            if let ast::ExprKind::Unary(op, ref sub_rhs) = rhs.node {
                if let Some(eq_snippet) = snippet_opt(cx, eq_span) {
                    let op = ast::UnOp::to_string(op);
                    let eqop_span = lhs.span.between(sub_rhs.span);
                    if eq_snippet.ends_with('=') {
                        span_note_and_lint(
                            cx,
                            SUSPICIOUS_ASSIGNMENT_FORMATTING,
                            eqop_span,
                            &format!(
                                "this looks like you are trying to use `.. {op}= ..`, but you \
                                 really are doing `.. = ({op} ..)`",
                                op = op
                            ),
                            eqop_span,
                            &format!("to remove this lint, use either `{op}=` or `= {op}`", op = op),
                        );
                    }
                }
            }
        }
    }
}

/// Implementation of the `SUSPICIOUS_ELSE_FORMATTING` lint for weird `else`.
fn check_else(cx: &EarlyContext<'_>, expr: &ast::Expr) {
    if let Some((then, &Some(ref else_))) = unsugar_if(expr) {
        if (is_block(else_) || unsugar_if(else_).is_some())
            && !differing_macro_contexts(then.span, else_.span)
            && !in_macro(then.span)
            && !in_external_macro(cx.sess, expr.span)
        {
            // workaround for rust-lang/rust#43081
            if expr.span.lo().0 == 0 && expr.span.hi().0 == 0 {
                return;
            }

            // this will be a span from the closing ‘}’ of the “then” block (excluding) to
            // the
            // “if” of the “else if” block (excluding)
            let else_span = then.span.between(else_.span);

            // the snippet should look like " else \n    " with maybe comments anywhere
            // it’s bad when there is a ‘\n’ after the “else”
            if let Some(else_snippet) = snippet_opt(cx, else_span) {
                let else_pos = else_snippet.find("else").expect("there must be a `else` here");

                if else_snippet[else_pos..].contains('\n') {
                    let else_desc = if unsugar_if(else_).is_some() { "if" } else { "{..}" };

                    span_note_and_lint(
                        cx,
                        SUSPICIOUS_ELSE_FORMATTING,
                        else_span,
                        &format!("this is an `else {}` but the formatting might hide it", else_desc),
                        else_span,
                        &format!(
                            "to remove this lint, remove the `else` or remove the new line between \
                             `else` and `{}`",
                            else_desc,
                        ),
                    );
                }
            }
        }
    }
}

fn has_unary_equivalent(bin_op: ast::BinOpKind) -> bool {
    // &, *, -
    bin_op == ast::BinOpKind::And || bin_op == ast::BinOpKind::Mul || bin_op == ast::BinOpKind::Sub
}

/// Implementation of the `POSSIBLE_MISSING_COMMA` lint for array
fn check_array(cx: &EarlyContext<'_>, expr: &ast::Expr) {
    if let ast::ExprKind::Array(ref array) = expr.node {
        for element in array {
            if let ast::ExprKind::Binary(ref op, ref lhs, _) = element.node {
                if has_unary_equivalent(op.node) && !differing_macro_contexts(lhs.span, op.span) {
                    let space_span = lhs.span.between(op.span);
                    if let Some(space_snippet) = snippet_opt(cx, space_span) {
                        let lint_span = lhs.span.with_lo(lhs.span.hi());
                        if space_snippet.contains('\n') {
                            span_note_and_lint(
                                cx,
                                POSSIBLE_MISSING_COMMA,
                                lint_span,
                                "possibly missing a comma here",
                                lint_span,
                                "to remove this lint, add a comma or write the expr in a single line",
                            );
                        }
                    }
                }
            }
        }
    }
}

fn check_missing_else(cx: &EarlyContext<'_>, first: &ast::Expr, second: &ast::Expr) {
    if !differing_macro_contexts(first.span, second.span)
        && !in_macro(first.span)
        && unsugar_if(first).is_some()
        && (is_block(second) || unsugar_if(second).is_some())
    {
        // where the else would be
        let else_span = first.span.between(second.span);

        if let Some(else_snippet) = snippet_opt(cx, else_span) {
            if !else_snippet.contains('\n') {
                let (looks_like, next_thing) = if unsugar_if(second).is_some() {
                    ("an `else if`", "the second `if`")
                } else {
                    ("an `else {..}`", "the next block")
                };

                span_note_and_lint(
                    cx,
                    SUSPICIOUS_ELSE_FORMATTING,
                    else_span,
                    &format!("this looks like {} but the `else` is missing", looks_like),
                    else_span,
                    &format!(
                        "to remove this lint, add the missing `else` or add a new line before {}",
                        next_thing,
                    ),
                );
            }
        }
    }
}

fn is_block(expr: &ast::Expr) -> bool {
    if let ast::ExprKind::Block(..) = expr.node {
        true
    } else {
        false
    }
}

/// Match `if` or `if let` expressions and return the `then` and `else` block.
fn unsugar_if(expr: &ast::Expr) -> Option<(&P<ast::Block>, &Option<P<ast::Expr>>)> {
    match expr.node {
        ast::ExprKind::If(_, ref then, ref else_) | ast::ExprKind::IfLet(_, _, ref then, ref else_) => {
            Some((then, else_))
        },
        _ => None,
    }
}
<commit_msg>Auto merge of #3960 - phansch:fix_except, r=flip1995<commit_after>use crate::utils::{differing_macro_contexts, in_macro, snippet_opt, span_note_and_lint};
use if_chain::if_chain;
use rustc::lint::{in_external_macro, EarlyContext, EarlyLintPass, LintArray, LintPass};
use rustc::{declare_tool_lint, lint_array};
use syntax::ast;
use syntax::ptr::P;

declare_clippy_lint! {
    /// **What it does:** Checks for use of the non-existent `=*`, `=!` and `=-`
    /// operators.
    ///
    /// **Why is this bad?** This is either a typo of `*=`, `!=` or `-=` or
    /// confusing.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust,ignore
    /// a =- 42; // confusing, should it be `a -= 42` or `a = -42`?
    /// ```
    pub SUSPICIOUS_ASSIGNMENT_FORMATTING,
    style,
    "suspicious formatting of `*=`, `-=` or `!=`"
}

declare_clippy_lint! {
    /// **What it does:** Checks for formatting of `else`. It lints if the `else`
    /// is followed immediately by a newline or the `else` seems to be missing.
    ///
    /// **Why is this bad?** This is probably some refactoring remnant, even if the
    /// code is correct, it might look confusing.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust,ignore
    /// if foo {
    /// } { // looks like an `else` is missing here
    /// }
    ///
    /// if foo {
    /// } if bar { // looks like an `else` is missing here
    /// }
    ///
    /// if foo {
    /// } else
    ///
    /// { // this is the `else` block of the previous `if`, but should it be?
    /// }
    ///
    /// if foo {
    /// } else
    ///
    /// if bar { // this is the `else` block of the previous `if`, but should it be?
    /// }
    /// ```
    pub SUSPICIOUS_ELSE_FORMATTING,
    style,
    "suspicious formatting of `else`"
}

declare_clippy_lint! {
    /// **What it does:** Checks for possible missing comma in an array. It lints if
    /// an array element is a binary operator expression and it lies on two lines.
    ///
    /// **Why is this bad?** This could lead to unexpected results.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust,ignore
    /// let a = &[
    ///     -1, -2, -3 // <= no comma here
    ///     -4, -5, -6
    /// ];
    /// ```
    pub POSSIBLE_MISSING_COMMA,
    correctness,
    "possible missing comma in array"
}

#[derive(Copy, Clone)]
pub struct Formatting;

impl LintPass for Formatting {
    fn get_lints(&self) -> LintArray {
        lint_array!(
            SUSPICIOUS_ASSIGNMENT_FORMATTING,
            SUSPICIOUS_ELSE_FORMATTING,
            POSSIBLE_MISSING_COMMA
        )
    }

    fn name(&self) -> &'static str {
        "Formatting"
    }
}

impl EarlyLintPass for Formatting {
    fn check_block(&mut self, cx: &EarlyContext<'_>, block: &ast::Block) {
        for w in block.stmts.windows(2) {
            match (&w[0].node, &w[1].node) {
                (&ast::StmtKind::Expr(ref first), &ast::StmtKind::Expr(ref second))
                | (&ast::StmtKind::Expr(ref first), &ast::StmtKind::Semi(ref second)) => {
                    check_missing_else(cx, first, second);
                },
                _ => (),
            }
        }
    }

    fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &ast::Expr) {
        check_assign(cx, expr);
        check_else(cx, expr);
        check_array(cx, expr);
    }
}

/// Implementation of the `SUSPICIOUS_ASSIGNMENT_FORMATTING` lint.
fn check_assign(cx: &EarlyContext<'_>, expr: &ast::Expr) {
    if let ast::ExprKind::Assign(ref lhs, ref rhs) = expr.node {
        if !differing_macro_contexts(lhs.span, rhs.span) && !in_macro(lhs.span) {
            let eq_span = lhs.span.between(rhs.span);
            if let ast::ExprKind::Unary(op, ref sub_rhs) = rhs.node {
                if let Some(eq_snippet) = snippet_opt(cx, eq_span) {
                    let op = ast::UnOp::to_string(op);
                    let eqop_span = lhs.span.between(sub_rhs.span);
                    if eq_snippet.ends_with('=') {
                        span_note_and_lint(
                            cx,
                            SUSPICIOUS_ASSIGNMENT_FORMATTING,
                            eqop_span,
                            &format!(
                                "this looks like you are trying to use `.. {op}= ..`, but you \
                                 really are doing `.. = ({op} ..)`",
                                op = op
                            ),
                            eqop_span,
                            &format!("to remove this lint, use either `{op}=` or `= {op}`", op = op),
                        );
                    }
                }
            }
        }
    }
}

/// Implementation of the `SUSPICIOUS_ELSE_FORMATTING` lint for weird `else`.
fn check_else(cx: &EarlyContext<'_>, expr: &ast::Expr) {
    if_chain! {
        if let Some((then, &Some(ref else_))) = unsugar_if(expr);
        if is_block(else_) || unsugar_if(else_).is_some();
        if !differing_macro_contexts(then.span, else_.span);
        if !in_macro(then.span) && !in_external_macro(cx.sess, expr.span);

        // workaround for rust-lang/rust#43081
        if expr.span.lo().0 != 0 && expr.span.hi().0 != 0;

        // this will be a span from the closing ‘}’ of the “then” block (excluding) to
        // the “if” of the “else if” block (excluding)
        let else_span = then.span.between(else_.span);

        // the snippet should look like " else \n    " with maybe comments anywhere
        // it’s bad when there is a ‘\n’ after the “else”
        if let Some(else_snippet) = snippet_opt(cx, else_span);
        if let Some(else_pos) = else_snippet.find("else");
        if else_snippet[else_pos..].contains('\n');
        let else_desc = if unsugar_if(else_).is_some() { "if" } else { "{..}" };

        then {
            span_note_and_lint(
                cx,
                SUSPICIOUS_ELSE_FORMATTING,
                else_span,
                &format!("this is an `else {}` but the formatting might hide it", else_desc),
                else_span,
                &format!(
                    "to remove this lint, remove the `else` or remove the new line between \
                     `else` and `{}`",
                    else_desc,
                ),
            );
        }
    }
}

fn has_unary_equivalent(bin_op: ast::BinOpKind) -> bool {
    // &, *, -
    bin_op == ast::BinOpKind::And || bin_op == ast::BinOpKind::Mul || bin_op == ast::BinOpKind::Sub
}

/// Implementation of the `POSSIBLE_MISSING_COMMA` lint for array
fn check_array(cx: &EarlyContext<'_>, expr: &ast::Expr) {
    if let ast::ExprKind::Array(ref array) = expr.node {
        for element in array {
            if let ast::ExprKind::Binary(ref op, ref lhs, _) = element.node {
                if has_unary_equivalent(op.node) && !differing_macro_contexts(lhs.span, op.span) {
                    let space_span = lhs.span.between(op.span);
                    if let Some(space_snippet) = snippet_opt(cx, space_span) {
                        let lint_span = lhs.span.with_lo(lhs.span.hi());
                        if space_snippet.contains('\n') {
                            span_note_and_lint(
                                cx,
                                POSSIBLE_MISSING_COMMA,
                                lint_span,
                                "possibly missing a comma here",
                                lint_span,
                                "to remove this lint, add a comma or write the expr in a single line",
                            );
                        }
                    }
                }
            }
        }
    }
}

fn check_missing_else(cx: &EarlyContext<'_>, first: &ast::Expr, second: &ast::Expr) {
    if !differing_macro_contexts(first.span, second.span)
        && !in_macro(first.span)
        && unsugar_if(first).is_some()
        && (is_block(second) || unsugar_if(second).is_some())
    {
        // where the else would be
        let else_span = first.span.between(second.span);

        if let Some(else_snippet) = snippet_opt(cx, else_span) {
            if !else_snippet.contains('\n') {
                let (looks_like, next_thing) = if unsugar_if(second).is_some() {
                    ("an `else if`", "the second `if`")
                } else {
                    ("an `else {..}`", "the next block")
                };

                span_note_and_lint(
                    cx,
                    SUSPICIOUS_ELSE_FORMATTING,
                    else_span,
                    &format!("this looks like {} but the `else` is missing", looks_like),
                    else_span,
                    &format!(
                        "to remove this lint, add the missing `else` or add a new line before {}",
                        next_thing,
                    ),
                );
            }
        }
    }
}

fn is_block(expr: &ast::Expr) -> bool {
    if let ast::ExprKind::Block(..) = expr.node {
        true
    } else {
        false
    }
}

/// Match `if` or `if let` expressions and return the `then` and `else` block.
fn unsugar_if(expr: &ast::Expr) -> Option<(&P<ast::Block>, &Option<P<ast::Expr>>)> {
    match expr.node {
        ast::ExprKind::If(_, ref then, ref else_) | ast::ExprKind::IfLet(_, _, ref then, ref else_) => {
            Some((then, else_))
        },
        _ => None,
    }
}
<|endoftext|>


<commit_before>// A Regex Operation
//
// Used on the stack to distinguish an operator
// (defined as an opcode), from a parsed
// regular expression.

pub enum OpCode {
  OpConcatenation,
  OpAlternation,
  OpKleine,
  OpZeroOrOne,
  OpOneOrMore,
  OpLineStart,
  OpLineEnd,
  OpLeftParen,
  OpRightParen
}

// Regexp Literal
//
struct RegexpLiteral {
  value: ~str
}

impl RegexpLiteral {
  pub fn new(s: &str) -> RegexpLiteral {
    RegexpLiteral { value: s.clone().to_owned() }
  }
}

struct Regexp {
  op: OpCode,
  state0: Option<~RegexpStack>,
  state1: Option<~RegexpStack>
}

impl Regexp {
  fn new(op: OpCode, state0: Option<~RegexpStack>, state1:
         Option<~RegexpStack>) -> Regexp {
    Regexp { op: op, state0: state0, state1: state1 }
  }
}

// Representations of regexp on
// the stack
enum RegexpStack {
  ReOp(OpCode),
  ReLiteral(RegexpLiteral),
  ReExpression(Regexp)
}

//
pub struct RegexpState {
  priv stack: ~[RegexpStack],
  priv nparen: uint
}

impl RegexpState {
  pub fn new() -> RegexpState {
    RegexpState { stack: ~[], nparen: 0 }
  }
}

impl RegexpState {
  pub fn pushLiteral(&mut self, s: &str) -> () {
    match self.stack.pop_opt() {
      Some(ReLiteral(l)) => {
        self.stack.push(ReLiteral(RegexpLiteral::new(l.value + s)));
      }
      Some(r) => {
        self.stack.push(r);
        self.stack.push(ReLiteral(RegexpLiteral::new(s)));
      }
      None => {
        self.stack.push(ReLiteral(RegexpLiteral::new(s)));
      }
    }
  }
  pub fn pushOperation(&mut self, op: OpCode) -> () {
    self.stack.push(ReOp(op));
  }
  pub fn pushExpression(&mut self, r: Regexp) -> () {
    self.stack.push(ReExpression(r));
  }
}

impl RegexpState {
  pub fn pushAlternation(&mut self) -> () {
    self.pushOperation(OpAlternation);
  }
  pub fn pushLeftParen(&mut self) -> () {
    self.pushOperation(OpLeftParen);
  }
}

impl RegexpState {
  pub fn doAlternation(&mut self) -> Result<bool, &'static str> {
    // try to pop off two items from the stack.
    // these should be branches that you can take.
    //     -> state0
    // s |
    //     -> state1
    //
    // with alternation occuring sequentially, we can
    // make an equivalent model by having state0 or state1
    // also be a regexp with a alternation operand applied to it.
    //     -> state0
    // s |              -> 'state0
    //     -> state1  |
    //                  -> 'state1
    //
    // the operand should be pushed onto the stack before we start parsing
    // the right hand side of the alternation, so the stack should look something
    // like this before doAlternation() is called:
    //
    // state0
    // OpCode(OpAlternation)
    // state1
    let branch1 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to alternate")
    };
    match self.stack.pop_opt() {
      Some(ReOp(OpAlternation)) => { },
      _ => return Err("No alternation operand on the stack, but expected one")
    };
    let branch2 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to alternate")
    };
    let r = Regexp::new(OpAlternation, Some(~branch1), Some(~branch2));

    self.pushExpression(r);

    Ok(true)
  }
  pub fn doConcatenation(&mut self) -> Result<bool, &'static str> {
    // try to take two items off the stack to
    // concatenate.
    //
    // state0 -> state1
    let branch1 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to concatenate")
    };
    let branch2 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to concatenate")
    };
    let r = Regexp::new(OpConcatenation, Some(~branch1), Some(~branch2));

    self.pushExpression(r);

    Ok(true)
  }
  pub fn tryConcatenation(&mut self) {
    // tries to perform a concatenation. this simply
    // means performing a concatenation operation
    // if two or more items are on the stack.
    if (self.stack.len() > 1) {
      self.doConcatenation();
    }
  }
  pub fn doLeftParen(&mut self) {

  }
  pub fn doRightParen(&mut self) {

  }
  pub fn doKleine(&mut self) -> Result<bool, &'static str> {
    match self.stack.pop_opt() {
      Some(r) => {
        let mut r = r;
        // check to see if the expr on the top
        // of the stack has some repition
        // op applied to it.
        let opcode = match &r {
          &ReExpression(ref e) => {
            match e.op {
              OpKleine      => Some(OpKleine),
              OpOneOrMore   => Some(OpOneOrMore),
              OpZeroOrMore  => Some(OpZeroOrMore)
            }
          },
          _ => {
            None
          }
        };
        // if we found that the expr already had
        // a repition op, try to condense (by collapsing
        // overlapping cases...i.e, *?, *+, **, can really
        // just be *).
        //
        // note: in the match block below,
        // the (_) case will cover cases
        // Some(OpOneOrMore), Some(OpZeroOrMore)
        //
        // otherwise, we can make a new expression.
        match opcode {
          Some(OpKleine) => {
            self.stack.push(r);
          }
          None => {
            let expr = Regexp::new(OpKleine, Some(~r), None);
            self.pushExpression(expr);
          }
          _ => {
            match r {
              ReExpression(ref mut e) => {
                e.op = OpKleine;
              }
              _ => { }
            }
            self.stack.push(r);
          }
        }
      }
      _ => {
        return Err("'*' not applied to any state.")
      }
    }
    Ok(true)
  }
}

impl RegexpState {
  pub fn trace(&mut self) {
    println("--STACK--");
    for e in self.stack.iter() {
      println(fmt!("%?", e));
    }
  }
}
<commit_msg>added comments<commit_after>// A Regex Operation
//
pub enum OpCode {
  OpConcatenation,
  OpAlternation,
  OpKleine,
  OpZeroOrOne,
  OpOneOrMore,
  OpLineStart,
  OpLineEnd,
  OpLeftParen,
  OpRightParen
}

// Regexp Literal
// a literal character in a regex string (i.e 'abcd')
struct RegexpLiteral {
  value: ~str
}

impl RegexpLiteral {
  pub fn new(s: &str) -> RegexpLiteral {
    RegexpLiteral { value: s.clone().to_owned() }
  }
}

// Regexp
// represents a variable number of states with an
// operator applied to them.
struct Regexp {
  op: OpCode,
  state0: Option<~RegexpStack>,
  state1: Option<~RegexpStack>
}

impl Regexp {
  fn new(op: OpCode, state0: Option<~RegexpStack>, state1:
         Option<~RegexpStack>) -> Regexp {
    Regexp { op: op, state0: state0, state1: state1 }
  }
}

// RegexpCharClass
// represents a character class (i.e '[a-z123]')
struct RegexpCharClass;

// different representations of expressions on
// the stack
enum RegexpStack {
  ReOp(OpCode),
  ReLiteral(RegexpLiteral),
  ReExpression(Regexp)
}

//
pub struct RegexpState {
  priv stack: ~[RegexpStack],
  priv nparen: uint
}

impl RegexpState {
  pub fn new() -> RegexpState {
    RegexpState { stack: ~[], nparen: 0 }
  }
}

impl RegexpState {
  pub fn pushLiteral(&mut self, s: &str) -> () {
    // special cases to consider that we can optimize for:
    //   * if we have multiple literals in a row, we can condense
    //     that expression into one expression (i.e 'abc' can just
    //     be ReLiteral(abc))
    match self.stack.pop_opt() {
      Some(ReLiteral(l)) => {
        self.stack.push(ReLiteral(RegexpLiteral::new(l.value + s)));
      }
      Some(r) => {
        self.stack.push(r);
        self.stack.push(ReLiteral(RegexpLiteral::new(s)));
      }
      None => {
        self.stack.push(ReLiteral(RegexpLiteral::new(s)));
      }
    }
  }
  pub fn pushOperation(&mut self, op: OpCode) -> () {
    self.stack.push(ReOp(op));
  }
  pub fn pushExpression(&mut self, r: Regexp) -> () {
    self.stack.push(ReExpression(r));
  }
}

impl RegexpState {
  pub fn pushAlternation(&mut self) -> () {
    self.pushOperation(OpAlternation);
  }
  pub fn pushLeftParen(&mut self) -> () {
    self.pushOperation(OpLeftParen);
  }
}

impl RegexpState {
  pub fn doAlternation(&mut self) -> Result<bool, &'static str> {
    // try to pop off two items from the stack.
    // these should be branches that you can take.
    //     -> state0
    // s |
    //     -> state1
    //
    // with alternation occuring sequentially, we can
    // make an equivalent model by having state0 or state1
    // also be a regexp with a alternation operand applied to it.
    //     -> state0
    // s |              -> 'state0
    //     -> state1  |
    //                  -> 'state1
    //
    // the operand should be pushed onto the stack before we start parsing
    // the right hand side of the alternation, so the stack should look something
    // like this before doAlternation() is called:
    //
    // state0
    // OpCode(OpAlternation)
    // state1
    let branch1 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to alternate")
    };
    match self.stack.pop_opt() {
      Some(ReOp(OpAlternation)) => { },
      _ => return Err("No alternation operand on the stack, but expected one")
    };
    let branch2 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to alternate")
    };
    let r = Regexp::new(OpAlternation, Some(~branch1), Some(~branch2));

    self.pushExpression(r);

    Ok(true)
  }
  pub fn doConcatenation(&mut self) -> Result<bool, &'static str> {
    // try to take two items off the stack to
    // concatenate.
    //
    // state0 -> state1
    let branch1 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to concatenate")
    };
    let branch2 = match self.stack.pop_opt() {
      Some(s) => s,
      None => return Err("Nothing to concatenate")
    };
    let r = Regexp::new(OpConcatenation, Some(~branch1), Some(~branch2));

    self.pushExpression(r);

    Ok(true)
  }
  pub fn tryConcatenation(&mut self) {
    // tries to perform a concatenation. this simply
    // means performing a concatenation operation
    // if two or more items are on the stack.
    if (self.stack.len() > 1) {
      self.doConcatenation();
    }
  }
  pub fn doLeftParen(&mut self) {

  }
  pub fn doRightParen(&mut self) {

  }
  pub fn doKleine(&mut self) -> Result<bool, &'static str> {
    match self.stack.pop_opt() {
      Some(r) => {
        let mut r = r;
        // check to see if the expr on the top
        // of the stack has some repition
        // op applied to it.
        let opcode = match &r {
          &ReExpression(ref e) => {
            match e.op {
              OpKleine      => Some(OpKleine),
              OpOneOrMore   => Some(OpOneOrMore),
              OpZeroOrMore  => Some(OpZeroOrMore)
            }
          },
          _ => {
            None
          }
        };
        // if we found that the expr already had
        // a repition op, try to condense (by collapsing
        // overlapping cases...i.e, *?, *+, **, can really
        // just be *).
        //
        // note: in the match block below,
        // the (_) case will cover cases
        // Some(OpOneOrMore), Some(OpZeroOrMore)
        //
        // otherwise, we can make a new expression.
        match opcode {
          Some(OpKleine) => {
            self.stack.push(r);
          }
          None => {
            let expr = Regexp::new(OpKleine, Some(~r), None);
            self.pushExpression(expr);
          }
          _ => {
            match r {
              ReExpression(ref mut e) => {
                e.op = OpKleine;
              }
              _ => { }
            }
            self.stack.push(r);
          }
        }
      }
      _ => {
        return Err("'*' not applied to any state.")
      }
    }
    Ok(true)
  }
}

impl RegexpState {
  pub fn trace(&mut self) {
    println("--STACK--");
    for e in self.stack.iter() {
      println(fmt!("%?", e));
    }
  }
}
<|endoftext|>


<commit_before>#![feature(new_uninit)]
#![feature(slice_as_chunks)]
#![feature(slice_partition_dedup)]

use std::slice;

fn slice_of_zst() {
    fn foo<T>(v: &[T]) -> Option<&[T]> {
        let mut it = v.iter();
        for _ in 0..5 {
            it.next();
        }
        Some(it.as_slice())
    }

    fn foo_mut<T>(v: &mut [T]) -> Option<&mut [T]> {
        let mut it = v.iter_mut();
        for _ in 0..5 {
            it.next();
        }
        Some(it.into_slice())
    }

    // In a slice of zero-size elements the pointer is meaningless.
    // Ensure iteration still works even if the pointer is at the end of the address space.
    let slice: &[()] = unsafe { slice::from_raw_parts(-5isize as *const (), 10) };
    assert_eq!(slice.len(), 10);
    assert_eq!(slice.iter().count(), 10);

    // .nth() on the iterator should also behave correctly
    let mut it = slice.iter();
    assert!(it.nth(5).is_some());
    assert_eq!(it.count(), 4);

    // Converting Iter to a slice should never have a null pointer
    assert!(foo(slice).is_some());

    // Test mutable iterators as well
    let slice: &mut [()] = unsafe { slice::from_raw_parts_mut(-5isize as *mut (), 10) };
    assert_eq!(slice.len(), 10);
    assert_eq!(slice.iter_mut().count(), 10);

    {
        let mut it = slice.iter_mut();
        assert!(it.nth(5).is_some());
        assert_eq!(it.count(), 4);
    }

    assert!(foo_mut(slice).is_some())
}

fn test_iter_ref_consistency() {
    use std::fmt::Debug;

    fn test<T : Copy + Debug + PartialEq>(x : T) {
        let v : &[T] = &[x, x, x];
        let v_ptrs : [*const T; 3] = match v {
            [ref v1, ref v2, ref v3] => [v1 as *const _, v2 as *const _, v3 as *const _],
            _ => unreachable!()
        };
        let len = v.len();

        // nth(i)
        for i in 0..len {
            assert_eq!(&v[i] as *const _, v_ptrs[i]); // check the v_ptrs array, just to be sure
            let nth = v.iter().nth(i).unwrap();
            assert_eq!(nth as *const _, v_ptrs[i]);
        }
        assert_eq!(v.iter().nth(len), None, "nth(len) should return None");

        // stepping through with nth(0)
        {
            let mut it = v.iter();
            for i in 0..len {
                let next = it.nth(0).unwrap();
                assert_eq!(next as *const _, v_ptrs[i]);
            }
            assert_eq!(it.nth(0), None);
        }

        // next()
        {
            let mut it = v.iter();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let next = it.next().unwrap();
                assert_eq!(next as *const _, v_ptrs[i]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next(), None, "The final call to next() should return None");
        }

        // next_back()
        {
            let mut it = v.iter();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let prev = it.next_back().unwrap();
                assert_eq!(prev as *const _, v_ptrs[remaining-1]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next_back(), None, "The final call to next_back() should return None");
        }
    }

    fn test_mut<T : Copy + Debug + PartialEq>(x : T) {
        let v : &mut [T] = &mut [x, x, x];
        let v_ptrs : [*mut T; 3] = match v {
            [ref v1, ref v2, ref v3] =>
              [v1 as *const _ as *mut _, v2 as *const _ as *mut _, v3 as *const _ as *mut _],
            _ => unreachable!()
        };
        let len = v.len();

        // nth(i)
        for i in 0..len {
            assert_eq!(&mut v[i] as *mut _, v_ptrs[i]); // check the v_ptrs array, just to be sure
            let nth = v.iter_mut().nth(i).unwrap();
            assert_eq!(nth as *mut _, v_ptrs[i]);
        }
        assert_eq!(v.iter().nth(len), None, "nth(len) should return None");

        // stepping through with nth(0)
        {
            let mut it = v.iter();
            for i in 0..len {
                let next = it.nth(0).unwrap();
                assert_eq!(next as *const _, v_ptrs[i]);
            }
            assert_eq!(it.nth(0), None);
        }

        // next()
        {
            let mut it = v.iter_mut();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let next = it.next().unwrap();
                assert_eq!(next as *mut _, v_ptrs[i]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next(), None, "The final call to next() should return None");
        }

        // next_back()
        {
            let mut it = v.iter_mut();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let prev = it.next_back().unwrap();
                assert_eq!(prev as *mut _, v_ptrs[remaining-1]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next_back(), None, "The final call to next_back() should return None");
        }
    }

    // Make sure iterators and slice patterns yield consistent addresses for various types,
    // including ZSTs.
    test(0u32);
    test(());
    test([0u32; 0]); // ZST with alignment > 0
    test_mut(0u32);
    test_mut(());
    test_mut([0u32; 0]); // ZST with alignment > 0
}

fn uninit_slice() {
    let mut values = Box::<[Box<u32>]>::new_uninit_slice(3);

    let values = unsafe {
        // Deferred initialization:
        values[0].as_mut_ptr().write(Box::new(1));
        values[1].as_mut_ptr().write(Box::new(2));
        values[2].as_mut_ptr().write(Box::new(3));

        values.assume_init()
    };

    assert_eq!(values.iter().map(|x| **x).collect::<Vec<_>>(), vec![1, 2, 3])
}

/// Regression tests for slice methods in the Rust core library where raw pointers are obtained
/// from mutable references.
fn test_for_invalidated_pointers() {
    let mut buffer = [0usize; 64];
    let len = buffer.len();

    // These regression tests (indirectly) call every slice method which contains a `buffer.as_mut_ptr()`.
    // `<[T]>::as_mut_ptr(&mut self)` takes a mutable reference (tagged Unique), which will invalidate all
    // the other pointers that were previously derived from it according to the Stacked Borrows model.
    // An example of where this could go wrong is a prior bug inside `<[T]>::copy_within`:
    //
    //      unsafe {
    //          core::ptr::copy(self.as_ptr().add(src_start), self.as_mut_ptr().add(dest), count);
    //      }
    //
    // The arguments to `core::ptr::copy` are evaluated from left to right. `self.as_ptr()` creates
    // an immutable reference (which is tagged as `SharedReadOnly` by Stacked Borrows) to the array
    // and derives a valid `*const` pointer from it. When jumping to the next argument,
    // `self.as_mut_ptr()` creates a mutable reference (tagged as `Unique`) to the array, which
    // invalidates the existing `SharedReadOnly` reference and any pointers derived from it.
    // The invalidated `*const` pointer (the first argument to `core::ptr::copy`) is then used
    // after the fact when `core::ptr::copy` is called, which triggers undefined behavior.

    unsafe { assert_eq!(0, *buffer.as_mut_ptr_range().start ); }
    // Check that the pointer range is in-bounds, while we're at it
    let range = buffer.as_mut_ptr_range();
    unsafe { assert_eq!(*range.start, *range.end.sub(len)); }

    buffer.reverse();

    // Calls `fn as_chunks_unchecked_mut` internally (requires unstable `#![feature(slice_as_chunks)]`):
    assert_eq!(2, buffer.as_chunks_mut::<32>().0.len());

    // Calls `fn split_at_mut_unchecked` internally:
    let split_mut = buffer.split_at_mut(32);
    assert_eq!(split_mut.0, split_mut.1);

    // Calls `fn partition_dedup_by` internally (requires unstable `#![feature(slice_partition_dedup)]`):
    assert_eq!(1, buffer.partition_dedup().0.len());

    buffer.rotate_left(8);
    buffer.rotate_right(16);

    buffer.copy_from_slice(&[1usize; 64]);
    buffer.swap_with_slice(&mut [2usize; 64]);

    assert_eq!(0, unsafe { buffer.align_to_mut::<u8>().1[1] });

    buffer.copy_within(1.., 0);
}

fn main() {
    slice_of_zst();
    test_iter_ref_consistency();
    uninit_slice();
    test_for_invalidated_pointers();
}
<commit_msg>use references so that potential aliasing bugs are triggered during regression test<commit_after>// compile-flags: -Zmiri-track-raw-pointers
#![feature(new_uninit)]
#![feature(slice_as_chunks)]
#![feature(slice_partition_dedup)]

use std::slice;

fn slice_of_zst() {
    fn foo<T>(v: &[T]) -> Option<&[T]> {
        let mut it = v.iter();
        for _ in 0..5 {
            it.next();
        }
        Some(it.as_slice())
    }

    fn foo_mut<T>(v: &mut [T]) -> Option<&mut [T]> {
        let mut it = v.iter_mut();
        for _ in 0..5 {
            it.next();
        }
        Some(it.into_slice())
    }

    // In a slice of zero-size elements the pointer is meaningless.
    // Ensure iteration still works even if the pointer is at the end of the address space.
    let slice: &[()] = unsafe { slice::from_raw_parts(-5isize as *const (), 10) };
    assert_eq!(slice.len(), 10);
    assert_eq!(slice.iter().count(), 10);

    // .nth() on the iterator should also behave correctly
    let mut it = slice.iter();
    assert!(it.nth(5).is_some());
    assert_eq!(it.count(), 4);

    // Converting Iter to a slice should never have a null pointer
    assert!(foo(slice).is_some());

    // Test mutable iterators as well
    let slice: &mut [()] = unsafe { slice::from_raw_parts_mut(-5isize as *mut (), 10) };
    assert_eq!(slice.len(), 10);
    assert_eq!(slice.iter_mut().count(), 10);

    {
        let mut it = slice.iter_mut();
        assert!(it.nth(5).is_some());
        assert_eq!(it.count(), 4);
    }

    assert!(foo_mut(slice).is_some())
}

fn test_iter_ref_consistency() {
    use std::fmt::Debug;

    fn test<T : Copy + Debug + PartialEq>(x : T) {
        let v : &[T] = &[x, x, x];
        let v_ptrs : [*const T; 3] = match v {
            [ref v1, ref v2, ref v3] => [v1 as *const _, v2 as *const _, v3 as *const _],
            _ => unreachable!()
        };
        let len = v.len();

        // nth(i)
        for i in 0..len {
            assert_eq!(&v[i] as *const _, v_ptrs[i]); // check the v_ptrs array, just to be sure
            let nth = v.iter().nth(i).unwrap();
            assert_eq!(nth as *const _, v_ptrs[i]);
        }
        assert_eq!(v.iter().nth(len), None, "nth(len) should return None");

        // stepping through with nth(0)
        {
            let mut it = v.iter();
            for i in 0..len {
                let next = it.nth(0).unwrap();
                assert_eq!(next as *const _, v_ptrs[i]);
            }
            assert_eq!(it.nth(0), None);
        }

        // next()
        {
            let mut it = v.iter();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let next = it.next().unwrap();
                assert_eq!(next as *const _, v_ptrs[i]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next(), None, "The final call to next() should return None");
        }

        // next_back()
        {
            let mut it = v.iter();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let prev = it.next_back().unwrap();
                assert_eq!(prev as *const _, v_ptrs[remaining-1]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next_back(), None, "The final call to next_back() should return None");
        }
    }

    fn test_mut<T : Copy + Debug + PartialEq>(x : T) {
        let v : &mut [T] = &mut [x, x, x];
        let v_ptrs : [*mut T; 3] = match v {
            [ref v1, ref v2, ref v3] =>
              [v1 as *const _ as *mut _, v2 as *const _ as *mut _, v3 as *const _ as *mut _],
            _ => unreachable!()
        };
        let len = v.len();

        // nth(i)
        for i in 0..len {
            assert_eq!(&mut v[i] as *mut _, v_ptrs[i]); // check the v_ptrs array, just to be sure
            let nth = v.iter_mut().nth(i).unwrap();
            assert_eq!(nth as *mut _, v_ptrs[i]);
        }
        assert_eq!(v.iter().nth(len), None, "nth(len) should return None");

        // stepping through with nth(0)
        {
            let mut it = v.iter();
            for i in 0..len {
                let next = it.nth(0).unwrap();
                assert_eq!(next as *const _, v_ptrs[i]);
            }
            assert_eq!(it.nth(0), None);
        }

        // next()
        {
            let mut it = v.iter_mut();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let next = it.next().unwrap();
                assert_eq!(next as *mut _, v_ptrs[i]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next(), None, "The final call to next() should return None");
        }

        // next_back()
        {
            let mut it = v.iter_mut();
            for i in 0..len {
                let remaining = len - i;
                assert_eq!(it.size_hint(), (remaining, Some(remaining)));

                let prev = it.next_back().unwrap();
                assert_eq!(prev as *mut _, v_ptrs[remaining-1]);
            }
            assert_eq!(it.size_hint(), (0, Some(0)));
            assert_eq!(it.next_back(), None, "The final call to next_back() should return None");
        }
    }

    // Make sure iterators and slice patterns yield consistent addresses for various types,
    // including ZSTs.
    test(0u32);
    test(());
    test([0u32; 0]); // ZST with alignment > 0
    test_mut(0u32);
    test_mut(());
    test_mut([0u32; 0]); // ZST with alignment > 0
}

fn uninit_slice() {
    let mut values = Box::<[Box<u32>]>::new_uninit_slice(3);

    let values = unsafe {
        // Deferred initialization:
        values[0].as_mut_ptr().write(Box::new(1));
        values[1].as_mut_ptr().write(Box::new(2));
        values[2].as_mut_ptr().write(Box::new(3));

        values.assume_init()
    };

    assert_eq!(values.iter().map(|x| **x).collect::<Vec<_>>(), vec![1, 2, 3])
}

/// Regression tests for slice methods in the Rust core library where raw pointers are obtained
/// from mutable references.
fn test_for_invalidated_pointers() {
    let mut buffer = [0usize; 64];
    let len = buffer.len();

    // These regression tests (indirectly) call every slice method which contains a `buffer.as_mut_ptr()`.
    // `<[T]>::as_mut_ptr(&mut self)` takes a mutable reference (tagged Unique), which will invalidate all
    // the other pointers that were previously derived from it according to the Stacked Borrows model.
    // An example of where this could go wrong is a prior bug inside `<[T]>::copy_within`:
    //
    //      unsafe {
    //          core::ptr::copy(self.as_ptr().add(src_start), self.as_mut_ptr().add(dest), count);
    //      }
    //
    // The arguments to `core::ptr::copy` are evaluated from left to right. `self.as_ptr()` creates
    // an immutable reference (which is tagged as `SharedReadOnly` by Stacked Borrows) to the array
    // and derives a valid `*const` pointer from it. When jumping to the next argument,
    // `self.as_mut_ptr()` creates a mutable reference (tagged as `Unique`) to the array, which
    // invalidates the existing `SharedReadOnly` reference and any pointers derived from it.
    // The invalidated `*const` pointer (the first argument to `core::ptr::copy`) is then used
    // after the fact when `core::ptr::copy` is called, which triggers undefined behavior.

    unsafe { assert_eq!(0, *buffer.as_mut_ptr_range().start ); }
    // Check that the pointer range is in-bounds, while we're at it
    let range = buffer.as_mut_ptr_range();
    unsafe { assert_eq!(*range.start, *range.end.sub(len)); }

    buffer.reverse();

    // Calls `fn as_chunks_unchecked_mut` internally (requires unstable `#![feature(slice_as_chunks)]`):
    assert_eq!(2, buffer.as_chunks_mut::<32>().0.len());
    for chunk in buffer.as_chunks_mut::<32>().0 {
        for elem in chunk {
            *elem += 1;
        }
    }

    // Calls `fn split_at_mut_unchecked` internally:
    let split_mut = buffer.split_at_mut(32);
    assert_eq!(split_mut.0, split_mut.1);

    // Calls `fn partition_dedup_by` internally (requires unstable `#![feature(slice_partition_dedup)]`):
    let partition_dedup = buffer.partition_dedup();
    assert_eq!(1, partition_dedup.0.len());
    partition_dedup.0[0] += 1;
    for elem in partition_dedup.1 {
        *elem += 1;
    }

    buffer.rotate_left(8);
    buffer.rotate_right(16);

    buffer.copy_from_slice(&[1usize; 64]);
    buffer.swap_with_slice(&mut [2usize; 64]);

    assert_eq!(0, unsafe { buffer.align_to_mut::<u8>().1[1] });

    buffer.copy_within(1.., 0);
}

fn main() {
    slice_of_zst();
    test_iter_ref_consistency();
    uninit_slice();
    test_for_invalidated_pointers();
}
<|endoftext|>


<commit_before>//! `mbe` (short for Macro By Example) crate contains code for handling
//! `macro_rules` macros. It uses `TokenTree` (from `ra_tt` package) as the
//! interface, although it contains some code to bridge `SyntaxNode`s and
//! `TokenTree`s as well!

mod parser;
mod mbe_expander;
mod syntax_bridge;
mod tt_iter;
mod subtree_source;

pub use tt::{Delimiter, Punct};

use crate::{
    parser::{parse_pattern, Op},
    tt_iter::TtIter,
};

#[derive(Debug, PartialEq, Eq)]
pub enum ParseError {
    Expected(String),
}

#[derive(Debug, PartialEq, Eq)]
pub enum ExpandError {
    NoMatchingRule,
    UnexpectedToken,
    BindingError(String),
    ConversionError,
    InvalidRepeat,
}

pub use crate::syntax_bridge::{
    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, TokenMap,
};

/// This struct contains AST for a single `macro_rules` definition. What might
/// be very confusing is that AST has almost exactly the same shape as
/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MacroRules {
    rules: Vec<Rule>,
    /// Highest id of the token we have in TokenMap
    shift: Shift,
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct Rule {
    lhs: tt::Subtree,
    rhs: tt::Subtree,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Shift(u32);

impl Shift {
    fn new(tt: &tt::Subtree) -> Shift {
        // Note that TokenId is started from zero,
        // We have to add 1 to prevent duplication.
        let value = max_id(tt).map_or(0, |it| it + 1);
        return Shift(value);

        // Find the max token id inside a subtree
        fn max_id(subtree: &tt::Subtree) -> Option<u32> {
            subtree
                .token_trees
                .iter()
                .filter_map(|tt| match tt {
                    tt::TokenTree::Subtree(subtree) => max_id(subtree),
                    tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
                        if ident.id != tt::TokenId::unspecified() =>
                    {
                        Some(ident.id.0)
                    }
                    _ => None,
                })
                .max()
        }
    }

    /// Shift given TokenTree token id
    fn shift_all(self, tt: &mut tt::Subtree) {
        for t in tt.token_trees.iter_mut() {
            match t {
                tt::TokenTree::Leaf(leaf) => match leaf {
                    tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id),
                    _ => (),
                },
                tt::TokenTree::Subtree(tt) => self.shift_all(tt),
            }
        }
    }

    fn shift(self, id: tt::TokenId) -> tt::TokenId {
        if id == tt::TokenId::unspecified() {
            return id;
        }
        tt::TokenId(id.0 + self.0)
    }

    fn unshift(self, id: tt::TokenId) -> Option<tt::TokenId> {
        id.0.checked_sub(self.0).map(tt::TokenId)
    }
}

#[derive(Debug, Eq, PartialEq)]
pub enum Origin {
    Def,
    Call,
}

impl MacroRules {
    pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
        // Note: this parsing can be implemented using mbe machinery itself, by
        // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
        // manually seems easier.
        let mut src = TtIter::new(tt);
        let mut rules = Vec::new();
        while src.len() > 0 {
            let rule = Rule::parse(&mut src)?;
            rules.push(rule);
            if let Err(()) = src.expect_char(';') {
                if src.len() > 0 {
                    return Err(ParseError::Expected("expected `:`".to_string()));
                }
                break;
            }
        }

        for rule in rules.iter() {
            validate(&rule.lhs)?;
        }

        Ok(MacroRules { rules, shift: Shift::new(tt) })
    }

    pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
        // apply shift
        let mut tt = tt.clone();
        self.shift.shift_all(&mut tt);
        mbe_expander::expand(self, &tt)
    }

    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
        self.shift.shift(id)
    }

    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) {
        match self.shift.unshift(id) {
            Some(id) => (id, Origin::Call),
            None => (id, Origin::Def),
        }
    }
}

impl Rule {
    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
        let mut lhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        lhs.delimiter = None;
        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
        let mut rhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        rhs.delimiter = None;
        Ok(crate::Rule { lhs, rhs })
    }
}

fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
    for op in parse_pattern(pattern) {
        let op = match op {
            Ok(it) => it,
            Err(e) => {
                let msg = match e {
                    ExpandError::InvalidRepeat => "invalid repeat".to_string(),
                    _ => "invalid macro definition".to_string(),
                };
                return Err(ParseError::Expected(msg));
            }
        };
        match op {
            Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
                validate(subtree)?
            }
            _ => (),
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests;
<commit_msg>Fix shift id for delim and other tokens<commit_after>//! `mbe` (short for Macro By Example) crate contains code for handling
//! `macro_rules` macros. It uses `TokenTree` (from `ra_tt` package) as the
//! interface, although it contains some code to bridge `SyntaxNode`s and
//! `TokenTree`s as well!

mod parser;
mod mbe_expander;
mod syntax_bridge;
mod tt_iter;
mod subtree_source;

pub use tt::{Delimiter, Punct};

use crate::{
    parser::{parse_pattern, Op},
    tt_iter::TtIter,
};

#[derive(Debug, PartialEq, Eq)]
pub enum ParseError {
    Expected(String),
}

#[derive(Debug, PartialEq, Eq)]
pub enum ExpandError {
    NoMatchingRule,
    UnexpectedToken,
    BindingError(String),
    ConversionError,
    InvalidRepeat,
}

pub use crate::syntax_bridge::{
    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, TokenMap,
};

/// This struct contains AST for a single `macro_rules` definition. What might
/// be very confusing is that AST has almost exactly the same shape as
/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MacroRules {
    rules: Vec<Rule>,
    /// Highest id of the token we have in TokenMap
    shift: Shift,
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct Rule {
    lhs: tt::Subtree,
    rhs: tt::Subtree,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Shift(u32);

impl Shift {
    fn new(tt: &tt::Subtree) -> Shift {
        // Note that TokenId is started from zero,
        // We have to add 1 to prevent duplication.
        let value = max_id(tt).map_or(0, |it| it + 1);
        return Shift(value);

        // Find the max token id inside a subtree
        fn max_id(subtree: &tt::Subtree) -> Option<u32> {
            subtree
                .token_trees
                .iter()
                .filter_map(|tt| match tt {
                    tt::TokenTree::Subtree(subtree) => {
                        let tree_id = max_id(subtree);
                        match subtree.delimiter {
                            Some(it) if it.id != tt::TokenId::unspecified() => {
                                Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0)))
                            }
                            _ => tree_id,
                        }
                    }
                    tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
                        if ident.id != tt::TokenId::unspecified() =>
                    {
                        Some(ident.id.0)
                    }
                    _ => None,
                })
                .max()
        }
    }

    /// Shift given TokenTree token id
    fn shift_all(self, tt: &mut tt::Subtree) {
        for t in tt.token_trees.iter_mut() {
            match t {
                tt::TokenTree::Leaf(leaf) => match leaf {
                    tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id),
                    tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id),
                    tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id),
                },
                tt::TokenTree::Subtree(tt) => {
                    tt.delimiter.as_mut().map(|it: &mut Delimiter| it.id = self.shift(it.id));
                    self.shift_all(tt)
                }
            }
        }
    }

    fn shift(self, id: tt::TokenId) -> tt::TokenId {
        if id == tt::TokenId::unspecified() {
            return id;
        }
        tt::TokenId(id.0 + self.0)
    }

    fn unshift(self, id: tt::TokenId) -> Option<tt::TokenId> {
        id.0.checked_sub(self.0).map(tt::TokenId)
    }
}

#[derive(Debug, Eq, PartialEq)]
pub enum Origin {
    Def,
    Call,
}

impl MacroRules {
    pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
        // Note: this parsing can be implemented using mbe machinery itself, by
        // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
        // manually seems easier.
        let mut src = TtIter::new(tt);
        let mut rules = Vec::new();
        while src.len() > 0 {
            let rule = Rule::parse(&mut src)?;
            rules.push(rule);
            if let Err(()) = src.expect_char(';') {
                if src.len() > 0 {
                    return Err(ParseError::Expected("expected `:`".to_string()));
                }
                break;
            }
        }

        for rule in rules.iter() {
            validate(&rule.lhs)?;
        }

        Ok(MacroRules { rules, shift: Shift::new(tt) })
    }

    pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
        // apply shift
        let mut tt = tt.clone();
        self.shift.shift_all(&mut tt);
        mbe_expander::expand(self, &tt)
    }

    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
        self.shift.shift(id)
    }

    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) {
        match self.shift.unshift(id) {
            Some(id) => (id, Origin::Call),
            None => (id, Origin::Def),
        }
    }
}

impl Rule {
    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
        let mut lhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        lhs.delimiter = None;
        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
        let mut rhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        rhs.delimiter = None;
        Ok(crate::Rule { lhs, rhs })
    }
}

fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
    for op in parse_pattern(pattern) {
        let op = match op {
            Ok(it) => it,
            Err(e) => {
                let msg = match e {
                    ExpandError::InvalidRepeat => "invalid repeat".to_string(),
                    _ => "invalid macro definition".to_string(),
                };
                return Err(ParseError::Expected(msg));
            }
        };
        match op {
            Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
                validate(subtree)?
            }
            _ => (),
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests;
<|endoftext|>


<commit_before>mod builtin_functions;
mod compiler;
mod interpreter;
mod language_server;

use crate::compiler::ast_to_hir::CompileVecAstsToHir;
use crate::compiler::cst_to_ast::LowerCstToAst;
use crate::compiler::string_to_cst::StringToCst;
use crate::interpreter::fiber::FiberStatus;
use crate::interpreter::*;
use language_server::CandyLanguageServer;
use log;
use lspower::{LspService, Server};
use simplelog::{
    ColorChoice, CombinedLogger, Config, LevelFilter, TermLogger, TerminalMode, WriteLogger,
};
use std::{fs::File, path::PathBuf};
use structopt::StructOpt;

#[derive(StructOpt, Debug)]
#[structopt(name = "candy", about = "The 🍭 Candy CLI.")]
enum CandyOptions {
    Run(CandyRunOptions),
    Lsp,
}

#[derive(StructOpt, Debug)]
struct CandyRunOptions {
    #[structopt(long)]
    print_cst: bool,

    #[structopt(long)]
    print_ast: bool,

    #[structopt(long)]
    print_hir: bool,

    #[structopt(long)]
    no_run: bool,

    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[tokio::main]
async fn main() {
    match CandyOptions::from_args() {
        CandyOptions::Run(options) => run(options),
        CandyOptions::Lsp => lsp().await,
    }
}

fn run(options: CandyRunOptions) {
    init_logger(TerminalMode::Mixed);
    log::debug!("Running `{}`.\n", options.file.to_string_lossy());

    let test_code = std::fs::read_to_string(options.file.clone())
        .unwrap_or_else(|_| panic!("File `{}` not found.", options.file.to_string_lossy()));

    log::info!("Parsing string to CST…");
    let cst = test_code.parse_cst();
    if options.print_cst {
        log::info!("CST: {:?}", cst);
    }

    log::info!("Lowering CST to AST…");
    let (asts, errors) = cst.into_ast();
    if options.print_ast {
        log::info!("AST: {:?}", asts);
    }
    if !errors.is_empty() {
        log::error!("Errors occurred while lowering CST to AST:\n{:?}", errors);
        return;
    }

    log::info!("Compiling AST to HIR…");
    let lambda = asts.compile_to_hir();
    if options.print_hir {
        log::info!("HIR: {:?}", lambda);
    }

    if !options.no_run {
        log::info!("Executing code…");
        let mut fiber = fiber::Fiber::new(lambda);
        fiber.run();
        match fiber.status() {
            FiberStatus::Running => log::info!("Fiber is still running."),
            FiberStatus::Done(value) => log::info!("Fiber is done: {:?}", value),
            FiberStatus::Panicked(value) => log::error!("Fiber panicked: {:?}", value),
        }
    }
}

async fn lsp() {
    init_logger(TerminalMode::Stderr);
    log::info!("Starting language server…");
    let (service, messages) = LspService::new(|client| CandyLanguageServer::from_client(client));
    Server::new(tokio::io::stdin(), tokio::io::stdout())
        .interleave(messages)
        .serve(service)
        .await;
}

fn init_logger(terminal_mode: TerminalMode) {
    TermLogger::init(
        LevelFilter::Trace,
        Config::default(),
        terminal_mode,
        ColorChoice::Auto,
    )
    .unwrap();
}
<commit_msg>feat(compiler): format output pretty<commit_after>mod builtin_functions;
mod compiler;
mod interpreter;
mod language_server;

use crate::compiler::ast_to_hir::CompileVecAstsToHir;
use crate::compiler::cst_to_ast::LowerCstToAst;
use crate::compiler::string_to_cst::StringToCst;
use crate::interpreter::fiber::FiberStatus;
use crate::interpreter::*;
use language_server::CandyLanguageServer;
use log;
use lspower::{LspService, Server};
use simplelog::{
    ColorChoice, CombinedLogger, Config, LevelFilter, TermLogger, TerminalMode, WriteLogger,
};
use std::{fs::File, path::PathBuf};
use structopt::StructOpt;

#[derive(StructOpt, Debug)]
#[structopt(name = "candy", about = "The 🍭 Candy CLI.")]
enum CandyOptions {
    Run(CandyRunOptions),
    Lsp,
}

#[derive(StructOpt, Debug)]
struct CandyRunOptions {
    #[structopt(long)]
    print_cst: bool,

    #[structopt(long)]
    print_ast: bool,

    #[structopt(long)]
    print_hir: bool,

    #[structopt(long)]
    no_run: bool,

    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[tokio::main]
async fn main() {
    match CandyOptions::from_args() {
        CandyOptions::Run(options) => run(options),
        CandyOptions::Lsp => lsp().await,
    }
}

fn run(options: CandyRunOptions) {
    init_logger(TerminalMode::Mixed);
    log::debug!("Running `{}`.\n", options.file.to_string_lossy());

    let test_code = std::fs::read_to_string(options.file.clone())
        .unwrap_or_else(|_| panic!("File `{}` not found.", options.file.to_string_lossy()));

    log::info!("Parsing string to CST…");
    let cst = test_code.parse_cst();
    if options.print_cst {
        log::info!("CST: {:#?}", cst);
    }

    log::info!("Lowering CST to AST…");
    let (asts, errors) = cst.into_ast();
    if options.print_ast {
        log::info!("AST: {:#?}", asts);
    }
    if !errors.is_empty() {
        log::error!("Errors occurred while lowering CST to AST:\n{:#?}", errors);
        return;
    }

    log::info!("Compiling AST to HIR…");
    let lambda = asts.compile_to_hir();
    if options.print_hir {
        log::info!("HIR: {:#?}", lambda);
    }

    if !options.no_run {
        log::info!("Executing code…");
        let mut fiber = fiber::Fiber::new(lambda);
        fiber.run();
        match fiber.status() {
            FiberStatus::Running => log::info!("Fiber is still running."),
            FiberStatus::Done(value) => log::info!("Fiber is done: {:#?}", value),
            FiberStatus::Panicked(value) => log::error!("Fiber panicked: {:#?}", value),
        }
    }
}

async fn lsp() {
    init_logger(TerminalMode::Stderr);
    log::info!("Starting language server…");
    let (service, messages) = LspService::new(|client| CandyLanguageServer::from_client(client));
    Server::new(tokio::io::stdin(), tokio::io::stdout())
        .interleave(messages)
        .serve(service)
        .await;
}

fn init_logger(terminal_mode: TerminalMode) {
    TermLogger::init(
        LevelFilter::Trace,
        Config::default(),
        terminal_mode,
        ColorChoice::Auto,
    )
    .unwrap();
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::fs::File;
use std::io;
use std::io::BufReader;
use std::io::BufWriter;
use std::io::Read;
use std::io::Write;
use std::io::Seek;
use std::path::Path;

extern crate env_logger;
extern crate tools;
use tools::find_source_file;
use tools::file_format::analysis::{read_analysis, read_source, read_jumps};
use tools::format::format_file_data;
use tools::git_ops;
use tools::config;
use tools::languages;
use tools::describe;
use languages::FormatAs;

use tools::output::{PanelItem, PanelSection};

extern crate rustc_serialize;
use rustc_serialize::json;

fn read_json_from_file(path: &str) -> Option<json::Object> {
    let components_file = File::open(path).ok()?;
    let mut reader = BufReader::new(&components_file);
    json::Json::from_reader(&mut reader).ok()?.into_object()
}

/// For a given path, looks up the bugzilla product and component and
/// returns it in a tuple if it could be found. The JSON data format
/// comes from https://searchfox.org/mozilla-central/rev/47edbd91c43db6229cf32d1fc4bae9b325b9e2d0/python/mozbuild/mozbuild/frontend/mach_commands.py#209-223,243
/// and is fairly straightforward.
fn get_bugzilla_component<'a>(bugzilla_data: &'a json::Object, path: &str) -> Option<(&'a str, &'a str)> {
    let mut path_obj = bugzilla_data.get("paths")?;
    for path_component in path.split('/') {
        path_obj = path_obj.as_object()?.get(path_component)?;
    }
    let component_id = path_obj.as_i64()?.to_string();
    let mut result_iter = bugzilla_data.get("components")?.as_object()?.get(&component_id)?.as_array()?.iter();
    let product = result_iter.next()?.as_string()?;
    let component = result_iter.next()?.as_string()?;
    Some((product, component))
}

fn main() {
    env_logger::init();

    let args: Vec<_> = env::args().collect();
    let (base_args, fname_args) = args.split_at(3);

    let cfg = config::load(&base_args[1], true);
    println!("Config file read");

    let tree_name = &base_args[2];
    let tree_config = cfg.trees.get(tree_name).unwrap();

    let jumps_fname = format!("{}/jumps", tree_config.paths.index_path);
    //let jumps : std::collections::HashMap<String, tools::analysis::Jump> = std::collections::HashMap::new();
    let jumps = read_jumps(&jumps_fname);
    println!("Jumps read");

    let bugzilla_fname = format!("{}/bugzilla-components.json", tree_config.paths.index_path);
    let bugzilla_data = read_json_from_file(&bugzilla_fname);
    if bugzilla_data.is_some() {
        println!("Bugzilla components read");
    } else {
        println!("No bugzilla-components.json file found");
    }

    let (blame_commit, head_oid) = match &tree_config.git {
        &Some(ref git) => {
            let head_oid = git.repo.refname_to_id("HEAD").unwrap();
            let blame_commit = if let Some(ref blame_repo) = git.blame_repo {
                let blame_oid = blame_repo.refname_to_id("HEAD").unwrap();
                Some(blame_repo.find_commit(blame_oid).unwrap())
            } else {
                None
            };
            (blame_commit, Some(head_oid))
        },
        &None => (None, None),
    };

    let head_commit = head_oid
        .and_then(|oid| tree_config.git.as_ref().unwrap().repo.find_commit(oid).ok());

    let mut extension_mapping = HashMap::new();
    extension_mapping.insert("cpp", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cc", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cxx", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("h", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hh", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hpp", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hxx", ("source", vec!["cpp", "cc", "cxx"]));

    let mut diff_cache = git_ops::TreeDiffCache::new();
    for path in fname_args {
        println!("File {}", path);

        let output_fname = format!("{}/file/{}", tree_config.paths.index_path, path);
        let source_fname = find_source_file(path, &tree_config.paths.files_path, &tree_config.paths.objdir_path);

        let format = languages::select_formatting(path);

        let output_file = File::create(output_fname).unwrap();
        let mut writer = BufWriter::new(output_file);

        let source_file = match File::open(source_fname.clone()) {
            Ok(f) => f,
            Err(_) => {
                println!("Unable to open file");
                continue;
            },
        };

        let path_wrapper = Path::new(&source_fname);
        let metadata = fs::symlink_metadata(path_wrapper).unwrap();
        if metadata.file_type().is_symlink() {
            let dest = fs::read_link(path_wrapper).unwrap();
            write!(writer, "Symlink to {}", dest.to_str().unwrap()).unwrap();
            continue;
        }

        let mut reader = BufReader::new(&source_file);

        match format {
            FormatAs::Binary => {
                let _ = io::copy(&mut reader, &mut writer);
                continue;
            },
            _ => {},
        };

        let analysis_fname = format!("{}/analysis/{}", tree_config.paths.index_path, path);
        let analysis = read_analysis(&analysis_fname, &mut read_source);

        let mut input = String::new();
        match reader.read_to_string(&mut input) {
            Ok(_) => {},
            Err(_) => {
                let mut bytes = Vec::new();
                reader.seek(std::io::SeekFrom::Start(0)).unwrap();
                match reader.read_to_end(&mut bytes) {
                    Ok(_) => {
                        input.push_str(&bytes.iter().map(|c| *c as char).collect::<String>());
                    },
                    Err(e) => {
                        println!("Unable to read file: {:?}", e);
                        continue;
                    }
                }
            }
        }

        if let Some(file_description) = describe::describe_file(&input, &path_wrapper, &format) {
            let description_fname = format!("{}/description/{}", tree_config.paths.index_path, path);
            let description_file = File::create(description_fname).unwrap();
            let mut desc_writer = BufWriter::new(description_file);
            write!(desc_writer, "{}", file_description).unwrap();
        }

        let extension = path_wrapper.extension().unwrap_or(&OsStr::new("")).to_str().unwrap();
        let show_header = match extension_mapping.get(extension) {
            Some(&(ref description, ref try_extensions)) => {
                let mut result = None;
                for try_ext in try_extensions {
                    let try_buf = path_wrapper.with_extension(try_ext);
                    let try_path = try_buf.as_path();
                    if try_path.exists() {
                        let (path_base, _) = path.split_at(path.len() - extension.len() - 1);
                        result = Some((description.to_owned(), format!("/{}/source/{}.{}", tree_name, path_base, try_ext)));
                        break;
                    }
                }
                result
            },
            None => {
                None
            }
        };

        let mut panel = vec![];

        let mut source_panel_items = vec![];
        if let Some((other_desc, other_path)) = show_header {
            source_panel_items.push(PanelItem {
                title: format!("Go to {} file", other_desc),
                link: other_path,
                update_link_lineno: false,
                accel_key: None,
            });
        };
        if let Some(ref bugzilla) = bugzilla_data {
            if !path.contains("__GENERATED__") {
                if let Some((product, component)) = get_bugzilla_component(bugzilla, &path) {
                    source_panel_items.push(PanelItem {
                        title: "File a bug for this file".to_owned(),
                        link: format!("https://bugzilla.mozilla.org/enter_bug.cgi?product={}&component={}",
                            product.replace("&", "%26"),
                            component.replace("&", "%26")),
                        update_link_lineno: false,
                        accel_key: None,
                    });
                }
            }
        }

        if !source_panel_items.is_empty() {
            panel.push(PanelSection {
                name: "Source code".to_owned(),
                items: source_panel_items,
            });
        };

        if let Some(oid) = head_oid {
            if !path.contains("__GENERATED__") {
                panel.push(PanelSection {
                    name: "Revision control".to_owned(),
                    items: vec![PanelItem {
                        title: "Permalink".to_owned(),
                        link: format!("/{}/rev/{}/{}", tree_name, oid, path),
                        update_link_lineno: true,
                        accel_key: Some('Y'),
                    }, PanelItem {
                        title: "Log".to_owned(),
                        link: format!("{}/log/tip/{}", config::get_hg_root(tree_config), path),
                        update_link_lineno: false,
                        accel_key: Some('L'),
                     }, PanelItem {
                        title: "Raw".to_owned(),
                        link: format!("{}/raw-file/tip/{}", config::get_hg_root(tree_config), path),
                        update_link_lineno: false,
                        accel_key: Some('R'),
                    }, PanelItem {
                        title: "Blame".to_owned(),
                        link: "javascript:alert('Hover over the gray bar on the left to see blame information.')".to_owned(),
                        update_link_lineno: false,
                        accel_key: None,
                    }],
                });
            }
        }

        format_file_data(&cfg,
                         tree_name,
                         &panel,
                         &head_commit,
                         &blame_commit,
                         path,
                         input,
                         &jumps,
                         &analysis,
                         &mut writer,
                         Some(&mut diff_cache)).unwrap();
    }
}
<commit_msg>Bug 1510341 - Display the bugzilla component in the link text<commit_after>use std::collections::HashMap;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::fs::File;
use std::io;
use std::io::BufReader;
use std::io::BufWriter;
use std::io::Read;
use std::io::Write;
use std::io::Seek;
use std::path::Path;

extern crate env_logger;
extern crate tools;
use tools::find_source_file;
use tools::file_format::analysis::{read_analysis, read_source, read_jumps};
use tools::format::format_file_data;
use tools::git_ops;
use tools::config;
use tools::languages;
use tools::describe;
use languages::FormatAs;

use tools::output::{PanelItem, PanelSection};

extern crate rustc_serialize;
use rustc_serialize::json;

fn read_json_from_file(path: &str) -> Option<json::Object> {
    let components_file = File::open(path).ok()?;
    let mut reader = BufReader::new(&components_file);
    json::Json::from_reader(&mut reader).ok()?.into_object()
}

/// For a given path, looks up the bugzilla product and component and
/// returns it in a tuple if it could be found. The JSON data format
/// comes from https://searchfox.org/mozilla-central/rev/47edbd91c43db6229cf32d1fc4bae9b325b9e2d0/python/mozbuild/mozbuild/frontend/mach_commands.py#209-223,243
/// and is fairly straightforward.
fn get_bugzilla_component<'a>(bugzilla_data: &'a json::Object, path: &str) -> Option<(&'a str, &'a str)> {
    let mut path_obj = bugzilla_data.get("paths")?;
    for path_component in path.split('/') {
        path_obj = path_obj.as_object()?.get(path_component)?;
    }
    let component_id = path_obj.as_i64()?.to_string();
    let mut result_iter = bugzilla_data.get("components")?.as_object()?.get(&component_id)?.as_array()?.iter();
    let product = result_iter.next()?.as_string()?;
    let component = result_iter.next()?.as_string()?;
    Some((product, component))
}

fn main() {
    env_logger::init();

    let args: Vec<_> = env::args().collect();
    let (base_args, fname_args) = args.split_at(3);

    let cfg = config::load(&base_args[1], true);
    println!("Config file read");

    let tree_name = &base_args[2];
    let tree_config = cfg.trees.get(tree_name).unwrap();

    let jumps_fname = format!("{}/jumps", tree_config.paths.index_path);
    //let jumps : std::collections::HashMap<String, tools::analysis::Jump> = std::collections::HashMap::new();
    let jumps = read_jumps(&jumps_fname);
    println!("Jumps read");

    let bugzilla_fname = format!("{}/bugzilla-components.json", tree_config.paths.index_path);
    let bugzilla_data = read_json_from_file(&bugzilla_fname);
    if bugzilla_data.is_some() {
        println!("Bugzilla components read");
    } else {
        println!("No bugzilla-components.json file found");
    }

    let (blame_commit, head_oid) = match &tree_config.git {
        &Some(ref git) => {
            let head_oid = git.repo.refname_to_id("HEAD").unwrap();
            let blame_commit = if let Some(ref blame_repo) = git.blame_repo {
                let blame_oid = blame_repo.refname_to_id("HEAD").unwrap();
                Some(blame_repo.find_commit(blame_oid).unwrap())
            } else {
                None
            };
            (blame_commit, Some(head_oid))
        },
        &None => (None, None),
    };

    let head_commit = head_oid
        .and_then(|oid| tree_config.git.as_ref().unwrap().repo.find_commit(oid).ok());

    let mut extension_mapping = HashMap::new();
    extension_mapping.insert("cpp", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cc", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cxx", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("h", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hh", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hpp", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hxx", ("source", vec!["cpp", "cc", "cxx"]));

    let mut diff_cache = git_ops::TreeDiffCache::new();
    for path in fname_args {
        println!("File {}", path);

        let output_fname = format!("{}/file/{}", tree_config.paths.index_path, path);
        let source_fname = find_source_file(path, &tree_config.paths.files_path, &tree_config.paths.objdir_path);

        let format = languages::select_formatting(path);

        let output_file = File::create(output_fname).unwrap();
        let mut writer = BufWriter::new(output_file);

        let source_file = match File::open(source_fname.clone()) {
            Ok(f) => f,
            Err(_) => {
                println!("Unable to open file");
                continue;
            },
        };

        let path_wrapper = Path::new(&source_fname);
        let metadata = fs::symlink_metadata(path_wrapper).unwrap();
        if metadata.file_type().is_symlink() {
            let dest = fs::read_link(path_wrapper).unwrap();
            write!(writer, "Symlink to {}", dest.to_str().unwrap()).unwrap();
            continue;
        }

        let mut reader = BufReader::new(&source_file);

        match format {
            FormatAs::Binary => {
                let _ = io::copy(&mut reader, &mut writer);
                continue;
            },
            _ => {},
        };

        let analysis_fname = format!("{}/analysis/{}", tree_config.paths.index_path, path);
        let analysis = read_analysis(&analysis_fname, &mut read_source);

        let mut input = String::new();
        match reader.read_to_string(&mut input) {
            Ok(_) => {},
            Err(_) => {
                let mut bytes = Vec::new();
                reader.seek(std::io::SeekFrom::Start(0)).unwrap();
                match reader.read_to_end(&mut bytes) {
                    Ok(_) => {
                        input.push_str(&bytes.iter().map(|c| *c as char).collect::<String>());
                    },
                    Err(e) => {
                        println!("Unable to read file: {:?}", e);
                        continue;
                    }
                }
            }
        }

        if let Some(file_description) = describe::describe_file(&input, &path_wrapper, &format) {
            let description_fname = format!("{}/description/{}", tree_config.paths.index_path, path);
            let description_file = File::create(description_fname).unwrap();
            let mut desc_writer = BufWriter::new(description_file);
            write!(desc_writer, "{}", file_description).unwrap();
        }

        let extension = path_wrapper.extension().unwrap_or(&OsStr::new("")).to_str().unwrap();
        let show_header = match extension_mapping.get(extension) {
            Some(&(ref description, ref try_extensions)) => {
                let mut result = None;
                for try_ext in try_extensions {
                    let try_buf = path_wrapper.with_extension(try_ext);
                    let try_path = try_buf.as_path();
                    if try_path.exists() {
                        let (path_base, _) = path.split_at(path.len() - extension.len() - 1);
                        result = Some((description.to_owned(), format!("/{}/source/{}.{}", tree_name, path_base, try_ext)));
                        break;
                    }
                }
                result
            },
            None => {
                None
            }
        };

        let mut panel = vec![];

        let mut source_panel_items = vec![];
        if let Some((other_desc, other_path)) = show_header {
            source_panel_items.push(PanelItem {
                title: format!("Go to {} file", other_desc),
                link: other_path,
                update_link_lineno: false,
                accel_key: None,
            });
        };
        if let Some(ref bugzilla) = bugzilla_data {
            if !path.contains("__GENERATED__") {
                if let Some((product, component)) = get_bugzilla_component(bugzilla, &path) {
                    source_panel_items.push(PanelItem {
                        title: format!("File a bug in {} :: {}", product, component),
                        link: format!("https://bugzilla.mozilla.org/enter_bug.cgi?product={}&component={}",
                            product.replace("&", "%26"),
                            component.replace("&", "%26")),
                        update_link_lineno: false,
                        accel_key: None,
                    });
                }
            }
        }

        if !source_panel_items.is_empty() {
            panel.push(PanelSection {
                name: "Source code".to_owned(),
                items: source_panel_items,
            });
        };

        if let Some(oid) = head_oid {
            if !path.contains("__GENERATED__") {
                panel.push(PanelSection {
                    name: "Revision control".to_owned(),
                    items: vec![PanelItem {
                        title: "Permalink".to_owned(),
                        link: format!("/{}/rev/{}/{}", tree_name, oid, path),
                        update_link_lineno: true,
                        accel_key: Some('Y'),
                    }, PanelItem {
                        title: "Log".to_owned(),
                        link: format!("{}/log/tip/{}", config::get_hg_root(tree_config), path),
                        update_link_lineno: false,
                        accel_key: Some('L'),
                     }, PanelItem {
                        title: "Raw".to_owned(),
                        link: format!("{}/raw-file/tip/{}", config::get_hg_root(tree_config), path),
                        update_link_lineno: false,
                        accel_key: Some('R'),
                    }, PanelItem {
                        title: "Blame".to_owned(),
                        link: "javascript:alert('Hover over the gray bar on the left to see blame information.')".to_owned(),
                        update_link_lineno: false,
                        accel_key: None,
                    }],
                });
            }
        }

        format_file_data(&cfg,
                         tree_name,
                         &panel,
                         &head_commit,
                         &blame_commit,
                         path,
                         input,
                         &jumps,
                         &analysis,
                         &mut writer,
                         Some(&mut diff_cache)).unwrap();
    }
}
<|endoftext|>


<commit_before>//! A trivial example. Copied from ocl-core crate repository.
use af_opencl_interop as afcl;
use arrayfire as af;

use ocl_core::{ContextProperties, Event};

fn main() {
    // Set the arrayfire backend to use OpenCL first,
    // because CUDA is the automatically preferred if available
    af::set_backend(af::Backend::OPENCL);

    // Choose platform & device(s) to use. Create a context, queue,
    let platform_id = ocl_core::default_platform().unwrap();
    let device_ids = ocl_core::get_device_ids(&platform_id, None, None).unwrap();
    let device_id = device_ids[0];
    let context_properties = ContextProperties::new().platform(platform_id);
    let context =
        ocl_core::create_context(Some(&context_properties), &[device_id], None, None).unwrap();
    let queue = ocl_core::create_command_queue(&context, &device_id, None).unwrap();
    let dims = [8, 1, 1];

    // Create a `Buffer`:
    let mut vec = vec![0.0f32; dims[0]];
    let buffer = unsafe {
        ocl_core::create_buffer(
            &context,
            ocl_core::MEM_READ_WRITE | ocl_core::MEM_COPY_HOST_PTR,
            dims[0],
            Some(&vec),
        )
        .unwrap()
    };
    ocl_core::finish(&queue).unwrap(); //sync up before switching to arrayfire

    // Add custom device, context and associated queue to ArrayFire
    afcl::add_device_context(device_id.as_raw(), context.as_ptr(), queue.as_ptr());
    afcl::set_device_context(device_id.as_raw(), context.as_ptr());
    af::info();

    let mut af_buffer = af::Array::new_from_device_ptr(
        buffer.as_ptr() as *mut f32,
        af::Dim4::new(&[dims[0] as u64, 1, 1, 1]),
    );

    af::af_print!("GPU Buffer before modification:", af_buffer);

    af_buffer = af_buffer + 10f32;

    af::sync(af::get_device());
    unsafe {
        let ptr = af_buffer.device_ptr();
        let obuf = ocl_core::Mem::from_raw_copied_ptr(ptr);

        // Read results from the device into a vector:
        ocl_core::enqueue_read_buffer(
            &queue,
            &obuf,
            true,
            0,
            &mut vec,
            None::<Event>,
            None::<&mut Event>,
        )
        .unwrap();
    }
    println!("GPU buffer on host after ArrayFire operation: {:?}", vec);

    // Remove device from ArrayFire management towards Application Exit
    af::set_device(0); // Cannot pop when in Use, hence switch to another device
    afcl::delete_device_context(device_id.as_raw(), context.as_ptr());
}
<commit_msg>Fix the missing cl_mem retain in opencl-af example<commit_after>//! A trivial example. Copied from ocl-core crate repository.
use af_opencl_interop as afcl;
use arrayfire as af;

use ocl_core::{retain_mem_object, ContextProperties, Event};

fn main() {
    // Set the arrayfire backend to use OpenCL first,
    // because CUDA is the automatically preferred if available
    af::set_backend(af::Backend::OPENCL);

    // Choose platform & device(s) to use. Create a context, queue,
    let platform_id = ocl_core::default_platform().unwrap();
    let device_ids = ocl_core::get_device_ids(&platform_id, None, None).unwrap();
    let device_id = device_ids[0];
    let context_properties = ContextProperties::new().platform(platform_id);
    let context =
        ocl_core::create_context(Some(&context_properties), &[device_id], None, None).unwrap();
    let queue = ocl_core::create_command_queue(&context, &device_id, None).unwrap();
    let dims = [8, 1, 1];

    // Create a `Buffer`:
    let mut vec = vec![1.0f32; dims[0]];
    let buffer = unsafe {
        ocl_core::create_buffer(
            &context,
            ocl_core::MEM_READ_WRITE | ocl_core::MEM_COPY_HOST_PTR,
            dims[0],
            Some(&vec),
        )
        .unwrap()
    };
    ocl_core::finish(&queue).unwrap(); //sync up before switching to arrayfire

    // Add custom device, context and associated queue to ArrayFire
    afcl::add_device_context(device_id.as_raw(), context.as_ptr(), queue.as_ptr());
    afcl::set_device_context(device_id.as_raw(), context.as_ptr());
    af::info();

    unsafe {
        retain_mem_object(&buffer).unwrap();
    }
    let mut af_buffer = af::Array::new_from_device_ptr(
        buffer.as_ptr() as *mut f32,
        af::Dim4::new(&[dims[0] as u64, 1, 1, 1]),
    );

    af::af_print!("GPU Buffer before modification:", af_buffer);

    af_buffer = af_buffer + 10f32;

    af::sync(af::get_device());
    unsafe {
        let ptr = af_buffer.device_ptr();
        let obuf = ocl_core::Mem::from_raw_copied_ptr(ptr);

        // Read results from the device into a vector:
        ocl_core::enqueue_read_buffer(
            &queue,
            &obuf,
            true,
            0,
            &mut vec,
            None::<Event>,
            None::<&mut Event>,
        )
        .unwrap();
    }
    println!("GPU buffer on host after ArrayFire operation: {:?}", vec);

    // Remove device from ArrayFire management towards Application Exit
    af::set_device(0); // Cannot pop when in Use, hence switch to another device
    afcl::delete_device_context(device_id.as_raw(), context.as_ptr());
}
<|endoftext|>