<commit_before>use rustc::lint::*;
use syntax::ptr::P;
use rustc_front::hir::*;
use reexport::*;
use rustc_front::util::{is_comparison_binop, binop_to_string};
use syntax::codemap::{Span, Spanned};
use rustc_front::intravisit::FnKind;
use rustc::middle::ty;
use rustc::middle::const_eval::ConstVal::Float;
use rustc::middle::const_eval::eval_const_expr_partial;
use rustc::middle::const_eval::EvalHint::ExprTypeChecked;

use utils::{get_item_name, match_path, snippet, span_lint, walk_ptrs_ty, is_integer_literal};
use utils::span_help_and_lint;

/// **What it does:** This lint checks for function arguments and let bindings denoted as `ref`. It is `Warn` by default.
///
/// **Why is this bad?** The `ref` declaration makes the function take an owned value, but turns the argument into a reference (which means that the value is destroyed when exiting the function). This adds not much value: either take a reference type, or take an owned value and create references in the body.
///
/// For let bindings, `let x = &foo;` is preferred over `let ref x = foo`. The type of `x` is more obvious with the former.
///
/// **Known problems:** If the argument is dereferenced within the function, removing the `ref` will lead to errors. This can be fixed by removing the dereferences, e.g. changing `*x` to `x` within the function.
///
/// **Example:** `fn foo(ref x: u8) -> bool { .. }`
declare_lint!(pub TOPLEVEL_REF_ARG, Warn,
              "An entire binding was declared as `ref`, in a function argument (`fn foo(ref x: Bar)`), \
               or a `let` statement (`let ref x = foo()`). In such cases, it is preferred to take \
               references with `&`.");

#[allow(missing_copy_implementations)]
pub struct TopLevelRefPass;

impl LintPass for TopLevelRefPass {
    fn get_lints(&self) -> LintArray {
        lint_array!(TOPLEVEL_REF_ARG)
    }
}

impl LateLintPass for TopLevelRefPass {
    fn check_fn(&mut self, cx: &LateContext, k: FnKind, decl: &FnDecl, _: &Block, _: Span, _: NodeId) {
        if let FnKind::Closure = k {
            // Does not apply to closures
            return
        }
        for ref arg in &decl.inputs {
            if let PatIdent(BindByRef(_), _, _) = arg.pat.node {
                span_lint(cx,
                    TOPLEVEL_REF_ARG,
                    arg.pat.span,
                    "`ref` directly on a function argument is ignored. Consider using a reference type instead."
                );
            }
        }
    }
    fn check_stmt(&mut self, cx: &LateContext, s: &Stmt) {
        if_let_chain! {
            [
            let StmtDecl(ref d, _) = s.node,
            let DeclLocal(ref l) = d.node,
            let PatIdent(BindByRef(_), i, None) = l.pat.node,
            let Some(ref init) = l.init
            ], {
                let tyopt = if let Some(ref ty) = l.ty {
                    format!(": {:?} ", ty)
                } else {
                    "".to_owned()
                };
                span_help_and_lint(cx,
                    TOPLEVEL_REF_ARG,
                    l.pat.span,
                    "`ref` on an entire `let` pattern is discouraged, take a reference with & instead",
                    &format!("try `let {} {}= &{};`", snippet(cx, i.span, "_"),
                             tyopt, snippet(cx, init.span, "_"))
                );
            }
        };
    }
}

/// **What it does:** This lint checks for comparisons to NAN. It is `Deny` by default.
///
/// **Why is this bad?** NAN does not compare meaningfully to anything – not even itself – so those comparisons are simply wrong.
///
/// **Known problems:** None
///
/// **Example:** `x == NAN`
declare_lint!(pub CMP_NAN, Deny,
              "comparisons to NAN (which will always return false, which is probably not intended)");

#[derive(Copy,Clone)]
pub struct CmpNan;

impl LintPass for CmpNan {
    fn get_lints(&self) -> LintArray {
        lint_array!(CMP_NAN)
    }
}

impl LateLintPass for CmpNan {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, ref left, ref right) = expr.node {
            if is_comparison_binop(cmp.node) {
                if let ExprPath(_, ref path) = left.node {
                    check_nan(cx, path, expr.span);
                }
                if let ExprPath(_, ref path) = right.node {
                    check_nan(cx, path, expr.span);
                }
            }
        }
    }
}

fn check_nan(cx: &LateContext, path: &Path, span: Span) {
    path.segments.last().map(|seg| if seg.identifier.name.as_str() == "NAN" {
        span_lint(cx, CMP_NAN, span,
            "doomed comparison with NAN, use `std::{f32,f64}::is_nan()` instead");
    });
}

/// **What it does:** This lint checks for (in-)equality comparisons on floating-point values (apart from zero), except in functions called `*eq*` (which probably implement equality for a type involving floats). It is `Warn` by default.
///
/// **Why is this bad?** Floating point calculations are usually imprecise, so asking if two values are *exactly* equal is asking for trouble. For a good guide on what to do, see [the floating point guide](http://www.floating-point-gui.de/errors/comparison).
///
/// **Known problems:** None
///
/// **Example:** `y == 1.23f64`
declare_lint!(pub FLOAT_CMP, Warn,
              "using `==` or `!=` on float values (as floating-point operations \
               usually involve rounding errors, it is always better to check for approximate \
               equality within small bounds)");

#[derive(Copy,Clone)]
pub struct FloatCmp;

impl LintPass for FloatCmp {
    fn get_lints(&self) -> LintArray {
        lint_array!(FLOAT_CMP)
    }
}

impl LateLintPass for FloatCmp {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, ref left, ref right) = expr.node {
            let op = cmp.node;
            if (op == BiEq || op == BiNe) && (is_float(cx, left) || is_float(cx, right)) {
                if is_allowed(cx, left) || is_allowed(cx, right) { return; }
                if let Some(name) = get_item_name(cx, expr) {
                    let name = name.as_str();
                    if name == "eq" || name == "ne" || name == "is_nan" ||
                            name.starts_with("eq_") ||
                            name.ends_with("_eq") {
                        return;
                    }
                }
                span_lint(cx, FLOAT_CMP, expr.span, &format!(
                    "{}-comparison of f32 or f64 detected. Consider changing this to \
                     `abs({} - {}) < epsilon` for some suitable value of epsilon",
                    binop_to_string(op), snippet(cx, left.span, ".."),
                    snippet(cx, right.span, "..")));
            }
        }
    }
}

fn is_allowed(cx: &LateContext, expr: &Expr) -> bool {
    let res = eval_const_expr_partial(cx.tcx, expr, ExprTypeChecked, None);
    if let Ok(Float(val)) = res {
        val == 0.0 || val == ::std::f64::INFINITY || val == ::std::f64::NEG_INFINITY
    } else { false }
}

fn is_float(cx: &LateContext, expr: &Expr) -> bool {
    if let ty::TyFloat(_) = walk_ptrs_ty(cx.tcx.expr_ty(expr)).sty {
        true
    } else {
        false
    }
}

/// **What it does:** This lint checks for conversions to owned values just for the sake of a comparison. It is `Warn` by default.
///
/// **Why is this bad?** The comparison can operate on a reference, so creating an owned value effectively throws it away directly afterwards, which is needlessly consuming code and heap space.
///
/// **Known problems:** None
///
/// **Example:** `x.to_owned() == y`
declare_lint!(pub CMP_OWNED, Warn,
              "creating owned instances for comparing with others, e.g. `x == \"foo\".to_string()`");

#[derive(Copy,Clone)]
pub struct CmpOwned;

impl LintPass for CmpOwned {
    fn get_lints(&self) -> LintArray {
        lint_array!(CMP_OWNED)
    }
}

impl LateLintPass for CmpOwned {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, ref left, ref right) = expr.node {
            if is_comparison_binop(cmp.node) {
                check_to_owned(cx, left, right.span, true, cmp.span);
                check_to_owned(cx, right, left.span, false, cmp.span)
            }
        }
    }
}

fn check_to_owned(cx: &LateContext, expr: &Expr, other_span: Span, left: bool, op: Span) {
    let snip = match expr.node {
        ExprMethodCall(Spanned{node: ref name, ..}, _, ref args) if args.len() == 1 => {
            if name.as_str() == "to_string" ||
                name.as_str() == "to_owned" && is_str_arg(cx, args) {
                    snippet(cx, args[0].span, "..")
                } else {
                    return
                }
        }
        ExprCall(ref path, ref v) if v.len() == 1 => {
            if let ExprPath(None, ref path) = path.node {
                if match_path(path, &["String", "from_str"]) ||
                    match_path(path, &["String", "from"]) {
                            snippet(cx, v[0].span, "..")
                    } else {
                        return
                    }
            } else {
                return
            }
        }
        _ => return
    };
    if left {
        span_lint(cx, CMP_OWNED, expr.span, &format!(
        "this creates an owned instance just for comparison. Consider using \
        `{} {} {}` to compare without allocation", snip,
        snippet(cx, op, "=="), snippet(cx, other_span, "..")));
    } else {
        span_lint(cx, CMP_OWNED, expr.span, &format!(
        "this creates an owned instance just for comparison. Consider using \
        `{} {} {}` to compare without allocation",
        snippet(cx, other_span, ".."), snippet(cx, op, "=="),  snip));
    }

}

fn is_str_arg(cx: &LateContext, args: &[P<Expr>]) -> bool {
    args.len() == 1 && if let ty::TyStr =
        walk_ptrs_ty(cx.tcx.expr_ty(&args[0])).sty { true } else { false }
}

/// **What it does:** This lint checks for getting the remainder of a division by one. It is `Warn` by default.
///
/// **Why is this bad?** The result can only ever be zero. No one will write such code deliberately, unless trying to win an Underhanded Rust Contest. Even for that contest, it's probably a bad idea. Use something more underhanded.
///
/// **Known problems:** None
///
/// **Example:** `x % 1`
declare_lint!(pub MODULO_ONE, Warn, "taking a number modulo 1, which always returns 0");

#[derive(Copy,Clone)]
pub struct ModuloOne;

impl LintPass for ModuloOne {
    fn get_lints(&self) -> LintArray {
        lint_array!(MODULO_ONE)
    }
}

impl LateLintPass for ModuloOne {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, _, ref right) = expr.node {
            if let Spanned {node: BinOp_::BiRem, ..} = *cmp {
                if is_integer_literal(right, 1) {
                    cx.span_lint(MODULO_ONE, expr.span, "any number modulo 1 will be 0");
                }
            }
        }
    }
}

/// **What it does:** This lint checks for patterns in the form `name @ _`.
///
/// **Why is this bad?** It's almost always more readable to just use direct bindings.
///
/// **Known problems:** None
///
/// **Example**:
/// ```
/// match v {
///     Some(x) => (),
///     y @ _   => (), // easier written as `y`,
/// }
/// ```
declare_lint!(pub REDUNDANT_PATTERN, Warn, "using `name @ _` in a pattern");

#[derive(Copy,Clone)]
pub struct PatternPass;

impl LintPass for PatternPass {
    fn get_lints(&self) -> LintArray {
        lint_array!(REDUNDANT_PATTERN)
    }
}

impl LateLintPass for PatternPass {
    fn check_pat(&mut self, cx: &LateContext, pat: &Pat) {
        if let PatIdent(_, ref ident, Some(ref right)) = pat.node {
            if right.node == PatWild {
                cx.span_lint(REDUNDANT_PATTERN, pat.span, &format!(
                    "the `{} @ _` pattern can be written as just `{}`",
                    ident.node.name, ident.node.name));
            }
        }
    }
}
<commit_msg>Add used_underscore_binding lint<commit_after>use rustc::lint::*;
use syntax::ptr::P;
use rustc_front::hir::*;
use reexport::*;
use rustc_front::util::{is_comparison_binop, binop_to_string};
use syntax::codemap::{Span, Spanned};
use rustc_front::intravisit::FnKind;
use rustc::middle::ty;
use rustc::middle::const_eval::ConstVal::Float;
use rustc::middle::const_eval::eval_const_expr_partial;
use rustc::middle::const_eval::EvalHint::ExprTypeChecked;
use rustc::middle::def::Def;

use utils::{get_item_name, match_path, snippet, span_lint, walk_ptrs_ty, is_integer_literal};
use utils::span_help_and_lint;

/// **What it does:** This lint checks for function arguments and let bindings denoted as `ref`. It is `Warn` by default.
///
/// **Why is this bad?** The `ref` declaration makes the function take an owned value, but turns the argument into a reference (which means that the value is destroyed when exiting the function). This adds not much value: either take a reference type, or take an owned value and create references in the body.
///
/// For let bindings, `let x = &foo;` is preferred over `let ref x = foo`. The type of `x` is more obvious with the former.
///
/// **Known problems:** If the argument is dereferenced within the function, removing the `ref` will lead to errors. This can be fixed by removing the dereferences, e.g. changing `*x` to `x` within the function.
///
/// **Example:** `fn foo(ref x: u8) -> bool { .. }`
declare_lint!(pub TOPLEVEL_REF_ARG, Warn,
              "An entire binding was declared as `ref`, in a function argument (`fn foo(ref x: Bar)`), \
               or a `let` statement (`let ref x = foo()`). In such cases, it is preferred to take \
               references with `&`.");

#[allow(missing_copy_implementations)]
pub struct TopLevelRefPass;

impl LintPass for TopLevelRefPass {
    fn get_lints(&self) -> LintArray {
        lint_array!(TOPLEVEL_REF_ARG)
    }
}

impl LateLintPass for TopLevelRefPass {
    fn check_fn(&mut self, cx: &LateContext, k: FnKind, decl: &FnDecl, _: &Block, _: Span, _: NodeId) {
        if let FnKind::Closure = k {
            // Does not apply to closures
            return
        }
        for ref arg in &decl.inputs {
            if let PatIdent(BindByRef(_), _, _) = arg.pat.node {
                span_lint(cx,
                    TOPLEVEL_REF_ARG,
                    arg.pat.span,
                    "`ref` directly on a function argument is ignored. Consider using a reference type instead."
                );
            }
        }
    }
    fn check_stmt(&mut self, cx: &LateContext, s: &Stmt) {
        if_let_chain! {
            [
            let StmtDecl(ref d, _) = s.node,
            let DeclLocal(ref l) = d.node,
            let PatIdent(BindByRef(_), i, None) = l.pat.node,
            let Some(ref init) = l.init
            ], {
                let tyopt = if let Some(ref ty) = l.ty {
                    format!(": {:?} ", ty)
                } else {
                    "".to_owned()
                };
                span_help_and_lint(cx,
                    TOPLEVEL_REF_ARG,
                    l.pat.span,
                    "`ref` on an entire `let` pattern is discouraged, take a reference with & instead",
                    &format!("try `let {} {}= &{};`", snippet(cx, i.span, "_"),
                             tyopt, snippet(cx, init.span, "_"))
                );
            }
        };
    }
}

/// **What it does:** This lint checks for comparisons to NAN. It is `Deny` by default.
///
/// **Why is this bad?** NAN does not compare meaningfully to anything – not even itself – so those comparisons are simply wrong.
///
/// **Known problems:** None
///
/// **Example:** `x == NAN`
declare_lint!(pub CMP_NAN, Deny,
              "comparisons to NAN (which will always return false, which is probably not intended)");

#[derive(Copy,Clone)]
pub struct CmpNan;

impl LintPass for CmpNan {
    fn get_lints(&self) -> LintArray {
        lint_array!(CMP_NAN)
    }
}

impl LateLintPass for CmpNan {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, ref left, ref right) = expr.node {
            if is_comparison_binop(cmp.node) {
                if let ExprPath(_, ref path) = left.node {
                    check_nan(cx, path, expr.span);
                }
                if let ExprPath(_, ref path) = right.node {
                    check_nan(cx, path, expr.span);
                }
            }
        }
    }
}

fn check_nan(cx: &LateContext, path: &Path, span: Span) {
    path.segments.last().map(|seg| if seg.identifier.name.as_str() == "NAN" {
        span_lint(cx, CMP_NAN, span,
            "doomed comparison with NAN, use `std::{f32,f64}::is_nan()` instead");
    });
}

/// **What it does:** This lint checks for (in-)equality comparisons on floating-point values (apart from zero), except in functions called `*eq*` (which probably implement equality for a type involving floats). It is `Warn` by default.
///
/// **Why is this bad?** Floating point calculations are usually imprecise, so asking if two values are *exactly* equal is asking for trouble. For a good guide on what to do, see [the floating point guide](http://www.floating-point-gui.de/errors/comparison).
///
/// **Known problems:** None
///
/// **Example:** `y == 1.23f64`
declare_lint!(pub FLOAT_CMP, Warn,
              "using `==` or `!=` on float values (as floating-point operations \
               usually involve rounding errors, it is always better to check for approximate \
               equality within small bounds)");

#[derive(Copy,Clone)]
pub struct FloatCmp;

impl LintPass for FloatCmp {
    fn get_lints(&self) -> LintArray {
        lint_array!(FLOAT_CMP)
    }
}

impl LateLintPass for FloatCmp {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, ref left, ref right) = expr.node {
            let op = cmp.node;
            if (op == BiEq || op == BiNe) && (is_float(cx, left) || is_float(cx, right)) {
                if is_allowed(cx, left) || is_allowed(cx, right) { return; }
                if let Some(name) = get_item_name(cx, expr) {
                    let name = name.as_str();
                    if name == "eq" || name == "ne" || name == "is_nan" ||
                            name.starts_with("eq_") ||
                            name.ends_with("_eq") {
                        return;
                    }
                }
                span_lint(cx, FLOAT_CMP, expr.span, &format!(
                    "{}-comparison of f32 or f64 detected. Consider changing this to \
                     `abs({} - {}) < epsilon` for some suitable value of epsilon",
                    binop_to_string(op), snippet(cx, left.span, ".."),
                    snippet(cx, right.span, "..")));
            }
        }
    }
}

fn is_allowed(cx: &LateContext, expr: &Expr) -> bool {
    let res = eval_const_expr_partial(cx.tcx, expr, ExprTypeChecked, None);
    if let Ok(Float(val)) = res {
        val == 0.0 || val == ::std::f64::INFINITY || val == ::std::f64::NEG_INFINITY
    } else { false }
}

fn is_float(cx: &LateContext, expr: &Expr) -> bool {
    if let ty::TyFloat(_) = walk_ptrs_ty(cx.tcx.expr_ty(expr)).sty {
        true
    } else {
        false
    }
}

/// **What it does:** This lint checks for conversions to owned values just for the sake of a comparison. It is `Warn` by default.
///
/// **Why is this bad?** The comparison can operate on a reference, so creating an owned value effectively throws it away directly afterwards, which is needlessly consuming code and heap space.
///
/// **Known problems:** None
///
/// **Example:** `x.to_owned() == y`
declare_lint!(pub CMP_OWNED, Warn,
              "creating owned instances for comparing with others, e.g. `x == \"foo\".to_string()`");

#[derive(Copy,Clone)]
pub struct CmpOwned;

impl LintPass for CmpOwned {
    fn get_lints(&self) -> LintArray {
        lint_array!(CMP_OWNED)
    }
}

impl LateLintPass for CmpOwned {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, ref left, ref right) = expr.node {
            if is_comparison_binop(cmp.node) {
                check_to_owned(cx, left, right.span, true, cmp.span);
                check_to_owned(cx, right, left.span, false, cmp.span)
            }
        }
    }
}

fn check_to_owned(cx: &LateContext, expr: &Expr, other_span: Span, left: bool, op: Span) {
    let snip = match expr.node {
        ExprMethodCall(Spanned{node: ref name, ..}, _, ref args) if args.len() == 1 => {
            if name.as_str() == "to_string" ||
                name.as_str() == "to_owned" && is_str_arg(cx, args) {
                    snippet(cx, args[0].span, "..")
                } else {
                    return
                }
        }
        ExprCall(ref path, ref v) if v.len() == 1 => {
            if let ExprPath(None, ref path) = path.node {
                if match_path(path, &["String", "from_str"]) ||
                    match_path(path, &["String", "from"]) {
                            snippet(cx, v[0].span, "..")
                    } else {
                        return
                    }
            } else {
                return
            }
        }
        _ => return
    };
    if left {
        span_lint(cx, CMP_OWNED, expr.span, &format!(
        "this creates an owned instance just for comparison. Consider using \
        `{} {} {}` to compare without allocation", snip,
        snippet(cx, op, "=="), snippet(cx, other_span, "..")));
    } else {
        span_lint(cx, CMP_OWNED, expr.span, &format!(
        "this creates an owned instance just for comparison. Consider using \
        `{} {} {}` to compare without allocation",
        snippet(cx, other_span, ".."), snippet(cx, op, "=="),  snip));
    }

}

fn is_str_arg(cx: &LateContext, args: &[P<Expr>]) -> bool {
    args.len() == 1 && if let ty::TyStr =
        walk_ptrs_ty(cx.tcx.expr_ty(&args[0])).sty { true } else { false }
}

/// **What it does:** This lint checks for getting the remainder of a division by one. It is `Warn` by default.
///
/// **Why is this bad?** The result can only ever be zero. No one will write such code deliberately, unless trying to win an Underhanded Rust Contest. Even for that contest, it's probably a bad idea. Use something more underhanded.
///
/// **Known problems:** None
///
/// **Example:** `x % 1`
declare_lint!(pub MODULO_ONE, Warn, "taking a number modulo 1, which always returns 0");

#[derive(Copy,Clone)]
pub struct ModuloOne;

impl LintPass for ModuloOne {
    fn get_lints(&self) -> LintArray {
        lint_array!(MODULO_ONE)
    }
}

impl LateLintPass for ModuloOne {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        if let ExprBinary(ref cmp, _, ref right) = expr.node {
            if let Spanned {node: BinOp_::BiRem, ..} = *cmp {
                if is_integer_literal(right, 1) {
                    cx.span_lint(MODULO_ONE, expr.span, "any number modulo 1 will be 0");
                }
            }
        }
    }
}

/// **What it does:** This lint checks for patterns in the form `name @ _`.
///
/// **Why is this bad?** It's almost always more readable to just use direct bindings.
///
/// **Known problems:** None
///
/// **Example**:
/// ```
/// match v {
///     Some(x) => (),
///     y @ _   => (), // easier written as `y`,
/// }
/// ```
declare_lint!(pub REDUNDANT_PATTERN, Warn, "using `name @ _` in a pattern");

#[derive(Copy,Clone)]
pub struct PatternPass;

impl LintPass for PatternPass {
    fn get_lints(&self) -> LintArray {
        lint_array!(REDUNDANT_PATTERN)
    }
}

impl LateLintPass for PatternPass {
    fn check_pat(&mut self, cx: &LateContext, pat: &Pat) {
        if let PatIdent(_, ref ident, Some(ref right)) = pat.node {
            if right.node == PatWild {
                cx.span_lint(REDUNDANT_PATTERN, pat.span, &format!(
                    "the `{} @ _` pattern can be written as just `{}`",
                    ident.node.name, ident.node.name));
            }
        }
    }
}

declare_lint!(pub USED_UNDERSCORE_BINDING, Warn,
              "using a binding which is prefixed with an underscore");

#[derive(Copy, Clone)]
pub struct UsedUnderscoreBinding;

impl LintPass for UsedUnderscoreBinding {
    fn get_lints(&self) -> LintArray {
        lint_array!(USED_UNDERSCORE_BINDING)
    }
}

impl LateLintPass for UsedUnderscoreBinding {
    fn check_expr(&mut self, cx: &LateContext, expr: &Expr) {
        let needs_lint = match expr.node {
            ExprPath(_, ref path) => {
                path.segments.last().unwrap().identifier.name.as_str().chars().next() == Some('_') &&
                (cx.tcx.def_map.borrow()).values().any(|res| match res.base_def {
                                                  Def::DefLocal(_, _) => true,
                                                  _ => false
                                            })
            },
            ExprField(_, spanned) => spanned.node.as_str().chars().next() == Some('_'),
            _ => false
        };
        if needs_lint {
            cx.span_lint(USED_UNDERSCORE_BINDING, expr.span, &format!(
                "Used binding which is prefixed with an underscore. A leading underscore signals\
                 that a binding will not be used."));
        }
    }
}
<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # NDFA
//!
//! The acronym NDFA stands for 'Non-Deterministic Finite Automaton'. An NDFA is a state machine where each state can have
//! transitions for more than one target state - this allows it to be in more than one state at once, which is the property
//! that makes it non-deterministic.
//!
//! NDFAs can match patterns conforming to a regular language - that is, they match regular expressions. However, their
//! non-deterministic nature means that they are inconvenient to evaluate. Fortunately, for every NDFA there can be found
//! a DFA - a *Deterministic* Finite Automaton - which can be used to rapidly match strings of symbols against the expression.
//!
//! Every implementation of the `StateMachine` trait represents an NDFA. This particular implementation is useful because it
//! also implements `MutableStateMachine` so can be used as a means to build state machines that can be used to match patterns.
//!
//! An NDFA is created by calling the constructor:
//!
//! ```
//! # use ndfa::*;
//! let mut ndfa: Ndfa<u32, u32> = Ndfa::new();
//! ```
//!

use super::state_machine::*;
use std::collections::HashMap;

///
/// Represents a non-deterministic finite-state automata
///
#[derive(Clone)]
pub struct Ndfa<InputSymbol, OutputSymbol> where InputSymbol : Clone {
    /// Highest known state ID
    max_state: StateId,

    /// Transitions for each state
    transitions: Vec<Vec<(InputSymbol, StateId)>>,

    /// The states that are joined with the state specified by the index
    joined_with: Vec<Vec<StateId>>,

    /// Output symbols for each state
    output_symbols: HashMap<StateId, OutputSymbol>
}

impl<InputSymbol : Clone, OutputSymbol> Ndfa<InputSymbol, OutputSymbol> {
    pub fn new() -> Ndfa<InputSymbol, OutputSymbol> {
        Ndfa { max_state: 0, transitions: vec![], joined_with: vec![], output_symbols: HashMap::new() }
    }
}

impl<InputSymbol : Clone, OutputSymbol> StateMachine<InputSymbol, OutputSymbol> for Ndfa<InputSymbol, OutputSymbol> {
    fn count_states(&self) -> StateId {
        self.max_state + 1
    }

    fn get_transitions_for_state(&self, state: StateId) -> Vec<(InputSymbol, StateId)> {
        if (state as usize) >= self.transitions.len() {
            vec![]
        } else {
            self.transitions[state as usize].clone()
        }
    }

    fn output_symbol_for_state(&self, state: StateId) -> Option<&OutputSymbol> {
        self.output_symbols.get(&state)
    }
}

impl<InputSymbol : Clone, OutputSymbol> MutableStateMachine<InputSymbol, OutputSymbol> for Ndfa<InputSymbol, OutputSymbol> {
    fn add_transition(&mut self, state: StateId, for_symbol: InputSymbol, new_state: StateId) {
        // Make sure that max_state reflects the highest state added by the user
        if new_state > self.max_state {
            self.max_state = new_state;
        }

        if state > self.max_state {
            self.max_state = state;
        }

        // Expand the transitions vector so that the new state has somewhere to go
        while self.transitions.len() <= state as usize {
            self.transitions.push(vec![]);
        }

        self.transitions[state as usize].push((for_symbol, new_state));
    }

    fn set_output_symbol(&mut self, state: StateId, new_output_symbol: OutputSymbol) {
        // Output symbols update the max state too
        if state > self.max_state {
            self.max_state = state;
        }

        self.output_symbols.insert(state, new_output_symbol);
    }

    fn join_states(&mut self, first_state: StateId, second_state: StateId) {
        // Joining states updates the overall state count_states
        if first_state > self.max_state {
            self.max_state = first_state;
        }

        if second_state > self.max_state {
            self.max_state = second_state;
        }

        // Expand the join table so we can add the first state
        while self.joined_with.len() <= first_state as usize {
            self.joined_with.push(vec![]);
        }

        // Join the second state to the first state
        self.joined_with[first_state as usize].push(second_state);
    }
}

#[cfg(test)]
mod tests {
    use super::super::state_machine::*;
    use super::*;

    #[test]
    fn there_is_initially_one_state() {
        let ndfa: Ndfa<u32, u32> = Ndfa::new();

        assert!(ndfa.count_states() == 1);
    }

    #[test]
    fn adding_transition_updates_max_state_to_target() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn adding_output_symbol_updates_max_state() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.set_output_symbol(1, 128);
        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn adding_transition_updates_max_state_to_source() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(1, 42, 0);
        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn can_retrieve_transition() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(1, 42, 0);
        assert!(ndfa.get_transitions_for_state(1).len() == 1);
        assert!(ndfa.get_transitions_for_state(1)[0] == (42, 0));
    }

    #[test]
    fn can_add_multiple_transitions() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(1, 42, 0);
        ndfa.add_transition(1, 42, 2);

        assert!(ndfa.get_transitions_for_state(1).len() == 2);
        assert!(ndfa.get_transitions_for_state(1).contains(&(42, 0)));
        assert!(ndfa.get_transitions_for_state(1).contains(&(42, 2)));
    }

    #[test]
    fn output_symbol_is_none_by_default() {
        let ndfa: Ndfa<u32, u32> = Ndfa::new();

        assert!(ndfa.output_symbol_for_state(0) == None);
        assert!(ndfa.output_symbol_for_state(1) == None);
    }

    #[test]
    fn can_set_output_symbol() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.set_output_symbol(0, 64);

        assert!(ndfa.output_symbol_for_state(0) == Some(&64));
        assert!(ndfa.output_symbol_for_state(1) == None);
    }

    #[test]
    fn join_states_attaches_transitions_to_first_state() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.join_states(0, 1);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));
    }

    #[test]
    fn adding_transition_after_join_updates_target() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.join_states(0, 1);
        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));
    }

    #[test]
    fn join_updates_state_count() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.join_states(0, 1);

        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn join_states_does_not_attach_to_second_state() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.join_states(0, 1);

        assert!(!ndfa.get_transitions_for_state(1).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(1).contains(&(43, 2)));
    }

    #[test]
    fn join_states_recurses_to_further_states() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.add_transition(2, 44, 3);
        ndfa.join_states(0, 1);
        ndfa.join_states(1, 2);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(44, 3)));
    }

    #[test]
    fn join_loop_attaches_to_both_states() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.join_states(0, 1);
        ndfa.join_states(1, 0);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));

        assert!(ndfa.get_transitions_for_state(1).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(1).contains(&(43, 2)));
    }
}
<commit_msg>Implement joining in the get_transitions_for_state method<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # NDFA
//!
//! The acronym NDFA stands for 'Non-Deterministic Finite Automaton'. An NDFA is a state machine where each state can have
//! transitions for more than one target state - this allows it to be in more than one state at once, which is the property
//! that makes it non-deterministic.
//!
//! NDFAs can match patterns conforming to a regular language - that is, they match regular expressions. However, their
//! non-deterministic nature means that they are inconvenient to evaluate. Fortunately, for every NDFA there can be found
//! a DFA - a *Deterministic* Finite Automaton - which can be used to rapidly match strings of symbols against the expression.
//!
//! Every implementation of the `StateMachine` trait represents an NDFA. This particular implementation is useful because it
//! also implements `MutableStateMachine` so can be used as a means to build state machines that can be used to match patterns.
//!
//! An NDFA is created by calling the constructor:
//!
//! ```
//! # use ndfa::*;
//! let mut ndfa: Ndfa<u32, u32> = Ndfa::new();
//! ```
//!

use super::state_machine::*;
use std::collections::HashMap;
use std::collections::HashSet;

///
/// Represents a non-deterministic finite-state automata
///
#[derive(Clone)]
pub struct Ndfa<InputSymbol, OutputSymbol> where InputSymbol : Clone {
    /// Highest known state ID
    max_state: StateId,

    /// Transitions for each state
    transitions: Vec<Vec<(InputSymbol, StateId)>>,

    /// The states that are joined with the state specified by the index
    joined_with: Vec<Vec<StateId>>,

    /// Output symbols for each state
    output_symbols: HashMap<StateId, OutputSymbol>
}

impl<InputSymbol : Clone, OutputSymbol> Ndfa<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new non-deterministic finite Automaton
    ///
    /// The NDFA will initially just consist of the start state 0. The methods in `MutableStateMachine` should be called to
    /// build it into a more useful structure.
    ///
    pub fn new() -> Ndfa<InputSymbol, OutputSymbol> {
        Ndfa { max_state: 0, transitions: vec![], joined_with: vec![], output_symbols: HashMap::new() }
    }

    ///
    /// Retrieves the complete set of states whose transitions should be returned due to joining for a given state
    ///
    #[inline]
    fn get_join_closure(&self, state: StateId) -> HashSet<StateId> {
        // The result is initially empty. We'll add the initial state in the first pass
        let mut result: HashSet<StateId> = HashSet::new();

        // Add in any joined states
        let mut stack = vec![];
        stack.push(state);

        while let Some(next_state) = stack.pop() {
            if !result.contains(&next_state) {
                // Add to the result
                result.insert(next_state);

                // Process any states that are joined to this one
                if (next_state as usize) < self.joined_with.len() {
                    let ref join_states = self.joined_with[next_state as usize];

                    for join_to in join_states {
                        stack.push(*join_to);
                    }
                }
            }
        }

        result
    }
}

impl<InputSymbol : Clone, OutputSymbol> StateMachine<InputSymbol, OutputSymbol> for Ndfa<InputSymbol, OutputSymbol> {
    fn count_states(&self) -> StateId {
        self.max_state + 1
    }

    fn get_transitions_for_state(&self, state: StateId) -> Vec<(InputSymbol, StateId)> {
        let empty           = vec![];

        // The transitions are all the transitions for this state, plus all the transitions in the states that are joined to it
        let joined_states   = self.get_join_closure(state);
        let merged          = joined_states.iter().flat_map(|join_state| {
            if (*join_state as usize) < self.transitions.len() {
                self.transitions[*join_state as usize].iter()
            } else {
                empty.iter()
            }
        });

        merged.map(|item| item.clone()).collect()
    }

    fn output_symbol_for_state(&self, state: StateId) -> Option<&OutputSymbol> {
        self.output_symbols.get(&state)
    }
}

impl<InputSymbol : Clone, OutputSymbol> MutableStateMachine<InputSymbol, OutputSymbol> for Ndfa<InputSymbol, OutputSymbol> {
    fn add_transition(&mut self, state: StateId, for_symbol: InputSymbol, new_state: StateId) {
        // Make sure that max_state reflects the highest state added by the user
        if new_state > self.max_state {
            self.max_state = new_state;
        }

        if state > self.max_state {
            self.max_state = state;
        }

        // Expand the transitions vector so that the new state has somewhere to go
        while self.transitions.len() <= state as usize {
            self.transitions.push(vec![]);
        }

        self.transitions[state as usize].push((for_symbol, new_state));
    }

    fn set_output_symbol(&mut self, state: StateId, new_output_symbol: OutputSymbol) {
        // Output symbols update the max state too
        if state > self.max_state {
            self.max_state = state;
        }

        self.output_symbols.insert(state, new_output_symbol);
    }

    fn join_states(&mut self, first_state: StateId, second_state: StateId) {
        // Joining states updates the overall state count_states
        if first_state > self.max_state {
            self.max_state = first_state;
        }

        if second_state > self.max_state {
            self.max_state = second_state;
        }

        // Expand the join table so we can add the first state
        while self.joined_with.len() <= first_state as usize {
            self.joined_with.push(vec![]);
        }

        // Join the second state to the first state
        self.joined_with[first_state as usize].push(second_state);
    }
}

#[cfg(test)]
mod tests {
    use super::super::state_machine::*;
    use super::*;

    #[test]
    fn there_is_initially_one_state() {
        let ndfa: Ndfa<u32, u32> = Ndfa::new();

        assert!(ndfa.count_states() == 1);
    }

    #[test]
    fn adding_transition_updates_max_state_to_target() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn adding_output_symbol_updates_max_state() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.set_output_symbol(1, 128);
        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn adding_transition_updates_max_state_to_source() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(1, 42, 0);
        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn can_retrieve_transition() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(1, 42, 0);
        assert!(ndfa.get_transitions_for_state(1).len() == 1);
        assert!(ndfa.get_transitions_for_state(1)[0] == (42, 0));
    }

    #[test]
    fn can_add_multiple_transitions() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(1, 42, 0);
        ndfa.add_transition(1, 42, 2);

        assert!(ndfa.get_transitions_for_state(1).len() == 2);
        assert!(ndfa.get_transitions_for_state(1).contains(&(42, 0)));
        assert!(ndfa.get_transitions_for_state(1).contains(&(42, 2)));
    }

    #[test]
    fn output_symbol_is_none_by_default() {
        let ndfa: Ndfa<u32, u32> = Ndfa::new();

        assert!(ndfa.output_symbol_for_state(0) == None);
        assert!(ndfa.output_symbol_for_state(1) == None);
    }

    #[test]
    fn can_set_output_symbol() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.set_output_symbol(0, 64);

        assert!(ndfa.output_symbol_for_state(0) == Some(&64));
        assert!(ndfa.output_symbol_for_state(1) == None);
    }

    #[test]
    fn join_states_attaches_transitions_to_first_state() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.join_states(0, 1);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));
    }

    #[test]
    fn adding_transition_after_join_updates_target() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.join_states(0, 1);
        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));
    }

    #[test]
    fn join_updates_state_count() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.join_states(0, 1);

        assert!(ndfa.count_states() == 2);
    }

    #[test]
    fn join_states_does_not_attach_to_second_state() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.join_states(0, 1);

        assert!(!ndfa.get_transitions_for_state(1).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(1).contains(&(43, 2)));
    }

    #[test]
    fn join_states_recurses_to_further_states() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.add_transition(2, 44, 3);
        ndfa.join_states(0, 1);
        ndfa.join_states(1, 2);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(44, 3)));
    }

    #[test]
    fn join_loop_attaches_to_both_states() {
        let mut ndfa: Ndfa<u32, u32> = Ndfa::new();

        ndfa.add_transition(0, 42, 1);
        ndfa.add_transition(1, 43, 2);
        ndfa.join_states(0, 1);
        ndfa.join_states(1, 0);

        assert!(ndfa.get_transitions_for_state(0).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(0).contains(&(43, 2)));

        assert!(ndfa.get_transitions_for_state(1).contains(&(42, 1)));
        assert!(ndfa.get_transitions_for_state(1).contains(&(43, 2)));
    }
}
<|endoftext|>


<commit_before>use af;
use af::{Dim4, Array};
use af::MatProp;

use activations;
use initializations;
use params::{LSTMIndex, Input, Params};

pub struct LSTM {
  pub input_size: usize,
  pub output_size: usize,
  pub return_sequences: bool,
}

pub enum ActivationIndex {
  Inner,
  Outer,
}

impl RTRL for LSTM {
  pub fn rtrl(&self, dW_tm1: &mut Array  // previous W derivatives for [I, F, Ct]
              , dU_tm1: &mut Array       // previous U derivatives for [I, F, Ct]
              , db_tm1: &mut Array       // previous b derivatives for [I, F, Ct]
              , z_t: &Array              // current time activation
              , inputs: &Input)          // x_t & h_{t-1}
  {
    let block_size = z_t.dims().unwrap()[0]; // the batch size * 5
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    // chunk out i, f, ct, c_tm1
    let if_t = activations::get_activation(self.inner_activation,
                                            , &af::rows(z_t, 0, 2*chunk_size).unwrap).unwrap();
    let ct_t = activations::get_activation(self.outer_activation,
                                           , &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap).unwrap();
    let c_tm1 = af::rows(z_t, 4*chunk_size, 5*chunk_size).unwrap();

    // diff(z_i), diff(z_f), diff(z_ct)
    let dz = vec![&activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, 0, chunk_size).unwrap()).unwrap() //d(z)
                  , &activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, chunk_size, 2*chunk_size).unwrap()).unwrap()
                  , &activations::get_activation_derivative(self.outer_activation, &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap()).unwrap()];
    let ct_ctm1_i = vec![&ct_t, &c_tm1, &af::rows(if_t, 0, chunk_size).unwrap()];

    // [Ct_t; C_{t-1}; i_t] * dz
    let dzprod = af::mul(&af::join_many(0, ct_ctm1_i).unwrap()
                         , af::join_many(0, dzvec).unwrap(), false).unwrap();

    // dC_t/dWi  = (dC_{t-1}/dWi  * f_t) + ct_t  * inner_activation(z_i) * x_t
    // dC_t/dWf  = (dC_{t-1}/dWf  * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // dC_t/dWct = (dC_{t-1}/dWct * f_t) + i_t   * outer_activation(Ct)  * x_t
    let w_lhs = af::mul(dW_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dW * f_t
    let w_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Input]);
    dW_tm1 = af::add(&w_lhs, &w_rhs).unwrap();

    // dC_t/dUi = (dC_{t-1}/dUi * f_t) + ct_t * inner_activation(z_i) * h_{t-1}
    // dC_t/dUf = (dC_{t-1}/dUf * f_t) + c_tm1 * inner_activation(z_f) * h_{t-1}
    // dC_t/dUct = (dC_{t-1}/dUct * f_t) + outer_activation(Ct) * x_t * h_{t-1}
    let u_lhs = af::mul(dU_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dU * f_t
    let u_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Recurrence]);
    dU_tm1 = af::add(&u_lhs, &u_rhs).unwrap();

    // dC_t/dbi = (dC_{t-1}/dbi * f_t) + ct_t * inner_activation(z_i)
    // dC_t/dbf = (dC_{t-1}/dbf * f_t) + c_tm1 * inner_activation(z_f)
    // dC_t/dbct = (dC_{t-1}/dbct * f_t) + outer_activation(Ct)
    let b_lhs = af::mul(db_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/db * f_t
    dW_tm1 = af::add(&b_lhs, &dzprod).unwrap();
  }
}

impl Layer for LSTM {
  fn forward(&self, params: &mut Params
             , inputs: &Input
             , recurrence: &Option<Input>) -> (Input, Option<Input>)
  {
    // keep previous layer's outputs
    assert!(inputs.data.dims().unwrap()[2] == 1);
    params.inputs[0] = vec![inputs.clone()];

    // apply the activation to the previous layer [Optimization: Memory saving]
    // the input activation is the activation of the previous output [outer]
    let activated_input = activations::get_activation(inputs.activation        // self.inputs.activation[ActivationIndex::Inner]
                                                      , inputs.data).unwrap(); //&self.inputs.data[DataIndex::Input]).unwrap();

    let ifo_tm1; // input, forget, output @ t-1
    let ct_tm1;  // cell internal @ t-1
    let c_tm2;   // cell output @ t-1

    if recurrence.is_some() {
      // extract the sub-block of each gate [i_tm1, f_tm1, o_tm1, ct_tm1, c_tm2]
      let block_size = recurrence.data.dims().unwrap()[0];
      assert!(block_size as f32 % 5.0f32 == 0);
      let chunk_size = block_size / 5;

      // i_{t-1} = inner_activation(zi_{t-1})
      // f_{t-1} = inner_activation(zf_{t-1})
      // o_{t-1} = inner_activation(zo_{t-1})
      ifo_tm1 = activations::get_activation(recurrence.activation                                                             //inputs.activation[ActivationIndex::Inner]
                                            , &af::rows(&recurrence.data, 0, 3 * chunk_size).unwrap).unwrap();                //&inputs.data[DataIndex::Recurrence], 0, 3 * chunk_size).unwrap).unwrap();
      // Ct_{t-1} = outer_activation(zct_{t-1})
      ct_tm1 = activations::get_activation(inputs.activation                                                                  //inputs.activation[ActivationIndex::Outer]
                                           , &af::rows(&recurrence.data , 3 * chunk_size, 4 * chunk_size).unwrap).unwrap();   //&inputs.data[DataIndex::Recurrence], 3 * chunk_size, 4 * chunk_size).unwrap).unwrap();
      // C_{t-2} = last_chunk(recurrence)
      c_tm2 = af::rows(&recurrence.data, 4 * chunk_size, 5 * chunk_size).unwrap();                                            //&inputs.data[DataIndex::Recurrence], 4 * chunk_size, 5 * chunk_size).unwrap();
    }else { // this is the first node in the recurrence
      let batch_size = inputs.data.dims().unwrap()[0];
      let ifo_recurrence_dims = Dim4::new([3*batch_size, self.output_size, 1, 1]).unwrap();
      let ct_recurrence_dims = Dim4::new([batch_size, self.output_size, 1, 1]).unwrap();
      let c_recurrence_dims = Dim4::new([batch_size, self.output_size, 1, 1]).unwrap();
      ifo_tm1 = initializations::get_initialization("zeros", ifo_recurrence_dims).unwrap();
      ct_tm1 = initializations::get_initialization("zeros", ct_recurrence_dims).unwrap();
      c_tm2 = initializations::get_initialization("zeros", c_recurrence_dims).unwrap();
    }

    // C_{t-1} = i_{t-1} * Ct_{t-1} + f_{t-1} * C_{t-2}
    // h_{t-1} = o_{t-1} * outer_activation(C_{t-1})
    let c_tm1 = af::add(&af::mul(&af::rows(&ifo_tm1, 0, chunk_size).unwrap(), &ct_tm1, false).unwrap()
                        , &af::mul(&af::rows(&ifo_tm1, chunk_size, 2 * chunk_size).unwrap(), &c_tm2, false).unwrap()
                        , false).unwrap();
    let h_tm1 = af::mul(&o_tm1, activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                                            , &c_tm1), false).unwrap();

    // forward pass in a batch for performance
    let weights_ref    = vec![&params.weights[LSTMIndex::Input]
                              , &params.weights[LSTMIndex::Forget]
                              , &params.weights[LSTMIndex::Output]
                              , &params.weights[LSTMIndex::CellTilda]];
    let offset = 3; // the offset from weights --> recurrent weights
    let recurrents_ref = vec![&params.weights[LSTMIndex::Input as usize + offset]
                              , &params.weights[LSTMIndex::Forget as usize + offset]
                              , &params.weights[LSTMIndex::Output as usize + offset]
                              , &params.weights[LSTMIndex::CellTilda as usize + offset]];
    let bias_ref       = vec![&params.biases[LSTMIndex::Input]
                              , &params.biases[LSTMIndex::Forget]
                              , &params.biases[LSTMIndex::Output]
                              , &params.biases[LSTMIndex::CellTilda]];
    // [z(i,f,o,ct)_t] = W*x + U*h_tm1 + b
    let z_t = af::add(&af::add(&af::matmul(&af::join_many(0, weights_ref).unwrap(), &activated_input).unwrap()
                               , &af::matmul(&af::join_many(0, recurrents_ref).unwrap(), &h_tm1).unwrap(), false).unwrap()
                      , &af::join_many(0, bias_ref).unwrap(), true).unwrap();
    self.rtrl(&self.dW, &self.dU, &self.db, &z_t, inputs);

    // since we are RTRL'ing i, f, Ct w.r.t. C we technically just need to pass z_o & c_t
    if self.return_sequences {
      (Input { data: af::join_many(0, vec![&z_t, &c_tm1]).unwrap()
              , activation: self.inner_activation, self.outer_activation] }
    }else {
      Input { data: vec![z_t.clone()]
              , activation: vec![self.inner_activation, self.outer_activation] }
    }
  }

  fn backward(&mut self, delta: &Array) -> Array {
    self.delta = delta.clone();

    let activation_prev = activations::get_activation(self.inputs.activation[0], &self.inputs.data[DataIndex::Input]).unwrap();
    let d_activation_prev = activations::get_activation_derivative(self.inputs.activation[0], &activation_prev).unwrap();
    let delta_prev = af::mul(&af::matmul(&params.weights[0], delta, af::MatProp::TRANS, af::MatProp::NONE).unwrap()
                             , &d_activation_prev, false).unwrap();
    delta_prev
  }
}
<commit_msg>intermediary commit<commit_after>use af;
use af::{Dim4, Array};
use af::MatProp;

use activations;
use initializations;
use params::{LSTMIndex, Input, Params};

pub struct LSTM {
  pub input_size: usize,
  pub output_size: usize,
  pub return_sequences: bool,
}

pub enum ActivationIndex {
  Inner,
  Outer,
}

impl RTRL for LSTM {
  pub fn rtrl(&self, dW_tm1: &mut Array  // previous W derivatives for [I, F, Ct]
              , dU_tm1: &mut Array       // previous U derivatives for [I, F, Ct]
              , db_tm1: &mut Array       // previous b derivatives for [I, F, Ct]
              , z_t: &Array              // current time activation
              , inputs: &Input           // x_t
              , recurrences: &Input      // h_{t-1}
  {
    let block_size = z_t.dims().unwrap()[0]; // the batch size * 5
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    // chunk out zi, zf, zct, zc_tm1
    let zi_t = af::rows(z_t, 0, chunk_size).unwrap();
    let zf_t = af::rows(z_t, chunk_size, 2*chunk_size).unwrap();
    let zct_t = af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap();

    // compute their activations
    let i_t =  activations::get_activation(self.inner_activation, &zi_t).unwrap();
    let f_t =  activations::get_activation(self.inner_activation, &zf_t).unwrap();
    let ct_t = activations::get_activation(self.outer_activation, &zct_t).unwrap();
    let c_tm1 = af::rows(z_t, 4*chunk_size, 5*chunk_size).unwrap();

    // compute their derivatives [diff(z_i), diff(z_f), diff(z_ct)]
    let dz = vec![&activations::get_activation_derivative(self.inner_activation, &zi_t).unwrap()
                  , &activations::get_activation_derivative(self.inner_activation, &zf_t).unwrap()
                  , &activations::get_activation_derivative(self.outer_activation, &zct_t).unwrap()];
    let ct_ctm1_it = vec![&ct_t, &c_tm1, &i_t).unwrap()];

    // [Ct_t; C_{t-1}; i_t] * dz
    let dzprod = af::mul(&af::join_many(0, ct_ctm1_it).unwrap()
                         , af::join_many(0, dzvec).unwrap(), false).unwrap();

    // dC_t/dWi  = (dC_{t-1}/dWi  * f_t) + ct_t  * inner_activation(z_i) * x_t
    // dC_t/dWf  = (dC_{t-1}/dWf  * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // dC_t/dWct = (dC_{t-1}/dWct * f_t) + i_t   * outer_activation(Ct)  * x_t
    let w_lhs = af::mul(dW_tm1, &f_t, true).unwrap(); // dC_{t-1}/dW * f_t
    let w_rhs = af::mul(&dzprod, &inputs.data, true).unwrap();
    dW_tm1 = af::add(&w_lhs, &w_rhs, false).unwrap();

    // dC_t/dUi = (dC_{t-1}/dUi * f_t) + ct_t * inner_activation(z_i) * h_{t-1}
    // dC_t/dUf = (dC_{t-1}/dUf * f_t) + c_tm1 * inner_activation(z_f) * h_{t-1}
    // dC_t/dUct = (dC_{t-1}/dUct * f_t) + outer_activation(Ct) * x_t * h_{t-1}
    let u_lhs = af::mul(dU_tm1, &f_t, true).unwrap(); // dC_{t-1}/dU * f_t
    let u_rhs = af::mul(&dzprod, &recurrences.data, true).unwrap();
    dU_tm1 = af::add(&u_lhs, &u_rhs, false).unwrap();

    // dC_t/dbi = (dC_{t-1}/dbi * f_t) + ct_t * inner_activation(z_i)
    // dC_t/dbf = (dC_{t-1}/dbf * f_t) + c_tm1 * inner_activation(z_f)
    // dC_t/dbct = (dC_{t-1}/dbct * f_t) + outer_activation(Ct)
    let b_lhs = af::mul(db_tm1, &f_t, true).unwrap(); // dC_{t-1}/db * f_t
    dW_tm1 = af::add(&b_lhs, &dzprod, false).unwrap();
  }
}

impl Layer for LSTM {
  fn forward(&self, params: &mut Params
             , inputs: &Input
             , recurrence: &Option<Input>) -> (Input, Option<Input>)
  {
    // keep previous layer's outputs
    assert!(inputs.data.dims().unwrap()[2] == 1);
    params.inputs[0] = vec![inputs.clone()];

    // apply the activation to the previous layer [Optimization: Memory saving]
    // the input activation is the activation of the previous output [outer]
    let activated_input = activations::get_activation(inputs.activation        // self.inputs.activation[ActivationIndex::Inner]
                                                      , inputs.data).unwrap(); //&self.inputs.data[DataIndex::Input]).unwrap();

    let i_tm1;   // input gate @ t-1
    let f_tm1;   // forget gate @ t-1
    let o_tm1;   // output gate @ t-1
    let ct_tm1;  // cell internal @ t-1
    let c_tm2;   // cell output @ t-2

    if recurrence.is_some() {
      // extract the sub-block of each gate [i_tm1, f_tm1, o_tm1, ct_tm1, c_tm2]
      let block_size = recurrence.data.dims().unwrap()[0];
      assert!(block_size as f32 % 5.0f32 == 0);
      let chunk_size = block_size / 5;

      // i_{t-1} = inner_activation(zi_{t-1})
      // f_{t-1} = inner_activation(zf_{t-1})
      // o_{t-1} = inner_activation(zo_{t-1})
      ifo_tm1 = activations::get_activation(recurrence.activation, &af::rows(&recurrence.data, 0, 3 * chunk_size).unwrap()).unwrap();

      // Ct_{t-1} = outer_activation(zct_{t-1})
      ct_tm1 = activations::get_activation(inputs.activation
                                           , &af::rows(&recurrence.data , 3 * chunk_size, 4 * chunk_size).unwrap()).unwrap();
      // C_{t-2} = last_chunk(recurrence)
      c_tm2 = af::rows(&recurrence.data, 4 * chunk_size, 5 * chunk_size).unwrap();
    }else { // this is the first node in the recurrence
      let batch_size = inputs.data.dims().unwrap()[0];
      ifo_recurrence_dims = Dim4::new([3*batch_size, self.output_size, 1, 1]).unwrap();
      ct_recurrence_dims = Dim4::new([batch_size, self.output_size, 1, 1]).unwrap();
      c_recurrence_dims = Dim4::new([batch_size, self.output_size, 1, 1]).unwrap();
      ifo_tm1 = initializations::get_initialization("zeros", ifo_recurrence_dims).unwrap();
      ct_tm1 = initializations::get_initialization("zeros", ct_recurrence_dims).unwrap();
      c_tm2 = initializations::get_initialization("zeros", c_recurrence_dims).unwrap();
    }

    // extract these gate values
    let i_tm1 = af::rows(&ifo_tm1, 0, chunk_size).unwrap();
    let f_tm1 = af::rows(&ifo_tm1, chunk_size, 2*chunk_size).unwrap();
    let o_tm1 = af::rows(&ifo_tm1, 2*chunk_size, 3*chunk_size).unwrap();

    // C_{t-1} = i_{t-1} * Ct_{t-1} + f_{t-1} * C_{t-2}
    // h_{t-1} = o_{t-1} * outer_activation(C_{t-1})
    let c_tm1 = af::add(&af::mul(&i_tm1, &ct_tm1, false).unwrap()
                        , &af::mul(&f_tm1, &c_tm2, false).unwrap()
                        , false).unwrap();
    let h_tm1 = af::mul(&o_tm1, activations::get_activation(inputs.activation, &c_tm1).unwrap(), false).unwrap();

    // forward pass in a batch for performance
    let weights_ref    = vec![&params.weights[LSTMIndex::Input]
                              , &params.weights[LSTMIndex::Forget]
                              , &params.weights[LSTMIndex::Output]
                              , &params.weights[LSTMIndex::CellTilda]];
    let offset = 3; // the offset from weights --> recurrent weights
    let recurrents_ref = vec![&params.weights[LSTMIndex::Input as usize + offset]
                              , &params.weights[LSTMIndex::Forget as usize + offset]
                              , &params.weights[LSTMIndex::Output as usize + offset]
                              , &params.weights[LSTMIndex::CellTilda as usize + offset]];
    let bias_ref       = vec![&params.biases[LSTMIndex::Input]
                              , &params.biases[LSTMIndex::Forget]
                              , &params.biases[LSTMIndex::Output]
                              , &params.biases[LSTMIndex::CellTilda]];
    // [z(i,f,o,ct)_t] = W*x + U*h_tm1 + b
    let z_t = af::add(&af::add(&af::matmul(&af::join_many(0, weights_ref).unwrap(), &activated_input).unwrap()
                               , &af::matmul(&af::join_many(0, recurrents_ref).unwrap(), &h_tm1).unwrap(), false).unwrap()
                      , &af::join_many(0, bias_ref).unwrap(), true).unwrap();
    self.rtrl(&self.dW, &self.dU, &self.db, &z_t, inputs);

    // since we are RTRL'ing i, f, Ct w.r.t. C we technically just need to pass z_o & c_t
    if self.return_sequences {
      (Input { data: af::join_many(0, vec![&z_t, &c_tm1]).unwrap()
              , activation: self.inner_activation, self.outer_activation] }
    }else {
      Input { data: vec![z_t.clone()]
              , activation: vec![self.inner_activation, self.outer_activation] }
    }
  }

  fn backward(&mut self, delta: &Array, recurrences: &Option<Input>) -> Array {
    self.delta = delta.clone();

    // calculate d_o first
    let d_o = activations::get_activation_derivative(

    let activation_prev = activations::get_activation(self.inputs.activation[0], &self.inputs.data[DataIndex::Input]).unwrap();
    let d_activation_prev = activations::get_activation_derivative(self.inputs.activation[0], &activation_prev).unwrap();
    let delta_prev = af::mul(&af::matmul(&params.weights[0], delta, af::MatProp::TRANS, af::MatProp::NONE).unwrap()
                             , &d_activation_prev, false).unwrap();
    delta_prev
  }
}
<|endoftext|>


<commit_before>mod token;

use std::str::Chars;

use self::token::Token;

#[derive(Debug)]
struct Lexed<'a> {
    chars: Chars<'a>,
}

impl<'a> Lexed<'a> {
    fn new(chars: Chars<'a>) -> Lexed {
        Lexed {
            chars: chars,
        }
    }
}

impl<'a> Iterator for Lexed<'a> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.chars.next() {
            Some(c) => {
                let mut s = String::new();
                s.push(c);
                Some(Token::IntTok(s))
            },
            None => None,
        }
    }
}

fn lex(chars: Chars) -> Lexed {
    Lexed::new(chars)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn assert_lex(input: &str, output: &[Token]) {
        let actual: Vec<Token> = lex(input.chars()).collect();
        let actual: &[Token] = actual.as_ref();
        assert_eq!(actual, output);
    }

    fn intt(chars: &str) -> Token {
        Token::IntTok(String::from(chars))
    }

    #[test]
    fn single_character_int() {
        assert_lex("3", &[intt("3")]);
        assert_lex("9", &[intt("9")]);
    }
}
<commit_msg>Break int lexing into a separate function<commit_after>mod token;

use std::str::Chars;

use self::token::Token;

#[derive(Debug)]
struct Lexed<'a> {
    chars: Chars<'a>,
}

impl<'a> Lexed<'a> {
    fn new(chars: Chars<'a>) -> Lexed {
        Lexed {
            chars: chars,
        }
    }
}

impl<'a> Iterator for Lexed<'a> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.chars.next() {
            Some(c) => Some(next_int(c, &mut self.chars)),
            None    => None,
        }
    }

}

fn next_int(first_char: char, chars: &mut Chars) -> Token {
    let mut s = String::new();
    s.push(first_char);
    Token::IntTok(s)
}

fn lex(chars: Chars) -> Lexed {
    Lexed::new(chars)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn assert_lex(input: &str, output: &[Token]) {
        let actual: Vec<Token> = lex(input.chars()).collect();
        let actual: &[Token] = actual.as_ref();
        assert_eq!(actual, output);
    }

    fn intt(chars: &str) -> Token {
        Token::IntTok(String::from(chars))
    }

    #[test]
    fn single_character_int() {
        assert_lex("3", &[intt("3")]);
        assert_lex("9", &[intt("9")]);
    }

//    #[test]
//    fn multiple_character_int() {
//        assert_lex("31", &[intt("31")]);
//
//        let long_int = "01234567891111111111012345678911111111110123456789";
//        assert_lex(long_int, &[intt(long_int)]);
//    }
}
<|endoftext|>


<commit_before>use std::fmt::{self, Display};
use std::path::PathBuf;
use std::str::FromStr;

use structopt::clap::AppSettings;
use structopt::StructOpt;
use syn_select::Selector;

#[derive(StructOpt)]
#[structopt(bin_name = "cargo")]
pub enum Opts {
    /// Show the result of macro expansion.
    #[structopt(
        name = "expand",
        raw(
            setting = "AppSettings::UnifiedHelpMessage",
            setting = "AppSettings::DeriveDisplayOrder",
            setting = "AppSettings::DontCollapseArgsInUsage"
        )
    )]
    Expand(Args),
}

#[derive(StructOpt, Debug)]
#[structopt(rename_all = "kebab-case")]
pub struct Args {
    /// Space-separated list of features to activate
    #[structopt(long, value_name = "FEATURES")]
    pub features: Option<String>,

    /// Activate all available features
    #[structopt(long)]
    pub all_features: bool,

    /// Do not activate the `default` feature
    #[structopt(long)]
    pub no_default_features: bool,

    /// Expand only this package's library
    #[structopt(long)]
    pub lib: bool,

    /// Expand only the specified binary
    #[structopt(long, value_name = "NAME")]
    pub bin: Option<String>,

    /// Expand only the specified example
    #[structopt(long, value_name = "NAME")]
    pub example: Option<String>,

    /// Expand only the specified test target
    #[structopt(long, value_name = "NAME")]
    pub test: Option<String>,

    /// Include tests when expanding the lib or bin
    #[structopt(long)]
    pub tests: bool,

    /// Expand only the specified bench target
    #[structopt(long, value_name = "NAME")]
    pub bench: Option<String>,

    /// Target triple which compiles will be for
    #[structopt(long, value_name = "TARGET")]
    pub target: Option<String>,

    /// Directory for all generated artifacts
    #[structopt(long, value_name = "DIRECTORY", parse(from_os_str))]
    pub target_dir: Option<PathBuf>,

    /// Path to Cargo.toml
    #[structopt(long, value_name = "PATH", parse(from_os_str))]
    pub manifest_path: Option<PathBuf>,

    /// Number of parallel jobs, defaults to # of CPUs
    #[structopt(short, long, value_name = "N")]
    pub jobs: Option<u64>,

    /// Print command lines as they are executed
    #[structopt(long)]
    pub verbose: bool,

    /// Coloring: auto, always, never
    #[structopt(long, value_name = "WHEN")]
    pub color: Option<Coloring>,

    /// Require Cargo.lock and cache are up to date
    #[structopt(long)]
    pub frozen: bool,

    /// Require Cargo.lock is up to date
    #[structopt(long)]
    pub locked: bool,

    /// Unstable (nightly-only) flags to Cargo
    #[structopt(short = "Z", value_name = "FLAG")]
    pub unstable_flags: Vec<String>,

    /// Do not attempt to run rustfmt
    #[structopt(long)]
    pub ugly: bool,

    /// Select syntax highlighting theme
    #[structopt(long, value_name = "NAME")]
    pub theme: Option<String>,

    /// Print available syntax highlighting theme names
    #[structopt(long)]
    pub themes: bool,

    /// Local path to module or other named item to expand, e.g. os::unix::ffi
    #[structopt(value_name = "ITEM")]
    pub item: Option<Selector>,
}

#[derive(Debug)]
pub enum Coloring {
    Auto,
    Always,
    Never,
}

impl FromStr for Coloring {
    type Err = String;

    fn from_str(name: &str) -> Result<Self, Self::Err> {
        match name {
            "auto" => Ok(Coloring::Auto),
            "always" => Ok(Coloring::Always),
            "never" => Ok(Coloring::Never),
            other => Err(format!(
                "must be auto, always, or never, but found `{}`",
                other,
            )),
        }
    }
}

impl Display for Coloring {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let name = match self {
            Coloring::Auto => "auto",
            Coloring::Always => "always",
            Coloring::Never => "never",
        };
        formatter.write_str(name)
    }
}
<commit_msg>Omit author from help text<commit_after>use std::fmt::{self, Display};
use std::path::PathBuf;
use std::str::FromStr;

use structopt::clap::AppSettings;
use structopt::StructOpt;
use syn_select::Selector;

#[derive(StructOpt)]
#[structopt(bin_name = "cargo", author = "")]
pub enum Opts {
    /// Show the result of macro expansion.
    #[structopt(
        name = "expand",
        author = "",
        raw(
            setting = "AppSettings::UnifiedHelpMessage",
            setting = "AppSettings::DeriveDisplayOrder",
            setting = "AppSettings::DontCollapseArgsInUsage"
        )
    )]
    Expand(Args),
}

#[derive(StructOpt, Debug)]
#[structopt(rename_all = "kebab-case", author = "")]
pub struct Args {
    /// Space-separated list of features to activate
    #[structopt(long, value_name = "FEATURES")]
    pub features: Option<String>,

    /// Activate all available features
    #[structopt(long)]
    pub all_features: bool,

    /// Do not activate the `default` feature
    #[structopt(long)]
    pub no_default_features: bool,

    /// Expand only this package's library
    #[structopt(long)]
    pub lib: bool,

    /// Expand only the specified binary
    #[structopt(long, value_name = "NAME")]
    pub bin: Option<String>,

    /// Expand only the specified example
    #[structopt(long, value_name = "NAME")]
    pub example: Option<String>,

    /// Expand only the specified test target
    #[structopt(long, value_name = "NAME")]
    pub test: Option<String>,

    /// Include tests when expanding the lib or bin
    #[structopt(long)]
    pub tests: bool,

    /// Expand only the specified bench target
    #[structopt(long, value_name = "NAME")]
    pub bench: Option<String>,

    /// Target triple which compiles will be for
    #[structopt(long, value_name = "TARGET")]
    pub target: Option<String>,

    /// Directory for all generated artifacts
    #[structopt(long, value_name = "DIRECTORY", parse(from_os_str))]
    pub target_dir: Option<PathBuf>,

    /// Path to Cargo.toml
    #[structopt(long, value_name = "PATH", parse(from_os_str))]
    pub manifest_path: Option<PathBuf>,

    /// Number of parallel jobs, defaults to # of CPUs
    #[structopt(short, long, value_name = "N")]
    pub jobs: Option<u64>,

    /// Print command lines as they are executed
    #[structopt(long)]
    pub verbose: bool,

    /// Coloring: auto, always, never
    #[structopt(long, value_name = "WHEN")]
    pub color: Option<Coloring>,

    /// Require Cargo.lock and cache are up to date
    #[structopt(long)]
    pub frozen: bool,

    /// Require Cargo.lock is up to date
    #[structopt(long)]
    pub locked: bool,

    /// Unstable (nightly-only) flags to Cargo
    #[structopt(short = "Z", value_name = "FLAG")]
    pub unstable_flags: Vec<String>,

    /// Do not attempt to run rustfmt
    #[structopt(long)]
    pub ugly: bool,

    /// Select syntax highlighting theme
    #[structopt(long, value_name = "NAME")]
    pub theme: Option<String>,

    /// Print available syntax highlighting theme names
    #[structopt(long)]
    pub themes: bool,

    /// Local path to module or other named item to expand, e.g. os::unix::ffi
    #[structopt(value_name = "ITEM")]
    pub item: Option<Selector>,
}

#[derive(Debug)]
pub enum Coloring {
    Auto,
    Always,
    Never,
}

impl FromStr for Coloring {
    type Err = String;

    fn from_str(name: &str) -> Result<Self, Self::Err> {
        match name {
            "auto" => Ok(Coloring::Auto),
            "always" => Ok(Coloring::Always),
            "never" => Ok(Coloring::Never),
            other => Err(format!(
                "must be auto, always, or never, but found `{}`",
                other,
            )),
        }
    }
}

impl Display for Coloring {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let name = match self {
            Coloring::Auto => "auto",
            Coloring::Always => "always",
            Coloring::Never => "never",
        };
        formatter.write_str(name)
    }
}
<|endoftext|>


<commit_before>use std::io::Write;
use std::io::Read;
use std::fmt;
use std::cmp;

const DEFAULT_NUMBER_OF_SAMPLES:usize = 31;
const DEFAULT_NUMBER_OF_ROWS_PER_PATTERN:usize = 64;
const DEFAULT_NUMBER_OF_CHANNELS_PER_ROW:usize = 4;
const MAGIC_MK:[u8; 4] =['M' as u8, '.' as u8, 'K' as u8, '.' as u8];

#[derive(Debug)]
pub struct SampleInfo {
	pub name: String, // [char; 22],
	pub length: u16, // In words, so multiply by 2
	pub finetune: u8,
	pub volume: u8, // 0-64
	pub repeat_start: u16, // In words from start of sample data, so multiply by 2
	pub repeat_length: u16, // Number of words to loop, multiply by 2
}

impl SampleInfo {
	pub fn new() -> SampleInfo {
		SampleInfo{name: String::new(), length:0, finetune:0, volume:0, repeat_start:0, repeat_length:0}
	}
}

#[derive(Debug)]
pub struct Channel {
	pub period: u16, // Really u12
	pub sample_number: u8,
	pub effect: u16 // Really u12
}

impl Channel {
	pub fn new() -> Channel {
		Channel{period: 0, sample_number: 0, effect: 0}
	}
}

#[derive(Debug)]
pub struct Row {
	pub channels: Vec<Channel>
}

impl Row {
	pub fn new() -> Row {
		let mut row = Row{channels: Vec::new()};
		for _ in 0..DEFAULT_NUMBER_OF_CHANNELS_PER_ROW {
			row.channels.push(Channel::new());
		}

		row
	}
}

#[derive(Debug)]
pub struct Pattern {
	pub rows: Vec<Row>
}

impl Pattern {
	pub fn new() -> Pattern {
		let mut p = Pattern{rows: Vec::new()};

		for _ in 0..DEFAULT_NUMBER_OF_ROWS_PER_PATTERN {
			p.rows.push(Row::new());
		}

		p
	}
}

pub struct Positions {
	pub data: [u8; 128]
}

impl fmt::Debug for Positions {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        self.data[..].fmt(formatter)
    }
}

#[derive(Debug)]
pub struct PTModule {
	pub name: String, // [char; 20],
	pub sample_info: Vec<SampleInfo>, // [SampleInfo; DEFAULT_NUMBER_OF_SAMPLES]
	pub length: u8, // Length 1-128
	pub nt_restart: u8, // Set to 127
	pub positions: Positions, // The order in which to play patterns
	pub mk: [u8; 4], // Set to M.K.
	pub patterns: Vec<Pattern>,
	pub sample_data: Vec<Vec<u8>> // The bytes in the samples -127 - 127
}

impl PTModule {
	pub fn new() -> PTModule {
		let mut ptmod = PTModule{name: String::new(), sample_info: Vec::new(), length:0,
			nt_restart: 127, positions: Positions{data: [0; 128]}, mk: MAGIC_MK,
			patterns: Vec::new(), sample_data: Vec::new() };

		for _ in 0..DEFAULT_NUMBER_OF_SAMPLES {
			ptmod.sample_info.push(SampleInfo::new());
		}

		ptmod
	}
}

fn read_or_panic(reader: &mut Read, data: &mut [u8]) {
	let mut pos = 0;
	while pos < data.len() {
		let slice = &mut data[pos..];
		match reader.read(slice) {
			Ok(n) => pos += n,
			_ => panic!("Failed to read bytes")
		};
	}
}

fn write_or_panic(writer: &mut Write, data: &[u8]) {
	match writer.write_all(&data) {
		Ok(_) => (),
		_ => panic!("Failed to write bytes")
	};
}

fn read_0_padded_string(reader: &mut Read, len: usize) -> String {
	let mut data = vec![0u8; len];
	read_or_panic(reader,&mut data);
	let mut str = String::new();
	for byte in data {
		if byte != 0 {
			str.push(byte as char);
		}
	}

	str
}

fn write_0_padded_string(writer: &mut Write, str: &String, len: usize) {
	let mut data = vec![0u8; len];
	let str_buf = str.as_bytes();
	let m = cmp::min(data.len(),str_buf.len());
	for i in 0..m {
		data[i] = str_buf[i];
	}

	write_or_panic(writer, &data);
}

fn read_big_endian_u16(reader: &mut Read) -> u16 {
	let mut data_arr = [0u8; 2];
	read_or_panic(reader,&mut data_arr);

	let mut data:u16 = 0;

	for n in data_arr.iter() {
		data = (data << 8) + *n as u16;
	}

	data
}

fn write_big_endian_u16(writer: &mut Write, val: u16) {
	let mut data = [0u8; 2];
	data[0] = (val >> 8) as u8;
	data[1] = (val & 0xff) as u8;

	write_or_panic(writer, &data);
}

fn read_u8(reader: &mut Read) -> u8 {
	let mut data = [0u8; 1];

	read_or_panic(reader, &mut data);

	data[0]
}

fn write_u8(writer: &mut Write, val: u8) {
	let mut data = [0u8; 1];
	data[0] = val;

	write_or_panic(writer, &data);
}

/// Write a 31 sample Amiga ProTracker mod-file
/// Panics on all errors
pub fn write_mod(writer: &mut Write, module: &mut PTModule) {

	// First write songname, 20 bytes, pad with 0
	write_0_padded_string(writer,&module.name,20);
	// Then write all 32 samples
	// TODO Handle the case when the vector has less than 31 samples
	let mut num_samples = 0;
	for i in 0..DEFAULT_NUMBER_OF_SAMPLES {
		let ref si = module.sample_info[i];
		// Sample name
		write_0_padded_string(writer, &si.name, 22);
		// Sample length
		write_big_endian_u16(writer, si.length);
		if si.length > 0 {
			num_samples += 1;
		}
		// Finetune
		write_u8(writer, si.finetune);
		// Volume
		write_u8(writer, si.volume);
		// Repeat start
		write_big_endian_u16(writer, si.repeat_start);
		// Repeat length
		write_big_endian_u16(writer, si.repeat_length);
	}

	let num_samples = num_samples;
	// Songlength
	write_u8(writer, module.length);
	// nt_restart
	write_u8(writer, module.nt_restart);
	// Song positions
	write_or_panic(writer,&module.positions.data);
	// M.K.
	write_or_panic(writer,&module.mk);

	// All patterns
	for pattern_it in module.patterns.iter() {
		// All rows
		for row_it in pattern_it.rows.iter() {
			// All channels
			for channel_it in row_it.channels.iter() {
				let mut data = [0u8; 4];

				data[0] = (channel_it.sample_number & 0xf0) | ((channel_it.period & 0xf00) >> 8) as u8;
				data[1] = (channel_it.period & 0xff) as u8;
				data[2] = ((channel_it.sample_number & 0x0f) << 4) | ((channel_it.effect & 0xf00) >> 8) as u8;
				data[3] = (channel_it.effect & 0xff) as u8;

				write_or_panic(writer,&data);
			}
		}
	}

	// And finally all samples
	for sample_it in module.sample_data.iter() {
		write_or_panic(writer,sample_it);
	}

	if num_samples != module.sample_data.len() {
		panic!("Warning! Number of samples does not match sample data");
	}
}

/// Read a 31 sample Amiga ProTracker mod-file
/// Panics on all errors
pub fn read_mod(reader: &mut Read) -> PTModule {
	let mut module = PTModule::new();

	// First read 20 bytes songname
	module.name = read_0_padded_string(reader, 20);
	// Read all sample info
	// TODO Handle 15 sample files
	for i in 0..DEFAULT_NUMBER_OF_SAMPLES {
		let si = &mut module.sample_info[i];
		// Sample name
		si.name = read_0_padded_string(reader, 22);
		si.length = read_big_endian_u16(reader);
		// Finetune
		si.finetune = read_u8(reader);
		// Volume
		si.volume = read_u8(reader);
		// Repeat start
		si.repeat_start = read_big_endian_u16(reader);
		// Repeat length
		si.repeat_length = read_big_endian_u16(reader);
	}

	// Songlength
	module.length = read_u8(reader);
	// nt_restart
	module.nt_restart = read_u8(reader);
	// Song positions
	read_or_panic(reader,&mut module.positions.data);
	// M.K.
	read_or_panic(reader,&mut module.mk);
	if module.mk != MAGIC_MK {
		panic!("Unknown format {:?}", module.mk);
	}

	// Read all patterns
	let mut num_patterns = 0;
	for n in module.positions.data.iter() {
		num_patterns = cmp::max(num_patterns,*n);
	}
	num_patterns += 1;
	for _ in 0..num_patterns {
		let mut pattern = Pattern::new();
		for mut row in &mut pattern.rows {
			for channel in &mut row.channels {
				let mut data  = [0u8;4];
				read_or_panic(reader,&mut data);
				channel.sample_number = (data[0] & 0xf0) | ((data[2] & 0xf0) >> 4);
				channel.period = (((data[0] & 0x0f) as u16) << 8) | (data[1] as u16);
				channel.effect = (((data[2] & 0x0f) as u16) << 8) | (data[3] as u16);
			}
		}
		module.patterns.push(pattern);
	}

	// Read all samples
	for si in &module.sample_info {
		if si.length > 0 {
			let length_in_bytes = si.length * 2;
			let mut data = vec![0u8; length_in_bytes as usize];
			read_or_panic(reader,&mut data);
			module.sample_data.push(data);
		}
	}

	// Now we have read all data.
	// Sanity check that the reader is empty
	let mut data = [0u8; 1];
	match reader.read(&mut data) {
		Ok(n) if n == data.len() => panic!("Unread data left in file"),
		_ => ()
	};

	module
}
<commit_msg>Started replacing panic with try<commit_after>use std::io::Write;
use std::io::Read;
use std::fmt;
use std::cmp;
use std::io;

const DEFAULT_NUMBER_OF_SAMPLES:usize = 31;
const DEFAULT_NUMBER_OF_ROWS_PER_PATTERN:usize = 64;
const DEFAULT_NUMBER_OF_CHANNELS_PER_ROW:usize = 4;
const MAGIC_MK:[u8; 4] =['M' as u8, '.' as u8, 'K' as u8, '.' as u8];

#[derive(Debug)]
pub struct SampleInfo {
	pub name: String, // [char; 22],
	pub length: u16, // In words, so multiply by 2
	pub finetune: u8,
	pub volume: u8, // 0-64
	pub repeat_start: u16, // In words from start of sample data, so multiply by 2
	pub repeat_length: u16, // Number of words to loop, multiply by 2
}

impl SampleInfo {
	pub fn new() -> SampleInfo {
		SampleInfo{name: String::new(), length:0, finetune:0, volume:0, repeat_start:0, repeat_length:0}
	}
}

#[derive(Debug)]
pub struct Channel {
	pub period: u16, // Really u12
	pub sample_number: u8,
	pub effect: u16 // Really u12
}

impl Channel {
	pub fn new() -> Channel {
		Channel{period: 0, sample_number: 0, effect: 0}
	}
}

#[derive(Debug)]
pub struct Row {
	pub channels: Vec<Channel>
}

impl Row {
	pub fn new() -> Row {
		let mut row = Row{channels: Vec::new()};
		for _ in 0..DEFAULT_NUMBER_OF_CHANNELS_PER_ROW {
			row.channels.push(Channel::new());
		}

		row
	}
}

#[derive(Debug)]
pub struct Pattern {
	pub rows: Vec<Row>
}

impl Pattern {
	pub fn new() -> Pattern {
		let mut p = Pattern{rows: Vec::new()};

		for _ in 0..DEFAULT_NUMBER_OF_ROWS_PER_PATTERN {
			p.rows.push(Row::new());
		}

		p
	}
}

pub struct Positions {
	pub data: [u8; 128]
}

impl fmt::Debug for Positions {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        self.data[..].fmt(formatter)
    }
}

#[derive(Debug)]
pub struct PTModule {
	pub name: String, // [char; 20],
	pub sample_info: Vec<SampleInfo>, // [SampleInfo; DEFAULT_NUMBER_OF_SAMPLES]
	pub length: u8, // Length 1-128
	pub nt_restart: u8, // Set to 127
	pub positions: Positions, // The order in which to play patterns
	pub mk: [u8; 4], // Set to M.K.
	pub patterns: Vec<Pattern>,
	pub sample_data: Vec<Vec<u8>> // The bytes in the samples -127 - 127
}

impl PTModule {
	pub fn new() -> PTModule {
		let mut ptmod = PTModule{name: String::new(), sample_info: Vec::new(), length:0,
			nt_restart: 127, positions: Positions{data: [0; 128]}, mk: MAGIC_MK,
			patterns: Vec::new(), sample_data: Vec::new() };

		for _ in 0..DEFAULT_NUMBER_OF_SAMPLES {
			ptmod.sample_info.push(SampleInfo::new());
		}

		ptmod
	}
}

fn read_all(reader: &mut Read, data: &mut [u8]) -> io::Result<usize> {
	let mut pos = 0;
	while pos < data.len() {
		let slice = &mut data[pos..];
		let n = try!(reader.read(slice));
		pos += n;
	}

	Ok(pos)
}

fn write_or_panic(writer: &mut Write, data: &[u8]) {
	match writer.write_all(&data) {
		Ok(_) => (),
		_ => panic!("Failed to write bytes")
	};
}

fn read_0_padded_string(reader: &mut Read, len: usize) -> String {
	let mut data = vec![0u8; len];
	read_all(reader,&mut data);
	let mut str = String::new();
	for byte in data {
		if byte != 0 {
			str.push(byte as char);
		}
	}

	str
}

fn write_0_padded_string(writer: &mut Write, str: &String, len: usize) {
	let mut data = vec![0u8; len];
	let str_buf = str.as_bytes();
	let m = cmp::min(data.len(),str_buf.len());
	for i in 0..m {
		data[i] = str_buf[i];
	}

	write_or_panic(writer, &data);
}

fn read_big_endian_u16(reader: &mut Read) -> u16 {
	let mut data_arr = [0u8; 2];
	read_all(reader,&mut data_arr);

	let mut data:u16 = 0;

	for n in data_arr.iter() {
		data = (data << 8) + *n as u16;
	}

	data
}

fn write_big_endian_u16(writer: &mut Write, val: u16) {
	let mut data = [0u8; 2];
	data[0] = (val >> 8) as u8;
	data[1] = (val & 0xff) as u8;

	write_or_panic(writer, &data);
}

fn read_u8(reader: &mut Read) -> u8 {
	let mut data = [0u8; 1];

	read_all(reader, &mut data);

	data[0]
}

fn write_u8(writer: &mut Write, val: u8) {
	let mut data = [0u8; 1];
	data[0] = val;

	write_or_panic(writer, &data);
}

/// Write a 31 sample Amiga ProTracker mod-file
/// Panics on all errors
pub fn write_mod(writer: &mut Write, module: &mut PTModule) {

	// First write songname, 20 bytes, pad with 0
	write_0_padded_string(writer,&module.name,20);
	// Then write all 32 samples
	// TODO Handle the case when the vector has less than 31 samples
	let mut num_samples = 0;
	for i in 0..DEFAULT_NUMBER_OF_SAMPLES {
		let ref si = module.sample_info[i];
		// Sample name
		write_0_padded_string(writer, &si.name, 22);
		// Sample length
		write_big_endian_u16(writer, si.length);
		if si.length > 0 {
			num_samples += 1;
		}
		// Finetune
		write_u8(writer, si.finetune);
		// Volume
		write_u8(writer, si.volume);
		// Repeat start
		write_big_endian_u16(writer, si.repeat_start);
		// Repeat length
		write_big_endian_u16(writer, si.repeat_length);
	}

	let num_samples = num_samples;
	// Songlength
	write_u8(writer, module.length);
	// nt_restart
	write_u8(writer, module.nt_restart);
	// Song positions
	write_or_panic(writer,&module.positions.data);
	// M.K.
	write_or_panic(writer,&module.mk);

	// All patterns
	for pattern_it in module.patterns.iter() {
		// All rows
		for row_it in pattern_it.rows.iter() {
			// All channels
			for channel_it in row_it.channels.iter() {
				let mut data = [0u8; 4];

				data[0] = (channel_it.sample_number & 0xf0) | ((channel_it.period & 0xf00) >> 8) as u8;
				data[1] = (channel_it.period & 0xff) as u8;
				data[2] = ((channel_it.sample_number & 0x0f) << 4) | ((channel_it.effect & 0xf00) >> 8) as u8;
				data[3] = (channel_it.effect & 0xff) as u8;

				write_or_panic(writer,&data);
			}
		}
	}

	// And finally all samples
	for sample_it in module.sample_data.iter() {
		write_or_panic(writer,sample_it);
	}

	if num_samples != module.sample_data.len() {
		panic!("Warning! Number of samples does not match sample data");
	}
}

/// Read a 31 sample Amiga ProTracker mod-file
/// Panics on all errors
pub fn read_mod(reader: &mut Read) -> PTModule {
	let mut module = PTModule::new();

	// First read 20 bytes songname
	module.name = read_0_padded_string(reader, 20);
	// Read all sample info
	// TODO Handle 15 sample files
	for i in 0..DEFAULT_NUMBER_OF_SAMPLES {
		let si = &mut module.sample_info[i];
		// Sample name
		si.name = read_0_padded_string(reader, 22);
		si.length = read_big_endian_u16(reader);
		// Finetune
		si.finetune = read_u8(reader);
		// Volume
		si.volume = read_u8(reader);
		// Repeat start
		si.repeat_start = read_big_endian_u16(reader);
		// Repeat length
		si.repeat_length = read_big_endian_u16(reader);
	}

	// Songlength
	module.length = read_u8(reader);
	// nt_restart
	module.nt_restart = read_u8(reader);
	// Song positions
	read_all(reader,&mut module.positions.data);
	// M.K.
	read_all(reader,&mut module.mk);
	if module.mk != MAGIC_MK {
		panic!("Unknown format {:?}", module.mk);
	}

	// Read all patterns
	let mut num_patterns = 0;
	for n in module.positions.data.iter() {
		num_patterns = cmp::max(num_patterns,*n);
	}
	num_patterns += 1;
	for _ in 0..num_patterns {
		let mut pattern = Pattern::new();
		for mut row in &mut pattern.rows {
			for channel in &mut row.channels {
				let mut data  = [0u8;4];
				read_all(reader,&mut data);
				channel.sample_number = (data[0] & 0xf0) | ((data[2] & 0xf0) >> 4);
				channel.period = (((data[0] & 0x0f) as u16) << 8) | (data[1] as u16);
				channel.effect = (((data[2] & 0x0f) as u16) << 8) | (data[3] as u16);
			}
		}
		module.patterns.push(pattern);
	}

	// Read all samples
	for si in &module.sample_info {
		if si.length > 0 {
			let length_in_bytes = si.length * 2;
			let mut data = vec![0u8; length_in_bytes as usize];
			read_all(reader,&mut data);
			module.sample_data.push(data);
		}
	}

	// Now we have read all data.
	// Sanity check that the reader is empty
	let mut data = [0u8; 1];
	match reader.read(&mut data) {
		Ok(n) if n == data.len() => panic!("Unread data left in file"),
		_ => ()
	};

	module
}
<|endoftext|>