<commit_before>#[ path="../functional.rs"] mod functional;
#[ path="helpers.rs"]       mod helpers;
#[ path="shellprocess.rs"]  mod shellprocess;
#[ path="parser.rs"]        mod parser;

pub mod shell {
    use std::run::Process;
    use std::os;
    use std::io::{stdin, stdout, stdio};
    use std::io::buffered::BufferedReader;
    use std::io::signal::{Listener, Interrupt};
    use std::task::try;

    use std::libc::consts::os::posix88::{STDOUT_FILENO, STDIN_FILENO};
    use std::libc::types::os::arch::posix88::pid_t;
    use std::libc;

    use helpers::helpers::{split_words, input_redirect, output_redirect, pipe_redirect};
    use functional::borrowed_maybe;
    use shellprocess::fg::FgProcess;
    use shellprocess::bg::BgProcess;
    use parser::parser::{lex,parse};
    use parser::lineelem::LineElem;
    use parser::pathtype::{Read, Write};

    extern {
        pub fn kill(pid: pid_t, sig: libc::c_int) -> libc::c_int;
    }

    pub struct Shell {
        cmd_prompt : ~str,
        history    : ~[~str],
        processes  : ~[~BgProcess],
        broken : bool,
    }

    impl Shell {
        pub fn new(prompt_str: &str) -> Shell {
            Shell {
                cmd_prompt: prompt_str.to_owned(),
                history: ~[],
                processes: ~[],
                broken: false,
            }
        }

        // Start the shell with an interrupt handler. Only needed when
        // an interactive shell is used.
        pub fn start(&mut self) {
            // Setup the interrupt handler. Has to happen here, or it won't
            // retain control over interrupts.
            let mut listener = Listener::new();
            listener.register(Interrupt);
            let port = listener.port;
            spawn(proc() {
                try(proc() {
                    loop {
                        match port.recv_opt() {
                            Some(Interrupt) => {
                            }
                            None => {
                                break;
                            }
                            _ => {
                            }
                        }
                    }
                    return 0
                });
                return ();
            });
            self.display_prompt();
        }

        // Show the prompt. If called by itself, without start(), no interrupt
        // handling will occur.
        fn display_prompt(&mut self) {
            // Standard input reader
            let mut stdin = BufferedReader::new(stdin());
            // Show the prompt
            print(self.cmd_prompt);
            stdio::flush();

            let line = stdin.read_line().unwrap();
            let cmd_line = line.trim().to_owned();
            let program = cmd_line.splitn(' ', 1).nth(0).expect("no program");
            self.disown_dead();

            match program {
                "" =>  {
                    self.display_prompt();
                }
                "exit" =>  {
                    self.kill_all();
                }
                "history" => {
                    self.show_hist();
                    self.display_prompt();
                }
                "jobs" => {
                    self.push_hist(cmd_line);
                    self.jobs();
                    self.display_prompt();
                }
                "cd" =>  {
                    self.push_hist(cmd_line);
                    self.chdir(cmd_line);
                    self.display_prompt();
                }
                _ => {
                    self.push_hist(cmd_line);
                    self.run_cmdline(cmd_line);
                    self.display_prompt();
                }
            }
        }

        fn _run(&mut self, elem : ~LineElem) -> Option<~Process> {
            if elem.pipe.is_some() {
                let left = self.parse_process(elem.clone(), None, None).expect("Couldn't spawn!");
                Some(elem.iter().fold(left, |left, right| {
                    let right = self.pipe_file(right).expect("Couldn't spawn!");
                    pipe_redirect(left,right)
                }))
            }
            else {
                self.pipe_file(elem)
            }
        }

        // "Pipe" output to or from a file. If no file is available, just return
        // the created process.
        fn pipe_file(&mut self, elem : ~LineElem) -> Option<~Process> {
            match elem.clone().file {
                Some(file) => {
                    match file.mode {
                        Read => {
                            let process = self.to_process(elem).expect("Couldn't spawn!");
                            Some(input_redirect(process, &file.path))
                        }
                        Write => {
                            let process = self.parse_process(elem, None, None).expect("Couldn't spawn!");
                            output_redirect(process, &file.path);
                            None
                        }
                    }
                }
                None => {
                    self.to_process(elem)
                }
            }
        }

        // Make a process from a LineElem. Sets the output to stdout if the "last"
        // field is true.
        fn to_process(&mut self, elem : ~LineElem) -> Option<~Process> {
                self.parse_process(elem.clone(),
                                None,
                                if elem.last { Some(STDOUT_FILENO) }
                                else { None })
        }

        // Determine the type of the current block, and send it to the right
        // parsing function.
        pub fn run_cmdline(&mut self, cmd_line: &str) {
            let lex = lex(cmd_line);
            let parse = parse(lex);
            if parse.pipe.is_none() && parse.file.is_none() {
                self.parse_process(parse, Some(STDIN_FILENO), Some(STDOUT_FILENO));
            }
            else {
                self._run(parse);
            }
        }

        // Parse a new lone process. Background/foreground it appropriately.
        fn parse_process(&mut self,
                        cmd: ~LineElem,
                        stdin: Option<i32>,
                        stdout:Option<i32>)
                        -> Option<~Process> {
            if (cmd.argv.len() > 0 && cmd.argv.last() == &~"&") {
                let mut argv = cmd.argv.to_owned();
                argv.pop();
                self.make_bg_process(cmd);
                None
            }
            else {
                Some(~(FgProcess::new(cmd, stdin, stdout).run()))
            }
        }

        // background processes.
        fn make_bg_process(&mut self, cmd : ~LineElem) {
            let name = cmd.program.to_owned();
            let mut process = BgProcess::new(cmd);
            match process.run() {
                Some(pid) => {
                    println!("{:s} {:i}", name, pid);
                    self.processes.push(~process);
                }
                None => {
                }
            }
        }

        // Nice extra feature: list running jobs.
        fn jobs(&mut self) {
            for cmd in self.processes.iter() {
                print!("{:s}", cmd.command);
                match cmd.pid {
                    Some(pid) => {
                        println!(" {:i}", pid);
                    }
                    None => {
                        println("");
                    }
                }
            }
        }

        // Remove dead processes from the list of processes.
        fn disown_dead(&mut self) {
            let mut dead : ~[uint];
            dead = ~[];
            let mut i = 0;
            for cmd in self.processes.iter() {
                borrowed_maybe(&cmd.exit_port, |port| match port.try_recv() {
                    Some(_) => {dead.push(i); Some(i)},
                    _ => None,
                });
                i += 1;
            }
            for &p in dead.iter() {
                self.processes.remove(p);
            }
        }

        // Kill all the background jobs. Used when "exit" is called at the CLI.
        fn kill_all(&mut self) {
            for p in self.processes.iter() {
                match p.pid {
                    Some(pid) => {
                        unsafe {
                            // Kill perhaps isn't the best way to do this, but
                            // without it we have no way of killing our background
                            // processes with only the PID - we'd need the process
                            // object itself, and that can't be moved out of the
                            // spawn() that it's trapped inside.
                            kill(pid, 15);
                        }
                    }
                    None => {
                    }
                }
            }
        }

        // Push a new command onto history.
        fn push_hist(&mut self, cmd_line: &str) {
            &self.history.push(cmd_line.to_owned());
        }

        // Show the history.
        fn show_hist(&mut self) {
            let mut hist = ~"";
            for i in self.history.iter() {
                let s = i.to_owned();
                match hist {
                    ~"" => { hist = s; }
                    _   => { hist = hist + "\n" + s; }
                }
            }
            println!("{:s}", hist);
        }

        // Change directories.
        fn chdir(&mut self, cmd_line: &str) {
            let mut argv: ~[~str] = split_words(cmd_line);
            if argv.len() > 0 {
                argv.remove(0);
                let path = Path::new(argv[0]);
                os::change_dir(&path);
            }
        }
    }
}
<commit_msg>rename Shell._run() to Shell.run()<commit_after>#[ path="../functional.rs"] mod functional;
#[ path="helpers.rs"]       mod helpers;
#[ path="shellprocess.rs"]  mod shellprocess;
#[ path="parser.rs"]        mod parser;

pub mod shell {
    use std::run::Process;
    use std::os;
    use std::io::{stdin, stdout, stdio};
    use std::io::buffered::BufferedReader;
    use std::io::signal::{Listener, Interrupt};
    use std::task::try;

    use std::libc::consts::os::posix88::{STDOUT_FILENO, STDIN_FILENO};
    use std::libc::types::os::arch::posix88::pid_t;
    use std::libc;

    use helpers::helpers::{split_words, input_redirect, output_redirect, pipe_redirect};
    use functional::borrowed_maybe;
    use shellprocess::fg::FgProcess;
    use shellprocess::bg::BgProcess;
    use parser::parser::{lex,parse};
    use parser::lineelem::LineElem;
    use parser::pathtype::{Read, Write};

    extern {
        pub fn kill(pid: pid_t, sig: libc::c_int) -> libc::c_int;
    }

    pub struct Shell {
        cmd_prompt : ~str,
        history    : ~[~str],
        processes  : ~[~BgProcess],
        broken : bool,
    }

    impl Shell {
        pub fn new(prompt_str: &str) -> Shell {
            Shell {
                cmd_prompt: prompt_str.to_owned(),
                history: ~[],
                processes: ~[],
                broken: false,
            }
        }

        // Start the shell with an interrupt handler. Only needed when
        // an interactive shell is used.
        pub fn start(&mut self) {
            // Setup the interrupt handler. Has to happen here, or it won't
            // retain control over interrupts.
            let mut listener = Listener::new();
            listener.register(Interrupt);
            let port = listener.port;
            spawn(proc() {
                try(proc() {
                    loop {
                        match port.recv_opt() {
                            Some(Interrupt) => {
                            }
                            None => {
                                break;
                            }
                            _ => {
                            }
                        }
                    }
                    return 0
                });
                return ();
            });
            self.display_prompt();
        }

        // Show the prompt. If called by itself, without start(), no interrupt
        // handling will occur.
        fn display_prompt(&mut self) {
            // Standard input reader
            let mut stdin = BufferedReader::new(stdin());
            // Show the prompt
            print(self.cmd_prompt);
            stdio::flush();

            let line = stdin.read_line().unwrap();
            let cmd_line = line.trim().to_owned();
            let program = cmd_line.splitn(' ', 1).nth(0).expect("no program");
            self.disown_dead();

            match program {
                "" =>  {
                    self.display_prompt();
                }
                "exit" =>  {
                    self.kill_all();
                }
                "history" => {
                    self.show_hist();
                    self.display_prompt();
                }
                "jobs" => {
                    self.push_hist(cmd_line);
                    self.jobs();
                    self.display_prompt();
                }
                "cd" =>  {
                    self.push_hist(cmd_line);
                    self.chdir(cmd_line);
                    self.display_prompt();
                }
                _ => {
                    self.push_hist(cmd_line);
                    self.run_cmdline(cmd_line);
                    self.display_prompt();
                }
            }
        }

        fn run(&mut self, elem : ~LineElem) -> Option<~Process> {
            if elem.pipe.is_some() {
                let left = self.parse_process(elem.clone(), None, None).expect("Couldn't spawn!");
                Some(elem.iter().fold(left, |left, right| {
                    let right = self.pipe_file(right).expect("Couldn't spawn!");
                    pipe_redirect(left,right)
                }))
            }
            else {
                self.pipe_file(elem)
            }
        }

        // "Pipe" output to or from a file. If no file is available, just return
        // the created process.
        fn pipe_file(&mut self, elem : ~LineElem) -> Option<~Process> {
            match elem.clone().file {
                Some(file) => {
                    match file.mode {
                        Read => {
                            let process = self.to_process(elem).expect("Couldn't spawn!");
                            Some(input_redirect(process, &file.path))
                        }
                        Write => {
                            let process = self.parse_process(elem, None, None).expect("Couldn't spawn!");
                            output_redirect(process, &file.path);
                            None
                        }
                    }
                }
                None => {
                    self.to_process(elem)
                }
            }
        }

        // Make a process from a LineElem. Sets the output to stdout if the "last"
        // field is true.
        fn to_process(&mut self, elem : ~LineElem) -> Option<~Process> {
                self.parse_process(elem.clone(),
                                None,
                                if elem.last { Some(STDOUT_FILENO) }
                                else { None })
        }

        // Determine the type of the current block, and send it to the right
        // parsing function.
        pub fn run_cmdline(&mut self, cmd_line: &str) {
            let lex = lex(cmd_line);
            let parse = parse(lex);
            if parse.pipe.is_none() && parse.file.is_none() {
                self.parse_process(parse, Some(STDIN_FILENO), Some(STDOUT_FILENO));
            }
            else {
                self.run(parse);
            }
        }

        // Parse a new lone process. Background/foreground it appropriately.
        fn parse_process(&mut self,
                        cmd: ~LineElem,
                        stdin: Option<i32>,
                        stdout:Option<i32>)
                        -> Option<~Process> {
            if (cmd.argv.len() > 0 && cmd.argv.last() == &~"&") {
                let mut argv = cmd.argv.to_owned();
                argv.pop();
                self.make_bg_process(cmd);
                None
            }
            else {
                Some(~(FgProcess::new(cmd, stdin, stdout).run()))
            }
        }

        // background processes.
        fn make_bg_process(&mut self, cmd : ~LineElem) {
            let name = cmd.program.to_owned();
            let mut process = BgProcess::new(cmd);
            match process.run() {
                Some(pid) => {
                    println!("{:s} {:i}", name, pid);
                    self.processes.push(~process);
                }
                None => {
                }
            }
        }

        // Nice extra feature: list running jobs.
        fn jobs(&mut self) {
            for cmd in self.processes.iter() {
                print!("{:s}", cmd.command);
                match cmd.pid {
                    Some(pid) => {
                        println!(" {:i}", pid);
                    }
                    None => {
                        println("");
                    }
                }
            }
        }

        // Remove dead processes from the list of processes.
        fn disown_dead(&mut self) {
            let mut dead : ~[uint];
            dead = ~[];
            let mut i = 0;
            for cmd in self.processes.iter() {
                borrowed_maybe(&cmd.exit_port, |port| match port.try_recv() {
                    Some(_) => {dead.push(i); Some(i)},
                    _ => None,
                });
                i += 1;
            }
            for &p in dead.iter() {
                self.processes.remove(p);
            }
        }

        // Kill all the background jobs. Used when "exit" is called at the CLI.
        fn kill_all(&mut self) {
            for p in self.processes.iter() {
                match p.pid {
                    Some(pid) => {
                        unsafe {
                            // Kill perhaps isn't the best way to do this, but
                            // without it we have no way of killing our background
                            // processes with only the PID - we'd need the process
                            // object itself, and that can't be moved out of the
                            // spawn() that it's trapped inside.
                            kill(pid, 15);
                        }
                    }
                    None => {
                    }
                }
            }
        }

        // Push a new command onto history.
        fn push_hist(&mut self, cmd_line: &str) {
            &self.history.push(cmd_line.to_owned());
        }

        // Show the history.
        fn show_hist(&mut self) {
            let mut hist = ~"";
            for i in self.history.iter() {
                let s = i.to_owned();
                match hist {
                    ~"" => { hist = s; }
                    _   => { hist = hist + "\n" + s; }
                }
            }
            println!("{:s}", hist);
        }

        // Change directories.
        fn chdir(&mut self, cmd_line: &str) {
            let mut argv: ~[~str] = split_words(cmd_line);
            if argv.len() > 0 {
                argv.remove(0);
                let path = Path::new(argv[0]);
                os::change_dir(&path);
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// Detecting language items.
//
// Language items are items that represent concepts intrinsic to the language
// itself. Examples are:
//
// * Traits that specify "kinds"; e.g. "Sync", "Send".
//
// * Traits that represent operators; e.g. "Add", "Sub", "Index".
//
// * Functions called by the compiler itself.

pub use self::LangItem::*;

use session::Session;
use metadata::csearch::each_lang_item;
use middle::ty;
use middle::weak_lang_items;
use util::nodemap::FnvHashMap;

use syntax::ast;
use syntax::ast_util::local_def;
use syntax::attr::AttrMetaMethods;
use syntax::codemap::{DUMMY_SP, Span};
use syntax::parse::token::InternedString;
use syntax::visit::Visitor;
use syntax::visit;

use std::iter::Enumerate;
use std::num::FromPrimitive;
use std::slice;

// The actual lang items defined come at the end of this file in one handy table.
// So you probably just want to nip down to the end.
macro_rules! lets_do_this {
    (
        $( $variant:ident, $name:expr, $method:ident; )*
    ) => {

#[derive(Copy, FromPrimitive, PartialEq, Eq, Hash)]
pub enum LangItem {
    $($variant),*
}

pub struct LanguageItems {
    pub items: Vec<Option<ast::DefId>>,
    pub missing: Vec<LangItem>,
}

impl LanguageItems {
    pub fn new() -> LanguageItems {
        fn foo(_: LangItem) -> Option<ast::DefId> { None }

        LanguageItems {
            items: vec!($(foo($variant)),*),
            missing: Vec::new(),
        }
    }

    pub fn items<'a>(&'a self) -> Enumerate<slice::Iter<'a, Option<ast::DefId>>> {
        self.items.iter().enumerate()
    }

    pub fn item_name(index: uint) -> &'static str {
        let item: Option<LangItem> = FromPrimitive::from_uint(index);
        match item {
            $( Some($variant) => $name, )*
            None => "???"
        }
    }

    pub fn require(&self, it: LangItem) -> Result<ast::DefId, String> {
        match self.items[it as uint] {
            Some(id) => Ok(id),
            None => {
                Err(format!("requires `{}` lang_item",
                            LanguageItems::item_name(it as uint)))
            }
        }
    }

    pub fn from_builtin_kind(&self, bound: ty::BuiltinBound)
                             -> Result<ast::DefId, String>
    {
        match bound {
            ty::BoundSend => self.require(SendTraitLangItem),
            ty::BoundSized => self.require(SizedTraitLangItem),
            ty::BoundCopy => self.require(CopyTraitLangItem),
            ty::BoundSync => self.require(SyncTraitLangItem),
        }
    }

    pub fn to_builtin_kind(&self, id: ast::DefId) -> Option<ty::BuiltinBound> {
        if Some(id) == self.send_trait() {
            Some(ty::BoundSend)
        } else if Some(id) == self.sized_trait() {
            Some(ty::BoundSized)
        } else if Some(id) == self.copy_trait() {
            Some(ty::BoundCopy)
        } else if Some(id) == self.sync_trait() {
            Some(ty::BoundSync)
        } else {
            None
        }
    }

    pub fn fn_trait_kind(&self, id: ast::DefId) -> Option<ty::ClosureKind> {
        let def_id_kinds = [
            (self.fn_trait(), ty::FnClosureKind),
            (self.fn_mut_trait(), ty::FnMutClosureKind),
            (self.fn_once_trait(), ty::FnOnceClosureKind),
            ];

        for &(opt_def_id, kind) in def_id_kinds.iter() {
            if Some(id) == opt_def_id {
                return Some(kind);
            }
        }

        None
    }

    $(
        #[allow(dead_code)]
        pub fn $method(&self) -> Option<ast::DefId> {
            self.items[$variant as uint]
        }
    )*
}

struct LanguageItemCollector<'a> {
    items: LanguageItems,

    session: &'a Session,

    item_refs: FnvHashMap<&'static str, uint>,
}

impl<'a, 'v> Visitor<'v> for LanguageItemCollector<'a> {
    fn visit_item(&mut self, item: &ast::Item) {
        match extract(item.attrs.as_slice()) {
            Some(value) => {
                let item_index = self.item_refs.get(value.get()).map(|x| *x);

                match item_index {
                    Some(item_index) => {
                        self.collect_item(item_index, local_def(item.id), item.span)
                    }
                    None => {}
                }
            }
            None => {}
        }

        visit::walk_item(self, item);
    }
}

impl<'a> LanguageItemCollector<'a> {
    pub fn new(session: &'a Session) -> LanguageItemCollector<'a> {
        let mut item_refs = FnvHashMap();

        $( item_refs.insert($name, $variant as uint); )*

        LanguageItemCollector {
            session: session,
            items: LanguageItems::new(),
            item_refs: item_refs
        }
    }

    pub fn collect_item(&mut self, item_index: uint,
                        item_def_id: ast::DefId, span: Span) {
        // Check for duplicates.
        match self.items.items[item_index] {
            Some(original_def_id) if original_def_id != item_def_id => {
                span_err!(self.session, span, E0152,
                    "duplicate entry for `{}`", LanguageItems::item_name(item_index));
            }
            Some(_) | None => {
                // OK.
            }
        }

        // Matched.
        self.items.items[item_index] = Some(item_def_id);
    }

    pub fn collect_local_language_items(&mut self, krate: &ast::Crate) {
        visit::walk_crate(self, krate);
    }

    pub fn collect_external_language_items(&mut self) {
        let crate_store = &self.session.cstore;
        crate_store.iter_crate_data(|crate_number, _crate_metadata| {
            each_lang_item(crate_store, crate_number, |node_id, item_index| {
                let def_id = ast::DefId { krate: crate_number, node: node_id };
                self.collect_item(item_index, def_id, DUMMY_SP);
                true
            });
        })
    }

    pub fn collect(&mut self, krate: &ast::Crate) {
        self.collect_local_language_items(krate);
        self.collect_external_language_items();
    }
}

pub fn extract(attrs: &[ast::Attribute]) -> Option<InternedString> {
    for attribute in attrs.iter() {
        match attribute.value_str() {
            Some(ref value) if attribute.check_name("lang") => {
                return Some(value.clone());
            }
            _ => {}
        }
    }

    return None;
}

pub fn collect_language_items(krate: &ast::Crate,
                              session: &Session) -> LanguageItems {
    let mut collector = LanguageItemCollector::new(session);
    collector.collect(krate);
    let LanguageItemCollector { mut items, .. } = collector;
    weak_lang_items::check_crate(krate, session, &mut items);
    session.abort_if_errors();
    items
}

// End of the macro
    }
}

lets_do_this! {
//  Variant name,                    Name,                      Method name;
    SendTraitLangItem,               "send",                    send_trait;
    SizedTraitLangItem,              "sized",                   sized_trait;
    CopyTraitLangItem,               "copy",                    copy_trait;
    SyncTraitLangItem,               "sync",                    sync_trait;

    DropTraitLangItem,               "drop",                    drop_trait;

    AddTraitLangItem,                "add",                     add_trait;
    SubTraitLangItem,                "sub",                     sub_trait;
    MulTraitLangItem,                "mul",                     mul_trait;
    DivTraitLangItem,                "div",                     div_trait;
    RemTraitLangItem,                "rem",                     rem_trait;
    NegTraitLangItem,                "neg",                     neg_trait;
    NotTraitLangItem,                "not",                     not_trait;
    BitXorTraitLangItem,             "bitxor",                  bitxor_trait;
    BitAndTraitLangItem,             "bitand",                  bitand_trait;
    BitOrTraitLangItem,              "bitor",                   bitor_trait;
    ShlTraitLangItem,                "shl",                     shl_trait;
    ShrTraitLangItem,                "shr",                     shr_trait;
    IndexTraitLangItem,              "index",                   index_trait;
    IndexMutTraitLangItem,           "index_mut",               index_mut_trait;
    RangeStructLangItem,             "range",                   range_struct;
    RangeFromStructLangItem,         "range_from",              range_from_struct;
    RangeToStructLangItem,           "range_to",                range_to_struct;
    RangeFullStructLangItem,         "range_full",              range_full_struct;

    UnsafeTypeLangItem,              "unsafe",                  unsafe_type;

    DerefTraitLangItem,              "deref",                   deref_trait;
    DerefMutTraitLangItem,           "deref_mut",               deref_mut_trait;

    FnTraitLangItem,                 "fn",                      fn_trait;
    FnMutTraitLangItem,              "fn_mut",                  fn_mut_trait;
    FnOnceTraitLangItem,             "fn_once",                 fn_once_trait;

    EqTraitLangItem,                 "eq",                      eq_trait;
    OrdTraitLangItem,                "ord",                     ord_trait;

    StrEqFnLangItem,                 "str_eq",                  str_eq_fn;

    // A number of panic-related lang items. The `panic` item corresponds to
    // divide-by-zero and various panic cases with `match`. The
    // `panic_bounds_check` item is for indexing arrays.
    //
    // The `begin_unwind` lang item has a predefined symbol name and is sort of
    // a "weak lang item" in the sense that a crate is not required to have it
    // defined to use it, but a final product is required to define it
    // somewhere. Additionally, there are restrictions on crates that use a weak
    // lang item, but do not have it defined.
    PanicFnLangItem,                 "panic",                   panic_fn;
    PanicBoundsCheckFnLangItem,      "panic_bounds_check",      panic_bounds_check_fn;
    PanicFmtLangItem,                "panic_fmt",               panic_fmt;

    ExchangeMallocFnLangItem,        "exchange_malloc",         exchange_malloc_fn;
    ExchangeFreeFnLangItem,          "exchange_free",           exchange_free_fn;
    StrDupUniqFnLangItem,            "strdup_uniq",             strdup_uniq_fn;

    StartFnLangItem,                 "start",                   start_fn;

    TyDescStructLangItem,            "ty_desc",                 ty_desc;
    OpaqueStructLangItem,            "opaque",                  opaque;

    EhPersonalityLangItem,           "eh_personality",          eh_personality;

    ExchangeHeapLangItem,            "exchange_heap",           exchange_heap;
    OwnedBoxLangItem,                "owned_box",               owned_box;

    CovariantTypeItem,               "covariant_type",          covariant_type;
    ContravariantTypeItem,           "contravariant_type",      contravariant_type;
    InvariantTypeItem,               "invariant_type",          invariant_type;

    CovariantLifetimeItem,           "covariant_lifetime",      covariant_lifetime;
    ContravariantLifetimeItem,       "contravariant_lifetime",  contravariant_lifetime;
    InvariantLifetimeItem,           "invariant_lifetime",      invariant_lifetime;

    NoCopyItem,                      "no_copy_bound",           no_copy_bound;
    ManagedItem,                     "managed_bound",           managed_bound;

    NonZeroItem,                     "non_zero",                non_zero;

    IteratorItem,                    "iterator",                iterator;

    StackExhaustedLangItem,          "stack_exhausted",         stack_exhausted;

    DebugTraitLangItem,              "debug_trait",             debug_trait;
}
<commit_msg>remove the `Iterator` lang item<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// Detecting language items.
//
// Language items are items that represent concepts intrinsic to the language
// itself. Examples are:
//
// * Traits that specify "kinds"; e.g. "Sync", "Send".
//
// * Traits that represent operators; e.g. "Add", "Sub", "Index".
//
// * Functions called by the compiler itself.

pub use self::LangItem::*;

use session::Session;
use metadata::csearch::each_lang_item;
use middle::ty;
use middle::weak_lang_items;
use util::nodemap::FnvHashMap;

use syntax::ast;
use syntax::ast_util::local_def;
use syntax::attr::AttrMetaMethods;
use syntax::codemap::{DUMMY_SP, Span};
use syntax::parse::token::InternedString;
use syntax::visit::Visitor;
use syntax::visit;

use std::iter::Enumerate;
use std::num::FromPrimitive;
use std::slice;

// The actual lang items defined come at the end of this file in one handy table.
// So you probably just want to nip down to the end.
macro_rules! lets_do_this {
    (
        $( $variant:ident, $name:expr, $method:ident; )*
    ) => {

#[derive(Copy, FromPrimitive, PartialEq, Eq, Hash)]
pub enum LangItem {
    $($variant),*
}

pub struct LanguageItems {
    pub items: Vec<Option<ast::DefId>>,
    pub missing: Vec<LangItem>,
}

impl LanguageItems {
    pub fn new() -> LanguageItems {
        fn foo(_: LangItem) -> Option<ast::DefId> { None }

        LanguageItems {
            items: vec!($(foo($variant)),*),
            missing: Vec::new(),
        }
    }

    pub fn items<'a>(&'a self) -> Enumerate<slice::Iter<'a, Option<ast::DefId>>> {
        self.items.iter().enumerate()
    }

    pub fn item_name(index: uint) -> &'static str {
        let item: Option<LangItem> = FromPrimitive::from_uint(index);
        match item {
            $( Some($variant) => $name, )*
            None => "???"
        }
    }

    pub fn require(&self, it: LangItem) -> Result<ast::DefId, String> {
        match self.items[it as uint] {
            Some(id) => Ok(id),
            None => {
                Err(format!("requires `{}` lang_item",
                            LanguageItems::item_name(it as uint)))
            }
        }
    }

    pub fn from_builtin_kind(&self, bound: ty::BuiltinBound)
                             -> Result<ast::DefId, String>
    {
        match bound {
            ty::BoundSend => self.require(SendTraitLangItem),
            ty::BoundSized => self.require(SizedTraitLangItem),
            ty::BoundCopy => self.require(CopyTraitLangItem),
            ty::BoundSync => self.require(SyncTraitLangItem),
        }
    }

    pub fn to_builtin_kind(&self, id: ast::DefId) -> Option<ty::BuiltinBound> {
        if Some(id) == self.send_trait() {
            Some(ty::BoundSend)
        } else if Some(id) == self.sized_trait() {
            Some(ty::BoundSized)
        } else if Some(id) == self.copy_trait() {
            Some(ty::BoundCopy)
        } else if Some(id) == self.sync_trait() {
            Some(ty::BoundSync)
        } else {
            None
        }
    }

    pub fn fn_trait_kind(&self, id: ast::DefId) -> Option<ty::ClosureKind> {
        let def_id_kinds = [
            (self.fn_trait(), ty::FnClosureKind),
            (self.fn_mut_trait(), ty::FnMutClosureKind),
            (self.fn_once_trait(), ty::FnOnceClosureKind),
            ];

        for &(opt_def_id, kind) in def_id_kinds.iter() {
            if Some(id) == opt_def_id {
                return Some(kind);
            }
        }

        None
    }

    $(
        #[allow(dead_code)]
        pub fn $method(&self) -> Option<ast::DefId> {
            self.items[$variant as uint]
        }
    )*
}

struct LanguageItemCollector<'a> {
    items: LanguageItems,

    session: &'a Session,

    item_refs: FnvHashMap<&'static str, uint>,
}

impl<'a, 'v> Visitor<'v> for LanguageItemCollector<'a> {
    fn visit_item(&mut self, item: &ast::Item) {
        match extract(item.attrs.as_slice()) {
            Some(value) => {
                let item_index = self.item_refs.get(value.get()).map(|x| *x);

                match item_index {
                    Some(item_index) => {
                        self.collect_item(item_index, local_def(item.id), item.span)
                    }
                    None => {}
                }
            }
            None => {}
        }

        visit::walk_item(self, item);
    }
}

impl<'a> LanguageItemCollector<'a> {
    pub fn new(session: &'a Session) -> LanguageItemCollector<'a> {
        let mut item_refs = FnvHashMap();

        $( item_refs.insert($name, $variant as uint); )*

        LanguageItemCollector {
            session: session,
            items: LanguageItems::new(),
            item_refs: item_refs
        }
    }

    pub fn collect_item(&mut self, item_index: uint,
                        item_def_id: ast::DefId, span: Span) {
        // Check for duplicates.
        match self.items.items[item_index] {
            Some(original_def_id) if original_def_id != item_def_id => {
                span_err!(self.session, span, E0152,
                    "duplicate entry for `{}`", LanguageItems::item_name(item_index));
            }
            Some(_) | None => {
                // OK.
            }
        }

        // Matched.
        self.items.items[item_index] = Some(item_def_id);
    }

    pub fn collect_local_language_items(&mut self, krate: &ast::Crate) {
        visit::walk_crate(self, krate);
    }

    pub fn collect_external_language_items(&mut self) {
        let crate_store = &self.session.cstore;
        crate_store.iter_crate_data(|crate_number, _crate_metadata| {
            each_lang_item(crate_store, crate_number, |node_id, item_index| {
                let def_id = ast::DefId { krate: crate_number, node: node_id };
                self.collect_item(item_index, def_id, DUMMY_SP);
                true
            });
        })
    }

    pub fn collect(&mut self, krate: &ast::Crate) {
        self.collect_local_language_items(krate);
        self.collect_external_language_items();
    }
}

pub fn extract(attrs: &[ast::Attribute]) -> Option<InternedString> {
    for attribute in attrs.iter() {
        match attribute.value_str() {
            Some(ref value) if attribute.check_name("lang") => {
                return Some(value.clone());
            }
            _ => {}
        }
    }

    return None;
}

pub fn collect_language_items(krate: &ast::Crate,
                              session: &Session) -> LanguageItems {
    let mut collector = LanguageItemCollector::new(session);
    collector.collect(krate);
    let LanguageItemCollector { mut items, .. } = collector;
    weak_lang_items::check_crate(krate, session, &mut items);
    session.abort_if_errors();
    items
}

// End of the macro
    }
}

lets_do_this! {
//  Variant name,                    Name,                      Method name;
    SendTraitLangItem,               "send",                    send_trait;
    SizedTraitLangItem,              "sized",                   sized_trait;
    CopyTraitLangItem,               "copy",                    copy_trait;
    SyncTraitLangItem,               "sync",                    sync_trait;

    DropTraitLangItem,               "drop",                    drop_trait;

    AddTraitLangItem,                "add",                     add_trait;
    SubTraitLangItem,                "sub",                     sub_trait;
    MulTraitLangItem,                "mul",                     mul_trait;
    DivTraitLangItem,                "div",                     div_trait;
    RemTraitLangItem,                "rem",                     rem_trait;
    NegTraitLangItem,                "neg",                     neg_trait;
    NotTraitLangItem,                "not",                     not_trait;
    BitXorTraitLangItem,             "bitxor",                  bitxor_trait;
    BitAndTraitLangItem,             "bitand",                  bitand_trait;
    BitOrTraitLangItem,              "bitor",                   bitor_trait;
    ShlTraitLangItem,                "shl",                     shl_trait;
    ShrTraitLangItem,                "shr",                     shr_trait;
    IndexTraitLangItem,              "index",                   index_trait;
    IndexMutTraitLangItem,           "index_mut",               index_mut_trait;
    RangeStructLangItem,             "range",                   range_struct;
    RangeFromStructLangItem,         "range_from",              range_from_struct;
    RangeToStructLangItem,           "range_to",                range_to_struct;
    RangeFullStructLangItem,         "range_full",              range_full_struct;

    UnsafeTypeLangItem,              "unsafe",                  unsafe_type;

    DerefTraitLangItem,              "deref",                   deref_trait;
    DerefMutTraitLangItem,           "deref_mut",               deref_mut_trait;

    FnTraitLangItem,                 "fn",                      fn_trait;
    FnMutTraitLangItem,              "fn_mut",                  fn_mut_trait;
    FnOnceTraitLangItem,             "fn_once",                 fn_once_trait;

    EqTraitLangItem,                 "eq",                      eq_trait;
    OrdTraitLangItem,                "ord",                     ord_trait;

    StrEqFnLangItem,                 "str_eq",                  str_eq_fn;

    // A number of panic-related lang items. The `panic` item corresponds to
    // divide-by-zero and various panic cases with `match`. The
    // `panic_bounds_check` item is for indexing arrays.
    //
    // The `begin_unwind` lang item has a predefined symbol name and is sort of
    // a "weak lang item" in the sense that a crate is not required to have it
    // defined to use it, but a final product is required to define it
    // somewhere. Additionally, there are restrictions on crates that use a weak
    // lang item, but do not have it defined.
    PanicFnLangItem,                 "panic",                   panic_fn;
    PanicBoundsCheckFnLangItem,      "panic_bounds_check",      panic_bounds_check_fn;
    PanicFmtLangItem,                "panic_fmt",               panic_fmt;

    ExchangeMallocFnLangItem,        "exchange_malloc",         exchange_malloc_fn;
    ExchangeFreeFnLangItem,          "exchange_free",           exchange_free_fn;
    StrDupUniqFnLangItem,            "strdup_uniq",             strdup_uniq_fn;

    StartFnLangItem,                 "start",                   start_fn;

    TyDescStructLangItem,            "ty_desc",                 ty_desc;
    OpaqueStructLangItem,            "opaque",                  opaque;

    EhPersonalityLangItem,           "eh_personality",          eh_personality;

    ExchangeHeapLangItem,            "exchange_heap",           exchange_heap;
    OwnedBoxLangItem,                "owned_box",               owned_box;

    CovariantTypeItem,               "covariant_type",          covariant_type;
    ContravariantTypeItem,           "contravariant_type",      contravariant_type;
    InvariantTypeItem,               "invariant_type",          invariant_type;

    CovariantLifetimeItem,           "covariant_lifetime",      covariant_lifetime;
    ContravariantLifetimeItem,       "contravariant_lifetime",  contravariant_lifetime;
    InvariantLifetimeItem,           "invariant_lifetime",      invariant_lifetime;

    NoCopyItem,                      "no_copy_bound",           no_copy_bound;
    ManagedItem,                     "managed_bound",           managed_bound;

    NonZeroItem,                     "non_zero",                non_zero;

    StackExhaustedLangItem,          "stack_exhausted",         stack_exhausted;

    DebugTraitLangItem,              "debug_trait",             debug_trait;
}
<|endoftext|>


<commit_before>//! Neural Network module
//!
//! Contains implementation of simple feed forward neural network.
//!
//! # Usage
//!
//! ```
//! use rusty_machine::learning::nnet::NeuralNet;
//! use rusty_machine::linalg::matrix::Matrix;
//! use rusty_machine::learning::SupModel;
//!
//! let data = Matrix::new(5,3, vec![1.,1.,1.,2.,2.,2.,3.,3.,3.,
//! 								4.,4.,4.,5.,5.,5.,]);
//! let outputs = Matrix::new(5,3, vec![1.,0.,0.,0.,1.,0.,0.,0.,1.,
//! 									0.,0.,1.,0.,0.,1.]);
//!
//! let layers = &[3,5,11,7,3];
//! let mut model = NeuralNet::default(layers);
//!
//! model.train(&data, &outputs);
//!
//! let test_data = Matrix::new(2,3, vec![1.5,1.5,1.5,5.1,5.1,5.1]);
//!
//! model.predict(&test_data);
//! ```

use linalg::matrix::Matrix;
use learning::SupModel;
use learning::toolkit::activ_fn;
use learning::toolkit::activ_fn::ActivationFunc;
use learning::toolkit::cost_fn;
use learning::toolkit::cost_fn::CostFunc;
use learning::optim::{Optimizable, OptimAlgorithm};
use learning::optim::fmincg::ConjugateGD;

use rand::{Rng, thread_rng};

/// Neural Network struct
pub struct NeuralNet<'a, T: Criterion> {
    layer_sizes: &'a [usize],
    weights: Vec<f64>,
    gd: ConjugateGD,
    criterion: T,
}

impl<'a> NeuralNet<'a, BCECriterion> {

    /// Creates a neural network with the specified layer sizes.
    ///
    /// Uses the default settings (gradient descent and sigmoid activation function).
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::nnet::NeuralNet;
    ///
    /// // Create a neural net with 4 layers, 3 neurons in each.
    /// let layers = &[3; 4];
    /// let mut a = NeuralNet::default(layers);
    /// ```
    pub fn default(layer_sizes: &[usize]) -> NeuralNet<BCECriterion> {
        NeuralNet {
            layer_sizes: layer_sizes,
            weights: NeuralNet::<BCECriterion>::create_weights(layer_sizes),
            gd: ConjugateGD::default(),
            criterion: BCECriterion,
        }
    }
}
impl<'a, T: Criterion> NeuralNet<'a, T> {
    /// Create a new neural network with the specified layer sizes.
    ///
    /// The layer sizes slice should include the input, hidden layers, and output layer sizes.
    /// The type of activation function must be specified.
    ///
    /// Currently defaults to simple batch Gradient Descent for optimization.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::nnet::BCECriterion;
    /// use rusty_machine::learning::nnet::NeuralNet;
    ///
    /// // Create a neural net with 4 layers, 3 neurons in each.
    /// let layers = &[3; 4];
    /// let mut a = NeuralNet::new(layers, BCECriterion);
    /// ```
    pub fn new(layer_sizes: &[usize], criterion: T) -> NeuralNet<T> {
        NeuralNet {
            layer_sizes: layer_sizes,
            weights: NeuralNet::<T>::create_weights(layer_sizes),
            gd: ConjugateGD::default(),
            criterion: criterion,
        }
    }

    /// Creates initial weights for all neurons in the network.
    fn create_weights(layer_sizes: &[usize]) -> Vec<f64> {
        let total_layers = layer_sizes.len();

        let mut capacity = 0usize;

        for l in 0..total_layers - 1 {
            capacity += (layer_sizes[l] + 1) * layer_sizes[l + 1]
        }

        let mut layers = Vec::with_capacity(capacity);

        for l in 0..total_layers - 1 {
            layers.append(&mut NeuralNet::<T>::initialize_weights(layer_sizes[l] + 1,
                                                             layer_sizes[l + 1]));
        }

        layers
    }

    /// Initializes the weights for a single layer in the network.
    fn initialize_weights(rows: usize, cols: usize) -> Vec<f64> {
        let mut weights = Vec::with_capacity(rows * cols);
        let eps_init = (6f64 / (rows + cols) as f64).sqrt();

        let mut rng = thread_rng();

        for _i in 0..rows * cols {
            let w = (rng.gen_range(0f64, 1f64) * 2f64 * eps_init) - eps_init;
            weights.push(w);
        }

        weights
    }

    /// Gets matrix of weights between specified layer and forward layer for the weights.
    fn get_layer_weights(&self, weights: &[f64], idx: usize) -> Matrix<f64> {
        assert!(idx < self.layer_sizes.len() - 1);

        // Check that the weights are the right size.
        let mut full_size = 0usize;
        for l in 0..self.layer_sizes.len() - 1 {
            full_size += (self.layer_sizes[l] + 1) * self.layer_sizes[l + 1];
        }

        assert_eq!(full_size, weights.len());

        let mut start = 0usize;

        for l in 0..idx {
            start += (self.layer_sizes[l] + 1) * self.layer_sizes[l + 1]
        }

        let capacity = (self.layer_sizes[idx] + 1) * self.layer_sizes[idx + 1];

        let mut layer_weights = Vec::with_capacity((self.layer_sizes[idx] + 1) *
                                                   self.layer_sizes[idx + 1]);
        unsafe {
            for i in start..start + capacity {
                layer_weights.push(*weights.get_unchecked(i));
            }
        }

        Matrix::new(self.layer_sizes[idx] + 1,
                    self.layer_sizes[idx + 1],
                    layer_weights)

    }

    /// Gets matrix of weights between specified layer and forward layer.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::nnet::NeuralNet;
    ///
    /// // Create a neural net with 4 layers, 3 neurons in each.
    /// let layers = &[3; 4];
    /// let mut a = NeuralNet::default(layers);
    ///
    /// let w = &a.get_net_weights(2);
    /// assert_eq!(w.rows(), 4);
    /// assert_eq!(w.cols(), 3);
    /// ```
    pub fn get_net_weights(&self, idx: usize) -> Matrix<f64> {
        self.get_layer_weights(&self.weights[..], idx)
    }

    /// Compute the gradient using the back propagation algorithm.
    fn compute_grad(&self, weights: &[f64], data: &Matrix<f64>, outputs: &Matrix<f64>) -> (f64, Vec<f64>) {
        assert_eq!(data.cols(), self.layer_sizes[0]);

        let mut forward_weights = Vec::with_capacity(self.layer_sizes.len() - 1);
        let mut activations = Vec::with_capacity(self.layer_sizes.len());

        let net_data = Matrix::ones(data.rows(), 1).hcat(data);

        activations.push(net_data.clone());

        // Forward propagation
        {
            let mut z = net_data * self.get_layer_weights(weights, 0);
            forward_weights.push(z.clone());

            for l in 1..self.layer_sizes.len() - 1 {
                let mut a = self.criterion.activate(z.clone());
                let ones = Matrix::ones(a.rows(), 1);

                a = ones.hcat(&a);
                activations.push(a.clone());
                z = a * self.get_layer_weights(weights, l);
                forward_weights.push(z.clone());
            }

            activations.push(self.criterion.activate(z));
        }

        // Compute cost using the last activation.

        let mut deltas = Vec::with_capacity(self.layer_sizes.len() - 1);
        // Backward propagation
        {
            let z = forward_weights[self.layer_sizes.len() - 2].clone();
            let g = self.criterion.grad_activ(z);

            // Take GRAD_cost to compute this delta.
            let mut delta = self.criterion.cost_grad(&activations[self.layer_sizes.len() - 1], outputs).elemul(&g);

            deltas.push(delta.clone());

            for l in (1..self.layer_sizes.len() - 1).rev() {
                let mut z = forward_weights[l - 1].clone();
                let ones = Matrix::ones(z.rows(), 1);
                z = ones.hcat(&z);

                let g = self.criterion.grad_activ(z);
                delta = (delta * self.get_layer_weights(weights, l).transpose()).elemul(&g);

                let non_one_rows = &(1..delta.cols()).collect::<Vec<usize>>()[..];
                delta = delta.select_cols(non_one_rows);
                deltas.push(delta.clone());
            }
        }

        let mut grad = Vec::with_capacity(self.layer_sizes.len() - 1);
        let mut capacity = 0;

        for l in 0..self.layer_sizes.len() - 1 {
            let g = deltas[self.layer_sizes.len() - 2 - l].transpose() * activations[l].clone();
            capacity += g.cols() * g.rows();
            grad.push(g / (data.rows() as f64));
        }

        let mut gradients = Vec::with_capacity(capacity);

        for g in grad {
            gradients.append(&mut g.data.clone());
        }

        (self.criterion.cost(&activations[activations.len() - 1], outputs), gradients)
    }

    /// Forward propagation of the model weights to get the outputs.
    fn forward_prop(&self, data: &Matrix<f64>) -> Matrix<f64> {
        assert_eq!(data.cols(), self.layer_sizes[0]);

        let net_data = Matrix::ones(data.rows(), 1).hcat(data);

        let mut z = net_data * self.get_net_weights(0);
        let mut a = self.criterion.activate(z.clone());

        for l in 1..self.layer_sizes.len() - 1 {
            let ones = Matrix::ones(a.rows(), 1);
            a = ones.hcat(&a);
            z = a * self.get_net_weights(l);
            a = self.criterion.activate(z.clone());
        }

        a
    }
}

impl<'a, T: Criterion> Optimizable for NeuralNet<'a, T> {
    type Data = Matrix<f64>;
	type Target = Matrix<f64>;

    /// Compute the gradient of the neural network.
    fn compute_grad(&self, params: &[f64], data: &Matrix<f64>, target: &Matrix<f64>) -> (f64, Vec<f64>) {
        self.compute_grad(params, data, target)
    }
}

impl<'a, T: Criterion> SupModel<Matrix<f64>, Matrix<f64>> for NeuralNet<'a, T> {
    /// Predict neural network output using forward propagation.
    fn predict(&self, data: &Matrix<f64>) -> Matrix<f64> {
        self.forward_prop(data)
    }

    /// Train the model using gradient optimization and back propagation.
    fn train(&mut self, data: &Matrix<f64>, values: &Matrix<f64>) {
        let start = self.weights.clone();
        let optimal_w = self.gd.optimize(self, &start[..], data, values);
        self.weights = optimal_w;
    }
}

pub trait Criterion {
    type ActFunc: ActivationFunc;
    type Cost: CostFunc<Matrix<f64>>;

    fn activate(&self, mat: Matrix<f64>) -> Matrix<f64> {
        mat.apply(&Self::ActFunc::func)
    }

    fn grad_activ(&self, mat: Matrix<f64>) -> Matrix<f64> {
        mat.apply(&Self::ActFunc::func_grad)
    }

    fn cost(&self, output: &Matrix<f64>, target: &Matrix<f64>) -> f64 {
        Self::Cost::cost(output, target)
    }

    fn cost_grad(&self, output: &Matrix<f64>, target: &Matrix<f64>) -> Matrix<f64> {
        Self::Cost::grad_cost(output, target)
    }
}

pub struct BCECriterion;

impl Criterion for BCECriterion {
    type ActFunc = activ_fn::Sigmoid;
    type Cost = cost_fn::CrossEntropyError;
}

pub struct MSECriterion;

impl Criterion for MSECriterion {
    type ActFunc = activ_fn::Linear;
    type Cost = cost_fn::MeanSqError;
}<commit_msg>Clean up<commit_after>//! Neural Network module
//!
//! Contains implementation of simple feed forward neural network.
//!
//! # Usage
//!
//! ```
//! use rusty_machine::learning::nnet::NeuralNet;
//! use rusty_machine::linalg::matrix::Matrix;
//! use rusty_machine::learning::SupModel;
//!
//! let data = Matrix::new(5,3, vec![1.,1.,1.,2.,2.,2.,3.,3.,3.,
//! 								4.,4.,4.,5.,5.,5.,]);
//! let outputs = Matrix::new(5,3, vec![1.,0.,0.,0.,1.,0.,0.,0.,1.,
//! 									0.,0.,1.,0.,0.,1.]);
//!
//! let layers = &[3,5,11,7,3];
//! let mut model = NeuralNet::default(layers);
//!
//! model.train(&data, &outputs);
//!
//! let test_data = Matrix::new(2,3, vec![1.5,1.5,1.5,5.1,5.1,5.1]);
//!
//! model.predict(&test_data);
//! ```

use linalg::matrix::Matrix;
use learning::SupModel;
use learning::toolkit::activ_fn;
use learning::toolkit::activ_fn::ActivationFunc;
use learning::toolkit::cost_fn;
use learning::toolkit::cost_fn::CostFunc;
use learning::optim::{Optimizable, OptimAlgorithm};
use learning::optim::fmincg::ConjugateGD;

use rand::{Rng, thread_rng};

/// Neural Network struct
pub struct NeuralNet<'a, T: Criterion> {
    layer_sizes: &'a [usize],
    weights: Vec<f64>,
    gd: ConjugateGD,
    criterion: T,
}

impl<'a> NeuralNet<'a, BCECriterion> {

    /// Creates a neural network with the specified layer sizes.
    ///
    /// Uses the default settings (gradient descent and sigmoid activation function).
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::nnet::NeuralNet;
    ///
    /// // Create a neural net with 4 layers, 3 neurons in each.
    /// let layers = &[3; 4];
    /// let mut a = NeuralNet::default(layers);
    /// ```
    pub fn default(layer_sizes: &[usize]) -> NeuralNet<BCECriterion> {
        NeuralNet {
            layer_sizes: layer_sizes,
            weights: NeuralNet::<BCECriterion>::create_weights(layer_sizes),
            gd: ConjugateGD::default(),
            criterion: BCECriterion,
        }
    }
}
impl<'a, T: Criterion> NeuralNet<'a, T> {
    /// Create a new neural network with the specified layer sizes.
    ///
    /// The layer sizes slice should include the input, hidden layers, and output layer sizes.
    /// The type of activation function must be specified.
    ///
    /// Currently defaults to simple batch Gradient Descent for optimization.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::nnet::BCECriterion;
    /// use rusty_machine::learning::nnet::NeuralNet;
    ///
    /// // Create a neural net with 4 layers, 3 neurons in each.
    /// let layers = &[3; 4];
    /// let mut a = NeuralNet::new(layers, BCECriterion);
    /// ```
    pub fn new(layer_sizes: &[usize], criterion: T) -> NeuralNet<T> {
        NeuralNet {
            layer_sizes: layer_sizes,
            weights: NeuralNet::<T>::create_weights(layer_sizes),
            gd: ConjugateGD::default(),
            criterion: criterion,
        }
    }

    /// Creates initial weights for all neurons in the network.
    fn create_weights(layer_sizes: &[usize]) -> Vec<f64> {
        let total_layers = layer_sizes.len();

        let mut capacity = 0usize;

        for l in 0..total_layers - 1 {
            capacity += (layer_sizes[l] + 1) * layer_sizes[l + 1]
        }

        let mut layers = Vec::with_capacity(capacity);

        for l in 0..total_layers - 1 {
            layers.append(&mut NeuralNet::<T>::initialize_weights(layer_sizes[l] + 1,
                                                             layer_sizes[l + 1]));
        }

        layers
    }

    /// Initializes the weights for a single layer in the network.
    fn initialize_weights(l_in: usize, l_out: usize) -> Vec<f64> {
        let mut weights = Vec::with_capacity(l_in * l_out);
        let eps_init = (6f64 / (l_in + l_out) as f64).sqrt();

        let mut rng = thread_rng();

        for _i in 0..l_in * l_out {
            let w = (rng.gen_range(0f64, 1f64) * 2f64 * eps_init) - eps_init;
            weights.push(w);
        }

        weights
    }

    /// Gets matrix of weights between specified layer and forward layer for the weights.
    fn get_layer_weights(&self, weights: &[f64], idx: usize) -> Matrix<f64> {
        assert!(idx < self.layer_sizes.len() - 1);

        // Check that the weights are the right size.
        let mut full_size = 0usize;
        for l in 0..self.layer_sizes.len() - 1 {
            full_size += (self.layer_sizes[l] + 1) * self.layer_sizes[l + 1];
        }

        assert_eq!(full_size, weights.len());

        let mut start = 0usize;

        for l in 0..idx {
            start += (self.layer_sizes[l] + 1) * self.layer_sizes[l + 1]
        }

        let capacity = (self.layer_sizes[idx] + 1) * self.layer_sizes[idx + 1];

        let mut layer_weights = Vec::with_capacity((self.layer_sizes[idx] + 1) *
                                                   self.layer_sizes[idx + 1]);
        unsafe {
            for i in start..start + capacity {
                layer_weights.push(*weights.get_unchecked(i));
            }
        }

        Matrix::new(self.layer_sizes[idx] + 1,
                    self.layer_sizes[idx + 1],
                    layer_weights)

    }

    /// Gets matrix of weights between specified layer and forward layer.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::nnet::NeuralNet;
    ///
    /// // Create a neural net with 4 layers, 3 neurons in each.
    /// let layers = &[3; 4];
    /// let mut a = NeuralNet::default(layers);
    ///
    /// let w = &a.get_net_weights(2);
    /// assert_eq!(w.rows(), 4);
    /// assert_eq!(w.cols(), 3);
    /// ```
    pub fn get_net_weights(&self, idx: usize) -> Matrix<f64> {
        self.get_layer_weights(&self.weights[..], idx)
    }

    /// Compute the gradient using the back propagation algorithm.
    fn compute_grad(&self, weights: &[f64], data: &Matrix<f64>, outputs: &Matrix<f64>) -> (f64, Vec<f64>) {
        assert_eq!(data.cols(), self.layer_sizes[0]);

        let mut forward_weights = Vec::with_capacity(self.layer_sizes.len() - 1);
        let mut activations = Vec::with_capacity(self.layer_sizes.len());

        let net_data = Matrix::ones(data.rows(), 1).hcat(data);

        activations.push(net_data.clone());

        // Forward propagation
        {
            let mut z = net_data * self.get_layer_weights(weights, 0);
            forward_weights.push(z.clone());

            for l in 1..self.layer_sizes.len() - 1 {
                let mut a = self.criterion.activate(z.clone());
                let ones = Matrix::ones(a.rows(), 1);

                a = ones.hcat(&a);
                activations.push(a.clone());
                z = a * self.get_layer_weights(weights, l);
                forward_weights.push(z.clone());
            }

            activations.push(self.criterion.activate(z));
        }

        let mut deltas = Vec::with_capacity(self.layer_sizes.len() - 1);
        // Backward propagation
        {
            let z = forward_weights[self.layer_sizes.len() - 2].clone();
            let g = self.criterion.grad_activ(z);

            // Take GRAD_cost to compute this delta.
            let mut delta = self.criterion.cost_grad(&activations[self.layer_sizes.len() - 1], outputs).elemul(&g);

            deltas.push(delta.clone());

            for l in (1..self.layer_sizes.len() - 1).rev() {
                let mut z = forward_weights[l - 1].clone();
                let ones = Matrix::ones(z.rows(), 1);
                z = ones.hcat(&z);

                let g = self.criterion.grad_activ(z);
                delta = (delta * self.get_layer_weights(weights, l).transpose()).elemul(&g);

                let non_one_rows = &(1..delta.cols()).collect::<Vec<usize>>()[..];
                delta = delta.select_cols(non_one_rows);
                deltas.push(delta.clone());
            }
        }

        let mut grad = Vec::with_capacity(self.layer_sizes.len() - 1);
        let mut capacity = 0;

        for l in 0..self.layer_sizes.len() - 1 {
            let g = deltas[self.layer_sizes.len() - 2 - l].transpose() * activations[l].clone();
            capacity += g.cols() * g.rows();
            grad.push(g / (data.rows() as f64));
        }

        let mut gradients = Vec::with_capacity(capacity);

        for g in grad {
            gradients.append(&mut g.data.clone());
        }

        (self.criterion.cost(&activations[activations.len() - 1], outputs), gradients)
    }

    /// Forward propagation of the model weights to get the outputs.
    fn forward_prop(&self, data: &Matrix<f64>) -> Matrix<f64> {
        assert_eq!(data.cols(), self.layer_sizes[0]);

        let net_data = Matrix::ones(data.rows(), 1).hcat(data);

        let mut z = net_data * self.get_net_weights(0);
        let mut a = self.criterion.activate(z.clone());

        for l in 1..self.layer_sizes.len() - 1 {
            let ones = Matrix::ones(a.rows(), 1);
            a = ones.hcat(&a);
            z = a * self.get_net_weights(l);
            a = self.criterion.activate(z.clone());
        }

        a
    }
}

impl<'a, T: Criterion> Optimizable for NeuralNet<'a, T> {
    type Data = Matrix<f64>;
	type Target = Matrix<f64>;

    /// Compute the gradient of the neural network.
    fn compute_grad(&self, params: &[f64], data: &Matrix<f64>, target: &Matrix<f64>) -> (f64, Vec<f64>) {
        self.compute_grad(params, data, target)
    }
}

impl<'a, T: Criterion> SupModel<Matrix<f64>, Matrix<f64>> for NeuralNet<'a, T> {
    /// Predict neural network output using forward propagation.
    fn predict(&self, data: &Matrix<f64>) -> Matrix<f64> {
        self.forward_prop(data)
    }

    /// Train the model using gradient optimization and back propagation.
    fn train(&mut self, data: &Matrix<f64>, values: &Matrix<f64>) {
        let start = self.weights.clone();
        let optimal_w = self.gd.optimize(self, &start[..], data, values);
        self.weights = optimal_w;
    }
}

pub trait Criterion {
    type ActFunc: ActivationFunc;
    type Cost: CostFunc<Matrix<f64>>;

    fn activate(&self, mat: Matrix<f64>) -> Matrix<f64> {
        mat.apply(&Self::ActFunc::func)
    }

    fn grad_activ(&self, mat: Matrix<f64>) -> Matrix<f64> {
        mat.apply(&Self::ActFunc::func_grad)
    }

    fn cost(&self, output: &Matrix<f64>, target: &Matrix<f64>) -> f64 {
        Self::Cost::cost(output, target)
    }

    fn cost_grad(&self, output: &Matrix<f64>, target: &Matrix<f64>) -> Matrix<f64> {
        Self::Cost::grad_cost(output, target)
    }
}

pub struct BCECriterion;

impl Criterion for BCECriterion {
    type ActFunc = activ_fn::Sigmoid;
    type Cost = cost_fn::CrossEntropyError;
}

pub struct MSECriterion;

impl Criterion for MSECriterion {
    type ActFunc = activ_fn::Linear;
    type Cost = cost_fn::MeanSqError;
}<|endoftext|>


<commit_before>extern crate sdl2;
extern crate sdl2_ttf;

use std::env;
use std::path::Path;

use sdl2::event::Event;
use sdl2::keyboard::Keycode;
use sdl2::rect::Rect;
use sdl2::render::TextureQuery;
use sdl2::pixels::Color;

static SCREEN_WIDTH : u32 = 800;
static SCREEN_HEIGHT : u32 = 600;

// handle the annoying Rect i32
macro_rules! rect(
    ($x:expr, $y:expr, $w:expr, $h:expr) => (
        Rect::new_unwrap($x as i32, $y as i32, $w as u32, $h as u32)
    )
);

// Scale fonts to a reasonable size when they're too big (though they might look less smooth)
fn get_centered_rect(rect_width: u32, rect_height: u32, cons_width: u32, cons_height: u32) -> Rect {
    let wr = rect_width as f32 / cons_width as f32;
    let hr = rect_height as f32 / cons_height as f32;

    let (w, h) = if wr > 1f32 || hr > 1f32 {
        if wr > hr {
            println!("Scaling down! The text will look worse!");
            let h = (rect_height as f32 / wr) as i32;
            (cons_width as i32, h)
        } else {
            println!("Scaling down! The text will look worse!");
            let w = (rect_width as f32 / hr) as i32;
            (w, cons_height as i32)
        }
    } else {
        (rect_width as i32, rect_height as i32)
    };

    let cx = (SCREEN_WIDTH as i32 - w) / 2;
    let cy = (SCREEN_HEIGHT as i32 - h) / 2;
    rect!(cx, cy, w, h)
}

fn run(font_path: &Path) {
    let sdl_context = sdl2::init().unwrap();
    let video_subsys = sdl_context.video().unwrap();
    let ttf_context = sdl2_ttf::init().unwrap();

    let window = video_subsys.window("SDL2_TTF Example", SCREEN_WIDTH, SCREEN_HEIGHT)
        .position_centered()
        .opengl()
        .build()
        .unwrap();

    let mut renderer = window.renderer().build().unwrap();

    // Load a font
    let font = ttf_context.load_font(font_path, 128).unwrap();

    // render a surface, and convert it to a texture bound to the renderer
    let surface = font.render("Hello Rust!")
        .blended(Color::RGBA(255, 0, 0, 255)).unwrap();
    let mut texture = renderer.create_texture_from_surface(&surface).unwrap();

    renderer.set_draw_color(Color::RGBA(195, 217, 255, 255));
    renderer.clear();

    let TextureQuery { width, height, .. } = texture.query();

    // If the example text is too big for the screen, downscale it (and center irregardless)
    let padding = 64;
    let target = get_centered_rect(width, height, SCREEN_WIDTH - padding, SCREEN_HEIGHT - padding);

    renderer.copy(&mut texture, None, Some(target));
    renderer.present();

    'mainloop: loop {
        for event in sdl_context.event_pump().unwrap().poll_iter() {
            match event {
                Event::Quit{..} => break 'mainloop,
                Event::KeyDown {keycode: Some(Keycode::Escape), ..} => break 'mainloop,
                _ => {}
            }
        }
    }
}

fn main() {
    let args: Vec<_> = env::args().collect();

    println!("linked sdl2_ttf: {}", sdl2_ttf::get_linked_version());

    if args.len() < 2 {
        println!("Usage: ./demo font.[ttf|ttc|fon]")
    } else {
        let path: &Path = Path::new(&args[1]);
        run(path);
    }
}
<commit_msg>Fix demo example<commit_after>extern crate sdl2;
extern crate sdl2_ttf;

use std::env;
use std::path::Path;

use sdl2::event::Event;
use sdl2::keyboard::Keycode;
use sdl2::rect::Rect;
use sdl2::render::TextureQuery;
use sdl2::pixels::Color;

static SCREEN_WIDTH : u32 = 800;
static SCREEN_HEIGHT : u32 = 600;

// handle the annoying Rect i32
macro_rules! rect(
    ($x:expr, $y:expr, $w:expr, $h:expr) => (
        Rect::new($x as i32, $y as i32, $w as u32, $h as u32)
    )
);

// Scale fonts to a reasonable size when they're too big (though they might look less smooth)
fn get_centered_rect(rect_width: u32, rect_height: u32, cons_width: u32, cons_height: u32) -> Rect {
    let wr = rect_width as f32 / cons_width as f32;
    let hr = rect_height as f32 / cons_height as f32;

    let (w, h) = if wr > 1f32 || hr > 1f32 {
        if wr > hr {
            println!("Scaling down! The text will look worse!");
            let h = (rect_height as f32 / wr) as i32;
            (cons_width as i32, h)
        } else {
            println!("Scaling down! The text will look worse!");
            let w = (rect_width as f32 / hr) as i32;
            (w, cons_height as i32)
        }
    } else {
        (rect_width as i32, rect_height as i32)
    };

    let cx = (SCREEN_WIDTH as i32 - w) / 2;
    let cy = (SCREEN_HEIGHT as i32 - h) / 2;
    rect!(cx, cy, w, h)
}

fn run(font_path: &Path) {
    let sdl_context = sdl2::init().unwrap();
    let video_subsys = sdl_context.video().unwrap();
    let ttf_context = sdl2_ttf::init().unwrap();

    let window = video_subsys.window("SDL2_TTF Example", SCREEN_WIDTH, SCREEN_HEIGHT)
        .position_centered()
        .opengl()
        .build()
        .unwrap();

    let mut renderer = window.renderer().build().unwrap();

    // Load a font
    let font = ttf_context.load_font(font_path, 128).unwrap();

    // render a surface, and convert it to a texture bound to the renderer
    let surface = font.render("Hello Rust!")
        .blended(Color::RGBA(255, 0, 0, 255)).unwrap();
    let mut texture = renderer.create_texture_from_surface(&surface).unwrap();

    renderer.set_draw_color(Color::RGBA(195, 217, 255, 255));
    renderer.clear();

    let TextureQuery { width, height, .. } = texture.query();

    // If the example text is too big for the screen, downscale it (and center irregardless)
    let padding = 64;
    let target = get_centered_rect(width, height, SCREEN_WIDTH - padding, SCREEN_HEIGHT - padding);

    renderer.copy(&mut texture, None, Some(target));
    renderer.present();

    'mainloop: loop {
        for event in sdl_context.event_pump().unwrap().poll_iter() {
            match event {
                Event::Quit{..} => break 'mainloop,
                Event::KeyDown {keycode: Some(Keycode::Escape), ..} => break 'mainloop,
                _ => {}
            }
        }
    }
}

fn main() {
    let args: Vec<_> = env::args().collect();

    println!("linked sdl2_ttf: {}", sdl2_ttf::get_linked_version());

    if args.len() < 2 {
        println!("Usage: ./demo font.[ttf|ttc|fon]")
    } else {
        let path: &Path = Path::new(&args[1]);
        run(path);
    }
}
<|endoftext|>


<commit_before>extern crate mp4parse;

use std::env;
use std::fs::File;
use std::io::{Read, Seek, Take};
use std::io::Cursor;
use std::thread;

fn limit<'a, T: Read>(f: &'a mut T, h: &mp4parse::BoxHeader) -> Take<&'a mut T> {
    f.take(h.size - h.offset)
}

fn read_box<T: Read + Seek>(f: &mut T) {
    mp4parse::read_box_header(f).and_then(|h| {
        match &(mp4parse::fourcc_to_string(h.name))[..] {
            "ftyp" => {
                let mut content = limit(f, &h);
                let ftyp = mp4parse::read_ftyp(&mut content, &h).unwrap();
                println!("{}", ftyp);
            },
            "moov" => {
                println!("{} -- recursing", h);
                let buf: Vec<u8> = limit(f, &h)
                    .bytes()
                    .map(|u| u.unwrap())
                    .collect();
                let mut content = Cursor::new(buf);
                read_box(&mut content);
                println!("{} -- end", h);
            },
            "mvhd" => {
                let mut content = limit(f, &h);
                let mvhd = mp4parse::read_mvhd(&mut content, &h).unwrap();
                println!("  {}", mvhd);
            },
            _ => {
                println!("{}", h);
                mp4parse::skip_box_content(f, &h);
            },
        };
        Some(()) // and_then needs a Option.
    });
}

fn dump_file(filename: String) {
    let mut f = File::open(filename).unwrap();
    let task = thread::spawn(move || {
        loop {
            read_box(&mut f);
        }
    });
    // Catch and ignore any panics in the thread.
    task.join().ok();
}

fn main() {
    for filename in env::args().skip(1) {
        println!("-- dump of '{}' --", filename);
        dump_file(filename);
    }
}
<commit_msg>Unwrap the return value of skip_box_content().<commit_after>extern crate mp4parse;

use std::env;
use std::fs::File;
use std::io::{Read, Seek, Take};
use std::io::Cursor;
use std::thread;

fn limit<'a, T: Read>(f: &'a mut T, h: &mp4parse::BoxHeader) -> Take<&'a mut T> {
    f.take(h.size - h.offset)
}

fn read_box<T: Read + Seek>(f: &mut T) {
    mp4parse::read_box_header(f).and_then(|h| {
        match &(mp4parse::fourcc_to_string(h.name))[..] {
            "ftyp" => {
                let mut content = limit(f, &h);
                let ftyp = mp4parse::read_ftyp(&mut content, &h).unwrap();
                println!("{}", ftyp);
            },
            "moov" => {
                println!("{} -- recursing", h);
                let buf: Vec<u8> = limit(f, &h)
                    .bytes()
                    .map(|u| u.unwrap())
                    .collect();
                let mut content = Cursor::new(buf);
                read_box(&mut content);
                println!("{} -- end", h);
            },
            "mvhd" => {
                let mut content = limit(f, &h);
                let mvhd = mp4parse::read_mvhd(&mut content, &h).unwrap();
                println!("  {}", mvhd);
            },
            _ => {
                // Skip the contents of unknown chunks.
                println!("{}", h);
                mp4parse::skip_box_content(f, &h).unwrap();
            },
        };
        Some(()) // and_then needs a Option.
    });
}

fn dump_file(filename: String) {
    let mut f = File::open(filename).unwrap();
    let task = thread::spawn(move || {
        loop {
            read_box(&mut f);
        }
    });
    // Catch and ignore any panics in the thread.
    task.join().ok();
}

fn main() {
    for filename in env::args().skip(1) {
        println!("-- dump of '{}' --", filename);
        dump_file(filename);
    }
}
<|endoftext|>


<commit_before>
//! Macros for used by tests to construct transactions and blocks

//! tx_builder must be invoked before tx and blk macros can be used
//! Usage:

//!
//!
//! ```
//! #[macro_use]
//! extern crate bitcrust_lib;
//! use hash;
//!
//! fn main() {
//!
//!     tx_builder!(bld); // needed local "cache" to construct transactions
//!
//!     //let tx1 = coinbase!(bld;   => a);
//!     //let tx2 = tx!      (bld; a => b, c );
//! }
//!
//! ```
//!
//!

/// This macro setups a tx-builder variable
/// It initializes a hashmap that will be used to map output variables
#[macro_export]
macro_rules! tx_builder { ($b:ident ) =>
    (let mut $b: ::std::collections::HashMap<&str, Vec<u8>>    = ::std::collections::HashMap::new(); ) }

#[macro_export]
macro_rules! coinbase {
    ($bld:ident ; => $output:ident $( ; $amount:expr )* )
    =>
    (
    {
        const default_amount: u8 = 50_u8;
        let amount: u8 = * vec![$( $amount)*].get(0).unwrap_or(& default_amount);

        // first output is block reward
        let outp1 = vec![
            amount, 0u8,0u8,0u8,0u8,0u8,0u8,0u8,  /* the amount as little endian i64 */
            1, 81 /* a one byte script consisting of OP_TRUE */
        ];

        // we count the number of items in bld, and at that as OP_RETURN to
        // output 2 to make this unique
        // TODO support blockheight in this output
        let bytes_bld_len: [u8;8] = unsafe { ::std::mem::transmute::<_,[u8;8]>( $bld.len()) };
        let mut outp2 = vec![
            0u8, 0u8,0u8,0u8,0u8,0u8,0u8,0u8,  /* the amount as little endian i64 */
            10, 106, 8 /* 10-bytes script, OP_RETURN, OP_PUSHDATA(8) */

        ];

        outp2.extend(bytes_bld_len.iter());

        let mut tx = vec![
            0x01_u8, 0_u8, 0_u8, 0_u8, /* tx-version */
            0x00, /* 0-inputs */
            0x02 /* 1-output */

        ];

        tx.extend(outp1.iter());
        tx.extend(outp2.iter());
        tx.extend([0u8;4].iter()); // locktime=0

        // now we have the tx we can create the input that references this
        let hash = ::hash::Hash32Buf::double_sha256(&tx);

        let mut txin: Vec<u8> = Vec::new();
        txin.extend(hash.as_ref().0.iter());
        txin.extend([0u8;4].iter()); // index=0
        txin.extend([0u8;1].iter()); // script is empty
        txin.extend([0u8;4].iter()); // sequence = 0

        $bld.insert(stringify!($output), txin);

        tx
    }
    )
}

/// Creates a transaction
#[macro_export]
macro_rules! tx {

    ( $bld:ident;  $($input:ident),* => $($output:ident $( ; $amount:expr )* ),+)
    =>
    ( {
        let b1 = $bld.clone();

        let txs_in  = vec![ $(
            b1.get(stringify!($input)).unwrap(),
        )* ];
        let txs_out = vec![ $(
            {
                const default_amount: u8 = 50_u8;
                let amount: u8 = * vec![$( $amount)*].get(0).unwrap_or(& default_amount);

                let outp = vec![
                    amount, 0u8,0u8,0u8,0u8,0u8,0u8,0u8,  /* the amount as little endian i64 */
                    1, 81 /* a one byte script consisting of OP_TRUE */
                ];

                outp
            }
            ,
        )* ];

        let mut tx = vec![
            0x01_u8, 0_u8, 0_u8, 0_u8, /* tx-version */
            txs_in.len() as u8 /* inputs */
        ];

        for ref tx_in in txs_in {
            tx.extend(tx_in.iter());
        }
        tx.push(txs_out.len() as u8);
        for ref tx_out in txs_out {
            tx.extend(tx_out.iter());
        }

        tx.extend([0u8;4].iter()); // locktime=0

        // now we have the tx we can create the input that references this
        let hash = ::hash::Hash32Buf::double_sha256(&tx);
        let mut idx = 0_u8;
        $(
            {

                let mut txin: Vec<u8> = Vec::new();
                txin.extend(hash.as_ref().0.iter());
                txin.extend([idx, 0u8, 0u8, 0u8].iter()); // index
                txin.extend([0u8;1].iter()); // script is empty
                txin.extend([0u8;4].iter()); // sequence = 0
                let x = stringify!($output);
                $bld.insert(stringify!($output), txin);
            }
            idx += 1;
        )*

        tx
    } );

}

#[cfg(test)]
mod tests {

    use buffer::*;
    use transaction::Transaction;

    #[test]
    fn test_tx_builders() {

        tx_builder!(bld);

        let tx1 = coinbase!(bld; => a);
        let tx2 = tx!(bld; a     => b, c );
        let tx3 = tx!(bld; a,b   => c );

        println!("{:?}", tx2);

        let tx1p = Transaction::parse(&mut Buffer::new(&tx1)).unwrap();
        let tx2p = Transaction::parse(&mut Buffer::new(&tx2)).unwrap();
        let tx3p = Transaction::parse(&mut Buffer::new(&tx3)).unwrap();

        assert_eq!(tx1p.txs_out.len(), 2);
        assert_eq!(tx1p.txs_in.len(), 0);

        assert_eq!(tx2p.txs_in.len(), 1);
        assert_eq!(tx2p.txs_out.len(), 2);
        assert_eq!(tx3p.txs_in.len(), 2);
        assert_eq!(tx3p.txs_out.len(), 1);

    }

    #[test]
    fn test_blk_builders() {

    }<commit_msg>Fix build<commit_after>
//! Macros for used by tests to construct transactions and blocks

//! tx_builder must be invoked before tx and blk macros can be used
//! Usage:

//!
//!
//! ```
//! #[macro_use]
//! extern crate bitcrust_lib;
//!
//! fn main() {
//!
//!     tx_builder!(bld); // needed local "cache" to construct transactions
//!
//!     //let tx1 = coinbase!(bld;   => a);
//!     //let tx2 = tx!      (bld; a => b, c );
//! }
//!
//! ```
//!
//!

/// This macro setups a tx-builder variable
/// It initializes a hashmap that will be used to map output variables
#[macro_export]
macro_rules! tx_builder { ($b:ident ) =>
    (let mut $b: ::std::collections::HashMap<&str, Vec<u8>>    = ::std::collections::HashMap::new(); ) }

#[macro_export]
macro_rules! coinbase {
    ($bld:ident ; => $output:ident $( ; $amount:expr )* )
    =>
    (
    {
        const default_amount: u8 = 50_u8;
        let amount: u8 = * vec![$( $amount)*].get(0).unwrap_or(& default_amount);

        // first output is block reward
        let outp1 = vec![
            amount, 0u8,0u8,0u8,0u8,0u8,0u8,0u8,  /* the amount as little endian i64 */
            1, 81 /* a one byte script consisting of OP_TRUE */
        ];

        // we count the number of items in bld, and at that as OP_RETURN to
        // output 2 to make this unique
        // TODO support blockheight in this output
        let bytes_bld_len: [u8;8] = unsafe { ::std::mem::transmute::<_,[u8;8]>( $bld.len()) };
        let mut outp2 = vec![
            0u8, 0u8,0u8,0u8,0u8,0u8,0u8,0u8,  /* the amount as little endian i64 */
            10, 106, 8 /* 10-bytes script, OP_RETURN, OP_PUSHDATA(8) */

        ];

        outp2.extend(bytes_bld_len.iter());

        let mut tx = vec![
            0x01_u8, 0_u8, 0_u8, 0_u8, /* tx-version */
            0x00, /* 0-inputs */
            0x02 /* 1-output */

        ];

        tx.extend(outp1.iter());
        tx.extend(outp2.iter());
        tx.extend([0u8;4].iter()); // locktime=0

        // now we have the tx we can create the input that references this
        let hash = ::hash::Hash32Buf::double_sha256(&tx);

        let mut txin: Vec<u8> = Vec::new();
        txin.extend(hash.as_ref().0.iter());
        txin.extend([0u8;4].iter()); // index=0
        txin.extend([0u8;1].iter()); // script is empty
        txin.extend([0u8;4].iter()); // sequence = 0

        $bld.insert(stringify!($output), txin);

        tx
    }
    )
}

/// Creates a transaction
#[macro_export]
macro_rules! tx {

    ( $bld:ident;  $($input:ident),* => $($output:ident $( ; $amount:expr )* ),+)
    =>
    ( {
        let b1 = $bld.clone();

        let txs_in  = vec![ $(
            b1.get(stringify!($input)).unwrap(),
        )* ];
        let txs_out = vec![ $(
            {
                const default_amount: u8 = 50_u8;
                let amount: u8 = * vec![$( $amount)*].get(0).unwrap_or(& default_amount);

                let outp = vec![
                    amount, 0u8,0u8,0u8,0u8,0u8,0u8,0u8,  /* the amount as little endian i64 */
                    1, 81 /* a one byte script consisting of OP_TRUE */
                ];

                outp
            }
            ,
        )* ];

        let mut tx = vec![
            0x01_u8, 0_u8, 0_u8, 0_u8, /* tx-version */
            txs_in.len() as u8 /* inputs */
        ];

        for ref tx_in in txs_in {
            tx.extend(tx_in.iter());
        }
        tx.push(txs_out.len() as u8);
        for ref tx_out in txs_out {
            tx.extend(tx_out.iter());
        }

        tx.extend([0u8;4].iter()); // locktime=0

        // now we have the tx we can create the input that references this
        let hash = ::hash::Hash32Buf::double_sha256(&tx);
        let mut idx = 0_u8;
        $(
            {

                let mut txin: Vec<u8> = Vec::new();
                txin.extend(hash.as_ref().0.iter());
                txin.extend([idx, 0u8, 0u8, 0u8].iter()); // index
                txin.extend([0u8;1].iter()); // script is empty
                txin.extend([0u8;4].iter()); // sequence = 0
                let x = stringify!($output);
                $bld.insert(stringify!($output), txin);
            }
            idx += 1;
        )*

        tx
    } );

}

#[macro_export]
macro_rules! blk {
    ( $($txvec:ident),* )
=>
    (
    {

    }

    )
}

#[cfg(test)]
mod tests {

    use buffer::*;
    use transaction::Transaction;

    #[test]
    fn test_tx_builders() {

        tx_builder!(bld);

        let tx1 = coinbase!(bld; => a);
        let tx2 = tx!(bld; a     => b, c );
        let tx3 = tx!(bld; a,b   => c );

        println!("{:?}", tx2);

        let tx1p = Transaction::parse(&mut Buffer::new(&tx1)).unwrap();
        let tx2p = Transaction::parse(&mut Buffer::new(&tx2)).unwrap();
        let tx3p = Transaction::parse(&mut Buffer::new(&tx3)).unwrap();

        assert_eq!(tx1p.txs_out.len(), 2);
        assert_eq!(tx1p.txs_in.len(), 0);

        assert_eq!(tx2p.txs_in.len(), 1);
        assert_eq!(tx2p.txs_out.len(), 2);
        assert_eq!(tx3p.txs_in.len(), 2);
        assert_eq!(tx3p.txs_out.len(), 1);

    }

    #[test]
    fn test_blk_builders() {

    }
}<|endoftext|>


<commit_before>use std::{
    sync::Arc,
    time::Instant,
};

use rustc_hash::FxHashMap;
use ra_syntax::{
    AstNode,  SyntaxNode, SmolStr,
    ast::{self, FnDef, FnDefNode, NameOwner, ModuleItemOwner}
};
use ra_db::{SourceRootId, FileId, Cancelable,};

use crate::{
        FnId,
        SourceFileItems, SourceItemId,
        db::HirDatabase,
        function::FnScopes,
        module::{
            ModuleSource, ModuleSourceNode, ModuleId,
            imp::Submodule,
            nameres::{InputModuleItems, ItemMap, Resolver},
        },
};

/// Resolve `FnId` to the corresponding `SyntaxNode`
pub(super) fn fn_syntax(db: &impl HirDatabase, fn_id: FnId) -> FnDefNode {
    let item_id = fn_id.loc(db);
    let syntax = db.file_item(item_id);
    FnDef::cast(syntax.borrowed()).unwrap().owned()
}

pub(super) fn fn_scopes(db: &impl HirDatabase, fn_id: FnId) -> Arc<FnScopes> {
    let syntax = db.fn_syntax(fn_id);
    let res = FnScopes::new(syntax.borrowed());
    Arc::new(res)
}

pub(super) fn file_items(db: &impl HirDatabase, file_id: FileId) -> Arc<SourceFileItems> {
    let source_file = db.source_file(file_id);
    let source_file = source_file.borrowed();
    let mut res = SourceFileItems::default();
    source_file
        .syntax()
        .descendants()
        .filter_map(ast::ModuleItem::cast)
        .map(|it| it.syntax().owned())
        .for_each(|it| {
            res.alloc(it);
        });
    Arc::new(res)
}

pub(super) fn file_item(db: &impl HirDatabase, source_item_id: SourceItemId) -> SyntaxNode {
    db.file_items(source_item_id.file_id)[source_item_id.item_id].clone()
}

pub(crate) fn submodules(
    db: &impl HirDatabase,
    source: ModuleSource,
) -> Cancelable<Arc<Vec<Submodule>>> {
    db.check_canceled()?;
    let file_id = source.file_id();
    let submodules = match source.resolve(db) {
        ModuleSourceNode::SourceFile(it) => collect_submodules(db, file_id, it.borrowed()),
        ModuleSourceNode::Module(it) => it
            .borrowed()
            .item_list()
            .map(|it| collect_submodules(db, file_id, it))
            .unwrap_or_else(Vec::new),
    };
    return Ok(Arc::new(submodules));

    fn collect_submodules<'a>(
        db: &impl HirDatabase,
        file_id: FileId,
        root: impl ast::ModuleItemOwner<'a>,
    ) -> Vec<Submodule> {
        modules(root)
            .map(|(name, m)| {
                if m.has_semi() {
                    Submodule::Declaration(name)
                } else {
                    let src = ModuleSource::new_inline(db, file_id, m);
                    Submodule::Definition(name, src)
                }
            })
            .collect()
    }
}

pub(crate) fn modules<'a>(
    root: impl ast::ModuleItemOwner<'a>,
) -> impl Iterator<Item = (SmolStr, ast::Module<'a>)> {
    root.items()
        .filter_map(|item| match item {
            ast::ModuleItem::Module(m) => Some(m),
            _ => None,
        })
        .filter_map(|module| {
            let name = module.name()?.text();
            Some((name, module))
        })
}

pub(super) fn input_module_items(
    db: &impl HirDatabase,
    source_root: SourceRootId,
    module_id: ModuleId,
) -> Cancelable<Arc<InputModuleItems>> {
    let module_tree = db.module_tree(source_root)?;
    let source = module_id.source(&module_tree);
    let file_items = db.file_items(source.file_id());
    let res = match source.resolve(db) {
        ModuleSourceNode::SourceFile(it) => {
            let items = it.borrowed().items();
            InputModuleItems::new(&file_items, items)
        }
        ModuleSourceNode::Module(it) => {
            let items = it
                .borrowed()
                .item_list()
                .into_iter()
                .flat_map(|it| it.items());
            InputModuleItems::new(&file_items, items)
        }
    };
    Ok(Arc::new(res))
}

pub(super) fn item_map(
    db: &impl HirDatabase,
    source_root: SourceRootId,
) -> Cancelable<Arc<ItemMap>> {
    let start = Instant::now();
    let module_tree = db.module_tree(source_root)?;
    let input = module_tree
        .modules()
        .map(|id| {
            let items = db.input_module_items(source_root, id)?;
            Ok((id, items))
        })
        .collect::<Cancelable<FxHashMap<_, _>>>()?;
    let resolver = Resolver {
        db: db,
        input: &input,
        source_root,
        module_tree,
        result: ItemMap::default(),
    };
    let res = resolver.resolve()?;
    let elapsed = start.elapsed();
    log::info!("item_map: {:?}", elapsed);
    Ok(Arc::new(res))
}
<commit_msg>include file itself in SourceFileItems<commit_after>use std::{
    sync::Arc,
    time::Instant,
};

use rustc_hash::FxHashMap;
use ra_syntax::{
    AstNode,  SyntaxNode, SmolStr,
    ast::{self, FnDef, FnDefNode, NameOwner, ModuleItemOwner}
};
use ra_db::{SourceRootId, FileId, Cancelable,};

use crate::{
        FnId,
        SourceFileItems, SourceItemId,
        db::HirDatabase,
        function::FnScopes,
        module::{
            ModuleSource, ModuleSourceNode, ModuleId,
            imp::Submodule,
            nameres::{InputModuleItems, ItemMap, Resolver},
        },
};

/// Resolve `FnId` to the corresponding `SyntaxNode`
pub(super) fn fn_syntax(db: &impl HirDatabase, fn_id: FnId) -> FnDefNode {
    let item_id = fn_id.loc(db);
    let syntax = db.file_item(item_id);
    FnDef::cast(syntax.borrowed()).unwrap().owned()
}

pub(super) fn fn_scopes(db: &impl HirDatabase, fn_id: FnId) -> Arc<FnScopes> {
    let syntax = db.fn_syntax(fn_id);
    let res = FnScopes::new(syntax.borrowed());
    Arc::new(res)
}

pub(super) fn file_items(db: &impl HirDatabase, file_id: FileId) -> Arc<SourceFileItems> {
    let mut res = SourceFileItems::default();
    let source_file = db.source_file(file_id);
    res.alloc(source_file.syntax().owned());
    let source_file = source_file.borrowed();
    source_file
        .syntax()
        .descendants()
        .filter_map(ast::ModuleItem::cast)
        .map(|it| it.syntax().owned())
        .for_each(|it| {
            res.alloc(it);
        });
    Arc::new(res)
}

pub(super) fn file_item(db: &impl HirDatabase, source_item_id: SourceItemId) -> SyntaxNode {
    db.file_items(source_item_id.file_id)[source_item_id.item_id].clone()
}

pub(crate) fn submodules(
    db: &impl HirDatabase,
    source: ModuleSource,
) -> Cancelable<Arc<Vec<Submodule>>> {
    db.check_canceled()?;
    let file_id = source.file_id();
    let submodules = match source.resolve(db) {
        ModuleSourceNode::SourceFile(it) => collect_submodules(db, file_id, it.borrowed()),
        ModuleSourceNode::Module(it) => it
            .borrowed()
            .item_list()
            .map(|it| collect_submodules(db, file_id, it))
            .unwrap_or_else(Vec::new),
    };
    return Ok(Arc::new(submodules));

    fn collect_submodules<'a>(
        db: &impl HirDatabase,
        file_id: FileId,
        root: impl ast::ModuleItemOwner<'a>,
    ) -> Vec<Submodule> {
        modules(root)
            .map(|(name, m)| {
                if m.has_semi() {
                    Submodule::Declaration(name)
                } else {
                    let src = ModuleSource::new_inline(db, file_id, m);
                    Submodule::Definition(name, src)
                }
            })
            .collect()
    }
}

pub(crate) fn modules<'a>(
    root: impl ast::ModuleItemOwner<'a>,
) -> impl Iterator<Item = (SmolStr, ast::Module<'a>)> {
    root.items()
        .filter_map(|item| match item {
            ast::ModuleItem::Module(m) => Some(m),
            _ => None,
        })
        .filter_map(|module| {
            let name = module.name()?.text();
            Some((name, module))
        })
}

pub(super) fn input_module_items(
    db: &impl HirDatabase,
    source_root: SourceRootId,
    module_id: ModuleId,
) -> Cancelable<Arc<InputModuleItems>> {
    let module_tree = db.module_tree(source_root)?;
    let source = module_id.source(&module_tree);
    let file_items = db.file_items(source.file_id());
    let res = match source.resolve(db) {
        ModuleSourceNode::SourceFile(it) => {
            let items = it.borrowed().items();
            InputModuleItems::new(&file_items, items)
        }
        ModuleSourceNode::Module(it) => {
            let items = it
                .borrowed()
                .item_list()
                .into_iter()
                .flat_map(|it| it.items());
            InputModuleItems::new(&file_items, items)
        }
    };
    Ok(Arc::new(res))
}

pub(super) fn item_map(
    db: &impl HirDatabase,
    source_root: SourceRootId,
) -> Cancelable<Arc<ItemMap>> {
    let start = Instant::now();
    let module_tree = db.module_tree(source_root)?;
    let input = module_tree
        .modules()
        .map(|id| {
            let items = db.input_module_items(source_root, id)?;
            Ok((id, items))
        })
        .collect::<Cancelable<FxHashMap<_, _>>>()?;
    let resolver = Resolver {
        db: db,
        input: &input,
        source_root,
        module_tree,
        result: ItemMap::default(),
    };
    let res = resolver.resolve()?;
    let elapsed = start.elapsed();
    log::info!("item_map: {:?}", elapsed);
    Ok(Arc::new(res))
}
<|endoftext|>