<commit_before>extern crate rff;
extern crate clap;

use std::io::{self, Write, BufWriter};
use rff::{stdin, matcher, scorer};
use clap::{App, Arg};

type Match<'a> = (&'a str, f64);

fn main() {
    let matches = App::new("rff").
        version(env!("CARGO_PKG_VERSION")).
        author("Andrew S. <andrew@stwrt.ca>").
        about("A fuzzy finder.").
        arg(
            Arg::with_name("QUERY").
                help("Term to search for").
                required(true).
                index(1)
        ).
        arg(
            Arg::with_name("benchmark").
                long("benchmark").
                help("Run rff in benchmark mode")
        ).
        get_matches();

    let query = matches.value_of("QUERY").unwrap();
    let lines = stdin::slurp();

    if matches.is_present("benchmark") {
        // in benchmark mode, we run the match/score/sort loop 100 times
        for _ in 0..100 {
            lines
                .iter()
                .filter_map(|line| match_and_score(query, line))
                .collect::<Vec<_>>()
                .sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().reverse());
        }
    } else {
        let mut lines: Vec<_> = lines
            .iter()
            .filter_map(|line| match_and_score(query, line))
            .collect();

        lines.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().reverse());

        let stdout = io::stdout();
        let mut stdout = BufWriter::new(stdout.lock());

        for line in &lines {
            writeln!(stdout, "{}", line.0).unwrap();
        }
    }
}

fn match_and_score<'a>(needle: &str, haystack: &'a str) -> Option<Match<'a>> {
    if matcher::matches(needle, haystack) {
        Some((haystack, scorer::score(needle, haystack)))
    } else {
        None
    }
}
<commit_msg>Extract work out of main()<commit_after>extern crate rff;
extern crate clap;

use std::io::{self, Write, BufWriter};
use rff::{stdin, matcher, scorer};
use clap::{App, Arg};

type Match<'a> = (&'a str, f64);

fn main() {
    let matches = App::new("rff").
        version(env!("CARGO_PKG_VERSION")).
        author("Andrew S. <andrew@stwrt.ca>").
        about("A fuzzy finder.").
        arg(
            Arg::with_name("QUERY").
                help("Term to search for").
                required(true).
                index(1)
        ).
        arg(
            Arg::with_name("benchmark").
                long("benchmark").
                help("Run rff in benchmark mode")
        ).
        get_matches();

    let query = matches.value_of("QUERY").unwrap();
    let lines = stdin::slurp();

    if matches.is_present("benchmark") {
        benchmark(query, lines);
    } else {
        search(query, lines);
    }
}

fn match_and_score<'a>(needle: &str, haystack: &'a str) -> Option<Match<'a>> {
    if matcher::matches(needle, haystack) {
        Some((haystack, scorer::score(needle, haystack)))
    } else {
        None
    }
}

fn benchmark(needle: &str, lines: Vec<String>) {
    // in benchmark mode, we run the match/score/sort loop 100 times
    for _ in 0..100 {
        lines
            .iter()
            .filter_map(|line| match_and_score(needle, line))
            .collect::<Vec<_>>()
            .sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().reverse());
    }
}

fn search(needle: &str, lines: Vec<String>) {
    let mut lines: Vec<_> = lines
        .iter()
        .filter_map(|line| match_and_score(needle, line))
        .collect();

    lines.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().reverse());

    let stdout = io::stdout();
    let mut stdout = BufWriter::new(stdout.lock());

    for line in &lines {
        writeln!(stdout, "{}", line.0).unwrap();
    }
}
<|endoftext|>


<commit_before>// Example story: He was so [adjective] that he [verb]ed John in the face, sending him into a/an [noun].

use std::io::File;

fn main() {
    let story = File::open(&Path::new("story.txt")).read_to_end();
    match story {
        Ok(story) => println!("{}", story),
        Err(msg) => println!("Oops: {}", msg),
    }
}
<commit_msg>file input line by line is working, non-unicode - CONCAT NOT WORKING<commit_after>// Example story: He was so [adjective] that he [verb]ed John in the face,
// sending him into a/an [noun].

use std::io;

fn main() {
    let mut story = "";
    println!("Please enter a line of the story. Enter a blank line to stop");
    for line in io::stdin().lines() {
        match line.to_string().as_slice() {
            "Ok(\n)" => break,
            _ => {
                println!("Please enter a line of the story. Enter a blank line to stop");
                //story = concat!(story, _);
            },
        }
    }
}
<|endoftext|>


<commit_before>extern crate libc;
extern crate termios;

use libc::STDIN_FILENO;
use termios::{Termios, tcsetattr, TCSAFLUSH, VMIN, VTIME};
use termios::{BRKINT, ICRNL, INPCK, ISTRIP, IXON};
use termios::{OPOST, CS8};
use termios::{ECHO, ICANON, IEXTEN, ISIG};

use std::io::{self, Stdin, Stdout, Read, ErrorKind, Write};

macro_rules! ctrl_key {
    ($k:expr) => ($k & 0x1f);
}

struct TermReset {
    orig_term: Termios
}

impl Drop for TermReset {
    fn drop(&mut self) {
        tcsetattr(STDIN_FILENO, TCSAFLUSH, &self.orig_term).unwrap()
    }
}

fn enable_raw_mode() -> io::Result<TermReset> {
    let mut term = Termios::from_fd(STDIN_FILENO)?;
    let mode = TermReset { orig_term: term.clone() };

    term.c_iflag &= !(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
    term.c_oflag &= !OPOST;
    term.c_cflag |= CS8;
    term.c_lflag &= !(ECHO | ICANON | IEXTEN | ISIG);
    term.c_cc[VMIN] = 0;
    term.c_cc[VTIME] = 1;

    tcsetattr(STDIN_FILENO, TCSAFLUSH, &term)?;
    Ok(mode)
}

fn editor_read_key(stdin: &mut Stdin) -> u8 {
    let mut buf = [0; 1];
    loop {
        let n = stdin.read(&mut buf)
                 .or_else(|e| if e.kind() == ErrorKind::WouldBlock { Ok(0) } else { Err(e) })
                 .expect("read");
        if n == 1 {
            return buf[0];
        }
    }
}

fn clear_screen(stdout: &mut Stdout) -> io::Result<()> {
    stdout.write(b"\x1b[2J")?;
    stdout.write(b"\x1b[H")?;
    stdout.flush()
}

fn editor_draw_rows(stdout: &mut Stdout) -> io::Result<()> {
  for _ in 0..24 {
    stdout.write(b"~\r\n")?;
  }
  Ok(())
}

fn editor_refresh_screen(stdout: &mut Stdout) -> io::Result<()> {
    clear_screen(stdout)?;

    editor_draw_rows(stdout)?;

    stdout.write(b"\x1b[H")?;
    stdout.flush()
}

fn editor_process_keypress(stdin: &mut Stdin) -> bool {
    let c = editor_read_key(stdin);

    match c {
        k if k == ctrl_key!(b'q') => {
            false
        },
        _ => true
    }
}

fn main() {
    let _reset = enable_raw_mode().unwrap();

    let mut stdin = io::stdin();
    let mut stdout = io::stdout();
    editor_refresh_screen(&mut stdout).unwrap();
    while editor_process_keypress(&mut stdin) {
        editor_refresh_screen(&mut stdout).unwrap();
    }
    clear_screen(&mut stdout).unwrap();
}
<commit_msg>Add EditorConfig struct<commit_after>extern crate libc;
extern crate termios;

use libc::STDIN_FILENO;
use termios::{Termios, tcsetattr, TCSAFLUSH, VMIN, VTIME};
use termios::{BRKINT, ICRNL, INPCK, ISTRIP, IXON};
use termios::{OPOST, CS8};
use termios::{ECHO, ICANON, IEXTEN, ISIG};

use std::io::{self, Stdin, Stdout, Read, ErrorKind, Write};

macro_rules! ctrl_key {
    ($k:expr) => ($k & 0x1f);
}

struct TermReset {
    orig_term: Termios
}

struct EditorConfig {
    _term: TermReset
}

impl Drop for TermReset {
    fn drop(&mut self) {
        tcsetattr(STDIN_FILENO, TCSAFLUSH, &self.orig_term).unwrap()
    }
}

impl EditorConfig {
    fn new() -> EditorConfig {
        let term = enable_raw_mode().unwrap();
        EditorConfig { _term: term }
    }
}

fn enable_raw_mode() -> io::Result<TermReset> {
    let mut term = Termios::from_fd(STDIN_FILENO)?;
    let mode = TermReset { orig_term: term.clone() };

    term.c_iflag &= !(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
    term.c_oflag &= !OPOST;
    term.c_cflag |= CS8;
    term.c_lflag &= !(ECHO | ICANON | IEXTEN | ISIG);
    term.c_cc[VMIN] = 0;
    term.c_cc[VTIME] = 1;

    tcsetattr(STDIN_FILENO, TCSAFLUSH, &term)?;
    Ok(mode)
}

fn editor_read_key(stdin: &mut Stdin) -> u8 {
    let mut buf = [0; 1];
    loop {
        let n = stdin.read(&mut buf)
                 .or_else(|e| if e.kind() == ErrorKind::WouldBlock { Ok(0) } else { Err(e) })
                 .expect("read");
        if n == 1 {
            return buf[0];
        }
    }
}

fn clear_screen(stdout: &mut Stdout) -> io::Result<()> {
    stdout.write(b"\x1b[2J")?;
    stdout.write(b"\x1b[H")?;
    stdout.flush()
}

fn editor_draw_rows(stdout: &mut Stdout) -> io::Result<()> {
  for _ in 0..24 {
    stdout.write(b"~\r\n")?;
  }
  Ok(())
}

fn editor_refresh_screen(stdout: &mut Stdout) -> io::Result<()> {
    clear_screen(stdout)?;

    editor_draw_rows(stdout)?;

    stdout.write(b"\x1b[H")?;
    stdout.flush()
}

fn editor_process_keypress(stdin: &mut Stdin) -> bool {
    let c = editor_read_key(stdin);

    match c {
        k if k == ctrl_key!(b'q') => {
            false
        },
        _ => true
    }
}

fn main() {
    let _e = EditorConfig::new();

    let mut stdin = io::stdin();
    let mut stdout = io::stdout();
    editor_refresh_screen(&mut stdout).unwrap();
    while editor_process_keypress(&mut stdin) {
        editor_refresh_screen(&mut stdout).unwrap();
    }
    clear_screen(&mut stdout).unwrap();
}
<|endoftext|>


<commit_before>extern crate devicemapper;
#[macro_use]
extern crate clap;

use std::io::Result;

use devicemapper as devm;
use clap::{App, Arg, SubCommand, ArgMatches};

fn list(args: &ArgMatches) -> Result<()> {
    println!("hello from list()");
    Ok(())
}

fn add(args: &ArgMatches) -> Result<()> {
    println!("hello from add()");
    Ok(())
}

fn main() {

    let matches = App::new("froyo")
        .version(&crate_version!())
        .author("Andy Grover <andy@groveronline.com>")
        .about("Drobo + Free + YOLO")
        .subcommand(SubCommand::with_name("list")
                    .about("lists stuff")
                    .arg(Arg::with_name("poolname")
                         .help("optional pool to list")
                         .index(1)))
        .subcommand(SubCommand::with_name("add")
                    .about("add a device to a pool")
                    .arg(Arg::with_name("poolname")
                         .help("pool to list")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("device")
                         .help("device to add")
                         .required(true)
                         .index(2))
                    )
        .get_matches();

    match matches.subcommand() {
        ("list", Some(matches)) => list(matches),
        ("add", Some(matches)) => add(matches),
        _ => unreachable!(),
    };
}
<commit_msg>Flesh out cmdline options<commit_after>extern crate devicemapper;
#[macro_use]
extern crate clap;

use std::io::Result;
use std::error::Error;
use std::process::exit;

use devicemapper as devm;
use clap::{App, Arg, SubCommand, ArgMatches};

fn list(args: &ArgMatches) -> Result<()> {
    println!("hello from list()");
    Ok(())
}

fn add(args: &ArgMatches) -> Result<()> {
    println!("hello from add()");
    Ok(())
}

fn main() {

    let matches = App::new("froyo")
        .version(&crate_version!())
        .author("Andy Grover <andy@groveronline.com>")
        .about("Drobo + Free + YOLO")
        .arg(Arg::with_name("debug")
             .short("d")
             .help("Print additional output for debugging"))
        .subcommand(SubCommand::with_name("list")
                    .about("List all froyodevs")
                    .arg(Arg::with_name("long")
                         .short("l")
                         .help("Use a long listing format")))
        .subcommand(SubCommand::with_name("status")
                    .about("Get the status of a single froyodev")
                    .arg(Arg::with_name("froyodevname")
                         .required(true)
                         .help("Froyodev to get info on")
                         .index(1)))
        .subcommand(SubCommand::with_name("add")
                    .about("Add one or more additional block devices to a froyodev")
                    .arg(Arg::with_name("froyodevname")
                         .help("Froyodev to add the device to")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("devices")
                         .help("device to add")
                         .multiple(true)
                         .required(true)
                         .index(2))
                    )
        .subcommand(SubCommand::with_name("remove")
                    .about("Remove a block device from a froyodev")
                    .arg(Arg::with_name("froyodevname")
                         .help("Froyodev to remove the device from")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("device")
                         .help("Block device to remove")
                         .required(true)
                         .index(2))
                    )
        .subcommand(SubCommand::with_name("create")
                    .about("Create a new froyodev")
                    .arg(Arg::with_name("froyodevname")
                         .help("Name of the new froyodev")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("devices")
                         .help("Initial block device(s) to use")
                         .multiple(true)
                         .required(true)
                         .index(2))
                    )
        .get_matches();

    let r = match matches.subcommand() {
        ("list", Some(matches)) => list(matches),
        ("add", Some(matches)) => add(matches),
        ("", None) => {
            println!("No command given, try \"help\"");
            Ok(())
        }
        _ => unreachable!(),
    };

    if let Err(r) = r {
        println!("there was an error: {}", r.description());
        exit(1);
    };
}
<|endoftext|>


<commit_before>use cargo_tarpaulin::cargo::{rust_flags, rustdoc_flags};
use cargo_tarpaulin::config::{Config, ConfigWrapper, OutputFile, RunType, Mode};
use cargo_tarpaulin::run;
use clap::{crate_version, App, Arg, ArgSettings, SubCommand};
use std::collections::HashMap;
use std::path::Path;
use tracing::{debug, info, trace};
use tracing_subscriber::{filter::LevelFilter, EnvFilter};

const RUST_LOG_ENV: &str = "RUST_LOG";

fn print_env(seen_rustflags: HashMap<String, Vec<String>>, prefix: &str, default_val: &str) {
    info!("Printing `{}`", prefix);
    if seen_rustflags.is_empty() {
        info!("No configs provided printing default RUSTFLAGS");
        println!("{}={}", prefix, default_val);
    } else if seen_rustflags.len() == 1 {
        let flags = seen_rustflags.keys().next().unwrap();
        println!(r#"{}="{}""#, prefix, flags);
    } else {
        for (k, v) in seen_rustflags.iter() {
            info!("RUSTFLAGS for configs {:?}", v);
            println!(r#"{}="{}""#, prefix, k);
        }
    }
}

fn is_dir(d: String) -> Result<(), String> {
    if Path::new(&d).is_dir() {
        Ok(())
    } else {
        Err(String::from("root must be a directory"))
    }
}

fn set_up_logging(debug: bool, verbose: bool) {
    //By default, we set tarpaulin to info,debug,trace while all dependencies stay at INFO
    let base_exceptions = |env: EnvFilter| {
        if debug {
            env.add_directive("cargo_tarpaulin=trace".parse().unwrap())
        } else if verbose {
            env.add_directive("cargo_tarpaulin=debug".parse().unwrap())
        } else {
            env.add_directive("cargo_tarpaulin=info".parse().unwrap())
        }
        .add_directive(LevelFilter::INFO.into())
    };

    //If RUST_LOG is set, then first apply our default directives (which are controlled by debug an verbose).
    // Then RUST_LOG will overwrite those default directives.
    // e.g. `RUST_LOG="trace" cargo-tarpaulin` will end up printing TRACE for everything
    // `cargo-tarpaulin -v` will print DEBUG for tarpaulin and INFO for everything else.
    // `RUST_LOG="error" cargo-tarpaulin -v` will print ERROR for everything.
    let filter = match std::env::var_os(RUST_LOG_ENV).map(|s| s.into_string()) {
        Some(Ok(env)) => {
            let mut filter = base_exceptions(EnvFilter::new(""));
            for s in env.split(',').into_iter() {
                match s.parse() {
                    Ok(d) => filter = filter.add_directive(d),
                    Err(err) => println!("WARN ignoring log directive: `{}`: {}", s, err),
                };
            }
            filter
        }
        _ => base_exceptions(EnvFilter::from_env(RUST_LOG_ENV)),
    };

    tracing_subscriber::FmtSubscriber::builder()
        .with_max_level(tracing::Level::ERROR)
        .with_env_filter(filter)
        .init();

    debug!("set up logging");
}

const CI_SERVER_HELP: &str = "Name of service, supported services are:
travis-ci, travis-pro, circle-ci, semaphore, jenkins and codeship.
If you are interfacing with coveralls.io or another site you can \
also specify a name that they will recognise. Refer to their documentation for this.";

fn main() -> Result<(), String> {
    let args = App::new("cargo-tarpaulin")
        .author("Daniel McKenna, <danielmckenna93@gmail.com>")
        .about("Tool to analyse test coverage of cargo projects")
        .version(concat!("version: ", crate_version!()))
        .bin_name("cargo")
        .subcommand(SubCommand::with_name("tarpaulin")
            .about("Tool to analyse test coverage of cargo projects")
            .version(concat!("version: ", crate_version!()))
            .args_from_usage(
                 "--config [FILE] 'Path to a toml file specifying a list of options this will override any other options set'
                 --ignore-config 'Ignore any project config files'
                 --lib 'Test only this package's library unit tests'
                 --bin [NAME]... 'Test only the specified binary`
                 --bins 'Test all binaries'
                 --example [NAME]... 'Test only the specified example'
                 --examples 'Test all examples'
                 --test [NAME]... 'Test only the specified test target'
                 --tests 'Test all tests'
                 --bench [NAME]... 'Test only the specified bench target'
                 --benches 'Test all benches'
                 --doc 'Test only this library's documentation'
                 --all-targets 'Test all targets'
                 --no-fail-fast 'Run all tests regardless of failure'
                 --profile [NAME] 'Build artefacts with the specified profile'
                 --debug 'Show debug output - this is used for diagnosing issues with tarpaulin'
                 --dump-traces 'Log tracing events and save to a json file. Also, enabled when --debug is used'
                 --verbose -v 'Show extra output'
                 --ignore-tests 'Ignore lines of test functions when collecting coverage'
                 --ignore-panics 'Ignore panic macros in tests'
                 --count   'Counts the number of hits during coverage'
                 --ignored -i 'Run ignored tests as well'
                 --line -l    'Line coverage'
                 --force-clean 'Adds a clean stage to work around cargo bugs that may affect coverage results'
                 --fail-under [PERCENTAGE] 'Sets a percentage threshold for failure ranging from 0-100, if coverage is below exit with a non-zero code'
                 --branch -b  'Branch coverage: NOT IMPLEMENTED'
                 --forward -f 'Forwards unexpected signals to test. Tarpaulin will still take signals it is expecting.'
                 --coveralls [KEY]  'Coveralls key, either the repo token, or if you're using travis use $TRAVIS_JOB_ID and specify travis-{ci|pro} in --ciserver'
                 --report-uri [URI] 'URI to send report to, only used if the option --coveralls is used'
                 --no-default-features 'Do not include default features'
                 --features [FEATURES]... 'Features to be included in the target project'
                 --all-features 'Build all available features'
                 --all        'Alias for --workspace (deprecated)'
                 --workspace 'Test all packages in the workspace'
                 --packages -p [PACKAGE]... 'Package id specifications for which package should be build. See cargo help pkgid for more info'
                 --exclude -e [PACKAGE]... 'Package id specifications to exclude from coverage. See cargo help pkgid for more info'
                 --exclude-files [FILE]... 'Exclude given files from coverage results has * wildcard'
                 --timeout -t [SECONDS] 'Integer for the maximum time in seconds without response from test before timeout (default is 1 minute).'
                 --release   'Build in release mode.'
                 --no-run 'Compile tests but don't run coverage'
                 --locked 'Do not update Cargo.lock'
                 --frozen 'Do not update Cargo.lock or any caches'
                 --target [TRIPLE] 'Compilation target triple'
                 --target-dir [DIR] 'Directory for all generated artifacts'
                 --offline 'Run without accessing the network'
                 --print-rust-flags 'Print the RUSTFLAGS options that tarpaulin will compile your program with and exit'
                 --print-rustdoc-flags 'Print the RUSTDOCFLAGS options that tarpaulin will compile any doctests with and exit'
                 -Z [FEATURES]...   'List of unstable nightly only flags'")
            .args(&[
                Arg::from_usage("--out -o [FMT]   'Output format of coverage report'")
                    .possible_values(&OutputFile::variants())
                    .multiple(true),
                Arg::from_usage("--output-dir [PATH] 'Specify a custom directory to write report files'"),
                Arg::from_usage("--run-types [TYPE]... 'Type of the coverage run'")
                    .possible_values(&RunType::variants())
                    .multiple(true),
                Arg::from_usage("--command [CMD] 'cargo subcommand to run. So far only test and build are supported'")
                    .possible_values(&Mode::variants()),
                Arg::from_usage("--root -r [DIR]  'Calculates relative paths to root directory. If --manifest-path isn't specified it will look for a Cargo.toml in root'")
                    .validator(is_dir),
                Arg::from_usage("--manifest-path [PATH] 'Path to Cargo.toml'"),
                Arg::from_usage("--ciserver [SERVICE] 'CI server being used, if unspecified tarpaulin may automatically infer for coveralls uploads'")
                    .help(CI_SERVER_HELP),
                Arg::with_name("args")
                    .set(ArgSettings::Last)
                    .multiple(true)
                    .help("Arguments to be passed to the test executables can be used to filter or skip certain tests")
            ]))
        .get_matches();

    let args = args.subcommand_matches("tarpaulin").unwrap_or(&args);

    set_up_logging(args.is_present("debug"), args.is_present("verbose"));
    let config = ConfigWrapper::from(args);
    let mut run_coverage = true;
    if args.is_present("print-rust-flags") {
        run_coverage = false;
        let mut seen_rustflags = HashMap::new();
        for config in &config.0 {
            let rustflags = rust_flags(config);
            seen_rustflags
                .entry(rustflags)
                .or_insert(vec![])
                .push(config.name.clone());
        }
        let default = Config::default();
        print_env(seen_rustflags, "RUSTFLAGS", &rust_flags(&default));
    }
    if args.is_present("print-rustdoc-flags") {
        run_coverage = false;
        let mut seen_rustflags = HashMap::new();
        for config in &config.0 {
            let rustflags = rustdoc_flags(config);
            seen_rustflags
                .entry(rustflags)
                .or_insert(vec![])
                .push(config.name.clone());
        }
        let default = Config::default();
        print_env(seen_rustflags, "RUSTDOCFLAGS", &rustdoc_flags(&default));
    }
    if run_coverage {
        trace!("Debug mode activated");
        // Since this is the last function we run and don't do any error mitigations (other than
        // printing the error to the user it's fine to unwrap here
        run(&config.0).map_err(|e| e.to_string())
    } else {
        Ok(())
    }
}
<commit_msg>Fix formating<commit_after>use cargo_tarpaulin::cargo::{rust_flags, rustdoc_flags};
use cargo_tarpaulin::config::{Config, ConfigWrapper, Mode, OutputFile, RunType};
use cargo_tarpaulin::run;
use clap::{crate_version, App, Arg, ArgSettings, SubCommand};
use std::collections::HashMap;
use std::path::Path;
use tracing::{debug, info, trace};
use tracing_subscriber::{filter::LevelFilter, EnvFilter};

const RUST_LOG_ENV: &str = "RUST_LOG";

fn print_env(seen_rustflags: HashMap<String, Vec<String>>, prefix: &str, default_val: &str) {
    info!("Printing `{}`", prefix);
    if seen_rustflags.is_empty() {
        info!("No configs provided printing default RUSTFLAGS");
        println!("{}={}", prefix, default_val);
    } else if seen_rustflags.len() == 1 {
        let flags = seen_rustflags.keys().next().unwrap();
        println!(r#"{}="{}""#, prefix, flags);
    } else {
        for (k, v) in seen_rustflags.iter() {
            info!("RUSTFLAGS for configs {:?}", v);
            println!(r#"{}="{}""#, prefix, k);
        }
    }
}

fn is_dir(d: String) -> Result<(), String> {
    if Path::new(&d).is_dir() {
        Ok(())
    } else {
        Err(String::from("root must be a directory"))
    }
}

fn set_up_logging(debug: bool, verbose: bool) {
    //By default, we set tarpaulin to info,debug,trace while all dependencies stay at INFO
    let base_exceptions = |env: EnvFilter| {
        if debug {
            env.add_directive("cargo_tarpaulin=trace".parse().unwrap())
        } else if verbose {
            env.add_directive("cargo_tarpaulin=debug".parse().unwrap())
        } else {
            env.add_directive("cargo_tarpaulin=info".parse().unwrap())
        }
        .add_directive(LevelFilter::INFO.into())
    };

    //If RUST_LOG is set, then first apply our default directives (which are controlled by debug an verbose).
    // Then RUST_LOG will overwrite those default directives.
    // e.g. `RUST_LOG="trace" cargo-tarpaulin` will end up printing TRACE for everything
    // `cargo-tarpaulin -v` will print DEBUG for tarpaulin and INFO for everything else.
    // `RUST_LOG="error" cargo-tarpaulin -v` will print ERROR for everything.
    let filter = match std::env::var_os(RUST_LOG_ENV).map(|s| s.into_string()) {
        Some(Ok(env)) => {
            let mut filter = base_exceptions(EnvFilter::new(""));
            for s in env.split(',').into_iter() {
                match s.parse() {
                    Ok(d) => filter = filter.add_directive(d),
                    Err(err) => println!("WARN ignoring log directive: `{}`: {}", s, err),
                };
            }
            filter
        }
        _ => base_exceptions(EnvFilter::from_env(RUST_LOG_ENV)),
    };

    tracing_subscriber::FmtSubscriber::builder()
        .with_max_level(tracing::Level::ERROR)
        .with_env_filter(filter)
        .init();

    debug!("set up logging");
}

const CI_SERVER_HELP: &str = "Name of service, supported services are:
travis-ci, travis-pro, circle-ci, semaphore, jenkins and codeship.
If you are interfacing with coveralls.io or another site you can \
also specify a name that they will recognise. Refer to their documentation for this.";

fn main() -> Result<(), String> {
    let args = App::new("cargo-tarpaulin")
        .author("Daniel McKenna, <danielmckenna93@gmail.com>")
        .about("Tool to analyse test coverage of cargo projects")
        .version(concat!("version: ", crate_version!()))
        .bin_name("cargo")
        .subcommand(SubCommand::with_name("tarpaulin")
            .about("Tool to analyse test coverage of cargo projects")
            .version(concat!("version: ", crate_version!()))
            .args_from_usage(
                 "--config [FILE] 'Path to a toml file specifying a list of options this will override any other options set'
                 --ignore-config 'Ignore any project config files'
                 --lib 'Test only this package's library unit tests'
                 --bin [NAME]... 'Test only the specified binary`
                 --bins 'Test all binaries'
                 --example [NAME]... 'Test only the specified example'
                 --examples 'Test all examples'
                 --test [NAME]... 'Test only the specified test target'
                 --tests 'Test all tests'
                 --bench [NAME]... 'Test only the specified bench target'
                 --benches 'Test all benches'
                 --doc 'Test only this library's documentation'
                 --all-targets 'Test all targets'
                 --no-fail-fast 'Run all tests regardless of failure'
                 --profile [NAME] 'Build artefacts with the specified profile'
                 --debug 'Show debug output - this is used for diagnosing issues with tarpaulin'
                 --dump-traces 'Log tracing events and save to a json file. Also, enabled when --debug is used'
                 --verbose -v 'Show extra output'
                 --ignore-tests 'Ignore lines of test functions when collecting coverage'
                 --ignore-panics 'Ignore panic macros in tests'
                 --count   'Counts the number of hits during coverage'
                 --ignored -i 'Run ignored tests as well'
                 --line -l    'Line coverage'
                 --force-clean 'Adds a clean stage to work around cargo bugs that may affect coverage results'
                 --fail-under [PERCENTAGE] 'Sets a percentage threshold for failure ranging from 0-100, if coverage is below exit with a non-zero code'
                 --branch -b  'Branch coverage: NOT IMPLEMENTED'
                 --forward -f 'Forwards unexpected signals to test. Tarpaulin will still take signals it is expecting.'
                 --coveralls [KEY]  'Coveralls key, either the repo token, or if you're using travis use $TRAVIS_JOB_ID and specify travis-{ci|pro} in --ciserver'
                 --report-uri [URI] 'URI to send report to, only used if the option --coveralls is used'
                 --no-default-features 'Do not include default features'
                 --features [FEATURES]... 'Features to be included in the target project'
                 --all-features 'Build all available features'
                 --all        'Alias for --workspace (deprecated)'
                 --workspace 'Test all packages in the workspace'
                 --packages -p [PACKAGE]... 'Package id specifications for which package should be build. See cargo help pkgid for more info'
                 --exclude -e [PACKAGE]... 'Package id specifications to exclude from coverage. See cargo help pkgid for more info'
                 --exclude-files [FILE]... 'Exclude given files from coverage results has * wildcard'
                 --timeout -t [SECONDS] 'Integer for the maximum time in seconds without response from test before timeout (default is 1 minute).'
                 --release   'Build in release mode.'
                 --no-run 'Compile tests but don't run coverage'
                 --locked 'Do not update Cargo.lock'
                 --frozen 'Do not update Cargo.lock or any caches'
                 --target [TRIPLE] 'Compilation target triple'
                 --target-dir [DIR] 'Directory for all generated artifacts'
                 --offline 'Run without accessing the network'
                 --print-rust-flags 'Print the RUSTFLAGS options that tarpaulin will compile your program with and exit'
                 --print-rustdoc-flags 'Print the RUSTDOCFLAGS options that tarpaulin will compile any doctests with and exit'
                 -Z [FEATURES]...   'List of unstable nightly only flags'")
            .args(&[
                Arg::from_usage("--out -o [FMT]   'Output format of coverage report'")
                    .possible_values(&OutputFile::variants())
                    .multiple(true),
                Arg::from_usage("--output-dir [PATH] 'Specify a custom directory to write report files'"),
                Arg::from_usage("--run-types [TYPE]... 'Type of the coverage run'")
                    .possible_values(&RunType::variants())
                    .multiple(true),
                Arg::from_usage("--command [CMD] 'cargo subcommand to run. So far only test and build are supported'")
                    .possible_values(&Mode::variants()),
                Arg::from_usage("--root -r [DIR]  'Calculates relative paths to root directory. If --manifest-path isn't specified it will look for a Cargo.toml in root'")
                    .validator(is_dir),
                Arg::from_usage("--manifest-path [PATH] 'Path to Cargo.toml'"),
                Arg::from_usage("--ciserver [SERVICE] 'CI server being used, if unspecified tarpaulin may automatically infer for coveralls uploads'")
                    .help(CI_SERVER_HELP),
                Arg::with_name("args")
                    .set(ArgSettings::Last)
                    .multiple(true)
                    .help("Arguments to be passed to the test executables can be used to filter or skip certain tests")
            ]))
        .get_matches();

    let args = args.subcommand_matches("tarpaulin").unwrap_or(&args);

    set_up_logging(args.is_present("debug"), args.is_present("verbose"));
    let config = ConfigWrapper::from(args);
    let mut run_coverage = true;
    if args.is_present("print-rust-flags") {
        run_coverage = false;
        let mut seen_rustflags = HashMap::new();
        for config in &config.0 {
            let rustflags = rust_flags(config);
            seen_rustflags
                .entry(rustflags)
                .or_insert(vec![])
                .push(config.name.clone());
        }
        let default = Config::default();
        print_env(seen_rustflags, "RUSTFLAGS", &rust_flags(&default));
    }
    if args.is_present("print-rustdoc-flags") {
        run_coverage = false;
        let mut seen_rustflags = HashMap::new();
        for config in &config.0 {
            let rustflags = rustdoc_flags(config);
            seen_rustflags
                .entry(rustflags)
                .or_insert(vec![])
                .push(config.name.clone());
        }
        let default = Config::default();
        print_env(seen_rustflags, "RUSTDOCFLAGS", &rustdoc_flags(&default));
    }
    if run_coverage {
        trace!("Debug mode activated");
        // Since this is the last function we run and don't do any error mitigations (other than
        // printing the error to the user it's fine to unwrap here
        run(&config.0).map_err(|e| e.to_string())
    } else {
        Ok(())
    }
}
<|endoftext|>


<commit_before>// ZSnapMgr :: ZFS snapshot and backup manager
//
// Copyright (c) 2016 by William R. Fraser
//

#![allow(unknown_lints)] // for Clippy

use std::env;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::iter::Iterator;
use std::io;
use std::io::{Read, Write};
use std::ops::Deref;
use std::path::Path;
use std::process;

extern crate regex;
use regex::Regex;

extern crate termios;
use termios::*;

extern crate zsnapmgr;
use zsnapmgr::ZSnapMgr;

mod table;
use table::Table;

mod backups;
use backups::{Backup, Backups};

static USE_SUDO: bool = true;

// Print and flush.
macro_rules! printf {
    ( $fmt:expr ) => {
        print!($fmt);
        io::stdout().flush().unwrap();
    };
    ( $fmt:expr, $($arg:tt)* ) => {
        print!($fmt, $($arg)*);
        io::stdout().flush().unwrap();
    }
}

trait OptionDeref<T: Deref> {
    fn as_deref(&self) -> Option<&T::Target>;
}

impl<T: Deref> OptionDeref<T> for Option<T> {
    fn as_deref(&self) -> Option<&T::Target> {
        self.as_ref().map(Deref::deref)
    }
}

trait VecDeref<T: Deref> {
    fn as_deref(&self) -> Vec<&T::Target>;
}

impl<T: Deref> VecDeref<T> for Vec<T> {
    fn as_deref(&self) -> Vec<&T::Target> {
        self.iter().map(Deref::deref).collect()
    }
}

fn enumerate_files(path: &Path) -> Result<Box<dyn Iterator<Item = String>>, io::Error> {
    let readdir = fs::read_dir(path)?;
    let iter = readdir.filter_map(|entry_result| {
        match entry_result {
            Ok(entry) => {
                match entry.file_type() {
                    Ok(ft) => {
                        if ft.is_file() {
                            let path = entry.path();
                            let filename: &OsStr = path.file_name().unwrap();
                            match filename.to_str() {
                                Some(s) => Some(s.to_owned()),
                                None => {
                                    println!("error: filename {:?} is invalid UTF-8!", filename);
                                    None
                                }
                            }
                        } else {
                            None
                        }
                    },
                    Err(e) => {
                        println!("error getting file type of {:?}: {}",
                            entry.path(), e);
                        None
                    }
                }
            },
            Err(e) => {
                println!("error enumerating files: {}", e);
                None
            }
        }
    });
    Ok(Box::new(iter))
}

fn gather_volumes(path: &Path) -> Vec<Backup> {
    let z = ZSnapMgr::new(USE_SUDO);
    let snapshots: Vec<String> = match z.get_snapshots(None) {
        Ok(s) => s,
        Err(e) => {
            println!("Error getting snapshots from ZFS: {}", e);
            return vec![];
        }
    };

    let volumes: Vec<String> = match z.get_volumes() {
        Ok(v) => v,
        Err(e) => {
            println!("Error getting volumes from ZFS: {}", e);
            return vec![];
        }
    };

    let mut backups = Backups::new();

    let file_iter = match enumerate_files(path) {
        Ok(x) => x,
        Err(e) => {
            println!("Error enumerating snapshot files: {}", e);
            return vec![];
        }
    };

    for file_path in file_iter {
        if let Some(zfs_pos) = file_path.find(".zfs") {
            if !(&file_path).ends_with("_partial") {
                let parts = file_path[0..zfs_pos].splitn(2, '@').collect::<Vec<&str>>();
                if parts.len() != 2 {
                    println!("ERROR: malformed ZFS filename: {:?}", file_path);
                    continue;
                }
                let volume_name = parts[0].replace("_", "/");
                let backup_snap = parts[1];

                if volumes.binary_search(&volume_name).is_ok() {
                    backups.insert(parts[0].to_string(),
                                   volume_name.to_string(),
                                   Some(backup_snap.to_string()));
                } else {
                    let volume_name_mod = "/".to_string() + &volume_name;
                    let matches: Vec<&str> = volumes.iter()
                                                    .filter(|vol| {
                                                        vol.ends_with(&volume_name_mod)
                                                    })
                                                    .map(Deref::deref)
                                                    .collect();

                    if matches.len() == 1 {
                        backups.insert(volume_name.to_string(),
                                       matches[0].to_string(),
                                       Some(backup_snap.to_string()));
                    } else {
                        print!("Backup filename \"{}\" ", file_path);
                        if matches.len() > 1 {
                            println!("matches more than one volume.\nIt could be any of: {:?}",
                                     matches);
                        } else {
                            println!("doesn't match any volumes.");
                        }
                        println!("Skipping it.\n");
                    }
                }
            }
        }
    }

    let date_regex = Regex::new(r"^\d{4}-\d{2}-\d{2}$").unwrap();

    // Now fill in the latest snapshot available for each volume in the proposed backups.
    for backup in backups.iter_mut() {
        let volume_at = backup.volume.clone() + "@";
        let mut volume_snaps: Vec<&str> = snapshots
                .iter()
                .filter(|snap| snap.starts_with(&volume_at)
                        && date_regex.is_match(snap.split_at(volume_at.len()).1))
                .map(Deref::deref)
                .collect();
        volume_snaps.sort();

        if backup.start_snapshot.is_some() {
            // Check that the start snapshot still exists.
            let start_snapshot = volume_at.clone() + backup.start_snapshot.as_ref().unwrap();

            if volume_snaps.binary_search(&start_snapshot.deref()).is_err() {
                println!("Snapshot \"{}\" doesn't exist any more; doing full backup instead.\n",
                         start_snapshot);
                backup.start_snapshot = None;
            }
        }

        let last_snapshot: &str = volume_snaps.last().unwrap().splitn(2, '@').last().unwrap();

        if backup.start_snapshot.as_deref() != Some(last_snapshot) {
            backup.end_snapshot = Some(last_snapshot.to_owned());
        } else if backup.start_snapshot.is_some() {
            println!("Backup of \"{}\" is up to date (@{}). Skipping.\n",
                     backup.volume,
                     backup.start_snapshot.as_ref().unwrap());
            backup.end_snapshot = None;
        }
    }

    backups.into_values()
}

fn getpass(prompt: &str) -> io::Result<String> {
    let mut termios = Termios::from_fd(0).expect("failed to get termios settings");

    let old = termios.c_lflag;
    termios.c_lflag &= !ECHO;   // disable echo
    termios.c_lflag &= !ICANON; // disable line-buffering
    tcsetattr(0, TCSAFLUSH, &termios).expect("failed to set termios settings");

    printf!("{}", prompt);

    let stdin = io::stdin();
    let mut bytes = stdin.lock().bytes();
    let mut line = String::new();
    let mut utf8 = Vec::<u8>::new();
    loop {
        match bytes.next().or_else(|| Some(Err(io::Error::new(io::ErrorKind::Other, "EOF in getpass!")))) {
            Some(Ok(byte)) => {
                // 0x4 is EOT; aka ctrl-D
                if byte == 0x4 && utf8.is_empty() {
                    return Err(io::Error::new(io::ErrorKind::Other, "EOF in getpass!"));
                }

                utf8.push(byte);

                let mut valid_utf8 = false;
                if let Ok(c) = std::str::from_utf8(&utf8) {
                    if c == "\n" {
                        println!();
                        break;
                    } else {
                        valid_utf8 = true;
                        line.push_str(c);
                        printf!("*");
                    }
                }

                if valid_utf8 {
                    utf8.clear();
                }
            }
            Some(Err(e)) => {
                return Err(e);
            }
            _ => unreachable!(),
        }
    }

    termios.c_lflag = old;
    tcsetattr(0, TCSAFLUSH, &termios).expect("failed to reset termios settings");

    Ok(line)
}

fn do_backups(backups: &[Backup], path: &Path) {
    if backups.is_empty() {
        println!("Nothing to do.");
        return;
    }

    let passphrase: String;
    loop {
        let pass1 = getpass("GPG passphrase: ").unwrap();
        let pass2 = getpass("again: ").unwrap();
        if pass1 != pass2 {
            println!("Passphrases do not match.");
        } else {
            passphrase = pass1;
            break;
        }
    }

    for backup in backups {
        let z = ZSnapMgr::new(USE_SUDO);

        let snapshot = format!("{}@{}",
                               backup.volume,
                               backup.end_snapshot.as_deref().unwrap());

        println!("\nBacking up: {}", snapshot);

        z.backup(path,
                 &snapshot,
                 &passphrase,
                 backup.start_snapshot.as_deref())
         .err()
         .and_then(|e| {
             println!("failed backup of {}: {}", backup.volume, e);
             Some(())
         });
    }
}

fn interactive_backup(backups_dir: &Path) {
    let z = ZSnapMgr::new(USE_SUDO);
    let mut backups: Vec<Backup> = gather_volumes(backups_dir);
    loop {
        let mut table = Table::new(&["_", "volume", "incremental", "snapshot date"]);
        for (i, backup) in backups.iter().enumerate() {
            let start = if backup.start_snapshot.is_none() {
                "full backup".to_string()
            } else {
                backup.start_snapshot.as_ref().unwrap().clone()
            };

            table.push(vec![(i + 1).to_string(),
                            backup.volume.clone(),
                            start,
                            backup.end_snapshot.as_ref().unwrap().clone()]);
        }

        println!("Volumes to backup:\n{}", table);

        printf!(concat!("Enter a number to make changes,\n",
                         "\t'+' to add a volume,\n",
                         "\t'-' to remove one,\n",
                         "\t'd' to change all dates,\n",
                         "\tor <return> to start backup: "));

        let mut input = String::new();
        match io::stdin().read_line(&mut input) {
            Ok(bytes_read) => {
                if bytes_read == 0 {
                    panic!("EOF");
                }
                input.pop();
            }
            Err(e) => panic!(e),
        }

        if input == "+" {
            printf!("Volume: ");

            let mut vol = String::new();
            io::stdin().read_line(&mut vol).unwrap();
            vol.pop();

            let latest_snap: String = match z.get_snapshots(Some(&vol))
                   .and_then(|ref mut snaps| {
                       Ok(snaps.pop()
                            .and_then(|full_name| {
                                full_name.rsplitn(2, '@')
                                    .next()
                                    .map(|s| s.to_owned())
                            }))
                   })
            {
                Ok(Some(date)) => date,
                Ok(None) => {
                    println!("No snapshots available for that volume.\n");
                    continue;
                }
                Err(e) => {
                    println!("Error listing snapshots: {}\n", e);
                    continue;
                }
            };

            backups.push(Backup {
                filename_base: vol.replace("/", "_").to_string(),
                volume: vol.clone(),
                start_snapshot: None,
                end_snapshot: Some(latest_snap),
            });

        } else if input.starts_with('-') {

            let index: usize;

            if input.len() > 1 {
                match input[1..].parse::<usize>() {
                    Ok(n) => {
                        index = n;
                    },
                    Err(e) => {
                        println!("Invalid number: {}\n", e);
                        continue;
                    }
                }
            } else {
                printf!("Remove which one?: ");

                input.clear();
                io::stdin().read_line(&mut input).unwrap();
                input.pop();

                match input.parse::<usize>() {
                    Ok(n) => {
                        index = n;
                    },
                    Err(e) => {
                        println!("Invalid number: {}\n", e);
                        continue;
                    }
                }
            }

            if backups.len() < index || index == 0 {
                println!("Number out of range.\n");
                continue;
            }

            backups.remove(index - 1);

        } else if input.starts_with('d') || input.starts_with('D') {

            printf!("Snapshot date (yyyy-MM-dd): ");

            let mut date = String::new();
            io::stdin().read_line(&mut date).unwrap();
            date.pop();

            for backup in &mut backups {
                backup.end_snapshot = Some(date.clone());
                //TODO: check that the snapshot exists for that volume
            }

        } else if input.is_empty() {
            println!("Starting backups.\n");
            do_backups(&backups, backups_dir);
            break;
        } else {
            let index: usize;

            match input.parse::<usize>() {
                Ok(n) => {
                    index = n;
                }
                Err(e) => {
                    println!("Invalid number: {}\n", e);
                    continue;
                }
            }

            if backups.len() < index || index == 0 {
                println!("Number out of range.\n");
                continue;
            }

            let vol = &mut backups[index - 1];

            printf!("Change (I)ncremental starting snapshot, (S)napshot date: ");

            input.clear();
            io::stdin().read_line(&mut input).unwrap();
            input.pop();

            let start: bool;
            if input == "I" || input == "i" {
                start = true;
            } else if input == "S" || input == "s" {
                start = false;
            } else {
                println!("Invalid selection.\n");
                continue;
            }

            printf!("Date (yyyy-MM-dd");
            if start {
                printf!(" or 'none' for full backup): ");
            } else {
                printf!("): ");
            }

            input.clear();
            io::stdin().read_line(&mut input).unwrap();
            input.pop();

            if start {
                vol.start_snapshot = if input == "none" {
                    None
                } else {
                    Some(input)
                }
            } else {
                vol.end_snapshot = Some(input);
            }
        }
    }
}

fn snapshot_automanage() {
    let z = ZSnapMgr::new(USE_SUDO);
    z.snapshot_automanage().unwrap();
}

fn main() {
    let args: Vec<OsString> = env::args_os().collect();
    let program_name = Path::new(&args[0]);

    let command = if args.len() < 2 {
        OsStr::new("help")
    } else {
        args[1].as_ref()
    };

    match command.to_str() {
        Some("backup") => {
            if args.len() == 3 {
                interactive_backup(Path::new(&args[2]));
            } else {
                println!("usage: {} backup <backups_location>", program_name.display());
                process::exit(-1);
            }
        }
        Some("automanage") => {
            snapshot_automanage();
        }
        _ => {
            if command != "help" {
                println!("unknown command \"{}\"", command.to_str().unwrap_or("[bad utf8]"));
            }
            println!("usage: {} <backup | automanage> [options]", program_name.display());
            process::exit(-1);
        }
    }
}
<commit_msg>allow symlinks when looking for previous backups<commit_after>// ZSnapMgr :: ZFS snapshot and backup manager
//
// Copyright (c) 2016 by William R. Fraser
//

#![allow(unknown_lints)] // for Clippy

use std::env;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::iter::Iterator;
use std::io;
use std::io::{Read, Write};
use std::ops::Deref;
use std::path::Path;
use std::process;

extern crate regex;
use regex::Regex;

extern crate termios;
use termios::*;

extern crate zsnapmgr;
use zsnapmgr::ZSnapMgr;

mod table;
use table::Table;

mod backups;
use backups::{Backup, Backups};

static USE_SUDO: bool = true;

// Print and flush.
macro_rules! printf {
    ( $fmt:expr ) => {
        print!($fmt);
        io::stdout().flush().unwrap();
    };
    ( $fmt:expr, $($arg:tt)* ) => {
        print!($fmt, $($arg)*);
        io::stdout().flush().unwrap();
    }
}

trait OptionDeref<T: Deref> {
    fn as_deref(&self) -> Option<&T::Target>;
}

impl<T: Deref> OptionDeref<T> for Option<T> {
    fn as_deref(&self) -> Option<&T::Target> {
        self.as_ref().map(Deref::deref)
    }
}

trait VecDeref<T: Deref> {
    fn as_deref(&self) -> Vec<&T::Target>;
}

impl<T: Deref> VecDeref<T> for Vec<T> {
    fn as_deref(&self) -> Vec<&T::Target> {
        self.iter().map(Deref::deref).collect()
    }
}

fn enumerate_files(path: &Path) -> Result<Box<dyn Iterator<Item = String>>, io::Error> {
    let readdir = fs::read_dir(path)?;
    let iter = readdir.filter_map(|entry_result| {
        match entry_result {
            Ok(entry) => {
                match entry.file_type() {
                    Ok(ft) => {
                        if ft.is_file() || ft.is_symlink() {
                            let path = entry.path();
                            let filename: &OsStr = path.file_name().unwrap();
                            match filename.to_str() {
                                Some(s) => Some(s.to_owned()),
                                None => {
                                    println!("error: filename {:?} is invalid UTF-8!", filename);
                                    None
                                }
                            }
                        } else {
                            None
                        }
                    },
                    Err(e) => {
                        println!("error getting file type of {:?}: {}",
                            entry.path(), e);
                        None
                    }
                }
            },
            Err(e) => {
                println!("error enumerating files: {}", e);
                None
            }
        }
    });
    Ok(Box::new(iter))
}

fn gather_volumes(path: &Path) -> Vec<Backup> {
    let z = ZSnapMgr::new(USE_SUDO);
    let snapshots: Vec<String> = match z.get_snapshots(None) {
        Ok(s) => s,
        Err(e) => {
            println!("Error getting snapshots from ZFS: {}", e);
            return vec![];
        }
    };

    let volumes: Vec<String> = match z.get_volumes() {
        Ok(v) => v,
        Err(e) => {
            println!("Error getting volumes from ZFS: {}", e);
            return vec![];
        }
    };

    let mut backups = Backups::new();

    let file_iter = match enumerate_files(path) {
        Ok(x) => x,
        Err(e) => {
            println!("Error enumerating snapshot files: {}", e);
            return vec![];
        }
    };

    for file_path in file_iter {
        if let Some(zfs_pos) = file_path.find(".zfs") {
            if !(&file_path).ends_with("_partial") {
                let parts = file_path[0..zfs_pos].splitn(2, '@').collect::<Vec<&str>>();
                if parts.len() != 2 {
                    println!("ERROR: malformed ZFS filename: {:?}", file_path);
                    continue;
                }
                let volume_name = parts[0].replace("_", "/");
                let backup_snap = parts[1];

                if volumes.binary_search(&volume_name).is_ok() {
                    backups.insert(parts[0].to_string(),
                                   volume_name.to_string(),
                                   Some(backup_snap.to_string()));
                } else {
                    let volume_name_mod = "/".to_string() + &volume_name;
                    let matches: Vec<&str> = volumes.iter()
                                                    .filter(|vol| {
                                                        vol.ends_with(&volume_name_mod)
                                                    })
                                                    .map(Deref::deref)
                                                    .collect();

                    if matches.len() == 1 {
                        backups.insert(volume_name.to_string(),
                                       matches[0].to_string(),
                                       Some(backup_snap.to_string()));
                    } else {
                        print!("Backup filename \"{}\" ", file_path);
                        if matches.len() > 1 {
                            println!("matches more than one volume.\nIt could be any of: {:?}",
                                     matches);
                        } else {
                            println!("doesn't match any volumes.");
                        }
                        println!("Skipping it.\n");
                    }
                }
            }
        }
    }

    let date_regex = Regex::new(r"^\d{4}-\d{2}-\d{2}$").unwrap();

    // Now fill in the latest snapshot available for each volume in the proposed backups.
    for backup in backups.iter_mut() {
        let volume_at = backup.volume.clone() + "@";
        let mut volume_snaps: Vec<&str> = snapshots
                .iter()
                .filter(|snap| snap.starts_with(&volume_at)
                        && date_regex.is_match(snap.split_at(volume_at.len()).1))
                .map(Deref::deref)
                .collect();
        volume_snaps.sort();

        if backup.start_snapshot.is_some() {
            // Check that the start snapshot still exists.
            let start_snapshot = volume_at.clone() + backup.start_snapshot.as_ref().unwrap();

            if volume_snaps.binary_search(&start_snapshot.deref()).is_err() {
                println!("Snapshot \"{}\" doesn't exist any more; doing full backup instead.\n",
                         start_snapshot);
                backup.start_snapshot = None;
            }
        }

        let last_snapshot: &str = volume_snaps.last().unwrap().splitn(2, '@').last().unwrap();

        if backup.start_snapshot.as_deref() != Some(last_snapshot) {
            backup.end_snapshot = Some(last_snapshot.to_owned());
        } else if backup.start_snapshot.is_some() {
            println!("Backup of \"{}\" is up to date (@{}). Skipping.\n",
                     backup.volume,
                     backup.start_snapshot.as_ref().unwrap());
            backup.end_snapshot = None;
        }
    }

    backups.into_values()
}

fn getpass(prompt: &str) -> io::Result<String> {
    let mut termios = Termios::from_fd(0).expect("failed to get termios settings");

    let old = termios.c_lflag;
    termios.c_lflag &= !ECHO;   // disable echo
    termios.c_lflag &= !ICANON; // disable line-buffering
    tcsetattr(0, TCSAFLUSH, &termios).expect("failed to set termios settings");

    printf!("{}", prompt);

    let stdin = io::stdin();
    let mut bytes = stdin.lock().bytes();
    let mut line = String::new();
    let mut utf8 = Vec::<u8>::new();
    loop {
        match bytes.next().or_else(|| Some(Err(io::Error::new(io::ErrorKind::Other, "EOF in getpass!")))) {
            Some(Ok(byte)) => {
                // 0x4 is EOT; aka ctrl-D
                if byte == 0x4 && utf8.is_empty() {
                    return Err(io::Error::new(io::ErrorKind::Other, "EOF in getpass!"));
                }

                utf8.push(byte);

                let mut valid_utf8 = false;
                if let Ok(c) = std::str::from_utf8(&utf8) {
                    if c == "\n" {
                        println!();
                        break;
                    } else {
                        valid_utf8 = true;
                        line.push_str(c);
                        printf!("*");
                    }
                }

                if valid_utf8 {
                    utf8.clear();
                }
            }
            Some(Err(e)) => {
                return Err(e);
            }
            _ => unreachable!(),
        }
    }

    termios.c_lflag = old;
    tcsetattr(0, TCSAFLUSH, &termios).expect("failed to reset termios settings");

    Ok(line)
}

fn do_backups(backups: &[Backup], path: &Path) {
    if backups.is_empty() {
        println!("Nothing to do.");
        return;
    }

    let passphrase: String;
    loop {
        let pass1 = getpass("GPG passphrase: ").unwrap();
        let pass2 = getpass("again: ").unwrap();
        if pass1 != pass2 {
            println!("Passphrases do not match.");
        } else {
            passphrase = pass1;
            break;
        }
    }

    for backup in backups {
        let z = ZSnapMgr::new(USE_SUDO);

        let snapshot = format!("{}@{}",
                               backup.volume,
                               backup.end_snapshot.as_deref().unwrap());

        println!("\nBacking up: {}", snapshot);

        z.backup(path,
                 &snapshot,
                 &passphrase,
                 backup.start_snapshot.as_deref())
         .err()
         .and_then(|e| {
             println!("failed backup of {}: {}", backup.volume, e);
             Some(())
         });
    }
}

fn interactive_backup(backups_dir: &Path) {
    let z = ZSnapMgr::new(USE_SUDO);
    let mut backups: Vec<Backup> = gather_volumes(backups_dir);
    loop {
        let mut table = Table::new(&["_", "volume", "incremental", "snapshot date"]);
        for (i, backup) in backups.iter().enumerate() {
            let start = if backup.start_snapshot.is_none() {
                "full backup".to_string()
            } else {
                backup.start_snapshot.as_ref().unwrap().clone()
            };

            table.push(vec![(i + 1).to_string(),
                            backup.volume.clone(),
                            start,
                            backup.end_snapshot.as_ref().unwrap().clone()]);
        }

        println!("Volumes to backup:\n{}", table);

        printf!(concat!("Enter a number to make changes,\n",
                         "\t'+' to add a volume,\n",
                         "\t'-' to remove one,\n",
                         "\t'd' to change all dates,\n",
                         "\tor <return> to start backup: "));

        let mut input = String::new();
        match io::stdin().read_line(&mut input) {
            Ok(bytes_read) => {
                if bytes_read == 0 {
                    panic!("EOF");
                }
                input.pop();
            }
            Err(e) => panic!(e),
        }

        if input == "+" {
            printf!("Volume: ");

            let mut vol = String::new();
            io::stdin().read_line(&mut vol).unwrap();
            vol.pop();

            let latest_snap: String = match z.get_snapshots(Some(&vol))
                   .and_then(|ref mut snaps| {
                       Ok(snaps.pop()
                            .and_then(|full_name| {
                                full_name.rsplitn(2, '@')
                                    .next()
                                    .map(|s| s.to_owned())
                            }))
                   })
            {
                Ok(Some(date)) => date,
                Ok(None) => {
                    println!("No snapshots available for that volume.\n");
                    continue;
                }
                Err(e) => {
                    println!("Error listing snapshots: {}\n", e);
                    continue;
                }
            };

            backups.push(Backup {
                filename_base: vol.replace("/", "_").to_string(),
                volume: vol.clone(),
                start_snapshot: None,
                end_snapshot: Some(latest_snap),
            });

        } else if input.starts_with('-') {

            let index: usize;

            if input.len() > 1 {
                match input[1..].parse::<usize>() {
                    Ok(n) => {
                        index = n;
                    },
                    Err(e) => {
                        println!("Invalid number: {}\n", e);
                        continue;
                    }
                }
            } else {
                printf!("Remove which one?: ");

                input.clear();
                io::stdin().read_line(&mut input).unwrap();
                input.pop();

                match input.parse::<usize>() {
                    Ok(n) => {
                        index = n;
                    },
                    Err(e) => {
                        println!("Invalid number: {}\n", e);
                        continue;
                    }
                }
            }

            if backups.len() < index || index == 0 {
                println!("Number out of range.\n");
                continue;
            }

            backups.remove(index - 1);

        } else if input.starts_with('d') || input.starts_with('D') {

            printf!("Snapshot date (yyyy-MM-dd): ");

            let mut date = String::new();
            io::stdin().read_line(&mut date).unwrap();
            date.pop();

            for backup in &mut backups {
                backup.end_snapshot = Some(date.clone());
                //TODO: check that the snapshot exists for that volume
            }

        } else if input.is_empty() {
            println!("Starting backups.\n");
            do_backups(&backups, backups_dir);
            break;
        } else {
            let index: usize;

            match input.parse::<usize>() {
                Ok(n) => {
                    index = n;
                }
                Err(e) => {
                    println!("Invalid number: {}\n", e);
                    continue;
                }
            }

            if backups.len() < index || index == 0 {
                println!("Number out of range.\n");
                continue;
            }

            let vol = &mut backups[index - 1];

            printf!("Change (I)ncremental starting snapshot, (S)napshot date: ");

            input.clear();
            io::stdin().read_line(&mut input).unwrap();
            input.pop();

            let start: bool;
            if input == "I" || input == "i" {
                start = true;
            } else if input == "S" || input == "s" {
                start = false;
            } else {
                println!("Invalid selection.\n");
                continue;
            }

            printf!("Date (yyyy-MM-dd");
            if start {
                printf!(" or 'none' for full backup): ");
            } else {
                printf!("): ");
            }

            input.clear();
            io::stdin().read_line(&mut input).unwrap();
            input.pop();

            if start {
                vol.start_snapshot = if input == "none" {
                    None
                } else {
                    Some(input)
                }
            } else {
                vol.end_snapshot = Some(input);
            }
        }
    }
}

fn snapshot_automanage() {
    let z = ZSnapMgr::new(USE_SUDO);
    z.snapshot_automanage().unwrap();
}

fn main() {
    let args: Vec<OsString> = env::args_os().collect();
    let program_name = Path::new(&args[0]);

    let command = if args.len() < 2 {
        OsStr::new("help")
    } else {
        args[1].as_ref()
    };

    match command.to_str() {
        Some("backup") => {
            if args.len() == 3 {
                interactive_backup(Path::new(&args[2]));
            } else {
                println!("usage: {} backup <backups_location>", program_name.display());
                process::exit(-1);
            }
        }
        Some("automanage") => {
            snapshot_automanage();
        }
        _ => {
            if command != "help" {
                println!("unknown command \"{}\"", command.to_str().unwrap_or("[bad utf8]"));
            }
            println!("usage: {} <backup | automanage> [options]", program_name.display());
            process::exit(-1);
        }
    }
}
<|endoftext|>


<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate docopt;

use docopt::Docopt;

use std::fs::File;
use std::thread;
use std::io::{Read, BufRead, BufReader};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;

use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
use graphmap::GraphMMap;

mod sorting;
use sorting::{SegmentList, radix_sort_32};

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      file containing list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let addresses: Vec<_> = if hosts != "" {
            let reader = BufReader::new(File::open(hosts).unwrap());
            reader.lines().take(processes as usize).map(|x| x.unwrap()).collect()
        }
        else {
            (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect()
        };

        if addresses.len() != processes as usize { panic!("only {} hosts for -p: {}", addresses.len(), processes); }

        let communicators = initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking");

        pagerank_spawn(communicators, source);
    }
    else if workers > 1 { pagerank_spawn(ProcessCommunicator::new_vector(workers), source); }
    else { pagerank_spawn(vec![ThreadCommunicator], source); };
}

fn pagerank_spawn<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank_thread(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: String, _workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();

    let nodes = GraphMMap::new(&filename).nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut going = start;

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let ranks = edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges (should only be iter 0)
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            // all inputs received for iter, commence multiplication
            while let Some((iter, _)) = notificator.next() {

                // if the very first iteration, prepare some stuff.
                // specifically, transpose edges and sort by destination.
                if iter.inner == 0 {
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a; rev = b; trn = c;
                    src = vec![0.0f32; deg.len()];
                }

                // record some timings in order to estimate per-iteration times
                if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                // prepare src for transmitting to destinations
                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                // wander through destinations
                let mut trn_slice = &trn[..];
                let mut rev_slice = &rev[..];
                while rev_slice.len() > 0 {
                    // TODO: session should flush automatically...
                    let mut session = output.session(&iter);
                    let next = std::cmp::min(200_000, rev_slice.len());
                    for &(dst, deg) in &rev_slice[..next] {
                        let mut accum = 0.0;
                        for &s in &trn_slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        trn_slice = &trn_slice[deg as usize..];
                        session.give((dst, accum));
                    }
                    rev_slice = &rev_slice[next..];
                }

                // clean out src to accumulate into
                for s in &mut src { *s = 0.0; }
            }

            // receive data from workers, accumulate in src
            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for x in data.drain_temp() {
                    src[x.0 as usize / 64] += x.1;
                }
            }
        });

        // // optionally, do process-local accumulation
        // let local_base = _workers * (index / _workers);
        // let local_index = index % _workers;
        // let mut acc = vec![0.0; (nodes / _workers) + 1];   // holds ranks
        // let ranks = ranks.unary_notify(
        //     Exchange::new(move |x: &(u32,f32)| (local_base as u64 + (x.0 as u64 % _workers as u64))),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for x in data.drain_temp() {
        //                 acc[x.0 as usize / _workers] += x.rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| ((u * _workers + local_index) as u32, f)));
        //
        //             for _ in 0..(1 + (nodes/_workers)) { acc.push(0.0); }
        //         }
        //     }
        // );

        ranks.connect_loop(cycle);

        input
    });

    // introduce edges into the computation;
    // allow mmaped file to drop
    {
        let graph = GraphMMap::new(&filename);

        let mut edges = Vec::new();
        for node in 0..graph.nodes() {
            if node % peers == index {
                for dst in graph.edges(node) {
                    edges.push((node as u32, *dst as u32));
                }
                if edges.len() > 100_000 {
                    input.send_at(0, edges.drain_temp());
                    root.step();
                }
            }
        }

        input.send_at(0, edges.drain_temp());
    }
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}
<commit_msg>unhardwired workers<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate docopt;

use docopt::Docopt;

use std::fs::File;
use std::thread;
use std::io::{Read, BufRead, BufReader};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;

use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
use graphmap::GraphMMap;

mod sorting;
use sorting::{SegmentList, radix_sort_32};

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      file containing list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let addresses: Vec<_> = if hosts != "" {
            let reader = BufReader::new(File::open(hosts).unwrap());
            reader.lines().take(processes as usize).map(|x| x.unwrap()).collect()
        }
        else {
            (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect()
        };

        if addresses.len() != processes as usize { panic!("only {} hosts for -p: {}", addresses.len(), processes); }

        let communicators = initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking");

        pagerank_spawn(communicators, source);
    }
    else if workers > 1 { pagerank_spawn(ProcessCommunicator::new_vector(workers), source); }
    else { pagerank_spawn(vec![ThreadCommunicator], source); };
}

fn pagerank_spawn<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank_thread(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: String, _workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();

    let nodes = GraphMMap::new(&filename).nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut going = start;

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let ranks = edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges (should only be iter 0)
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            // all inputs received for iter, commence multiplication
            while let Some((iter, _)) = notificator.next() {

                // if the very first iteration, prepare some stuff.
                // specifically, transpose edges and sort by destination.
                if iter.inner == 0 {
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a; rev = b; trn = c;
                    src = vec![0.0f32; deg.len()];
                }

                // record some timings in order to estimate per-iteration times
                if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                // prepare src for transmitting to destinations
                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                // wander through destinations
                let mut trn_slice = &trn[..];
                let mut rev_slice = &rev[..];
                while rev_slice.len() > 0 {
                    // TODO: session should flush automatically...
                    let mut session = output.session(&iter);
                    let next = std::cmp::min(200_000, rev_slice.len());
                    for &(dst, deg) in &rev_slice[..next] {
                        let mut accum = 0.0;
                        for &s in &trn_slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        trn_slice = &trn_slice[deg as usize..];
                        session.give((dst, accum));
                    }
                    rev_slice = &rev_slice[next..];
                }

                // clean out src to accumulate into
                for s in &mut src { *s = 0.0; }
            }

            // receive data from workers, accumulate in src
            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for x in data.drain_temp() {
                    src[x.0 as usize / peers] += x.1;
                }
            }
        });

        // // optionally, do process-local accumulation
        // let local_base = _workers * (index / _workers);
        // let local_index = index % _workers;
        // let mut acc = vec![0.0; (nodes / _workers) + 1];   // holds ranks
        // let ranks = ranks.unary_notify(
        //     Exchange::new(move |x: &(u32,f32)| (local_base as u64 + (x.0 as u64 % _workers as u64))),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for x in data.drain_temp() {
        //                 acc[x.0 as usize / _workers] += x.rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| ((u * _workers + local_index) as u32, f)));
        //
        //             for _ in 0..(1 + (nodes/_workers)) { acc.push(0.0); }
        //         }
        //     }
        // );

        ranks.connect_loop(cycle);

        input
    });

    // introduce edges into the computation;
    // allow mmaped file to drop
    {
        let graph = GraphMMap::new(&filename);

        let mut edges = Vec::new();
        for node in 0..graph.nodes() {
            if node % peers == index {
                for dst in graph.edges(node) {
                    edges.push((node as u32, *dst as u32));
                }
                if edges.len() > 100_000 {
                    input.send_at(0, edges.drain_temp());
                    root.step();
                }
            }
        }

        input.send_at(0, edges.drain_temp());
    }
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}
<|endoftext|>


<commit_before>//! Executable toolkit for working with sdf files.

extern crate docopt;
extern crate rustc_serialize;
extern crate sdf;

use std::process::exit;
use std::u32;

use docopt::Docopt;

const USAGE: &'static str = "
Read and process .sdf files.

Usage:
    sdf info <infile> \
                             [--brief]
    sdf record <infile> <index>
    sdf block <infile> \
                             <index> <block>
    sdf (-h | --help)
    sdf --version

Options:
    \
                             -h --help   Show this screen.
    --version   Show sdf-rs and sdfifc \
                             library versions.
    --brief     Only provide file information from \
                             the header, do not inspect the file itself.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_brief: bool,
    flag_version: bool,
    arg_block: usize,
    arg_index: u32,
    arg_infile: String,
    cmd_block: bool,
    cmd_info: bool,
    cmd_record: bool,
}

#[cfg_attr(test, allow(dead_code))]
fn main() {
    let args: Args = Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    if args.flag_version {
        let library_version = sdf::library_version().unwrap_or_else(|e| {
            println!("ERROR: Unable to get library version: {}", e);
            exit(1);
        });
        println!("      sdf-rs version: {}", env!("CARGO_PKG_VERSION"));
        println!("  sdfifc api version: {}.{}",
                 library_version.api_major,
                 library_version.api_minor);
        println!("sdfifc build version: {}", library_version.build_version);
        println!("    sdfifc build tag: {}", library_version.build_tag);
        exit(0);
    }

    let mut file = sdf::File::open(args.arg_infile.clone()).unwrap_or_else(|e| {
        println!("ERROR: Unable to open file: {}", e);
        exit(1)
    });
    if !args.flag_brief {
        file.reindex().unwrap_or_else(|e| {
            println!("ERROR: Unable to reindex file: {}", e);
            exit(1);
        });
    }

    if args.cmd_info {
        let info = file.info().unwrap_or_else(|e| {
            println!("ERROR: Unable to retrieve file info: {}", e);
            exit(1);
        });
        println!("      instrument: {}", info.instrument);
        println!("          serial: {}", info.serial);
        println!("           epoch: {}", info.epoch);
        println!("  group velocity: {}", info.v_group);
        println!("   sampling time: {}", info.sampling_time);
        println!("gps synchronized: {}", info.gps_synchronized);
        println!("number of facets: {}", info.num_facets);
        if args.flag_brief {
            exit(0);
        }

        let record = file.read().unwrap_or_else(|e| {
            println!("ERROR: Unable to read first point: {}", e);
            exit(1);
        });
        let start_time = record.time_external;
        println!("      start time: {}", start_time);
        file.seek(u32::MAX).unwrap_or_else(|e| {
            println!("ERROR: Unable to seek to end of file: {}", e);
            exit(1);
        });
        let record = file.read().unwrap_or_else(|e| {
            println!("ERROR: Unable to read last point: {}", e);
            exit(1);
        });
        let end_time = record.time_external;
        println!("        end time: {}", end_time);
        let npoints = file.tell().unwrap_or_else(|e| {
            println!("ERROR: Unable to get index of next record: {}", e);
            exit(1);
        });
        println!("number of points: {}", npoints);

        exit(0);
    }

    if args.cmd_record {
        file.seek(args.arg_index).unwrap_or_else(|e| {
            println!("ERROR: Unable to seek to index {}: {}", args.arg_index, e);
            exit(1);
        });
        let record = file.read().unwrap_or_else(|e| {
            println!("ERROR: Unable to read point: {}", e);
            exit(1);
        });
        println!("{}", record);
        exit(0);
    }

    if args.cmd_block {
        file.seek(args.arg_index).unwrap_or_else(|e| {
            println!("ERROR: Unable to seek to index {}: {}", args.arg_index, e);
            exit(1);
        });
        let record = file.read().unwrap_or_else(|e| {
            println!("ERROR: Unable to read point: {}", e);
            exit(1);
        });
        let ref block = record.blocks[args.arg_block];
        println!("{}", block);
        exit(0);
    }
    unreachable!()
}
<commit_msg>Slightly clean up error handling in main<commit_after>//! Executable toolkit for working with sdf files.

extern crate docopt;
extern crate rustc_serialize;
extern crate sdf;

use std::process::exit;
use std::u32;

use docopt::Docopt;

use sdf::error::SdfError;

const USAGE: &'static str = "
Read and process .sdf files.

Usage:
    sdf info <infile> \
                             [--brief]
    sdf record <infile> <index>
    sdf block <infile> \
                             <index> <block>
    sdf (-h | --help)
    sdf --version

Options:
    \
                             -h --help   Show this screen.
    --version   Show sdf-rs and sdfifc \
                             library versions.
    --brief     Only provide file information from \
                             the header, do not inspect the file itself.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_brief: bool,
    flag_version: bool,
    arg_block: usize,
    arg_index: u32,
    arg_infile: String,
    cmd_block: bool,
    cmd_info: bool,
    cmd_record: bool,
}

fn error_exit(message: &str, err: SdfError) -> ! {
    println!("ERROR: {}: {}", message, err);
    exit(1);
}

#[cfg_attr(test, allow(dead_code))]
fn main() {
    let args: Args = Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    if args.flag_version {
        let library_version = sdf::library_version().unwrap_or_else(|e| {
            error_exit("Unable to get library version", e)
        });
        println!("      sdf-rs version: {}", env!("CARGO_PKG_VERSION"));
        println!("  sdfifc api version: {}.{}",
                 library_version.api_major,
                 library_version.api_minor);
        println!("sdfifc build version: {}", library_version.build_version);
        println!("    sdfifc build tag: {}", library_version.build_tag);
        exit(0);
    }

    let mut file = sdf::File::open(args.arg_infile.clone())
                       .unwrap_or_else(|e| error_exit("Unable to open file", e));
    if !args.flag_brief {
        file.reindex().unwrap_or_else(|e| error_exit("Unable to reindex file", e));
    }

    if args.cmd_info {
        let info = file.info().unwrap_or_else(|e| error_exit("Unable to retrieve file info", e));
        println!("      instrument: {}", info.instrument);
        println!("          serial: {}", info.serial);
        println!("           epoch: {}", info.epoch);
        println!("  group velocity: {}", info.v_group);
        println!("   sampling time: {}", info.sampling_time);
        println!("gps synchronized: {}", info.gps_synchronized);
        println!("number of facets: {}", info.num_facets);
        if args.flag_brief {
            exit(0);
        }

        let record = file.read().unwrap_or_else(|e| error_exit("Unable to read first point", e));
        let start_time = record.time_external;
        println!("      start time: {}", start_time);
        file.seek(u32::MAX).unwrap_or_else(|e| error_exit("Unable to seek to end of file", e));
        let record = file.read().unwrap_or_else(|e| error_exit("Unable to read last point", e));
        let end_time = record.time_external;
        println!("        end time: {}", end_time);
        let npoints = file.tell()
                          .unwrap_or_else(|e| error_exit("Unable to get index of next record", e));
        println!("number of points: {}", npoints);

        exit(0);
    }

    if args.cmd_record {
        file.seek(args.arg_index)
            .unwrap_or_else(|e| {
                error_exit(&format!("Unable to seek to index {}", args.arg_index)[..],
                           e)
            });
        let record = file.read().unwrap_or_else(|e| error_exit("Unable to read point", e));
        println!("{}", record);
        exit(0);
    }

    if args.cmd_block {
        file.seek(args.arg_index)
            .unwrap_or_else(|e| {
                error_exit(&format!("Unable to seek to index {}", args.arg_index)[..],
                           e)
            });
        let record = file.read().unwrap_or_else(|e| error_exit("Unable to read point", e));
        let ref block = record.blocks[args.arg_block];
        println!("{}", block);
        exit(0);
    }
    unreachable!()
}
<|endoftext|>