<commit_before>mod app;
mod error;
mod exec;
mod exit_codes;
mod filesystem;
mod filetypes;
mod filter;
mod options;
mod output;
mod regex_helper;
mod walk;

use std::env;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time;

use anyhow::{anyhow, Context, Result};
use atty::Stream;
use globset::GlobBuilder;
use lscolors::LsColors;
use regex::bytes::{RegexBuilder, RegexSetBuilder};

use crate::exec::CommandTemplate;
use crate::exit_codes::ExitCode;
use crate::filetypes::FileTypes;
use crate::filter::{SizeFilter, TimeFilter};
use crate::options::Options;
use crate::regex_helper::pattern_has_uppercase_char;

// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481
#[cfg(all(not(windows), not(target_env = "musl")))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

fn run() -> Result<ExitCode> {
    let matches = app::build_app().get_matches_from(env::args_os());

    // Set the current working directory of the process
    if let Some(base_directory) = matches.value_of_os("base-directory") {
        let base_directory = Path::new(base_directory);
        if !filesystem::is_dir(base_directory) {
            return Err(anyhow!(
                "The '--base-directory' path '{}' is not a directory.",
                base_directory.to_string_lossy()
            ));
        }
        env::set_current_dir(base_directory).with_context(|| {
            format!(
                "Could not set '{}' as the current working directory.",
                base_directory.to_string_lossy()
            )
        })?;
    }

    let current_directory = Path::new(".");
    if !filesystem::is_dir(current_directory) {
        return Err(anyhow!(
            "Could not retrieve current directory (has it been deleted?)."
        ));
    }

    // Get the search pattern
    let pattern = matches
        .value_of_os("pattern")
        .map(|p| {
            p.to_str().ok_or(anyhow!(
                "The search pattern includes invalid UTF-8 sequences."
            ))
        })
        .transpose()?
        .unwrap_or("");

    // Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches
        .values_of_os("path")
        .or_else(|| matches.values_of_os("search-path"))
    {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if filesystem::is_dir(&path_buffer) {
                    Ok(path_buffer)
                } else {
                    Err(anyhow!(
                        "Search path '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ))
                }
            })
            .collect::<Result<Vec<_>>>()?,
        None => vec![current_directory.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| {
                path_buffer
                    .canonicalize()
                    .and_then(|pb| filesystem::absolute_path(pb.as_path()))
                    .unwrap()
            })
            .collect();
    }

    // Detect if the user accidentally supplied a path instead of a search pattern
    if !matches.is_present("full-path")
        && pattern.contains(std::path::MAIN_SEPARATOR)
        && filesystem::is_dir(Path::new(pattern))
    {
        return Err(anyhow!(
            "The search pattern '{pattern}' contains a path-separation character ('{sep}') \
             and will not lead to any search results.\n\n\
             If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n  \
             fd . '{pattern}'\n\n\
             Instead, if you want your pattern to match the full file path, use:\n\n  \
             fd --full-path '{pattern}'",
            pattern = pattern,
            sep = std::path::MAIN_SEPARATOR,
        ));
    }

    let pattern_regex = if matches.is_present("glob") {
        let glob = GlobBuilder::new(pattern).literal_separator(true).build()?;
        glob.regex().to_owned()
    } else if matches.is_present("fixed-strings") {
        // Treat pattern as literal string if '--fixed-strings' is used
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let interactive_terminal = atty::is(Stream::Stdout);
    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => env::var_os("NO_COLOR").is_none() && interactive_terminal,
    };

    let path_separator = matches.value_of("path-separator").map(|str| str.to_owned());

    #[cfg(windows)]
    ansi_term::enable_ansi_support().ok();

    let ls_colors = if colored_output {
        Some(LsColors::from_env().unwrap_or_default())
    } else {
        None
    };

    let command = if let Some(args) = matches.values_of("exec") {
        Some(CommandTemplate::new(args))
    } else if let Some(args) = matches.values_of("exec-batch") {
        Some(CommandTemplate::new_batch(args)?)
    } else if matches.is_present("list-details") {
        let color = matches.value_of("color").unwrap_or("auto");
        #[allow(unused)]
        let color_arg = ["--color=", color].concat();

        let cmd: Vec<&str> = if cfg!(unix) {
            if !cfg!(target_os = "macos") {
                // Non-MacOS Unix
                vec![
                    "ls",
                    "-l",               // long listing format
                    "--human-readable", // human readable file sizes
                    "--directory",      // list directories themselves, not their contents
                    &color_arg,
                ]
            } else {
                // MacOS
                use std::process::Command;

                // Use GNU ls, if available
                let gnu_ls_exists = Command::new("gls").arg("--version").status().is_ok();

                if gnu_ls_exists {
                    vec![
                        "gls",
                        "-l",               // long listing format
                        "--human-readable", // human readable file sizes
                        "--directory",      // list directories themselves, not their contents
                        &color_arg,
                    ]
                } else {
                    let mut cmd = vec![
                        "ls", // MacOS version of ls
                        "-l", // long listing format
                        "-h", // '--human-readable' is not available on MacOS, '-h' is
                        "-d", // '--directory' is not available, but '-d' is
                    ];

                    if colored_output {
                        cmd.push("-G");
                    }

                    cmd
                }
            }
        } else {
            return Err(anyhow!(
                "'fd --list-details' is not supported on this platform."
            ));
        };

        Some(CommandTemplate::new_batch(&cmd).unwrap())
    } else {
        None
    };

    let size_limits = if let Some(vs) = matches.values_of("size") {
        vs.map(|sf| {
            SizeFilter::from_string(sf).ok_or(anyhow!(
                "'{}' is not a valid size constraint. See 'fd --help'.",
                sf
            ))
        })
        .collect::<Result<Vec<_>>>()?
    } else {
        vec![]
    };

    let now = time::SystemTime::now();
    let mut time_constraints: Vec<TimeFilter> = Vec::new();
    if let Some(t) = matches.value_of("changed-within") {
        if let Some(f) = TimeFilter::after(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }
    if let Some(t) = matches.value_of("changed-before") {
        if let Some(f) = TimeFilter::before(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }

    let config = Options {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_fdignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_vcsignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        follow_links: matches.is_present("follow"),
        one_file_system: matches.is_present("one-file-system"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("max-depth")
            .or_else(|| matches.value_of("rg-depth"))
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        min_depth: matches
            .value_of("min-depth")
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        interactive_terminal,
        file_types: matches.values_of("file-type").map(|values| {
            let mut file_types = FileTypes::default();
            for value in values {
                match value {
                    "f" | "file" => file_types.files = true,
                    "d" | "directory" => file_types.directories = true,
                    "l" | "symlink" => file_types.symlinks = true,
                    "x" | "executable" => {
                        file_types.executables_only = true;
                        file_types.files = true;
                    }
                    "e" | "empty" => {
                        file_types.empty_only = true;
                    }
                    _ => unreachable!(),
                }
            }

            // If only 'empty' was specified, search for both files and directories:
            if file_types.empty_only && !(file_types.files || file_types.directories) {
                file_types.files = true;
                file_types.directories = true;
            }

            file_types
        }),
        extensions: matches
            .values_of("extension")
            .map(|exts| {
                let patterns = exts
                    .map(|e| e.trim_start_matches('.'))
                    .map(|e| format!(r".\.{}$", regex::escape(e)));
                RegexSetBuilder::new(patterns)
                    .case_insensitive(true)
                    .build()
            })
            .transpose()?,
        command: command.map(Arc::new),
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
        ignore_files: matches
            .values_of("ignore-file")
            .map(|vs| vs.map(PathBuf::from).collect())
            .unwrap_or_else(|| vec![]),
        size_constraints: size_limits,
        time_constraints,
        show_filesystem_errors: matches.is_present("show-errors"),
        path_separator,
        max_results: matches
            .value_of("max-results")
            .and_then(|n| usize::from_str_radix(n, 10).ok())
            .filter(|&n| n != 0)
            .or_else(|| {
                if matches.is_present("max-one-result") {
                    Some(1)
                } else {
                    None
                }
            }),
    };

    let re = RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
        .map_err(|e| {
            anyhow!(
                "{}\n\nNote: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression. Alternatively, you can \
                 also use the '--glob' option to match on a glob pattern.",
                e.to_string()
            )
        })?;

    walk::scan(&dir_vec, Arc::new(re), Arc::new(config))
}

fn main() {
    let result = run();
    match result {
        Ok(exit_code) => {
            process::exit(exit_code.into());
        }
        Err(err) => {
            eprintln!("[fd error]: {}", err);
            process::exit(ExitCode::GeneralError.into());
        }
    }
}
<commit_msg>Do not show output of 'gls --version'<commit_after>mod app;
mod error;
mod exec;
mod exit_codes;
mod filesystem;
mod filetypes;
mod filter;
mod options;
mod output;
mod regex_helper;
mod walk;

use std::env;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time;

use anyhow::{anyhow, Context, Result};
use atty::Stream;
use globset::GlobBuilder;
use lscolors::LsColors;
use regex::bytes::{RegexBuilder, RegexSetBuilder};

use crate::exec::CommandTemplate;
use crate::exit_codes::ExitCode;
use crate::filetypes::FileTypes;
use crate::filter::{SizeFilter, TimeFilter};
use crate::options::Options;
use crate::regex_helper::pattern_has_uppercase_char;

// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481
#[cfg(all(not(windows), not(target_env = "musl")))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

fn run() -> Result<ExitCode> {
    let matches = app::build_app().get_matches_from(env::args_os());

    // Set the current working directory of the process
    if let Some(base_directory) = matches.value_of_os("base-directory") {
        let base_directory = Path::new(base_directory);
        if !filesystem::is_dir(base_directory) {
            return Err(anyhow!(
                "The '--base-directory' path '{}' is not a directory.",
                base_directory.to_string_lossy()
            ));
        }
        env::set_current_dir(base_directory).with_context(|| {
            format!(
                "Could not set '{}' as the current working directory.",
                base_directory.to_string_lossy()
            )
        })?;
    }

    let current_directory = Path::new(".");
    if !filesystem::is_dir(current_directory) {
        return Err(anyhow!(
            "Could not retrieve current directory (has it been deleted?)."
        ));
    }

    // Get the search pattern
    let pattern = matches
        .value_of_os("pattern")
        .map(|p| {
            p.to_str().ok_or(anyhow!(
                "The search pattern includes invalid UTF-8 sequences."
            ))
        })
        .transpose()?
        .unwrap_or("");

    // Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches
        .values_of_os("path")
        .or_else(|| matches.values_of_os("search-path"))
    {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if filesystem::is_dir(&path_buffer) {
                    Ok(path_buffer)
                } else {
                    Err(anyhow!(
                        "Search path '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ))
                }
            })
            .collect::<Result<Vec<_>>>()?,
        None => vec![current_directory.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| {
                path_buffer
                    .canonicalize()
                    .and_then(|pb| filesystem::absolute_path(pb.as_path()))
                    .unwrap()
            })
            .collect();
    }

    // Detect if the user accidentally supplied a path instead of a search pattern
    if !matches.is_present("full-path")
        && pattern.contains(std::path::MAIN_SEPARATOR)
        && filesystem::is_dir(Path::new(pattern))
    {
        return Err(anyhow!(
            "The search pattern '{pattern}' contains a path-separation character ('{sep}') \
             and will not lead to any search results.\n\n\
             If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n  \
             fd . '{pattern}'\n\n\
             Instead, if you want your pattern to match the full file path, use:\n\n  \
             fd --full-path '{pattern}'",
            pattern = pattern,
            sep = std::path::MAIN_SEPARATOR,
        ));
    }

    let pattern_regex = if matches.is_present("glob") {
        let glob = GlobBuilder::new(pattern).literal_separator(true).build()?;
        glob.regex().to_owned()
    } else if matches.is_present("fixed-strings") {
        // Treat pattern as literal string if '--fixed-strings' is used
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let interactive_terminal = atty::is(Stream::Stdout);
    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => env::var_os("NO_COLOR").is_none() && interactive_terminal,
    };

    let path_separator = matches.value_of("path-separator").map(|str| str.to_owned());

    #[cfg(windows)]
    ansi_term::enable_ansi_support().ok();

    let ls_colors = if colored_output {
        Some(LsColors::from_env().unwrap_or_default())
    } else {
        None
    };

    let command = if let Some(args) = matches.values_of("exec") {
        Some(CommandTemplate::new(args))
    } else if let Some(args) = matches.values_of("exec-batch") {
        Some(CommandTemplate::new_batch(args)?)
    } else if matches.is_present("list-details") {
        let color = matches.value_of("color").unwrap_or("auto");
        #[allow(unused)]
        let color_arg = ["--color=", color].concat();

        let cmd: Vec<&str> = if cfg!(unix) {
            if !cfg!(target_os = "macos") {
                // Non-MacOS Unix
                vec![
                    "ls",
                    "-l",               // long listing format
                    "--human-readable", // human readable file sizes
                    "--directory",      // list directories themselves, not their contents
                    &color_arg,
                ]
            } else {
                // MacOS
                use std::process::{Command, Stdio};

                // Use GNU ls, if available
                let gnu_ls_exists = Command::new("gls")
                    .arg("--version")
                    .stdout(Stdio::null())
                    .stderr(Stdio::null())
                    .status()
                    .is_ok();

                if gnu_ls_exists {
                    vec![
                        "gls",
                        "-l",               // long listing format
                        "--human-readable", // human readable file sizes
                        "--directory",      // list directories themselves, not their contents
                        &color_arg,
                    ]
                } else {
                    let mut cmd = vec![
                        "ls", // MacOS version of ls
                        "-l", // long listing format
                        "-h", // '--human-readable' is not available on MacOS, '-h' is
                        "-d", // '--directory' is not available, but '-d' is
                    ];

                    if colored_output {
                        cmd.push("-G");
                    }

                    cmd
                }
            }
        } else {
            return Err(anyhow!(
                "'fd --list-details' is not supported on this platform."
            ));
        };

        Some(CommandTemplate::new_batch(&cmd).unwrap())
    } else {
        None
    };

    let size_limits = if let Some(vs) = matches.values_of("size") {
        vs.map(|sf| {
            SizeFilter::from_string(sf).ok_or(anyhow!(
                "'{}' is not a valid size constraint. See 'fd --help'.",
                sf
            ))
        })
        .collect::<Result<Vec<_>>>()?
    } else {
        vec![]
    };

    let now = time::SystemTime::now();
    let mut time_constraints: Vec<TimeFilter> = Vec::new();
    if let Some(t) = matches.value_of("changed-within") {
        if let Some(f) = TimeFilter::after(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }
    if let Some(t) = matches.value_of("changed-before") {
        if let Some(f) = TimeFilter::before(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }

    let config = Options {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_fdignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_vcsignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        follow_links: matches.is_present("follow"),
        one_file_system: matches.is_present("one-file-system"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("max-depth")
            .or_else(|| matches.value_of("rg-depth"))
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        min_depth: matches
            .value_of("min-depth")
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        interactive_terminal,
        file_types: matches.values_of("file-type").map(|values| {
            let mut file_types = FileTypes::default();
            for value in values {
                match value {
                    "f" | "file" => file_types.files = true,
                    "d" | "directory" => file_types.directories = true,
                    "l" | "symlink" => file_types.symlinks = true,
                    "x" | "executable" => {
                        file_types.executables_only = true;
                        file_types.files = true;
                    }
                    "e" | "empty" => {
                        file_types.empty_only = true;
                    }
                    _ => unreachable!(),
                }
            }

            // If only 'empty' was specified, search for both files and directories:
            if file_types.empty_only && !(file_types.files || file_types.directories) {
                file_types.files = true;
                file_types.directories = true;
            }

            file_types
        }),
        extensions: matches
            .values_of("extension")
            .map(|exts| {
                let patterns = exts
                    .map(|e| e.trim_start_matches('.'))
                    .map(|e| format!(r".\.{}$", regex::escape(e)));
                RegexSetBuilder::new(patterns)
                    .case_insensitive(true)
                    .build()
            })
            .transpose()?,
        command: command.map(Arc::new),
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
        ignore_files: matches
            .values_of("ignore-file")
            .map(|vs| vs.map(PathBuf::from).collect())
            .unwrap_or_else(|| vec![]),
        size_constraints: size_limits,
        time_constraints,
        show_filesystem_errors: matches.is_present("show-errors"),
        path_separator,
        max_results: matches
            .value_of("max-results")
            .and_then(|n| usize::from_str_radix(n, 10).ok())
            .filter(|&n| n != 0)
            .or_else(|| {
                if matches.is_present("max-one-result") {
                    Some(1)
                } else {
                    None
                }
            }),
    };

    let re = RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
        .map_err(|e| {
            anyhow!(
                "{}\n\nNote: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression. Alternatively, you can \
                 also use the '--glob' option to match on a glob pattern.",
                e.to_string()
            )
        })?;

    walk::scan(&dir_vec, Arc::new(re), Arc::new(config))
}

fn main() {
    let result = run();
    match result {
        Ok(exit_code) => {
            process::exit(exit_code.into());
        }
        Err(err) => {
            eprintln!("[fd error]: {}", err);
            process::exit(ExitCode::GeneralError.into());
        }
    }
}
<|endoftext|>


<commit_before>extern crate clap;
extern crate walkdir;
extern crate twox_hash;

use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::os::linux::fs::MetadataExt;

use clap::{App, Arg, ArgMatches};
use walkdir::WalkDir;
use twox_hash::XxHash;
use std::hash::Hasher;

#[derive(Default)]
struct Stats {
    file_count: u64,
    total_size: u64,
    duplicate_count: u64,
}

fn parse_args<'a>() -> ArgMatches<'a> {
    App::new("dupr")
        .version("0.1.0")
        .author("Mike Sampson")
        .about("Duplicate file finder")
        .arg(
            Arg::with_name("DIR")
                .help("Directory to process")
                .required(true),
        )
        .arg(Arg::with_name("noempty").short("n").long("noempty").help(
            "Exclude zero length files",
        ))
        .arg(Arg::with_name("summary").short("s").long("summary").help(
            "Print out summary information after duplicates",
        ))
        .get_matches()
}

fn collect_paths(path: &str, noempty: bool, stats: &mut Stats) -> HashMap<u64, Vec<PathBuf>> {

    let mut length_paths = HashMap::new();
    let spinner = ["|", "/", "-", "\\"];

    for (idx, entry) in WalkDir::new(path).into_iter().enumerate() {
        eprint!("\rBuilding file list {}", spinner[idx % spinner.len()]);
        let entry = match entry {
            Ok(e) => e,
            Err(err) => {
                eprintln!("dupr: {}", err);
                continue;
            }
        };

        if !entry.file_type().is_file() {
            continue;
        }

        let metadata = match entry.metadata() {
            Ok(e) => e,
            Err(err) => {
                eprintln!("dupr: {}", err);
                continue;
            }
        };

        if noempty && metadata.len() == 0 {
            continue;
        }

        stats.file_count += 1;
        stats.total_size += metadata.len();

        let path = entry.path().to_path_buf();

        length_paths
            .entry(metadata.len())
            .or_insert(Vec::new())
            .push(path);
    }
    eprint!("\r");
    length_paths
}

fn main() {
    let matches = parse_args();

    let mut stats: Stats = Default::default();

    let files = collect_paths(
        matches.value_of("DIR").unwrap(),
        matches.is_present("noempty"),
        &mut stats,
    );

    let mut len_hash_path = HashMap::new();

    // iterate over length buckets when no. files > 1
    for (len, paths) in files.iter().filter(|&(_, v)| v.len() > 1) {
        let mut inode_paths = HashMap::new();

        for path in paths {
            let metadata = match path.metadata() {
                Ok(m) => m,
                Err(err) => {
                    eprintln!("dupr: {}", err);
                    continue;
                }
            };

            let dev_inode = (metadata.st_dev(), metadata.st_ino());

            inode_paths.entry(dev_inode).or_insert(Vec::new()).push(
                path,
            );
        }

        if inode_paths.len() > 1 {
            for paths in inode_paths.values() {
                len_hash_path
                    .entry((len, hash_file(paths[0])))
                    .or_insert(Vec::new())
                    .push(paths[0]);
            }
        }
    }

    for paths in len_hash_path.values().filter(|p| p.len() > 1) {
        for path in paths {
            println!("{}", path.display());
        }
        println!();
    }

    if matches.is_present("summary") {
        println!(
            "Processed {} files with a total size of {} bytes. {} duplicates found.",
            stats.file_count,
            stats.total_size,
            stats.duplicate_count
        );
    }
}

fn hash_file<P: AsRef<Path>>(path: P) -> u64 {
    let mut file = File::open(path).unwrap();

    let mut hash = XxHash::with_seed(0);

    let mut buffer = [0; 32 * 1024];

    while let Ok(n) = file.read(&mut buffer) {
        if n == 0 {
            break;
        }
        hash.write(&buffer[0..n]);
    }

    hash.finish()
}
<commit_msg>Tweak status messages<commit_after>extern crate clap;
extern crate walkdir;
extern crate twox_hash;

use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::os::linux::fs::MetadataExt;

use clap::{App, Arg, ArgMatches};
use walkdir::WalkDir;
use twox_hash::XxHash;
use std::hash::Hasher;

#[derive(Default)]
struct Stats {
    file_count: u64,
    total_size: u64,
    duplicate_count: u64,
}

fn parse_args<'a>() -> ArgMatches<'a> {
    App::new("dupr")
        .version("0.1.0")
        .author("Mike Sampson")
        .about("Duplicate file finder")
        .arg(
            Arg::with_name("DIR")
                .help("Directory to process")
                .required(true),
        )
        .arg(Arg::with_name("noempty").short("n").long("noempty").help(
            "Exclude zero length files",
        ))
        .arg(Arg::with_name("summary").short("s").long("summary").help(
            "Print out summary information after duplicates",
        ))
        .get_matches()
}

fn collect_paths(path: &str, noempty: bool, stats: &mut Stats) -> HashMap<u64, Vec<PathBuf>> {

    let mut length_paths = HashMap::new();
    let spinner = ["|", "/", "-", "\\"];

    for (idx, entry) in WalkDir::new(path).into_iter().enumerate() {
        eprint!("\rBuilding file list {} ", spinner[idx % spinner.len()]);
        let entry = match entry {
            Ok(e) => e,
            Err(err) => {
                eprintln!("dupr: {}", err);
                continue;
            }
        };

        if !entry.file_type().is_file() {
            continue;
        }

        let metadata = match entry.metadata() {
            Ok(e) => e,
            Err(err) => {
                eprintln!("dupr: {}", err);
                continue;
            }
        };

        if noempty && metadata.len() == 0 {
            continue;
        }

        stats.file_count += 1;
        stats.total_size += metadata.len();

        let path = entry.path().to_path_buf();

        length_paths
            .entry(metadata.len())
            .or_insert(Vec::new())
            .push(path);
    }
    eprint!("\r{:40}", " ");

    length_paths
}

fn main() {
    let matches = parse_args();

    let mut stats: Stats = Default::default();

    let files = collect_paths(
        matches.value_of("DIR").unwrap(),
        matches.is_present("noempty"),
        &mut stats,
    );

    let file_count = files.len();

    let mut len_hash_path = HashMap::new();

    for (idx, (len, paths)) in files.iter().enumerate() {

        eprint!(
            "\rProgress [{}/{}] {:.0}%",
            idx,
            file_count,
            100.0 * idx as f64 / file_count as f64
        );

        if paths.len() < 2 {
            continue;
        }

        let mut inode_paths = HashMap::new();

        for path in paths {
            let metadata = match path.metadata() {
                Ok(m) => m,
                Err(err) => {
                    eprintln!("dupr: {}", err);
                    continue;
                }
            };

            let dev_inode = (metadata.st_dev(), metadata.st_ino());

            inode_paths.entry(dev_inode).or_insert(Vec::new()).push(
                path,
            );
        }

        if inode_paths.len() > 1 {
            for paths in inode_paths.values() {
                len_hash_path
                    .entry((len, hash_file(paths[0])))
                    .or_insert(Vec::new())
                    .push(paths[0]);
            }
        }
    }
    eprint!("\r{:40}", " ");

    for paths in len_hash_path.values().filter(|p| p.len() > 1) {
        for path in paths {
            println!("{}", path.display());
        }
        println!();
    }

    if matches.is_present("summary") {
        println!(
            "Processed {} files with a total size of {} bytes. {} duplicates found.",
            stats.file_count,
            stats.total_size,
            stats.duplicate_count
        );
    }
}

fn hash_file<P: AsRef<Path>>(path: P) -> u64 {
    let mut file = File::open(path).unwrap();

    let mut hash = XxHash::with_seed(0);

    let mut buffer = [0; 32 * 1024];

    while let Ok(n) = file.read(&mut buffer) {
        if n == 0 {
            break;
        }
        hash.write(&buffer[0..n]);
    }

    hash.finish()
}
<|endoftext|>


<commit_before>extern crate hyper;
extern crate getopts;
extern crate serde;
extern crate serde_json;
use std::io::Read;
use std::env;
use hyper::Client;
use std::collections::btree_map::BTreeMap;
use serde_json::Value;
use getopts::Options;
const KEY: &'static str = "1e940957819058fe3ec7c59d43c09504b400110db7faa0509";
const TKEY: &'static str = "e415520c671c26518df498d8f4736cac";
struct Dict<'a> {
    key: &'a str,
    tkey: &'a str,
    httpclient: Client
}
impl <'a> Dict <'a> {
    fn new<'c> (key: &'c str,tkey: &'c str) -> Dict<'c> {
        Dict {key: key, httpclient: Client::new(),tkey:tkey}
    }
    fn get_definitions(&self,word: &str) -> Result<Vec<String>,&str> {
        let url = format!("http://api.wordnik.com/v4/word.json/{word}/definitions?api_key={key}",key = self.key, word = word);
        let mut resp = self.httpclient.get(&url).send().expect("Failed to send request");
        let mut body = String::new();
        resp.read_to_string(&mut body).unwrap();
        let word: Value = serde_json::from_str(&body).unwrap();
        let decoded = word.as_array().unwrap().clone();
        if decoded.len()==0 {
            return Err("No definitions found")
        }
        let mut definitions = Vec::new();
        for word in &decoded {
            definitions.push(word.as_object().unwrap().get("text").unwrap().as_string().unwrap().to_string());
        }
        Ok(definitions)
    }
    fn get_thesaurus(&self,word: &str) -> Result<BTreeMap<String,Value>,&str> {
        let url = format!("http://words.bighugelabs.com/api/2/{key}/{word}/json", key = self.tkey, word = word);
        let mut resp = self.httpclient.get(&url).send().expect("Failed to get thesaurus");
        let mut body = String::new();
        resp.read_to_string(&mut body).unwrap();
        if &body == "" {
            return Err("No synonyms found")
        }
        let mut synonyms: BTreeMap<String,Value> = serde_json::from_str(&body).unwrap();
        synonyms = synonyms.clone();
        Ok(synonyms)
    }
}

fn main() {
    let wordclient: Dict = Dict::new(KEY,TKEY);
    let args: Vec<String> = env::args().collect();
    let mut parser = Options::new();
    parser.optflag("t","thesaurus","use thesaurus");
    let opts = parser.parse(&args[1..]).unwrap();
    for word in &opts.free {
        println!("{}:",word.to_uppercase());
        let definitions = wordclient.get_definitions(word);
        match definitions {
            Ok(defs) => {println!("{}",defs[0]);},
            Err(err) => {println!("{}",err);}
        };
        if opts.opt_present("t") {
            let syns = wordclient.get_thesaurus(&word);
            match syns {
                Ok(map) => {
                   if map.get("noun").is_some() {
                        let mut nounstr = String::new();
                        let mut first: bool = true;
                        for noun in map.get("noun").unwrap().as_object().unwrap().get("syn").unwrap().as_array().unwrap() {
                            if first {
                                nounstr = nounstr + &(noun.as_string().unwrap());
                                first = false;
                            }
                            else {
                                nounstr = nounstr + ", " + &(noun.as_string().unwrap());
                            }
                   }
                       println!("NOUNS:\n{}",nounstr);
                   }
                },
                Err(err) => {println!("{}",err);
            }
        }}

    }
}
<commit_msg>Added experimental download feature<commit_after>#![allow(dead_code)]
#![allow(unused_must_use)]
extern crate hyper;
extern crate getopts;
extern crate serde;
extern crate serde_json;
use std::io::{Read,Write};
use std::env;
use hyper::Client;
use std::collections::btree_map::BTreeMap;
use serde_json::Value;
use getopts::Options;
use std::process;
use std::fs::File;
const KEY: &'static str = "1e940957819058fe3ec7c59d43c09504b400110db7faa0509";
const TKEY: &'static str = "e415520c671c26518df498d8f4736cac";

struct Dict<'a> {
    key: &'a str,
    tkey: &'a str,
    httpclient: Client
}
impl <'a> Dict <'a> {
    fn new<'c> (key: &'c str,tkey: &'c str) -> Dict<'c> {
        Dict {key: key, httpclient: Client::new(),tkey:tkey}
    }
    fn get_definitions(&self,word: &str) -> Result<Vec<String>,&str> {
        let url = format!("http://api.wordnik.com/v4/word.json/{word}/definitions?api_key={key}",key = self.key, word = word);
        let mut resp = self.httpclient.get(&url).send().expect("Failed to send request");
        let mut body = String::new();
        resp.read_to_string(&mut body).unwrap();
        let word: Value = serde_json::from_str(&body).unwrap();
        let decoded = word.as_array().unwrap().clone();
        if decoded.len()==0 {
            return Err("No definitions found")
        }
        let mut definitions = Vec::new();
        for word in &decoded {
            definitions.push(word.as_object().unwrap().get("text").unwrap().as_string().unwrap().to_string());
        }
        Ok(definitions)
    }
    fn get_thesaurus(&self,word: &str) -> Result<BTreeMap<String,Value>,&str> {
        let url = format!("http://words.bighugelabs.com/api/2/{key}/{word}/json", key = self.tkey, word = word);
        let mut resp = self.httpclient.get(&url).send().expect("Failed to get thesaurus");
        let mut body = String::new();
        resp.read_to_string(&mut body).unwrap();
        if &body == "" {
            return Err("No synonyms found")
        }
        let mut synonyms: BTreeMap<String,Value> = serde_json::from_str(&body).unwrap();
        synonyms = synonyms.clone();
        Ok(synonyms)
    }

    fn download_audio(&self,word: &str) -> Result<(),std::io::Error> { //downloads the audio to /tmp/audio.mp3 for playback
        let url = format!("http://api.wordnik.com:80/v4/word.json/{word}/audio?api_key={key}", word = word, key = self.key);
        let audio = self.httpclient.get(&url).send();
        let mut audio = match audio {
            Ok(val) => val,
            Err(_) => {println!("Could not download audio, exiting."); process::exit(1);}
        };
        let mut body = String::new();
        audio.read_to_string(&mut body).unwrap();
        let decoded: Vec<BTreeMap<String,Value>> = serde_json::from_str(&body).unwrap();
        let audiourl = decoded[0].get("fileUrl").unwrap().as_string().unwrap();
        let mut audiofile = try!(File::create("/tmp/audio.mp3"));
        let mut audio = match self.httpclient.get(audiourl).send() {
            Ok(val) => val,
            Err(_) => {println!("Could not download audio, exiting."); process::exit(1);}
            };
        let mut audiob = vec![0; 10];
        audio.read_to_end(&mut audiob).unwrap();
        audiofile.write_all(&audiob);
        Ok(())
    }
}

fn program_exists(program_name: &str) -> bool {
    let output = process::Command::new("whereis").arg(program_name).output().unwrap_or_else(|err| { panic!("{}",err); }).stdout;
    let stdout = String::from_utf8(output).unwrap();
    let stdout = stdout.trim();
    if stdout==format!("{}:",program_name) {
        false
    }
    else {
        true
    }
}

fn main() {
    let wordclient: Dict = Dict::new(KEY,TKEY);
    let args: Vec<String> = env::args().collect();
    let mut parser = Options::new();
    parser.optflag("t","thesaurus","use thesaurus");
    let opts = parser.parse(&args[1..]).unwrap();
    for word in &opts.free {
        println!("{}:",word.to_uppercase());
        let definitions = wordclient.get_definitions(word);
        match definitions {
            Ok(defs) => {println!("{}",defs[0]);},
            Err(err) => {println!("{}",err);}
        };
        if opts.opt_present("t") {
            let syns = wordclient.get_thesaurus(&word);
            match syns {
                Ok(map) => {
                   if map.get("noun").is_some() {
                        let mut nounstr = String::new();
                        let mut first: bool = true;
                        for noun in map.get("noun").unwrap().as_object().unwrap().get("syn").unwrap().as_array().unwrap() {
                            if first {
                                nounstr = nounstr + &(noun.as_string().unwrap());
                                first = false;
                            }
                            else {
                                nounstr = nounstr + ", " + &(noun.as_string().unwrap());
                            }
                   }
                       println!("NOUNS:\n{}",nounstr);
                   }
                },
                Err(err) => {println!("{}",err);
            }
        }
        }
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate nom;

mod lib;

fn main() {
    println!("Not implemented");
}
<commit_msg>Temporary stop using lib module<commit_after>fn main() {
    println!("Not implemented");
}
<|endoftext|>


<commit_before>extern crate rustc_serialize;
#[macro_use]
extern crate aws_abacus;
#[macro_use]
extern crate log;
extern crate walkdir;
extern crate env_logger;
extern crate docopt;
use docopt::Docopt;
use std::path;
use aws_abacus::elb_log_files;
extern crate chrono;
use chrono::{DateTime, UTC};
use aws_abacus::elb_log_files::ParsingResult;
use std::collections::HashMap;
extern crate urlparse;
use urlparse::urlparse;

fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.decode())
                            .unwrap_or_else(|e| e.exit());

    let log_location = &path::Path::new(&args.arg_log_location);
    debug!("Running summary on {}.", log_location.to_str().unwrap());

    let start: Option<DateTime<UTC>> = if args.flag_benchmark {
        Some(UTC::now())
    } else {
        None
    };

    let mut number_of_files = 0;
    let mut number_of_records = 0;
    let mut filenames = Vec::new();
    let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();
    match elb_log_files::file_list(log_location, &mut filenames) {
        Ok(num_files) => {
            number_of_files = num_files;
            debug!("Found {} files.", number_of_files);
            number_of_records = elb_log_files::process_files(&filenames, &mut |parsing_result: ParsingResult| {
                parsing_result_handler(parsing_result, &mut agg);
            });
            debug!("Processed {} records in {} files.", number_of_records, num_files);
        },
        Err(e) => {
            println!("ERROR: {}", e);
        },
    };

    for (aggregate, total) in &agg {
        println!("{},{},{},{}", aggregate.system_name, aggregate.day, aggregate.client_address, total);
    };

    match start {
        Some(s) => {
            let end = UTC::now();
            let time = end - s;
            println!("Processed {} files having {} records in {} milliseconds and produced {} aggregates.",
                number_of_files,
                number_of_records,
                time.num_milliseconds(),
                agg.len()
            );
        },
        None => {},
    };
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct AggregateELBRecord {
    day: String,
    client_address: String,
    system_name: String,
    // public_id: u64,
}

fn parsing_result_handler(parsing_result: ParsingResult, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    match parsing_result {
        Ok(elb_record) => {
            let url = urlparse(&elb_record.request_url);
            let system = match url.get_parsed_query() {
                Some(query_map) => {
                    match query_map.get("system") {
                        Some(systems) => {
                            systems[0].clone()
                        },

                        None => {
                            "".to_owned()
                        }
                    }
                },

                None => {
                    "".to_owned()
                }
            };

            let aer = AggregateELBRecord {
                day: elb_record.timestamp.format("%Y-%m-%d").to_string(),
                client_address: elb_record.client_address.ip().to_string(),
                system_name: system,
                // public_id: elb_record.sent_bytes
            };
            aggregate_record(aer, aggregation);
        },

        Err(errors) => {

        }
    }
}

fn aggregate_record(aggregate_record: AggregateELBRecord, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    let total = aggregation.entry(aggregate_record).or_insert(0);
    *total += 1;
}

const USAGE: &'static str = "
aws-abacus

Usage:
  aws-abacus <log-location>
  aws-abacus (-d | --debug | -b | --benchmark) <log-location>
  aws-abacus (-h | --help)

Options:
  -h --help     Show this screen.
  -d --debug    Turn on debug output
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_log_location: String,
    flag_debug: bool,
    flag_benchmark: bool,
}

#[cfg(test)]
mod tests {

    use std::collections::HashMap;
    use ::AggregateELBRecord;
    use ::aggregate_record;

    const TEST_RECORD: &'static str = "2015-08-15T23:43:05.302180Z elb-name 172.16.1.6:54814 \
    172.16.1.5:9000 0.000039 0.145507 0.00003 200 200 0 7582 \
    \"GET http://some.domain.com:80/path0/path1?param0=p0&param1=p1 HTTP/1.1\"\
    ";

    #[test]
	fn inserting_two_records_with_different_values_creates_two_entries_each_recorded_once() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8880
        };

        let ar1 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg.len(), 2);
        for (_, total) in agg {
            assert_eq!(total, 1)
        }
	}

    #[test]
	fn inserting_two_records_with_the_same_values_increases_the_total_correctly() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        let ar1 = ar0.clone();
        let ar3 = ar0.clone();

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg[&ar3], 2);
	}
}
<commit_msg>Factored out the parsing of the system name to its own function.  Experiment with how to handle errors during parsing of the URL.<commit_after>extern crate rustc_serialize;
#[macro_use]
extern crate aws_abacus;
#[macro_use]
extern crate log;
extern crate walkdir;
extern crate env_logger;
extern crate docopt;
use docopt::Docopt;
use std::path;
use aws_abacus::elb_log_files;
extern crate chrono;
use chrono::{DateTime, UTC};
use aws_abacus::elb_log_files::ParsingResult;
use std::collections::HashMap;
extern crate urlparse;
use urlparse::{Url, urlparse};

fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.decode())
                            .unwrap_or_else(|e| e.exit());

    let log_location = &path::Path::new(&args.arg_log_location);
    debug!("Running summary on {}.", log_location.to_str().unwrap());

    let start: Option<DateTime<UTC>> = if args.flag_benchmark {
        Some(UTC::now())
    } else {
        None
    };

    let mut number_of_files = 0;
    let mut number_of_records = 0;
    let mut filenames = Vec::new();
    let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();
    match elb_log_files::file_list(log_location, &mut filenames) {
        Ok(num_files) => {
            number_of_files = num_files;
            debug!("Found {} files.", number_of_files);
            number_of_records = elb_log_files::process_files(&filenames, &mut |parsing_result: ParsingResult| {
                parsing_result_handler(parsing_result, &mut agg);
            });
            debug!("Processed {} records in {} files.", number_of_records, num_files);
        },
        Err(e) => {
            println!("ERROR: {}", e);
        },
    };

    for (aggregate, total) in &agg {
        println!("{},{},{},{}", aggregate.system_name, aggregate.day, aggregate.client_address, total);
    };

    match start {
        Some(s) => {
            let end = UTC::now();
            let time = end - s;
            println!("Processed {} files having {} records in {} milliseconds and produced {} aggregates.",
                number_of_files,
                number_of_records,
                time.num_milliseconds(),
                agg.len()
            );
        },
        None => {},
    };
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct AggregateELBRecord {
    day: String,
    client_address: String,
    system_name: String,
    // public_id: u64,
}

fn parsing_result_handler(parsing_result: ParsingResult, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    match parsing_result {
        Ok(elb_record) => {
            let url = urlparse(&elb_record.request_url);
            let aer = AggregateELBRecord {
                day: elb_record.timestamp.format("%Y-%m-%d").to_string(),
                client_address: elb_record.client_address.ip().to_string(),
                system_name: parse_system_name(&url).unwrap_or("UNDEFINED_SYSTEM".to_owned()),
                // public_id: elb_record.sent_bytes
            };
            aggregate_record(aer, aggregation);
        },

        Err(errors) => {

        }
    }
}

fn parse_system_name(url: &Url) -> Option<String> {
    match url.get_parsed_query() {
        Some(query_map) => {
            match query_map.get("system") {
                Some(systems) => {
                    Some(systems[0].clone())
                },

                None => {
                    None
                }
            }
        },

        None => {
            None
        }
    }
}

fn aggregate_record(aggregate_record: AggregateELBRecord, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    let total = aggregation.entry(aggregate_record).or_insert(0);
    *total += 1;
}

const USAGE: &'static str = "
aws-abacus

Usage:
  aws-abacus <log-location>
  aws-abacus (-d | --debug | -b | --benchmark) <log-location>
  aws-abacus (-h | --help)

Options:
  -h --help     Show this screen.
  -d --debug    Turn on debug output
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_log_location: String,
    flag_debug: bool,
    flag_benchmark: bool,
}

#[cfg(test)]
mod tests {

    use std::collections::HashMap;
    use ::AggregateELBRecord;
    use ::aggregate_record;

    const TEST_RECORD: &'static str = "2015-08-15T23:43:05.302180Z elb-name 172.16.1.6:54814 \
    172.16.1.5:9000 0.000039 0.145507 0.00003 200 200 0 7582 \
    \"GET http://some.domain.com:80/path0/path1?param0=p0&param1=p1 HTTP/1.1\"\
    ";

    #[test]
	fn inserting_two_records_with_different_values_creates_two_entries_each_recorded_once() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8880
        };

        let ar1 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg.len(), 2);
        for (_, total) in agg {
            assert_eq!(total, 1)
        }
	}

    #[test]
	fn inserting_two_records_with_the_same_values_increases_the_total_correctly() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        let ar1 = ar0.clone();
        let ar3 = ar0.clone();

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg[&ar3], 2);
	}
}
<|endoftext|>


<commit_before>extern crate rgmk;
extern crate clap;

use clap::{App, SubCommand, AppSettings, Arg};
use std::io::{self, stdin, stdout, stderr, BufReader};
use std::io::prelude::*;
use rgmk::GameData;
use std::path::Path;

fn main() {
    std::process::exit(run());
}

fn run() -> i32 {
    let strings_subcommand = SubCommand::with_name("strings")
                                 .about("Manipulate strings")
                                 .setting(AppSettings::SubcommandRequiredElseHelp)
                                 .setting(AppSettings::ArgRequiredElseHelp)
                                 .subcommand(SubCommand::with_name("dump")
                                                 .about("Dump strings.")
                                                 .args_from_usage("[OUTPUT_FILE] 'The file to \
                                                                   write the strings to. By \
                                                                   default, standard output.'"))
                                 .subcommand(SubCommand::with_name("repack")
                                                 .about("Repack strings")
                                                 .args_from_usage("[INPUT_FILE] 'The file \
                                                                   containing the strings to \
                                                                   repack. By default, standard \
                                                                   input.'"));
    let textures_subcommand = SubCommand::with_name("textures")
                                  .about("Manipulate textures")
                                  .setting(AppSettings::SubcommandRequiredElseHelp)
                                  .subcommand(SubCommand::with_name("dump").about("Dump textures"))
                                  .subcommand(SubCommand::with_name("repack")
                                                  .about("Repack textures"));
    let matches = App::new("gmktool")
                      .setting(AppSettings::SubcommandRequiredElseHelp)
                      .version(env!("CARGO_PKG_VERSION"))
                      .about("Tool for manipulating Game Maker Stdudio data files")
                      .arg(Arg::with_name("BACKUP")
                               .short("b")
                               .long("backup")
                               .help("Create a backup (.bk) before writing to the data file."))
                      .args_from_usage("<DATA_FILE> 'The game maker data file to operate on'")
                      .subcommand(strings_subcommand)
                      .subcommand(textures_subcommand)
                      .get_matches();
    let data_file_path = matches.value_of("DATA_FILE").unwrap();
    let backup = matches.is_present("BACKUP");
    let backup_path = format!("{}.bk", data_file_path);
    let backup_path = matches.value_of("BACKUP_FILE").unwrap_or(&backup_path);
    let mut game_data = match GameData::from_file(data_file_path) {
        Ok(g) => g,
        Err(e) => {
            writeln!(stderr(),
                     "Failed to load game data for {:?}: {}",
                     data_file_path,
                     e)
                .unwrap();
            return 1;
        }
    };
    if let Some(matches) = matches.subcommand_matches("strings") {
        if let Some(matches) = matches.subcommand_matches("dump") {
            match matches.value_of("OUTPUT_FILE") {
                Some(path) => {
                    let mut file = match std::fs::File::create(path) {
                        Ok(f) => f,
                        Err(e) => {
                            writeln!(stderr(), "Failed to create output file: {}", e).unwrap();
                            return 1;
                        }
                    };
                    if let Err(e) = dump_strings(&game_data, &mut file) {
                        writeln!(stderr(), "Failed to dump strings: {}", e).unwrap();
                    }
                }
                None => {
                    let stdout = stdout();
                    let mut lock = stdout.lock();
                    if let Err(e) = dump_strings(&game_data, &mut lock) {
                        writeln!(stderr(), "Failed to dump strings: {}", e).unwrap();
                    }
                }
            }
        } else if let Some(matches) = matches.subcommand_matches("repack") {
            if backup {
                if let Err(e) = std::fs::copy(data_file_path, backup_path) {
                    writeln!(stderr(), "Failed to create backup: {}", e).unwrap();
                }
            }
            match matches.value_of("INPUT_FILE") {
                Some(path) => {
                    let file = match std::fs::File::open(path) {
                        Ok(f) => f,
                        Err(e) => {
                            writeln!(stderr(), "Failed to open input file: {}", e).unwrap();
                            return 1;
                        }
                    };
                    if let Err(e) = repack_strings(&mut game_data,
                                                   &mut BufReader::new(file),
                                                   data_file_path) {
                        writeln!(stderr(), "Failed to repack strings: {}", e).unwrap();
                    }
                }
                None => {
                    if let Err(e) = repack_strings(&mut game_data,
                                                   &mut BufReader::new(stdin()),
                                                   data_file_path) {
                        writeln!(stderr(), "Failed to repack strings: {}", e).unwrap();
                    }
                }
            }
        }
    } else if let Some(matches) = matches.subcommand_matches("textures") {
        if let Some(_) = matches.subcommand_matches("dump") {
            if let Err(e) = dump_textures(&game_data) {
                writeln!(stderr(), "Failed to dump textures: {}", e).unwrap();
                return 1;
            }
        } else if let Some(_) = matches.subcommand_matches("repack") {
            if let Err(e) = repack_textures(&mut game_data, data_file_path) {
                writeln!(stderr(), "Failed to repack textures: {}", e).unwrap();
            }
        }
    }
    0
}

fn dump_strings<W: Write>(game_data: &GameData, writer: &mut W) -> io::Result<()> {
    for s in &game_data.strings.strings {
        try!(writeln!(writer, "{}", s));
    }
    Ok(())
}

fn repack_strings<R: BufRead, P: AsRef<Path>>(game_data: &mut GameData,
                                              reader: &mut R,
                                              path: P)
                                              -> io::Result<()> {
    for (s, l) in game_data.strings.strings.iter_mut().zip(reader.lines()) {
        *s = try!(l);
    }
    game_data.save_to_file(path)
}

fn dump_textures(game_data: &GameData) -> io::Result<()> {
    for (i, t) in game_data.textures.textures.iter().enumerate() {
        let path = format!("{}.png", i);
        let mut f = try!(std::fs::File::create(path));
        try!(f.write_all(&t.png_data));
    }
    Ok(())
}

fn repack_textures<P: AsRef<Path>>(game_data: &mut GameData, path: P) -> io::Result<()> {
    for (i, t) in game_data.textures.textures.iter_mut().enumerate() {
        let path = format!("{}.png", i);
        let mut f = try!(std::fs::File::open(path));
        let mut png_data = Vec::new();
        try!(f.read_to_end(&mut png_data));
        (*t).png_data = png_data;
    }
    game_data.save_to_file(path)
}
<commit_msg>Remove the concept of "dump" to "repack'<commit_after>extern crate rgmk;
extern crate clap;

use clap::{App, SubCommand, AppSettings, Arg};
use std::io::{self, stdin, stdout, stderr, BufReader};
use std::io::prelude::*;
use rgmk::GameData;
use std::path::Path;

fn main() {
    std::process::exit(run());
}

fn run() -> i32 {
    let strings_subcommand = SubCommand::with_name("strings")
                                 .about("Manipulate strings")
                                 .setting(AppSettings::SubcommandRequiredElseHelp)
                                 .setting(AppSettings::ArgRequiredElseHelp)
                                 .subcommand(SubCommand::with_name("unpack")
                                                 .about("Unpack strings.")
                                                 .args_from_usage("[OUTPUT_FILE] 'The file to \
                                                                   write the strings to. By \
                                                                   default, standard output.'"))
                                 .subcommand(SubCommand::with_name("repack")
                                                 .about("Repack strings")
                                                 .args_from_usage("[INPUT_FILE] 'The file \
                                                                   containing the strings to \
                                                                   repack. By default, standard \
                                                                   input.'"));
    let textures_subcommand = SubCommand::with_name("textures")
                                  .about("Manipulate textures")
                                  .setting(AppSettings::SubcommandRequiredElseHelp)
                                  .subcommand(SubCommand::with_name("unpack")
                                                  .about("Unpack textures"))
                                  .subcommand(SubCommand::with_name("repack")
                                                  .about("Repack textures"));
    let matches = App::new("gmktool")
                      .setting(AppSettings::SubcommandRequiredElseHelp)
                      .version(env!("CARGO_PKG_VERSION"))
                      .about("Tool for manipulating Game Maker Stdudio data files")
                      .arg(Arg::with_name("BACKUP")
                               .short("b")
                               .long("backup")
                               .help("Create a backup (.bk) before writing to the data file."))
                      .args_from_usage("<DATA_FILE> 'The game maker data file to operate on'")
                      .subcommand(strings_subcommand)
                      .subcommand(textures_subcommand)
                      .get_matches();
    let data_file_path = matches.value_of("DATA_FILE").unwrap();
    let backup = matches.is_present("BACKUP");
    let backup_path = format!("{}.bk", data_file_path);
    let backup_path = matches.value_of("BACKUP_FILE").unwrap_or(&backup_path);
    let mut game_data = match GameData::from_file(data_file_path) {
        Ok(g) => g,
        Err(e) => {
            writeln!(stderr(),
                     "Failed to load game data for {:?}: {}",
                     data_file_path,
                     e)
                .unwrap();
            return 1;
        }
    };
    if let Some(matches) = matches.subcommand_matches("strings") {
        if let Some(matches) = matches.subcommand_matches("unpack") {
            match matches.value_of("OUTPUT_FILE") {
                Some(path) => {
                    let mut file = match std::fs::File::create(path) {
                        Ok(f) => f,
                        Err(e) => {
                            writeln!(stderr(), "Failed to create output file: {}", e).unwrap();
                            return 1;
                        }
                    };
                    if let Err(e) = unpack_strings(&game_data, &mut file) {
                        writeln!(stderr(), "Failed to unpack strings: {}", e).unwrap();
                    }
                }
                None => {
                    let stdout = stdout();
                    let mut lock = stdout.lock();
                    if let Err(e) = unpack_strings(&game_data, &mut lock) {
                        writeln!(stderr(), "Failed to unpack strings: {}", e).unwrap();
                    }
                }
            }
        } else if let Some(matches) = matches.subcommand_matches("repack") {
            if backup {
                if let Err(e) = std::fs::copy(data_file_path, backup_path) {
                    writeln!(stderr(), "Failed to create backup: {}", e).unwrap();
                }
            }
            match matches.value_of("INPUT_FILE") {
                Some(path) => {
                    let file = match std::fs::File::open(path) {
                        Ok(f) => f,
                        Err(e) => {
                            writeln!(stderr(), "Failed to open input file: {}", e).unwrap();
                            return 1;
                        }
                    };
                    if let Err(e) = repack_strings(&mut game_data,
                                                   &mut BufReader::new(file),
                                                   data_file_path) {
                        writeln!(stderr(), "Failed to repack strings: {}", e).unwrap();
                    }
                }
                None => {
                    if let Err(e) = repack_strings(&mut game_data,
                                                   &mut BufReader::new(stdin()),
                                                   data_file_path) {
                        writeln!(stderr(), "Failed to repack strings: {}", e).unwrap();
                    }
                }
            }
        }
    } else if let Some(matches) = matches.subcommand_matches("textures") {
        if let Some(_) = matches.subcommand_matches("unpack") {
            if let Err(e) = unpack_textures(&game_data) {
                writeln!(stderr(), "Failed to unpack textures: {}", e).unwrap();
                return 1;
            }
        } else if let Some(_) = matches.subcommand_matches("repack") {
            if let Err(e) = repack_textures(&mut game_data, data_file_path) {
                writeln!(stderr(), "Failed to repack textures: {}", e).unwrap();
            }
        }
    }
    0
}

fn unpack_strings<W: Write>(game_data: &GameData, writer: &mut W) -> io::Result<()> {
    for s in &game_data.strings.strings {
        try!(writeln!(writer, "{}", s));
    }
    Ok(())
}

fn repack_strings<R: BufRead, P: AsRef<Path>>(game_data: &mut GameData,
                                              reader: &mut R,
                                              path: P)
                                              -> io::Result<()> {
    for (s, l) in game_data.strings.strings.iter_mut().zip(reader.lines()) {
        *s = try!(l);
    }
    game_data.save_to_file(path)
}

fn unpack_textures(game_data: &GameData) -> io::Result<()> {
    for (i, t) in game_data.textures.textures.iter().enumerate() {
        let path = format!("{}.png", i);
        let mut f = try!(std::fs::File::create(path));
        try!(f.write_all(&t.png_data));
    }
    Ok(())
}

fn repack_textures<P: AsRef<Path>>(game_data: &mut GameData, path: P) -> io::Result<()> {
    for (i, t) in game_data.textures.textures.iter_mut().enumerate() {
        let path = format!("{}.png", i);
        let mut f = try!(std::fs::File::open(path));
        let mut png_data = Vec::new();
        try!(f.read_to_end(&mut png_data));
        (*t).png_data = png_data;
    }
    game_data.save_to_file(path)
}
<|endoftext|>


<commit_before>const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const AUTHORS: &'static str = env!("CARGO_PKG_AUTHORS");

static FORMAT: &'static str = "{request-time}: {method} {uri} {status} {response-time}";

#[macro_use]
extern crate clap;
use clap::{Arg, ArgMatches, App, AppSettings, SubCommand};

extern crate toml;
extern crate rustc_serialize;
extern crate regex;

extern crate iron;
use iron::prelude::*;
use iron::Handler;
use iron::method::Method;
use std::str::FromStr;

extern crate router;
use router::Router;

extern crate staticfile;
use staticfile::Static;

extern crate mount;
use mount::Mount;

#[macro_use]
extern crate log;
extern crate simplelog;
use simplelog::{SimpleLogger, LogLevelFilter};

extern crate logger;
use logger::Logger;
use logger::Format;

extern crate notify;

use notify::{RecommendedWatcher, Watcher, RecursiveMode};
use std::sync::mpsc::{channel, Receiver};
use std::time::Duration;

mod config;
use config::{Config, ServerConfig, RequestConfig, ResponseConfig};
use std::collections::HashMap;
use std::ascii::AsciiExt;
mod error;
use error::HaxoniteError;
mod handlers;
use handlers::*;
mod utils;

use std::path::Path;

fn main() {
    let _ = SimpleLogger::init(LogLevelFilter::Info, simplelog::Config::default());

    match run() {
        Ok(_) => {}
        Err(err) => {
            error!("{}!", err);
            std::process::exit(-1);
        }
    }
}

fn run() -> Result<(), HaxoniteError> {
    let matches = define_command_line_options().get_matches();
    if let Some(matches) = matches.subcommand_matches("new") {
        let generate_full_project = matches.is_present("full");
        if let Some(project_name) = matches.value_of("project_name") {
            try!(utils::create_new_project(project_name, generate_full_project));
        }
        return Ok(());
    }

    if let Some(matches) = matches.subcommand_matches("serve") {
        try!(serve(matches, None));
    }
    if let Some(matches) = matches.subcommand_matches("watch") {
        try!(watch(matches));
    }
    Ok(())
}

fn serve(matches: &ArgMatches, rx_server: Option<Receiver<&str>>) -> Result<(), HaxoniteError> {
    let config_file = matches.value_of("config_file").unwrap_or("config.toml");
    loop {
        let config_content = try!(config::read_config(config_file));
        let config: Config = toml::decode_str(config_content.as_ref()).unwrap(); // TODO: remove unwrap to handle malformed config
        debug!("Using config: {:?}!", config);

        let server_config = config.server.unwrap_or_else(ServerConfig::default);
        let host = match value_t!(matches, "host", String) { // TODO: rewrite these matches
            Ok(host) => host,
            Err(_) => server_config.host.unwrap_or_else(config::default_host),
        };
        let port_number = match value_t!(matches, "port_number", u16) {
            Ok(port_number) => port_number,
            Err(_) => server_config.port.unwrap_or_else(config::default_port),
        };

        // Initialize Iron's router and pass it to config processing function to define routes
        let mut router = Router::new();
        let mut mount = Mount::new();
        match config.requests {
            Some(requests) => {
                try!(process_config_requests(&requests, &mut mount, &mut router));
            }
            None => return Err(HaxoniteError::NoRequestDefined),
        }
        mount.mount("/", router);

        // Insert logger middlewares before and after router
        let mut chain = Chain::new(mount);
        let format = Format::new(FORMAT);
        let (logger_before, logger_after) = Logger::new(Some(format.unwrap()));
        chain.link_before(logger_before);
        chain.link_after(logger_after);

        // Initialize Iron to process requests
        let mut _iron = try!(Iron::new(chain).http((host.as_ref(), port_number)));
        info!("Haxonite running on port: {}!", port_number);
        if let Some(ref rx_server) = rx_server {
        info!("RECEIVER: {:?}!", rx_server);
            match rx_server.recv() {
                Ok(_) => continue,
                Err(_) => break
            }
        }
    }
    Ok(())
}

fn watch(matches: &ArgMatches) -> Result<(), HaxoniteError> {
    let config_file = matches.value_of("config_file").unwrap_or("config.toml");

    let (tx_watcher, rx_watcher) = channel();
    let (tx_server, rx_server) = channel();
    let mut watcher: RecommendedWatcher = try!(Watcher::new(tx_watcher, Duration::from_secs(1)));
    try!(watcher.watch(config_file, RecursiveMode::Recursive));
    try!(serve(matches, Some(rx_server)));
    loop {
        match rx_watcher.recv() {
            Ok(event) => {
                info!("{:?}", event);
                tx_server.send("reload");
            },
            Err(e) => error!("watch error: {:?}", e),
        }
    }
}

fn process_config_requests(requests: &HashMap<String, RequestConfig>,
                           mut mount: &mut Mount,
                           mut router: &mut Router)
                           -> Result<(), HaxoniteError> {
    info!("Processing config!");
    for (request_name, request_config) in requests {
        info!("Processing config for {}: {:?}!",
              request_name,
              request_config);
        try!(define_route(request_name.clone(),
                          request_config.clone(),
                          &mut mount,
                          &mut router));
    }
    Ok(())
}

fn define_route(request_name: String, request_config: RequestConfig, mount: &mut Mount, router: &mut Router) -> Result<(), HaxoniteError> {
    let method = request_config.method.unwrap_or_else(config::default_method);
    let path = match request_config.path {
        Some(path) => path,
        None => return Err(HaxoniteError::NoPathDefined(request_name.clone())),
    };

    let type_ = request_config.type_.unwrap_or_else(config::default_type);
    let response_configs = match request_config.responses {
        Some(response_configs) => response_configs,
        None => return Err(HaxoniteError::NoResponseDefined(request_name.clone())),
    };
    let handler: Box<Handler> = match type_.as_ref() {
        "single" => {
            let response_config = try!(response_configs.first()
                .ok_or(HaxoniteError::NoResponseDefined(request_name.clone())));
            Box::new(try!(get_single_response_handler(request_name.clone(), response_config.clone())))
        }
        "random" => {
            let mut handler = RandomResponse::new();
            for response_config in &response_configs {
                let weight = response_config.weight.unwrap_or_else(config::default_weight);
                handler.add_handler(weight,
                                    try!(get_single_response_handler(request_name.clone(), response_config.clone())));
            }
            Box::new(handler)
        }
        "roundrobin" => {
            let mut handler = IteratingResponse::new("roundrobin");
            for response_config in &response_configs {
                handler.add_handler(try!(get_single_response_handler(request_name.clone(), response_config.clone())));
            }
            Box::new(handler)
        }
        "chain" => {
            let mut handler = IteratingResponse::new("chain");
            for response_config in &response_configs {
                handler.add_handler(try!(get_single_response_handler(request_name.clone(), response_config.clone())));
            }
            Box::new(handler)
        }
        "static" => {
            let response_config = try!(response_configs.first()
                .ok_or(HaxoniteError::NoResponseDefined(request_name.clone())));
            let response = match response_config.response {
                Some(ref response) => response,
                None => return Err(HaxoniteError::NoResponseDefined(request_name.clone())),
            };
            let handler = Static::new(Path::new(&response));
            Box::new(handler)
        }
        _ => return Err(HaxoniteError::InvalidType(request_name.clone())),
    };

    match Method::from_str(method.to_ascii_uppercase().as_ref()) {
        Ok(Method::Extension(_)) |
        Err(_) => return Err(HaxoniteError::InvalidHTTPMethod(request_name.clone())),
        Ok(a) => {
            match type_.as_ref() {
                "static" => {
                    info!("Mounting static for: {} using {} type of handler.\n",
                          path,
                          type_);
                    mount.mount(path.as_ref(), handler);
                }
                _ => {
                    info!("Defining route for: {} using {} type of handler.\n",
                          path,
                          type_);
                    router.route(a, path.clone(), handler, request_name);
                }
            }
        }
    };
    Ok(())
}

fn get_single_response_handler(request_name: String, response_config: ResponseConfig) -> Result<SingleResponse, HaxoniteError> {
    let headers = process_headers(response_config.headers);
    let status = match response_config.status {
        Some(status) if status >= 100 && status <= 599 => status, // switch to Range's contains() when it will be stable
        Some(status) => return Err(HaxoniteError::InvalidHTTPStatus(request_name.clone(), status)),
        None => config::default_status(),
    };
    let response = match response_config.response {
        Some(response) => response,
        None => return Err(HaxoniteError::NoResponseDefined(request_name.clone())),
    };
    if !Path::new(&response).exists() {
        return Err(HaxoniteError::ResponseDoesNotExist(request_name.clone()));
    }
    Ok(SingleResponse::new(status, headers, response, response_config.delay))
}

fn process_headers(headers: Option<Vec<String>>) -> Vec<Vec<String>> {
    // TODO: This method should be optimized to not compile regex on every run,
    // not match on header when the default is used (it's valid)
    use regex::Regex;
    let re = Regex::new(r"([\w-]+):\s+([\w/]+)").unwrap();

    let headers = headers.unwrap_or_else(config::default_headers);
    headers.iter()
        .filter(|header| {
            if re.is_match(header) {
                true
            } else {
                warn!("Header {} is invalid (does not match \": \"). Skipping.",
                      header);
                false
            }
        })
        .map(|header| {
            header.split(": ")
                .map(// TODO: This is quite stupid and could be done with &str,
                     // but I decided to use String to not fight borrow checker for now...
                     |hdr| hdr.to_string())
                .collect()
        })
        .collect()
}

fn define_command_line_options<'a, 'b>() -> App<'a, 'b> {
    App::new("Haxonite")
        .setting(AppSettings::SubcommandRequired)
        .version(VERSION)
        .author(AUTHORS)
        .about("Easy API mocking")
        .subcommand(SubCommand::with_name("serve")
            .about("Starts Haxonite server")
            .version(VERSION)
            .author(AUTHORS)
            .arg(Arg::with_name("host")
                 .short("h")
                 .long("host")
                 .value_name("HOST")
                 .help("Run Haxonite on the host. Default: localhost.")
                 .takes_value(true))
            .arg(Arg::with_name("port_number")
                 .short("p")
                 .long("port")
                 .value_name("PORT")
                 .help("Run Haxonite on the specified port. Default: 4000.")
                 .takes_value(true))
            .arg(Arg::with_name("config_file")
                 .short("c")
                 .long("config")
                 .value_name("FILE")
                 .help("Sets a custom config file. Default: config.toml.")
                 .takes_value(true)))
        .subcommand(SubCommand::with_name("watch")
            .about("Starts Haxonite server with dynamic reload on change")
            .version(VERSION)
            .author(AUTHORS)
            .arg(Arg::with_name("host")
                 .short("h")
                 .long("host")
                 .value_name("HOST")
                 .help("Run Haxonite on the host. Default: localhost.")
                 .takes_value(true))
            .arg(Arg::with_name("port_number")
                 .short("p")
                 .long("port")
                 .value_name("PORT")
                 .help("Run Haxonite on the specified port. Default: 4000.")
                 .takes_value(true))
            .arg(Arg::with_name("config_file")
                 .short("c")
                 .long("config")
                 .value_name("FILE")
                 .help("Sets a custom config file. Default: config.toml.")
                 .takes_value(true)))
        .subcommand(SubCommand::with_name("new")
            .about("Create new Haxonite project")
            .version(VERSION)
            .author(AUTHORS)
            .arg(Arg::with_name("full")
                .help("Use to create a project with full config.toml file")
                .short("f")
                .long("full"))
            .arg(Arg::with_name("project_name")
                .help("Name of the project (will be used as a directory name)")
                .index(1)
                .required(true)))
}
<commit_msg>#6: Fix watch in application<commit_after>const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const AUTHORS: &'static str = env!("CARGO_PKG_AUTHORS");

static FORMAT: &'static str = "{request-time}: {method} {uri} {status} {response-time}";

#[macro_use]
extern crate clap;
use clap::{Arg, ArgMatches, App, AppSettings, SubCommand};

extern crate toml;
extern crate rustc_serialize;
extern crate regex;

extern crate iron;
use iron::prelude::*;
use iron::Handler;
use iron::method::Method;
use std::str::FromStr;

extern crate router;
use router::Router;

extern crate staticfile;
use staticfile::Static;

extern crate mount;
use mount::Mount;

#[macro_use]
extern crate log;
extern crate simplelog;
use simplelog::{SimpleLogger, LogLevelFilter};

extern crate logger;
use logger::Logger;
use logger::Format;

extern crate notify;

use notify::{RecommendedWatcher, Watcher, RecursiveMode};
use notify::DebouncedEvent::{Create, Write};
use std::sync::mpsc::{channel, Receiver};
use std::time::Duration;

use std::thread;

mod config;
use config::{Config, ServerConfig, RequestConfig, ResponseConfig};
use std::collections::HashMap;
use std::ascii::AsciiExt;
mod error;
use error::HaxoniteError;
mod handlers;
use handlers::*;
mod utils;

use std::path::Path;

fn main() {
    let _ = SimpleLogger::init(LogLevelFilter::Info, simplelog::Config::default());

    match run() {
        Ok(_) => {}
        Err(err) => {
            error!("{}!", err);
            std::process::exit(-1);
        }
    }
}

fn run() -> Result<(), HaxoniteError> {
    let matches = define_command_line_options().get_matches();
    if let Some(matches) = matches.subcommand_matches("new") {
        let generate_full_project = matches.is_present("full");
        if let Some(project_name) = matches.value_of("project_name") {
            try!(utils::create_new_project(project_name, generate_full_project));
        }
        return Ok(());
    }

    if let Some(matches) = matches.subcommand_matches("serve") {
        try!(serve(matches, None));
    }
    if let Some(matches) = matches.subcommand_matches("watch") {
        try!(watch(matches));
    }
    Ok(())
}

fn serve(matches: &ArgMatches, rx_server: Option<Receiver<&str>>) -> Result<(), HaxoniteError> {
    let config_file = matches.value_of("config_file").unwrap_or("config.toml");
    loop {
        let config_content = try!(config::read_config(config_file));
        let config: Config = toml::decode_str(config_content.as_ref()).unwrap(); // TODO: remove unwrap to handle malformed config
        debug!("Using config: {:?}!", config);

        let server_config = config.server.unwrap_or_else(ServerConfig::default);
        let host = match value_t!(matches, "host", String) { // TODO: rewrite these matches
            Ok(host) => host,
            Err(_) => server_config.host.unwrap_or_else(config::default_host),
        };
        let port_number = match value_t!(matches, "port_number", u16) {
            Ok(port_number) => port_number,
            Err(_) => server_config.port.unwrap_or_else(config::default_port),
        };

        // Initialize Iron's router and pass it to config processing function to define routes
        let mut router = Router::new();
        let mut mount = Mount::new();
        match config.requests {
            Some(requests) => {
                try!(process_config_requests(&requests, &mut mount, &mut router));
            }
            None => return Err(HaxoniteError::NoRequestDefined),
        }
        mount.mount("/", router);

        // Insert logger middlewares before and after router
        let mut chain = Chain::new(mount);
        let format = Format::new(FORMAT);
        let (logger_before, logger_after) = Logger::new(Some(format.unwrap()));
        chain.link_before(logger_before);
        chain.link_after(logger_after);

        // Initialize Iron to process requests
        let mut _iron = try!(Iron::new(chain).http((host.as_ref(), port_number)));
        info!("Haxonite running on port: {}!", port_number);

        if let Some(ref rx_server) = rx_server {
            match rx_server.recv() {
                Ok(_) => {
                    _iron.close().expect("Iron cannot be closed");
                    thread::sleep(Duration::from_secs(1));
                    continue
                },
                Err(_) => break
            }
        }
    }
    Ok(())
}

fn watch(matches: &ArgMatches) -> Result<(), HaxoniteError> {
    let config_file = matches.value_of("config_file").unwrap_or("config.toml");

    let (tx_watcher, rx_watcher) = channel();
    let (tx_server, rx_server) = channel();

    let mut watcher: RecommendedWatcher = try!(Watcher::new(tx_watcher, Duration::from_secs(1)));
    try!(watcher.watch(config_file, RecursiveMode::NonRecursive)); // NonRecursie as we only watch config file
    thread::spawn(move || {
        loop {
            match rx_watcher.recv() {
                Ok(event) => {
                    match event {
                        Create(_) | Write(_) => tx_server.send("reload").unwrap(), // reload only on file save or create
                        _ => continue
                    }
                },
                Err(e) => error!("watch error: {:?}", e),
            }
        }
    });
    serve(matches, Some(rx_server))
}

fn process_config_requests(requests: &HashMap<String, RequestConfig>,
                           mut mount: &mut Mount,
                           mut router: &mut Router)
                           -> Result<(), HaxoniteError> {
    info!("Processing config!");
    for (request_name, request_config) in requests {
        info!("Processing config for {}: {:?}!",
              request_name,
              request_config);
        try!(define_route(request_name.clone(),
                          request_config.clone(),
                          &mut mount,
                          &mut router));
    }
    Ok(())
}

fn define_route(request_name: String, request_config: RequestConfig, mount: &mut Mount, router: &mut Router) -> Result<(), HaxoniteError> {
    let method = request_config.method.unwrap_or_else(config::default_method);
    let path = match request_config.path {
        Some(path) => path,
        None => return Err(HaxoniteError::NoPathDefined(request_name.clone())),
    };

    let type_ = request_config.type_.unwrap_or_else(config::default_type);
    let response_configs = match request_config.responses {
        Some(response_configs) => response_configs,
        None => return Err(HaxoniteError::NoResponseDefined(request_name.clone())),
    };
    let handler: Box<Handler> = match type_.as_ref() {
        "single" => {
            let response_config = try!(response_configs.first()
                .ok_or(HaxoniteError::NoResponseDefined(request_name.clone())));
            Box::new(try!(get_single_response_handler(request_name.clone(), response_config.clone())))
        }
        "random" => {
            let mut handler = RandomResponse::new();
            for response_config in &response_configs {
                let weight = response_config.weight.unwrap_or_else(config::default_weight);
                handler.add_handler(weight,
                                    try!(get_single_response_handler(request_name.clone(), response_config.clone())));
            }
            Box::new(handler)
        }
        "roundrobin" => {
            let mut handler = IteratingResponse::new("roundrobin");
            for response_config in &response_configs {
                handler.add_handler(try!(get_single_response_handler(request_name.clone(), response_config.clone())));
            }
            Box::new(handler)
        }
        "chain" => {
            let mut handler = IteratingResponse::new("chain");
            for response_config in &response_configs {
                handler.add_handler(try!(get_single_response_handler(request_name.clone(), response_config.clone())));
            }
            Box::new(handler)
        }
        "static" => {
            let response_config = try!(response_configs.first()
                .ok_or(HaxoniteError::NoResponseDefined(request_name.clone())));
            let response = match response_config.response {
                Some(ref response) => response,
                None => return Err(HaxoniteError::NoResponseDefined(request_name.clone())),
            };
            let handler = Static::new(Path::new(&response));
            Box::new(handler)
        }
        _ => return Err(HaxoniteError::InvalidType(request_name.clone())),
    };

    match Method::from_str(method.to_ascii_uppercase().as_ref()) {
        Ok(Method::Extension(_)) |
        Err(_) => return Err(HaxoniteError::InvalidHTTPMethod(request_name.clone())),
        Ok(a) => {
            match type_.as_ref() {
                "static" => {
                    info!("Mounting static for: {} using {} type of handler.\n",
                          path,
                          type_);
                    mount.mount(path.as_ref(), handler);
                }
                _ => {
                    info!("Defining route for: {} using {} type of handler.\n",
                          path,
                          type_);
                    router.route(a, path.clone(), handler, request_name);
                }
            }
        }
    };
    Ok(())
}

fn get_single_response_handler(request_name: String, response_config: ResponseConfig) -> Result<SingleResponse, HaxoniteError> {
    let headers = process_headers(response_config.headers);
    let status = match response_config.status {
        Some(status) if status >= 100 && status <= 599 => status, // switch to Range's contains() when it will be stable
        Some(status) => return Err(HaxoniteError::InvalidHTTPStatus(request_name.clone(), status)),
        None => config::default_status(),
    };
    let response = match response_config.response {
        Some(response) => response,
        None => return Err(HaxoniteError::NoResponseDefined(request_name.clone())),
    };
    if !Path::new(&response).exists() {
        return Err(HaxoniteError::ResponseDoesNotExist(request_name.clone()));
    }
    Ok(SingleResponse::new(status, headers, response, response_config.delay))
}

fn process_headers(headers: Option<Vec<String>>) -> Vec<Vec<String>> {
    // TODO: This method should be optimized to not compile regex on every run,
    // not match on header when the default is used (it's valid)
    use regex::Regex;
    let re = Regex::new(r"([\w-]+):\s+([\w/]+)").unwrap();

    let headers = headers.unwrap_or_else(config::default_headers);
    headers.iter()
        .filter(|header| {
            if re.is_match(header) {
                true
            } else {
                warn!("Header {} is invalid (does not match \": \"). Skipping.",
                      header);
                false
            }
        })
        .map(|header| {
            header.split(": ")
                .map(// TODO: This is quite stupid and could be done with &str,
                     // but I decided to use String to not fight borrow checker for now...
                     |hdr| hdr.to_string())
                .collect()
        })
        .collect()
}

fn define_command_line_options<'a, 'b>() -> App<'a, 'b> {
    App::new("Haxonite")
        .setting(AppSettings::SubcommandRequired)
        .version(VERSION)
        .author(AUTHORS)
        .about("Easy API mocking")
        .subcommand(SubCommand::with_name("serve")
            .about("Starts Haxonite server")
            .version(VERSION)
            .author(AUTHORS)
            .arg(Arg::with_name("host")
                 .short("h")
                 .long("host")
                 .value_name("HOST")
                 .help("Run Haxonite on the host. Default: localhost.")
                 .takes_value(true))
            .arg(Arg::with_name("port_number")
                 .short("p")
                 .long("port")
                 .value_name("PORT")
                 .help("Run Haxonite on the specified port. Default: 4000.")
                 .takes_value(true))
            .arg(Arg::with_name("config_file")
                 .short("c")
                 .long("config")
                 .value_name("FILE")
                 .help("Sets a custom config file. Default: config.toml.")
                 .takes_value(true)))
        .subcommand(SubCommand::with_name("watch")
            .about("Starts Haxonite server with dynamic reload on change")
            .version(VERSION)
            .author(AUTHORS)
            .arg(Arg::with_name("host")
                 .short("h")
                 .long("host")
                 .value_name("HOST")
                 .help("Run Haxonite on the host. Default: localhost.")
                 .takes_value(true))
            .arg(Arg::with_name("port_number")
                 .short("p")
                 .long("port")
                 .value_name("PORT")
                 .help("Run Haxonite on the specified port. Default: 4000.")
                 .takes_value(true))
            .arg(Arg::with_name("config_file")
                 .short("c")
                 .long("config")
                 .value_name("FILE")
                 .help("Sets a custom config file. Default: config.toml.")
                 .takes_value(true)))
        .subcommand(SubCommand::with_name("new")
            .about("Create new Haxonite project")
            .version(VERSION)
            .author(AUTHORS)
            .arg(Arg::with_name("full")
                .help("Use to create a project with full config.toml file")
                .short("f")
                .long("full"))
            .arg(Arg::with_name("project_name")
                .help("Name of the project (will be used as a directory name)")
                .index(1)
                .required(true)))
}
<|endoftext|>