<commit_before>mod app;
mod error;
mod exec;
mod exit_codes;
mod filesystem;
mod filetypes;
mod filter;
mod options;
mod output;
mod regex_helper;
mod walk;

use std::env;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time;

use anyhow::{anyhow, Context, Result};
use atty::Stream;
use globset::GlobBuilder;
use lscolors::LsColors;
use regex::bytes::{RegexBuilder, RegexSetBuilder};

use crate::exec::CommandTemplate;
use crate::exit_codes::ExitCode;
use crate::filetypes::FileTypes;
use crate::filter::{SizeFilter, TimeFilter};
use crate::options::Options;
use crate::regex_helper::pattern_has_uppercase_char;

// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481
#[cfg(all(not(windows), not(target_env = "musl")))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

fn run() -> Result<ExitCode> {
    let matches = app::build_app().get_matches_from(env::args_os());

    // Set the current working directory of the process
    if let Some(base_directory) = matches.value_of_os("base-directory") {
        let base_directory = Path::new(base_directory);
        if !filesystem::is_dir(base_directory) {
            return Err(anyhow!(
                "The '--base-directory' path '{}' is not a directory.",
                base_directory.to_string_lossy()
            ));
        }
        env::set_current_dir(base_directory).with_context(|| {
            format!(
                "Could not set '{}' as the current working directory.",
                base_directory.to_string_lossy()
            )
        })?;
    }

    let current_directory = Path::new(".");
    if !filesystem::is_dir(current_directory) {
        return Err(anyhow!(
            "Could not retrieve current directory (has it been deleted?)."
        ));
    }

    // Get the search pattern
    let pattern = matches
        .value_of_os("pattern")
        .map(|p| {
            p.to_str().ok_or(anyhow!(
                "The search pattern includes invalid UTF-8 sequences."
            ))
        })
        .transpose()?
        .unwrap_or("");

    // Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches
        .values_of_os("path")
        .or_else(|| matches.values_of_os("search-path"))
    {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if filesystem::is_dir(&path_buffer) {
                    Ok(path_buffer)
                } else {
                    Err(anyhow!(
                        "Search path '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ))
                }
            })
            .collect::<Result<Vec<_>>>()?,
        None => vec![current_directory.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| {
                path_buffer
                    .canonicalize()
                    .and_then(|pb| filesystem::absolute_path(pb.as_path()))
                    .unwrap()
            })
            .collect();
    }

    // Detect if the user accidentally supplied a path instead of a search pattern
    if !matches.is_present("full-path")
        && pattern.contains(std::path::MAIN_SEPARATOR)
        && filesystem::is_dir(Path::new(pattern))
    {
        return Err(anyhow!(
            "The search pattern '{pattern}' contains a path-separation character ('{sep}') \
             and will not lead to any search results.\n\n\
             If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n  \
             fd . '{pattern}'\n\n\
             Instead, if you want your pattern to match the full file path, use:\n\n  \
             fd --full-path '{pattern}'",
            pattern = pattern,
            sep = std::path::MAIN_SEPARATOR,
        ));
    }

    let pattern_regex = if matches.is_present("glob") {
        let glob = GlobBuilder::new(pattern).literal_separator(true).build()?;
        glob.regex().to_owned()
    } else if matches.is_present("fixed-strings") {
        // Treat pattern as literal string if '--fixed-strings' is used
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let interactive_terminal = atty::is(Stream::Stdout);
    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => env::var_os("NO_COLOR").is_none() && interactive_terminal,
    };

    let path_separator = matches.value_of("path-separator").map(|str| str.to_owned());

    #[cfg(windows)]
    ansi_term::enable_ansi_support().ok();

    let ls_colors = if colored_output {
        Some(LsColors::from_env().unwrap_or_default())
    } else {
        None
    };

    let command = if let Some(args) = matches.values_of("exec") {
        Some(CommandTemplate::new(args))
    } else if let Some(args) = matches.values_of("exec-batch") {
        Some(CommandTemplate::new_batch(args)?)
    } else if matches.is_present("list-details") {
        let color = matches.value_of("color").unwrap_or("auto");
        let color_arg = ["--color=", color].concat();

        #[allow(unused)]
        let gnu_ls = |command_name| {
            vec![
                command_name,
                "-l",               // long listing format
                "--human-readable", // human readable file sizes
                "--directory",      // list directories themselves, not their contents
                &color_arg,
            ]
        };

        let cmd: Vec<&str> = if cfg!(unix) {
            if !cfg!(any(target_os = "macos", target_os = "dragonfly", target_os = "freebsd")) {
                // Assume ls is GNU ls
                gnu_ls("ls")
            } else {
                // MacOS, DragonFlyBSD, FreeBSD
                use std::process::{Command, Stdio};

                // Use GNU ls, if available (support for --color=auto, better LS_COLORS support)
                let gnu_ls_exists = Command::new("gls")
                    .arg("--version")
                    .stdout(Stdio::null())
                    .stderr(Stdio::null())
                    .status()
                    .is_ok();

                if gnu_ls_exists {
                    gnu_ls("gls")
                } else {
                    let mut cmd = vec![
                        "ls", // BSD version of ls
                        "-l", // long listing format
                        "-h", // '--human-readable' is not available, '-h' is
                        "-d", // '--directory' is not available, but '-d' is
                    ];

                    if colored_output {
                        cmd.push("-G");
                    }

                    cmd
                }
            }
        } else if cfg!(windows) {
            use std::process::{Command, Stdio};

            // Use GNU ls, if available
            let gnu_ls_exists = Command::new("ls")
                .arg("--version")
                .stdout(Stdio::null())
                .stderr(Stdio::null())
                .status()
                .is_ok();

            if gnu_ls_exists {
                gnu_ls("ls")
            } else {
                return Err(anyhow!(
                    "'fd --list-details' is not supported on Windows unless GNU 'ls' is installed."
                ));
            }
        } else {
            return Err(anyhow!(
                "'fd --list-details' is not supported on this platform."
            ));
        };

        Some(CommandTemplate::new_batch(&cmd).unwrap())
    } else {
        None
    };

    let size_limits = if let Some(vs) = matches.values_of("size") {
        vs.map(|sf| {
            SizeFilter::from_string(sf).ok_or(anyhow!(
                "'{}' is not a valid size constraint. See 'fd --help'.",
                sf
            ))
        })
        .collect::<Result<Vec<_>>>()?
    } else {
        vec![]
    };

    let now = time::SystemTime::now();
    let mut time_constraints: Vec<TimeFilter> = Vec::new();
    if let Some(t) = matches.value_of("changed-within") {
        if let Some(f) = TimeFilter::after(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }
    if let Some(t) = matches.value_of("changed-before") {
        if let Some(f) = TimeFilter::before(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }

    let config = Options {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_fdignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_vcsignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        follow_links: matches.is_present("follow"),
        one_file_system: matches.is_present("one-file-system"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("max-depth")
            .or_else(|| matches.value_of("rg-depth"))
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        min_depth: matches
            .value_of("min-depth")
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        interactive_terminal,
        file_types: matches.values_of("file-type").map(|values| {
            let mut file_types = FileTypes::default();
            for value in values {
                match value {
                    "f" | "file" => file_types.files = true,
                    "d" | "directory" => file_types.directories = true,
                    "l" | "symlink" => file_types.symlinks = true,
                    "x" | "executable" => {
                        file_types.executables_only = true;
                        file_types.files = true;
                    }
                    "e" | "empty" => file_types.empty_only = true,
                    "s" | "socket" => file_types.sockets = true,
                    "p" | "pipe" => file_types.pipes = true,
                    _ => unreachable!(),
                }
            }

            // If only 'empty' was specified, search for both files and directories:
            if file_types.empty_only && !(file_types.files || file_types.directories) {
                file_types.files = true;
                file_types.directories = true;
            }

            file_types
        }),
        extensions: matches
            .values_of("extension")
            .map(|exts| {
                let patterns = exts
                    .map(|e| e.trim_start_matches('.'))
                    .map(|e| format!(r".\.{}$", regex::escape(e)));
                RegexSetBuilder::new(patterns)
                    .case_insensitive(true)
                    .build()
            })
            .transpose()?,
        command: command.map(Arc::new),
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
        ignore_files: matches
            .values_of("ignore-file")
            .map(|vs| vs.map(PathBuf::from).collect())
            .unwrap_or_else(|| vec![]),
        size_constraints: size_limits,
        time_constraints,
        show_filesystem_errors: matches.is_present("show-errors"),
        path_separator,
        max_results: matches
            .value_of("max-results")
            .and_then(|n| usize::from_str_radix(n, 10).ok())
            .filter(|&n| n != 0)
            .or_else(|| {
                if matches.is_present("max-one-result") {
                    Some(1)
                } else {
                    None
                }
            }),
    };

    let re = RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
        .map_err(|e| {
            anyhow!(
                "{}\n\nNote: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression. Alternatively, you can \
                 also use the '--glob' option to match on a glob pattern.",
                e.to_string()
            )
        })?;

    walk::scan(&dir_vec, Arc::new(re), Arc::new(config))
}

fn main() {
    let result = run();
    match result {
        Ok(exit_code) => {
            process::exit(exit_code.into());
        }
        Err(err) => {
            eprintln!("[fd error]: {}", err);
            process::exit(ExitCode::GeneralError.into());
        }
    }
}
<commit_msg>Also add NetBSD and OpenBSD<commit_after>mod app;
mod error;
mod exec;
mod exit_codes;
mod filesystem;
mod filetypes;
mod filter;
mod options;
mod output;
mod regex_helper;
mod walk;

use std::env;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time;

use anyhow::{anyhow, Context, Result};
use atty::Stream;
use globset::GlobBuilder;
use lscolors::LsColors;
use regex::bytes::{RegexBuilder, RegexSetBuilder};

use crate::exec::CommandTemplate;
use crate::exit_codes::ExitCode;
use crate::filetypes::FileTypes;
use crate::filter::{SizeFilter, TimeFilter};
use crate::options::Options;
use crate::regex_helper::pattern_has_uppercase_char;

// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481
#[cfg(all(not(windows), not(target_env = "musl")))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

fn run() -> Result<ExitCode> {
    let matches = app::build_app().get_matches_from(env::args_os());

    // Set the current working directory of the process
    if let Some(base_directory) = matches.value_of_os("base-directory") {
        let base_directory = Path::new(base_directory);
        if !filesystem::is_dir(base_directory) {
            return Err(anyhow!(
                "The '--base-directory' path '{}' is not a directory.",
                base_directory.to_string_lossy()
            ));
        }
        env::set_current_dir(base_directory).with_context(|| {
            format!(
                "Could not set '{}' as the current working directory.",
                base_directory.to_string_lossy()
            )
        })?;
    }

    let current_directory = Path::new(".");
    if !filesystem::is_dir(current_directory) {
        return Err(anyhow!(
            "Could not retrieve current directory (has it been deleted?)."
        ));
    }

    // Get the search pattern
    let pattern = matches
        .value_of_os("pattern")
        .map(|p| {
            p.to_str().ok_or(anyhow!(
                "The search pattern includes invalid UTF-8 sequences."
            ))
        })
        .transpose()?
        .unwrap_or("");

    // Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches
        .values_of_os("path")
        .or_else(|| matches.values_of_os("search-path"))
    {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if filesystem::is_dir(&path_buffer) {
                    Ok(path_buffer)
                } else {
                    Err(anyhow!(
                        "Search path '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ))
                }
            })
            .collect::<Result<Vec<_>>>()?,
        None => vec![current_directory.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| {
                path_buffer
                    .canonicalize()
                    .and_then(|pb| filesystem::absolute_path(pb.as_path()))
                    .unwrap()
            })
            .collect();
    }

    // Detect if the user accidentally supplied a path instead of a search pattern
    if !matches.is_present("full-path")
        && pattern.contains(std::path::MAIN_SEPARATOR)
        && filesystem::is_dir(Path::new(pattern))
    {
        return Err(anyhow!(
            "The search pattern '{pattern}' contains a path-separation character ('{sep}') \
             and will not lead to any search results.\n\n\
             If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n  \
             fd . '{pattern}'\n\n\
             Instead, if you want your pattern to match the full file path, use:\n\n  \
             fd --full-path '{pattern}'",
            pattern = pattern,
            sep = std::path::MAIN_SEPARATOR,
        ));
    }

    let pattern_regex = if matches.is_present("glob") {
        let glob = GlobBuilder::new(pattern).literal_separator(true).build()?;
        glob.regex().to_owned()
    } else if matches.is_present("fixed-strings") {
        // Treat pattern as literal string if '--fixed-strings' is used
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let interactive_terminal = atty::is(Stream::Stdout);
    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => env::var_os("NO_COLOR").is_none() && interactive_terminal,
    };

    let path_separator = matches.value_of("path-separator").map(|str| str.to_owned());

    #[cfg(windows)]
    ansi_term::enable_ansi_support().ok();

    let ls_colors = if colored_output {
        Some(LsColors::from_env().unwrap_or_default())
    } else {
        None
    };

    let command = if let Some(args) = matches.values_of("exec") {
        Some(CommandTemplate::new(args))
    } else if let Some(args) = matches.values_of("exec-batch") {
        Some(CommandTemplate::new_batch(args)?)
    } else if matches.is_present("list-details") {
        let color = matches.value_of("color").unwrap_or("auto");
        let color_arg = ["--color=", color].concat();

        #[allow(unused)]
        let gnu_ls = |command_name| {
            vec![
                command_name,
                "-l",               // long listing format
                "--human-readable", // human readable file sizes
                "--directory",      // list directories themselves, not their contents
                &color_arg,
            ]
        };

        let cmd: Vec<&str> = if cfg!(unix) {
            if !cfg!(any(target_os = "macos", target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd")) {
                // Assume ls is GNU ls
                gnu_ls("ls")
            } else {
                // MacOS, DragonFlyBSD, FreeBSD
                use std::process::{Command, Stdio};

                // Use GNU ls, if available (support for --color=auto, better LS_COLORS support)
                let gnu_ls_exists = Command::new("gls")
                    .arg("--version")
                    .stdout(Stdio::null())
                    .stderr(Stdio::null())
                    .status()
                    .is_ok();

                if gnu_ls_exists {
                    gnu_ls("gls")
                } else {
                    let mut cmd = vec![
                        "ls", // BSD version of ls
                        "-l", // long listing format
                        "-h", // '--human-readable' is not available, '-h' is
                        "-d", // '--directory' is not available, but '-d' is
                    ];

                    if !cfg!(any(target_os = "netbsd", target_os = "openbsd")) && colored_output {
                        // -G is not available in NetBSD's and OpenBSD's ls
                        cmd.push("-G");
                    }

                    cmd
                }
            }
        } else if cfg!(windows) {
            use std::process::{Command, Stdio};

            // Use GNU ls, if available
            let gnu_ls_exists = Command::new("ls")
                .arg("--version")
                .stdout(Stdio::null())
                .stderr(Stdio::null())
                .status()
                .is_ok();

            if gnu_ls_exists {
                gnu_ls("ls")
            } else {
                return Err(anyhow!(
                    "'fd --list-details' is not supported on Windows unless GNU 'ls' is installed."
                ));
            }
        } else {
            return Err(anyhow!(
                "'fd --list-details' is not supported on this platform."
            ));
        };

        Some(CommandTemplate::new_batch(&cmd).unwrap())
    } else {
        None
    };

    let size_limits = if let Some(vs) = matches.values_of("size") {
        vs.map(|sf| {
            SizeFilter::from_string(sf).ok_or(anyhow!(
                "'{}' is not a valid size constraint. See 'fd --help'.",
                sf
            ))
        })
        .collect::<Result<Vec<_>>>()?
    } else {
        vec![]
    };

    let now = time::SystemTime::now();
    let mut time_constraints: Vec<TimeFilter> = Vec::new();
    if let Some(t) = matches.value_of("changed-within") {
        if let Some(f) = TimeFilter::after(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }
    if let Some(t) = matches.value_of("changed-before") {
        if let Some(f) = TimeFilter::before(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }

    let config = Options {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_fdignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_vcsignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        follow_links: matches.is_present("follow"),
        one_file_system: matches.is_present("one-file-system"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("max-depth")
            .or_else(|| matches.value_of("rg-depth"))
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        min_depth: matches
            .value_of("min-depth")
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        interactive_terminal,
        file_types: matches.values_of("file-type").map(|values| {
            let mut file_types = FileTypes::default();
            for value in values {
                match value {
                    "f" | "file" => file_types.files = true,
                    "d" | "directory" => file_types.directories = true,
                    "l" | "symlink" => file_types.symlinks = true,
                    "x" | "executable" => {
                        file_types.executables_only = true;
                        file_types.files = true;
                    }
                    "e" | "empty" => file_types.empty_only = true,
                    "s" | "socket" => file_types.sockets = true,
                    "p" | "pipe" => file_types.pipes = true,
                    _ => unreachable!(),
                }
            }

            // If only 'empty' was specified, search for both files and directories:
            if file_types.empty_only && !(file_types.files || file_types.directories) {
                file_types.files = true;
                file_types.directories = true;
            }

            file_types
        }),
        extensions: matches
            .values_of("extension")
            .map(|exts| {
                let patterns = exts
                    .map(|e| e.trim_start_matches('.'))
                    .map(|e| format!(r".\.{}$", regex::escape(e)));
                RegexSetBuilder::new(patterns)
                    .case_insensitive(true)
                    .build()
            })
            .transpose()?,
        command: command.map(Arc::new),
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
        ignore_files: matches
            .values_of("ignore-file")
            .map(|vs| vs.map(PathBuf::from).collect())
            .unwrap_or_else(|| vec![]),
        size_constraints: size_limits,
        time_constraints,
        show_filesystem_errors: matches.is_present("show-errors"),
        path_separator,
        max_results: matches
            .value_of("max-results")
            .and_then(|n| usize::from_str_radix(n, 10).ok())
            .filter(|&n| n != 0)
            .or_else(|| {
                if matches.is_present("max-one-result") {
                    Some(1)
                } else {
                    None
                }
            }),
    };

    let re = RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
        .map_err(|e| {
            anyhow!(
                "{}\n\nNote: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression. Alternatively, you can \
                 also use the '--glob' option to match on a glob pattern.",
                e.to_string()
            )
        })?;

    walk::scan(&dir_vec, Arc::new(re), Arc::new(config))
}

fn main() {
    let result = run();
    match result {
        Ok(exit_code) => {
            process::exit(exit_code.into());
        }
        Err(err) => {
            eprintln!("[fd error]: {}", err);
            process::exit(ExitCode::GeneralError.into());
        }
    }
}
<|endoftext|>


<commit_before>extern crate regex;
extern crate clap;
extern crate walkdir;
extern crate ansi_term;

use std::io::Write;

macro_rules! println_stderr(
    ($($arg:tt)*) => { {
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

use clap::{Arg, App, AppSettings};
use regex::Regex;
use walkdir::WalkDir;
use ansi_term::Colour::Blue;

use std::env;
use std::path::PathBuf;

fn find(rgx: Regex, dirs: Vec<PathBuf>, paint: bool) {
    let painter;
    let replacement = if paint {
        painter = Blue.paint("$search");
        format!("{}", painter)
    } else {
        "$search".to_string()
    };

    for d in dirs.iter() {
        let wkd = WalkDir::new(d);

        for f in wkd.into_iter() {
            let entry;
            match f {
                Ok(e) => entry = e,
                Err(err) => {
                    println_stderr!("{}", err);
                    continue;
                }
            }

            // paths are ascii so should always be valid unicode
            // the edge case is super rare so just ignore it
            let s = &entry.path().to_string_lossy();

            if rgx.is_match(s) {
                let display = rgx.replace_all(s, replacement.as_str());
                println!("{}", display);
            }
        }
    }
}

fn main() {
    let matches = App::new("fnd, a simpler way to find files")
        .version("0.2.0")
        .author("Mathew Robinson <mrobinson@praelatus.io>")
        .about("Finds files")
        .setting(AppSettings::TrailingVarArg)
        .arg(Arg::with_name("color")
             .long("no_color")
             .short("nc")
             .help("When specified will not highlight matches with ansi \
                    term color codes this is useful when piping output"))
        .arg(Arg::with_name("REGEX")
             .required(true)
             .index(1)
             .help("The REGEX to search for, defaults to fuzzy finding"))
        .arg(Arg::with_name("DIRS")
            .multiple(true)
            .help("Where to search for SEARCH")
            .default_value("."))
        .get_matches();

    let search = matches.value_of("REGEX").unwrap();
    let dirs: Vec<&str> = matches.values_of("DIRS").unwrap().collect();

    let d: Vec<PathBuf> = dirs.iter()
        .map(|x| {
            if *x == "." {
                return env::current_dir().unwrap().to_path_buf();
            }

            PathBuf::from(x)
        })
        .collect();

    let re = format!("(?P<search>{})", search);
    let rgx = Regex::new(&re).unwrap();

    find(rgx, d, !matches.is_present("color"));
}
<commit_msg>make color optional, this improves performance for large regexes<commit_after>extern crate regex;
extern crate clap;
extern crate walkdir;
extern crate ansi_term;

use std::io::Write;

macro_rules! println_stderr(
    ($($arg:tt)*) => { {
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

use clap::{Arg, App, AppSettings};
use regex::Regex;
use walkdir::WalkDir;
use ansi_term::Colour::Blue;

use std::env;
use std::path::PathBuf;

fn find(rgx: Regex, dirs: Vec<PathBuf>, paint: bool) {
    let painter;
    let replacement = if paint {
        painter = Blue.paint("$search");
        format!("{}", painter)
    } else {
        "$search".to_string()
    };

    for d in dirs.iter() {
        let wkd = WalkDir::new(d);

        for f in wkd.into_iter() {
            let entry;
            match f {
                Ok(e) => entry = e,
                Err(err) => {
                    println_stderr!("{}", err);
                    continue;
                }
            }

            // paths are ascii so should always be valid unicode
            // the edge case is super rare so just ignore it
            let s = &entry.path().to_string_lossy();

            if rgx.is_match(s) {
                let display = if paint {
                    rgx.replace_all(s, replacement.as_str()).to_owned()
                } else {
                    s.to_owned()
                };

                println!("{}", display);
            }
        }
    }
}

fn main() {
    let matches = App::new("fnd, a simpler way to find files")
        .version("0.2.0")
        .author("Mathew Robinson <mrobinson@praelatus.io>")
        .about("Finds files")
        .setting(AppSettings::TrailingVarArg)
        .arg(Arg::with_name("color")
             .long("color")
             .short("c")
             .help("When specified will highlight matches with ansi \
                    term color codes. Note that for large regexes or \
                    regexes which match a large portion of text this \
                    can negatively affect performance."))
        .arg(Arg::with_name("REGEX")
             .required(true)
             .index(1)
             .help("The REGEX to search for, defaults to fuzzy finding"))
        .arg(Arg::with_name("DIRS")
            .multiple(true)
            .help("Where to search for SEARCH")
            .default_value("."))
        .get_matches();

    let search = matches.value_of("REGEX").unwrap();
    let dirs: Vec<&str> = matches.values_of("DIRS").unwrap().collect();

    let d: Vec<PathBuf> = dirs.iter()
        .map(|x| {
            if *x == "." {
                return env::current_dir().unwrap().to_path_buf();
            }

            PathBuf::from(x)
        })
        .collect();

    let re = format!("(?P<search>{})", search);
    let rgx = Regex::new(&re).unwrap();

    find(rgx, d, matches.is_present("color"));
}
<|endoftext|>


<commit_before>extern crate discord;

use std::env;
use discord::{Discord, State};
use discord::model::Event;

pub fn main() {
	// Log in to Discord using a bot token from the environment
	let discord = Discord::from_bot_token(
		&env::var("DISCORD_TOKEN").expect("Expected token"),
	).expect("login failed");

	// establish websocket and voice connection
	let (mut connection, ready) = discord.connect().expect("connect failed");
	println!("[Ready] {} is serving {} servers", ready.user.username, ready.servers.len());
	let mut state = State::new(ready);
	connection.sync_calls(&state.all_private_channels());

	// receive events forever
	loop {
		let event = match connection.recv_event() {
			Ok(event) => event,
			Err(err) => {
				println!("[Warning] Receive error: {:?}", err);
				if let discord::Error::WebSocket(..) = err {
					// Handle the websocket connection being dropped
					let (new_connection, ready) = discord.connect().expect("connect failed");
					connection = new_connection;
					state = State::new(ready);
					println!("[Ready] Reconnected successfully.");
				}
				if let discord::Error::Closed(..) = err {
					break
				}
				continue
			},
		};
		state.update(&event);

		match event {
			Event::MessageCreate(message) => {
				use std::ascii::AsciiExt;
				// safeguard: stop if the message is from us
				if message.author.id == state.user().id {
					continue
				}

				// reply to a command if there was one
				let mut split = message.content.split(' ');
				let first_word = split.next().unwrap_or("");
				let argument = split.next().unwrap_or("");

				//Section of avaliable commands.
                if first_word.eq_ignore_ascii_case("!help") {
                    if argument.eq_ignore_ascii_case("commands") {
                        warn(discord.send_message(message.channel_id, "help, about, uber, distrowiki, and sourcecode are the avaliable commands. (!dj is buggy)", "", false))
                    }
		    		else if argument.eq_ignore_ascii_case("about") {
			 			warn(discord.send_message(message.channel_id, "Displays info about me. No arguments available.", "", false))
		    		}
		    		else if argument.eq_ignore_ascii_case("uber") {
						warn(discord.send_message(message.channel_id, "Uber an object (I'm picky).", "", false))
		    		}
		    		else if argument.eq_ignore_ascii_case("distrowiki") {
						warn(discord.send_message(message.channel_id, "Finds the wiki of the specified distro. Arguments include Arch, Ubuntu, Debian, Antergos, CentOS, OpenBSD, Opensuse, Sabayon, KaliLinux, and Gentoo.", "", false))
		    		}
		    		else if argument.eq_ignore_ascii_case("sourcecode") {
						warn(discord.send_message(message.channel_id, "Displays where I am on GitHub. No arguments available", "", false))
	            	}
		    		else if argument.eq_ignore_ascii_case("help") {
						warn(discord.send_message(message.channel_id, "Are you trying to break me?", "", false))
	            	}
		    		else if argument.eq_ignore_ascii_case("[args]") {
						warn(discord.send_message(message.channel_id, "You're helpless.", "", false))
	            	}
                    else {
                    warn(discord.send_message(message.channel_id, "Umm, you gotta use it like this. !help [args] Try using the `commands` argument.", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!about") {
                    warn(discord.send_message(message.channel_id, "I have no clue. All I know is that I was written by Arroz in a language called Rust, and my code is still being developed.", "", false))
                }

                if first_word.eq_ignore_ascii_case("!uber"){
					if argument.eq_ignore_ascii_case("ddog75") {
                        warn(discord.send_message(message.channel_id, "Alright, 10 minutes.", "", false))
                    }
					else if argument.eq_ignore_ascii_case("aspire") {
						warn(discord.send_message(message.channel_id, "Not the meatballs.", "", false))
					}
					else if argument.eq_ignore_ascii_case("nic") {
						warn(discord.send_message(message.channel_id, "NOT THE BEES!!!", "", false))
					}
					 else {
                        warn(discord.send_message(message.channel_id, "Fuck off!", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!sourcecode") {
                    warn(discord.send_message(message.channel_id, "Okay, here you go. https://github.com/aarroz/rice_bot2", "", false))
                }
				//Section that prints wikis for linux distributions.
				if first_word.eq_ignore_ascii_case("!distrowiki") {
					if argument.eq_ignore_ascii_case("arch") {
						warn(discord.send_message(message.channel_id, "Here's the Arch Wiki! https://wiki.archlinux.org/", "", false))
					}
					else if argument.eq_ignore_ascii_case("ubuntu") {
						warn(discord.send_message(message.channel_id, "Here's the Ubuntu Wiki! https://wiki.ubuntu.com/", "", false))
					}
					else if argument.eq_ignore_ascii_case("debian") {
						warn(discord.send_message(message.channel_id, "Here's the Debian Wiki! https://wiki.debian.org/", "", false))
					}
					else if argument.eq_ignore_ascii_case("antergos") {
						warn(discord.send_message(message.channel_id, "Here's the Antergos Wiki! https://antergos.com/wiki/", "", false))
					}
					else if argument.eq_ignore_ascii_case("centos") {
						warn(discord.send_message(message.channel_id, "Here's the CentOS Wiki! https://wiki.centos.org/", "", false))
					}
					//Sorry OpenBSD users.
					else if argument.eq_ignore_ascii_case("openbsd") {
						warn(discord.send_message(message.channel_id, "No, just no.", "", false))
					}
					//Not sorry.
					else if argument.eq_ignore_ascii_case("gentoo") {
						warn(discord.send_message(message.channel_id, "Here's the Gentoo Wiki! https://wiki.gentoo.org/wiki/Main_Page", "", false))
					}
					else if argument.eq_ignore_ascii_case("opensuse") {
						warn(discord.send_message(message.channel_id, "Here's the OpenSuse Wiki! https://en.opensuse.org/Portal:Wiki", "", false))
					}
					else if argument.eq_ignore_ascii_case("sabayon") {
						warn(discord.send_message(message.channel_id, "Here's the Sabayon Wiki! https://wiki.sabayon.org/", "", false))
					}
					else if argument.eq_ignore_ascii_case("kalilinux") {
						warn(discord.send_message(message.channel_id, "Heres's the KaliLinux Wiki! https://en.wikipedia.org/wiki/Kali_Linux", "", false))
					}
					else if argument.eq_ignore_ascii_case("mint") {
						warn(discord.send_message(message.channel_id, "Here's the Arch install page! https://www.archlinux.org/download/", "", false))
					}
					else if argument.eq_ignore_ascii_case("windows") {
						warn(discord.send_message(message.channel_id, "Thats not a distro.", "", false))
					}
					else if argument.eq_ignore_ascii_case("macos") {
						warn(discord.send_message(message.channel_id, "Sure its close to linux, but I'm too tired to have that argument.", "", false))
					}
					else {
						warn(discord.send_message(message.channel_id, "Please use a distro from !help distrowiki.", "", false))
					}
				}

                // Command for using dj.
				if first_word.eq_ignore_ascii_case("!dj") {
					let vchan = state.find_voice_user(message.author.id);
					if argument.eq_ignore_ascii_case("stop") {
						vchan.map(|(sid, _)| connection.voice(sid).stop());
					} else if argument.eq_ignore_ascii_case("quit") {
						vchan.map(|(sid, _)| connection.drop_voice(sid));
					} else {
						let output = if let Some((server_id, channel_id)) = vchan {
							match discord::voice::open_ytdl_stream(argument) {
								Ok(stream) => {
									let voice = connection.voice(server_id);
									voice.set_deaf(true);
									voice.connect(channel_id);
									voice.play(stream);
									String::new()
								},
								Err(error) => format!("Error: {}", error),
							}
						} else {
							"You must be in a voice channel to DJ".to_owned()
						};
						if !output.is_empty() {
							warn(discord.send_message(message.channel_id, &output, "", false));
						}
					}
				}
			}
			Event::VoiceStateUpdate(server_id, _) => {
				// If someone moves/hangs up, and we are in a voice channel,
				if let Some(cur_channel) = connection.voice(server_id).current_channel() {
					// and our current voice channel is empty, disconnect from voice
					match server_id {
						Some(server_id) => if let Some(srv) = state.servers().iter().find(|srv| srv.id == server_id) {
							if srv.voice_states.iter().filter(|vs| vs.channel_id == Some(cur_channel)).count() <= 1 {
								connection.voice(Some(server_id)).disconnect();
							}
						},
						None => if let Some(call) = state.calls().get(&cur_channel) {
							if call.voice_states.len() <= 1 {
								connection.voice(server_id).disconnect();
							}
						}
					}
				}
			}
			_ => {}, // discard other events
		}
	}
}

fn warn<T, E: ::std::fmt::Debug>(result: Result<T, E>) {
	match result {
		Ok(_) => {},
		Err(err) => println!("[Warning] {:?}", err)
	}
}
<commit_msg>added solos, changed mint output<commit_after>extern crate discord;

use std::env;
use discord::{Discord, State};
use discord::model::Event;

pub fn main() {
	// Log in to Discord using a bot token from the environment
	let discord = Discord::from_bot_token(
		&env::var("DISCORD_TOKEN").expect("Expected token"),
	).expect("login failed");

	// establish websocket and voice connection
	let (mut connection, ready) = discord.connect().expect("connect failed");
	println!("[Ready] {} is serving {} servers", ready.user.username, ready.servers.len());
	let mut state = State::new(ready);
	connection.sync_calls(&state.all_private_channels());

	// receive events forever
	loop {
		let event = match connection.recv_event() {
			Ok(event) => event,
			Err(err) => {
				println!("[Warning] Receive error: {:?}", err);
				if let discord::Error::WebSocket(..) = err {
					// Handle the websocket connection being dropped
					let (new_connection, ready) = discord.connect().expect("connect failed");
					connection = new_connection;
					state = State::new(ready);
					println!("[Ready] Reconnected successfully.");
				}
				if let discord::Error::Closed(..) = err {
					break
				}
				continue
			},
		};
		state.update(&event);

		match event {
			Event::MessageCreate(message) => {
				use std::ascii::AsciiExt;
				// safeguard: stop if the message is from us
				if message.author.id == state.user().id {
					continue
				}

				// reply to a command if there was one
				let mut split = message.content.split(' ');
				let first_word = split.next().unwrap_or("");
				let argument = split.next().unwrap_or("");

				//Section of avaliable commands.
                if first_word.eq_ignore_ascii_case("!help") {
                    if argument.eq_ignore_ascii_case("commands") {
                        warn(discord.send_message(message.channel_id, "help, about, uber, distrowiki, and sourcecode are the avaliable commands. (!dj is buggy)", "", false))
                    }
		    		else if argument.eq_ignore_ascii_case("about") {
			 			warn(discord.send_message(message.channel_id, "Displays info about me. No arguments available.", "", false))
		    		}
		    		else if argument.eq_ignore_ascii_case("uber") {
						warn(discord.send_message(message.channel_id, "Uber an object (I'm picky).", "", false))
		    		}
		    		else if argument.eq_ignore_ascii_case("distrowiki") {
						warn(discord.send_message(message.channel_id, "Finds the wiki of the specified distro. Arguments include Arch, Ubuntu, Debian, Antergos, Mint, CentOS, OpenBSD, Opensuse, Sabayon, KaliLinux, Solus, and Gentoo.", "", false))
		    		}
		    		else if argument.eq_ignore_ascii_case("sourcecode") {
						warn(discord.send_message(message.channel_id, "Displays where I am on GitHub. No arguments available", "", false))
	            	}
		    		else if argument.eq_ignore_ascii_case("help") {
						warn(discord.send_message(message.channel_id, "Are you trying to break me?", "", false))
	            	}
		    		else if argument.eq_ignore_ascii_case("[args]") {
						warn(discord.send_message(message.channel_id, "You're helpless.", "", false))
	            	}
                    else {
                    warn(discord.send_message(message.channel_id, "Umm, you gotta use it like this. !help [args] Try using the `commands` argument.", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!about") {
                    warn(discord.send_message(message.channel_id, "I have no clue. All I know is that I was written by Arroz in a language called Rust, and my code is still being developed.", "", false))
                }

                if first_word.eq_ignore_ascii_case("!uber"){
					if argument.eq_ignore_ascii_case("ddog75") {
                        warn(discord.send_message(message.channel_id, "Alright, 10 minutes.", "", false))
                    }
					else if argument.eq_ignore_ascii_case("aspire") {
						warn(discord.send_message(message.channel_id, "Not the meatballs.", "", false))
					}
					else if argument.eq_ignore_ascii_case("nic") {
						warn(discord.send_message(message.channel_id, "NOT THE BEES!!!", "", false))
					}
					 else {
                        warn(discord.send_message(message.channel_id, "Fuck off!", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!sourcecode") {
                    warn(discord.send_message(message.channel_id, "Okay, here you go. https://github.com/aarroz/rice_bot2", "", false))
                }
				//Section that prints wikis for linux distributions.
				if first_word.eq_ignore_ascii_case("!distrowiki") {
					if argument.eq_ignore_ascii_case("arch") {
						warn(discord.send_message(message.channel_id, "Here's the Arch Wiki! https://wiki.archlinux.org/", "", false))
					}
					else if argument.eq_ignore_ascii_case("ubuntu") {
						warn(discord.send_message(message.channel_id, "Here's the Ubuntu Wiki! https://wiki.ubuntu.com/", "", false))
					}
					else if argument.eq_ignore_ascii_case("debian") {
						warn(discord.send_message(message.channel_id, "Here's the Debian Wiki! https://wiki.debian.org/", "", false))
					}
					else if argument.eq_ignore_ascii_case("antergos") {
						warn(discord.send_message(message.channel_id, "Here's the Antergos Wiki! https://antergos.com/wiki/", "", false))
					}
					else if argument.eq_ignore_ascii_case("centos") {
						warn(discord.send_message(message.channel_id, "Here's the CentOS Wiki! https://wiki.centos.org/", "", false))
					}
					//Sorry OpenBSD users.
					else if argument.eq_ignore_ascii_case("openbsd") {
						warn(discord.send_message(message.channel_id, "No, just no.", "", false))
					}
					//Not sorry.
					else if argument.eq_ignore_ascii_case("gentoo") {
						warn(discord.send_message(message.channel_id, "Here's the Gentoo Wiki! https://wiki.gentoo.org/wiki/Main_Page", "", false))
					}
					else if argument.eq_ignore_ascii_case("opensuse") {
						warn(discord.send_message(message.channel_id, "Here's the OpenSuse Wiki! https://en.opensuse.org/Portal:Wiki", "", false))
					}
					else if argument.eq_ignore_ascii_case("sabayon") {
						warn(discord.send_message(message.channel_id, "Here's the Sabayon Wiki! https://wiki.sabayon.org/", "", false))
					}
					else if argument.eq_ignore_ascii_case("kalilinux") {
						warn(discord.send_message(message.channel_id, "Heres's the KaliLinux Wiki! https://en.wikipedia.org/wiki/Kali_Linux", "", false))
					}
					else if argument.eq_ignore_ascii_case("mint") {
						warn(discord.send_message(message.channel_id, "Here's the Mint newbie page! https://forums.linuxmint.com/viewforum.php?f=90", "", false))
					}
					else if argument.eq_ignore_ascii_case("solus") {
						warn(discord.send_message(message.channel_id, "There is no wiki for solus, however there is the help page! https://solus-project.com/help-center/home/", "", false))
					}
					else if argument.eq_ignore_ascii_case("windows") {
						warn(discord.send_message(message.channel_id, "Thats not a distro.", "", false))
					}
					else if argument.eq_ignore_ascii_case("macos") {
						warn(discord.send_message(message.channel_id, "Sure its close to linux, but I'm too tired to have that argument.", "", false))
					}
					else {
						warn(discord.send_message(message.channel_id, "Please use a distro from !help distrowiki.", "", false))
					}
				}

                // Command for using dj.
				if first_word.eq_ignore_ascii_case("!dj") {
					let vchan = state.find_voice_user(message.author.id);
					if argument.eq_ignore_ascii_case("stop") {
						vchan.map(|(sid, _)| connection.voice(sid).stop());
					} else if argument.eq_ignore_ascii_case("quit") {
						vchan.map(|(sid, _)| connection.drop_voice(sid));
					} else {
						let output = if let Some((server_id, channel_id)) = vchan {
							match discord::voice::open_ytdl_stream(argument) {
								Ok(stream) => {
									let voice = connection.voice(server_id);
									voice.set_deaf(true);
									voice.connect(channel_id);
									voice.play(stream);
									String::new()
								},
								Err(error) => format!("Error: {}", error),
							}
						} else {
							"You must be in a voice channel to DJ".to_owned()
						};
						if !output.is_empty() {
							warn(discord.send_message(message.channel_id, &output, "", false));
						}
					}
				}
			}
			Event::VoiceStateUpdate(server_id, _) => {
				// If someone moves/hangs up, and we are in a voice channel,
				if let Some(cur_channel) = connection.voice(server_id).current_channel() {
					// and our current voice channel is empty, disconnect from voice
					match server_id {
						Some(server_id) => if let Some(srv) = state.servers().iter().find(|srv| srv.id == server_id) {
							if srv.voice_states.iter().filter(|vs| vs.channel_id == Some(cur_channel)).count() <= 1 {
								connection.voice(Some(server_id)).disconnect();
							}
						},
						None => if let Some(call) = state.calls().get(&cur_channel) {
							if call.voice_states.len() <= 1 {
								connection.voice(server_id).disconnect();
							}
						}
					}
				}
			}
			_ => {}, // discard other events
		}
	}
}

fn warn<T, E: ::std::fmt::Debug>(result: Result<T, E>) {
	match result {
		Ok(_) => {},
		Err(err) => println!("[Warning] {:?}", err)
	}
}
<|endoftext|>


<commit_before>extern crate futures;
extern crate glob;
extern crate hyper;
extern crate hyper_rustls;
#[macro_use]
extern crate lazy_static;
extern crate libc;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
#[macro_use]
extern crate structopt;
extern crate tokio_core;
extern crate toml;

mod ghclient;
mod statics;
mod structs;
#[macro_use]
mod syslog;

use statics::CONF_PATH;
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
enum Command {
    /// Gets user public key
    #[structopt(name = "key")]
    Key {
        #[structopt(parse(from_str))]
        user: String,
    },
    /// Executes pam check
    #[structopt(name = "pam")]
    Pam,
    /// Check configuration
    #[structopt(name = "check")]
    Check {
        #[structopt(parse(from_os_str))]
        confpath: std::path::PathBuf,
    },
    /// Cleans caches up
    #[structopt(name = "cleanup")]
    CleanUp,
    /// Displays version details
    #[structopt(name = "version")]
    Version,
}

fn main() {
    let command = Command::from_args();

    use std::env;
    use std::process;

    match command {
        Command::Check { confpath } => match structs::Config::new(&confpath) {
            Ok(_) => process::exit(0),
            Err(_) => process::exit(11),
        },
        Command::Key { user } => {
            match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                  .and_then(|client| client.print_user_public_key(&user))
            {
                Ok(_) => process::exit(0),
                Err(_) => process::exit(21),
            }
        }
        Command::Pam => match env::var("PAM_USER") {
            Ok(user) => match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                              .and_then(|client| client.check_pam(&user))
            {
                Ok(true) => process::exit(0),
                Ok(false) => process::exit(1),
                Err(_) => process::exit(31),
            },
            Err(_) => process::exit(41),
        },
        Command::CleanUp => {
            match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                  .and_then(|client| client.clear_all_caches())
            {
                Ok(_) => process::exit(0),
                Err(_) => process::exit(51),
            }
        }
        Command::Version => {
            println!("{}",
                     concat!(env!("CARGO_PKG_VERSION"),
                             include_str!(concat!(env!("OUT_DIR"), "/commit-info.txt"))));
        }
    };
}
<commit_msg>Add a subcommand to get rate_limit of github api<commit_after>extern crate futures;
extern crate glob;
extern crate hyper;
extern crate hyper_rustls;
#[macro_use]
extern crate lazy_static;
extern crate libc;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
#[macro_use]
extern crate structopt;
extern crate tokio_core;
extern crate toml;

mod ghclient;
mod statics;
mod structs;
#[macro_use]
mod syslog;

use statics::CONF_PATH;
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
enum Command {
    /// Gets user public key
    #[structopt(name = "key")]
    Key {
        #[structopt(parse(from_str))]
        user: String,
    },
    /// Executes pam check
    #[structopt(name = "pam")]
    Pam,
    /// Check configuration
    #[structopt(name = "check")]
    Check {
        #[structopt(parse(from_os_str))]
        confpath: std::path::PathBuf,
    },
    /// Cleans caches up
    #[structopt(name = "cleanup")]
    CleanUp,
    /// Get rate limit for github api
    #[structopt(name = "ratelimit")]
    RateLimit,
    /// Displays version details
    #[structopt(name = "version")]
    Version,
}

fn main() {
    let command = Command::from_args();

    use std::env;
    use std::process;

    match command {
        Command::Check { confpath } => match structs::Config::new(&confpath) {
            Ok(_) => process::exit(0),
            Err(_) => process::exit(11),
        },
        Command::Key { user } => {
            match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                  .and_then(|client| client.print_user_public_key(&user))
            {
                Ok(_) => process::exit(0),
                Err(_) => process::exit(21),
            }
        }
        Command::Pam => match env::var("PAM_USER") {
            Ok(user) => match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                              .and_then(|client| client.check_pam(&user))
            {
                Ok(true) => process::exit(0),
                Ok(false) => process::exit(1),
                Err(_) => process::exit(31),
            },
            Err(_) => process::exit(41),
        },
        Command::CleanUp => {
            match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                  .and_then(|client| client.clear_all_caches())
            {
                Ok(_) => process::exit(0),
                Err(_) => process::exit(51),
            }
        }
        Command::RateLimit => {
            match structs::Config::new(&CONF_PATH).and_then(|conf| Ok(ghclient::GithubClient::new(&conf)))
                                                  .and_then(|client| client.print_rate_limit())
            {
                Ok(_) => process::exit(0),
                Err(_) => process::exit(61),
            }
        }
        Command::Version => {
            println!("{}",
                     concat!(env!("CARGO_PKG_VERSION"),
                             include_str!(concat!(env!("OUT_DIR"), "/commit-info.txt"))));
        }
    };
}
<|endoftext|>


<commit_before>extern crate gfx_texture;
extern crate image;
extern crate piston;
extern crate piston_window;
extern crate texture;

use gfx_texture::Texture;
use image::ImageBuffer;
use piston::input::*;
use piston_window::*;
use texture::TextureSettings;

const WIDTH: u32 = 600;
const HEIGHT: u32 = 600;

fn main() {
	let mut window : PistonWindow = WindowSettings::new("conway", [WIDTH, HEIGHT])
		.exit_on_esc(true)
		.build()
		.expect("error making window");

	let buf = vec![0u8; (WIDTH * HEIGHT * 4) as usize];
	let mut img = ImageBuffer::from_raw(WIDTH, HEIGHT, buf).unwrap();
	let tset = TextureSettings::new();
	let mut tex = Texture::from_image(&mut window.factory, &img, &tset).unwrap();

	while let Some(e) = window.next() {
		if let Some(Button::Keyboard(key)) = e.press_args() {
			if key == Key::Space {
				for p in img.enumerate_pixels_mut() {
					p.2.data = [0xFF, 0xFF, 0xFF, 0xFF];
				}
				tex = Texture::from_image(&mut window.factory, &img, &tset).unwrap();
			}
		}
		window.draw_2d(&e, |c, g| {
			clear([0.0, 0.0, 0.0, 1.0], g);
			image(&tex, c.transform, g);
		});
	}
}
<commit_msg>Magnify the texture so as to render one cell per pixel<commit_after>extern crate gfx_texture;
extern crate image;
extern crate piston;
extern crate piston_window;
extern crate texture;

use gfx_texture::Texture;
use image::ImageBuffer;
use piston::input::*;
use piston_window::*;
use texture::TextureSettings;

const WIDTH: u32 = 60;  // Width of window, in cells.
const HEIGHT: u32 = 60;  // Height of window, in cells.
const CELL_SIZE: u32 = 10;  // Effectively a zoom factor.  CELL_SIZE * WIDTH => width of the window in pixels.

fn main() {
	let mut window : PistonWindow = WindowSettings::new("conway", [WIDTH * CELL_SIZE, HEIGHT * CELL_SIZE])
		.exit_on_esc(true)
		.build()
		.expect("error making window");

	// Since we're modeling one cell per pixel, the texture gets magnified.  Setting the
	// mag filter to Nearest gives us sharp edges on the cells.
	let tset = TextureSettings::new()
		.mag(texture::Filter::Nearest);

	let buf = vec![0u8; (WIDTH * HEIGHT * 4) as usize];
	let mut cells = ImageBuffer::from_raw(WIDTH, HEIGHT, buf).unwrap();
	let mut tex = Texture::from_image(&mut window.factory, &cells, &tset).unwrap();

	while let Some(e) = window.next() {
		if let Some(Button::Keyboard(key)) = e.press_args() {
			if key == Key::Space {
				for p in cells.enumerate_pixels_mut() {
					if p.0 % 2 == p.1 % 2 {
						p.2.data = [0xFF, 0xFF, 0xFF, 0xFF];
					}
				}
				tex = Texture::from_image(&mut window.factory, &cells, &tset).unwrap();
			}
		}
		window.draw_2d(&e, |c, g| {
			clear([0.0, 0.0, 0.0, 1.0], g);
			image(&tex, c.transform.scale(CELL_SIZE as f64, CELL_SIZE as f64), g);
		});
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Athena Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![feature(path_ext)]

extern crate gfx;
extern crate gfx_device_gl;
extern crate gfx_window_glutin;
extern crate glutin;
extern crate phosphorus;
extern crate rustc_serialize;

use std::cell::RefCell;
use std::env;
use std::fs::{File, PathExt};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::rc::Rc;
use gfx::traits::*;
use phosphorus::Gui;
use phosphorus::widget::{ButtonBuilder, Layout, LayoutBuilder, TextBuilder};
use rustc_serialize::json;

struct SharedData {
    canceled: bool,
    queued_layout: Option<Layout<gfx_device_gl::Resources>>
}

#[derive(RustcDecodable, RustcEncodable, Clone)]
struct EntityEntry;

#[derive(RustcDecodable, RustcEncodable, Clone)]
struct WorldModel {
    entities: Vec<EntityEntry>
}

#[derive(RustcDecodable, RustcEncodable, Clone)]
struct Model {
    worlds: Vec<WorldModel>
}

fn main() {
    // This is all a bunch of placeholder code that needs to be refactored into something coherent

    // Get the target path to open the editor at
    let path = match env::args().nth(1) {
        Some(v) => v,
        None => {
            display_error("No path given!");
            return;
        }
    };

    // Get the actual path of the data file
    let mut path = PathBuf::from(path);
    path.push("editor.json");

    let model = Rc::new(RefCell::new(if !path.exists() {
        println!("Creating new editor.json...");
        let model = Model { worlds: vec![] };

        save_model(&path, &model);

        model
    } else {
        println!("Loading in editor.json...");
        let mut file = File::open(path.clone()).unwrap();
        let mut file_data = String::new();
        file.read_to_string(&mut file_data).unwrap();

        json::decode::<Model>(&file_data).unwrap()
    }));

    // Set up our Phosphorus UI
    let data = Rc::new(RefCell::new(SharedData { canceled: false, queued_layout: None }));
    let layout = generate_view(path, data.clone(), model);

    data.borrow_mut().queued_layout = Some(layout);

    display_gui(data);
}

fn save_model(proj_path: &PathBuf, model: &Model){
    let mut file = File::create(proj_path).unwrap();
    file.write_all(&json::encode(&model).unwrap().as_bytes()).unwrap();
}

fn generate_view(proj_path: PathBuf, data: Rc<RefCell<SharedData>>, model: Rc<RefCell<Model>>) -> Layout<gfx_device_gl::Resources> {
    save_model(&proj_path, &model.borrow());

    let mut builder = LayoutBuilder::<gfx_device_gl::Resources>::new()
        .with_background_color([21, 23, 24]);

    let mut wnum = 0;
    for world in &model.borrow_mut().worlds {
        builder = builder
            .with_widget(TextBuilder::new()
                .with_text(&format!("== World #{} ==", wnum))
                .build_boxed());

        let mut ennum = 0;
        for entity in &world.entities {
            builder = builder
                .with_widget(TextBuilder::new()
                    .with_text(&format!("Entity #{}", ennum))
                    .build_boxed());
            ennum += 1;
        }

        let tmp_model = model.clone();
        let tmp_data = data.clone();
        let tmp_path = proj_path.clone();
        builder = builder
            .with_widget(ButtonBuilder::new()
                .with_text("Add Entity")
                .with_callback(Box::new(move || {
                    {
                        let mut m = tmp_model.borrow_mut();
                        let w = m.worlds.get_mut(wnum).unwrap();
                        w.entities.push(EntityEntry);
                    }

                    tmp_data.borrow_mut().queued_layout = Some(generate_view(tmp_path.clone(), tmp_data.clone(), tmp_model.clone()));
                }))
            .build_boxed());
        wnum += 1;

        // Placeholder text as padding
        builder = builder.with_widget(TextBuilder::new().build_boxed());
    }

    builder
        .with_widget(ButtonBuilder::new()
            .with_text("Add World")
            .with_callback(Box::new(move || {
                {
                    let mut m = model.borrow_mut();
                    m.worlds.push(WorldModel { entities: vec![] });
                }

                data.borrow_mut().queued_layout = Some(generate_view(proj_path.clone(), data.clone(), model.clone()));
            }))
            .build_boxed())
        .build()
}

fn display_error(text: &str) {
    let data = Rc::new(RefCell::new(SharedData { canceled: false, queued_layout: None }));
    let data_clone = data.clone();
    let layout = LayoutBuilder::<gfx_device_gl::Resources>::new()
        .with_background_color([21, 23, 24])
        .with_widget(TextBuilder::new()
            .with_text(text)
            .build_boxed())
        .with_widget(ButtonBuilder::new()
            .with_text("Ok")
            .with_callback(Box::new(move || data_clone.borrow_mut().canceled = true))
            .build_boxed())
        .build();

    data.borrow_mut().queued_layout = Some(layout);

    display_gui(data);
}

fn display_gui(data: Rc<RefCell<SharedData>>)
{
    // Set up our window
    let (mut stream, mut device, mut factory) = {
        let window = glutin::WindowBuilder::new()
            .with_vsync()
            .with_dimensions(600, 500)
            .with_title(String::from("Phosphorus Widgets"))
            .build_strict().unwrap();
        gfx_window_glutin::init(window)
    };

    let mut gui = Gui::new(
        &mut device,
        data.borrow_mut().queued_layout.take().unwrap(),
        |d: &mut gfx_device_gl::Device| d.spawn_factory());

    // Run our actual UI loop
    'main: loop {
        {
            let mut d = data.borrow_mut();

            if d.canceled {
                break 'main;
            }

            if let Some(layout) = d.queued_layout.take() {
                gui.set_root(layout);
            }
        }

        // Quit when the window is closed
        for event in stream.out.window.poll_events() {
            match event {
                glutin::Event::Closed => break 'main,
                glutin::Event::MouseMoved((x, y)) =>
                    gui.raise_event(&stream, phosphorus::Event::MouseMoved([x, y])),
                glutin::Event::MouseInput(glutin::ElementState::Released, _) =>
                    gui.raise_event(&stream, phosphorus::Event::MouseClick),
                _ => (),
            }
        }

        stream.clear(gfx::ClearData {color: [1.0, 1.0, 1.0, 1.0], depth: 1.0, stencil: 0});

        // Render our actual GUI
        gui.render(&mut factory, &mut stream);

        // Show the rendered to buffer on the screen
        //stream.present(&mut device); ICE!
        stream.flush(&mut device);
        stream.out.window.swap_buffers();
        device.cleanup();
    }
}
<commit_msg>Slightly improve error reporting on failure to open<commit_after>// Copyright 2015 The Athena Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![feature(path_ext)]

extern crate gfx;
extern crate gfx_device_gl;
extern crate gfx_window_glutin;
extern crate glutin;
extern crate phosphorus;
extern crate rustc_serialize;

use std::cell::RefCell;
use std::env;
use std::fs::{File, PathExt};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::rc::Rc;
use gfx::traits::*;
use phosphorus::Gui;
use phosphorus::widget::{ButtonBuilder, Layout, LayoutBuilder, TextBuilder};
use rustc_serialize::json;

struct SharedData {
    canceled: bool,
    queued_layout: Option<Layout<gfx_device_gl::Resources>>
}

#[derive(RustcDecodable, RustcEncodable, Clone)]
struct EntityEntry;

#[derive(RustcDecodable, RustcEncodable, Clone)]
struct WorldModel {
    entities: Vec<EntityEntry>
}

#[derive(RustcDecodable, RustcEncodable, Clone)]
struct Model {
    worlds: Vec<WorldModel>
}

fn main() {
    // This is all a bunch of placeholder code that needs to be refactored into something coherent

    // Get the target path to open the editor at
    let path = match env::args().nth(1) {
        Some(v) => v,
        None => {
            display_error("No path given!");
            return;
        }
    };

    // Get the actual path of the data file
    let mut path = PathBuf::from(path);
    path.push("editor.json");

    let model = Rc::new(RefCell::new(if !path.exists() {
        println!("Creating new editor.json...");
        let model = Model { worlds: vec![] };

        save_model(&path, &model);

        model
    } else {
        println!("Loading in editor.json...");
        let mut file = File::open(path.clone()).unwrap();
        let mut file_data = String::new();
        file.read_to_string(&mut file_data).unwrap();

        json::decode::<Model>(&file_data).unwrap()
    }));

    // Set up our Phosphorus UI
    let data = Rc::new(RefCell::new(SharedData { canceled: false, queued_layout: None }));
    let layout = generate_view(path, data.clone(), model);

    data.borrow_mut().queued_layout = Some(layout);

    display_gui(data);
}

fn save_model(proj_path: &PathBuf, model: &Model){
    let mut file = match File::create(proj_path) {
        Ok(f) => f,
        Err(e) => panic!(format!("Folder does not exist or is not accessable, error: {:?}", e))
    };
    file.write_all(&json::encode(&model).unwrap().as_bytes()).unwrap();
}

fn generate_view(proj_path: PathBuf, data: Rc<RefCell<SharedData>>, model: Rc<RefCell<Model>>) -> Layout<gfx_device_gl::Resources> {
    save_model(&proj_path, &model.borrow());

    let mut builder = LayoutBuilder::<gfx_device_gl::Resources>::new()
        .with_background_color([21, 23, 24]);

    let mut wnum = 0;
    for world in &model.borrow_mut().worlds {
        builder = builder
            .with_widget(TextBuilder::new()
                .with_text(&format!("== World #{} ==", wnum))
                .build_boxed());

        let mut ennum = 0;
        for entity in &world.entities {
            builder = builder
                .with_widget(TextBuilder::new()
                    .with_text(&format!("Entity #{}", ennum))
                    .build_boxed());
            ennum += 1;
        }

        let tmp_model = model.clone();
        let tmp_data = data.clone();
        let tmp_path = proj_path.clone();
        builder = builder
            .with_widget(ButtonBuilder::new()
                .with_text("Add Entity")
                .with_callback(Box::new(move || {
                    {
                        let mut m = tmp_model.borrow_mut();
                        let w = m.worlds.get_mut(wnum).unwrap();
                        w.entities.push(EntityEntry);
                    }

                    tmp_data.borrow_mut().queued_layout = Some(generate_view(tmp_path.clone(), tmp_data.clone(), tmp_model.clone()));
                }))
            .build_boxed());
        wnum += 1;

        // Placeholder text as padding
        builder = builder.with_widget(TextBuilder::new().build_boxed());
    }

    builder
        .with_widget(ButtonBuilder::new()
            .with_text("Add World")
            .with_callback(Box::new(move || {
                {
                    let mut m = model.borrow_mut();
                    m.worlds.push(WorldModel { entities: vec![] });
                }

                data.borrow_mut().queued_layout = Some(generate_view(proj_path.clone(), data.clone(), model.clone()));
            }))
            .build_boxed())
        .build()
}

fn display_error(text: &str) {
    let data = Rc::new(RefCell::new(SharedData { canceled: false, queued_layout: None }));
    let data_clone = data.clone();
    let layout = LayoutBuilder::<gfx_device_gl::Resources>::new()
        .with_background_color([21, 23, 24])
        .with_widget(TextBuilder::new()
            .with_text(text)
            .build_boxed())
        .with_widget(ButtonBuilder::new()
            .with_text("Ok")
            .with_callback(Box::new(move || data_clone.borrow_mut().canceled = true))
            .build_boxed())
        .build();

    data.borrow_mut().queued_layout = Some(layout);

    display_gui(data);
}

fn display_gui(data: Rc<RefCell<SharedData>>)
{
    // Set up our window
    let (mut stream, mut device, mut factory) = {
        let window = glutin::WindowBuilder::new()
            .with_vsync()
            .with_dimensions(600, 500)
            .with_title(String::from("Phosphorus Widgets"))
            .build_strict().unwrap();
        gfx_window_glutin::init(window)
    };

    let mut gui = Gui::new(
        &mut device,
        data.borrow_mut().queued_layout.take().unwrap(),
        |d: &mut gfx_device_gl::Device| d.spawn_factory());

    // Run our actual UI loop
    'main: loop {
        {
            let mut d = data.borrow_mut();

            if d.canceled {
                break 'main;
            }

            if let Some(layout) = d.queued_layout.take() {
                gui.set_root(layout);
            }
        }

        // Quit when the window is closed
        for event in stream.out.window.poll_events() {
            match event {
                glutin::Event::Closed => break 'main,
                glutin::Event::MouseMoved((x, y)) =>
                    gui.raise_event(&stream, phosphorus::Event::MouseMoved([x, y])),
                glutin::Event::MouseInput(glutin::ElementState::Released, _) =>
                    gui.raise_event(&stream, phosphorus::Event::MouseClick),
                _ => (),
            }
        }

        stream.clear(gfx::ClearData {color: [1.0, 1.0, 1.0, 1.0], depth: 1.0, stencil: 0});

        // Render our actual GUI
        gui.render(&mut factory, &mut stream);

        // Show the rendered to buffer on the screen
        //stream.present(&mut device); ICE!
        stream.flush(&mut device);
        stream.out.window.swap_buffers();
        device.cleanup();
    }
}
<|endoftext|>


<commit_before>//    Copyright 2017 Vector Creations Limited
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

// extern crate base64;
// extern crate blake2_rfc;
#[macro_use]
extern crate clap;
extern crate humansize;
extern crate linear_map;
extern crate rand;
extern crate rusqlite;
extern crate twox_hash;
extern crate walkdir;
extern crate indicatif;

// use blake2_rfc::blake2b::Blake2b;
use humansize::{FileSize, file_size_opts as options};
use clap::{App, Arg};
use linear_map::LinearMap;
use rand::Rng;
use std::io;
use std::io::Read;
use std::fs::File;
use std::hash::Hasher;
use std::collections::BTreeMap;
use std::path::PathBuf;
use walkdir::WalkDir;

fn copy<R: io::Read, W: Hasher>(reader: &mut R, writer: &mut W) -> io::Result<u64> {
    let mut buf = [0; 64 * 1024];
    let mut written = 0;
    loop {
        let len = match reader.read(&mut buf) {
            Ok(0) => return Ok(written),
            Ok(len) => len,
            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
            Err(e) => return Err(e),
        };
        writer.write(&buf[..len]);
        written += len as u64;
    }
}

fn to_hash(path: &PathBuf) -> u64 {
    let mut file = File::open(path).unwrap();
    let mut hasher = twox_hash::XxHash::default();
    copy(&mut file, &mut hasher).unwrap();
    hasher.finish()
}

fn read_file(path: &PathBuf) -> Vec<u8> {
    let mut file = File::open(path).unwrap();
    let mut vec = Vec::new();
    file.read_to_end(&mut vec).unwrap();
    vec
}

fn partition_by<I, F, R>(paths: I, f: F) -> LinearMap<R, Vec<PathBuf>>
    where I: Iterator<Item=PathBuf>, F: Fn(&PathBuf) -> R, R: Eq
{
    let mut map = LinearMap::with_capacity(paths.size_hint().0);
    for path in paths {
        let key = f(&path);
        map.entry(key).or_insert_with(Vec::new).push(path);
    }
    map
}

const DB_TABLE_SCHEMA: &'static str = r#"
CREATE TABLE files (
    hash BIGINT NOT NULL,
    path TEXT NOT NULL,
    size BIGINT NOT NULL
);
"#;

fn main() {
    let matches = App::new(crate_name!())
        .version(crate_version!())
        .author(crate_authors!("\n"))
        .arg(Arg::with_name("media_directory")
            .help("The location of the media store")
            .index(1)
            .multiple(true)
            .required(true))
        .get_matches();

    let paths_to_search = matches.values_of("media_directory").unwrap();

    let db = rusqlite::Connection::open_in_memory().expect("failed to open sqlite db");
    db.execute_batch(DB_TABLE_SCHEMA).expect("failed to create db schema");

    let mut paths_by_size = BTreeMap::new();
    let mut total_files = 0;
    let mut total_size = 0;

    let pb = indicatif::ProgressBar::new_spinner();
    pb.set_style(
        indicatif::ProgressStyle::default_spinner()
        .template("{spinner} Collected metadata for {pos} files [{elapsed}]")
    );

    for path in paths_to_search {
        for entry in WalkDir::new(path) {
            let entry = entry.unwrap();
            if !entry.file_type().is_file() {
                continue
            }

            let file_size = entry.metadata().unwrap().len() as usize;
            paths_by_size.entry(file_size).or_insert_with(Vec::new).push(entry.path().to_owned());

            total_files += 1;
            total_size += file_size;

            pb.inc(1);
        }
    }

    pb.finish();
    pb.set_position(total_files);

    // println!("  Collected metadata for {} files", total_files);

    let pb = indicatif::ProgressBar::new(total_files);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Searching for possible duplicates  {bar:40} {pos:>9}/{len:9} [{elapsed}]")
    );

    let mut possible_total_size = 0;
    let mut possible_duplicates = Vec::new();
    for (file_size, paths) in paths_by_size.into_iter() {
        if paths.len() > 1 {
            possible_total_size += file_size * paths.len();
            possible_duplicates.push((file_size, paths))
        }
        pb.inc(1);
    }

    pb.finish();

    // We shuffle the vector as otherwise we do the small files first, and
    // that can skew the progress estimates massively.
    let mut rng = rand::thread_rng();
    rng.shuffle(possible_duplicates.as_mut_slice());

    let pb = indicatif::ProgressBar::new(possible_total_size as u64);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Comparing hashes                   {bar:40} {bytes:>9}/{total_bytes:9} [{elapsed}]")
    );

    let mut total_wasted_size = 0;

    for (file_size, paths) in possible_duplicates {
        if paths.len() == 1 {
            continue
        }

        let by_hash = partition_by(paths.into_iter(), to_hash);

        for (hash, paths) in by_hash {
            if paths.len() == 1 {
                pb.inc(file_size as u64);
                continue
            }

            let by_contents = partition_by(paths.into_iter(), read_file);

            for (_, paths) in by_contents {
                if paths.len() == 1 {
                    pb.inc(file_size as u64);
                    continue
                }

                for path in &paths {
                    db.execute("INSERT INTO files (hash, path, size) VALUES (?, ?, ?)", &[&(hash as i64), &path.to_str().unwrap(), &(file_size as i64)]).expect("failed to write to db");
                }

                let wasted = file_size * (paths.len() - 1);

                total_wasted_size += wasted;

                pb.inc((file_size * paths.len()) as u64);
            }
        }
    }

    pb.finish();

    println!();
    println!(
        "Total wasted size: {} out of {}. Percentage: {:.2}%",
        total_wasted_size.file_size(options::CONVENTIONAL).unwrap(),
        total_size.file_size(options::CONVENTIONAL).unwrap(),
        (total_wasted_size * 100) as f64 / total_size as f64,
    );

    let mut disk_db = rusqlite::Connection::open("media_store_sizes.db").expect("failed to open sqlite db");
    let backup = rusqlite::backup::Backup::new(&db, &mut disk_db).expect("failed to create backup");
    backup.run_to_completion(5, std::time::Duration::from_millis(0), None).expect("failed to write to disk");
}
<commit_msg>Add option to save sqlite database to disk<commit_after>//    Copyright 2017 Vector Creations Limited
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

// extern crate base64;
// extern crate blake2_rfc;
#[macro_use]
extern crate clap;
extern crate humansize;
extern crate linear_map;
extern crate rand;
extern crate rusqlite;
extern crate twox_hash;
extern crate walkdir;
extern crate indicatif;

// use blake2_rfc::blake2b::Blake2b;
use humansize::{FileSize, file_size_opts as options};
use clap::{App, Arg};
use linear_map::LinearMap;
use rand::Rng;
use std::io;
use std::io::Read;
use std::fs::File;
use std::hash::Hasher;
use std::collections::BTreeMap;
use std::path::PathBuf;
use walkdir::WalkDir;

fn copy<R: io::Read, W: Hasher>(reader: &mut R, writer: &mut W) -> io::Result<u64> {
    let mut buf = [0; 64 * 1024];
    let mut written = 0;
    loop {
        let len = match reader.read(&mut buf) {
            Ok(0) => return Ok(written),
            Ok(len) => len,
            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
            Err(e) => return Err(e),
        };
        writer.write(&buf[..len]);
        written += len as u64;
    }
}

fn to_hash(path: &PathBuf) -> u64 {
    let mut file = File::open(path).unwrap();
    let mut hasher = twox_hash::XxHash::default();
    copy(&mut file, &mut hasher).unwrap();
    hasher.finish()
}

fn read_file(path: &PathBuf) -> Vec<u8> {
    let mut file = File::open(path).unwrap();
    let mut vec = Vec::new();
    file.read_to_end(&mut vec).unwrap();
    vec
}

fn partition_by<I, F, R>(paths: I, f: F) -> LinearMap<R, Vec<PathBuf>>
    where I: Iterator<Item=PathBuf>, F: Fn(&PathBuf) -> R, R: Eq
{
    let mut map = LinearMap::with_capacity(paths.size_hint().0);
    for path in paths {
        let key = f(&path);
        map.entry(key).or_insert_with(Vec::new).push(path);
    }
    map
}

const DB_TABLE_SCHEMA: &'static str = r#"
CREATE TABLE files (
    hash BIGINT NOT NULL,
    path TEXT NOT NULL,
    size BIGINT NOT NULL
);
"#;

fn main() {
    let matches = App::new(crate_name!())
        .version(crate_version!())
        .author(crate_authors!("\n"))
        .arg(Arg::with_name("media_directory")
            .help("The location of the media store")
            .index(1)
            .multiple(true)
            .required(true))
        .arg(Arg::with_name("output-db")
            .short("o")
            .long("output-db")
            .help("Where to write SQLite database to")
            .value_name("FILE")
            .takes_value(true))
        .get_matches();

    let paths_to_search = matches.values_of("media_directory").unwrap();
    let output_db_path = matches.value_of("output-db");

    let db = rusqlite::Connection::open_in_memory().expect("failed to open sqlite db");
    db.execute_batch(DB_TABLE_SCHEMA).expect("failed to create db schema");

    let mut paths_by_size = BTreeMap::new();
    let mut total_files = 0;
    let mut total_size = 0;

    let pb = indicatif::ProgressBar::new_spinner();
    pb.set_style(
        indicatif::ProgressStyle::default_spinner()
        .template("{spinner} Collected metadata for {pos} files [{elapsed}]")
    );

    for path in paths_to_search {
        for entry in WalkDir::new(path) {
            let entry = entry.unwrap();
            if !entry.file_type().is_file() {
                continue
            }

            let file_size = entry.metadata().unwrap().len() as usize;
            paths_by_size.entry(file_size).or_insert_with(Vec::new).push(entry.path().to_owned());

            total_files += 1;
            total_size += file_size;

            pb.inc(1);
        }
    }

    pb.finish();
    pb.set_position(total_files);

    // println!("  Collected metadata for {} files", total_files);

    let pb = indicatif::ProgressBar::new(total_files);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Searching for possible duplicates  {bar:40} {pos:>9}/{len:9} [{elapsed}]")
    );

    let mut possible_total_size = 0;
    let mut possible_duplicates = Vec::new();
    for (file_size, paths) in paths_by_size.into_iter() {
        if paths.len() > 1 {
            possible_total_size += file_size * paths.len();
            possible_duplicates.push((file_size, paths))
        }
        pb.inc(1);
    }

    pb.finish();

    // We shuffle the vector as otherwise we do the small files first, and
    // that can skew the progress estimates massively.
    let mut rng = rand::thread_rng();
    rng.shuffle(possible_duplicates.as_mut_slice());

    let pb = indicatif::ProgressBar::new(possible_total_size as u64);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Comparing hashes                   {bar:40} {bytes:>9}/{total_bytes:9} [{elapsed}]")
    );

    let mut total_wasted_size = 0;

    for (file_size, paths) in possible_duplicates {
        if paths.len() == 1 {
            continue
        }

        let by_hash = partition_by(paths.into_iter(), to_hash);

        for (hash, paths) in by_hash {
            if paths.len() == 1 {
                pb.inc(file_size as u64);
                continue
            }

            let by_contents = partition_by(paths.into_iter(), read_file);

            for (_, paths) in by_contents {
                if paths.len() == 1 {
                    pb.inc(file_size as u64);
                    continue
                }

                for path in &paths {
                    db.execute("INSERT INTO files (hash, path, size) VALUES (?, ?, ?)", &[&(hash as i64), &path.to_str().unwrap(), &(file_size as i64)]).expect("failed to write to db");
                }

                let wasted = file_size * (paths.len() - 1);

                total_wasted_size += wasted;

                pb.inc((file_size * paths.len()) as u64);
            }
        }
    }

    pb.finish();

    println!();
    println!(
        "Total wasted size: {} out of {}. Percentage: {:.2}%",
        total_wasted_size.file_size(options::CONVENTIONAL).unwrap(),
        total_size.file_size(options::CONVENTIONAL).unwrap(),
        (total_wasted_size * 100) as f64 / total_size as f64,
    );

    if let Some(path) = output_db_path {
        let mut disk_db = rusqlite::Connection::open(path).expect("failed to open sqlite db");
        let backup = rusqlite::backup::Backup::new(&db, &mut disk_db).expect("failed to create backup");
        backup.run_to_completion(5, std::time::Duration::from_millis(0), None).expect("failed to write to disk");
    }
}
<|endoftext|>