<commit_before>//  * This file is part of `fmt` from the uutils coreutils package.
//  *
//  * (c) kwantam <kwantam@gmail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) PSKIP linebreak ostream parasplit tabwidth xanti xprefix

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use std::cmp;
use std::fs::File;
use std::io::{stdin, stdout, Write};
use std::io::{BufReader, BufWriter, Read};

use self::linebreak::break_lines;
use self::parasplit::ParagraphStream;

macro_rules! silent_unwrap(
    ($exp:expr) => (
        match $exp {
            Ok(_) => (),
            Err(_) => ::std::process::exit(1),
        }
    )
);

mod linebreak;
mod parasplit;

static ABOUT: &str = "Reformat paragraphs from input files (or stdin) to stdout.";
static MAX_WIDTH: usize = 2500;

static OPT_CROWN_MARGIN: &str = "crown-margin";
static OPT_TAGGED_PARAGRAPH: &str = "tagged-paragraph";
static OPT_PRESERVE_HEADERS: &str = "preserve-headers";
static OPT_SPLIT_ONLY: &str = "split-only";
static OPT_UNIFORM_SPACING: &str = "uniform-spacing";
static OPT_PREFIX: &str = "prefix";
static OPT_SKIP_PREFIX: &str = "skip-prefix";
static OPT_EXACT_PREFIX: &str = "exact-prefix";
static OPT_EXACT_SKIP_PREFIX: &str = "exact-skip-prefix";
static OPT_WIDTH: &str = "width";
static OPT_GOAL: &str = "goal";
static OPT_QUICK: &str = "quick";
static OPT_TAB_WIDTH: &str = "tab-width";

static ARG_FILES: &str = "files";

fn get_usage() -> String {
    format!("{} [OPTION]... [FILE]...", executable!())
}

pub type FileOrStdReader = BufReader<Box<dyn Read + 'static>>;
pub struct FmtOptions {
    crown: bool,
    tagged: bool,
    mail: bool,
    split_only: bool,
    use_prefix: bool,
    prefix: String,
    xprefix: bool,
    use_anti_prefix: bool,
    anti_prefix: String,
    xanti_prefix: bool,
    uniform: bool,
    quick: bool,
    width: usize,
    goal: usize,
    tabwidth: usize,
}

#[allow(clippy::cognitive_complexity)]
pub fn uumain(args: impl uucore::Args) -> i32 {
    let usage = get_usage();

    let matches = uu_app().usage(&usage[..]).get_matches_from(args);

    let mut files: Vec<String> = matches
        .values_of(ARG_FILES)
        .map(|v| v.map(ToString::to_string).collect())
        .unwrap_or_default();

    let mut fmt_opts = FmtOptions {
        crown: false,
        tagged: false,
        mail: false,
        uniform: false,
        quick: false,
        split_only: false,
        use_prefix: false,
        prefix: String::new(),
        xprefix: false,
        use_anti_prefix: false,
        anti_prefix: String::new(),
        xanti_prefix: false,
        width: 79,
        goal: 74,
        tabwidth: 8,
    };

    fmt_opts.tagged = matches.is_present(OPT_TAGGED_PARAGRAPH);
    if matches.is_present(OPT_CROWN_MARGIN) {
        fmt_opts.crown = true;
        fmt_opts.tagged = false;
    }
    fmt_opts.mail = matches.is_present(OPT_PRESERVE_HEADERS);
    fmt_opts.uniform = matches.is_present(OPT_UNIFORM_SPACING);
    fmt_opts.quick = matches.is_present(OPT_QUICK);
    if matches.is_present(OPT_SPLIT_ONLY) {
        fmt_opts.split_only = true;
        fmt_opts.crown = false;
        fmt_opts.tagged = false;
    }
    fmt_opts.xprefix = matches.is_present(OPT_EXACT_PREFIX);
    fmt_opts.xanti_prefix = matches.is_present(OPT_SKIP_PREFIX);

    if let Some(s) = matches.value_of(OPT_PREFIX).map(String::from) {
        fmt_opts.prefix = s;
        fmt_opts.use_prefix = true;
    };

    if let Some(s) = matches.value_of(OPT_SKIP_PREFIX).map(String::from) {
        fmt_opts.anti_prefix = s;
        fmt_opts.use_anti_prefix = true;
    };

    if let Some(s) = matches.value_of(OPT_WIDTH) {
        fmt_opts.width = match s.parse::<usize>() {
            Ok(t) => t,
            Err(e) => {
                crash!(1, "Invalid WIDTH specification: `{}': {}", s, e);
            }
        };
        if fmt_opts.width > MAX_WIDTH {
            crash!(
                1,
                "invalid width: '{}': Numerical result out of range",
                fmt_opts.width
            );
        }
        fmt_opts.goal = cmp::min(fmt_opts.width * 94 / 100, fmt_opts.width - 3);
    };

    if let Some(s) = matches.value_of(OPT_GOAL) {
        fmt_opts.goal = match s.parse::<usize>() {
            Ok(t) => t,
            Err(e) => {
                crash!(1, "Invalid GOAL specification: `{}': {}", s, e);
            }
        };
        if !matches.is_present(OPT_WIDTH) {
            fmt_opts.width = cmp::max(fmt_opts.goal * 100 / 94, fmt_opts.goal + 3);
        } else if fmt_opts.goal > fmt_opts.width {
            crash!(1, "GOAL cannot be greater than WIDTH.");
        }
    };

    if let Some(s) = matches.value_of(OPT_TAB_WIDTH) {
        fmt_opts.tabwidth = match s.parse::<usize>() {
            Ok(t) => t,
            Err(e) => {
                crash!(1, "Invalid TABWIDTH specification: `{}': {}", s, e);
            }
        };
    };

    if fmt_opts.tabwidth < 1 {
        fmt_opts.tabwidth = 1;
    }

    // immutable now
    let fmt_opts = fmt_opts;

    if files.is_empty() {
        files.push("-".to_owned());
    }

    let mut ostream = BufWriter::new(stdout());

    for i in files.iter().map(|x| &x[..]) {
        let mut fp = match i {
            "-" => BufReader::new(Box::new(stdin()) as Box<dyn Read + 'static>),
            _ => match File::open(i) {
                Ok(f) => BufReader::new(Box::new(f) as Box<dyn Read + 'static>),
                Err(e) => {
                    show_warning!("{}: {}", i, e);
                    continue;
                }
            },
        };
        let p_stream = ParagraphStream::new(&fmt_opts, &mut fp);
        for para_result in p_stream {
            match para_result {
                Err(s) => {
                    silent_unwrap!(ostream.write_all(s.as_bytes()));
                    silent_unwrap!(ostream.write_all(b"\n"));
                }
                Ok(para) => break_lines(&para, &fmt_opts, &mut ostream),
            }
        }

        // flush the output after each file
        silent_unwrap!(ostream.flush());
    }

    0
}

pub fn uu_app() -> App<'static, 'static> {
    App::new(executable!())
        .version(crate_version!())
        .about(ABOUT)
        .arg(
            Arg::with_name(OPT_CROWN_MARGIN)
                .short("c")
                .long(OPT_CROWN_MARGIN)
                .help(
                    "First and second line of paragraph
        may have different indentations, in which
        case the first line's indentation is preserved,
        and each subsequent line's indentation matches the second line.",
                ),
        )
        .arg(
            Arg::with_name(OPT_TAGGED_PARAGRAPH)
                .short("t")
                .long("tagged-paragraph")
                .help(
                    "Like -c, except that the first and second line of a paragraph *must*
                    have different indentation or they are treated as separate paragraphs.",
                ),
        )
        .arg(
            Arg::with_name(OPT_PRESERVE_HEADERS)
                .short("m")
                .long("preserve-headers")
                .help(
                    "Attempt to detect and preserve mail headers in the input.
                    Be careful when combining this flag with -p.",
                ),
        )
        .arg(
            Arg::with_name(OPT_SPLIT_ONLY)
                .short("s")
                .long("split-only")
                .help("Split lines only, do not reflow."),
        )
        .arg(
            Arg::with_name(OPT_UNIFORM_SPACING)
                .short("u")
                .long("uniform-spacing")
                .help(
                    "Insert exactly one
                    space between words, and two between sentences.
                    Sentence breaks in the input are detected as [?!.]
                    followed by two spaces or a newline; other punctuation
                    is not interpreted as a sentence break.",
                ),
        )
        .arg(
            Arg::with_name(OPT_PREFIX)
                .short("p")
                .long("prefix")
                .help(
                    "Reformat only lines
                    beginning with PREFIX, reattaching PREFIX to reformatted lines.
                    Unless -x is specified, leading whitespace will be ignored
                    when matching PREFIX.",
                )
                .value_name("PREFIX"),
        )
        .arg(
            Arg::with_name(OPT_SKIP_PREFIX)
                .short("P")
                .long("skip-prefix")
                .help(
                    "Do not reformat lines
                    beginning with PSKIP. Unless -X is specified, leading whitespace
                    will be ignored when matching PSKIP",
                )
                .value_name("PSKIP"),
        )
        .arg(
            Arg::with_name(OPT_EXACT_PREFIX)
                .short("x")
                .long("exact-prefix")
                .help(
                    "PREFIX must match at the
                    beginning of the line with no preceding whitespace.",
                ),
        )
        .arg(
            Arg::with_name(OPT_EXACT_SKIP_PREFIX)
                .short("X")
                .long("exact-skip-prefix")
                .help(
                    "PSKIP must match at the
                    beginning of the line with no preceding whitespace.",
                ),
        )
        .arg(
            Arg::with_name(OPT_WIDTH)
                .short("w")
                .long("width")
                .help("Fill output lines up to a maximum of WIDTH columns, default 79.")
                .value_name("WIDTH"),
        )
        .arg(
            Arg::with_name(OPT_GOAL)
                .short("g")
                .long("goal")
                .help("Goal width, default ~0.94*WIDTH. Must be less than WIDTH.")
                .value_name("GOAL"),
        )
        .arg(Arg::with_name(OPT_QUICK).short("q").long("quick").help(
            "Break lines more quickly at the
            expense of a potentially more ragged appearance.",
        ))
        .arg(
            Arg::with_name(OPT_TAB_WIDTH)
                .short("T")
                .long("tab-width")
                .help(
                    "Treat tabs as TABWIDTH spaces for
                    determining line length, default 8. Note that this is used only for
                    calculating line lengths; tabs are preserved in the output.",
                )
                .value_name("TABWIDTH"),
        )
        .arg(Arg::with_name(ARG_FILES).multiple(true).takes_value(true))
}
<commit_msg>fmt: fix indentation for help<commit_after>//  * This file is part of `fmt` from the uutils coreutils package.
//  *
//  * (c) kwantam <kwantam@gmail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) PSKIP linebreak ostream parasplit tabwidth xanti xprefix

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use std::cmp;
use std::fs::File;
use std::io::{stdin, stdout, Write};
use std::io::{BufReader, BufWriter, Read};

use self::linebreak::break_lines;
use self::parasplit::ParagraphStream;

macro_rules! silent_unwrap(
    ($exp:expr) => (
        match $exp {
            Ok(_) => (),
            Err(_) => ::std::process::exit(1),
        }
    )
);

mod linebreak;
mod parasplit;

static ABOUT: &str = "Reformat paragraphs from input files (or stdin) to stdout.";
static MAX_WIDTH: usize = 2500;

static OPT_CROWN_MARGIN: &str = "crown-margin";
static OPT_TAGGED_PARAGRAPH: &str = "tagged-paragraph";
static OPT_PRESERVE_HEADERS: &str = "preserve-headers";
static OPT_SPLIT_ONLY: &str = "split-only";
static OPT_UNIFORM_SPACING: &str = "uniform-spacing";
static OPT_PREFIX: &str = "prefix";
static OPT_SKIP_PREFIX: &str = "skip-prefix";
static OPT_EXACT_PREFIX: &str = "exact-prefix";
static OPT_EXACT_SKIP_PREFIX: &str = "exact-skip-prefix";
static OPT_WIDTH: &str = "width";
static OPT_GOAL: &str = "goal";
static OPT_QUICK: &str = "quick";
static OPT_TAB_WIDTH: &str = "tab-width";

static ARG_FILES: &str = "files";

fn get_usage() -> String {
    format!("{} [OPTION]... [FILE]...", executable!())
}

pub type FileOrStdReader = BufReader<Box<dyn Read + 'static>>;
pub struct FmtOptions {
    crown: bool,
    tagged: bool,
    mail: bool,
    split_only: bool,
    use_prefix: bool,
    prefix: String,
    xprefix: bool,
    use_anti_prefix: bool,
    anti_prefix: String,
    xanti_prefix: bool,
    uniform: bool,
    quick: bool,
    width: usize,
    goal: usize,
    tabwidth: usize,
}

#[allow(clippy::cognitive_complexity)]
pub fn uumain(args: impl uucore::Args) -> i32 {
    let usage = get_usage();

    let matches = uu_app().usage(&usage[..]).get_matches_from(args);

    let mut files: Vec<String> = matches
        .values_of(ARG_FILES)
        .map(|v| v.map(ToString::to_string).collect())
        .unwrap_or_default();

    let mut fmt_opts = FmtOptions {
        crown: false,
        tagged: false,
        mail: false,
        uniform: false,
        quick: false,
        split_only: false,
        use_prefix: false,
        prefix: String::new(),
        xprefix: false,
        use_anti_prefix: false,
        anti_prefix: String::new(),
        xanti_prefix: false,
        width: 79,
        goal: 74,
        tabwidth: 8,
    };

    fmt_opts.tagged = matches.is_present(OPT_TAGGED_PARAGRAPH);
    if matches.is_present(OPT_CROWN_MARGIN) {
        fmt_opts.crown = true;
        fmt_opts.tagged = false;
    }
    fmt_opts.mail = matches.is_present(OPT_PRESERVE_HEADERS);
    fmt_opts.uniform = matches.is_present(OPT_UNIFORM_SPACING);
    fmt_opts.quick = matches.is_present(OPT_QUICK);
    if matches.is_present(OPT_SPLIT_ONLY) {
        fmt_opts.split_only = true;
        fmt_opts.crown = false;
        fmt_opts.tagged = false;
    }
    fmt_opts.xprefix = matches.is_present(OPT_EXACT_PREFIX);
    fmt_opts.xanti_prefix = matches.is_present(OPT_SKIP_PREFIX);

    if let Some(s) = matches.value_of(OPT_PREFIX).map(String::from) {
        fmt_opts.prefix = s;
        fmt_opts.use_prefix = true;
    };

    if let Some(s) = matches.value_of(OPT_SKIP_PREFIX).map(String::from) {
        fmt_opts.anti_prefix = s;
        fmt_opts.use_anti_prefix = true;
    };

    if let Some(s) = matches.value_of(OPT_WIDTH) {
        fmt_opts.width = match s.parse::<usize>() {
            Ok(t) => t,
            Err(e) => {
                crash!(1, "Invalid WIDTH specification: `{}': {}", s, e);
            }
        };
        if fmt_opts.width > MAX_WIDTH {
            crash!(
                1,
                "invalid width: '{}': Numerical result out of range",
                fmt_opts.width
            );
        }
        fmt_opts.goal = cmp::min(fmt_opts.width * 94 / 100, fmt_opts.width - 3);
    };

    if let Some(s) = matches.value_of(OPT_GOAL) {
        fmt_opts.goal = match s.parse::<usize>() {
            Ok(t) => t,
            Err(e) => {
                crash!(1, "Invalid GOAL specification: `{}': {}", s, e);
            }
        };
        if !matches.is_present(OPT_WIDTH) {
            fmt_opts.width = cmp::max(fmt_opts.goal * 100 / 94, fmt_opts.goal + 3);
        } else if fmt_opts.goal > fmt_opts.width {
            crash!(1, "GOAL cannot be greater than WIDTH.");
        }
    };

    if let Some(s) = matches.value_of(OPT_TAB_WIDTH) {
        fmt_opts.tabwidth = match s.parse::<usize>() {
            Ok(t) => t,
            Err(e) => {
                crash!(1, "Invalid TABWIDTH specification: `{}': {}", s, e);
            }
        };
    };

    if fmt_opts.tabwidth < 1 {
        fmt_opts.tabwidth = 1;
    }

    // immutable now
    let fmt_opts = fmt_opts;

    if files.is_empty() {
        files.push("-".to_owned());
    }

    let mut ostream = BufWriter::new(stdout());

    for i in files.iter().map(|x| &x[..]) {
        let mut fp = match i {
            "-" => BufReader::new(Box::new(stdin()) as Box<dyn Read + 'static>),
            _ => match File::open(i) {
                Ok(f) => BufReader::new(Box::new(f) as Box<dyn Read + 'static>),
                Err(e) => {
                    show_warning!("{}: {}", i, e);
                    continue;
                }
            },
        };
        let p_stream = ParagraphStream::new(&fmt_opts, &mut fp);
        for para_result in p_stream {
            match para_result {
                Err(s) => {
                    silent_unwrap!(ostream.write_all(s.as_bytes()));
                    silent_unwrap!(ostream.write_all(b"\n"));
                }
                Ok(para) => break_lines(&para, &fmt_opts, &mut ostream),
            }
        }

        // flush the output after each file
        silent_unwrap!(ostream.flush());
    }

    0
}

pub fn uu_app() -> App<'static, 'static> {
    App::new(executable!())
        .version(crate_version!())
        .about(ABOUT)
        .arg(
            Arg::with_name(OPT_CROWN_MARGIN)
                .short("c")
                .long(OPT_CROWN_MARGIN)
                .help(
                    "First and second line of paragraph \
                    may have different indentations, in which \
                    case the first line's indentation is preserved, \
                    and each subsequent line's indentation matches the second line.",
                ),
        )
        .arg(
            Arg::with_name(OPT_TAGGED_PARAGRAPH)
                .short("t")
                .long("tagged-paragraph")
                .help(
                    "Like -c, except that the first and second line of a paragraph *must* \
                    have different indentation or they are treated as separate paragraphs.",
                ),
        )
        .arg(
            Arg::with_name(OPT_PRESERVE_HEADERS)
                .short("m")
                .long("preserve-headers")
                .help(
                    "Attempt to detect and preserve mail headers in the input. \
                    Be careful when combining this flag with -p.",
                ),
        )
        .arg(
            Arg::with_name(OPT_SPLIT_ONLY)
                .short("s")
                .long("split-only")
                .help("Split lines only, do not reflow."),
        )
        .arg(
            Arg::with_name(OPT_UNIFORM_SPACING)
                .short("u")
                .long("uniform-spacing")
                .help(
                    "Insert exactly one \
                    space between words, and two between sentences. \
                    Sentence breaks in the input are detected as [?!.] \
                    followed by two spaces or a newline; other punctuation \
                    is not interpreted as a sentence break.",
                ),
        )
        .arg(
            Arg::with_name(OPT_PREFIX)
                .short("p")
                .long("prefix")
                .help(
                    "Reformat only lines \
                    beginning with PREFIX, reattaching PREFIX to reformatted lines. \
                    Unless -x is specified, leading whitespace will be ignored \
                    when matching PREFIX.",
                )
                .value_name("PREFIX"),
        )
        .arg(
            Arg::with_name(OPT_SKIP_PREFIX)
                .short("P")
                .long("skip-prefix")
                .help(
                    "Do not reformat lines \
                    beginning with PSKIP. Unless -X is specified, leading whitespace \
                    will be ignored when matching PSKIP",
                )
                .value_name("PSKIP"),
        )
        .arg(
            Arg::with_name(OPT_EXACT_PREFIX)
                .short("x")
                .long("exact-prefix")
                .help(
                    "PREFIX must match at the \
                    beginning of the line with no preceding whitespace.",
                ),
        )
        .arg(
            Arg::with_name(OPT_EXACT_SKIP_PREFIX)
                .short("X")
                .long("exact-skip-prefix")
                .help(
                    "PSKIP must match at the \
                    beginning of the line with no preceding whitespace.",
                ),
        )
        .arg(
            Arg::with_name(OPT_WIDTH)
                .short("w")
                .long("width")
                .help("Fill output lines up to a maximum of WIDTH columns, default 79.")
                .value_name("WIDTH"),
        )
        .arg(
            Arg::with_name(OPT_GOAL)
                .short("g")
                .long("goal")
                .help("Goal width, default ~0.94*WIDTH. Must be less than WIDTH.")
                .value_name("GOAL"),
        )
        .arg(Arg::with_name(OPT_QUICK).short("q").long("quick").help(
            "Break lines more quickly at the \
            expense of a potentially more ragged appearance.",
        ))
        .arg(
            Arg::with_name(OPT_TAB_WIDTH)
                .short("T")
                .long("tab-width")
                .help(
                    "Treat tabs as TABWIDTH spaces for \
                    determining line length, default 8. Note that this is used only for \
                    calculating line lengths; tabs are preserved in the output.",
                )
                .value_name("TABWIDTH"),
        )
        .arg(Arg::with_name(ARG_FILES).multiple(true).takes_value(true))
}
<|endoftext|>


<commit_before>extern crate glob;

use gitignore;
use std::io;
use std::path::{Path, PathBuf};

use self::glob::{Pattern, PatternError};

pub struct NotificationFilter {
    cwd: PathBuf,
    filters: Vec<Pattern>,
    ignores: Vec<Pattern>,
    ignore_file: Option<gitignore::PatternSet>,
}

#[derive(Debug)]
pub enum NotificationError {
    BadPattern(PatternError),
    Io(io::Error),
}

impl NotificationFilter {
    pub fn new(current_dir: &Path,
               ignore_file: Option<gitignore::PatternSet>)
               -> Result<NotificationFilter, io::Error> {
        let canonicalized = try!(current_dir.canonicalize());

        Ok(NotificationFilter {
            cwd: canonicalized,
            filters: vec![],
            ignores: vec![],
            ignore_file: ignore_file,
        })
    }

    pub fn add_filter(&mut self, pattern: &str) -> Result<(), NotificationError> {
        let compiled = try!(self.pattern_for(pattern));
        self.filters.push(compiled);

        debug!("Adding filter: {}", pattern);

        Ok(())
    }

    pub fn add_ignore(&mut self, pattern: &str) -> Result<(), NotificationError> {
        let compiled = try!(self.pattern_for(pattern));
        self.ignores.push(compiled);

        debug!("Adding ignore: {}", pattern);

        Ok(())
    }

    fn pattern_for(&self, p: &str) -> Result<Pattern, PatternError> {
        let mut path = PathBuf::from(p);
        if path.is_relative() {
            path = self.cwd.join(path.as_path());
        }

        if let Ok(metadata) = path.metadata() {
            if metadata.is_dir() {
                path = path.join("*");
            }
        }

        Pattern::new(path.to_str().unwrap())
    }

    pub fn is_excluded(&self, path: &Path) -> bool {
        let path_as_str = path.to_str().unwrap();

        if let Ok(metadata) = path.metadata() {
            if metadata.is_dir() {
                debug!("Ignoring {:?}: is a directory", path);
                return true;
            }
        }

        for pattern in &self.ignores {
            if pattern.matches(path_as_str) {
                debug!("Ignoring {:?}: matched ignore filter", path);
                return true;
            }
        }

        for pattern in &self.filters {
            if pattern.matches(path_as_str) {
                return false;
            }
        }

        if let Some(ref ignore_file) = self.ignore_file {
            if ignore_file.is_excluded(path) {
                debug!("Ignoring {:?}: matched gitignore file", path);
                return true;
            }
        }

        if !self.filters.is_empty() {
            debug!("Ignoring {:?}: did not match any given filters", path);
        }

        !self.filters.is_empty()
    }
}

impl From<io::Error> for NotificationError {
    fn from(err: io::Error) -> NotificationError {
        NotificationError::Io(err)
    }
}

impl From<PatternError> for NotificationError {
    fn from(err: PatternError) -> NotificationError {
        NotificationError::BadPattern(err)
    }
}
<commit_msg>PollWatcher sometimes reports directories with new/deleted files<commit_after>extern crate glob;

use gitignore;
use std::io;
use std::path::{Path, PathBuf};

use self::glob::{Pattern, PatternError};

pub struct NotificationFilter {
    cwd: PathBuf,
    filters: Vec<Pattern>,
    ignores: Vec<Pattern>,
    ignore_file: Option<gitignore::PatternSet>,
}

#[derive(Debug)]
pub enum NotificationError {
    BadPattern(PatternError),
    Io(io::Error),
}

impl NotificationFilter {
    pub fn new(current_dir: &Path,
               ignore_file: Option<gitignore::PatternSet>)
               -> Result<NotificationFilter, io::Error> {
        let canonicalized = try!(current_dir.canonicalize());

        Ok(NotificationFilter {
            cwd: canonicalized,
            filters: vec![],
            ignores: vec![],
            ignore_file: ignore_file,
        })
    }

    pub fn add_filter(&mut self, pattern: &str) -> Result<(), NotificationError> {
        let compiled = try!(self.pattern_for(pattern));
        self.filters.push(compiled);

        debug!("Adding filter: {}", pattern);

        Ok(())
    }

    pub fn add_ignore(&mut self, pattern: &str) -> Result<(), NotificationError> {
        let compiled = try!(self.pattern_for(pattern));
        self.ignores.push(compiled);

        debug!("Adding ignore: {}", pattern);

        Ok(())
    }

    fn pattern_for(&self, p: &str) -> Result<Pattern, PatternError> {
        let mut path = PathBuf::from(p);
        if path.is_relative() {
            path = self.cwd.join(path.as_path());
        }

        if let Ok(metadata) = path.metadata() {
            if metadata.is_dir() {
                path = path.join("*");
            }
        }

        Pattern::new(path.to_str().unwrap())
    }

    pub fn is_excluded(&self, path: &Path) -> bool {
        let path_as_str = path.to_str().unwrap();

        for pattern in &self.ignores {
            if pattern.matches(path_as_str) {
                debug!("Ignoring {:?}: matched ignore filter", path);
                return true;
            }
        }

        for pattern in &self.filters {
            if pattern.matches(path_as_str) {
                return false;
            }
        }

        if let Some(ref ignore_file) = self.ignore_file {
            if ignore_file.is_excluded(path) {
                debug!("Ignoring {:?}: matched gitignore file", path);
                return true;
            }
        }

        if !self.filters.is_empty() {
            debug!("Ignoring {:?}: did not match any given filters", path);
        }

        !self.filters.is_empty()
    }
}

impl From<io::Error> for NotificationError {
    fn from(err: io::Error) -> NotificationError {
        NotificationError::Io(err)
    }
}

impl From<PatternError> for NotificationError {
    fn from(err: PatternError) -> NotificationError {
        NotificationError::BadPattern(err)
    }
}
<|endoftext|>


<commit_before>use byteorder::{LittleEndian, ReadBytesExt};
use internal::prelude::*;
use serde_json;
use std::{
    ffi::OsStr,
    fs::File,
    io::{
        BufReader,
        ErrorKind as IoErrorKind,
        Read,
        Result as IoResult
    },
    process::{
        Child,
        Command,
        Stdio
    },
    result::Result as StdResult
};
use super::{
    AudioSource,
    AudioType,
    DcaError,
    DcaMetadata,
    VoiceError
};

struct ChildContainer(Child);

impl Read for ChildContainer {
    fn read(&mut self, buffer: &mut [u8]) -> IoResult<usize> {
        self.0.stdout.as_mut().unwrap().read(buffer)
    }
}

impl Drop for ChildContainer {
    fn drop (&mut self) {
        if let Err(e) = self.0.wait() {
            debug!("[Voice] Error awaiting child process: {:?}", e);
        }
    }
}

struct InputSource<R: Read + Send + 'static> {
    stereo: bool,
    reader: R,
    kind: AudioType,
}

impl<R: Read + Send> AudioSource for InputSource<R> {
    fn is_stereo(&mut self) -> bool { self.stereo }

    fn get_type(&self) -> AudioType { self.kind }

    fn read_pcm_frame(&mut self, buffer: &mut [i16]) -> Option<usize> {
        for (i, v) in buffer.iter_mut().enumerate() {
            *v = match self.reader.read_i16::<LittleEndian>() {
                Ok(v) => v,
                Err(ref e) => {
                    return if e.kind() == IoErrorKind::UnexpectedEof {
                        Some(i)
                    } else {
                        None
                    }
                },
            }
        }

        Some(buffer.len())
    }

    fn read_opus_frame(&mut self) -> Option<Vec<u8>> {
        match self.reader.read_i16::<LittleEndian>() {
            Ok(size) => {
                if size <= 0 {
                    warn!("Invalid opus frame size: {}", size);
                    return None;
                }

                let mut frame = Vec::with_capacity(size as usize);

                {
                    let reader = self.reader.by_ref();

                    if reader.take(size as u64).read_to_end(&mut frame).is_err() {
                        return None;
                    }
                }

                Some(frame)
            },
            Err(ref e) => if e.kind() == IoErrorKind::UnexpectedEof {
                Some(Vec::new())
            } else {
                None
            },
        }
    }
}

/// Opens an audio file through `ffmpeg` and creates an audio source.
pub fn ffmpeg<P: AsRef<OsStr>>(path: P) -> Result<Box<AudioSource>> {
    let path = path.as_ref();

    // Will fail if the path is not to a file on the fs. Likely a YouTube URI.
    let is_stereo = is_stereo(path).unwrap_or(false);
    let stereo_val = if is_stereo { "2" } else { "1" };

    let args = [
        "-f",
        "s16le",
        "-ac",
        stereo_val,
        "-ar",
        "48000",
        "-acodec",
        "pcm_s16le",
        "-",
    ];

    let command = Command::new("ffmpeg")
        .arg("-i")
        .arg(path)
        .args(&args)
        .stderr(Stdio::null())
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .spawn()?;

    Ok(pcm(is_stereo, ChildContainer(command)))
}

/// Creates a streamed audio source from a DCA file.
/// Currently only accepts the DCA1 format.
pub fn dca<P: AsRef<OsStr>>(path: P) -> StdResult<Box<AudioSource>, DcaError> {
    let file = File::open(path.as_ref()).map_err(DcaError::IoError)?;

    let mut reader = BufReader::new(file);

    let mut header = [0u8; 4];

    // Read in the magic number to verify it's a DCA file.
    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    if header != b"DCA1"[..] {
        return Err(DcaError::InvalidHeader);
    }

    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    let size = (&header[..]).read_i32::<LittleEndian>().unwrap();

    // Sanity check
    if size < 2 {
        return Err(DcaError::InvalidSize(size));
    }

    let mut raw_json = Vec::with_capacity(size as usize);

    {
        let json_reader = reader.by_ref();
        json_reader
            .take(size as u64)
            .read_to_end(&mut raw_json)
            .map_err(DcaError::IoError)?;
    }

    let metadata = serde_json::from_slice::<DcaMetadata>(raw_json.as_slice())
        .map_err(DcaError::InvalidMetadata)?;

    Ok(opus(metadata.is_stereo(), reader))
}

/// Creates an Opus audio source. This makes certain assumptions: namely, that the input stream
/// is composed ONLY of opus frames of the variety that Discord expects.
///
/// If you want to decode a `.opus` file, use [`ffmpeg`]
///
/// [`ffmpeg`]: fn.ffmpeg.html
pub fn opus<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Opus,
    })
}

/// Creates a PCM audio source.
pub fn pcm<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Pcm,
    })
}

/// Creates a streamed audio source with `youtube-dl` and `ffmpeg`.
pub fn ytdl(uri: &str) -> Result<Box<AudioSource>> {
    let args = [
        "-f",
        "webm[abr>0]/bestaudio/best",
        "--no-playlist",
        "--print-json",
        "--skip-download",
        uri,
    ];

    let out = Command::new("youtube-dl")
        .args(&args)
        .stdin(Stdio::null())
        .output()?;

    if !out.status.success() {
        return Err(Error::Voice(VoiceError::YouTubeDLRun(out)));
    }

    let value = serde_json::from_reader(&out.stdout[..])?;
    let mut obj = match value {
        Value::Object(obj) => obj,
        other => return Err(Error::Voice(VoiceError::YouTubeDLProcessing(other))),
    };

    let uri = match obj.remove("url") {
        Some(v) => match v {
            Value::String(uri) => uri,
            other => return Err(Error::Voice(VoiceError::YouTubeDLUrl(other))),
        },
        None => return Err(Error::Voice(VoiceError::YouTubeDLUrl(Value::Object(obj)))),
    };

    ffmpeg(&uri)
}

fn is_stereo(path: &OsStr) -> Result<bool> {
    let args = ["-v", "quiet", "-of", "json", "-show-streams", "-i"];

    let out = Command::new("ffprobe")
        .args(&args)
        .arg(path)
        .stdin(Stdio::null())
        .output()?;

    let value: Value = serde_json::from_reader(&out.stdout[..])?;

    let streams = value
        .as_object()
        .and_then(|m| m.get("streams"))
        .and_then(|v| v.as_array())
        .ok_or(Error::Voice(VoiceError::Streams))?;

    let check = streams.iter().any(|stream| {
        let channels = stream
            .as_object()
            .and_then(|m| m.get("channels").and_then(|v| v.as_i64()));

        channels == Some(2)
    });

    Ok(check)
}
<commit_msg>Fix voice hang on `.stop()`<commit_after>use byteorder::{LittleEndian, ReadBytesExt};
use internal::prelude::*;
use serde_json;
use std::{
    ffi::OsStr,
    fs::File,
    io::{
        BufReader,
        ErrorKind as IoErrorKind,
        Read,
        Result as IoResult
    },
    process::{
        Child,
        Command,
        Stdio
    },
    result::Result as StdResult
};
use super::{
    AudioSource,
    AudioType,
    DcaError,
    DcaMetadata,
    VoiceError
};

struct ChildContainer(Child);

impl Read for ChildContainer {
    fn read(&mut self, buffer: &mut [u8]) -> IoResult<usize> {
        self.0.stdout.as_mut().unwrap().read(buffer)
    }
}

impl Drop for ChildContainer {
    fn drop (&mut self) {
        if let Err(e) = self.0.kill() {
            debug!("[Voice] Error awaiting child process: {:?}", e);
        }
    }
}

struct InputSource<R: Read + Send + 'static> {
    stereo: bool,
    reader: R,
    kind: AudioType,
}

impl<R: Read + Send> AudioSource for InputSource<R> {
    fn is_stereo(&mut self) -> bool { self.stereo }

    fn get_type(&self) -> AudioType { self.kind }

    fn read_pcm_frame(&mut self, buffer: &mut [i16]) -> Option<usize> {
        for (i, v) in buffer.iter_mut().enumerate() {
            *v = match self.reader.read_i16::<LittleEndian>() {
                Ok(v) => v,
                Err(ref e) => {
                    return if e.kind() == IoErrorKind::UnexpectedEof {
                        Some(i)
                    } else {
                        None
                    }
                },
            }
        }

        Some(buffer.len())
    }

    fn read_opus_frame(&mut self) -> Option<Vec<u8>> {
        match self.reader.read_i16::<LittleEndian>() {
            Ok(size) => {
                if size <= 0 {
                    warn!("Invalid opus frame size: {}", size);
                    return None;
                }

                let mut frame = Vec::with_capacity(size as usize);

                {
                    let reader = self.reader.by_ref();

                    if reader.take(size as u64).read_to_end(&mut frame).is_err() {
                        return None;
                    }
                }

                Some(frame)
            },
            Err(ref e) => if e.kind() == IoErrorKind::UnexpectedEof {
                Some(Vec::new())
            } else {
                None
            },
        }
    }
}

/// Opens an audio file through `ffmpeg` and creates an audio source.
pub fn ffmpeg<P: AsRef<OsStr>>(path: P) -> Result<Box<AudioSource>> {
    let path = path.as_ref();

    // Will fail if the path is not to a file on the fs. Likely a YouTube URI.
    let is_stereo = is_stereo(path).unwrap_or(false);
    let stereo_val = if is_stereo { "2" } else { "1" };

    let args = [
        "-f",
        "s16le",
        "-ac",
        stereo_val,
        "-ar",
        "48000",
        "-acodec",
        "pcm_s16le",
        "-",
    ];

    let command = Command::new("ffmpeg")
        .arg("-i")
        .arg(path)
        .args(&args)
        .stderr(Stdio::null())
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .spawn()?;

    Ok(pcm(is_stereo, ChildContainer(command)))
}

/// Creates a streamed audio source from a DCA file.
/// Currently only accepts the DCA1 format.
pub fn dca<P: AsRef<OsStr>>(path: P) -> StdResult<Box<AudioSource>, DcaError> {
    let file = File::open(path.as_ref()).map_err(DcaError::IoError)?;

    let mut reader = BufReader::new(file);

    let mut header = [0u8; 4];

    // Read in the magic number to verify it's a DCA file.
    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    if header != b"DCA1"[..] {
        return Err(DcaError::InvalidHeader);
    }

    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    let size = (&header[..]).read_i32::<LittleEndian>().unwrap();

    // Sanity check
    if size < 2 {
        return Err(DcaError::InvalidSize(size));
    }

    let mut raw_json = Vec::with_capacity(size as usize);

    {
        let json_reader = reader.by_ref();
        json_reader
            .take(size as u64)
            .read_to_end(&mut raw_json)
            .map_err(DcaError::IoError)?;
    }

    let metadata = serde_json::from_slice::<DcaMetadata>(raw_json.as_slice())
        .map_err(DcaError::InvalidMetadata)?;

    Ok(opus(metadata.is_stereo(), reader))
}

/// Creates an Opus audio source. This makes certain assumptions: namely, that the input stream
/// is composed ONLY of opus frames of the variety that Discord expects.
///
/// If you want to decode a `.opus` file, use [`ffmpeg`]
///
/// [`ffmpeg`]: fn.ffmpeg.html
pub fn opus<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Opus,
    })
}

/// Creates a PCM audio source.
pub fn pcm<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Pcm,
    })
}

/// Creates a streamed audio source with `youtube-dl` and `ffmpeg`.
pub fn ytdl(uri: &str) -> Result<Box<AudioSource>> {
    let args = [
        "-f",
        "webm[abr>0]/bestaudio/best",
        "--no-playlist",
        "--print-json",
        "--skip-download",
        uri,
    ];

    let out = Command::new("youtube-dl")
        .args(&args)
        .stdin(Stdio::null())
        .output()?;

    if !out.status.success() {
        return Err(Error::Voice(VoiceError::YouTubeDLRun(out)));
    }

    let value = serde_json::from_reader(&out.stdout[..])?;
    let mut obj = match value {
        Value::Object(obj) => obj,
        other => return Err(Error::Voice(VoiceError::YouTubeDLProcessing(other))),
    };

    let uri = match obj.remove("url") {
        Some(v) => match v {
            Value::String(uri) => uri,
            other => return Err(Error::Voice(VoiceError::YouTubeDLUrl(other))),
        },
        None => return Err(Error::Voice(VoiceError::YouTubeDLUrl(Value::Object(obj)))),
    };

    ffmpeg(&uri)
}

fn is_stereo(path: &OsStr) -> Result<bool> {
    let args = ["-v", "quiet", "-of", "json", "-show-streams", "-i"];

    let out = Command::new("ffprobe")
        .args(&args)
        .arg(path)
        .stdin(Stdio::null())
        .output()?;

    let value: Value = serde_json::from_reader(&out.stdout[..])?;

    let streams = value
        .as_object()
        .and_then(|m| m.get("streams"))
        .and_then(|v| v.as_array())
        .ok_or(Error::Voice(VoiceError::Streams))?;

    let check = streams.iter().any(|stream| {
        let channels = stream
            .as_object()
            .and_then(|m| m.get("channels").and_then(|v| v.as_i64()));

        channels == Some(2)
    });

    Ok(check)
}
<|endoftext|>


<commit_before>use futures::{future, Future};
use futures_cpupool::Builder;
use rdkafka::error::KafkaResult;
use regex::Regex;
use rocket::State;

use cache::Cache;
use error::*;
use live_consumer::LiveConsumerStore;
use metadata::{CONSUMERS, ClusterId, TopicName};
use metrics::build_topic_metrics;
use offsets::OffsetStore;
use web_server::pages::omnisearch::OmnisearchFormParams;

use std::collections::HashMap;

//
// ********** TOPICS LIST **********
//

#[get("/api/clusters/<cluster_id>/topics?<timestamp>")]
pub fn cluster_topics(cluster_id: ClusterId, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let brokers = brokers.unwrap();
    let topics = cache.topics.filter_clone(|&(ref c, _)| c == &cluster_id);
    let topic_metrics = build_topic_metrics(&cluster_id, &brokers, topics.len(), &cache.metrics);

    let mut result_data = Vec::with_capacity(topics.len());
    for &((_, ref topic_name), ref partitions) in topics.iter() {
        let def = (-1f64, -1f64);
        let rate = topic_metrics.get(topic_name).unwrap_or(&def);
        let errors = partitions.iter().find(|p| p.error.is_some());
        // let err_str = format!("{:?}", errors);
        result_data.push(json!((topic_name, partitions.len(), &errors, rate.0.round(), rate.1.round())));
    }

    //Ok(json_gzip_response(json!({"data": result_data})))
    json!({"data": result_data}).to_string()
}

//
// ********** BROKERS LIST **********
//

#[get("/api/clusters/<cluster_id>/brokers?<timestamp>")]
pub fn brokers(cluster_id: ClusterId, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let brokers = brokers.unwrap();
    let mut result_data = Vec::with_capacity(brokers.len());
    for broker in brokers {
        let rate = cache.metrics.get(&(cluster_id.to_owned(), broker.id))
            .and_then(|b_metrics| { b_metrics.topics.get("__TOTAL__").cloned() })
            .unwrap_or((-1f64, -1f64)); // TODO null instead?
        result_data.push(json!((broker.id, broker.hostname, rate.0.round(), rate.1.round())));
    }

    json!({"data": result_data}).to_string()
}

//
// ********** GROUP **********
//

struct GroupInfo {
    state: String,
    members: usize,
    stored_offsets: usize,
}

impl GroupInfo {
    fn new(state: String, members: usize) -> GroupInfo {
        GroupInfo { state: state, members: members, stored_offsets: 0 }
    }

    fn new_empty() -> GroupInfo {
        GroupInfo { state: "Offsets only".to_owned(), members: 0, stored_offsets: 0 }
    }

    fn add_offset(&mut self) {
        self.stored_offsets += 1;
    }
}

// TODO: add doc
// TODO: add limit
fn build_group_list<F>(cache: &Cache, filter_fn: F) -> HashMap<(ClusterId, String), GroupInfo>
        where F: Fn(&ClusterId, &TopicName, &String) -> bool {

    let mut groups: HashMap<(ClusterId, String), GroupInfo> = cache.groups
        .lock_iter(|iter| {
            iter.filter(|&(&(ref c, ref t), ref g)| filter_fn(&c, &t, &g.name))
                .map(|(&(ref c, _), g)| ((c.clone(), g.name.clone()), GroupInfo::new(g.state.clone(), g.members.len())))
                .collect()
        });

    let offsets = cache.offsets.filter_clone_k(|&(ref c, ref g, ref t)| filter_fn(c, t, g));
    for (cluster_id, group, _) in offsets {
        (*groups.entry((cluster_id, group)).or_insert(GroupInfo::new_empty())).add_offset();
    }

    groups
}

#[get("/api/clusters/<cluster_id>/groups?<timestamp>")]
pub fn cluster_groups(cluster_id: ClusterId, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let groups = build_group_list(cache.inner(), |c, _, _| &cluster_id == c);

    let mut result_data = Vec::with_capacity(groups.len());
    for ((_cluster_id, group_name), info) in groups {
        result_data.push(json!((group_name, info.state, info.members, info.stored_offsets)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/clusters/<cluster_id>/topics/<topic_name>/groups?<timestamp>")]
pub fn topic_groups(cluster_id: ClusterId, topic_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let groups = build_group_list(cache.inner(), |c, t, _| &cluster_id == c && topic_name == t);

    let mut result_data = Vec::with_capacity(groups.len());
    for ((_cluster_id, group_name), info) in groups {
        result_data.push(json!((group_name, info.state, info.members, info.stored_offsets)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/clusters/<cluster_id>/groups/<group_name>/members?<timestamp>")]
pub fn group_members(cluster_id: ClusterId, group_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let group = cache.groups.get(&(cluster_id.clone(), group_name.to_owned()));
    if group.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let group = group.unwrap();

    let mut result_data = Vec::with_capacity(group.members.len());
    for member in group.members {
        result_data.push(json!((member.id, member.client_id, member.client_host)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/clusters/<cluster_id>/groups/<group_name>/offsets?<timestamp>")]
pub fn group_offsets(cluster_id: ClusterId, group_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let offsets = cache.offsets_by_cluster_group(&cluster_id, &group_name.to_owned());

    let wms = time!("fetch wms", fetch_watermarks(&cluster_id, &offsets));
    let wms = match wms {
        Ok(wms) => wms,
        Err(e) => {
            error!("Error while fetching watermarks: {}", e);
            return json!({"data": []}).to_string();
        }
    };

    let mut result_data = Vec::with_capacity(offsets.len());
    for ((_cluster_id, _group, topic), partitions) in offsets {
        for (partition_id, &curr_offset) in partitions.iter().enumerate() {
            let (low, high) = match wms.get(&(topic.clone(), partition_id as i32)) {
                Some(&Ok((low_mark, high_mark))) => (low_mark, high_mark),
                _ => (-1, -1),
            };
            let (lag_shown, perc_shown) = match (high - low, high - curr_offset) {
                (_, lag) if lag < 0 => ("Out of retention".to_owned(), "".to_owned()),
                (0, lag) => (lag.to_string(), "0.0%".to_owned()),
                (size, lag) => (lag.to_string(), format!("{:.1}%", (lag as f64) / (size as f64) * 100.0))
            };
            result_data.push(json!((topic.clone(), partition_id, high-low, low, high, curr_offset, lag_shown, perc_shown)));
        }
    }

    json!({"data": result_data}).to_string()
}

fn fetch_watermarks(cluster_id: &ClusterId, offsets: &Vec<((ClusterId, String, TopicName), Vec<i64>)>)
        -> Result<HashMap<(TopicName, i32), KafkaResult<(i64, i64)>>> {
    let consumer = CONSUMERS.get_err(&cluster_id)?;

    let cpu_pool = Builder::new().pool_size(32).create();

    let mut futures = Vec::new();

    for &((_, _, ref topic), ref partitions) in offsets {
        for partition_id in 0..partitions.len() {
            let consumer_clone = consumer.clone();
            let topic_clone = topic.clone();
            let wm_future = cpu_pool.spawn_fn(move || {
                let wms = consumer_clone.fetch_watermarks(&topic_clone, partition_id as i32, 10000);
                Ok::<_, ()>(((topic_clone, partition_id as i32), wms))  // never fail
            });
            futures.push(wm_future);
        }
    }

    let watermarks = future::join_all(futures).wait().unwrap()
        .into_iter()
        .collect::<HashMap<_, _>>();

    Ok(watermarks)
}

//
// ********** TOPIC TOPOLOGY **********
//

#[get("/api/clusters/<cluster_id>/topics/<topic_name>/topology?<timestamp>")]
pub fn topic_topology(cluster_id: ClusterId, topic_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let partitions = cache.topics.get(&(cluster_id.to_owned(), topic_name.to_owned()));
    if partitions.is_none() {
        return json!({"data": []}).to_string();
    }

    let partitions = partitions.unwrap();

    let mut result_data = Vec::with_capacity(partitions.len());
    for p in partitions {
        result_data.push(json!((p.id, p.leader, p.replicas, p.isr, p.error)));
    }

    json!({"data": result_data}).to_string()
}

//
// ********** SEARCH **********
//

#[get("/api/search/consumer?<search>")]
pub fn consumer_search(search: OmnisearchFormParams, cache: State<Cache>) -> String {
    let groups = if search.regex {
        Regex::new(&search.string)
            .map(|r| build_group_list(&cache, |_, _, g| r.is_match(g)))
            .unwrap_or(HashMap::new())
    } else {
        build_group_list(&cache, |_, _, g| g.contains(&search.string))
    };

    let mut result_data = Vec::with_capacity(groups.len());
    for ((cluster_id, group_name), info) in groups {
        result_data.push(json!((cluster_id, group_name, info.state, info.members, info.stored_offsets)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/search/topic?<search>")]
pub fn topic_search(search: OmnisearchFormParams, cache: State<Cache>) -> String {
    let topics = if search.regex {
        Regex::new(&search.string)
            .map(|r| cache.topics.filter_clone(|&(_, ref name)| r.is_match(name)))
            .unwrap_or(Vec::new())
    } else {
        cache.topics.filter_clone(|&(_, ref name)| name.contains(&search.string))
    };

    let mut metrics_map = HashMap::new();
    let mut result_data = Vec::new();
    for ((cluster_id, topic_name), partitions) in topics {
        let cluster_metrics = metrics_map.entry(cluster_id.clone())
            .or_insert_with(|| {
                cache.brokers.get(&cluster_id)
                    .map(|brokers| build_topic_metrics(&cluster_id, &brokers, 100, &cache.metrics))
            });
        let (b_rate, m_rate) = cluster_metrics.as_ref()
            .and_then(|c_metrics| c_metrics.get(&topic_name).cloned())
            .unwrap_or((-1f64, -1f64));
        let errors = partitions.iter().find(|p| p.error.is_some());
        result_data.push(json!((cluster_id, topic_name, partitions.len(), errors, b_rate, m_rate)));
    }

    json!({"data": result_data}).to_string()
}

//
// ********** INTERNALS **********
//

#[get("/api/internals/cache/brokers?<timestamp>")]
pub fn cache_brokers(cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let result_data = cache.brokers.lock_iter(|brokers_cache_entry| {
        brokers_cache_entry.map(|(cluster_id, brokers)| {
            ((cluster_id.clone(), brokers.iter().map(|b| b.id).collect::<Vec<_>>()))
        })
        .collect::<Vec<_>>()
    });

    json!({"data": result_data}).to_string()
}

#[get("/api/internals/cache/metrics?<timestamp>")]
pub fn cache_metrics(cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let result_data = cache.metrics.lock_iter(|metrics_cache_entry| {
        metrics_cache_entry
            .map(|(&(ref cluster_id, ref broker_id), metrics)| {
                ((cluster_id.clone(), broker_id.clone(), metrics.topics.len()))
            }).collect::<Vec<_>>()
    });

    json!({"data": result_data}).to_string()
}

#[get("/api/internals/live_consumers?<timestamp>")]
pub fn live_consumers(live_consumers: State<LiveConsumerStore>, timestamp: &str) -> String {
    let _ = timestamp;
    let result_data = live_consumers.consumers().iter()
        .map(|consumer| ((consumer.id(), consumer.cluster_id().to_owned(), consumer.topic().to_owned(),
                          consumer.last_poll().elapsed().as_secs())))
        .collect::<Vec<_>>();
    json!({"data": result_data}).to_string()
}
<commit_msg>Improve consumer lag output<commit_after>use futures::{future, Future};
use futures_cpupool::Builder;
use rdkafka::error::KafkaResult;
use regex::Regex;
use rocket::State;

use cache::Cache;
use error::*;
use live_consumer::LiveConsumerStore;
use metadata::{CONSUMERS, ClusterId, TopicName};
use metrics::build_topic_metrics;
use offsets::OffsetStore;
use web_server::pages::omnisearch::OmnisearchFormParams;

use std::collections::HashMap;

//
// ********** TOPICS LIST **********
//

#[get("/api/clusters/<cluster_id>/topics?<timestamp>")]
pub fn cluster_topics(cluster_id: ClusterId, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let brokers = brokers.unwrap();
    let topics = cache.topics.filter_clone(|&(ref c, _)| c == &cluster_id);
    let topic_metrics = build_topic_metrics(&cluster_id, &brokers, topics.len(), &cache.metrics);

    let mut result_data = Vec::with_capacity(topics.len());
    for &((_, ref topic_name), ref partitions) in topics.iter() {
        let def = (-1f64, -1f64);
        let rate = topic_metrics.get(topic_name).unwrap_or(&def);
        let errors = partitions.iter().find(|p| p.error.is_some());
        // let err_str = format!("{:?}", errors);
        result_data.push(json!((topic_name, partitions.len(), &errors, rate.0.round(), rate.1.round())));
    }

    //Ok(json_gzip_response(json!({"data": result_data})))
    json!({"data": result_data}).to_string()
}

//
// ********** BROKERS LIST **********
//

#[get("/api/clusters/<cluster_id>/brokers?<timestamp>")]
pub fn brokers(cluster_id: ClusterId, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let brokers = brokers.unwrap();
    let mut result_data = Vec::with_capacity(brokers.len());
    for broker in brokers {
        let rate = cache.metrics.get(&(cluster_id.to_owned(), broker.id))
            .and_then(|b_metrics| { b_metrics.topics.get("__TOTAL__").cloned() })
            .unwrap_or((-1f64, -1f64)); // TODO null instead?
        result_data.push(json!((broker.id, broker.hostname, rate.0.round(), rate.1.round())));
    }

    json!({"data": result_data}).to_string()
}

//
// ********** GROUP **********
//

struct GroupInfo {
    state: String,
    members: usize,
    stored_offsets: usize,
}

impl GroupInfo {
    fn new(state: String, members: usize) -> GroupInfo {
        GroupInfo { state: state, members: members, stored_offsets: 0 }
    }

    fn new_empty() -> GroupInfo {
        GroupInfo { state: "Offsets only".to_owned(), members: 0, stored_offsets: 0 }
    }

    fn add_offset(&mut self) {
        self.stored_offsets += 1;
    }
}

// TODO: add doc
// TODO: add limit
fn build_group_list<F>(cache: &Cache, filter_fn: F) -> HashMap<(ClusterId, String), GroupInfo>
        where F: Fn(&ClusterId, &TopicName, &String) -> bool {

    let mut groups: HashMap<(ClusterId, String), GroupInfo> = cache.groups
        .lock_iter(|iter| {
            iter.filter(|&(&(ref c, ref t), ref g)| filter_fn(&c, &t, &g.name))
                .map(|(&(ref c, _), g)| ((c.clone(), g.name.clone()), GroupInfo::new(g.state.clone(), g.members.len())))
                .collect()
        });

    let offsets = cache.offsets.filter_clone_k(|&(ref c, ref g, ref t)| filter_fn(c, t, g));
    for (cluster_id, group, _) in offsets {
        (*groups.entry((cluster_id, group)).or_insert(GroupInfo::new_empty())).add_offset();
    }

    groups
}

#[get("/api/clusters/<cluster_id>/groups?<timestamp>")]
pub fn cluster_groups(cluster_id: ClusterId, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let groups = build_group_list(cache.inner(), |c, _, _| &cluster_id == c);

    let mut result_data = Vec::with_capacity(groups.len());
    for ((_cluster_id, group_name), info) in groups {
        result_data.push(json!((group_name, info.state, info.members, info.stored_offsets)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/clusters/<cluster_id>/topics/<topic_name>/groups?<timestamp>")]
pub fn topic_groups(cluster_id: ClusterId, topic_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let brokers = cache.brokers.get(&cluster_id);
    if brokers.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let groups = build_group_list(cache.inner(), |c, t, _| &cluster_id == c && topic_name == t);

    let mut result_data = Vec::with_capacity(groups.len());
    for ((_cluster_id, group_name), info) in groups {
        result_data.push(json!((group_name, info.state, info.members, info.stored_offsets)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/clusters/<cluster_id>/groups/<group_name>/members?<timestamp>")]
pub fn group_members(cluster_id: ClusterId, group_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let group = cache.groups.get(&(cluster_id.clone(), group_name.to_owned()));
    if group.is_none() {  // TODO: Improve here
        return json!({"data": []}).to_string();
    }

    let group = group.unwrap();

    let mut result_data = Vec::with_capacity(group.members.len());
    for member in group.members {
        result_data.push(json!((member.id, member.client_id, member.client_host)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/clusters/<cluster_id>/groups/<group_name>/offsets?<timestamp>")]
pub fn group_offsets(cluster_id: ClusterId, group_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let offsets = cache.offsets_by_cluster_group(&cluster_id, &group_name.to_owned());

    let wms = time!("fetch wms", fetch_watermarks(&cluster_id, &offsets));
    let wms = match wms {
        Ok(wms) => wms,
        Err(e) => {
            error!("Error while fetching watermarks: {}", e);
            return json!({"data": []}).to_string();
        }
    };

    let mut result_data = Vec::with_capacity(offsets.len());
    for ((_cluster_id, _group, topic), partitions) in offsets {
        for (partition_id, &curr_offset) in partitions.iter().enumerate() {
            let (low, high) = match wms.get(&(topic.clone(), partition_id as i32)) {
                Some(&Ok((low_mark, high_mark))) => (low_mark, high_mark),
                _ => (-1, -1),
            };
            let (lag_shown, perc_shown) = match (high - low, high - curr_offset) {
                (0, lag) => ("Empty topic".to_owned(), "0.0%".to_owned()),
                (size, lag) if lag > size => ("Out of retention".to_owned(), "".to_owned()),
                (size, lag) => (lag.to_string(), format!("{:.1}%", (lag as f64) / (size as f64) * 100.0))
            };
            result_data.push(json!((topic.clone(), partition_id, high-low, low, high, curr_offset, lag_shown, perc_shown)));
        }
    }

    json!({"data": result_data}).to_string()
}

fn fetch_watermarks(cluster_id: &ClusterId, offsets: &Vec<((ClusterId, String, TopicName), Vec<i64>)>)
        -> Result<HashMap<(TopicName, i32), KafkaResult<(i64, i64)>>> {
    let consumer = CONSUMERS.get_err(&cluster_id)?;

    let cpu_pool = Builder::new().pool_size(32).create();

    let mut futures = Vec::new();

    for &((_, _, ref topic), ref partitions) in offsets {
        for partition_id in 0..partitions.len() {
            let consumer_clone = consumer.clone();
            let topic_clone = topic.clone();
            let wm_future = cpu_pool.spawn_fn(move || {
                let wms = consumer_clone.fetch_watermarks(&topic_clone, partition_id as i32, 10000);
                Ok::<_, ()>(((topic_clone, partition_id as i32), wms))  // never fail
            });
            futures.push(wm_future);
        }
    }

    let watermarks = future::join_all(futures).wait().unwrap()
        .into_iter()
        .collect::<HashMap<_, _>>();

    Ok(watermarks)
}

//
// ********** TOPIC TOPOLOGY **********
//

#[get("/api/clusters/<cluster_id>/topics/<topic_name>/topology?<timestamp>")]
pub fn topic_topology(cluster_id: ClusterId, topic_name: &str, cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let partitions = cache.topics.get(&(cluster_id.to_owned(), topic_name.to_owned()));
    if partitions.is_none() {
        return json!({"data": []}).to_string();
    }

    let partitions = partitions.unwrap();

    let mut result_data = Vec::with_capacity(partitions.len());
    for p in partitions {
        result_data.push(json!((p.id, p.leader, p.replicas, p.isr, p.error)));
    }

    json!({"data": result_data}).to_string()
}

//
// ********** SEARCH **********
//

#[get("/api/search/consumer?<search>")]
pub fn consumer_search(search: OmnisearchFormParams, cache: State<Cache>) -> String {
    let groups = if search.regex {
        Regex::new(&search.string)
            .map(|r| build_group_list(&cache, |_, _, g| r.is_match(g)))
            .unwrap_or(HashMap::new())
    } else {
        build_group_list(&cache, |_, _, g| g.contains(&search.string))
    };

    let mut result_data = Vec::with_capacity(groups.len());
    for ((cluster_id, group_name), info) in groups {
        result_data.push(json!((cluster_id, group_name, info.state, info.members, info.stored_offsets)));
    }

    json!({"data": result_data}).to_string()
}

#[get("/api/search/topic?<search>")]
pub fn topic_search(search: OmnisearchFormParams, cache: State<Cache>) -> String {
    let topics = if search.regex {
        Regex::new(&search.string)
            .map(|r| cache.topics.filter_clone(|&(_, ref name)| r.is_match(name)))
            .unwrap_or(Vec::new())
    } else {
        cache.topics.filter_clone(|&(_, ref name)| name.contains(&search.string))
    };

    let mut metrics_map = HashMap::new();
    let mut result_data = Vec::new();
    for ((cluster_id, topic_name), partitions) in topics {
        let cluster_metrics = metrics_map.entry(cluster_id.clone())
            .or_insert_with(|| {
                cache.brokers.get(&cluster_id)
                    .map(|brokers| build_topic_metrics(&cluster_id, &brokers, 100, &cache.metrics))
            });
        let (b_rate, m_rate) = cluster_metrics.as_ref()
            .and_then(|c_metrics| c_metrics.get(&topic_name).cloned())
            .unwrap_or((-1f64, -1f64));
        let errors = partitions.iter().find(|p| p.error.is_some());
        result_data.push(json!((cluster_id, topic_name, partitions.len(), errors, b_rate, m_rate)));
    }

    json!({"data": result_data}).to_string()
}

//
// ********** INTERNALS **********
//

#[get("/api/internals/cache/brokers?<timestamp>")]
pub fn cache_brokers(cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let result_data = cache.brokers.lock_iter(|brokers_cache_entry| {
        brokers_cache_entry.map(|(cluster_id, brokers)| {
            ((cluster_id.clone(), brokers.iter().map(|b| b.id).collect::<Vec<_>>()))
        })
        .collect::<Vec<_>>()
    });

    json!({"data": result_data}).to_string()
}

#[get("/api/internals/cache/metrics?<timestamp>")]
pub fn cache_metrics(cache: State<Cache>, timestamp: &str) -> String {
    let _ = timestamp;
    let result_data = cache.metrics.lock_iter(|metrics_cache_entry| {
        metrics_cache_entry
            .map(|(&(ref cluster_id, ref broker_id), metrics)| {
                ((cluster_id.clone(), broker_id.clone(), metrics.topics.len()))
            }).collect::<Vec<_>>()
    });

    json!({"data": result_data}).to_string()
}

#[get("/api/internals/live_consumers?<timestamp>")]
pub fn live_consumers(live_consumers: State<LiveConsumerStore>, timestamp: &str) -> String {
    let _ = timestamp;
    let result_data = live_consumers.consumers().iter()
        .map(|consumer| ((consumer.id(), consumer.cluster_id().to_owned(), consumer.topic().to_owned(),
                          consumer.last_poll().elapsed().as_secs())))
        .collect::<Vec<_>>();
    json!({"data": result_data}).to_string()
}
<|endoftext|>


<commit_before>//! Computes a perfect hash table using [the CHD
//! algorithm](http://cmph.sourceforge.net/papers/esa09.pdf).
//!
//! Strongly inspired by https://github.com/sfackler/rust-phf

extern crate time;

use std::rand::{XorShiftRng, Rng, mod};
use std::collections::HashMap;

static NOVAL: u32 = 0xFFFF_FFFF;

/// FNV
fn hash(s: &str, h: u64) -> u64 {
    let mut g = 0xcbf29ce484222325 ^ h;
    for b in s.bytes() { g ^= b as u64; g *= 0x100000001b3; }
    g
}

pub fn displace(f1: u32, f2: u32, d1: u32, d2: u32) -> u32 {
    d2 + f1 * d1 + f2
}
fn split(hash: u64) -> Hash {
    let bits = 21;
    let mask = (1 << bits) - 1;
    Hash {
        g: (hash & mask) as u32,
        f1: ((hash >> bits) & mask) as u32,
        f2: ((hash >> (2 * bits)) & mask) as u32
    }
}

struct Hash { g: u32, f1: u32, f2: u32 }

fn try_phf_table(values: &[(u32, String)],
                 lambda: uint, seed: u64) -> Option<(Vec<(u32, u32)>, Vec<u32>)> {

    let hashes: Vec<(Hash, u32)> =
        values.iter().map(|&(n, ref s)| (split(hash(s.as_slice(), seed)), n)).collect();

    let table_len = hashes.len();
    let buckets_len = (table_len + lambda - 1) / lambda;

    // group the elements into buckets of lambda (on average, for a
    // good hash) based on the suffix of their hash.
    let mut buckets = Vec::from_fn(buckets_len, |i| (i, vec![]));
    for (i, &(h, cp)) in hashes.iter().enumerate() {
        buckets.get_mut(h.g as uint % buckets_len).mut1().push((h, cp))
    }

    // place the large buckets first.
    buckets.sort_by(|&(_, ref a), &(_, ref b)| b.len().cmp(&a.len()));

    // this stores the final computed backing vector, i.e. getting the
    // value for `foo` is "just" `map[displace(hash(foo))]`, where
    // `displace` uses the pair of displacements that we computed
    // (stored in `disps`).
    let mut map = Vec::from_elem(table_len, NOVAL);
    let mut disps = Vec::from_elem(buckets_len, (0, 0));

    // the set of index -> value mappings for the next bucket to be
    // placed; we need it separate because it may not work, so we may
    // have to roll back.
    let mut try_map = HashMap::new();

    // heuristically avoiding doing everything in the same order seems
    // good? I dunno; but anyway, we get vectors of indices and
    // shuffle them.
    let mut d1s = Vec::from_fn(table_len, |i| i as u32);
    let mut d2s = d1s.clone();
    let mut rng: XorShiftRng = rand::random();

    // run through each bucket and try to fit the elements into the
    // array by choosing appropriate adjusting factors
    // ("displacements") that allow the other two parts of the hash to
    // be combined into an empty index.
    'next_bucket: for &(bkt_idx, ref bkt_keys) in buckets.iter() {
        rng.shuffle(d1s.as_mut_slice());
        rng.shuffle(d2s.as_mut_slice());

        // exhaustively search for a pair of displacements that work.
        for &d1 in d1s.iter() {
            'next_disp: for &d2 in d2s.iter() {
                try_map.clear();

                // run through the elements to see if they all fit
                for &(h, cp) in bkt_keys.iter() {
                    // adjust the index slightly using the
                    // displacements, hoping that this will allow us
                    // to avoid collisions.
                    let idx = (displace(h.f1, h.f2, d1, d2) % table_len as u32) as uint;

                    if map[idx] != NOVAL || try_map.contains_key(&idx) {
                        // nope, this one is taken, so this pair of
                        // displacements doesn't work.
                        continue 'next_disp
                    }
                    try_map.insert(idx, cp);
                }

                // everything works! let's lock it in and go to the
                // next bucket.
                *disps.get_mut(bkt_idx) = (d1, d2);
                for (idx, cp) in try_map.iter() {
                    *map.get_mut(*idx) = *cp
                }
                continue 'next_bucket
            }
        }

        // if we're here, we ran through all displacements for a
        // bucket and didn't find one that worked, so we can't make
        // the hash table.
        return None
    }

    return Some((disps, map))
}

pub fn create_phf(data: &[(u32, String)], lambda: uint,
                  max_tries: uint) -> (u64, Vec<(u32, u32)>, Vec<u32>) {
    let start = time::precise_time_s();

    for i in range(0, max_tries){
        let my_start = time::precise_time_s();
        println!("PHF #{}: starting {:.2}", i, my_start - start);

        let seed = rand::random();
        match try_phf_table(data, lambda, seed) {
            Some((disp, map)) => {
                let end = time::precise_time_s();
                println!("PHF took: total {:.2} s, successive {:.2} s",
                         end - start, end - my_start);
                return (seed, disp, map)
            }
            None => {}
        }
    }
    fail!("could not create a length {} PHF with {}, {}",
          data.len(), lambda, max_tries);
}
<commit_msg>Move shuffling; by-far the slowest part of PHF generation.<commit_after>//! Computes a perfect hash table using [the CHD
//! algorithm](http://cmph.sourceforge.net/papers/esa09.pdf).
//!
//! Strongly inspired by https://github.com/sfackler/rust-phf

extern crate time;

use std::rand::{XorShiftRng, Rng, mod};
use std::collections::HashMap;

static NOVAL: u32 = 0xFFFF_FFFF;

/// FNV
fn hash(s: &str, h: u64) -> u64 {
    let mut g = 0xcbf29ce484222325 ^ h;
    for b in s.bytes() { g ^= b as u64; g *= 0x100000001b3; }
    g
}

pub fn displace(f1: u32, f2: u32, d1: u32, d2: u32) -> u32 {
    d2 + f1 * d1 + f2
}
fn split(hash: u64) -> Hash {
    let bits = 21;
    let mask = (1 << bits) - 1;
    Hash {
        g: (hash & mask) as u32,
        f1: ((hash >> bits) & mask) as u32,
        f2: ((hash >> (2 * bits)) & mask) as u32
    }
}

struct Hash { g: u32, f1: u32, f2: u32 }

fn try_phf_table(values: &[(u32, String)],
                 lambda: uint, seed: u64) -> Option<(Vec<(u32, u32)>, Vec<u32>)> {

    let hashes: Vec<(Hash, u32)> =
        values.iter().map(|&(n, ref s)| (split(hash(s.as_slice(), seed)), n)).collect();

    let table_len = hashes.len();
    let buckets_len = (table_len + lambda - 1) / lambda;

    // group the elements into buckets of lambda (on average, for a
    // good hash) based on the suffix of their hash.
    let mut buckets = Vec::from_fn(buckets_len, |i| (i, vec![]));
    for (i, &(h, cp)) in hashes.iter().enumerate() {
        buckets.get_mut(h.g as uint % buckets_len).mut1().push((h, cp))
    }

    // place the large buckets first.
    buckets.sort_by(|&(_, ref a), &(_, ref b)| b.len().cmp(&a.len()));

    // this stores the final computed backing vector, i.e. getting the
    // value for `foo` is "just" `map[displace(hash(foo))]`, where
    // `displace` uses the pair of displacements that we computed
    // (stored in `disps`).
    let mut map = Vec::from_elem(table_len, NOVAL);
    let mut disps = Vec::from_elem(buckets_len, (0, 0));

    // the set of index -> value mappings for the next bucket to be
    // placed; we need it separate because it may not work, so we may
    // have to roll back.
    let mut try_map = HashMap::new();

    // heuristically avoiding doing everything in the same order seems
    // good? I dunno; but anyway, we get vectors of indices and
    // shuffle them.
    let mut d1s = Vec::from_fn(table_len, |i| i as u32);
    let mut d2s = d1s.clone();
    let mut rng: XorShiftRng = rand::random();
    rng.shuffle(d1s.as_mut_slice());
    rng.shuffle(d2s.as_mut_slice());

    // run through each bucket and try to fit the elements into the
    // array by choosing appropriate adjusting factors
    // ("displacements") that allow the other two parts of the hash to
    // be combined into an empty index.
    'next_bucket: for &(bkt_idx, ref bkt_keys) in buckets.iter() {
        // exhaustively search for a pair of displacements that work.
        for &d1 in d1s.iter() {
            'next_disp: for &d2 in d2s.iter() {
                try_map.clear();

                // run through the elements to see if they all fit
                for &(h, cp) in bkt_keys.iter() {
                    // adjust the index slightly using the
                    // displacements, hoping that this will allow us
                    // to avoid collisions.
                    let idx = (displace(h.f1, h.f2, d1, d2) % table_len as u32) as uint;

                    if map[idx] != NOVAL || try_map.contains_key(&idx) {
                        // nope, this one is taken, so this pair of
                        // displacements doesn't work.
                        continue 'next_disp
                    }
                    try_map.insert(idx, cp);
                }

                // everything works! let's lock it in and go to the
                // next bucket.
                *disps.get_mut(bkt_idx) = (d1, d2);
                for (idx, cp) in try_map.iter() {
                    *map.get_mut(*idx) = *cp
                }
                continue 'next_bucket
            }
        }

        // if we're here, we ran through all displacements for a
        // bucket and didn't find one that worked, so we can't make
        // the hash table.
        return None
    }

    return Some((disps, map))
}

pub fn create_phf(data: &[(u32, String)], lambda: uint,
                  max_tries: uint) -> (u64, Vec<(u32, u32)>, Vec<u32>) {
    let start = time::precise_time_s();

    for i in range(0, max_tries){
        let my_start = time::precise_time_s();
        println!("PHF #{}: starting {:.2}", i, my_start - start);

        let seed = rand::random();
        match try_phf_table(data, lambda, seed) {
            Some((disp, map)) => {
                let end = time::precise_time_s();
                println!("PHF took: total {:.2} s, successive {:.2} s",
                         end - start, end - my_start);
                return (seed, disp, map)
            }
            None => {}
        }
    }
    fail!("could not create a length {} PHF with {}, {}",
          data.len(), lambda, max_tries);
}
<|endoftext|>


<commit_before>use dirs;
use libc;
use logger::{self, Level};
use std::fs::{self, DirBuilder};
use std::io::{self, Read};
use std::os::unix::fs::DirBuilderExt;
use std::path::{Path, PathBuf};

pub const NEWSBOAT_SUBDIR_XDG: &str = "newsboat";
pub const NEWSBOAT_CONFIG_SUBDIR: &str = ".newsboat";
pub const NEWSBEUTER_SUBDIR_XDG: &str = "newsbeuter";
pub const NEWSBEUTER_CONFIG_SUBDIR: &str = ".newsbeuter";
pub const LOCK_SUFFIX: &str = ".lock";

#[derive(Debug)]
pub struct ConfigPaths {
    m_env_home: PathBuf,
    m_error_message: String,

    m_data_dir: PathBuf,
    m_config_dir: PathBuf,

    m_url_file: String,
    m_cache_file: String,
    m_config_file: String,
    m_lock_file: String,
    m_queue_file: String,
    m_search_file: String,
    m_cmdline_file: String,

    m_silent: bool,
    m_using_nonstandard_configs: bool,
}

impl ConfigPaths {
    pub fn new() -> ConfigPaths {
        let mut config_paths = ConfigPaths {
            m_env_home: PathBuf::new(),
            m_error_message: String::new(),

            m_data_dir: PathBuf::new(),
            m_config_dir: PathBuf::new(),

            m_url_file: String::from("urls"),
            m_cache_file: String::from("cache.db"),
            m_config_file: String::from("config"),
            m_lock_file: String::new(),
            m_queue_file: String::from("queue"),
            m_search_file: String::new(),
            m_cmdline_file: String::new(),

            m_silent: false,
            m_using_nonstandard_configs: false,
        };

        let m_env_home = dirs::home_dir();
        if m_env_home.is_none() {
            let uid = unsafe { libc::getuid() };

            // TODO: i18n
            config_paths.m_error_message = format!("Fatal error: couldn't determine home directory!\nPlease set the HOME environment variable or add a valid user for UID {}!", uid);

            return config_paths;
        }

        // hitting this branch means we found a home directory
        // we can now safely call unwrap on all other functions in the dirs crate

        config_paths.m_env_home = m_env_home.unwrap();
        config_paths.find_dirs();
        config_paths
    }

    fn migrate_data_from_newsbeuter_xdg(&mut self) -> bool {
        // This can't panic because we've tested we can find the home directory in ConfigPaths::new
        // This should be replaced with proper error handling after this is not used by c++ anymore
        let xdg_config_dir = dirs::config_dir().unwrap();
        let xdg_data_dir = dirs::data_dir().unwrap();

        let newsbeuter_config_dir = xdg_config_dir.join(NEWSBEUTER_SUBDIR_XDG);
        let newsbeuter_data_dir = xdg_data_dir.join(NEWSBEUTER_SUBDIR_XDG);

        let newsboat_config_dir = xdg_config_dir.join(NEWSBOAT_SUBDIR_XDG);
        let newsboat_data_dir = xdg_data_dir.join(NEWSBOAT_SUBDIR_XDG);

        if !newsbeuter_config_dir.is_dir() {
            return false;
        }

        fn exists(path: &Path) -> bool {
            let exists = path.exists();
            if exists {
                log!(
                    Level::Debug,
                    "{:?} already exists, aborting XDG migration.",
                    path
                );
            }
            return exists;
        }

        if exists(&newsboat_config_dir) {
            return false;
        }

        if exists(&newsboat_data_dir) {
            return false;
        }

        self.m_config_dir = newsboat_config_dir;
        self.m_data_dir = newsboat_data_dir;

        if !self.m_silent {
            eprintln!("Migrating configs and data from Newsbeuter's XDG dirs...");
        }

        if !try_mkdir(&self.m_config_dir) {
            return false;
        }

        if !try_mkdir(&self.m_data_dir) {
            return false;
        }

        // in config
        migrate_file(&newsbeuter_config_dir, &self.m_config_dir, "urls").is_ok();
        migrate_file(&newsbeuter_config_dir, &self.m_config_dir, "config").is_ok();

        // in data
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "cache.db").is_ok();
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "queue").is_ok();
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "history.search").is_ok();
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "history.cmdline").is_ok();

        return true;
    }

    fn migrate_data_from_newsbeuter_simple(&self) -> bool {
        let newsbeuter_dir = self.m_env_home.join(NEWSBEUTER_CONFIG_SUBDIR);

        if !newsbeuter_dir.is_dir() {
            return false;
        }

        let newsboat_dir = self.m_env_home.join(NEWSBOAT_CONFIG_SUBDIR);

        if newsboat_dir.exists() {
            log!(
                Level::Debug,
                "{:?} already exists, aborting migration.",
                newsboat_dir
            );
            return false;
        }

        if !self.m_silent {
            eprintln!("Migrating configs and data from Newsbeuter's dotdir...");
        }

        match mkdir(&newsboat_dir, 0o700) {
            Ok(_) => (),
            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => (),
            Err(err) => {
                if !self.m_silent {
                    eprintln!(
                        "Aborting migration because mkdir on {:?} failed: {}",
                        newsboat_dir, err
                    );
                }
                return false;
            }
        };

        migrate_file(&newsbeuter_dir, &newsboat_dir, "urls").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "cache.db").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "config").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "queue").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "history.search").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "history.cmdline").ok();

        return true;
    }

    fn migrate_data_from_newsbeuter(&mut self) {
        let mut migrated = self.migrate_data_from_newsbeuter_xdg();

        if migrated {
            // Re-running to pick up XDG dirs
            self.m_url_file = "urls".into();
            self.m_cache_file = "cache.db".into();
            self.m_config_file = "config".into();
            self.m_queue_file = "queue".into();
            self.find_dirs();
        } else {
            migrated = self.migrate_data_from_newsbeuter_simple();
        }

        if migrated {
            eprintln!("\nPlease check the results and press Enter to continue.");
            // only wait for enter, ignore actual input
            let mut buf = [0; 16];
            io::stdin().read(&mut buf).ok();
        }
    }

    fn create_dirs(&self) -> bool {
        return try_mkdir(&self.m_config_dir) && try_mkdir(&self.m_data_dir);
    }

    fn find_dirs(&mut self) {
        self.m_config_dir = self.m_env_home.join(NEWSBOAT_CONFIG_SUBDIR);

        self.m_data_dir = self.m_config_dir.clone();

        // Will change config_dir and data_dir to point to XDG if XDG
        // directories are available.
        self.find_dirs_xdg();

        // in config
        self.m_url_file = self
            .m_config_dir
            .join(&self.m_url_file)
            .to_string_lossy()
            .into();
        self.m_config_file = self
            .m_config_dir
            .join(&self.m_config_file)
            .to_string_lossy()
            .into();

        // in data
        self.m_cache_file = self
            .m_data_dir
            .join(&self.m_cache_file)
            .to_string_lossy()
            .into();
        self.m_lock_file = self.m_cache_file.clone() + LOCK_SUFFIX;
        self.m_queue_file = self
            .m_data_dir
            .join(&self.m_queue_file)
            .to_string_lossy()
            .into();
        self.m_search_file = self
            .m_data_dir
            .join("history.search")
            .to_string_lossy()
            .into();
        self.m_cmdline_file = self
            .m_data_dir
            .join("history.cmdline")
            .to_string_lossy()
            .into();
    }

    fn find_dirs_xdg(&mut self) {
        // This can't panic because we've tested we can find the home directory in ConfigPaths::new
        // This should be replaced with proper error handling after this is not used by c++ anymore
        let config_dir = dirs::config_dir().unwrap().join(NEWSBOAT_SUBDIR_XDG);
        let data_dir = dirs::data_dir().unwrap().join(NEWSBOAT_SUBDIR_XDG);

        if !config_dir.is_dir() {
            return;
        }

        self.m_config_dir = config_dir;
        self.m_data_dir = data_dir;
    }

    /// \brief Indicates if the object can be used.
    ///
    /// If this method returned `false`, the cause for initialization
    /// failure can be found using `error_message()`.
    pub fn initialized(&self) -> bool {
        !self.m_env_home.as_os_str().is_empty()
    }

    /// Returns explanation why initialization failed.
    ///
    /// \note You shouldn't call this unless `initialized()` returns
    /// `false`.
    pub fn error_message(&self) -> &str {
        &self.m_error_message
    }

    // TODO
    /*
    /// Initializes paths to config, cache etc. from CLI arguments.
    pub fn process_args(const CliArgsParser& args) {
        unimplemented!()
    }
    */

    /// Ensures all directories exist (migrating them from Newsbeuter if
    /// possible).
    pub fn setup_dirs(&mut self) -> bool {
        if !self.m_using_nonstandard_configs && !Path::new(&self.m_url_file).exists() {
            self.migrate_data_from_newsbeuter();
        }

        self.create_dirs()
    }

    /// Path to the URLs file.
    pub fn url_file(&self) -> String {
        self.m_url_file.clone()
    }

    /// Path to the cache file.
    pub fn cache_file(&self) -> String {
        self.m_cache_file.clone()
    }

    /// Sets path to the cache file.
    // FIXME: this is actually a kludge that lets Controller change the path
    // midway. That logic should be moved into ConfigPaths, and this method
    // removed.
    pub fn set_cache_file(&mut self, path: String) {
        self.m_cache_file = path.clone();
        self.m_lock_file = path + LOCK_SUFFIX;
    }

    /// Path to the config file.
    pub fn config_file(&self) -> String {
        self.m_config_file.clone()
    }

    /// Path to the lock file.
    ///
    /// \note This changes when path to config file changes.
    pub fn lock_file(&self) -> String {
        self.m_lock_file.clone()
    }

    /// \brief Path to the queue file.
    ///
    /// Queue file stores enqueued podcasts. It's written by Newsboat, and
    /// read by Podboat.
    pub fn queue_file(&self) -> String {
        self.m_queue_file.clone()
    }

    /// Path to the file with previous search queries.
    pub fn search_file(&self) -> String {
        self.m_search_file.clone()
    }

    /// Path to the file with command-line history.
    pub fn cmdline_file(&self) -> String {
        self.m_cmdline_file.clone()
    }
}

fn try_mkdir<R: AsRef<Path>>(path: R) -> bool {
    DirBuilder::new()
        .recursive(true)
        .mode(0o700)
        .create(path.as_ref())
        .is_ok()
}

fn mkdir<R: AsRef<Path>>(path: R, mode: u32) -> io::Result<()> {
    DirBuilder::new().mode(mode).create(path.as_ref())
}

fn migrate_file<R: AsRef<Path>>(newsbeuter_dir: R, newsboat_dir: R, file: &str) -> io::Result<()> {
    let input_filepath = newsbeuter_dir.as_ref().join(file);
    let output_filepath = newsboat_dir.as_ref().join(file);
    eprintln!("{:?} -> {:?}", input_filepath, output_filepath);
    fs::copy(input_filepath, output_filepath).map(|_| ())
}
<commit_msg>Port more comments from C++<commit_after>use dirs;
use libc;
use logger::{self, Level};
use std::fs::{self, DirBuilder};
use std::io::{self, Read};
use std::os::unix::fs::DirBuilderExt;
use std::path::{Path, PathBuf};

pub const NEWSBOAT_SUBDIR_XDG: &str = "newsboat";
pub const NEWSBOAT_CONFIG_SUBDIR: &str = ".newsboat";
pub const NEWSBEUTER_SUBDIR_XDG: &str = "newsbeuter";
pub const NEWSBEUTER_CONFIG_SUBDIR: &str = ".newsbeuter";
pub const LOCK_SUFFIX: &str = ".lock";

#[derive(Debug)]
pub struct ConfigPaths {
    m_env_home: PathBuf,
    m_error_message: String,

    m_data_dir: PathBuf,
    m_config_dir: PathBuf,

    m_url_file: String,
    m_cache_file: String,
    m_config_file: String,
    m_lock_file: String,
    m_queue_file: String,
    m_search_file: String,
    m_cmdline_file: String,

    m_silent: bool,
    m_using_nonstandard_configs: bool,
}

impl ConfigPaths {
    pub fn new() -> ConfigPaths {
        let mut config_paths = ConfigPaths {
            m_env_home: PathBuf::new(),
            m_error_message: String::new(),

            m_data_dir: PathBuf::new(),
            m_config_dir: PathBuf::new(),

            m_url_file: String::from("urls"),
            m_cache_file: String::from("cache.db"),
            m_config_file: String::from("config"),
            m_lock_file: String::new(),
            m_queue_file: String::from("queue"),
            m_search_file: String::new(),
            m_cmdline_file: String::new(),

            m_silent: false,
            m_using_nonstandard_configs: false,
        };

        let m_env_home = dirs::home_dir();
        if m_env_home.is_none() {
            let uid = unsafe { libc::getuid() };

            // TODO: i18n
            config_paths.m_error_message = format!("Fatal error: couldn't determine home directory!\nPlease set the HOME environment variable or add a valid user for UID {}!", uid);

            return config_paths;
        }

        // hitting this branch means we found a home directory
        // we can now safely call unwrap on all other functions in the dirs crate

        config_paths.m_env_home = m_env_home.unwrap();
        config_paths.find_dirs();
        config_paths
    }

    fn migrate_data_from_newsbeuter_xdg(&mut self) -> bool {
        // This can't panic because we've tested we can find the home directory in ConfigPaths::new
        // This should be replaced with proper error handling after this is not used by c++ anymore
        let xdg_config_dir = dirs::config_dir().unwrap();
        let xdg_data_dir = dirs::data_dir().unwrap();

        let newsbeuter_config_dir = xdg_config_dir.join(NEWSBEUTER_SUBDIR_XDG);
        let newsbeuter_data_dir = xdg_data_dir.join(NEWSBEUTER_SUBDIR_XDG);

        let newsboat_config_dir = xdg_config_dir.join(NEWSBOAT_SUBDIR_XDG);
        let newsboat_data_dir = xdg_data_dir.join(NEWSBOAT_SUBDIR_XDG);

        if !newsbeuter_config_dir.is_dir() {
            return false;
        }

        fn exists(path: &Path) -> bool {
            let exists = path.exists();
            if exists {
                log!(
                    Level::Debug,
                    "{:?} already exists, aborting XDG migration.",
                    path
                );
            }
            return exists;
        }

        if exists(&newsboat_config_dir) {
            return false;
        }

        if exists(&newsboat_data_dir) {
            return false;
        }

        self.m_config_dir = newsboat_config_dir;
        self.m_data_dir = newsboat_data_dir;

        if !self.m_silent {
            eprintln!("Migrating configs and data from Newsbeuter's XDG dirs...");
        }

        if !try_mkdir(&self.m_config_dir) {
            return false;
        }

        if !try_mkdir(&self.m_data_dir) {
            return false;
        }

        // in config
        migrate_file(&newsbeuter_config_dir, &self.m_config_dir, "urls").is_ok();
        migrate_file(&newsbeuter_config_dir, &self.m_config_dir, "config").is_ok();

        // in data
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "cache.db").is_ok();
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "queue").is_ok();
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "history.search").is_ok();
        migrate_file(&newsbeuter_data_dir, &self.m_data_dir, "history.cmdline").is_ok();

        return true;
    }

    fn migrate_data_from_newsbeuter_simple(&self) -> bool {
        let newsbeuter_dir = self.m_env_home.join(NEWSBEUTER_CONFIG_SUBDIR);

        if !newsbeuter_dir.is_dir() {
            return false;
        }

        let newsboat_dir = self.m_env_home.join(NEWSBOAT_CONFIG_SUBDIR);

        if newsboat_dir.exists() {
            log!(
                Level::Debug,
                "{:?} already exists, aborting migration.",
                newsboat_dir
            );
            return false;
        }

        if !self.m_silent {
            eprintln!("Migrating configs and data from Newsbeuter's dotdir...");
        }

        match mkdir(&newsboat_dir, 0o700) {
            Ok(_) => (),
            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => (),
            Err(err) => {
                if !self.m_silent {
                    eprintln!(
                        "Aborting migration because mkdir on {:?} failed: {}",
                        newsboat_dir, err
                    );
                }
                return false;
            }
        };

        migrate_file(&newsbeuter_dir, &newsboat_dir, "urls").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "cache.db").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "config").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "queue").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "history.search").ok();
        migrate_file(&newsbeuter_dir, &newsboat_dir, "history.cmdline").ok();

        return true;
    }

    fn migrate_data_from_newsbeuter(&mut self) {
        let mut migrated = self.migrate_data_from_newsbeuter_xdg();

        if migrated {
            // Re-running to pick up XDG dirs
            self.m_url_file = "urls".into();
            self.m_cache_file = "cache.db".into();
            self.m_config_file = "config".into();
            self.m_queue_file = "queue".into();
            self.find_dirs();
        } else {
            migrated = self.migrate_data_from_newsbeuter_simple();
        }

        if migrated {
            eprintln!("\nPlease check the results and press Enter to continue.");
            // only wait for enter, ignore actual input
            let mut buf = [0; 16];
            io::stdin().read(&mut buf).ok();
        }
    }

    fn create_dirs(&self) -> bool {
        return try_mkdir(&self.m_config_dir) && try_mkdir(&self.m_data_dir);
    }

    fn find_dirs(&mut self) {
        self.m_config_dir = self.m_env_home.join(NEWSBOAT_CONFIG_SUBDIR);

        self.m_data_dir = self.m_config_dir.clone();

        // Will change config_dir and data_dir to point to XDG if XDG
        // directories are available.
        self.find_dirs_xdg();

        // in config
        self.m_url_file = self
            .m_config_dir
            .join(&self.m_url_file)
            .to_string_lossy()
            .into();
        self.m_config_file = self
            .m_config_dir
            .join(&self.m_config_file)
            .to_string_lossy()
            .into();

        // in data
        self.m_cache_file = self
            .m_data_dir
            .join(&self.m_cache_file)
            .to_string_lossy()
            .into();
        self.m_lock_file = self.m_cache_file.clone() + LOCK_SUFFIX;
        self.m_queue_file = self
            .m_data_dir
            .join(&self.m_queue_file)
            .to_string_lossy()
            .into();
        self.m_search_file = self
            .m_data_dir
            .join("history.search")
            .to_string_lossy()
            .into();
        self.m_cmdline_file = self
            .m_data_dir
            .join("history.cmdline")
            .to_string_lossy()
            .into();
    }

    fn find_dirs_xdg(&mut self) {
        // This can't panic because we've tested we can find the home directory in ConfigPaths::new
        // This should be replaced with proper error handling after this is not used by c++ anymore
        let config_dir = dirs::config_dir().unwrap().join(NEWSBOAT_SUBDIR_XDG);
        let data_dir = dirs::data_dir().unwrap().join(NEWSBOAT_SUBDIR_XDG);

        if !config_dir.is_dir() {
            return;
        }

        /* Invariant: config dir exists.
         *
         * At this point, we're confident we'll be using XDG. We don't check if
         * data dir exists, because if it doesn't we'll create it. */

        self.m_config_dir = config_dir;
        self.m_data_dir = data_dir;
    }

    /// \brief Indicates if the object can be used.
    ///
    /// If this method returned `false`, the cause for initialization
    /// failure can be found using `error_message()`.
    pub fn initialized(&self) -> bool {
        !self.m_env_home.as_os_str().is_empty()
    }

    /// Returns explanation why initialization failed.
    ///
    /// \note You shouldn't call this unless `initialized()` returns
    /// `false`.
    pub fn error_message(&self) -> &str {
        &self.m_error_message
    }

    // TODO
    /*
    /// Initializes paths to config, cache etc. from CLI arguments.
    pub fn process_args(const CliArgsParser& args) {
        unimplemented!()
    }
    */

    /// Ensures all directories exist (migrating them from Newsbeuter if
    /// possible).
    pub fn setup_dirs(&mut self) -> bool {
        if !self.m_using_nonstandard_configs && !Path::new(&self.m_url_file).exists() {
            self.migrate_data_from_newsbeuter();
        }

        self.create_dirs()
    }

    /// Path to the URLs file.
    pub fn url_file(&self) -> String {
        self.m_url_file.clone()
    }

    /// Path to the cache file.
    pub fn cache_file(&self) -> String {
        self.m_cache_file.clone()
    }

    /// Sets path to the cache file.
    // FIXME: this is actually a kludge that lets Controller change the path
    // midway. That logic should be moved into ConfigPaths, and this method
    // removed.
    pub fn set_cache_file(&mut self, path: String) {
        self.m_cache_file = path.clone();
        self.m_lock_file = path + LOCK_SUFFIX;
    }

    /// Path to the config file.
    pub fn config_file(&self) -> String {
        self.m_config_file.clone()
    }

    /// Path to the lock file.
    ///
    /// \note This changes when path to config file changes.
    pub fn lock_file(&self) -> String {
        self.m_lock_file.clone()
    }

    /// \brief Path to the queue file.
    ///
    /// Queue file stores enqueued podcasts. It's written by Newsboat, and
    /// read by Podboat.
    pub fn queue_file(&self) -> String {
        self.m_queue_file.clone()
    }

    /// Path to the file with previous search queries.
    pub fn search_file(&self) -> String {
        self.m_search_file.clone()
    }

    /// Path to the file with command-line history.
    pub fn cmdline_file(&self) -> String {
        self.m_cmdline_file.clone()
    }
}

fn try_mkdir<R: AsRef<Path>>(path: R) -> bool {
    DirBuilder::new()
        .recursive(true)
        .mode(0o700)
        .create(path.as_ref())
        .is_ok()
}

fn mkdir<R: AsRef<Path>>(path: R, mode: u32) -> io::Result<()> {
    DirBuilder::new().mode(mode).create(path.as_ref())
}

fn migrate_file<R: AsRef<Path>>(newsbeuter_dir: R, newsboat_dir: R, file: &str) -> io::Result<()> {
    let input_filepath = newsbeuter_dir.as_ref().join(file);
    let output_filepath = newsboat_dir.as_ref().join(file);
    eprintln!("{:?} -> {:?}", input_filepath, output_filepath);
    fs::copy(input_filepath, output_filepath).map(|_| ())
}
<|endoftext|>