<commit_before>use super::{SrcOffset, BytePos, LineIdx, ColIdx, Loc};
use std::cell::RefCell;
use std::fmt;

/// Stores the content of a file and keeps track of some position meta data,
/// such as linebreaks.
pub struct FileMap {
    /// Original filename or dummy filename of the form "<...>"
    filename: String,
    /// The whole content of the file
    src: String,
    /// List of line beginnings. Wrapped in a `RefCell`, because `FileMap`s
    /// are shared a lot via `Rc` and this is the only thing requiring
    /// mutability. Borrowing it will never panic, because:
    /// - it's only borrowed within methods of `FileMap`
    /// - the borrow always ends with those methods
    /// - no method is called/active while another one is active (as long as
    ///   no borrowing method is calling another borrowing method)
    lines: RefCell<Vec<BytePos>>,
}

impl FileMap {
    /// Creates a new Filemap from existing buffers for the filename and
    /// content of the file.
    pub fn new<U, V>(filename: U, src: V) -> FileMap
        where U: Into<String>,
              V: Into<String>
    {
        FileMap {
            filename: filename.into(),
            src: src.into(),
            lines: RefCell::new(vec![BytePos(0)]),
        }
    }

    pub fn filename(&self) -> &str {
        &self.filename
    }

    pub fn src(&self) -> &str {
        &self.src
    }

    /// Adds a new line beginning with the given BytePos to the list. Line
    /// beginnings need to be added in order!
    pub fn add_line(&self, offset: BytePos) {
        self.lines.borrow_mut().push(offset);
    }

    /// Returns the number of lines.
    pub fn num_lines(&self) -> usize {
        self.lines.borrow().len()
    }

    /// Returns the (0-based) index of the line in which the byte with the
    /// given `offset` lives.
    fn get_line_idx(&self, offset: BytePos) -> LineIdx {
        // If `offset` is not a line beginning, `binary_search` returns the
        // index of the next line. Hence `-1`.
        LineIdx(self.lines.borrow()
            .binary_search(&offset)
            .unwrap_or_else(|e| e - 1) as u32)
    }

    /// Returns the location of the given bytes as line and col numbers within
    /// this file.
    pub fn get_loc(&self, offset: BytePos) -> Loc {
        let line = self.get_line_idx(offset);
        let col = offset - self.lines.borrow()[line.0 as usize];

        Loc { line: line, col: ColIdx(col.0) }
    }

    /// Returns the line with the given index or `None` if it is invalid.
    pub fn get_line(&self, line: LineIdx) -> Option<&str> {
        self.lines.borrow().get(line.0 as usize).map(|&BytePos(start)| {
            let end = self.src[start as usize..]
                          .find("\n")
                          .unwrap_or(self.src.len() - start as usize);
            &self.src[start as usize .. (end + start as usize)]
        })
    }

    /// Searches for line endings and collects all line beginnings in the
    /// source string. It starts searching at the latest line beginning in the
    /// list so far (or at the beginning of none was added yet).
    ///
    /// Normally this is done while lexing to avoid iterating
    /// over the whole string multiple times. Mostly handy for tests.
    pub fn find_lines(&self) {
        // We can unwrap here, because the vector contains at least one element
        let last_line_so_far = self.lines.borrow().last().unwrap().0 as usize;
        for (pos, c) in self.src[last_line_so_far..].char_indices() {
            // it doesn't matter if there was a '\n' or '\r\n'
            if c == '\n' {
                let line_start = (pos + c.len_utf8()) as SrcOffset;
                self.lines.borrow_mut().push(BytePos(line_start));
            }
        }
    }
}

impl fmt::Debug for FileMap {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        struct Dummy<'a>(&'a [BytePos]);
        impl<'a> fmt::Debug for Dummy<'a> {
            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
                let items = self.0.iter().map(|v| v.0).enumerate();
                f.debug_map().entries(items).finish()
            }
        }

        f.debug_struct("FileMap")
            .field("filename", &self.filename)
            .field("src", &format!("<long string> (len {})", self.src.len()))
            .field("lines", &Dummy(&self.lines.borrow()))
            .finish()
    }
}

// --- tests ---
#[test]
fn location_lookup() {
    let map = FileMap::new("<dummy>", "foo\r\nbär\nbaz");
    map.find_lines();

    // "foo\r\n" is 5 bytes. "bär\n" is 5 bytes, too.
    assert_eq!(*map.lines.borrow(),
        vec![BytePos(0), BytePos(5), BytePos(10)]
    );

    macro_rules! is_at {
        ($offset:expr => [$line:expr, $col:expr]) => {
            assert_eq!(map.get_loc(BytePos($offset)), Loc {
                line: LineIdx($line), col: ColIdx($col)
            });
        }
    }

    is_at!(0 => [0, 0]);
    is_at!(1 => [0, 1]);
    is_at!(4 => [0, 4]);
    is_at!(5 => [1, 0]);
    is_at!(9 => [1, 4]);
    is_at!(10 => [2, 0]);
    is_at!(12 => [2, 2]);
}
<commit_msg>Add additional method to `FileMap` and make an existing one public<commit_after>use super::{SrcOffset, BytePos, LineIdx, ColIdx, Loc};
use std::cell::RefCell;
use std::fmt;

/// Stores the content of a file and keeps track of some position meta data,
/// such as linebreaks.
pub struct FileMap {
    /// Original filename or dummy filename of the form "<...>"
    filename: String,
    /// The whole content of the file
    src: String,
    /// List of line beginnings. Wrapped in a `RefCell`, because `FileMap`s
    /// are shared a lot via `Rc` and this is the only thing requiring
    /// mutability. Borrowing it will never panic, because:
    /// - it's only borrowed within methods of `FileMap`
    /// - the borrow always ends with those methods
    /// - no method is called/active while another one is active (as long as
    ///   no borrowing method is calling another borrowing method)
    lines: RefCell<Vec<BytePos>>,
}

impl FileMap {
    /// Creates a new Filemap from existing buffers for the filename and
    /// content of the file.
    pub fn new<U, V>(filename: U, src: V) -> FileMap
        where U: Into<String>,
              V: Into<String>
    {
        FileMap {
            filename: filename.into(),
            src: src.into(),
            lines: RefCell::new(vec![BytePos(0)]),
        }
    }

    pub fn filename(&self) -> &str {
        &self.filename
    }

    pub fn src(&self) -> &str {
        &self.src
    }

    /// Adds a new line beginning with the given BytePos to the list. Line
    /// beginnings need to be added in order!
    pub fn add_line(&self, offset: BytePos) {
        self.lines.borrow_mut().push(offset);
    }

    /// Returns the number of lines.
    pub fn num_lines(&self) -> usize {
        self.lines.borrow().len()
    }

    /// Returns the (0-based) index of the line in which the byte with the
    /// given `offset` lives.
    pub fn get_line_idx(&self, offset: BytePos) -> LineIdx {
        // If `offset` is not a line beginning, `binary_search` returns the
        // index of the next line. Hence `-1`.
        LineIdx(self.lines.borrow()
            .binary_search(&offset)
            .unwrap_or_else(|e| e - 1) as u32)
    }

    /// Returns the location of the given bytes as line and col numbers within
    /// this file.
    pub fn get_loc(&self, offset: BytePos) -> Loc {
        let line = self.get_line_idx(offset);
        let col = offset - self.lines.borrow()[line.0 as usize];

        Loc { line: line, col: ColIdx(col.0) }
    }

    /// Returns the line with the given index or `None` if it is invalid.
    pub fn get_line(&self, line: LineIdx) -> Option<&str> {
        self.lines.borrow().get(line.0 as usize).map(|&BytePos(start)| {
            let end = self.src[start as usize..]
                          .find("\n")
                          .unwrap_or(self.src.len() - start as usize);
            &self.src[start as usize .. (end + start as usize)]
        })
    }

    /// Returns the byte offset of the first symbol in `line`
    pub fn get_line_start(&self, line: LineIdx) -> Option<BytePos> {
        self.lines.borrow().get(line.0 as usize).map(|&pos| pos)
    }

    /// Searches for line endings and collects all line beginnings in the
    /// source string. It starts searching at the latest line beginning in the
    /// list so far (or at the beginning of none was added yet).
    ///
    /// Normally this is done while lexing to avoid iterating
    /// over the whole string multiple times. Mostly handy for tests.
    pub fn find_lines(&self) {
        // We can unwrap here, because the vector contains at least one element
        let last_line_so_far = self.lines.borrow().last().unwrap().0 as usize;
        for (pos, c) in self.src[last_line_so_far..].char_indices() {
            // it doesn't matter if there was a '\n' or '\r\n'
            if c == '\n' {
                let line_start = (pos + c.len_utf8()) as SrcOffset;
                self.lines.borrow_mut().push(BytePos(line_start));
            }
        }
    }
}

impl fmt::Debug for FileMap {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        struct Dummy<'a>(&'a [BytePos]);
        impl<'a> fmt::Debug for Dummy<'a> {
            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
                let items = self.0.iter().map(|v| v.0).enumerate();
                f.debug_map().entries(items).finish()
            }
        }

        f.debug_struct("FileMap")
            .field("filename", &self.filename)
            .field("src", &format!("<long string> (len {})", self.src.len()))
            .field("lines", &Dummy(&self.lines.borrow()))
            .finish()
    }
}

// --- tests ---
#[test]
fn location_lookup() {
    let map = FileMap::new("<dummy>", "foo\r\nbär\nbaz");
    map.find_lines();

    // "foo\r\n" is 5 bytes. "bär\n" is 5 bytes, too.
    assert_eq!(*map.lines.borrow(),
        vec![BytePos(0), BytePos(5), BytePos(10)]
    );

    macro_rules! is_at {
        ($offset:expr => [$line:expr, $col:expr]) => {
            assert_eq!(map.get_loc(BytePos($offset)), Loc {
                line: LineIdx($line), col: ColIdx($col)
            });
        }
    }

    is_at!(0 => [0, 0]);
    is_at!(1 => [0, 1]);
    is_at!(4 => [0, 4]);
    is_at!(5 => [1, 0]);
    is_at!(9 => [1, 4]);
    is_at!(10 => [2, 0]);
    is_at!(12 => [2, 2]);
}
<|endoftext|>


<commit_before>//! Command line modules: has one function which takes input parameters from commandline
//! and parses them.
use super::config;

static APP_TITLE: &str = concat!("The audiobook finder (", env!("CARGO_PKG_NAME"), ")");

static ARG_NET: &str = "net";
static ARG_TUI: &str = "tui";
static ARG_WEBUI: &str = "webui";
static ARG_KEEP_ALIVE: &str = "keep";
static ARG_BROWSER: &str = "browser";
static ARG_BROWSER_PORT: &str = "port";

static INPUT_FOLDERS: &str = "folders";

const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const AUTHORS: &'static str = env!("CARGO_PKG_AUTHORS");
const HOMEPAGE: &'static str = env!("CARGO_PKG_HOMEPAGE");
const DESCRIPTION: &'static str = env!("CARGO_PKG_DESCRIPTION");

/// Get all start values which are passed from command line
///
/// Get all start values and returns the following tuple
/// ui_paths,
/// has_tui,
/// has_webui,
/// has_net,
/// keep_alive,
/// open_browser,
/// web_port,
/// has_ui,
pub fn get_start_values() -> (Vec<String>, bool, bool, bool, bool, bool, u16, bool) {
    let parse_args = clap::App::new(APP_TITLE)
        .version(VERSION)
        .author(AUTHORS)
        .about(DESCRIPTION)
        .long_about::<&str>(
            &[
                &DESCRIPTION,
                "\n\
                 It reads data from possibly multiple given path(s). Via local network it searches \
                 for other instances of the program, and will later exchange data securely.\n\
                 All information gathered will be used to find duplicates, versions of \
                 different quality, different tags for same content (spelling, \
                 incompleteness).\n\
                 For documentation see: ",
                &HOMEPAGE,
                "\n \
                 A program to learn, embrace, and love Rust! \n\
                 Have fun!",
            ]
            .concat(),
        )
        .arg(
            clap::Arg::with_name("config")
                .short("c")
                .long("config")
                .value_name("FILE")
                .help("Sets custom config file (not implemented yet)")
                .takes_value(true),
        )
        .arg(
            clap::Arg::with_name(ARG_TUI)
                .short("t")
                .long(ARG_TUI)
                .help("Run with TUI")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_WEBUI)
                .short("w")
                .long(ARG_WEBUI)
                .help("Run with-in a webui.")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_BROWSER_PORT)
                .short("p")
                .long(ARG_BROWSER_PORT)
                .help(
                    &vec![
                        "Define port for webui (only works with webui).\nDefault port is:"
                            .to_string(),
                        config::net::WEB_PORT_DEFAULT.to_string(),
                        " but please choose from 8080 to 8099)".to_string(),
                    ]
                    .join(""),
                )
                .takes_value(true),
        )
        .arg(
            clap::Arg::with_name(ARG_NET)
                .short("n")
                .long(ARG_NET)
                .help("With net search for other audiobookfinders running in local network.")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_KEEP_ALIVE)
                .short("k")
                .long(ARG_KEEP_ALIVE)
                .help("With keep alive process will continue even after search has been performed.")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_BROWSER)
                .short("b")
                .long(ARG_BROWSER)
                .help("Shall browser be openend automatically (only works with webui).")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(INPUT_FOLDERS)
                .help("Sets multiple input folder(s) to be searched for audio files.")
                .multiple(true)
                .required(false),
        )
        .get_matches();
    // tricky thing, but I really like that
    let all_pathes = if let Some(correct_input) = parse_args.values_of(INPUT_FOLDERS) {
        correct_input.collect()
    } else {
        vec!["."]
    };

    //
    // check argments if tui and net search is needed
    //
    let has_arg = |x: &str| parse_args.is_present(x);

    let has_tui = has_arg(ARG_TUI);
    let has_webui = has_arg(ARG_WEBUI);
    let has_net = has_arg(ARG_NET);
    let keep_alive = has_arg(ARG_KEEP_ALIVE);
    let open_browser = has_arg(ARG_BROWSER);

    let web_port = {
        let web_default_string = config::net::WEB_PORT_DEFAULT.to_string();
        // when to write to console
        let has_to_write_console = !has_tui && has_webui;

        let parsed_value = parse_args
            .value_of(ARG_BROWSER_PORT)
            .unwrap_or_else(|| {
                if has_to_write_console {
                    println!(
                        "Port argument was bad, using default port {}!",
                        config::net::WEB_ADDR
                    );
                }
                &web_default_string
            })
            .parse::<u16>()
            .unwrap_or_else(|_| {
                if has_to_write_console {
                    println!(
                        "Invalid port input, using default port {}!",
                        &web_default_string
                    );
                }
                config::net::WEB_PORT_DEFAULT
            });
        if parsed_value < config::net::WEB_PORT_MIN || parsed_value > config::net::WEB_PORT_MAX {
            if has_to_write_console {
                let web_max_string = config::net::WEB_PORT_MAX.to_string();
                let web_min_string = config::net::WEB_PORT_MIN.to_string();
                println!(
                    "Port not in range {} .. {}, using default {}!",
                    &web_min_string, &web_max_string, &web_default_string
                );
            }
            config::net::WEB_PORT_DEFAULT
        } else {
            parsed_value
        }
    };

    // extended help for certain option combinations
    if !has_tui && has_webui && !open_browser {
        println!(
            "Open http://{}:{} to start!",
            config::net::WEB_ADDR,
            config::net::WEB_PORT_DEFAULT
        );
        println!("The webui needs to get the start signal from there");
    }

    // either one will have a ui, representing data and error messages
    let has_ui = has_tui || has_webui;
    let ui_paths = all_pathes.iter().map(|s| s.to_string()).collect();
    (
        ui_paths,
        has_tui,
        has_webui,
        has_net,
        keep_alive,
        open_browser,
        web_port,
        has_ui,
    )
}
<commit_msg>[main] command line fix, that port if not given shouldn't be claimed wrong<commit_after>//! Command line modules: has one function which takes input parameters from commandline
//! and parses them.
use super::config;

static APP_TITLE: &str = concat!("The audiobook finder (", env!("CARGO_PKG_NAME"), ")");

static ARG_NET: &str = "net";
static ARG_TUI: &str = "tui";
static ARG_WEBUI: &str = "webui";
static ARG_KEEP_ALIVE: &str = "keep";
static ARG_BROWSER: &str = "browser";
static ARG_BROWSER_PORT: &str = "port";

static INPUT_FOLDERS: &str = "folders";

const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const AUTHORS: &'static str = env!("CARGO_PKG_AUTHORS");
const HOMEPAGE: &'static str = env!("CARGO_PKG_HOMEPAGE");
const DESCRIPTION: &'static str = env!("CARGO_PKG_DESCRIPTION");

/// Get all start values which are passed from command line
///
/// Get all start values and returns the following tuple
/// ui_paths,
/// has_tui,
/// has_webui,
/// has_net,
/// keep_alive,
/// open_browser,
/// web_port,
/// has_ui,
pub fn get_start_values() -> (Vec<String>, bool, bool, bool, bool, bool, u16, bool) {
    let parse_args = clap::App::new(APP_TITLE)
        .version(VERSION)
        .author(AUTHORS)
        .about(DESCRIPTION)
        .long_about::<&str>(
            &[
                &DESCRIPTION,
                "\n\
                 It reads data from possibly multiple given path(s). Via local network it searches \
                 for other instances of the program, and will later exchange data securely.\n\
                 All information gathered will be used to find duplicates, versions of \
                 different quality, different tags for same content (spelling, \
                 incompleteness).\n\
                 For documentation see: ",
                &HOMEPAGE,
                "\n \
                 A program to learn, embrace, and love Rust! \n\
                 Have fun!",
            ]
            .concat(),
        )
        .arg(
            clap::Arg::with_name("config")
                .short("c")
                .long("config")
                .value_name("FILE")
                .help("Sets custom config file (not implemented yet)")
                .takes_value(true),
        )
        .arg(
            clap::Arg::with_name(ARG_TUI)
                .short("t")
                .long(ARG_TUI)
                .help("Run with TUI")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_WEBUI)
                .short("w")
                .long(ARG_WEBUI)
                .help("Run with-in a webui.")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_BROWSER_PORT)
                .short("p")
                .long(ARG_BROWSER_PORT)
                .help(
                    &vec![
                        "Define port for webui (only works with webui).\nDefault port is:"
                            .to_string(),
                        config::net::WEB_PORT_DEFAULT.to_string(),
                        " but please choose from 8080 to 8099)".to_string(),
                    ]
                    .join(""),
                )
                .takes_value(true),
        )
        .arg(
            clap::Arg::with_name(ARG_NET)
                .short("n")
                .long(ARG_NET)
                .help("With net search for other audiobookfinders running in local network.")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_KEEP_ALIVE)
                .short("k")
                .long(ARG_KEEP_ALIVE)
                .help("With keep alive process will continue even after search has been performed.")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(ARG_BROWSER)
                .short("b")
                .long(ARG_BROWSER)
                .help("Shall browser be openend automatically (only works with webui).")
                .takes_value(false),
        )
        .arg(
            clap::Arg::with_name(INPUT_FOLDERS)
                .help("Sets multiple input folder(s) to be searched for audio files.")
                .multiple(true)
                .required(false),
        )
        .get_matches();
    // tricky thing, but I really like that
    let all_pathes = if let Some(correct_input) = parse_args.values_of(INPUT_FOLDERS) {
        correct_input.collect()
    } else {
        vec!["."]
    };

    //
    // check argments if tui and net search is needed
    //
    let has_arg = |x: &str| parse_args.is_present(x);

    let has_tui = has_arg(ARG_TUI);
    let has_webui = has_arg(ARG_WEBUI);
    let has_net = has_arg(ARG_NET);
    let has_port = has_arg(ARG_BROWSER_PORT);
    let keep_alive = has_arg(ARG_KEEP_ALIVE);
    let open_browser = has_arg(ARG_BROWSER);

    let web_port = {
        let web_default_string = config::net::WEB_PORT_DEFAULT.to_string();
        // when to write to console
        let has_to_write_console = !has_tui && has_webui;

        let parsed_value = parse_args
            .value_of(ARG_BROWSER_PORT)
            .unwrap_or_else(|| {
                if has_to_write_console && has_port {
                    println!(
                        "Port argument was bad, using default port {}!",
                        config::net::WEB_ADDR
                    );
                }
                &web_default_string
            })
            .parse::<u16>()
            .unwrap_or_else(|_| {
                if has_to_write_console && has_port {
                    println!(
                        "Invalid port input, using default port {}!",
                        &web_default_string
                    );
                }
                config::net::WEB_PORT_DEFAULT
            });
        if parsed_value < config::net::WEB_PORT_MIN || parsed_value > config::net::WEB_PORT_MAX {
            if has_to_write_console {
                let web_max_string = config::net::WEB_PORT_MAX.to_string();
                let web_min_string = config::net::WEB_PORT_MIN.to_string();
                println!(
                    "Port not in range {} .. {}, using default {}!",
                    &web_min_string, &web_max_string, &web_default_string
                );
            }
            config::net::WEB_PORT_DEFAULT
        } else {
            parsed_value
        }
    };

    // extended help for certain option combinations
    if !has_tui && has_webui && !open_browser {
        println!(
            "Open http://{}:{} to start!",
            config::net::WEB_ADDR,
            config::net::WEB_PORT_DEFAULT
        );
        println!("The webui needs to get the start signal from there");
    }

    // either one will have a ui, representing data and error messages
    let has_ui = has_tui || has_webui;
    let ui_paths = all_pathes.iter().map(|s| s.to_string()).collect();
    (
        ui_paths,
        has_tui,
        has_webui,
        has_net,
        keep_alive,
        open_browser,
        web_port,
        has_ui,
    )
}
<|endoftext|>


<commit_before>use core::pin::Pin;
use futures_core::future::{Future, FusedFuture};
use futures_core::task::{Context, Poll};
use pin_utils::unsafe_pinned;

/// Future for the [`fuse`](super::FutureExt::fuse) method.
#[derive(Debug)]
#[must_use = "futures do nothing unless polled"]
pub struct Fuse<Fut: Future> {
    future: Option<Fut>,
}

impl<Fut: Future> Fuse<Fut> {
    unsafe_pinned!(future: Option<Fut>);

    pub(super) fn new(f: Fut) -> Fuse<Fut> {
        Fuse {
            future: Some(f),
        }
    }
}

impl<Fut: Future> FusedFuture for Fuse<Fut> {
    fn is_terminated(&self) -> bool {
        self.future.is_none()
    }
}

impl<Fut: Future> Future for Fuse<Fut> {
    type Output = Fut::Output;

    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Fut::Output> {
        // safety: we use this &mut only for matching, not for movement
        let v = match self.as_mut().future().as_pin_mut() {
            Some(fut) => {
                // safety: this re-pinned future will never move before being dropped
                match fut.poll(cx) {
                    Poll::Pending => return Poll::Pending,
                    Poll::Ready(v) => v
                }
            }
            None => return Poll::Pending,
        };

        self.as_mut().future().set(None);
        Poll::Ready(v)
    }
}
<commit_msg>Add Fuse::terminated constructor<commit_after>use core::pin::Pin;
use futures_core::future::{Future, FusedFuture};
use futures_core::task::{Context, Poll};
use pin_utils::unsafe_pinned;

/// Future for the [`fuse`](super::FutureExt::fuse) method.
#[derive(Debug)]
#[must_use = "futures do nothing unless polled"]
pub struct Fuse<Fut: Future> {
    future: Option<Fut>,
}

impl<Fut: Future> Fuse<Fut> {
    unsafe_pinned!(future: Option<Fut>);

    pub(super) fn new(f: Fut) -> Fuse<Fut> {
        Fuse {
            future: Some(f),
        }
    }

    /// Creates a new `Fuse`-wrapped future which is already terminated.
    ///
    /// This can be useful in combination with looping and the `select!`
    /// macro, which bypasses terminated futures.
    ///
    /// # Examples
    ///
    /// ```
    /// #![feature(async_await, await_macro, futures_api)]
    /// # futures::executor::block_on(async {
    /// use futures::channel::mpsc;
    /// use futures::future::{Fuse, FusedFuture, FutureExt};
    /// use futures::select;
    /// use futures::stream::StreamExt;
    /// use pin_utils::pin_mut;
    ///
    /// let (sender, mut stream) = mpsc::unbounded();
    ///
    /// // Send a few messages into the stream
    /// sender.unbounded_send(()).unwrap();
    /// sender.unbounded_send(()).unwrap();
    /// drop(sender);
    ///
    /// // Use `Fuse::termianted()` to create an already-terminated future
    /// // which may be instantiated later.
    /// let foo_printer = Fuse::terminated();
    /// pin_mut!(foo_printer);
    ///
    /// loop {
    ///     select! {
    ///         _ = foo_printer => {},
    ///         () = stream.select_next_some() => {
    ///             if !foo_printer.is_terminated() {
    ///                 println!("Foo is already being printed!");
    ///             } else {
    ///                 foo_printer.set(async {
    ///                     // do some other async operations
    ///                     println!("Printing foo from `foo_printer` future");
    ///                 }.fuse());
    ///             }
    ///         },
    ///         complete => break, // `foo_printer` is terminated and the stream is done
    ///     }
    /// }
    /// # });
    /// ```
    pub fn terminated() -> Fuse<Fut> {
        Fuse { future: None }
    }
}

impl<Fut: Future> FusedFuture for Fuse<Fut> {
    fn is_terminated(&self) -> bool {
        self.future.is_none()
    }
}

impl<Fut: Future> Future for Fuse<Fut> {
    type Output = Fut::Output;

    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Fut::Output> {
        // safety: we use this &mut only for matching, not for movement
        let v = match self.as_mut().future().as_pin_mut() {
            Some(fut) => {
                // safety: this re-pinned future will never move before being dropped
                match fut.poll(cx) {
                    Poll::Pending => return Poll::Pending,
                    Poll::Ready(v) => v
                }
            }
            None => return Poll::Pending,
        };

        self.as_mut().future().set(None);
        Poll::Ready(v)
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate enum_map;

use enum_map::{EnumMap, IntoIter};

use std::cell::RefCell;
use std::collections::HashSet;

#[derive(Copy, Clone, Debug, EnumMap, PartialEq)]
enum Example {
    A,
    B,
    C,
}

#[test]
fn test_bool() {
    let mut map = enum_map! { false => 24, true => 42 };
    assert_eq!(map[false], 24);
    assert_eq!(map[true], 42);
    map[false] += 1;
    assert_eq!(map[false], 25);
    for (key, item) in &mut map {
        if !key {
            *item += 1;
        }
    }
    assert_eq!(map[false], 26);
    assert_eq!(map[true], 42);
}

#[test]
fn test_option_bool() {
    let mut map = enum_map! { None => 1, Some(false) => 2, Some(true) => 3};
    assert_eq!(map[None], 1);
    assert_eq!(map[Some(false)], 2);
    assert_eq!(map[Some(true)], 3);
    map[None] = 4;
    map[Some(false)] = 5;
    map[Some(true)] = 6;
    assert_eq!(map[None], 4);
    assert_eq!(map[Some(false)], 5);
    assert_eq!(map[Some(true)], 6);
    assert_eq!(map.as_slice(), [4, 5, 6]);
    assert_eq!(
        map.into_iter().collect::<Vec<_>>(),
        [(None, 4), (Some(false), 5), (Some(true), 6)]
    );
}

#[test]
fn test_clone() {
    let map = enum_map! { false => 3, true => 5 };
    assert_eq!(map.clone(), map);
}

#[test]
fn test_debug() {
    let map = enum_map! { false => 3, true => 5 };
    assert_eq!(format!("{:?}", map), "{false: 3, true: 5}");
}

#[test]
fn test_hash() {
    let map = enum_map! { false => 3, true => 5 };
    let mut set = HashSet::new();
    set.insert(map);
    assert!(set.contains(&map));
}

#[test]
fn discriminants() {
    #[derive(Debug, EnumMap, PartialEq)]
    enum Discriminants {
        A = 2000,
        B = 3000,
        C = 1000,
    }
    let mut map = EnumMap::new();
    map[Discriminants::A] = 3;
    map[Discriminants::B] = 2;
    map[Discriminants::C] = 1;
    let mut pairs = map.iter();
    assert_eq!(pairs.next(), Some((Discriminants::A, &3)));
    assert_eq!(pairs.next(), Some((Discriminants::B, &2)));
    assert_eq!(pairs.next(), Some((Discriminants::C, &1)));
    assert_eq!(pairs.next(), None);
}

#[test]
fn extend() {
    let mut map = enum_map! { _ => 0 };
    map.extend(vec![(Example::A, 3)]);
    map.extend(vec![(&Example::B, &4)]);
    assert_eq!(
        map,
        enum_map! { Example:: A => 3, Example:: B => 4, Example::C => 0 }
    );
}

#[test]
fn huge_enum() {
    #[derive(EnumMap)]
    enum Example {
        A,
        B,
        C,
        D,
        E,
        F,
        G,
        H,
        I,
        J,
        K,
        L,
        M,
        N,
        O,
        P,
        Q,
        R,
        S,
        T,
        U,
        V,
        W,
        X,
        Y,
        Z,
        Aa,
        Bb,
        Cc,
        Dd,
        Ee,
        Ff,
        Gg,
        Hh,
        Ii,
        Jj,
        Kk,
        Ll,
        Mm,
        Nn,
        Oo,
        Pp,
        Qq,
        Rr,
        Ss,
        Tt,
        Uu,
        Vv,
        Ww,
        Xx,
        Yy,
        Zz,
    }

    let map = enum_map! { _ => 2 };
    assert_eq!(map[Example::Xx], 2);
}

#[test]
fn iterator_len() {
    assert_eq!(
        enum_map! { Example::A | Example::B | Example::C => 0 }
            .iter()
            .len(),
        3
    );
}

#[test]
fn iter_mut_len() {
    assert_eq!(
        enum_map! { Example::A | Example::B | Example::C => 0 }
            .iter_mut()
            .len(),
        3
    );
}

#[test]
fn iterator_next_back() {
    assert_eq!(
        enum_map! { Example::A => 1, Example::B => 2, Example::C => 3 }
            .iter()
            .next_back(),
        Some((Example::C, &3))
    );
}

#[test]
fn iter_mut_next_back() {
    assert_eq!(
        enum_map! { Example::A => 1, Example::B => 2, Example::C => 3 }
            .iter()
            .next_back(),
        Some((Example::C, &mut 3))
    );
}

#[test]
fn into_iter() {
    let mut iter = enum_map! { true => 5, false => 7 }.into_iter();
    assert_eq!(iter.next(), Some((false, 7)));
    assert_eq!(iter.next(), Some((true, 5)));
    assert_eq!(iter.next(), None);
    assert_eq!(iter.next(), None);
}

struct DropReporter<'a> {
    into: &'a RefCell<Vec<usize>>,
    value: usize,
}

impl<'a> Drop for DropReporter<'a> {
    fn drop(&mut self) {
        self.into.borrow_mut().push(self.value);
    }
}

#[test]
fn into_iter_drop() {
    let dropped = RefCell::new(Vec::new());
    let mut a: IntoIter<Example, _> = enum_map! {
        k => DropReporter {
            into: &dropped,
            value: k as usize,
        },
    }.into_iter();
    assert_eq!(a.next().unwrap().0, Example::A);
    assert_eq!(*dropped.borrow(), &[0]);
    drop(a);
    assert_eq!(*dropped.borrow(), &[0, 1, 2]);
}

#[test]
fn test_u8() {
    let mut map = enum_map! { b'a' => 4, _ => 0 };
    map[b'c'] = 3;
    assert_eq!(map[b'a'], 4);
    assert_eq!(map[b'b'], 0);
    assert_eq!(map[b'c'], 3);
}

#[derive(EnumMap)]
enum Void {}

#[test]
fn empty_map() {
    let void: EnumMap<Void, Void> = enum_map!{};
    assert!(void.is_empty());
}

#[test]
#[should_panic]
fn empty_value() {
    let _void: EnumMap<bool, Void> = enum_map! { _ => unreachable!() };
}
<commit_msg>Avoid multiline assert_eq! to workaround a bug in Rust<commit_after>#[macro_use]
extern crate enum_map;

use enum_map::{EnumMap, IntoIter};

use std::cell::RefCell;
use std::collections::HashSet;

#[derive(Copy, Clone, Debug, EnumMap, PartialEq)]
enum Example {
    A,
    B,
    C,
}

#[test]
fn test_bool() {
    let mut map = enum_map! { false => 24, true => 42 };
    assert_eq!(map[false], 24);
    assert_eq!(map[true], 42);
    map[false] += 1;
    assert_eq!(map[false], 25);
    for (key, item) in &mut map {
        if !key {
            *item += 1;
        }
    }
    assert_eq!(map[false], 26);
    assert_eq!(map[true], 42);
}

#[test]
fn test_option_bool() {
    let mut map = enum_map! { None => 1, Some(false) => 2, Some(true) => 3};
    assert_eq!(map[None], 1);
    assert_eq!(map[Some(false)], 2);
    assert_eq!(map[Some(true)], 3);
    map[None] = 4;
    map[Some(false)] = 5;
    map[Some(true)] = 6;
    assert_eq!(map[None], 4);
    assert_eq!(map[Some(false)], 5);
    assert_eq!(map[Some(true)], 6);
    assert_eq!(map.as_slice(), [4, 5, 6]);
    let expected = [(None, 4), (Some(false), 5), (Some(true), 6)];
    assert_eq!(map.into_iter().collect::<Vec<_>>(), expected);
}

#[test]
fn test_clone() {
    let map = enum_map! { false => 3, true => 5 };
    assert_eq!(map.clone(), map);
}

#[test]
fn test_debug() {
    let map = enum_map! { false => 3, true => 5 };
    assert_eq!(format!("{:?}", map), "{false: 3, true: 5}");
}

#[test]
fn test_hash() {
    let map = enum_map! { false => 3, true => 5 };
    let mut set = HashSet::new();
    set.insert(map);
    assert!(set.contains(&map));
}

#[test]
fn discriminants() {
    #[derive(Debug, EnumMap, PartialEq)]
    enum Discriminants {
        A = 2000,
        B = 3000,
        C = 1000,
    }
    let mut map = EnumMap::new();
    map[Discriminants::A] = 3;
    map[Discriminants::B] = 2;
    map[Discriminants::C] = 1;
    let mut pairs = map.iter();
    assert_eq!(pairs.next(), Some((Discriminants::A, &3)));
    assert_eq!(pairs.next(), Some((Discriminants::B, &2)));
    assert_eq!(pairs.next(), Some((Discriminants::C, &1)));
    assert_eq!(pairs.next(), None);
}

#[test]
fn extend() {
    let mut map = enum_map! { _ => 0 };
    map.extend(vec![(Example::A, 3)]);
    map.extend(vec![(&Example::B, &4)]);
    assert_eq!(
        map,
        enum_map! { Example:: A => 3, Example:: B => 4, Example::C => 0 }
    );
}

#[test]
fn huge_enum() {
    #[derive(EnumMap)]
    enum Example {
        A,
        B,
        C,
        D,
        E,
        F,
        G,
        H,
        I,
        J,
        K,
        L,
        M,
        N,
        O,
        P,
        Q,
        R,
        S,
        T,
        U,
        V,
        W,
        X,
        Y,
        Z,
        Aa,
        Bb,
        Cc,
        Dd,
        Ee,
        Ff,
        Gg,
        Hh,
        Ii,
        Jj,
        Kk,
        Ll,
        Mm,
        Nn,
        Oo,
        Pp,
        Qq,
        Rr,
        Ss,
        Tt,
        Uu,
        Vv,
        Ww,
        Xx,
        Yy,
        Zz,
    }

    let map = enum_map! { _ => 2 };
    assert_eq!(map[Example::Xx], 2);
}

#[test]
fn iterator_len() {
    assert_eq!(
        enum_map! { Example::A | Example::B | Example::C => 0 }
            .iter()
            .len(),
        3
    );
}

#[test]
fn iter_mut_len() {
    assert_eq!(
        enum_map! { Example::A | Example::B | Example::C => 0 }
            .iter_mut()
            .len(),
        3
    );
}

#[test]
fn iterator_next_back() {
    assert_eq!(
        enum_map! { Example::A => 1, Example::B => 2, Example::C => 3 }
            .iter()
            .next_back(),
        Some((Example::C, &3))
    );
}

#[test]
fn iter_mut_next_back() {
    assert_eq!(
        enum_map! { Example::A => 1, Example::B => 2, Example::C => 3 }
            .iter()
            .next_back(),
        Some((Example::C, &mut 3))
    );
}

#[test]
fn into_iter() {
    let mut iter = enum_map! { true => 5, false => 7 }.into_iter();
    assert_eq!(iter.next(), Some((false, 7)));
    assert_eq!(iter.next(), Some((true, 5)));
    assert_eq!(iter.next(), None);
    assert_eq!(iter.next(), None);
}

struct DropReporter<'a> {
    into: &'a RefCell<Vec<usize>>,
    value: usize,
}

impl<'a> Drop for DropReporter<'a> {
    fn drop(&mut self) {
        self.into.borrow_mut().push(self.value);
    }
}

#[test]
fn into_iter_drop() {
    let dropped = RefCell::new(Vec::new());
    let mut a: IntoIter<Example, _> = enum_map! {
        k => DropReporter {
            into: &dropped,
            value: k as usize,
        },
    }.into_iter();
    assert_eq!(a.next().unwrap().0, Example::A);
    assert_eq!(*dropped.borrow(), &[0]);
    drop(a);
    assert_eq!(*dropped.borrow(), &[0, 1, 2]);
}

#[test]
fn test_u8() {
    let mut map = enum_map! { b'a' => 4, _ => 0 };
    map[b'c'] = 3;
    assert_eq!(map[b'a'], 4);
    assert_eq!(map[b'b'], 0);
    assert_eq!(map[b'c'], 3);
}

#[derive(EnumMap)]
enum Void {}

#[test]
fn empty_map() {
    let void: EnumMap<Void, Void> = enum_map!{};
    assert!(void.is_empty());
}

#[test]
#[should_panic]
fn empty_value() {
    let _void: EnumMap<bool, Void> = enum_map! { _ => unreachable!() };
}
<|endoftext|>


<commit_before>pub mod pep_243
{
    use reikna::prime::prime_sieve;
    use std::collections::BinaryHeap;
    use std::cmp::Reverse;

    pub fn solve() -> u64
    {
        let factors = prime_sieve(100);
        let mut heap = BinaryHeap::new();
        heap.push(Reverse((2, 0, 1)));

        loop {
            if let Some(Reverse((n, i, phi))) = heap.pop()
            {
                let f0 = factors[i];
                let f1 = factors[i + 1];
                heap.push(Reverse((f0 * n, i, f0 * phi)));
                heap.push(Reverse((f1 * n, i + 1, (f1 - 1) * phi)));
                if phi as f64 / (n as f64 - 1_f64) < 15499_f64 / 94744_f64 {
                    break n;
                }
            }
        }
    }
}
<commit_msg>housekeeping<commit_after>pub mod pep_243 {
    use reikna::prime::prime_sieve;
    use std::cmp::Reverse;
    use std::collections::BinaryHeap;

    pub fn solve() -> u64 {
        let factors = prime_sieve(100);
        let mut heap = BinaryHeap::new();
        heap.push(Reverse((2, 0, 1)));

        loop {
            if let Some(Reverse((n, i, phi))) = heap.pop() {
                let f0 = factors[i];
                let f1 = factors[i + 1];
                heap.push(Reverse((f0 * n, i, f0 * phi)));
                heap.push(Reverse((f1 * n, i + 1, (f1 - 1) * phi)));
                if phi as f64 / (n as f64 - 1_f64) < 15499_f64 / 94744_f64 {
                    break n;
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>#![feature(integer_atomics)]

extern crate abra;
extern crate rocksdb;
extern crate rustc_serialize;
#[macro_use]
extern crate maplit;
extern crate byteorder;

use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicU32, Ordering};
use std::collections::BTreeMap;

use rocksdb::{DB, WriteBatch, Writable, Options, MergeOperands};
use abra::{Term, Document};
use abra::schema::{Schema, FieldType, FieldRef, AddFieldError};
use rustc_serialize::{json, Encodable};
use byteorder::{ByteOrder, BigEndian};

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct TermRef(u32);

impl TermRef {
    pub fn ord(&self) -> u32 {
        self.0
    }
}

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct DocRef(u32, u16);

impl DocRef {
    pub fn chunk(&self) -> u32 {
        self.0
    }

    pub fn ord(&self) -> u16 {
        self.1
    }
}

fn merge_keys(key: &[u8], existing_val: Option<&[u8]>, operands: &mut MergeOperands) -> Vec<u8> {
    match key[0] {
        b'd' | b'x' => {
            // Sequence of two byte document ids
            // d = directory
            // x = deletion list

            // Allocate vec for new Value
            let new_size = match existing_val {
                Some(existing_val) => existing_val.len(),
                None => 0,
            } + operands.size_hint().0 * 2;

            let mut new_val = Vec::with_capacity(new_size);

            // Push existing value
            existing_val.map(|v| {
                for b in v {
                    new_val.push(*b);
                }
            });

            // Append new entries
            for op in operands {
                for b in op {
                    new_val.push(*b);
                }
            }

            new_val
        }
        b's' => {
            // Statistic
            // An i64 number that can be incremented or decremented
            let mut value = match existing_val {
                Some(existing_val) => BigEndian::read_i64(existing_val),
                None => 0
            };

            for op in operands {
                value += BigEndian::read_i64(op);
            }

            let mut buf = [0; 8];
            BigEndian::write_i64(&mut buf, value);
            buf.iter().cloned().collect()
        }
        _ => {
            // Unrecognised key, fallback to emulating a put operation (by taking the last value)
            operands.last().unwrap().iter().cloned().collect()
        }
    }
}

pub struct RocksDBIndexStore {
    schema: Arc<Schema>,
    db: DB,
    next_term_ref: AtomicU32,
    term_dictionary: RwLock<BTreeMap<Vec<u8>, TermRef>>,
    next_chunk: AtomicU32,
    doc_key_mapping: RwLock<BTreeMap<Vec<u8>, DocRef>>,
    deleted_docs: RwLock<Vec<DocRef>>,
}

impl RocksDBIndexStore {
    pub fn create(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        opts.create_if_missing(true);
        let db = try!(DB::open(&opts, path));

        // Schema
        let schema = Schema::new();
        db.put(b".schema", json::encode(&schema).unwrap().as_bytes());

        // Next term ref
        db.put(b".next_term_ref", b"1");

        // Next chunk
        db.put(b".next_chunk", b"1");

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(1),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(1),
            doc_key_mapping: RwLock::new(BTreeMap::new()),
            deleted_docs: RwLock::new(Vec::new()),
        })
    }

    pub fn open(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        let db = try!(DB::open(&opts, path));

        let schema = match db.get(b".schema") {
            Ok(Some(schema)) => {
                let schema = schema.to_utf8().unwrap().to_string();
                json::decode(&schema).unwrap()
            }
            Ok(None) => Schema::new(),  // TODO: error
            Err(_) => Schema::new(),  // TODO: error
        };

        let next_term_ref = match db.get(b".next_term_ref") {
            Ok(Some(next_term_ref)) => {
                next_term_ref.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        let next_chunk = match db.get(b".next_chunk") {
            Ok(Some(next_chunk)) => {
                next_chunk.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(next_term_ref),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(next_chunk),
            doc_key_mapping: RwLock::new(BTreeMap::new()),
            deleted_docs: RwLock::new(Vec::new()),
        })
    }

    pub fn add_field(&mut self, name: String, field_type: FieldType) -> Result<FieldRef, AddFieldError> {
        let mut schema_copy = (*self.schema).clone();
        let field_ref = try!(schema_copy.add_field(name, field_type));
        self.schema = Arc::new(schema_copy);

        self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());

        Ok(field_ref)
    }

    pub fn remove_field(&mut self, field_ref: &FieldRef) -> bool {
        let mut schema_copy = (*self.schema).clone();
        let field_removed = schema_copy.remove_field(field_ref);

        if field_removed {
            self.schema = Arc::new(schema_copy);
            self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());
        }

        field_removed
    }

    fn get_or_create_term(&mut self, term: &Term) -> TermRef {
        let term_bytes = term.to_bytes();

        if let Some(term_ref) = self.term_dictionary.read().unwrap().get(&term_bytes) {
            return *term_ref;
        }

        // Term doesn't exist in the term dictionary

        // Increment next_term_ref
        let next_term_ref = self.next_term_ref.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_term_ref", (next_term_ref + 1).to_string().as_bytes());

        // Create term ref
        let term_ref = TermRef(next_term_ref);

        // Get exclusive lock to term dictionary
        let mut term_dictionary = self.term_dictionary.write().unwrap();

        // It's possible that another thread has written the term to the dictionary
        // since we checked earlier. If this is the case, We should forget about
        // writing our TermRef and use the one that has been inserted already.
        if let Some(term_ref) = term_dictionary.get(&term_bytes) {
            return *term_ref;
        }

        // Write it to the on-disk term dictionary
        let mut key = Vec::with_capacity(1 + term_bytes.len());
        key.push(b't');
        for byte in term_bytes.iter() {
            key.push(*byte);
        }
        self.db.put(&key, next_term_ref.to_string().as_bytes());

        // Write it to the term dictionary
        term_dictionary.insert(term_bytes, term_ref);

        term_ref
    }

    pub fn insert_or_update_document(&mut self, doc: Document) {
        // Allocate a new chunk for the document

        // Chunk merges are very slow so we should avoid doing them at runtime
        // which is why each new document is created in a fresh chunk.
        // Later on, a background process will come and merge any small chunks
        // together.

        // For best performance, documents should be inserted in batches.

        // Increment next_chunk
        let next_chunk = self.next_chunk.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_chunk", (next_chunk + 1).to_string().as_bytes());

        // Create doc ref
        let doc_ref = DocRef(next_chunk, 0);

        // Start write batch
        let mut write_batch = WriteBatch::default();

        // Insert contents
        let mut token_count: i64 = 0;
        for (field_ref, tokens) in doc.fields.iter() {
            for token in tokens.iter() {
                token_count += 1;
                let term_ref = self.get_or_create_term(&token.term);

                let mut key = Vec::with_capacity(20);
                key.push(b'd');
                for byte in field_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in term_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in doc_ref.chunk().to_string().as_bytes() {
                    key.push(*byte);
                }

                let mut doc_id_bytes = [0; 2];
                BigEndian::write_u16(&mut doc_id_bytes, doc_ref.ord());
                write_batch.merge(&key, &doc_id_bytes);
            }
        }

        // Increment total docs
        let mut key = Vec::with_capacity(5);
        for byte in b"stotal_docs".iter() {
            key.push(*byte);
        }
        key.push(b'/');
        for byte in doc_ref.chunk().to_string().as_bytes() {
            key.push(*byte);
        }
        let mut inc_bytes = [0; 8];
        BigEndian::write_i64(&mut inc_bytes, 1);
        write_batch.merge(&key, &inc_bytes);

        // Increment total tokens
        let mut key = Vec::with_capacity(5);
        for byte in b"stotal_tokens".iter() {
            key.push(*byte);
        }
        key.push(b'/');
        for byte in doc_ref.chunk().to_string().as_bytes() {
            key.push(*byte);
        }
        let mut inc_bytes = [0; 8];
        BigEndian::write_i64(&mut inc_bytes, token_count);
        write_batch.merge(&key, &inc_bytes);

        // Write document data
        self.db.write(write_batch);

        // Update doc_key_mapping
        let mut write_batch = WriteBatch::default();
        let previous_doc_ref = self.doc_key_mapping.write().unwrap().insert(doc.key.as_bytes().iter().cloned().collect(), doc_ref);

        let mut key = Vec::with_capacity(1 + doc.key.len());
        key.push(b'k');
        for byte in doc.key.as_bytes() {
            key.push(*byte);
        }
        let mut doc_ref_bytes = [0; 6];
        BigEndian::write_u32(&mut doc_ref_bytes, doc_ref.chunk());
        BigEndian::write_u16(&mut doc_ref_bytes[4..], doc_ref.ord());
        write_batch.put(&key, &doc_ref_bytes);

        // If there was a document there previously, mark it as deleted
        if let Some(previous_doc_ref) = previous_doc_ref {
            self.deleted_docs.write().unwrap().push(previous_doc_ref);

            let mut key = Vec::with_capacity(5);
            key.push(b'x');
            for byte in previous_doc_ref.chunk().to_string().as_bytes() {
                key.push(*byte);
            }

            let mut previous_doc_id_bytes = [0; 2];
            BigEndian::write_u16(&mut previous_doc_id_bytes, previous_doc_ref.ord());
            write_batch.merge(&key, &previous_doc_id_bytes);
        }

        // Write document data
        self.db.write(write_batch);
    }
}

#[cfg(test)]
mod tests {
    use rocksdb::{DB, Options, IteratorMode};
    use abra::{Term, Token, Document};
    use abra::schema::{Schema, FieldType, FieldRef};

    use super::RocksDBIndexStore;

    fn clean_test_indices() {
        use std::fs::remove_dir_all;

        remove_dir_all("test_indices");
    }

    #[test]
    fn test_create() {
        clean_test_indices();

        let store = RocksDBIndexStore::create("test_indices/test_create");
        assert!(store.is_ok());
    }

    #[test]
    fn test_open() {
        clean_test_indices();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_err());

        // Create DB
        let mut opts = Options::default();
        opts.create_if_missing(true);
        DB::open(&opts, "test_indices/test_open").unwrap();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_ok());
    }

    fn make_test_store(path: &str) -> RocksDBIndexStore {
        let mut store = RocksDBIndexStore::create(path).unwrap();
        let mut title_field = store.add_field("title".to_string(), FieldType::Text).unwrap();
        let mut body_field = store.add_field("body".to_string(), FieldType::Text).unwrap();

        store.insert_or_update_document(Document {
            key: "test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("hello".to_string()), position: 1 },
                    Token { term: Term::String("world".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store.insert_or_update_document(Document {
            key: "another_test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("howdy".to_string()), position: 1 },
                    Token { term: Term::String("partner".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store
    }

    pub fn print_keys(db: &DB) {
        fn bytes_to_string(bytes: &Box<[u8]>) -> String {
            use std::char;

            let mut string = String::new();

            for byte in bytes.iter() {
                if *byte < 128 {
                    // ASCII character
                    string.push(char::from_u32(*byte as u32).unwrap());
                } else {
                    string.push('?');
                }
            }

            string
        }

        for (key, value) in db.iterator(IteratorMode::Start) {
            println!("{} = {:?}", bytes_to_string(&key), value);
        }
    }

    #[test]
    fn test() {
        clean_test_indices();

        let store = make_test_store("test_indices/test");

        print_keys(&store.db);

    }
}
<commit_msg>Make each test only remove its own directory<commit_after>#![feature(integer_atomics)]

extern crate abra;
extern crate rocksdb;
extern crate rustc_serialize;
#[macro_use]
extern crate maplit;
extern crate byteorder;

use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicU32, Ordering};
use std::collections::BTreeMap;

use rocksdb::{DB, WriteBatch, Writable, Options, MergeOperands};
use abra::{Term, Document};
use abra::schema::{Schema, FieldType, FieldRef, AddFieldError};
use rustc_serialize::{json, Encodable};
use byteorder::{ByteOrder, BigEndian};

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct TermRef(u32);

impl TermRef {
    pub fn ord(&self) -> u32 {
        self.0
    }
}

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct DocRef(u32, u16);

impl DocRef {
    pub fn chunk(&self) -> u32 {
        self.0
    }

    pub fn ord(&self) -> u16 {
        self.1
    }
}

fn merge_keys(key: &[u8], existing_val: Option<&[u8]>, operands: &mut MergeOperands) -> Vec<u8> {
    match key[0] {
        b'd' | b'x' => {
            // Sequence of two byte document ids
            // d = directory
            // x = deletion list

            // Allocate vec for new Value
            let new_size = match existing_val {
                Some(existing_val) => existing_val.len(),
                None => 0,
            } + operands.size_hint().0 * 2;

            let mut new_val = Vec::with_capacity(new_size);

            // Push existing value
            existing_val.map(|v| {
                for b in v {
                    new_val.push(*b);
                }
            });

            // Append new entries
            for op in operands {
                for b in op {
                    new_val.push(*b);
                }
            }

            new_val
        }
        b's' => {
            // Statistic
            // An i64 number that can be incremented or decremented
            let mut value = match existing_val {
                Some(existing_val) => BigEndian::read_i64(existing_val),
                None => 0
            };

            for op in operands {
                value += BigEndian::read_i64(op);
            }

            let mut buf = [0; 8];
            BigEndian::write_i64(&mut buf, value);
            buf.iter().cloned().collect()
        }
        _ => {
            // Unrecognised key, fallback to emulating a put operation (by taking the last value)
            operands.last().unwrap().iter().cloned().collect()
        }
    }
}

pub struct RocksDBIndexStore {
    schema: Arc<Schema>,
    db: DB,
    next_term_ref: AtomicU32,
    term_dictionary: RwLock<BTreeMap<Vec<u8>, TermRef>>,
    next_chunk: AtomicU32,
    doc_key_mapping: RwLock<BTreeMap<Vec<u8>, DocRef>>,
    deleted_docs: RwLock<Vec<DocRef>>,
}

impl RocksDBIndexStore {
    pub fn create(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        opts.create_if_missing(true);
        let db = try!(DB::open(&opts, path));

        // Schema
        let schema = Schema::new();
        db.put(b".schema", json::encode(&schema).unwrap().as_bytes());

        // Next term ref
        db.put(b".next_term_ref", b"1");

        // Next chunk
        db.put(b".next_chunk", b"1");

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(1),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(1),
            doc_key_mapping: RwLock::new(BTreeMap::new()),
            deleted_docs: RwLock::new(Vec::new()),
        })
    }

    pub fn open(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        let db = try!(DB::open(&opts, path));

        let schema = match db.get(b".schema") {
            Ok(Some(schema)) => {
                let schema = schema.to_utf8().unwrap().to_string();
                json::decode(&schema).unwrap()
            }
            Ok(None) => Schema::new(),  // TODO: error
            Err(_) => Schema::new(),  // TODO: error
        };

        let next_term_ref = match db.get(b".next_term_ref") {
            Ok(Some(next_term_ref)) => {
                next_term_ref.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        let next_chunk = match db.get(b".next_chunk") {
            Ok(Some(next_chunk)) => {
                next_chunk.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(next_term_ref),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(next_chunk),
            doc_key_mapping: RwLock::new(BTreeMap::new()),
            deleted_docs: RwLock::new(Vec::new()),
        })
    }

    pub fn add_field(&mut self, name: String, field_type: FieldType) -> Result<FieldRef, AddFieldError> {
        let mut schema_copy = (*self.schema).clone();
        let field_ref = try!(schema_copy.add_field(name, field_type));
        self.schema = Arc::new(schema_copy);

        self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());

        Ok(field_ref)
    }

    pub fn remove_field(&mut self, field_ref: &FieldRef) -> bool {
        let mut schema_copy = (*self.schema).clone();
        let field_removed = schema_copy.remove_field(field_ref);

        if field_removed {
            self.schema = Arc::new(schema_copy);
            self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());
        }

        field_removed
    }

    fn get_or_create_term(&mut self, term: &Term) -> TermRef {
        let term_bytes = term.to_bytes();

        if let Some(term_ref) = self.term_dictionary.read().unwrap().get(&term_bytes) {
            return *term_ref;
        }

        // Term doesn't exist in the term dictionary

        // Increment next_term_ref
        let next_term_ref = self.next_term_ref.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_term_ref", (next_term_ref + 1).to_string().as_bytes());

        // Create term ref
        let term_ref = TermRef(next_term_ref);

        // Get exclusive lock to term dictionary
        let mut term_dictionary = self.term_dictionary.write().unwrap();

        // It's possible that another thread has written the term to the dictionary
        // since we checked earlier. If this is the case, We should forget about
        // writing our TermRef and use the one that has been inserted already.
        if let Some(term_ref) = term_dictionary.get(&term_bytes) {
            return *term_ref;
        }

        // Write it to the on-disk term dictionary
        let mut key = Vec::with_capacity(1 + term_bytes.len());
        key.push(b't');
        for byte in term_bytes.iter() {
            key.push(*byte);
        }
        self.db.put(&key, next_term_ref.to_string().as_bytes());

        // Write it to the term dictionary
        term_dictionary.insert(term_bytes, term_ref);

        term_ref
    }

    pub fn insert_or_update_document(&mut self, doc: Document) {
        // Allocate a new chunk for the document

        // Chunk merges are very slow so we should avoid doing them at runtime
        // which is why each new document is created in a fresh chunk.
        // Later on, a background process will come and merge any small chunks
        // together.

        // For best performance, documents should be inserted in batches.

        // Increment next_chunk
        let next_chunk = self.next_chunk.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_chunk", (next_chunk + 1).to_string().as_bytes());

        // Create doc ref
        let doc_ref = DocRef(next_chunk, 0);

        // Start write batch
        let mut write_batch = WriteBatch::default();

        // Insert contents
        let mut token_count: i64 = 0;
        for (field_ref, tokens) in doc.fields.iter() {
            for token in tokens.iter() {
                token_count += 1;
                let term_ref = self.get_or_create_term(&token.term);

                let mut key = Vec::with_capacity(20);
                key.push(b'd');
                for byte in field_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in term_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in doc_ref.chunk().to_string().as_bytes() {
                    key.push(*byte);
                }

                let mut doc_id_bytes = [0; 2];
                BigEndian::write_u16(&mut doc_id_bytes, doc_ref.ord());
                write_batch.merge(&key, &doc_id_bytes);
            }
        }

        // Increment total docs
        let mut key = Vec::with_capacity(5);
        for byte in b"stotal_docs".iter() {
            key.push(*byte);
        }
        key.push(b'/');
        for byte in doc_ref.chunk().to_string().as_bytes() {
            key.push(*byte);
        }
        let mut inc_bytes = [0; 8];
        BigEndian::write_i64(&mut inc_bytes, 1);
        write_batch.merge(&key, &inc_bytes);

        // Increment total tokens
        let mut key = Vec::with_capacity(5);
        for byte in b"stotal_tokens".iter() {
            key.push(*byte);
        }
        key.push(b'/');
        for byte in doc_ref.chunk().to_string().as_bytes() {
            key.push(*byte);
        }
        let mut inc_bytes = [0; 8];
        BigEndian::write_i64(&mut inc_bytes, token_count);
        write_batch.merge(&key, &inc_bytes);

        // Write document data
        self.db.write(write_batch);

        // Update doc_key_mapping
        let mut write_batch = WriteBatch::default();
        let previous_doc_ref = self.doc_key_mapping.write().unwrap().insert(doc.key.as_bytes().iter().cloned().collect(), doc_ref);

        let mut key = Vec::with_capacity(1 + doc.key.len());
        key.push(b'k');
        for byte in doc.key.as_bytes() {
            key.push(*byte);
        }
        let mut doc_ref_bytes = [0; 6];
        BigEndian::write_u32(&mut doc_ref_bytes, doc_ref.chunk());
        BigEndian::write_u16(&mut doc_ref_bytes[4..], doc_ref.ord());
        write_batch.put(&key, &doc_ref_bytes);

        // If there was a document there previously, mark it as deleted
        if let Some(previous_doc_ref) = previous_doc_ref {
            self.deleted_docs.write().unwrap().push(previous_doc_ref);

            let mut key = Vec::with_capacity(5);
            key.push(b'x');
            for byte in previous_doc_ref.chunk().to_string().as_bytes() {
                key.push(*byte);
            }

            let mut previous_doc_id_bytes = [0; 2];
            BigEndian::write_u16(&mut previous_doc_id_bytes, previous_doc_ref.ord());
            write_batch.merge(&key, &previous_doc_id_bytes);
        }

        // Write document data
        self.db.write(write_batch);
    }
}

#[cfg(test)]
mod tests {
    use std::fs::remove_dir_all;

    use rocksdb::{DB, Options, IteratorMode};
    use abra::{Term, Token, Document};
    use abra::schema::{Schema, FieldType, FieldRef};

    use super::RocksDBIndexStore;

    #[test]
    fn test_create() {
        remove_dir_all("test_indices/test_create");

        let store = RocksDBIndexStore::create("test_indices/test_create");
        assert!(store.is_ok());
    }

    #[test]
    fn test_open() {
        remove_dir_all("test_indices/test_open");

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_err());

        // Create DB
        let mut opts = Options::default();
        opts.create_if_missing(true);
        DB::open(&opts, "test_indices/test_open").unwrap();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_ok());
    }

    fn make_test_store(path: &str) -> RocksDBIndexStore {
        let mut store = RocksDBIndexStore::create(path).unwrap();
        let mut title_field = store.add_field("title".to_string(), FieldType::Text).unwrap();
        let mut body_field = store.add_field("body".to_string(), FieldType::Text).unwrap();

        store.insert_or_update_document(Document {
            key: "test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("hello".to_string()), position: 1 },
                    Token { term: Term::String("world".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store.insert_or_update_document(Document {
            key: "another_test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("howdy".to_string()), position: 1 },
                    Token { term: Term::String("partner".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store
    }

    pub fn print_keys(db: &DB) {
        fn bytes_to_string(bytes: &Box<[u8]>) -> String {
            use std::char;

            let mut string = String::new();

            for byte in bytes.iter() {
                if *byte < 128 {
                    // ASCII character
                    string.push(char::from_u32(*byte as u32).unwrap());
                } else {
                    string.push('?');
                }
            }

            string
        }

        for (key, value) in db.iterator(IteratorMode::Start) {
            println!("{} = {:?}", bytes_to_string(&key), value);
        }
    }

    #[test]
    fn test() {
        remove_dir_all("test_indices/test");

        let store = make_test_store("test_indices/test");

        print_keys(&store.db);

    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::path::{Path, PathBuf};
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};

/// List of directories containing files to expand. The first tuple element is the source
/// directory, while the second tuple element is the destination directory.
#[rustfmt::skip]
static TO_EXPAND: &[(&str, &str)] = &[
    ("src/ci/github-actions", ".github/workflows"),
];

/// Name of a special key that will be removed from all the maps in expanded configuration files.
/// This key can then be used to contain shared anchors.
static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";

/// Message that will be included at the top of all the expanded files. {source} will be replaced
/// with the source filename relative to the base path.
static HEADER_MESSAGE: &str = "\
#############################################################
#   WARNING: automatically generated file, DO NOT CHANGE!   #
#############################################################

# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
#   {source}
#
# Once you make changes to that file you need to run:
#
#   ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.

";

enum Mode {
    Check,
    Generate,
}

struct App {
    mode: Mode,
    base: PathBuf,
}

impl App {
    fn from_args() -> Result<Self, Box<dyn Error>> {
        // Parse CLI arguments
        let args = std::env::args().skip(1).collect::<Vec<_>>();
        let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
            ["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
            ["check", ref base] => (Mode::Check, PathBuf::from(base)),
            _ => {
                eprintln!("usage: expand-yaml-anchors <source-dir> <dest-dir>");
                std::process::exit(1);
            }
        };

        Ok(App { mode, base })
    }

    fn run(&self) -> Result<(), Box<dyn Error>> {
        for (source, dest) in TO_EXPAND {
            let source = self.base.join(source);
            let dest = self.base.join(dest);
            for entry in std::fs::read_dir(&source)? {
                let path = entry?.path();
                if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
                    continue;
                }

                let dest_path = dest.join(path.file_name().unwrap());
                self.expand(&path, &dest_path).with_context(|| match self.mode {
                    Mode::Generate => format!(
                        "failed to expand {} into {}",
                        self.path(&path),
                        self.path(&dest_path)
                    ),
                    Mode::Check => format!("{} is not up to date", self.path(&dest_path)),
                })?;
            }
        }
        Ok(())
    }

    fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
        let content = std::fs::read_to_string(source)
            .with_context(|| format!("failed to read {}", self.path(source)))?;

        let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string());

        let documents = YamlLoader::load_from_str(&content)
            .with_context(|| format!("failed to parse {}", self.path(source)))?;
        for mut document in documents.into_iter() {
            document = yaml_merge_keys::merge_keys(document)
                .with_context(|| format!("failed to expand {}", self.path(source)))?;
            document = filter_document(document);

            YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
                context: "failed to serialize the expanded yaml".into(),
                source: Box::new(err),
            })?;
            buf.push('\n');
        }

        match self.mode {
            Mode::Check => {
                let old = std::fs::read_to_string(dest)
                    .with_context(|| format!("failed to read {}", self.path(dest)))?;
                if old != buf {
                    return Err(Box::new(StrError(format!(
                        "{} and {} are different",
                        self.path(source),
                        self.path(dest),
                    ))));
                }
            }
            Mode::Generate => {
                std::fs::write(dest, buf.as_bytes())
                    .with_context(|| format!("failed to write to {}", self.path(dest)))?;
            }
        }
        Ok(())
    }

    fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
        path.strip_prefix(&self.base).unwrap_or(path).display()
    }
}

fn filter_document(document: Yaml) -> Yaml {
    match document {
        Yaml::Hash(map) => Yaml::Hash(
            map.into_iter()
                .filter(|(key, _)| {
                    if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
                })
                .map(|(key, value)| (filter_document(key), filter_document(value)))
                .collect(),
        ),
        Yaml::Array(vec) => Yaml::Array(vec.into_iter().map(filter_document).collect()),
        other => other,
    }
}

fn main() {
    if let Err(err) = App::from_args().and_then(|app| app.run()) {
        eprintln!("error: {}", err);

        let mut source = err.as_ref() as &dyn Error;
        while let Some(err) = source.source() {
            eprintln!("caused by: {}", err);
            source = err;
        }

        std::process::exit(1);
    }
}

#[derive(Debug)]
struct StrError(String);

impl Error for StrError {}

impl std::fmt::Display for StrError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        std::fmt::Display::fmt(&self.0, f)
    }
}

#[derive(Debug)]
struct WithContext {
    context: String,
    source: Box<dyn Error>,
}

impl std::fmt::Display for WithContext {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.context)
    }
}

impl Error for WithContext {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        Some(self.source.as_ref())
    }
}

pub(crate) trait ResultExt<T> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
}

impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
        match self {
            Ok(ok) => Ok(ok),
            Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
        }
    }
}
<commit_msg>Rollup merge of #80026 - JohnTitor:separator-insensitive, r=Mark-Simulacrum<commit_after>use std::error::Error;
use std::path::{Path, PathBuf};
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};

/// List of directories containing files to expand. The first tuple element is the source
/// directory, while the second tuple element is the destination directory.
#[rustfmt::skip]
static TO_EXPAND: &[(&str, &str)] = &[
    ("src/ci/github-actions", ".github/workflows"),
];

/// Name of a special key that will be removed from all the maps in expanded configuration files.
/// This key can then be used to contain shared anchors.
static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";

/// Message that will be included at the top of all the expanded files. {source} will be replaced
/// with the source filename relative to the base path.
static HEADER_MESSAGE: &str = "\
#############################################################
#   WARNING: automatically generated file, DO NOT CHANGE!   #
#############################################################

# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
#   {source}
#
# Once you make changes to that file you need to run:
#
#   ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.

";

enum Mode {
    Check,
    Generate,
}

struct App {
    mode: Mode,
    base: PathBuf,
}

impl App {
    fn from_args() -> Result<Self, Box<dyn Error>> {
        // Parse CLI arguments
        let args = std::env::args().skip(1).collect::<Vec<_>>();
        let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
            ["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
            ["check", ref base] => (Mode::Check, PathBuf::from(base)),
            _ => {
                eprintln!("usage: expand-yaml-anchors <source-dir> <dest-dir>");
                std::process::exit(1);
            }
        };

        Ok(App { mode, base })
    }

    fn run(&self) -> Result<(), Box<dyn Error>> {
        for (source, dest) in TO_EXPAND {
            let source = self.base.join(source);
            let dest = self.base.join(dest);
            for entry in std::fs::read_dir(&source)? {
                let path = entry?.path();
                if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
                    continue;
                }

                let dest_path = dest.join(path.file_name().unwrap());
                self.expand(&path, &dest_path).with_context(|| match self.mode {
                    Mode::Generate => format!(
                        "failed to expand {} into {}",
                        self.path(&path),
                        self.path(&dest_path)
                    ),
                    Mode::Check => format!("{} is not up to date", self.path(&dest_path)),
                })?;
            }
        }
        Ok(())
    }

    fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
        let content = std::fs::read_to_string(source)
            .with_context(|| format!("failed to read {}", self.path(source)))?;

        let mut buf =
            HEADER_MESSAGE.replace("{source}", &self.path(source).to_string().replace("\\", "/"));

        let documents = YamlLoader::load_from_str(&content)
            .with_context(|| format!("failed to parse {}", self.path(source)))?;
        for mut document in documents.into_iter() {
            document = yaml_merge_keys::merge_keys(document)
                .with_context(|| format!("failed to expand {}", self.path(source)))?;
            document = filter_document(document);

            YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
                context: "failed to serialize the expanded yaml".into(),
                source: Box::new(err),
            })?;
            buf.push('\n');
        }

        match self.mode {
            Mode::Check => {
                let old = std::fs::read_to_string(dest)
                    .with_context(|| format!("failed to read {}", self.path(dest)))?;
                if old != buf {
                    return Err(Box::new(StrError(format!(
                        "{} and {} are different",
                        self.path(source),
                        self.path(dest),
                    ))));
                }
            }
            Mode::Generate => {
                std::fs::write(dest, buf.as_bytes())
                    .with_context(|| format!("failed to write to {}", self.path(dest)))?;
            }
        }
        Ok(())
    }

    fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
        path.strip_prefix(&self.base).unwrap_or(path).display()
    }
}

fn filter_document(document: Yaml) -> Yaml {
    match document {
        Yaml::Hash(map) => Yaml::Hash(
            map.into_iter()
                .filter(|(key, _)| {
                    if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
                })
                .map(|(key, value)| (filter_document(key), filter_document(value)))
                .collect(),
        ),
        Yaml::Array(vec) => Yaml::Array(vec.into_iter().map(filter_document).collect()),
        other => other,
    }
}

fn main() {
    if let Err(err) = App::from_args().and_then(|app| app.run()) {
        eprintln!("error: {}", err);

        let mut source = err.as_ref() as &dyn Error;
        while let Some(err) = source.source() {
            eprintln!("caused by: {}", err);
            source = err;
        }

        std::process::exit(1);
    }
}

#[derive(Debug)]
struct StrError(String);

impl Error for StrError {}

impl std::fmt::Display for StrError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        std::fmt::Display::fmt(&self.0, f)
    }
}

#[derive(Debug)]
struct WithContext {
    context: String,
    source: Box<dyn Error>,
}

impl std::fmt::Display for WithContext {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.context)
    }
}

impl Error for WithContext {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        Some(self.source.as_ref())
    }
}

pub(crate) trait ResultExt<T> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
}

impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
        match self {
            Ok(ok) => Ok(ok),
            Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
        }
    }
}
<|endoftext|>


<commit_before>#[cfg(feature = "extra-filters")]
use chrono::FixedOffset;

use value::Value;
use value::Scalar;
use interpreter::{FilterError, FilterResult};

use super::check_args_len;

pub fn date(input: &Value, args: &[Value]) -> FilterResult {
    check_args_len(args, 1, 0)?;

    let format = args[0].to_str();
    if format.is_empty() {
        return Ok(input.clone());
    }

    let date = input.as_scalar().and_then(Scalar::to_date);
    let date = match date {
        Some(d) => d,
        None => {
            return Ok(input.clone());
        }
    };

    Ok(Value::scalar(date.format(format.as_ref()).to_string()))
}

// liquid-rust proprietary

#[cfg(feature = "extra-filters")]
pub fn date_in_tz(input: &Value, args: &[Value]) -> FilterResult {
    check_args_len(args, 2, 0)?;

    let date = input
        .as_scalar()
        .and_then(Scalar::to_date)
        .ok_or_else(|| FilterError::InvalidType("Invalid date format".into()))?;

    let format = args[0].to_str();

    let n =
        args[1]
            .as_scalar()
            .and_then(Scalar::to_integer)
            .ok_or_else(|| FilterError::InvalidArgument(1, "Whole number expected".to_owned()))?;
    let timezone = FixedOffset::east(n * 3600);

    Ok(Value::scalar(date.with_timezone(&timezone)
                         .format(format.as_ref())
                         .to_string()))
}

#[cfg(test)]
mod tests {

    use super::*;

    macro_rules! unit {
        ( $a:ident, $b:expr ) => {{
            unit!($a, $b, &[])
        }};
        ( $a:ident, $b:expr , $c:expr) => {{
            $a(&$b, $c).unwrap()
        }};
    }

    macro_rules! failed {
        ( $a:ident, $b:expr ) => {{
            failed!($a, $b, &[])
        }};
        ( $a:ident, $b:expr, $c:expr ) => {{
            $a(&$b, $c).unwrap_err()
        }};
    }

    macro_rules! tos {
        ( $a:expr ) => {{
            Value::scalar($a.to_owned())
        }};
    }

    #[test]
    fn unit_date() {
        assert_eq!(unit!(date,
                         tos!("13 Jun 2016 02:30:00 +0300"),
                         &[tos!("%Y-%m-%d")]),
                   tos!("2016-06-13"));
    }

    #[test]
    fn unit_date_cobalt_format() {
        assert_eq!(unit!(date, tos!("2016-06-13 02:30:00 +0300"), &[tos!("%Y-%m-%d")]),
                   tos!("2016-06-13"));
    }

    #[test]
    fn unit_date_bad_input_type() {
        assert_eq!(unit!(date, Value::scalar(0f32), &[tos!("%Y-%m-%d")]),
                   Value::scalar(0f32));
    }

    #[test]
    fn unit_date_bad_input_format() {
        assert_eq!(unit!(date, tos!("blah blah blah"), &[tos!("%Y-%m-%d")]),
                   tos!("blah blah blah"));
    }

    #[test]
    fn unit_date_format_empty() {
        assert_eq!(unit!(date,
                         tos!("13 Jun 2016 02:30:00 +0300"),
                         &[Value::scalar("".to_owned())]),
                   tos!("13 Jun 2016 02:30:00 +0300"));
    }

    #[test]
    fn unit_date_bad_format_type() {
        assert_eq!(unit!(date,
                         tos!("13 Jun 2016 02:30:00 +0300"),
                         &[Value::scalar(0f32)]),
                   tos!("0"));
    }

    #[test]
    fn unit_date_missing_format() {
        assert_eq!(failed!(date, tos!("13 Jun 2016 02:30:00 +0300")),
                   FilterError::InvalidArgumentCount("expected at least 1, 0 given".to_owned()));
    }

    #[test]
    fn unit_date_extra_param() {
        assert_eq!(failed!(date,
                           tos!("13 Jun 2016 02:30:00 +0300"),
                           &[Value::scalar(0f32), Value::scalar(1f32)]),
                   FilterError::InvalidArgumentCount("expected at most 1, 2 given".to_owned()));
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_same_day() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(3i32)];
        let desired_result = tos!("2016-06-13 15:00:00 +0300");
        assert_eq!(unit!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_previous_day() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(-13i32)];
        let desired_result = tos!("2016-06-12 23:00:00 -1300");
        assert_eq!(unit!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_next_day() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(13i32)];
        let desired_result = tos!("2016-06-14 01:00:00 +1300");
        assert_eq!(unit!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_input_not_a_string() {
        let input = &Value::scalar(0f32);
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(0i32)];
        let desired_result = FilterError::InvalidType("Invalid date format".to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_input_not_a_date_string() {
        let input = &tos!("blah blah blah");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(0i32)];
        let desired_result = FilterError::InvalidType("Invalid date format".to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_offset_not_a_num() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), tos!("Hello")];
        let desired_result = FilterError::InvalidArgument(1, "Whole number expected".to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_zero_arguments() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[];
        let desired_result = FilterError::InvalidArgumentCount("expected at least 2, 0 given"
                                                                   .to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_one_argument() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z")];
        let desired_result = FilterError::InvalidArgumentCount("expected at least 2, 1 given"
                                                                   .to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_three_arguments() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"),
                     Value::scalar(0f32),
                     Value::scalar(1f32)];
        let desired_result = FilterError::InvalidArgumentCount("expected at most 2, 3 given"
                                                                   .to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }
}
<commit_msg>fix: Remove warning when no-default-features<commit_after>use value::Value;
use value::Scalar;
use interpreter::FilterResult;

use super::check_args_len;

#[cfg(feature = "extra-filters")]
use chrono::FixedOffset;
#[cfg(feature = "extra-filters")]
use interpreter::FilterError;

pub fn date(input: &Value, args: &[Value]) -> FilterResult {
    check_args_len(args, 1, 0)?;

    let format = args[0].to_str();
    if format.is_empty() {
        return Ok(input.clone());
    }

    let date = input.as_scalar().and_then(Scalar::to_date);
    let date = match date {
        Some(d) => d,
        None => {
            return Ok(input.clone());
        }
    };

    Ok(Value::scalar(date.format(format.as_ref()).to_string()))
}

// liquid-rust proprietary

#[cfg(feature = "extra-filters")]
pub fn date_in_tz(input: &Value, args: &[Value]) -> FilterResult {
    check_args_len(args, 2, 0)?;

    let date = input
        .as_scalar()
        .and_then(Scalar::to_date)
        .ok_or_else(|| FilterError::InvalidType("Invalid date format".into()))?;

    let format = args[0].to_str();

    let n =
        args[1]
            .as_scalar()
            .and_then(Scalar::to_integer)
            .ok_or_else(|| FilterError::InvalidArgument(1, "Whole number expected".to_owned()))?;
    let timezone = FixedOffset::east(n * 3600);

    Ok(Value::scalar(date.with_timezone(&timezone)
                         .format(format.as_ref())
                         .to_string()))
}

#[cfg(test)]
mod tests {

    use super::*;

    macro_rules! unit {
        ( $a:ident, $b:expr ) => {{
            unit!($a, $b, &[])
        }};
        ( $a:ident, $b:expr , $c:expr) => {{
            $a(&$b, $c).unwrap()
        }};
    }

    macro_rules! failed {
        ( $a:ident, $b:expr ) => {{
            failed!($a, $b, &[])
        }};
        ( $a:ident, $b:expr, $c:expr ) => {{
            $a(&$b, $c).unwrap_err()
        }};
    }

    macro_rules! tos {
        ( $a:expr ) => {{
            Value::scalar($a.to_owned())
        }};
    }

    #[test]
    fn unit_date() {
        assert_eq!(unit!(date,
                         tos!("13 Jun 2016 02:30:00 +0300"),
                         &[tos!("%Y-%m-%d")]),
                   tos!("2016-06-13"));
    }

    #[test]
    fn unit_date_cobalt_format() {
        assert_eq!(unit!(date, tos!("2016-06-13 02:30:00 +0300"), &[tos!("%Y-%m-%d")]),
                   tos!("2016-06-13"));
    }

    #[test]
    fn unit_date_bad_input_type() {
        assert_eq!(unit!(date, Value::scalar(0f32), &[tos!("%Y-%m-%d")]),
                   Value::scalar(0f32));
    }

    #[test]
    fn unit_date_bad_input_format() {
        assert_eq!(unit!(date, tos!("blah blah blah"), &[tos!("%Y-%m-%d")]),
                   tos!("blah blah blah"));
    }

    #[test]
    fn unit_date_format_empty() {
        assert_eq!(unit!(date,
                         tos!("13 Jun 2016 02:30:00 +0300"),
                         &[Value::scalar("".to_owned())]),
                   tos!("13 Jun 2016 02:30:00 +0300"));
    }

    #[test]
    fn unit_date_bad_format_type() {
        assert_eq!(unit!(date,
                         tos!("13 Jun 2016 02:30:00 +0300"),
                         &[Value::scalar(0f32)]),
                   tos!("0"));
    }

    #[test]
    fn unit_date_missing_format() {
        assert_eq!(failed!(date, tos!("13 Jun 2016 02:30:00 +0300")),
                   FilterError::InvalidArgumentCount("expected at least 1, 0 given".to_owned()));
    }

    #[test]
    fn unit_date_extra_param() {
        assert_eq!(failed!(date,
                           tos!("13 Jun 2016 02:30:00 +0300"),
                           &[Value::scalar(0f32), Value::scalar(1f32)]),
                   FilterError::InvalidArgumentCount("expected at most 1, 2 given".to_owned()));
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_same_day() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(3i32)];
        let desired_result = tos!("2016-06-13 15:00:00 +0300");
        assert_eq!(unit!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_previous_day() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(-13i32)];
        let desired_result = tos!("2016-06-12 23:00:00 -1300");
        assert_eq!(unit!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_next_day() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(13i32)];
        let desired_result = tos!("2016-06-14 01:00:00 +1300");
        assert_eq!(unit!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_input_not_a_string() {
        let input = &Value::scalar(0f32);
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(0i32)];
        let desired_result = FilterError::InvalidType("Invalid date format".to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_input_not_a_date_string() {
        let input = &tos!("blah blah blah");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), Value::scalar(0i32)];
        let desired_result = FilterError::InvalidType("Invalid date format".to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_offset_not_a_num() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"), tos!("Hello")];
        let desired_result = FilterError::InvalidArgument(1, "Whole number expected".to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_zero_arguments() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[];
        let desired_result = FilterError::InvalidArgumentCount("expected at least 2, 0 given"
                                                                   .to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_one_argument() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z")];
        let desired_result = FilterError::InvalidArgumentCount("expected at least 2, 1 given"
                                                                   .to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }

    #[test]
    #[cfg(feature = "extra-filters")]
    fn unit_date_in_tz_three_arguments() {
        let input = &tos!("13 Jun 2016 12:00:00 +0000");
        let args = &[tos!("%Y-%m-%d %H:%M:%S %z"),
                     Value::scalar(0f32),
                     Value::scalar(1f32)];
        let desired_result = FilterError::InvalidArgumentCount("expected at most 2, 3 given"
                                                                   .to_owned());
        assert_eq!(failed!(date_in_tz, input, args), desired_result);
    }
}
<|endoftext|>