<commit_before>// This is an X-Macro.
#[macro_export]
macro_rules! list_of_all_specifier_classes {
    ($your_macro:ident) => {
        $your_macro!($crate::ws_client_peer::WsClientClass);
        #[cfg(all(unix, feature = "ssl"))]
        $your_macro!($crate::ws_client_peer::WsClientSecureClass);
        $your_macro!($crate::ws_server_peer::WsTcpServerClass);
        $your_macro!($crate::ws_server_peer::WsInetdServerClass);
        $your_macro!($crate::ws_server_peer::WsUnixServerClass);
        $your_macro!($crate::ws_server_peer::WsAbstractUnixServerClass);
        $your_macro!($crate::ws_server_peer::WsServerClass);

        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::StdioClass);
        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::InetdClass);
        #[cfg(not(all(unix, feature = "unix_stdio")))]
        $your_macro!($crate::stdio_threaded_peer::ThreadedStdioSubstituteClass);
        #[cfg(not(all(unix, feature = "unix_stdio")))]
        $your_macro!($crate::stdio_threaded_peer::InetdClass);

        $your_macro!($crate::net_peer::TcpConnectClass);
        $your_macro!($crate::net_peer::TcpListenClass);

        #[cfg(feature = "tokio-process")]
        $your_macro!($crate::process_peer::ShCClass);
        #[cfg(feature = "tokio-process")]
        $your_macro!($crate::process_peer::CmdClass);
        #[cfg(feature = "tokio-process")]
        $your_macro!($crate::process_peer::ExecClass);

        $your_macro!($crate::file_peer::ReadFileClass);
        $your_macro!($crate::file_peer::WriteFileClass);
        $your_macro!($crate::file_peer::AppendFileClass);

        $your_macro!($crate::primitive_reuse_peer::ReuserClass);
        $your_macro!($crate::broadcast_reuse_peer::BroadcastReuserClass);
        $your_macro!($crate::reconnect_peer::AutoReconnectClass);

        $your_macro!($crate::ws_client_peer::WsConnectClass);

        $your_macro!($crate::net_peer::UdpConnectClass);
        $your_macro!($crate::net_peer::UdpListenClass);

        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::OpenAsyncClass);
        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::OpenFdAsyncClass);

        $your_macro!($crate::stdio_threaded_peer::ThreadedStdioClass);

        #[cfg(unix)]
        $your_macro!($crate::unix_peer::UnixConnectClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::UnixListenClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::UnixDgramClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::AbstractConnectClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::AbstractListenClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::AbstractDgramClass);

        $your_macro!($crate::line_peer::Message2LineClass);
        $your_macro!($crate::line_peer::Line2MessageClass);
        $your_macro!($crate::mirror_peer::MirrorClass);
        $your_macro!($crate::mirror_peer::LiteralReplyClass);
        $your_macro!($crate::trivial_peer::CloggedClass);
        $your_macro!($crate::trivial_peer::LiteralClass);
        $your_macro!($crate::trivial_peer::AssertClass);
        $your_macro!($crate::trivial_peer::Assert2Class);

        #[cfg(feature = "seqpacket")]
        $your_macro!($crate::unix_peer::SeqpacketConnectClass);
        #[cfg(feature = "seqpacket")]
        $your_macro!($crate::unix_peer::SeqpacketListenClass);

        /*
                                        $your_macro!($crate:: :: );
                                        $your_macro!($crate:: :: );
                                        $your_macro!($crate:: :: );
                                        */
    };
}
<commit_msg>Fix all_peers that limited SSL suppor to UNIX.<commit_after>// This is an X-Macro.
#[macro_export]
macro_rules! list_of_all_specifier_classes {
    ($your_macro:ident) => {
        $your_macro!($crate::ws_client_peer::WsClientClass);
        #[cfg(feature = "ssl")]
        $your_macro!($crate::ws_client_peer::WsClientSecureClass);
        $your_macro!($crate::ws_server_peer::WsTcpServerClass);
        $your_macro!($crate::ws_server_peer::WsInetdServerClass);
        $your_macro!($crate::ws_server_peer::WsUnixServerClass);
        $your_macro!($crate::ws_server_peer::WsAbstractUnixServerClass);
        $your_macro!($crate::ws_server_peer::WsServerClass);

        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::StdioClass);
        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::InetdClass);
        #[cfg(not(all(unix, feature = "unix_stdio")))]
        $your_macro!($crate::stdio_threaded_peer::ThreadedStdioSubstituteClass);
        #[cfg(not(all(unix, feature = "unix_stdio")))]
        $your_macro!($crate::stdio_threaded_peer::InetdClass);

        $your_macro!($crate::net_peer::TcpConnectClass);
        $your_macro!($crate::net_peer::TcpListenClass);

        #[cfg(feature = "tokio-process")]
        $your_macro!($crate::process_peer::ShCClass);
        #[cfg(feature = "tokio-process")]
        $your_macro!($crate::process_peer::CmdClass);
        #[cfg(feature = "tokio-process")]
        $your_macro!($crate::process_peer::ExecClass);

        $your_macro!($crate::file_peer::ReadFileClass);
        $your_macro!($crate::file_peer::WriteFileClass);
        $your_macro!($crate::file_peer::AppendFileClass);

        $your_macro!($crate::primitive_reuse_peer::ReuserClass);
        $your_macro!($crate::broadcast_reuse_peer::BroadcastReuserClass);
        $your_macro!($crate::reconnect_peer::AutoReconnectClass);

        $your_macro!($crate::ws_client_peer::WsConnectClass);

        $your_macro!($crate::net_peer::UdpConnectClass);
        $your_macro!($crate::net_peer::UdpListenClass);

        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::OpenAsyncClass);
        #[cfg(all(unix, feature = "unix_stdio"))]
        $your_macro!($crate::stdio_peer::OpenFdAsyncClass);

        $your_macro!($crate::stdio_threaded_peer::ThreadedStdioClass);

        #[cfg(unix)]
        $your_macro!($crate::unix_peer::UnixConnectClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::UnixListenClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::UnixDgramClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::AbstractConnectClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::AbstractListenClass);
        #[cfg(unix)]
        $your_macro!($crate::unix_peer::AbstractDgramClass);

        $your_macro!($crate::line_peer::Message2LineClass);
        $your_macro!($crate::line_peer::Line2MessageClass);
        $your_macro!($crate::mirror_peer::MirrorClass);
        $your_macro!($crate::mirror_peer::LiteralReplyClass);
        $your_macro!($crate::trivial_peer::CloggedClass);
        $your_macro!($crate::trivial_peer::LiteralClass);
        $your_macro!($crate::trivial_peer::AssertClass);
        $your_macro!($crate::trivial_peer::Assert2Class);

        #[cfg(feature = "seqpacket")]
        $your_macro!($crate::unix_peer::SeqpacketConnectClass);
        #[cfg(feature = "seqpacket")]
        $your_macro!($crate::unix_peer::SeqpacketListenClass);

        /*
                                        $your_macro!($crate:: :: );
                                        $your_macro!($crate:: :: );
                                        $your_macro!($crate:: :: );
                                        */
    };
}
<|endoftext|>


<commit_before>

#[cfg(test)]
mod benchmarks {
    use ::eval::eval_string;
    use french_press::init_gc;
    use test::Bencher;

    #[bench]
    fn bench_example(b: &mut Bencher) {
        let code = "\
        var x = 3;
        var y = 4;
        var z = 1000;
        while (z > 0) {
            z--;
            if (x < y) {
                x = x + 3;
            } else {
                x = x - y;
            }
        }
        ";
        let mut sm = init_gc();

        b.iter(|| eval_string(code, &mut sm));
    }
}
<commit_msg>added even simpler example<commit_after>

#[cfg(test)]
mod benchmarks {
    use ::eval::eval_string;
    use french_press::init_gc;
    use test::Bencher;

    #[bench]
    fn simple_example(b: &mut Bencher) {
        let code = "\
        var x = 3;
        var y = 4;
        var z = x + y;
        ";

        let mut sm = init_gc();
        b.iter(|| eval_string(code, &mut sm));
    }

    #[bench]
    fn complex_example(b: &mut Bencher) {
        let code = "\
        var x = 3;
        var y = 4;
        var z = 1000;
        while (z > 0) {
            z--;
            if (x < y) {
                x = x + 3;
            } else {
                x = x - y;
            }
        }
        ";

        let mut sm = init_gc();
        b.iter(|| eval_string(code, &mut sm));
    }
}
<|endoftext|>


<commit_before>// This file is part of the uutils coreutils package.
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
// spell-checker:ignore tldr

use clap::App;
use std::ffi::OsString;
use std::fs::File;
use std::io::Cursor;
use std::io::{self, Read, Seek, Write};
use zip::ZipArchive;

include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));

fn main() -> io::Result<()> {
    println!("Downloading tldr archive");
    let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip")
        .call()
        .unwrap()
        .into_reader();
    let mut buffer = Vec::new();
    zip_reader.read_to_end(&mut buffer).unwrap();
    let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap();

    let utils = util_map::<Box<dyn Iterator<Item = OsString>>>();
    match std::fs::create_dir("docs/src/utils/") {
        Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
        x => x,
    }?;

    let mut summary = File::create("docs/src/SUMMARY.md")?;

    let _ = write!(
        summary,
        "# Summary\n\
        \n\
        [Introduction](index.md)\n\
        * [Installation](installation.md)\n\
        * [Contributing](contributing.md)\n\
        * [GNU test coverage](test_coverage.md)\n\
        \n\
        # Reference\n\
        * [Multi-call binary](multicall.md)\n",
    );

    let mut utils = utils.entries().collect::<Vec<_>>();
    utils.sort();
    for (&name, (_, app)) in utils {
        if name == "[" {
            continue;
        }
        let p = format!("docs/src/utils/{}.md", name);
        if let Ok(f) = File::create(&p) {
            write_markdown(f, &mut app(), name, &mut tldr_zip)?;
            println!("Wrote to '{}'", p);
        } else {
            println!("Error writing to {}", p);
        }
        writeln!(summary, "* [{0}](utils/{0}.md)", name)?;
    }
    Ok(())
}

fn write_markdown(
    mut w: impl Write,
    app: &mut App,
    name: &str,
    tldr_zip: &mut zip::ZipArchive<impl Read + Seek>,
) -> io::Result<()> {
    write!(w, "# {}\n\n", name)?;
    write_version(&mut w, app)?;
    write_usage(&mut w, app, name)?;
    write_description(&mut w, app)?;
    write_examples(&mut w, name, tldr_zip)?;
    write_options(&mut w, app)
}

fn write_version(w: &mut impl Write, app: &App) -> io::Result<()> {
    writeln!(
        w,
        "
version: {}
",
        app.render_version().split_once(' ').unwrap().1
    )
}

fn write_usage(w: &mut impl Write, app: &mut App, name: &str) -> io::Result<()> {
    writeln!(w, "\n```")?;
    let mut usage: String = app
        .render_usage()
        .lines()
        .skip(1)
        .map(|l| l.trim())
        .filter(|l| !l.is_empty())
        .collect::<Vec<_>>()
        .join("\n");
    usage = usage.replace(app.get_name(), name);
    writeln!(w, "{}", usage)?;
    writeln!(w, "```")
}

fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> {
    if let Some(about) = app.get_long_about().or_else(|| app.get_about()) {
        writeln!(w, "{}", about)
    } else {
        Ok(())
    }
}

fn write_examples(
    w: &mut impl Write,
    name: &str,
    tldr_zip: &mut zip::ZipArchive<impl Read + Seek>,
) -> io::Result<()> {
    let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) {
        f
    } else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) {
        f
    } else {
        return Ok(());
    };

    writeln!(w, "## Examples")?;
    writeln!(w)?;
    for line in content.lines().skip_while(|l| !l.starts_with('-')) {
        if let Some(l) = line.strip_prefix("- ") {
            writeln!(w, "{}", l)?;
        } else if line.starts_with('`') {
            writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?;
        } else if line.is_empty() {
            writeln!(w)?;
        } else {
            println!("Not sure what to do with this line:");
            println!("{}", line);
        }
    }
    writeln!(w)?;
    writeln!(
        w,
        "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)."
    )?;
    writeln!(w, ">")?;
    writeln!(
        w,
        "> Please note that, as uutils is a work in progress, some examples might fail."
    )
}

fn get_zip_content(archive: &mut ZipArchive<impl Read + Seek>, name: &str) -> Option<String> {
    let mut s = String::new();
    archive.by_name(name).ok()?.read_to_string(&mut s).unwrap();
    Some(s)
}

fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> {
    writeln!(w, "
Options
")?;
    write!(w, "
")?;
    for arg in app.get_arguments() {
        write!(w, "
")?;
        let mut first = true;
        for l in arg.get_long_and_visible_aliases().unwrap_or_default() {
            if !first {
                write!(w, ", ")?;
            } else {
                first = false;
            }
            write!(w, "")?;
            write!(w, "--{}", l)?;
            if let Some(names) = arg.get_value_names() {
                write!(
                    w,
                    "={}",
                    names
                        .iter()
                        .map(|x| format!("<{}>", x))
                        .collect::<Vec<_>>()
                        .join(" ")
                )?;
            }
            write!(w, "")?;
        }
        for s in arg.get_short_and_visible_aliases().unwrap_or_default() {
            if !first {
                write!(w, ", ")?;
            } else {
                first = false;
            }
            write!(w, "")?;
            write!(w, "-{}", s)?;
            if let Some(names) = arg.get_value_names() {
                write!(
                    w,
                    " {}",
                    names
                        .iter()
                        .map(|x| format!("<{}>", x))
                        .collect::<Vec<_>>()
                        .join(" ")
                )?;
            }
            write!(w, "")?;
        }
        writeln!(w, "
")?;
        writeln!(w, "
\n\n{}\n\n
", arg.get_help().unwrap_or_default())?;
    }
    writeln!(w, "
")
}
<commit_msg>user docs: move the examples after the options<commit_after>// This file is part of the uutils coreutils package.
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
// spell-checker:ignore tldr

use clap::App;
use std::ffi::OsString;
use std::fs::File;
use std::io::Cursor;
use std::io::{self, Read, Seek, Write};
use zip::ZipArchive;

include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));

fn main() -> io::Result<()> {
    println!("Downloading tldr archive");
    let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip")
        .call()
        .unwrap()
        .into_reader();
    let mut buffer = Vec::new();
    zip_reader.read_to_end(&mut buffer).unwrap();
    let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap();

    let utils = util_map::<Box<dyn Iterator<Item = OsString>>>();
    match std::fs::create_dir("docs/src/utils/") {
        Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
        x => x,
    }?;

    let mut summary = File::create("docs/src/SUMMARY.md")?;

    let _ = write!(
        summary,
        "# Summary\n\
        \n\
        [Introduction](index.md)\n\
        * [Installation](installation.md)\n\
        * [Contributing](contributing.md)\n\
        * [GNU test coverage](test_coverage.md)\n\
        \n\
        # Reference\n\
        * [Multi-call binary](multicall.md)\n",
    );

    let mut utils = utils.entries().collect::<Vec<_>>();
    utils.sort();
    for (&name, (_, app)) in utils {
        if name == "[" {
            continue;
        }
        let p = format!("docs/src/utils/{}.md", name);
        if let Ok(f) = File::create(&p) {
            write_markdown(f, &mut app(), name, &mut tldr_zip)?;
            println!("Wrote to '{}'", p);
        } else {
            println!("Error writing to {}", p);
        }
        writeln!(summary, "* [{0}](utils/{0}.md)", name)?;
    }
    Ok(())
}

fn write_markdown(
    mut w: impl Write,
    app: &mut App,
    name: &str,
    tldr_zip: &mut zip::ZipArchive<impl Read + Seek>,
) -> io::Result<()> {
    write!(w, "# {}\n\n", name)?;
    write_version(&mut w, app)?;
    write_usage(&mut w, app, name)?;
    write_description(&mut w, app)?;
    write_options(&mut w, app)?;
    write_examples(&mut w, name, tldr_zip)
}

fn write_version(w: &mut impl Write, app: &App) -> io::Result<()> {
    writeln!(
        w,
        "
version: {}
",
        app.render_version().split_once(' ').unwrap().1
    )
}

fn write_usage(w: &mut impl Write, app: &mut App, name: &str) -> io::Result<()> {
    writeln!(w, "\n```")?;
    let mut usage: String = app
        .render_usage()
        .lines()
        .skip(1)
        .map(|l| l.trim())
        .filter(|l| !l.is_empty())
        .collect::<Vec<_>>()
        .join("\n");
    usage = usage.replace(app.get_name(), name);
    writeln!(w, "{}", usage)?;
    writeln!(w, "```")
}

fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> {
    if let Some(about) = app.get_long_about().or_else(|| app.get_about()) {
        writeln!(w, "{}", about)
    } else {
        Ok(())
    }
}

fn write_examples(
    w: &mut impl Write,
    name: &str,
    tldr_zip: &mut zip::ZipArchive<impl Read + Seek>,
) -> io::Result<()> {
    let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) {
        f
    } else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) {
        f
    } else {
        return Ok(());
    };

    writeln!(w, "## Examples")?;
    writeln!(w)?;
    for line in content.lines().skip_while(|l| !l.starts_with('-')) {
        if let Some(l) = line.strip_prefix("- ") {
            writeln!(w, "{}", l)?;
        } else if line.starts_with('`') {
            writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?;
        } else if line.is_empty() {
            writeln!(w)?;
        } else {
            println!("Not sure what to do with this line:");
            println!("{}", line);
        }
    }
    writeln!(w)?;
    writeln!(
        w,
        "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)."
    )?;
    writeln!(w, ">")?;
    writeln!(
        w,
        "> Please note that, as uutils is a work in progress, some examples might fail."
    )
}

fn get_zip_content(archive: &mut ZipArchive<impl Read + Seek>, name: &str) -> Option<String> {
    let mut s = String::new();
    archive.by_name(name).ok()?.read_to_string(&mut s).unwrap();
    Some(s)
}

fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> {
    writeln!(w, "
Options
")?;
    write!(w, "
")?;
    for arg in app.get_arguments() {
        write!(w, "
")?;
        let mut first = true;
        for l in arg.get_long_and_visible_aliases().unwrap_or_default() {
            if !first {
                write!(w, ", ")?;
            } else {
                first = false;
            }
            write!(w, "")?;
            write!(w, "--{}", l)?;
            if let Some(names) = arg.get_value_names() {
                write!(
                    w,
                    "={}",
                    names
                        .iter()
                        .map(|x| format!("<{}>", x))
                        .collect::<Vec<_>>()
                        .join(" ")
                )?;
            }
            write!(w, "")?;
        }
        for s in arg.get_short_and_visible_aliases().unwrap_or_default() {
            if !first {
                write!(w, ", ")?;
            } else {
                first = false;
            }
            write!(w, "")?;
            write!(w, "-{}", s)?;
            if let Some(names) = arg.get_value_names() {
                write!(
                    w,
                    " {}",
                    names
                        .iter()
                        .map(|x| format!("<{}>", x))
                        .collect::<Vec<_>>()
                        .join(" ")
                )?;
            }
            write!(w, "")?;
        }
        writeln!(w, "
")?;
        writeln!(w, "
\n\n{}\n\n
", arg.get_help().unwrap_or_default())?;
    }
    writeln!(w, "
\n")
}
<|endoftext|>


<commit_before>// Claxon -- A FLAC decoding library in Rust
// Copyright (C) 2014-2015 Ruud van Asseldonk
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

use std::old_io::IoResult;

/// Wraps a `Reader` to facilitate reading that is not byte-aligned.
pub struct Bitstream<'r> {
    /// The source where bits are read from.
    reader: &'r mut (Reader + 'r),
    /// Data read from the reader, but not yet fully consumed.
    data: u8,
    /// The number of bits of `data` that have not been consumed.
    bits_left: u8
}

impl<'r> Bitstream<'r> {
    /// Wraps the reader with a reader that facilitates reading individual bits.
    pub fn new(reader: &'r mut Reader) -> Bitstream<'r> {
        Bitstream { reader: reader, data: 0, bits_left: 0 }
    }

    /// Generates a bitmask with 1s in the `bits` most significant bits.
    fn mask_u8(bits: u8) -> u8 {
        debug_assert!(bits <= 8);
        0xffu8 << (8 - bits) as usize
    }

    /// Skips bits such that the next read will be byte-aligned.
    pub fn align_to_byte(&mut self) {
        self.bits_left = 0;
    }

    // TODO: Remove this method.
    pub fn dump_some(&mut self, bytes: u8) -> IoResult<()> {
        println!(">=== begin bitstream dump ===>");
        println!("     data = {:x}, {} bits left", self.data, self.bits_left);
        print!("     more: [");
        for _ in 0 .. bytes - 1 {
            print!("{:x}, ", try!(self.reader.read_byte()));
        }
        println!("{:x}]", try!(self.reader.read_byte()));
        println!("<=== end bitstream dump <===");
        Ok(())
    }

    /// Reads at most eight bits.
    pub fn read_leq_u8(&mut self, bits: u8) -> IoResult<u8> {
        // Of course we can read no more than 8 bits, but we do not want the
        // performance overhead of the assertion, so only do it in debug mode.
        debug_assert!(bits <= 8);

        // If not enough bits left, we will need to read the next byte.
        let result = if self.bits_left < bits {
            // Most significant bits are shifted to the right position already.
            let msb = self.data;

            // From the next byte, we take the additional bits that we need.
            // Those start at the most significant bit, so we need to shift so
            // that it does not overlap with what we have already.
            self.data = try!(self.reader.read_byte());
            let lsb = (self.data & Bitstream::mask_u8(bits - self.bits_left))
                    >> self.bits_left as usize;

            // Shift out the bits that we have consumed.
            self.data = self.data << (bits - self.bits_left) as usize;
            self.bits_left = 8 - (bits - self.bits_left);

            msb | lsb
        } else {
            let result = self.data & Bitstream::mask_u8(bits);

            // Shift out the bits that we have consumed.
            self.data = self.data << bits as usize;
            self.bits_left = self.bits_left - bits;

            result
        };

        // If there are more than 8 bits left, we read too far.
        debug_assert!(self.bits_left < 8);

        // The least significant bits should be zero.
        debug_assert_eq!(self.data & !Bitstream::mask_u8(self.bits_left), 0u8);

        // The resulting data is padded with zeroes in the least significant
        // bits, but we want to pad in the most significant bits, so shift.
        Ok(result >> (8 - bits) as usize)
    }

    /// Reads at most 16 bits.
    pub fn read_leq_u16(&mut self, bits: u8) -> IoResult<u16> {
        // As with read_leq_u8, this only makes sense if we read <= 16 bits.
        debug_assert!(bits <= 16);

        // Note: the following is not the most efficient implementation
        // possible, but it avoids duplicating the complexity of `read_leq_u8`.

        if bits <= 8 {
            let result = try!(self.read_leq_u8(bits));
            Ok(result as u16)
        } else {
            // First read the 8 most significant bits, then read what is left.
            let msb = try!(self.read_leq_u8(8)) as u16;
            let lsb = try!(self.read_leq_u8(bits - 8)) as u16;
            Ok((msb << (bits - 8) as usize) | lsb)
        }
    }

    /// Reads at most 32 bits.
    pub fn read_leq_u32(&mut self, bits: u8) -> IoResult<u32> {
        // As with read_leq_u8, this only makes sense if we read <= 32 bits.
        debug_assert!(bits <= 32);

        // Note: the following is not the most efficient implementation
        // possible, but it avoids duplicating the complexity of `read_leq_u8`.

        if bits <= 16 {
            let result = try!(self.read_leq_u16(bits));
            Ok(result as u32)
        } else {
            // First read the 16 most significant bits, then read what is left.
            let msb = try!(self.read_leq_u16(16)) as u32;
            let lsb = try!(self.read_leq_u16(bits - 16)) as u32;
            Ok((msb << (bits - 16) as usize) | lsb)
        }
    }
}

#[test]
fn verify_read_leq_u8() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0b1010_0101, 0b1110_0001,
                                       0b1101_0010, 0b0101_0101,
                                       0b0111_0011, 0b0011_1111,
                                       0b1010_1010, 0b0000_1100));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u8(0).unwrap(), 0);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 0);
    assert_eq!(bits.read_leq_u8(2).unwrap(), 0b10);
    assert_eq!(bits.read_leq_u8(2).unwrap(), 0b01);
    assert_eq!(bits.read_leq_u8(3).unwrap(), 0b011);
    assert_eq!(bits.read_leq_u8(3).unwrap(), 0b110);
    assert_eq!(bits.read_leq_u8(4).unwrap(), 0b0001);
    assert_eq!(bits.read_leq_u8(5).unwrap(), 0b11010);
    assert_eq!(bits.read_leq_u8(6).unwrap(), 0b010010);
    assert_eq!(bits.read_leq_u8(7).unwrap(), 0b1010101);
    assert_eq!(bits.read_leq_u8(8).unwrap(), 0b11001100);
    assert_eq!(bits.read_leq_u8(6).unwrap(), 0b111111);
    assert_eq!(bits.read_leq_u8(8).unwrap(), 0b10101010);
    assert_eq!(bits.read_leq_u8(4).unwrap(), 0b0000);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u8(2).unwrap(), 0b00);
}

#[test]
fn verify_read_leq_u16() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0b1010_0101, 0b1110_0001,
                                       0b1101_0010, 0b0101_0101));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u16(0).unwrap(), 0);
    assert_eq!(bits.read_leq_u16(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u16(13).unwrap(), 0b010_0101_1110_00);
    assert_eq!(bits.read_leq_u16(9).unwrap(), 0b01_1101_001);
}

#[test]
fn verify_read_leq_u32() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0b1010_0101, 0b1110_0001,
                                       0b1101_0010, 0b0101_0101));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u32(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u32(17).unwrap(), 0b010_0101_1110_0001_11);
    assert_eq!(bits.read_leq_u32(14).unwrap(), 0b01_0010_0101_0101);
}

#[test]
fn verify_read_mixed() {
    use std::old_io::MemReader;

    // The test data for this test is actual data from the warmup samples in an
    // LPC subframe.
    let mut data = MemReader::new(vec!(0x03, 0xc7, 0xbf, 0xe5, 0x9b, 0x74,
                                       0x1e, 0x3a, 0xdd, 0x7d, 0xc5, 0x5e,
                                       0xf6, 0xbf, 0x78, 0x1b, 0xbd));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u8(6).unwrap(), 0);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u32(16).unwrap() as u16, -14401_i16 as u16);
}

#[test]
fn verify_align() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0x00, 0xff));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u8(5).unwrap(), 0);
    bits.align_to_byte();
    assert_eq!(bits.read_leq_u8(3).unwrap(), 7);
}
<commit_msg>the bitstream works fine, bits per sample is wrong<commit_after>// Claxon -- A FLAC decoding library in Rust
// Copyright (C) 2014-2015 Ruud van Asseldonk
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

use std::old_io::IoResult;

/// Wraps a `Reader` to facilitate reading that is not byte-aligned.
pub struct Bitstream<'r> {
    /// The source where bits are read from.
    reader: &'r mut (Reader + 'r),
    /// Data read from the reader, but not yet fully consumed.
    data: u8,
    /// The number of bits of `data` that have not been consumed.
    bits_left: u8
}

impl<'r> Bitstream<'r> {
    /// Wraps the reader with a reader that facilitates reading individual bits.
    pub fn new(reader: &'r mut Reader) -> Bitstream<'r> {
        Bitstream { reader: reader, data: 0, bits_left: 0 }
    }

    /// Generates a bitmask with 1s in the `bits` most significant bits.
    fn mask_u8(bits: u8) -> u8 {
        debug_assert!(bits <= 8);
        0xffu8 << (8 - bits) as usize
    }

    /// Skips bits such that the next read will be byte-aligned.
    pub fn align_to_byte(&mut self) {
        self.bits_left = 0;
    }

    // TODO: Remove this method.
    pub fn dump_some(&mut self, bytes: u8) -> IoResult<()> {
        println!(">=== begin bitstream dump ===>");
        println!("     data = {:x}, {} bits left", self.data, self.bits_left);
        print!("     more: [");
        for _ in 0 .. bytes - 1 {
            print!("{:x}, ", try!(self.reader.read_byte()));
        }
        println!("{:x}]", try!(self.reader.read_byte()));
        println!("<=== end bitstream dump <===");
        Ok(())
    }

    /// Reads at most eight bits.
    pub fn read_leq_u8(&mut self, bits: u8) -> IoResult<u8> {
        // Of course we can read no more than 8 bits, but we do not want the
        // performance overhead of the assertion, so only do it in debug mode.
        debug_assert!(bits <= 8);

        // If not enough bits left, we will need to read the next byte.
        let result = if self.bits_left < bits {
            // Most significant bits are shifted to the right position already.
            let msb = self.data;

            // From the next byte, we take the additional bits that we need.
            // Those start at the most significant bit, so we need to shift so
            // that it does not overlap with what we have already.
            self.data = try!(self.reader.read_byte());
            let lsb = (self.data & Bitstream::mask_u8(bits - self.bits_left))
                    >> self.bits_left as usize;

            // Shift out the bits that we have consumed.
            self.data = self.data << (bits - self.bits_left) as usize;
            self.bits_left = 8 - (bits - self.bits_left);

            msb | lsb
        } else {
            let result = self.data & Bitstream::mask_u8(bits);

            // Shift out the bits that we have consumed.
            self.data = self.data << bits as usize;
            self.bits_left = self.bits_left - bits;

            result
        };

        // If there are more than 8 bits left, we read too far.
        debug_assert!(self.bits_left < 8);

        // The least significant bits should be zero.
        debug_assert_eq!(self.data & !Bitstream::mask_u8(self.bits_left), 0u8);

        // The resulting data is padded with zeroes in the least significant
        // bits, but we want to pad in the most significant bits, so shift.
        Ok(result >> (8 - bits) as usize)
    }

    /// Reads at most 16 bits.
    pub fn read_leq_u16(&mut self, bits: u8) -> IoResult<u16> {
        // As with read_leq_u8, this only makes sense if we read <= 16 bits.
        debug_assert!(bits <= 16);

        // Note: the following is not the most efficient implementation
        // possible, but it avoids duplicating the complexity of `read_leq_u8`.

        if bits <= 8 {
            let result = try!(self.read_leq_u8(bits));
            Ok(result as u16)
        } else {
            // First read the 8 most significant bits, then read what is left.
            let msb = try!(self.read_leq_u8(8)) as u16;
            let lsb = try!(self.read_leq_u8(bits - 8)) as u16;
            Ok((msb << (bits - 8) as usize) | lsb)
        }
    }

    /// Reads at most 32 bits.
    pub fn read_leq_u32(&mut self, bits: u8) -> IoResult<u32> {
        // As with read_leq_u8, this only makes sense if we read <= 32 bits.
        debug_assert!(bits <= 32);

        // Note: the following is not the most efficient implementation
        // possible, but it avoids duplicating the complexity of `read_leq_u8`.

        if bits <= 16 {
            let result = try!(self.read_leq_u16(bits));
            Ok(result as u32)
        } else {
            // First read the 16 most significant bits, then read what is left.
            let msb = try!(self.read_leq_u16(16)) as u32;
            let lsb = try!(self.read_leq_u16(bits - 16)) as u32;
            Ok((msb << (bits - 16) as usize) | lsb)
        }
    }
}

#[test]
fn verify_read_leq_u8() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0b1010_0101, 0b1110_0001,
                                       0b1101_0010, 0b0101_0101,
                                       0b0111_0011, 0b0011_1111,
                                       0b1010_1010, 0b0000_1100));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u8(0).unwrap(), 0);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 0);
    assert_eq!(bits.read_leq_u8(2).unwrap(), 0b10);
    assert_eq!(bits.read_leq_u8(2).unwrap(), 0b01);
    assert_eq!(bits.read_leq_u8(3).unwrap(), 0b011);
    assert_eq!(bits.read_leq_u8(3).unwrap(), 0b110);
    assert_eq!(bits.read_leq_u8(4).unwrap(), 0b0001);
    assert_eq!(bits.read_leq_u8(5).unwrap(), 0b11010);
    assert_eq!(bits.read_leq_u8(6).unwrap(), 0b010010);
    assert_eq!(bits.read_leq_u8(7).unwrap(), 0b1010101);
    assert_eq!(bits.read_leq_u8(8).unwrap(), 0b11001100);
    assert_eq!(bits.read_leq_u8(6).unwrap(), 0b111111);
    assert_eq!(bits.read_leq_u8(8).unwrap(), 0b10101010);
    assert_eq!(bits.read_leq_u8(4).unwrap(), 0b0000);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u8(2).unwrap(), 0b00);
}

#[test]
fn verify_read_leq_u16() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0b1010_0101, 0b1110_0001,
                                       0b1101_0010, 0b0101_0101));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u16(0).unwrap(), 0);
    assert_eq!(bits.read_leq_u16(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u16(13).unwrap(), 0b010_0101_1110_00);
    assert_eq!(bits.read_leq_u16(9).unwrap(), 0b01_1101_001);
}

#[test]
fn verify_read_leq_u32() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0b1010_0101, 0b1110_0001,
                                       0b1101_0010, 0b0101_0101));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u32(1).unwrap(), 1);
    assert_eq!(bits.read_leq_u32(17).unwrap(), 0b010_0101_1110_0001_11);
    assert_eq!(bits.read_leq_u32(14).unwrap(), 0b01_0010_0101_0101);
}

#[test]
fn verify_read_mixed() {
    use std::old_io::MemReader;

    // These test data are warm-up samples from an actual stream.
    let mut data = MemReader::new(vec!(0x03, 0xc7, 0xbf, 0xe5, 0x9b, 0x74,
                                       0x1e, 0x3a, 0xdd, 0x7d, 0xc5, 0x5e,
                                       0xf6, 0xbf, 0x78, 0x1b, 0xbd));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u8(6).unwrap(), 0);
    assert_eq!(bits.read_leq_u8(1).unwrap(), 1);
    let minus = 1u32 << 16;
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-14401_i16 as u16 as u32));
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-13514_i16 as u16 as u32));
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-12168_i16 as u16 as u32));
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-10517_i16 as u16 as u32));
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-09131_i16 as u16 as u32));
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-08489_i16 as u16 as u32));
    assert_eq!(bits.read_leq_u32(17).unwrap(), minus | (-08698_i16 as u16 as u32));
}

#[test]
fn verify_align() {
    use std::old_io::MemReader;

    let mut data = MemReader::new(vec!(0x00, 0xff));
    let mut bits = Bitstream::new(&mut data);

    assert_eq!(bits.read_leq_u8(5).unwrap(), 0);
    bits.align_to_byte();
    assert_eq!(bits.read_leq_u8(3).unwrap(), 7);
}
<|endoftext|>


<commit_before>use std::fmt::{self, Display, Formatter};
use std::path::{Path, PathBuf};
use std::collections::VecDeque;
use std::fs::File;
use std::io::{Read, Write};

use super::summary::{parse_summary, Link, SectionNumber, Summary, SummaryItem};
use config::BuildConfig;
use errors::*;

/// Load a book into memory from its `src/` directory.
pub fn load_book<P: AsRef<Path>>(src_dir: P, cfg: &BuildConfig) -> Result<Book> {
    let src_dir = src_dir.as_ref();
    let summary_md = src_dir.join("SUMMARY.md");

    let mut summary_content = String::new();
    File::open(summary_md)
        .chain_err(|| "Couldn't open SUMMARY.md")?
        .read_to_string(&mut summary_content)?;

    let summary = parse_summary(&summary_content).chain_err(|| "Summary parsing failed")?;

    if cfg.create_missing {
        create_missing(&src_dir, &summary).chain_err(|| "Unable to create missing chapters")?;
    }

    load_book_from_disk(&summary, src_dir)
}

fn create_missing(src_dir: &Path, summary: &Summary) -> Result<()> {
    let mut items: Vec<_> = summary
        .prefix_chapters
        .iter()
        .chain(summary.numbered_chapters.iter())
        .chain(summary.suffix_chapters.iter())
        .collect();

    while !items.is_empty() {
        let next = items.pop().expect("already checked");

        if let SummaryItem::Link(ref link) = *next {
            let filename = src_dir.join(&link.location);
            if !filename.exists() {
                debug!("[*] Creating missing file {}", filename.display());

                let mut f = File::create(&filename)?;
                writeln!(f, "# {}", link.name)?;
            }

            items.extend(&link.nested_items);
        }
    }

    Ok(())
}

/// A dumb tree structure representing a book.
///
/// For the moment a book is just a collection of `BookItems`.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Book {
    /// The sections in this book.
    pub sections: Vec<BookItem>,
}

impl Book {
    /// Create an empty book.
    pub fn new() -> Self {
        Default::default()
    }

    /// Get a depth-first iterator over the items in the book.
    pub fn iter(&self) -> BookItems {
        BookItems {
            items: self.sections.iter().collect(),
        }
    }
}

/// Enum representing any type of item which can be added to a book.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum BookItem {
    /// A nested chapter.
    Chapter(Chapter),
    /// A section separator.
    Separator,
}

/// The representation of a "chapter", usually mapping to a single file on
/// disk however it may contain multiple sub-chapters.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Chapter {
    /// The chapter's name.
    pub name: String,
    /// The chapter's contents.
    pub content: String,
    /// The chapter's section number, if it has one.
    pub number: Option<SectionNumber>,
    /// Nested items.
    pub sub_items: Vec<BookItem>,
    /// The chapter's location, relative to the `SUMMARY.md` file.
    pub path: PathBuf,
}

impl Chapter {
    /// Create a new chapter with the provided content.
    pub fn new<P: Into<PathBuf>>(name: &str, content: String, path: P) -> Chapter {
        Chapter {
            name: name.to_string(),
            content: content,
            path: path.into(),
            ..Default::default()
        }
    }
}

/// Use the provided `Summary` to load a `Book` from disk.
///
/// You need to pass in the book's source directory because all the links in
/// `SUMMARY.md` give the chapter locations relative to it.
fn load_book_from_disk<P: AsRef<Path>>(summary: &Summary, src_dir: P) -> Result<Book> {
    debug!("[*] Loading the book from disk");
    let src_dir = src_dir.as_ref();

    let prefix = summary.prefix_chapters.iter();
    let numbered = summary.numbered_chapters.iter();
    let suffix = summary.suffix_chapters.iter();

    let summary_items = prefix.chain(numbered).chain(suffix);

    let mut chapters = Vec::new();

    for summary_item in summary_items {
        let chapter = load_summary_item(summary_item, src_dir)?;
        chapters.push(chapter);
    }

    Ok(Book { sections: chapters })
}

fn load_summary_item<P: AsRef<Path>>(item: &SummaryItem, src_dir: P) -> Result<BookItem> {
    match *item {
        SummaryItem::Separator => Ok(BookItem::Separator),
        SummaryItem::Link(ref link) => load_chapter(link, src_dir).map(|c| BookItem::Chapter(c)),
    }
}

fn load_chapter<P: AsRef<Path>>(link: &Link, src_dir: P) -> Result<Chapter> {
    debug!("[*] Loading {} ({})", link.name, link.location.display());
    let src_dir = src_dir.as_ref();

    let location = if link.location.is_absolute() {
        link.location.clone()
    } else {
        src_dir.join(&link.location)
    };

    let mut f = File::open(&location)
        .chain_err(|| format!("Chapter file not found, {}", link.location.display()))?;

    let mut content = String::new();
    f.read_to_string(&mut content)?;

    let stripped = location
        .strip_prefix(&src_dir)
        .expect("Chapters are always inside a book");

    let mut ch = Chapter::new(&link.name, content, stripped);
    ch.number = link.number.clone();

    let sub_items = link.nested_items
        .iter()
        .map(|i| load_summary_item(i, src_dir))
        .collect::<Result<Vec<_>>>()?;

    ch.sub_items = sub_items;

    Ok(ch)
}

/// A depth-first iterator over the items in a book.
///
/// # Note
///
/// This struct shouldn't be created directly, instead prefer the
/// [`Book::iter()`] method.
///
/// [`Book::iter()`]: struct.Book.html#method.iter
pub struct BookItems<'a> {
    items: VecDeque<&'a BookItem>,
}

impl<'a> Iterator for BookItems<'a> {
    type Item = &'a BookItem;

    fn next(&mut self) -> Option<Self::Item> {
        let item = self.items.pop_front();

        if let Some(&BookItem::Chapter(ref ch)) = item {
            // if we wanted a breadth-first iterator we'd `extend()` here
            for sub_item in ch.sub_items.iter().rev() {
                self.items.push_front(sub_item);
            }
        }

        item
    }
}

impl Display for Chapter {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        if let Some(ref section_number) = self.number {
            write!(f, "{} ", section_number)?;
        }

        write!(f, "{}", self.name)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempdir::TempDir;
    use std::io::Write;

    const DUMMY_SRC: &'static str = "
# Dummy Chapter

this is some dummy text.

And here is some \
                                     more text.
";

    /// Create a dummy `Link` in a temporary directory.
    fn dummy_link() -> (Link, TempDir) {
        let temp = TempDir::new("book").unwrap();

        let chapter_path = temp.path().join("chapter_1.md");
        File::create(&chapter_path)
            .unwrap()
            .write(DUMMY_SRC.as_bytes())
            .unwrap();

        let link = Link::new("Chapter 1", chapter_path);

        (link, temp)
    }

    /// Create a nested `Link` written to a temporary directory.
    fn nested_links() -> (Link, TempDir) {
        let (mut root, temp_dir) = dummy_link();

        let second_path = temp_dir.path().join("second.md");

        File::create(&second_path)
            .unwrap()
            .write_all("Hello World!".as_bytes())
            .unwrap();

        let mut second = Link::new("Nested Chapter 1", &second_path);
        second.number = Some(SectionNumber(vec![1, 2]));

        root.nested_items.push(second.clone().into());
        root.nested_items.push(SummaryItem::Separator);
        root.nested_items.push(second.clone().into());

        (root, temp_dir)
    }

    #[test]
    fn load_a_single_chapter_from_disk() {
        let (link, temp_dir) = dummy_link();
        let should_be = Chapter::new("Chapter 1", DUMMY_SRC.to_string(), "chapter_1.md");

        let got = load_chapter(&link, temp_dir.path()).unwrap();
        assert_eq!(got, should_be);
    }

    #[test]
    fn cant_load_a_nonexistent_chapter() {
        let link = Link::new("Chapter 1", "/foo/bar/baz.md");

        let got = load_chapter(&link, "");
        assert!(got.is_err());
    }

    #[test]
    fn load_recursive_link_with_separators() {
        let (root, temp) = nested_links();

        let nested = Chapter {
            name: String::from("Nested Chapter 1"),
            content: String::from("Hello World!"),
            number: Some(SectionNumber(vec![1, 2])),
            path: PathBuf::from("second.md"),
            sub_items: Vec::new(),
        };
        let should_be = BookItem::Chapter(Chapter {
            name: String::from("Chapter 1"),
            content: String::from(DUMMY_SRC),
            number: None,
            path: PathBuf::from("chapter_1.md"),
            sub_items: vec![
                BookItem::Chapter(nested.clone()),
                BookItem::Separator,
                BookItem::Chapter(nested.clone()),
            ],
        });

        let got = load_summary_item(&SummaryItem::Link(root), temp.path()).unwrap();
        assert_eq!(got, should_be);
    }

    #[test]
    fn load_a_book_with_a_single_chapter() {
        let (link, temp) = dummy_link();
        let summary = Summary {
            numbered_chapters: vec![SummaryItem::Link(link)],
            ..Default::default()
        };
        let should_be = Book {
            sections: vec![
                BookItem::Chapter(Chapter {
                    name: String::from("Chapter 1"),
                    content: String::from(DUMMY_SRC),
                    path: PathBuf::from("chapter_1.md"),
                    ..Default::default()
                }),
            ],
        };

        let got = load_book_from_disk(&summary, temp.path()).unwrap();

        assert_eq!(got, should_be);
    }

    #[test]
    fn book_iter_iterates_over_sequential_items() {
        let book = Book {
            sections: vec![
                BookItem::Chapter(Chapter {
                    name: String::from("Chapter 1"),
                    content: String::from(DUMMY_SRC),
                    ..Default::default()
                }),
                BookItem::Separator,
            ],
        };

        let should_be: Vec<_> = book.sections.iter().collect();

        let got: Vec<_> = book.iter().collect();

        assert_eq!(got, should_be);
    }

    #[test]
    fn iterate_over_nested_book_items() {
        let book = Book {
            sections: vec![
                BookItem::Chapter(Chapter {
                    name: String::from("Chapter 1"),
                    content: String::from(DUMMY_SRC),
                    number: None,
                    path: PathBuf::from("Chapter_1/index.md"),
                    sub_items: vec![
                        BookItem::Chapter(Chapter::new(
                            "Hello World",
                            String::new(),
                            "Chapter_1/hello.md",
                        )),
                        BookItem::Separator,
                        BookItem::Chapter(Chapter::new(
                            "Goodbye World",
                            String::new(),
                            "Chapter_1/goodbye.md",
                        )),
                    ],
                }),
                BookItem::Separator,
            ],
        };

        let got: Vec<_> = book.iter().collect();

        assert_eq!(got.len(), 5);

        // checking the chapter names are in the order should be sufficient here...
        let chapter_names: Vec<String> = got.into_iter()
            .filter_map(|i| match *i {
                BookItem::Chapter(ref ch) => Some(ch.name.clone()),
                _ => None,
            })
            .collect();
        let should_be: Vec<_> = vec![
            String::from("Chapter 1"),
            String::from("Hello World"),
            String::from("Goodbye World"),
        ];

        assert_eq!(chapter_names, should_be);
    }
}
<commit_msg>Made sure create_missing also creates the parent directory<commit_after>use std::fmt::{self, Display, Formatter};
use std::path::{Path, PathBuf};
use std::collections::VecDeque;
use std::fs::{self, File};
use std::io::{Read, Write};

use super::summary::{parse_summary, Link, SectionNumber, Summary, SummaryItem};
use config::BuildConfig;
use errors::*;

/// Load a book into memory from its `src/` directory.
pub fn load_book<P: AsRef<Path>>(src_dir: P, cfg: &BuildConfig) -> Result<Book> {
    let src_dir = src_dir.as_ref();
    let summary_md = src_dir.join("SUMMARY.md");

    let mut summary_content = String::new();
    File::open(summary_md)
        .chain_err(|| "Couldn't open SUMMARY.md")?
        .read_to_string(&mut summary_content)?;

    let summary = parse_summary(&summary_content).chain_err(|| "Summary parsing failed")?;

    if cfg.create_missing {
        create_missing(&src_dir, &summary).chain_err(|| "Unable to create missing chapters")?;
    }

    load_book_from_disk(&summary, src_dir)
}

fn create_missing(src_dir: &Path, summary: &Summary) -> Result<()> {
    let mut items: Vec<_> = summary
        .prefix_chapters
        .iter()
        .chain(summary.numbered_chapters.iter())
        .chain(summary.suffix_chapters.iter())
        .collect();

    while !items.is_empty() {
        let next = items.pop().expect("already checked");

        if let SummaryItem::Link(ref link) = *next {
            let filename = src_dir.join(&link.location);
            if !filename.exists() {
                if let Some(parent) = filename.parent() {
                    if !parent.exists() {
                        fs::create_dir_all(parent)?;
                    }
                }
                debug!("[*] Creating missing file {}", filename.display());

                let mut f = File::create(&filename)?;
                writeln!(f, "# {}", link.name)?;
            }

            items.extend(&link.nested_items);
        }
    }

    Ok(())
}

/// A dumb tree structure representing a book.
///
/// For the moment a book is just a collection of `BookItems`.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Book {
    /// The sections in this book.
    pub sections: Vec<BookItem>,
}

impl Book {
    /// Create an empty book.
    pub fn new() -> Self {
        Default::default()
    }

    /// Get a depth-first iterator over the items in the book.
    pub fn iter(&self) -> BookItems {
        BookItems {
            items: self.sections.iter().collect(),
        }
    }
}

/// Enum representing any type of item which can be added to a book.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum BookItem {
    /// A nested chapter.
    Chapter(Chapter),
    /// A section separator.
    Separator,
}

/// The representation of a "chapter", usually mapping to a single file on
/// disk however it may contain multiple sub-chapters.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Chapter {
    /// The chapter's name.
    pub name: String,
    /// The chapter's contents.
    pub content: String,
    /// The chapter's section number, if it has one.
    pub number: Option<SectionNumber>,
    /// Nested items.
    pub sub_items: Vec<BookItem>,
    /// The chapter's location, relative to the `SUMMARY.md` file.
    pub path: PathBuf,
}

impl Chapter {
    /// Create a new chapter with the provided content.
    pub fn new<P: Into<PathBuf>>(name: &str, content: String, path: P) -> Chapter {
        Chapter {
            name: name.to_string(),
            content: content,
            path: path.into(),
            ..Default::default()
        }
    }
}

/// Use the provided `Summary` to load a `Book` from disk.
///
/// You need to pass in the book's source directory because all the links in
/// `SUMMARY.md` give the chapter locations relative to it.
fn load_book_from_disk<P: AsRef<Path>>(summary: &Summary, src_dir: P) -> Result<Book> {
    debug!("[*] Loading the book from disk");
    let src_dir = src_dir.as_ref();

    let prefix = summary.prefix_chapters.iter();
    let numbered = summary.numbered_chapters.iter();
    let suffix = summary.suffix_chapters.iter();

    let summary_items = prefix.chain(numbered).chain(suffix);

    let mut chapters = Vec::new();

    for summary_item in summary_items {
        let chapter = load_summary_item(summary_item, src_dir)?;
        chapters.push(chapter);
    }

    Ok(Book { sections: chapters })
}

fn load_summary_item<P: AsRef<Path>>(item: &SummaryItem, src_dir: P) -> Result<BookItem> {
    match *item {
        SummaryItem::Separator => Ok(BookItem::Separator),
        SummaryItem::Link(ref link) => load_chapter(link, src_dir).map(|c| BookItem::Chapter(c)),
    }
}

fn load_chapter<P: AsRef<Path>>(link: &Link, src_dir: P) -> Result<Chapter> {
    debug!("[*] Loading {} ({})", link.name, link.location.display());
    let src_dir = src_dir.as_ref();

    let location = if link.location.is_absolute() {
        link.location.clone()
    } else {
        src_dir.join(&link.location)
    };

    let mut f = File::open(&location)
        .chain_err(|| format!("Chapter file not found, {}", link.location.display()))?;

    let mut content = String::new();
    f.read_to_string(&mut content)?;

    let stripped = location
        .strip_prefix(&src_dir)
        .expect("Chapters are always inside a book");

    let mut ch = Chapter::new(&link.name, content, stripped);
    ch.number = link.number.clone();

    let sub_items = link.nested_items
        .iter()
        .map(|i| load_summary_item(i, src_dir))
        .collect::<Result<Vec<_>>>()?;

    ch.sub_items = sub_items;

    Ok(ch)
}

/// A depth-first iterator over the items in a book.
///
/// # Note
///
/// This struct shouldn't be created directly, instead prefer the
/// [`Book::iter()`] method.
///
/// [`Book::iter()`]: struct.Book.html#method.iter
pub struct BookItems<'a> {
    items: VecDeque<&'a BookItem>,
}

impl<'a> Iterator for BookItems<'a> {
    type Item = &'a BookItem;

    fn next(&mut self) -> Option<Self::Item> {
        let item = self.items.pop_front();

        if let Some(&BookItem::Chapter(ref ch)) = item {
            // if we wanted a breadth-first iterator we'd `extend()` here
            for sub_item in ch.sub_items.iter().rev() {
                self.items.push_front(sub_item);
            }
        }

        item
    }
}

impl Display for Chapter {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        if let Some(ref section_number) = self.number {
            write!(f, "{} ", section_number)?;
        }

        write!(f, "{}", self.name)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempdir::TempDir;
    use std::io::Write;

    const DUMMY_SRC: &'static str = "
# Dummy Chapter

this is some dummy text.

And here is some \
                                     more text.
";

    /// Create a dummy `Link` in a temporary directory.
    fn dummy_link() -> (Link, TempDir) {
        let temp = TempDir::new("book").unwrap();

        let chapter_path = temp.path().join("chapter_1.md");
        File::create(&chapter_path)
            .unwrap()
            .write(DUMMY_SRC.as_bytes())
            .unwrap();

        let link = Link::new("Chapter 1", chapter_path);

        (link, temp)
    }

    /// Create a nested `Link` written to a temporary directory.
    fn nested_links() -> (Link, TempDir) {
        let (mut root, temp_dir) = dummy_link();

        let second_path = temp_dir.path().join("second.md");

        File::create(&second_path)
            .unwrap()
            .write_all("Hello World!".as_bytes())
            .unwrap();

        let mut second = Link::new("Nested Chapter 1", &second_path);
        second.number = Some(SectionNumber(vec![1, 2]));

        root.nested_items.push(second.clone().into());
        root.nested_items.push(SummaryItem::Separator);
        root.nested_items.push(second.clone().into());

        (root, temp_dir)
    }

    #[test]
    fn load_a_single_chapter_from_disk() {
        let (link, temp_dir) = dummy_link();
        let should_be = Chapter::new("Chapter 1", DUMMY_SRC.to_string(), "chapter_1.md");

        let got = load_chapter(&link, temp_dir.path()).unwrap();
        assert_eq!(got, should_be);
    }

    #[test]
    fn cant_load_a_nonexistent_chapter() {
        let link = Link::new("Chapter 1", "/foo/bar/baz.md");

        let got = load_chapter(&link, "");
        assert!(got.is_err());
    }

    #[test]
    fn load_recursive_link_with_separators() {
        let (root, temp) = nested_links();

        let nested = Chapter {
            name: String::from("Nested Chapter 1"),
            content: String::from("Hello World!"),
            number: Some(SectionNumber(vec![1, 2])),
            path: PathBuf::from("second.md"),
            sub_items: Vec::new(),
        };
        let should_be = BookItem::Chapter(Chapter {
            name: String::from("Chapter 1"),
            content: String::from(DUMMY_SRC),
            number: None,
            path: PathBuf::from("chapter_1.md"),
            sub_items: vec![
                BookItem::Chapter(nested.clone()),
                BookItem::Separator,
                BookItem::Chapter(nested.clone()),
            ],
        });

        let got = load_summary_item(&SummaryItem::Link(root), temp.path()).unwrap();
        assert_eq!(got, should_be);
    }

    #[test]
    fn load_a_book_with_a_single_chapter() {
        let (link, temp) = dummy_link();
        let summary = Summary {
            numbered_chapters: vec![SummaryItem::Link(link)],
            ..Default::default()
        };
        let should_be = Book {
            sections: vec![
                BookItem::Chapter(Chapter {
                    name: String::from("Chapter 1"),
                    content: String::from(DUMMY_SRC),
                    path: PathBuf::from("chapter_1.md"),
                    ..Default::default()
                }),
            ],
        };

        let got = load_book_from_disk(&summary, temp.path()).unwrap();

        assert_eq!(got, should_be);
    }

    #[test]
    fn book_iter_iterates_over_sequential_items() {
        let book = Book {
            sections: vec![
                BookItem::Chapter(Chapter {
                    name: String::from("Chapter 1"),
                    content: String::from(DUMMY_SRC),
                    ..Default::default()
                }),
                BookItem::Separator,
            ],
        };

        let should_be: Vec<_> = book.sections.iter().collect();

        let got: Vec<_> = book.iter().collect();

        assert_eq!(got, should_be);
    }

    #[test]
    fn iterate_over_nested_book_items() {
        let book = Book {
            sections: vec![
                BookItem::Chapter(Chapter {
                    name: String::from("Chapter 1"),
                    content: String::from(DUMMY_SRC),
                    number: None,
                    path: PathBuf::from("Chapter_1/index.md"),
                    sub_items: vec![
                        BookItem::Chapter(Chapter::new(
                            "Hello World",
                            String::new(),
                            "Chapter_1/hello.md",
                        )),
                        BookItem::Separator,
                        BookItem::Chapter(Chapter::new(
                            "Goodbye World",
                            String::new(),
                            "Chapter_1/goodbye.md",
                        )),
                    ],
                }),
                BookItem::Separator,
            ],
        };

        let got: Vec<_> = book.iter().collect();

        assert_eq!(got.len(), 5);

        // checking the chapter names are in the order should be sufficient here...
        let chapter_names: Vec<String> = got.into_iter()
            .filter_map(|i| match *i {
                BookItem::Chapter(ref ch) => Some(ch.name.clone()),
                _ => None,
            })
            .collect();
        let should_be: Vec<_> = vec![
            String::from("Chapter 1"),
            String::from("Hello World"),
            String::from("Goodbye World"),
        ];

        assert_eq!(chapter_names, should_be);
    }
}
<|endoftext|>


<commit_before>
/// Get the port from a string (ip)
pub fn parse_port(string: &str) -> &str {
    let mut a = 0;
    let mut b = 0;

    for (n, c) in string.chars().enumerate() {
        match c {
            '0' | '1' |
            '2' | '3' |
            '4' | '5' |
            '6' | '7' |
            '8' | '9' => b += 1,
            ':' => a = n + 1,
            _ => break,
        }
    }

    &string[a..b + 1]
}

/// Get the host from a string (ip)
pub fn parse_host(string: &str) -> &str {
    // TODO: Username/Password syntax
    let pos = match string.find(':') {
        Some(pos) => pos + 1,
        None => match string.find('/') {
            Some(pos) => pos + 1,
            None => string.len(),
        },
    };

    &string[..pos]
}
<commit_msg>Minor cleanup in parse_port()<commit_after>
/// Get the port from a string (ip)
pub fn parse_port(string: &str) -> &str {
    let a = match string.find(':') {
        Some(pos) => pos + 1,
        None => 0,
    };

    let mut b = 0;
    for c in string.chars().skip(a) {
        match c {
            '0' | '1' |
            '2' | '3' |
            '4' | '5' |
            '6' | '7' |
            '8' | '9' => b += 1,
            _ => break,
        }
    }

    &string[a..b + 1]
}

/// Get the host from a string (ip)
pub fn parse_host(string: &str) -> &str {
    // TODO: Username/Password syntax
    let pos = match string.find(':') {
        Some(pos) => pos + 1,
        None => match string.find('/') {
            Some(pos) => pos + 1,
            None => string.len(),
        },
    };

    &string[..pos]
}
<|endoftext|>


<commit_before>use Result;
use core::Segment;
use core::SegmentId;
use core::SegmentComponent;
use schema::Term;
use common::HasLen;
use core::SegmentMeta;
use fastfield::{self, FastFieldNotAvailableError};
use fastfield::DeleteBitSet;
use store::StoreReader;
use schema::Document;
use directory::ReadOnlySource;
use DocId;
use std::str;
use termdict::TermDictionary;
use std::cmp;
use postings::TermInfo;
use termdict::TermDictionaryImpl;
use std::sync::Arc;
use std::fmt;
use schema::Field;
use postings::SegmentPostingsOption;
use postings::{SegmentPostings, BlockSegmentPostings};
use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader};
use schema::Schema;
use postings::FreqHandler;

/// Entry point to access all of the datastructures of the `Segment`
///
/// - term dictionary
/// - postings
/// - store
/// - fast field readers
/// - field norm reader
///
/// The segment reader has a very low memory footprint,
/// as close to all of the memory data is mmapped.
///
#[derive(Clone)]
pub struct SegmentReader {
    segment_id: SegmentId,
    segment_meta: SegmentMeta,
    terms: Arc<TermDictionaryImpl>,
    postings_data: ReadOnlySource,
    store_reader: StoreReader,
    fast_fields_reader: Arc<FastFieldsReader>,
    fieldnorms_reader: Arc<FastFieldsReader>,
    delete_bitset: DeleteBitSet,
    positions_data: ReadOnlySource,
    schema: Schema,
}

impl SegmentReader {
    /// Returns the highest document id ever attributed in
    /// this segment + 1.
    /// Today, `tantivy` does not handle deletes, so it happens
    /// to also be the number of documents in the index.
    pub fn max_doc(&self) -> DocId {
        self.segment_meta.max_doc()
    }

    /// Returns the number of documents.
    /// Deleted documents are not counted.
    ///
    /// Today, `tantivy` does not handle deletes so max doc and
    /// num_docs are the same.
    pub fn num_docs(&self) -> DocId {
        self.segment_meta.num_docs()
    }

    pub fn schema(&self) -> &Schema {
        &self.schema
    }

    /// Return the number of documents that have been
    /// deleted in the segment.
    pub fn num_deleted_docs(&self) -> DocId {
        self.delete_bitset.len() as DocId
    }

    #[doc(hidden)]
    pub fn fast_fields_reader(&self) -> &FastFieldsReader {
        &*self.fast_fields_reader
    }

    /// Accessor to a segment's fast field reader given a field.
    ///
    /// Returns the u64 fast value reader if the field
    /// is a u64 field indexed as "fast".
    ///
    /// Return a FastFieldNotAvailableError if the field is not
    /// declared as a fast field in the schema.
    ///
    /// # Panics
    /// May panic if the index is corrupted.
    pub fn get_fast_field_reader<TFastFieldReader: FastFieldReader>
        (&self,
         field: Field)
         -> fastfield::Result<TFastFieldReader> {
        let field_entry = self.schema.get_field_entry(field);
        if !TFastFieldReader::is_enabled(field_entry.field_type()) {
            Err(FastFieldNotAvailableError::new(field_entry))
        } else {
            Ok(self.fast_fields_reader
                   .open_reader(field)
                   .expect("Fast field file corrupted."))
        }
    }

    /// Accessor to the segment's `Field norms`'s reader.
    ///
    /// Field norms are the length (in tokens) of the fields.
    /// It is used in the computation of the [TfIdf]
    /// (https://fulmicoton.gitbooks.io/tantivy-doc/content/tfidf.html).
    ///
    /// They are simply stored as a fast field, serialized in
    /// the `.fieldnorm` file of the segment.
    pub fn get_fieldnorms_reader(&self, field: Field) -> Option<U64FastFieldReader> {
        self.fieldnorms_reader.open_reader(field)
    }

    /// Returns the number of documents containing the term.
    pub fn doc_freq(&self, term: &Term) -> u32 {
        match self.get_term_info(term) {
            Some(term_info) => term_info.doc_freq,
            None => 0,
        }
    }

    /// Accessor to the segment's `StoreReader`.
    pub fn get_store_reader(&self) -> &StoreReader {
        &self.store_reader
    }

    /// Open a new segment for reading.
    pub fn open(segment: Segment) -> Result<SegmentReader> {

        let source = segment.open_read(SegmentComponent::TERMS)?;
        let terms = TermDictionaryImpl::from_source(source)?;

        let store_source = segment.open_read(SegmentComponent::STORE)?;
        let store_reader = StoreReader::from_source(store_source);

        let postings_shared_mmap = segment.open_read(SegmentComponent::POSTINGS)?;

        let fast_field_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
        let fast_fields_reader = FastFieldsReader::from_source(fast_field_data)?;

        let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
        let fieldnorms_reader = FastFieldsReader::from_source(fieldnorms_data)?;

        let positions_data = segment
            .open_read(SegmentComponent::POSITIONS)
            .unwrap_or_else(|_| ReadOnlySource::empty());

        let delete_bitset = if segment.meta().has_deletes() {
            let delete_data = segment.open_read(SegmentComponent::DELETE)?;
            DeleteBitSet::open(delete_data)
        } else {
            DeleteBitSet::empty()
        };

        let schema = segment.schema();
        Ok(SegmentReader {
               segment_meta: segment.meta().clone(),
               postings_data: postings_shared_mmap,
               terms: Arc::new(terms),
               segment_id: segment.id(),
               store_reader: store_reader,
               fast_fields_reader: Arc::new(fast_fields_reader),
               fieldnorms_reader: Arc::new(fieldnorms_reader),
               delete_bitset: delete_bitset,
               positions_data: positions_data,
               schema: schema,
           })
    }

    /// Return the term dictionary datastructure.
    pub fn terms(&self) -> &TermDictionaryImpl {
        &self.terms
    }

    /// Returns the document (or to be accurate, its stored field)
    /// bearing the given doc id.
    /// This method is slow and should seldom be called from
    /// within a collector.
    pub fn doc(&self, doc_id: DocId) -> Result<Document> {
        self.store_reader.get(doc_id)
    }

    /// Returns the segment postings associated with the term, and with the given option,
    /// or `None` if the term has never been encounterred and indexed.
    ///
    /// If the field was not indexed with the indexing options that cover
    /// the requested options, the returned `SegmentPostings` the method does not fail
    /// and returns a `SegmentPostings` with as much information as possible.
    ///
    /// For instance, requesting `SegmentPostingsOption::FreqAndPositions` for a
    /// `TextIndexingOptions` that does not index position will return a `SegmentPostings`
    /// with `DocId`s and frequencies.
    pub fn read_postings(&self,
                         term: &Term,
                         option: SegmentPostingsOption)
                         -> Option<SegmentPostings> {
        let field = term.field();
        let field_entry = self.schema.get_field_entry(field);
        let term_info = get!(self.get_term_info(term));
        let maximum_option = get!(field_entry.field_type().get_segment_postings_option());
        let best_effort_option = cmp::min(maximum_option, option);
        Some(self.read_postings_from_terminfo(&term_info, best_effort_option))
    }

    /// Returns a posting object given a `term_info`.
    /// This method is for an advanced usage only.
    ///
    /// Most user should prefer using `read_postings` instead.
    pub fn read_postings_from_terminfo(&self,
                                       term_info: &TermInfo,
                                       option: SegmentPostingsOption)
                                       -> SegmentPostings {
        let block_postings = self.read_block_postings_from_terminfo(term_info, option);
        let delete_bitset = self.delete_bitset.clone();
        SegmentPostings::from_block_postings(block_postings, delete_bitset)
    }

    /// Returns a block postings given a `term_info`.
    /// This method is for an advanced usage only.
    ///
    /// Most user should prefer using `read_postings` instead.
    pub fn read_block_postings_from_terminfo(&self,
                                             term_info: &TermInfo,
                                             option: SegmentPostingsOption)
                                             -> BlockSegmentPostings {
        let offset = term_info.postings_offset as usize;
        let postings_data = &self.postings_data[offset..];
        let freq_handler = match option {
            SegmentPostingsOption::NoFreq => FreqHandler::new_without_freq(),
            SegmentPostingsOption::Freq => FreqHandler::new_with_freq(),
            SegmentPostingsOption::FreqAndPositions => {
                let offset = term_info.positions_offset as usize;
                let offseted_position_data = &self.positions_data[offset..];
                FreqHandler::new_with_freq_and_position(offseted_position_data)
            }
        };
        BlockSegmentPostings::from_data(term_info.doc_freq as usize, postings_data, freq_handler)
    }

    /// Resets the block segment to another position of the postings
    /// file.
    ///
    /// This is useful for enumerating through a list of terms,
    /// and consuming the associated posting lists while avoiding
    /// reallocating a `BlockSegmentPostings`.
    ///
    /// # Warning
    ///
    /// This does not reset the positions list.
    pub fn reset_block_postings_from_terminfo<'a>(&'a self,
                                                  term_info: &TermInfo,
                                                  block_postings: &mut BlockSegmentPostings<'a>) {
        let offset = term_info.postings_offset as usize;
        let postings_data: &'a [u8] = &self.postings_data[offset..];
        block_postings.reset(term_info.doc_freq as usize, postings_data);
    }

    /// Returns the term info associated with the term.
    pub fn get_term_info(&self, term: &Term) -> Option<TermInfo> {
        self.terms.get(term.as_slice())
    }

    /// Returns the segment id
    pub fn segment_id(&self) -> SegmentId {
        self.segment_id
    }

    /// Returns the bitset representing
    /// the documents that have been deleted.
    pub fn delete_bitset(&self) -> &DeleteBitSet {
        &self.delete_bitset
    }

    /// Returns true iff the `doc` is marked
    /// as deleted.
    pub fn is_deleted(&self, doc: DocId) -> bool {
        self.delete_bitset.is_deleted(doc)
    }
}

impl fmt::Debug for SegmentReader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "SegmentReader({:?})", self.segment_id)
    }
}
<commit_msg>Avoid exposign schema from segment_reader<commit_after>use Result;
use core::Segment;
use core::SegmentId;
use core::SegmentComponent;
use schema::Term;
use common::HasLen;
use core::SegmentMeta;
use fastfield::{self, FastFieldNotAvailableError};
use fastfield::DeleteBitSet;
use store::StoreReader;
use schema::Document;
use directory::ReadOnlySource;
use DocId;
use std::str;
use termdict::TermDictionary;
use std::cmp;
use postings::TermInfo;
use termdict::TermDictionaryImpl;
use std::sync::Arc;
use std::fmt;
use schema::Field;
use postings::SegmentPostingsOption;
use postings::{SegmentPostings, BlockSegmentPostings};
use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader};
use schema::Schema;
use postings::FreqHandler;

/// Entry point to access all of the datastructures of the `Segment`
///
/// - term dictionary
/// - postings
/// - store
/// - fast field readers
/// - field norm reader
///
/// The segment reader has a very low memory footprint,
/// as close to all of the memory data is mmapped.
///
#[derive(Clone)]
pub struct SegmentReader {
    segment_id: SegmentId,
    segment_meta: SegmentMeta,
    terms: Arc<TermDictionaryImpl>,
    postings_data: ReadOnlySource,
    store_reader: StoreReader,
    fast_fields_reader: Arc<FastFieldsReader>,
    fieldnorms_reader: Arc<FastFieldsReader>,
    delete_bitset: DeleteBitSet,
    positions_data: ReadOnlySource,
    schema: Schema,
}

impl SegmentReader {
    /// Returns the highest document id ever attributed in
    /// this segment + 1.
    /// Today, `tantivy` does not handle deletes, so it happens
    /// to also be the number of documents in the index.
    pub fn max_doc(&self) -> DocId {
        self.segment_meta.max_doc()
    }

    /// Returns the number of documents.
    /// Deleted documents are not counted.
    ///
    /// Today, `tantivy` does not handle deletes so max doc and
    /// num_docs are the same.
    pub fn num_docs(&self) -> DocId {
        self.segment_meta.num_docs()
    }

    /// Return the number of documents that have been
    /// deleted in the segment.
    pub fn num_deleted_docs(&self) -> DocId {
        self.delete_bitset.len() as DocId
    }

    #[doc(hidden)]
    pub fn fast_fields_reader(&self) -> &FastFieldsReader {
        &*self.fast_fields_reader
    }

    /// Accessor to a segment's fast field reader given a field.
    ///
    /// Returns the u64 fast value reader if the field
    /// is a u64 field indexed as "fast".
    ///
    /// Return a FastFieldNotAvailableError if the field is not
    /// declared as a fast field in the schema.
    ///
    /// # Panics
    /// May panic if the index is corrupted.
    pub fn get_fast_field_reader<TFastFieldReader: FastFieldReader>
        (&self,
         field: Field)
         -> fastfield::Result<TFastFieldReader> {
        let field_entry = self.schema.get_field_entry(field);
        if !TFastFieldReader::is_enabled(field_entry.field_type()) {
            Err(FastFieldNotAvailableError::new(field_entry))
        } else {
            Ok(self.fast_fields_reader
                   .open_reader(field)
                   .expect("Fast field file corrupted."))
        }
    }

    /// Accessor to the segment's `Field norms`'s reader.
    ///
    /// Field norms are the length (in tokens) of the fields.
    /// It is used in the computation of the [TfIdf]
    /// (https://fulmicoton.gitbooks.io/tantivy-doc/content/tfidf.html).
    ///
    /// They are simply stored as a fast field, serialized in
    /// the `.fieldnorm` file of the segment.
    pub fn get_fieldnorms_reader(&self, field: Field) -> Option<U64FastFieldReader> {
        self.fieldnorms_reader.open_reader(field)
    }

    /// Returns the number of documents containing the term.
    pub fn doc_freq(&self, term: &Term) -> u32 {
        match self.get_term_info(term) {
            Some(term_info) => term_info.doc_freq,
            None => 0,
        }
    }

    /// Accessor to the segment's `StoreReader`.
    pub fn get_store_reader(&self) -> &StoreReader {
        &self.store_reader
    }

    /// Open a new segment for reading.
    pub fn open(segment: Segment) -> Result<SegmentReader> {

        let source = segment.open_read(SegmentComponent::TERMS)?;
        let terms = TermDictionaryImpl::from_source(source)?;

        let store_source = segment.open_read(SegmentComponent::STORE)?;
        let store_reader = StoreReader::from_source(store_source);

        let postings_shared_mmap = segment.open_read(SegmentComponent::POSTINGS)?;

        let fast_field_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
        let fast_fields_reader = FastFieldsReader::from_source(fast_field_data)?;

        let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
        let fieldnorms_reader = FastFieldsReader::from_source(fieldnorms_data)?;

        let positions_data = segment
            .open_read(SegmentComponent::POSITIONS)
            .unwrap_or_else(|_| ReadOnlySource::empty());

        let delete_bitset = if segment.meta().has_deletes() {
            let delete_data = segment.open_read(SegmentComponent::DELETE)?;
            DeleteBitSet::open(delete_data)
        } else {
            DeleteBitSet::empty()
        };

        let schema = segment.schema();
        Ok(SegmentReader {
               segment_meta: segment.meta().clone(),
               postings_data: postings_shared_mmap,
               terms: Arc::new(terms),
               segment_id: segment.id(),
               store_reader: store_reader,
               fast_fields_reader: Arc::new(fast_fields_reader),
               fieldnorms_reader: Arc::new(fieldnorms_reader),
               delete_bitset: delete_bitset,
               positions_data: positions_data,
               schema: schema,
           })
    }

    /// Return the term dictionary datastructure.
    pub fn terms(&self) -> &TermDictionaryImpl {
        &self.terms
    }

    /// Returns the document (or to be accurate, its stored field)
    /// bearing the given doc id.
    /// This method is slow and should seldom be called from
    /// within a collector.
    pub fn doc(&self, doc_id: DocId) -> Result<Document> {
        self.store_reader.get(doc_id)
    }

    /// Returns the segment postings associated with the term, and with the given option,
    /// or `None` if the term has never been encounterred and indexed.
    ///
    /// If the field was not indexed with the indexing options that cover
    /// the requested options, the returned `SegmentPostings` the method does not fail
    /// and returns a `SegmentPostings` with as much information as possible.
    ///
    /// For instance, requesting `SegmentPostingsOption::FreqAndPositions` for a
    /// `TextIndexingOptions` that does not index position will return a `SegmentPostings`
    /// with `DocId`s and frequencies.
    pub fn read_postings(&self,
                         term: &Term,
                         option: SegmentPostingsOption)
                         -> Option<SegmentPostings> {
        let field = term.field();
        let field_entry = self.schema.get_field_entry(field);
        let term_info = get!(self.get_term_info(term));
        let maximum_option = get!(field_entry.field_type().get_segment_postings_option());
        let best_effort_option = cmp::min(maximum_option, option);
        Some(self.read_postings_from_terminfo(&term_info, best_effort_option))
    }

    /// Returns a posting object given a `term_info`.
    /// This method is for an advanced usage only.
    ///
    /// Most user should prefer using `read_postings` instead.
    pub fn read_postings_from_terminfo(&self,
                                       term_info: &TermInfo,
                                       option: SegmentPostingsOption)
                                       -> SegmentPostings {
        let block_postings = self.read_block_postings_from_terminfo(term_info, option);
        let delete_bitset = self.delete_bitset.clone();
        SegmentPostings::from_block_postings(block_postings, delete_bitset)
    }

    /// Returns a block postings given a `term_info`.
    /// This method is for an advanced usage only.
    ///
    /// Most user should prefer using `read_postings` instead.
    pub fn read_block_postings_from_terminfo(&self,
                                             term_info: &TermInfo,
                                             option: SegmentPostingsOption)
                                             -> BlockSegmentPostings {
        let offset = term_info.postings_offset as usize;
        let postings_data = &self.postings_data[offset..];
        let freq_handler = match option {
            SegmentPostingsOption::NoFreq => FreqHandler::new_without_freq(),
            SegmentPostingsOption::Freq => FreqHandler::new_with_freq(),
            SegmentPostingsOption::FreqAndPositions => {
                let offset = term_info.positions_offset as usize;
                let offseted_position_data = &self.positions_data[offset..];
                FreqHandler::new_with_freq_and_position(offseted_position_data)
            }
        };
        BlockSegmentPostings::from_data(term_info.doc_freq as usize, postings_data, freq_handler)
    }

    /// Resets the block segment to another position of the postings
    /// file.
    ///
    /// This is useful for enumerating through a list of terms,
    /// and consuming the associated posting lists while avoiding
    /// reallocating a `BlockSegmentPostings`.
    ///
    /// # Warning
    ///
    /// This does not reset the positions list.
    pub fn reset_block_postings_from_terminfo<'a>(&'a self,
                                                  term_info: &TermInfo,
                                                  block_postings: &mut BlockSegmentPostings<'a>) {
        let offset = term_info.postings_offset as usize;
        let postings_data: &'a [u8] = &self.postings_data[offset..];
        block_postings.reset(term_info.doc_freq as usize, postings_data);
    }

    /// Returns the term info associated with the term.
    pub fn get_term_info(&self, term: &Term) -> Option<TermInfo> {
        self.terms.get(term.as_slice())
    }

    /// Returns the segment id
    pub fn segment_id(&self) -> SegmentId {
        self.segment_id
    }

    /// Returns the bitset representing
    /// the documents that have been deleted.
    pub fn delete_bitset(&self) -> &DeleteBitSet {
        &self.delete_bitset
    }

    /// Returns true iff the `doc` is marked
    /// as deleted.
    pub fn is_deleted(&self, doc: DocId) -> bool {
        self.delete_bitset.is_deleted(doc)
    }
}

impl fmt::Debug for SegmentReader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "SegmentReader({:?})", self.segment_id)
    }
}
<|endoftext|>