<commit_before>// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// This defines a base target-configuration for native UEFI systems. The UEFI specification has
// quite detailed sections on the ABI of all the supported target architectures. In almost all
// cases it simply follows what Microsoft Windows does. Hence, whenever in doubt, see the MSDN
// documentation.
// UEFI uses COFF/PE32+ format for binaries. All binaries must be statically linked. No dynamic
// linker is supported. As native to COFF, binaries are position-dependent, but will be relocated
// by the loader if the pre-chosen memory location is already in use.
// UEFI forbids running code on anything but the boot-CPU. Not interrupts are allowed other than
// the timer-interrupt. Device-drivers are required to use polling-based models. Furthermore, all
// code runs in the same environment, no process separation is supported.

use spec::{LinkArgs, LinkerFlavor, LldFlavor, PanicStrategy, TargetOptions};
use std::default::Default;

pub fn opts() -> TargetOptions {
    let mut pre_link_args = LinkArgs::new();

    pre_link_args.insert(LinkerFlavor::Lld(LldFlavor::Link), vec![
            // Suppress the verbose logo and authorship debugging output, which would needlessly
            // clog any log files.
            "/NOLOGO".to_string(),

            // UEFI is fully compatible to non-executable data pages. Tell the compiler that
            // non-code sections can be marked as non-executable, including stack pages.
            "/NXCOMPAT".to_string(),

            // There is no runtime for UEFI targets, prevent them from being linked. UEFI targets
            // must be freestanding.
            "/nodefaultlib".to_string(),

            // Non-standard subsystems have no default entry-point in PE+ files. We have to define
            // one. "efi_main" seems to be a common choice amongst other implementations and the
            // spec.
            "/entry:efi_main".to_string(),

            // COFF images have a "Subsystem" field in their header, which defines what kind of
            // program it is. UEFI has 3 fields reserved, which are EFI_APPLICATION,
            // EFI_BOOT_SERVICE_DRIVER, and EFI_RUNTIME_DRIVER. We default to EFI_APPLICATION,
            // which is very likely the most common option. Individual projects can override this
            // with custom linker flags.
            // The subsystem-type only has minor effects on the application. It defines the memory
            // regions the application is loaded into (runtime-drivers need to be put into
            // reserved areas), as well as whether a return from the entry-point is treated as
            // exit (default for applications).
            "/subsystem:efi_application".to_string(),
        ]);

    TargetOptions {
        dynamic_linking: false,
        executables: true,
        disable_redzone: true,
        exe_suffix: ".efi".to_string(),
        allows_weak_linkage: false,
        panic_strategy: PanicStrategy::Abort,
        singlethread: true,
        emit_debug_gdb_scripts: false,

        linker: Some("lld-link".to_string()),
        lld_flavor: LldFlavor::Link,
        pre_link_args,

        .. Default::default()
    }
}
<commit_msg>Enable stack probes for UEFI images<commit_after>// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// This defines a base target-configuration for native UEFI systems. The UEFI specification has
// quite detailed sections on the ABI of all the supported target architectures. In almost all
// cases it simply follows what Microsoft Windows does. Hence, whenever in doubt, see the MSDN
// documentation.
// UEFI uses COFF/PE32+ format for binaries. All binaries must be statically linked. No dynamic
// linker is supported. As native to COFF, binaries are position-dependent, but will be relocated
// by the loader if the pre-chosen memory location is already in use.
// UEFI forbids running code on anything but the boot-CPU. Not interrupts are allowed other than
// the timer-interrupt. Device-drivers are required to use polling-based models. Furthermore, all
// code runs in the same environment, no process separation is supported.

use spec::{LinkArgs, LinkerFlavor, LldFlavor, PanicStrategy, TargetOptions};
use std::default::Default;

pub fn opts() -> TargetOptions {
    let mut pre_link_args = LinkArgs::new();

    pre_link_args.insert(LinkerFlavor::Lld(LldFlavor::Link), vec![
            // Suppress the verbose logo and authorship debugging output, which would needlessly
            // clog any log files.
            "/NOLOGO".to_string(),

            // UEFI is fully compatible to non-executable data pages. Tell the compiler that
            // non-code sections can be marked as non-executable, including stack pages.
            "/NXCOMPAT".to_string(),

            // There is no runtime for UEFI targets, prevent them from being linked. UEFI targets
            // must be freestanding.
            "/nodefaultlib".to_string(),

            // Non-standard subsystems have no default entry-point in PE+ files. We have to define
            // one. "efi_main" seems to be a common choice amongst other implementations and the
            // spec.
            "/entry:efi_main".to_string(),

            // COFF images have a "Subsystem" field in their header, which defines what kind of
            // program it is. UEFI has 3 fields reserved, which are EFI_APPLICATION,
            // EFI_BOOT_SERVICE_DRIVER, and EFI_RUNTIME_DRIVER. We default to EFI_APPLICATION,
            // which is very likely the most common option. Individual projects can override this
            // with custom linker flags.
            // The subsystem-type only has minor effects on the application. It defines the memory
            // regions the application is loaded into (runtime-drivers need to be put into
            // reserved areas), as well as whether a return from the entry-point is treated as
            // exit (default for applications).
            "/subsystem:efi_application".to_string(),
        ]);

    TargetOptions {
        dynamic_linking: false,
        executables: true,
        disable_redzone: true,
        exe_suffix: ".efi".to_string(),
        allows_weak_linkage: false,
        panic_strategy: PanicStrategy::Abort,
        stack_probes: true,
        singlethread: true,
        emit_debug_gdb_scripts: false,

        linker: Some("lld-link".to_string()),
        lld_flavor: LldFlavor::Link,
        pre_link_args,

        .. Default::default()
    }
}
<|endoftext|>


<commit_before>use crate::common::*;
use crate::data::packet::Packet;
use crate::data::value::*;
use std::any::Any;
use std::io::Write;
use std::sync::Arc;

use crate::error::*;

/// Used to implement muxing operations.
pub trait Muxer: Send {
    /// Configures a muxer.
    fn configure(&mut self) -> Result<()>;
    /// Writes a stream header into a data structure implementing
    /// the `Write` trait.
    fn write_header(&mut self, out: &mut dyn Write) -> Result<()>;
    /// Writes a stream packet into a data structure implementing
    /// the `Write` trait.
    fn write_packet(&mut self, out: &mut dyn Write, pkt: Arc<Packet>) -> Result<()>;
    /// Writes a stream trailer into a data structure implementing
    /// the `Write` trait.
    fn write_trailer(&mut self, out: &mut dyn Write) -> Result<()>;

    /// Sets global media file information for a muxer.
    fn set_global_info(&mut self, info: GlobalInfo) -> Result<()>;
    /// Sets a muxer option.
    ///
    /// This method should be called as many times as the number of options
    /// present in a muxer.
    fn set_option<'a>(&mut self, key: &str, val: Value<'a>) -> Result<()>;
}

/// Auxiliary structure to encapsulate a muxer object and
/// its additional data.
pub struct Context {
    muxer: Box<dyn Muxer + Send>,
    writer: Box<dyn Write + Send>,
    buf: Vec<u8>,
    /// User private data.
    ///
    /// This data cannot be cloned.
    pub user_private: Option<Box<dyn Any + Send + Sync>>,
}

impl Context {
    /// Creates a new `Context` instance.
    pub fn new<W: Write + 'static + Send>(muxer: Box<dyn Muxer + Send>, writer: Box<W>) -> Self {
        Context {
            muxer,
            writer,
            buf: Vec::new(),
            user_private: None,
        }
    }

    /// Configures a muxer.
    pub fn configure(&mut self) -> Result<()> {
        self.muxer.configure()
    }

    /// Writes a stream header to an internal buffer and returns how many
    /// bytes were written or an error.
    pub fn write_header(&mut self) -> Result<usize> {
        self.muxer.write_header(&mut self.buf)?;
        //FIXME: we should have proper management of the buffer's index
        match self.writer.write_all(&self.buf) {
            Ok(()) => {
                let len = self.buf.len();
                self.buf.clear();
                Ok(len)
            }
            Err(e) => Err(Error::Io(e)),
        }
    }

    /// Writes a stream packet to an internal buffer and returns how many
    /// bytes were written or an error.
    pub fn write_packet(&mut self, pkt: Arc<Packet>) -> Result<usize> {
        self.muxer.write_packet(&mut self.buf, pkt)?;
        //FIXME: we should have proper management of the buffer's index
        match self.writer.write_all(&self.buf) {
            Ok(()) => {
                let len = self.buf.len();
                self.buf.clear();
                Ok(len)
            }
            Err(e) => Err(Error::Io(e)),
        }
    }

    /// Writes a stream trailer to an internal buffer and returns how many
    /// bytes were written or an error.
    pub fn write_trailer(&mut self) -> Result<usize> {
        self.muxer.write_trailer(&mut self.buf)?;
        //FIXME: we should have proper management of the buffer's index
        match self.writer.write_all(&self.buf) {
            Ok(()) => {
                let len = self.buf.len();
                self.buf.clear();
                Ok(len)
            }
            Err(e) => Err(Error::Io(e)),
        }
    }

    /// Sets global media file information for a muxer.
    pub fn set_global_info(&mut self, info: GlobalInfo) -> Result<()> {
        self.muxer.set_global_info(info)
    }

    /// Sets a muxer option.
    ///
    /// This method should be called as many times as the number of options
    /// present in a muxer.
    pub fn set_option<'a, V>(&mut self, key: &str, val: V) -> Result<()>
    where
        V: Into<Value<'a>>,
    {
        self.muxer.set_option(key, val.into())
    }
}

/// Format descriptor.
///
/// Contains information on a format and its own muxer.
#[derive(Clone, Debug, PartialEq)]
pub struct Descr {
    /// Format name.
    pub name: &'static str,
    /// Muxer name.
    pub demuxer: &'static str,
    /// Format description.
    pub description: &'static str,
    /// Format media file extensions.
    pub extensions: &'static [&'static str],
    /// Format MIME.
    pub mime: &'static [&'static str],
}

/// Used to get a format descriptor and create a new muxer.
pub trait Descriptor {
    /// Creates a new muxer for the requested format.
    fn create(&self) -> Box<dyn Muxer>;
    /// Returns the descriptor of a format.
    fn describe(&self) -> &Descr;
}

/// Used to look for a specific format.
pub trait Lookup {
    /// Retrieves a specific format by name.
    fn by_name(&self, name: &str) -> Option<&'static dyn Descriptor>;
}

impl Lookup for [&'static dyn Descriptor] {
    fn by_name(&self, name: &str) -> Option<&'static dyn Descriptor> {
        self.iter().find(|&&d| d.describe().name == name).copied()
    }
}
<commit_msg>format: Add Writer for Seek and non-Seek output<commit_after>use crate::common::*;
use crate::data::packet::Packet;
use crate::data::value::*;
use std::any::Any;
use std::io::{ErrorKind, Seek, SeekFrom, Write};
use std::sync::Arc;

use crate::error::*;

pub trait WriteSeek: Write + Seek {}

impl<T: Write + Seek> WriteSeek for T {}

/// Runtime wrapper around either a [`Write`] or a [`WriteSeek`] trait object which supports querying
/// for seek support.
pub enum Writer {
    NonSeekable(Box<dyn Write>, u64),
    Seekable(Box<dyn WriteSeek>),
}

impl Writer {
    /// Creates a [`Writer`] from a trait object that implements both [`Write`] and [`Seek`].
    pub fn from_seekable(inner: Box<dyn WriteSeek>) -> Self {
        Self::Seekable(inner)
    }

    /// Creates a [`Writer`] from a trait object that implements the [`Write`] trait.
    pub fn from_nonseekable(inner: Box<dyn Write>) -> Self {
        Self::NonSeekable(inner, 0)
    }

    /// Returns whether the [`Writer`] can seek within the source.
    pub fn can_seek(&self) -> bool {
        matches!(self, Self::Seekable(_))
    }
}

impl Write for Writer {
    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
        match self {
            Self::NonSeekable(inner, ref mut index) => {
                let result = inner.write(bytes);

                if let Ok(written) = result {
                    *index += written as u64;
                }

                result
            }
            Self::Seekable(inner) => inner.write(bytes),
        }
    }

    fn flush(&mut self) -> std::io::Result<()> {
        match self {
            Self::NonSeekable(inner, ..) => inner.flush(),
            Self::Seekable(inner) => inner.flush(),
        }
    }
}

impl Seek for Writer {
    fn seek(&mut self, seek: SeekFrom) -> std::io::Result<u64> {
        match self {
            Self::NonSeekable(_, index) => {
                if let SeekFrom::Current(0) = seek {
                    Ok(*index)
                } else {
                    Err(std::io::Error::new(
                        ErrorKind::Other,
                        "Seeking not supported",
                    ))
                }
            }
            Self::Seekable(inner) => inner.seek(seek),
        }
    }
}

/// Used to implement muxing operations.
pub trait Muxer: Send {
    /// Configures a muxer.
    fn configure(&mut self) -> Result<()>;
    /// Writes a stream header into a data structure implementing
    /// the `Write` trait.
    fn write_header(&mut self, out: &mut Writer) -> Result<()>;
    /// Writes a stream packet into a data structure implementing
    /// the `Write` trait.
    fn write_packet(&mut self, out: &mut Writer, pkt: Arc<Packet>) -> Result<()>;
    /// Writes a stream trailer into a data structure implementing
    /// the `Write` trait.
    fn write_trailer(&mut self, out: &mut Writer) -> Result<()>;

    /// Sets global media file information for a muxer.
    fn set_global_info(&mut self, info: GlobalInfo) -> Result<()>;
    /// Sets a muxer option.
    ///
    /// This method should be called as many times as the number of options
    /// present in a muxer.
    fn set_option<'a>(&mut self, key: &str, val: Value<'a>) -> Result<()>;
}

/// Auxiliary structure to encapsulate a muxer object and
/// its additional data.
pub struct Context {
    muxer: Box<dyn Muxer + Send>,
    writer: Writer,
    /// User private data.
    ///
    /// This data cannot be cloned.
    pub user_private: Option<Box<dyn Any + Send + Sync>>,
}

impl Context {
    /// Creates a new `Context` instance.
    pub fn new(muxer: Box<dyn Muxer + Send>, writer: Writer) -> Self {
        Context {
            muxer,
            writer,
            user_private: None,
        }
    }

    /// Configures a muxer.
    pub fn configure(&mut self) -> Result<()> {
        self.muxer.configure()
    }

    /// Writes a stream header to an internal buffer and returns how many
    /// bytes were written or an error.
    pub fn write_header(&mut self) -> Result<()> {
        self.muxer.write_header(&mut self.writer)
    }

    /// Writes a stream packet to an internal buffer and returns how many
    /// bytes were written or an error.
    pub fn write_packet(&mut self, pkt: Arc<Packet>) -> Result<()> {
        self.muxer.write_packet(&mut self.writer, pkt)
    }

    /// Writes a stream trailer to an internal buffer and returns how many
    /// bytes were written or an error.
    pub fn write_trailer(&mut self) -> Result<()> {
        self.muxer.write_trailer(&mut self.writer)?;
        self.writer.flush()?;

        Ok(())
    }

    /// Sets global media file information for a muxer.
    pub fn set_global_info(&mut self, info: GlobalInfo) -> Result<()> {
        self.muxer.set_global_info(info)
    }

    /// Sets a muxer option.
    ///
    /// This method should be called as many times as the number of options
    /// present in a muxer.
    pub fn set_option<'a, V>(&mut self, key: &str, val: V) -> Result<()>
    where
        V: Into<Value<'a>>,
    {
        self.muxer.set_option(key, val.into())
    }
}

/// Format descriptor.
///
/// Contains information on a format and its own muxer.
#[derive(Clone, Debug, PartialEq)]
pub struct Descr {
    /// Format name.
    pub name: &'static str,
    /// Muxer name.
    pub demuxer: &'static str,
    /// Format description.
    pub description: &'static str,
    /// Format media file extensions.
    pub extensions: &'static [&'static str],
    /// Format MIME.
    pub mime: &'static [&'static str],
}

/// Used to get a format descriptor and create a new muxer.
pub trait Descriptor {
    /// Creates a new muxer for the requested format.
    fn create(&self) -> Box<dyn Muxer>;
    /// Returns the descriptor of a format.
    fn describe(&self) -> &Descr;
}

/// Used to look for a specific format.
pub trait Lookup {
    /// Retrieves a specific format by name.
    fn by_name(&self, name: &str) -> Option<&'static dyn Descriptor>;
}

impl Lookup for [&'static dyn Descriptor] {
    fn by_name(&self, name: &str) -> Option<&'static dyn Descriptor> {
        self.iter().find(|&&d| d.describe().name == name).copied()
    }
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2013 Jeremy Letang (letang.jeremy@gmail.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

extern crate pkg_config;

use std::path::Path;
use std::env;

#[cfg(all(unix, not(target_os = "linux")))]
use unix_platform as platform;

fn main() {
    if env::var("PORTAUDIO_ONLY_STATIC").is_err() {
        // If pkg-config finds a library on the system, we are done
        if pkg_config::Config::new().atleast_version("19").find("portaudio-2.0").is_ok() {
            return;
        }
    }

    build();
}

fn build() {
    // retrieve cargo deps out dir
    let out_dir_str = env::var("OUT_DIR").unwrap();
    let out_dir = Path::new(&out_dir_str);

    let static_lib = out_dir.join("lib/libportaudio.a");
    if let Err(_) = ::std::fs::metadata(static_lib) {
        platform::download();
        platform::build(out_dir);
    }

    platform::print_libs(out_dir);
}

#[allow(dead_code)]
mod unix_platform {
    use std::process::Command;
    use std::path::Path;

    use std::env;

    pub const PORTAUDIO_URL: &'static str = "http://www.portaudio.com/archives/pa_stable_v19_20140130.tgz";
    pub const PORTAUDIO_TAR: &'static str = "pa_stable_v19_20140130.tgz";
    pub const PORTAUDIO_FOLDER: &'static str = "portaudio";

    pub fn download() {
        match Command::new("curl").arg(PORTAUDIO_URL).arg("-O").output() {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }
    }

    pub fn build(out_dir: &Path) {
        // untar portaudio sources
        match Command::new("tar").arg("xvf").arg(PORTAUDIO_TAR).output() {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }

        // change dir to the portaudio folder
        match env::set_current_dir(PORTAUDIO_FOLDER) {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }

        // run portaudio autoconf
        Command::new("./configure")
            .args(&["--disable-shared", "--enable-static"]) // Only build static lib
            .args(&["--prefix", out_dir.to_str().unwrap()]) // Install on the outdir
            .arg("--with-pic") // Build position-independent code (required by Rust)
            .output()
            .unwrap();

        // then make
        match Command::new("make").output() {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }

        // "install" on the outdir
        match Command::new("make").arg("install").output() {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }

        // return to rust-portaudio root
        match env::set_current_dir("..") {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }

        // cleaning portaudio sources
        match Command::new("rm").arg("-rf").args(&[PORTAUDIO_TAR, PORTAUDIO_FOLDER]).output() {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }
    }

    pub fn print_libs(out_dir: &Path) {
        let out_str = out_dir.to_str().unwrap();
        println!("cargo:rustc-flags=-L native={}/lib -l static=portaudio", out_str);
    }
}

#[cfg(target_os = "linux")]
mod platform {
    use pkg_config;
    use std::process::Command;
    use super::unix_platform;
    use std::path::Path;

    pub fn download() {
        match Command::new("wget").arg(unix_platform::PORTAUDIO_URL).output() {
            Ok(_) => {},
            Err(e) => panic!("{}", e)
        }
    }

    pub fn build(out_dir: &Path) {
        unix_platform::build(out_dir);
    }

    pub fn print_libs(out_dir: &Path) {
        let portaudio_pc_file = out_dir.join("lib/pkgconfig/portaudio-2.0.pc");
        let portaudio_pc_file = portaudio_pc_file.to_str().unwrap();

        pkg_config::Config::new().statik(true).find(portaudio_pc_file).unwrap();
    }
}

#[cfg(windows)]
mod platform {
    use std::path::Path;
    use std::io::{self, Write};

    const PORTAUDIO_DOWNLOAD_URL: &'static str = "http://www.portaudio.com";

    fn print_lib_url() {
        let msg = format!("Don't know how to build portaudio on Windows yet. Sources and build instructions available at: {}", PORTAUDIO_DOWNLOAD_URL);
        io::stderr().write(msg.as_bytes()).unwrap();
    }

    pub fn download() {
        print_lib_url();
    }

    pub fn build(_: &Path) {
        print_lib_url();
    }

    pub fn print_libs(_: &Path) {
        print_lib_url();
    }
}
<commit_msg>Change build errors to panics<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2013 Jeremy Letang (letang.jeremy@gmail.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

extern crate pkg_config;

use std::path::Path;
use std::env;
use std::fmt::Display;

#[cfg(all(unix, not(target_os = "linux")))]
use unix_platform as platform;

fn main() {
    if env::var("PORTAUDIO_ONLY_STATIC").is_err() {
        // If pkg-config finds a library on the system, we are done
        if pkg_config::Config::new().atleast_version("19").find("portaudio-2.0").is_ok() {
            return;
        }
    }

    build();
}

fn build() {
    // retrieve cargo deps out dir
    let out_dir_str = env::var("OUT_DIR").unwrap();
    let out_dir = Path::new(&out_dir_str);

    let static_lib = out_dir.join("lib/libportaudio.a");
    if let Err(_) = ::std::fs::metadata(static_lib) {
        platform::download();
        platform::build(out_dir);
    }

    platform::print_libs(out_dir);
}

// Similar to unwrap, but panics on just the error value
#[allow(dead_code)]
fn err_to_panic<T, E: Display>(result: Result<T, E>) -> T {
    match result {
        Ok(x) => x,
        Err(e) => panic!("{}", e)
    }
}

#[allow(dead_code)]
mod unix_platform {
    use std::process::Command;
    use std::path::Path;

    use std::env;

    use super::err_to_panic;

    pub const PORTAUDIO_URL: &'static str = "http://www.portaudio.com/archives/pa_stable_v19_20140130.tgz";
    pub const PORTAUDIO_TAR: &'static str = "pa_stable_v19_20140130.tgz";
    pub const PORTAUDIO_FOLDER: &'static str = "portaudio";

    pub fn download() {
        err_to_panic(Command::new("curl").arg(PORTAUDIO_URL).arg("-O").output());
    }

    pub fn build(out_dir: &Path) {
        // untar portaudio sources
        err_to_panic(Command::new("tar").arg("xvf").arg(PORTAUDIO_TAR).output());

        // change dir to the portaudio folder
        err_to_panic(env::set_current_dir(PORTAUDIO_FOLDER));

        // run portaudio autoconf
        err_to_panic(Command::new("./configure")
            .args(&["--disable-shared", "--enable-static"]) // Only build static lib
            .args(&["--prefix", out_dir.to_str().unwrap()]) // Install on the outdir
            .arg("--with-pic") // Build position-independent code (required by Rust)
            .output());

        // then make
        err_to_panic(Command::new("make").output());

        // "install" on the outdir
        err_to_panic(Command::new("make").arg("install").output());

        // return to rust-portaudio root
        err_to_panic(env::set_current_dir(".."));

        // cleaning portaudio sources
        err_to_panic(Command::new("rm").arg("-rf")
            .args(&[PORTAUDIO_TAR, PORTAUDIO_FOLDER]).output());
    }

    pub fn print_libs(out_dir: &Path) {
        let out_str = out_dir.to_str().unwrap();
        println!("cargo:rustc-flags=-L native={}/lib -l static=portaudio", out_str);
    }
}

#[cfg(target_os = "linux")]
mod platform {
    use pkg_config;
    use std::process::Command;
    use super::unix_platform;
    use std::path::Path;

    use super::err_to_panic;

    pub fn download() {
        err_to_panic(Command::new("wget").arg(unix_platform::PORTAUDIO_URL).output());
    }

    pub fn build(out_dir: &Path) {
        unix_platform::build(out_dir);
    }

    pub fn print_libs(out_dir: &Path) {
        let portaudio_pc_file = out_dir.join("lib/pkgconfig/portaudio-2.0.pc");
        let portaudio_pc_file = portaudio_pc_file.to_str().unwrap();

        err_to_panic(pkg_config::Config::new().statik(true).find(portaudio_pc_file));
    }
}

#[cfg(windows)]
mod platform {
    use std::path::Path;

    const PORTAUDIO_DOWNLOAD_URL: &'static str = "http://www.portaudio.com";

    fn print_lib_url() {
        panic!("Don't know how to build portaudio on Windows yet. Sources and build instructions available at: {}", PORTAUDIO_DOWNLOAD_URL);
    }

    pub fn download() {
        print_lib_url();
    }

    pub fn build(_: &Path) {
        print_lib_url();
    }

    pub fn print_libs(_: &Path) {
        print_lib_url();
    }
}
<|endoftext|>


<commit_before>// spell-checker:ignore udev
use crate::common::util::*;

#[test]
fn test_df_compatible_no_size_arg() {
    new_ucmd!().arg("-a").succeeds();
}

#[test]
fn test_df_shortened_long_argument() {
    new_ucmd!().arg("--a").succeeds();
}

#[test]
fn test_df_compatible() {
    new_ucmd!().arg("-ah").succeeds();
}

#[test]
fn test_df_compatible_type() {
    new_ucmd!().arg("-aT").succeeds();
}

#[test]
fn test_df_compatible_si() {
    new_ucmd!().arg("-aH").succeeds();
}

#[test]
fn test_df_output() {
    // TODO These should fail because `-total` should have two dashes,
    // not just one. But they don't fail.
    if cfg!(target_os = "macos") {
        new_ucmd!().arg("-H").arg("-total").succeeds().
        stdout_only("Filesystem               Size         Used    Available     Capacity  Use% Mounted on       \n");
    } else {
        new_ucmd!().arg("-H").arg("-total").succeeds().stdout_only(
            "Filesystem               Size         Used    Available  Use% Mounted on       \n",
        );
    }
}

/// Test that the order of rows in the table does not change across executions.
#[test]
fn test_order_same() {
    // TODO When #3057 is resolved, we should just use
    //
    //     new_ucmd!().arg("--output=source").succeeds().stdout_move_str();
    //
    // instead of parsing the entire `df` table as a string.
    let output1 = new_ucmd!().succeeds().stdout_move_str();
    let output2 = new_ucmd!().succeeds().stdout_move_str();
    let output1: Vec<String> = output1
        .lines()
        .map(|l| String::from(l.split_once(' ').unwrap().0))
        .collect();
    let output2: Vec<String> = output2
        .lines()
        .map(|l| String::from(l.split_once(' ').unwrap().0))
        .collect();
    assert_eq!(output1, output2);
}

#[test]
fn test_output_conflict_options() {
    for option in ["-i", "-T", "-P"] {
        new_ucmd!().arg("--output=source").arg(option).fails();
    }
}

#[test]
fn test_output_option() {
    new_ucmd!().arg("--output").succeeds();
    new_ucmd!().arg("--output=source,target").succeeds();
    new_ucmd!().arg("--output=invalid_option").fails();
}

#[test]
fn test_type_option() {
    new_ucmd!().args(&["-t", "ext4", "-t", "ext3"]).succeeds();
}

#[test]
fn test_total() {
    // Example output:
    //
    //     Filesystem            1K-blocks     Used Available Use% Mounted on
    //     udev                    3858016        0   3858016   0% /dev
    //     ...
    //     /dev/loop14               63488    63488         0 100% /snap/core20/1361
    //     total                 258775268 98099712 148220200  40% -
    let output = new_ucmd!().arg("--total").succeeds().stdout_move_str();

    // Skip the header line.
    let lines: Vec<&str> = output.lines().skip(1).collect();

    // Parse the values from the last row.
    let last_line = lines.last().unwrap();
    let mut iter = last_line.split_whitespace();
    assert_eq!(iter.next().unwrap(), "total");
    let reported_total_size = iter.next().unwrap().parse().unwrap();
    let reported_total_used = iter.next().unwrap().parse().unwrap();
    let reported_total_avail = iter.next().unwrap().parse().unwrap();

    // Loop over each row except the last, computing the sum of each column.
    let mut computed_total_size = 0;
    let mut computed_total_used = 0;
    let mut computed_total_avail = 0;
    let n = lines.len();
    for line in &lines[..n - 1] {
        let mut iter = line.split_whitespace();
        iter.next().unwrap();
        computed_total_size += iter.next().unwrap().parse::<u64>().unwrap();
        computed_total_used += iter.next().unwrap().parse::<u64>().unwrap();
        computed_total_avail += iter.next().unwrap().parse::<u64>().unwrap();
    }

    // Check that the sum of each column matches the reported value in
    // the last row.
    assert_eq!(computed_total_size, reported_total_size);
    assert_eq!(computed_total_used, reported_total_used);
    assert_eq!(computed_total_avail, reported_total_avail);
}

#[test]
fn test_use_percentage() {
    // Example output:
    //
    //     Filesystem            1K-blocks     Used Available Use% Mounted on
    //     udev                    3858016        0   3858016   0% /dev
    //     ...
    //     /dev/loop14               63488    63488         0 100% /snap/core20/1361
    let output = new_ucmd!().succeeds().stdout_move_str();

    // Skip the header line.
    let lines: Vec<&str> = output.lines().skip(1).collect();

    for line in lines {
        let mut iter = line.split_whitespace();
        iter.next();
        let reported_size = iter.next().unwrap().parse::<f64>().unwrap();
        let reported_used = iter.next().unwrap().parse::<f64>().unwrap();
        // Skip "Available" column
        iter.next();
        if cfg!(target_os = "macos") {
            // Skip "Capacity" column
            iter.next();
        }
        let reported_percentage = iter.next().unwrap();
        let reported_percentage = reported_percentage[..reported_percentage.len() - 1]
            .parse::<u8>()
            .unwrap();
        let computed_percentage = (100.0 * (reported_used / reported_size)).ceil() as u8;

        assert_eq!(computed_percentage, reported_percentage);
    }
}

#[test]
fn test_block_size_1024() {
    fn get_header(block_size: u64) -> String {
        // TODO When #3057 is resolved, we should just use
        //
        //     new_ucmd!().arg("--output=size").succeeds().stdout_move_str();
        //
        // instead of parsing the entire `df` table as a string.
        let output = new_ucmd!()
            .args(&["-B", &format!("{}", block_size)])
            .succeeds()
            .stdout_move_str();
        let first_line = output.lines().next().unwrap();
        let mut column_labels = first_line.split_whitespace();
        let size_column_label = column_labels.nth(1).unwrap();
        size_column_label.into()
    }

    assert_eq!(get_header(1024), "1K-blocks");
    assert_eq!(get_header(2048), "2K-blocks");
    assert_eq!(get_header(4096), "4K-blocks");
    assert_eq!(get_header(1024 * 1024), "1M-blocks");
    assert_eq!(get_header(2 * 1024 * 1024), "2M-blocks");
    assert_eq!(get_header(1024 * 1024 * 1024), "1G-blocks");
    assert_eq!(get_header(34 * 1024 * 1024 * 1024), "34G-blocks");
}

// TODO The spacing does not match GNU df. Also we need to remove
// trailing spaces from the heading row.
#[test]
fn test_output_selects_columns() {
    let output = new_ucmd!()
        .args(&["--output=source"])
        .succeeds()
        .stdout_move_str();
    assert_eq!(output.lines().next().unwrap(), "Filesystem       ");

    let output = new_ucmd!()
        .args(&["--output=source,target"])
        .succeeds()
        .stdout_move_str();
    assert_eq!(
        output.lines().next().unwrap(),
        "Filesystem       Mounted on       "
    );

    let output = new_ucmd!()
        .args(&["--output=source,target,used"])
        .succeeds()
        .stdout_move_str();
    assert_eq!(
        output.lines().next().unwrap(),
        "Filesystem       Mounted on               Used "
    );
}

// ToDO: more tests...
<commit_msg>tests: update test_df.rs to use --output argument<commit_after>// spell-checker:ignore udev
use crate::common::util::*;

#[test]
fn test_df_compatible_no_size_arg() {
    new_ucmd!().arg("-a").succeeds();
}

#[test]
fn test_df_shortened_long_argument() {
    new_ucmd!().arg("--a").succeeds();
}

#[test]
fn test_df_compatible() {
    new_ucmd!().arg("-ah").succeeds();
}

#[test]
fn test_df_compatible_type() {
    new_ucmd!().arg("-aT").succeeds();
}

#[test]
fn test_df_compatible_si() {
    new_ucmd!().arg("-aH").succeeds();
}

#[test]
fn test_df_output() {
    // TODO These should fail because `-total` should have two dashes,
    // not just one. But they don't fail.
    if cfg!(target_os = "macos") {
        new_ucmd!().arg("-H").arg("-total").succeeds().
        stdout_only("Filesystem               Size         Used    Available     Capacity  Use% Mounted on       \n");
    } else {
        new_ucmd!().arg("-H").arg("-total").succeeds().stdout_only(
            "Filesystem               Size         Used    Available  Use% Mounted on       \n",
        );
    }
}

/// Test that the order of rows in the table does not change across executions.
#[test]
fn test_order_same() {
    let output1 = new_ucmd!()
        .arg("--output=source")
        .succeeds()
        .stdout_move_str();
    let output2 = new_ucmd!()
        .arg("--output=source")
        .succeeds()
        .stdout_move_str();
    assert_eq!(output1, output2);
}

#[test]
fn test_output_conflict_options() {
    for option in ["-i", "-T", "-P"] {
        new_ucmd!().arg("--output=source").arg(option).fails();
    }
}

#[test]
fn test_output_option() {
    new_ucmd!().arg("--output").succeeds();
    new_ucmd!().arg("--output=source,target").succeeds();
    new_ucmd!().arg("--output=invalid_option").fails();
}

#[test]
fn test_type_option() {
    new_ucmd!().args(&["-t", "ext4", "-t", "ext3"]).succeeds();
}

#[test]
fn test_total() {
    // Example output:
    //
    //     Filesystem            1K-blocks     Used Available Use% Mounted on
    //     udev                    3858016        0   3858016   0% /dev
    //     ...
    //     /dev/loop14               63488    63488         0 100% /snap/core20/1361
    //     total                 258775268 98099712 148220200  40% -
    let output = new_ucmd!().arg("--total").succeeds().stdout_move_str();

    // Skip the header line.
    let lines: Vec<&str> = output.lines().skip(1).collect();

    // Parse the values from the last row.
    let last_line = lines.last().unwrap();
    let mut iter = last_line.split_whitespace();
    assert_eq!(iter.next().unwrap(), "total");
    let reported_total_size = iter.next().unwrap().parse().unwrap();
    let reported_total_used = iter.next().unwrap().parse().unwrap();
    let reported_total_avail = iter.next().unwrap().parse().unwrap();

    // Loop over each row except the last, computing the sum of each column.
    let mut computed_total_size = 0;
    let mut computed_total_used = 0;
    let mut computed_total_avail = 0;
    let n = lines.len();
    for line in &lines[..n - 1] {
        let mut iter = line.split_whitespace();
        iter.next().unwrap();
        computed_total_size += iter.next().unwrap().parse::<u64>().unwrap();
        computed_total_used += iter.next().unwrap().parse::<u64>().unwrap();
        computed_total_avail += iter.next().unwrap().parse::<u64>().unwrap();
    }

    // Check that the sum of each column matches the reported value in
    // the last row.
    assert_eq!(computed_total_size, reported_total_size);
    assert_eq!(computed_total_used, reported_total_used);
    assert_eq!(computed_total_avail, reported_total_avail);
}

#[test]
fn test_use_percentage() {
    // Example output:
    //
    //     Filesystem            1K-blocks     Used Available Use% Mounted on
    //     udev                    3858016        0   3858016   0% /dev
    //     ...
    //     /dev/loop14               63488    63488         0 100% /snap/core20/1361
    let output = new_ucmd!().succeeds().stdout_move_str();

    // Skip the header line.
    let lines: Vec<&str> = output.lines().skip(1).collect();

    for line in lines {
        let mut iter = line.split_whitespace();
        iter.next();
        let reported_size = iter.next().unwrap().parse::<f64>().unwrap();
        let reported_used = iter.next().unwrap().parse::<f64>().unwrap();
        // Skip "Available" column
        iter.next();
        if cfg!(target_os = "macos") {
            // Skip "Capacity" column
            iter.next();
        }
        let reported_percentage = iter.next().unwrap();
        let reported_percentage = reported_percentage[..reported_percentage.len() - 1]
            .parse::<u8>()
            .unwrap();
        let computed_percentage = (100.0 * (reported_used / reported_size)).ceil() as u8;

        assert_eq!(computed_percentage, reported_percentage);
    }
}

#[test]
fn test_block_size_1024() {
    fn get_header(block_size: u64) -> String {
        // TODO When #3057 is resolved, we should just use
        //
        //     new_ucmd!().arg("--output=size").succeeds().stdout_move_str();
        //
        // instead of parsing the entire `df` table as a string.
        let output = new_ucmd!()
            .args(&["-B", &format!("{}", block_size)])
            .succeeds()
            .stdout_move_str();
        let first_line = output.lines().next().unwrap();
        let mut column_labels = first_line.split_whitespace();
        let size_column_label = column_labels.nth(1).unwrap();
        size_column_label.into()
    }

    assert_eq!(get_header(1024), "1K-blocks");
    assert_eq!(get_header(2048), "2K-blocks");
    assert_eq!(get_header(4096), "4K-blocks");
    assert_eq!(get_header(1024 * 1024), "1M-blocks");
    assert_eq!(get_header(2 * 1024 * 1024), "2M-blocks");
    assert_eq!(get_header(1024 * 1024 * 1024), "1G-blocks");
    assert_eq!(get_header(34 * 1024 * 1024 * 1024), "34G-blocks");
}

// TODO The spacing does not match GNU df. Also we need to remove
// trailing spaces from the heading row.
#[test]
fn test_output_selects_columns() {
    let output = new_ucmd!()
        .args(&["--output=source"])
        .succeeds()
        .stdout_move_str();
    assert_eq!(output.lines().next().unwrap(), "Filesystem       ");

    let output = new_ucmd!()
        .args(&["--output=source,target"])
        .succeeds()
        .stdout_move_str();
    assert_eq!(
        output.lines().next().unwrap(),
        "Filesystem       Mounted on       "
    );

    let output = new_ucmd!()
        .args(&["--output=source,target,used"])
        .succeeds()
        .stdout_move_str();
    assert_eq!(
        output.lines().next().unwrap(),
        "Filesystem       Mounted on               Used "
    );
}

// ToDO: more tests...
<|endoftext|>


<commit_before>use crate::controllers::prelude::*;
use crate::middleware::log_request::add_custom_metadata;
use crate::models::helpers::with_count::*;
use crate::util::errors::{bad_request, AppResult};
use crate::util::request_header;

use crate::App;
use diesel::pg::Pg;
use diesel::query_builder::*;
use diesel::query_dsl::LoadQuery;
use diesel::sql_types::BigInt;
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use std::sync::Arc;

const MAX_PAGE_BEFORE_SUSPECTED_BOT: u32 = 10;
const DEFAULT_PER_PAGE: u32 = 10;
const MAX_PER_PAGE: u32 = 100;

#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Page {
    Numeric(u32),
    Seek(RawSeekPayload),
    Unspecified,
}

#[derive(Debug, Clone)]
pub(crate) struct PaginationOptions {
    pub(crate) page: Page,
    pub(crate) per_page: u32,
}

impl PaginationOptions {
    pub(crate) fn builder() -> PaginationOptionsBuilder {
        PaginationOptionsBuilder {
            limit_page_numbers: None,
            enable_seek: false,
        }
    }

    pub(crate) fn offset(&self) -> Option<u32> {
        if let Page::Numeric(p) = self.page {
            Some((p - 1) * self.per_page)
        } else {
            None
        }
    }
}

pub(crate) struct PaginationOptionsBuilder {
    limit_page_numbers: Option<Arc<App>>,
    enable_seek: bool,
}

impl PaginationOptionsBuilder {
    pub(crate) fn limit_page_numbers(mut self, app: Arc<App>) -> Self {
        self.limit_page_numbers = Some(app);
        self
    }

    pub(crate) fn enable_seek(mut self, enable: bool) -> Self {
        self.enable_seek = enable;
        self
    }

    pub(crate) fn gather(self, req: &mut dyn RequestExt) -> AppResult<PaginationOptions> {
        let params = req.query();
        let page_param = params.get("page");
        let seek_param = params.get("seek");

        if seek_param.is_some() && page_param.is_some() {
            return Err(bad_request(
                "providing both ?page= and ?seek= is unsupported",
            ));
        }

        let page = if let Some(s) = page_param {
            let numeric_page = s.parse().map_err(|e| bad_request(&e))?;
            if numeric_page < 1 {
                return Err(bad_request(&format_args!(
                    "page indexing starts from 1, page {} is invalid",
                    numeric_page,
                )));
            }

            if numeric_page > MAX_PAGE_BEFORE_SUSPECTED_BOT {
                req.log_metadata("bot", "suspected");
            }

            // Block large offsets for known violators of the crawler policy
            if let Some(app) = self.limit_page_numbers {
                let config = &app.config;
                let user_agent = request_header(req, header::USER_AGENT);
                if numeric_page > config.max_allowed_page_offset
                    && config
                        .page_offset_ua_blocklist
                        .iter()
                        .any(|blocked| user_agent.contains(blocked))
                {
                    add_custom_metadata(req, "cause", "large page offset");
                    return Err(bad_request("requested page offset is too large"));
                }
            }

            Page::Numeric(numeric_page)
        } else if let Some(s) = seek_param {
            if self.enable_seek {
                Page::Seek(RawSeekPayload(s.clone()))
            } else {
                return Err(bad_request("?seek= is not supported for this request"));
            }
        } else {
            Page::Unspecified
        };

        let per_page = params
            .get("per_page")
            .map(|s| s.parse().map_err(|e| bad_request(&e)))
            .unwrap_or(Ok(DEFAULT_PER_PAGE))?;
        if per_page > MAX_PER_PAGE {
            return Err(bad_request(&format_args!(
                "cannot request more than {} items",
                MAX_PER_PAGE,
            )));
        }

        Ok(PaginationOptions { page, per_page })
    }
}

pub(crate) trait Paginate: Sized {
    fn pages_pagination(self, options: PaginationOptions) -> PaginatedQuery<Self> {
        PaginatedQuery {
            query: self,
            options,
        }
    }
}

impl<T> Paginate for T {}

pub struct Paginated<T> {
    records_and_total: Vec<WithCount<T>>,
    options: PaginationOptions,
}

impl<T> Paginated<T> {
    pub(crate) fn total(&self) -> i64 {
        self.records_and_total
            .get(0)
            .map(|row| row.total)
            .unwrap_or_default() // If there is no first row, then the total is zero.
    }

    pub(crate) fn next_page_params(&self) -> Option<IndexMap<String, String>> {
        if self.records_and_total.len() < self.options.per_page as usize {
            return None;
        }

        let mut opts = IndexMap::new();
        match self.options.page {
            Page::Numeric(n) => opts.insert("page".into(), (n + 1).to_string()),
            Page::Unspecified => opts.insert("page".into(), 2.to_string()),
            Page::Seek(_) => return None,
        };
        Some(opts)
    }

    pub(crate) fn prev_page_params(&self) -> Option<IndexMap<String, String>> {
        let mut opts = IndexMap::new();
        match self.options.page {
            Page::Numeric(1) | Page::Unspecified | Page::Seek(_) => return None,
            Page::Numeric(n) => opts.insert("page".into(), (n - 1).to_string()),
        };
        Some(opts)
    }

    pub(crate) fn iter(&self) -> impl Iterator<Item = &T> {
        self.records_and_total.iter().map(|row| &row.record)
    }
}

impl<T: 'static> IntoIterator for Paginated<T> {
    type IntoIter = Box<dyn Iterator<Item = Self::Item>>;
    type Item = T;

    fn into_iter(self) -> Self::IntoIter {
        Box::new(self.records_and_total.into_iter().map(|row| row.record))
    }
}

#[derive(Debug)]
pub(crate) struct PaginatedQuery<T> {
    query: T,
    options: PaginationOptions,
}

impl<T> PaginatedQuery<T> {
    pub(crate) fn load<U>(self, conn: &PgConnection) -> QueryResult<Paginated<U>>
    where
        Self: LoadQuery<PgConnection, WithCount<U>>,
    {
        let options = self.options.clone();
        let records_and_total = self.internal_load(conn)?;
        Ok(Paginated {
            records_and_total,
            options,
        })
    }
}

impl<T> QueryId for PaginatedQuery<T> {
    const HAS_STATIC_QUERY_ID: bool = false;
    type QueryId = ();
}

impl<T: Query> Query for PaginatedQuery<T> {
    type SqlType = (T::SqlType, BigInt);
}

impl<T, DB> RunQueryDsl<DB> for PaginatedQuery<T> {}

impl<T> QueryFragment<Pg> for PaginatedQuery<T>
where
    T: QueryFragment<Pg>,
{
    fn walk_ast(&self, mut out: AstPass<'_, Pg>) -> QueryResult<()> {
        out.push_sql("SELECT *, COUNT(*) OVER () FROM (");
        self.query.walk_ast(out.reborrow())?;
        out.push_sql(") t LIMIT ");
        out.push_bind_param::<BigInt, _>(&i64::from(self.options.per_page))?;
        if let Some(offset) = self.options.offset() {
            out.push_sql(" OFFSET ");
            out.push_bind_param::<BigInt, _>(&i64::from(offset))?;
        }
        Ok(())
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RawSeekPayload(String);

impl RawSeekPayload {
    pub(crate) fn decode<D: for<'a> Deserialize<'a>>(&self) -> AppResult<D> {
        decode_seek(&self.0)
    }
}

/// Encode a payload to be used as a seek key.
///
/// The payload is base64-encoded to hint that it shouldn't be manually constructed. There is no
/// technical measure to prevent API consumers for manually creating or modifying them, but
/// hopefully the base64 will be enough to convey that doing it is unsupported.
pub(crate) fn encode_seek<S: Serialize>(params: S) -> AppResult<String> {
    Ok(base64::encode_config(
        &serde_json::to_vec(&params)?,
        base64::URL_SAFE_NO_PAD,
    ))
}

/// Decode a list of params previously encoded with [`encode_seek`].
pub(crate) fn decode_seek<D: for<'a> Deserialize<'a>>(seek: &str) -> AppResult<D> {
    Ok(serde_json::from_slice(&base64::decode_config(
        seek,
        base64::URL_SAFE_NO_PAD,
    )?)?)
}

#[cfg(test)]
mod tests {
    use super::*;
    use conduit::StatusCode;
    use conduit_test::{MockRequest, ResponseExt};

    #[test]
    fn no_pagination_param() {
        let pagination = PaginationOptions::builder().gather(&mut mock("")).unwrap();
        assert_eq!(Page::Unspecified, pagination.page);
        assert_eq!(DEFAULT_PER_PAGE, pagination.per_page);
    }

    #[test]
    fn page_param_parsing() {
        let assert_error =
            |query, msg| assert_pagination_error(PaginationOptions::builder(), query, msg);

        assert_error("page=", "cannot parse integer from empty string");
        assert_error("page=not_a_number", "invalid digit found in string");
        assert_error("page=1.0", "invalid digit found in string");
        assert_error("page=0", "page indexing starts from 1, page 0 is invalid");

        let pagination = PaginationOptions::builder()
            .gather(&mut mock("page=5"))
            .unwrap();
        assert_eq!(Page::Numeric(5), pagination.page);
    }

    #[test]
    fn per_page_param_parsing() {
        let assert_error =
            |query, msg| assert_pagination_error(PaginationOptions::builder(), query, msg);

        assert_error("per_page=", "cannot parse integer from empty string");
        assert_error("per_page=not_a_number", "invalid digit found in string");
        assert_error("per_page=1.0", "invalid digit found in string");
        assert_error("per_page=101", "cannot request more than 100 items");

        let pagination = PaginationOptions::builder()
            .gather(&mut mock("per_page=5"))
            .unwrap();
        assert_eq!(5, pagination.per_page);
    }

    #[test]
    fn seek_param_parsing() {
        assert_pagination_error(
            PaginationOptions::builder(),
            "seek=OTg",
            "?seek= is not supported for this request",
        );

        let pagination = PaginationOptions::builder()
            .enable_seek(true)
            .gather(&mut mock("seek=OTg"))
            .unwrap();

        if let Page::Seek(raw) = pagination.page {
            assert_eq!(98, raw.decode::<i32>().unwrap());
        } else {
            panic!(
                "did not parse a seek page, parsed {:?} instead",
                pagination.page
            );
        }
    }

    #[test]
    fn both_page_and_seek() {
        assert_pagination_error(
            PaginationOptions::builder(),
            "page=1&seek=OTg",
            "providing both ?page= and ?seek= is unsupported",
        );
        assert_pagination_error(
            PaginationOptions::builder().enable_seek(true),
            "page=1&seek=OTg",
            "providing both ?page= and ?seek= is unsupported",
        );
    }

    #[test]
    fn test_seek_encode_and_decode() {
        // Encoding produces the results we expect
        assert_eq!("OTg", encode_seek(98).unwrap());
        assert_eq!("WyJmb28iLDQyXQ", encode_seek(("foo", 42)).unwrap());

        // Encoded values can be then decoded.
        assert_eq!(98, decode_seek::<i32>(&encode_seek(98).unwrap()).unwrap(),);
        assert_eq!(
            ("foo".into(), 42),
            decode_seek::<(String, i32)>(&encode_seek(("foo", 42)).unwrap()).unwrap(),
        );
    }

    fn mock(query: &str) -> MockRequest {
        let mut req = MockRequest::new(http::Method::GET, "");
        req.with_query(query);
        req
    }

    fn assert_pagination_error(options: PaginationOptionsBuilder, query: &str, message: &str) {
        let response = options
            .gather(&mut mock(query))
            .unwrap_err()
            .response()
            .unwrap();

        assert_eq!(StatusCode::BAD_REQUEST, response.status());

        #[derive(Deserialize)]
        struct Parsed {
            errors: [ParsedError; 1],
        }
        #[derive(Deserialize)]
        struct ParsedError {
            detail: String,
        }

        let parsed: Parsed = serde_json::from_slice(&response.into_cow()).unwrap();
        assert_eq!(message, parsed.errors[0].detail);
    }
}
<commit_msg>helpers/pagination: Implement `enable_pages` flag<commit_after>use crate::controllers::prelude::*;
use crate::middleware::log_request::add_custom_metadata;
use crate::models::helpers::with_count::*;
use crate::util::errors::{bad_request, AppResult};
use crate::util::request_header;

use crate::App;
use diesel::pg::Pg;
use diesel::query_builder::*;
use diesel::query_dsl::LoadQuery;
use diesel::sql_types::BigInt;
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use std::sync::Arc;

const MAX_PAGE_BEFORE_SUSPECTED_BOT: u32 = 10;
const DEFAULT_PER_PAGE: u32 = 10;
const MAX_PER_PAGE: u32 = 100;

#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Page {
    Numeric(u32),
    Seek(RawSeekPayload),
    Unspecified,
}

#[derive(Debug, Clone)]
pub(crate) struct PaginationOptions {
    pub(crate) page: Page,
    pub(crate) per_page: u32,
}

impl PaginationOptions {
    pub(crate) fn builder() -> PaginationOptionsBuilder {
        PaginationOptionsBuilder {
            limit_page_numbers: None,
            enable_seek: false,
            enable_pages: true,
        }
    }

    pub(crate) fn offset(&self) -> Option<u32> {
        if let Page::Numeric(p) = self.page {
            Some((p - 1) * self.per_page)
        } else {
            None
        }
    }
}

pub(crate) struct PaginationOptionsBuilder {
    limit_page_numbers: Option<Arc<App>>,
    enable_pages: bool,
    enable_seek: bool,
}

impl PaginationOptionsBuilder {
    pub(crate) fn limit_page_numbers(mut self, app: Arc<App>) -> Self {
        self.limit_page_numbers = Some(app);
        self
    }

    pub(crate) fn enable_pages(mut self, enable: bool) -> Self {
        self.enable_pages = enable;
        self
    }

    pub(crate) fn enable_seek(mut self, enable: bool) -> Self {
        self.enable_seek = enable;
        self
    }

    pub(crate) fn gather(self, req: &mut dyn RequestExt) -> AppResult<PaginationOptions> {
        let params = req.query();
        let page_param = params.get("page");
        let seek_param = params.get("seek");

        if seek_param.is_some() && page_param.is_some() {
            return Err(bad_request(
                "providing both ?page= and ?seek= is unsupported",
            ));
        }

        let page = if let Some(s) = page_param {
            if self.enable_pages {
                let numeric_page = s.parse().map_err(|e| bad_request(&e))?;
                if numeric_page < 1 {
                    return Err(bad_request(&format_args!(
                        "page indexing starts from 1, page {} is invalid",
                        numeric_page,
                    )));
                }

                if numeric_page > MAX_PAGE_BEFORE_SUSPECTED_BOT {
                    req.log_metadata("bot", "suspected");
                }

                // Block large offsets for known violators of the crawler policy
                if let Some(app) = self.limit_page_numbers {
                    let config = &app.config;
                    let user_agent = request_header(req, header::USER_AGENT);
                    if numeric_page > config.max_allowed_page_offset
                        && config
                            .page_offset_ua_blocklist
                            .iter()
                            .any(|blocked| user_agent.contains(blocked))
                    {
                        add_custom_metadata(req, "cause", "large page offset");
                        return Err(bad_request("requested page offset is too large"));
                    }
                }

                Page::Numeric(numeric_page)
            } else {
                return Err(bad_request("?page= is not supported for this request"));
            }
        } else if let Some(s) = seek_param {
            if self.enable_seek {
                Page::Seek(RawSeekPayload(s.clone()))
            } else {
                return Err(bad_request("?seek= is not supported for this request"));
            }
        } else {
            Page::Unspecified
        };

        let per_page = params
            .get("per_page")
            .map(|s| s.parse().map_err(|e| bad_request(&e)))
            .unwrap_or(Ok(DEFAULT_PER_PAGE))?;
        if per_page > MAX_PER_PAGE {
            return Err(bad_request(&format_args!(
                "cannot request more than {} items",
                MAX_PER_PAGE,
            )));
        }

        Ok(PaginationOptions { page, per_page })
    }
}

pub(crate) trait Paginate: Sized {
    fn pages_pagination(self, options: PaginationOptions) -> PaginatedQuery<Self> {
        PaginatedQuery {
            query: self,
            options,
        }
    }
}

impl<T> Paginate for T {}

pub struct Paginated<T> {
    records_and_total: Vec<WithCount<T>>,
    options: PaginationOptions,
}

impl<T> Paginated<T> {
    pub(crate) fn total(&self) -> i64 {
        self.records_and_total
            .get(0)
            .map(|row| row.total)
            .unwrap_or_default() // If there is no first row, then the total is zero.
    }

    pub(crate) fn next_page_params(&self) -> Option<IndexMap<String, String>> {
        if self.records_and_total.len() < self.options.per_page as usize {
            return None;
        }

        let mut opts = IndexMap::new();
        match self.options.page {
            Page::Numeric(n) => opts.insert("page".into(), (n + 1).to_string()),
            Page::Unspecified => opts.insert("page".into(), 2.to_string()),
            Page::Seek(_) => return None,
        };
        Some(opts)
    }

    pub(crate) fn prev_page_params(&self) -> Option<IndexMap<String, String>> {
        let mut opts = IndexMap::new();
        match self.options.page {
            Page::Numeric(1) | Page::Unspecified | Page::Seek(_) => return None,
            Page::Numeric(n) => opts.insert("page".into(), (n - 1).to_string()),
        };
        Some(opts)
    }

    pub(crate) fn iter(&self) -> impl Iterator<Item = &T> {
        self.records_and_total.iter().map(|row| &row.record)
    }
}

impl<T: 'static> IntoIterator for Paginated<T> {
    type IntoIter = Box<dyn Iterator<Item = Self::Item>>;
    type Item = T;

    fn into_iter(self) -> Self::IntoIter {
        Box::new(self.records_and_total.into_iter().map(|row| row.record))
    }
}

#[derive(Debug)]
pub(crate) struct PaginatedQuery<T> {
    query: T,
    options: PaginationOptions,
}

impl<T> PaginatedQuery<T> {
    pub(crate) fn load<U>(self, conn: &PgConnection) -> QueryResult<Paginated<U>>
    where
        Self: LoadQuery<PgConnection, WithCount<U>>,
    {
        let options = self.options.clone();
        let records_and_total = self.internal_load(conn)?;
        Ok(Paginated {
            records_and_total,
            options,
        })
    }
}

impl<T> QueryId for PaginatedQuery<T> {
    const HAS_STATIC_QUERY_ID: bool = false;
    type QueryId = ();
}

impl<T: Query> Query for PaginatedQuery<T> {
    type SqlType = (T::SqlType, BigInt);
}

impl<T, DB> RunQueryDsl<DB> for PaginatedQuery<T> {}

impl<T> QueryFragment<Pg> for PaginatedQuery<T>
where
    T: QueryFragment<Pg>,
{
    fn walk_ast(&self, mut out: AstPass<'_, Pg>) -> QueryResult<()> {
        out.push_sql("SELECT *, COUNT(*) OVER () FROM (");
        self.query.walk_ast(out.reborrow())?;
        out.push_sql(") t LIMIT ");
        out.push_bind_param::<BigInt, _>(&i64::from(self.options.per_page))?;
        if let Some(offset) = self.options.offset() {
            out.push_sql(" OFFSET ");
            out.push_bind_param::<BigInt, _>(&i64::from(offset))?;
        }
        Ok(())
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RawSeekPayload(String);

impl RawSeekPayload {
    pub(crate) fn decode<D: for<'a> Deserialize<'a>>(&self) -> AppResult<D> {
        decode_seek(&self.0)
    }
}

/// Encode a payload to be used as a seek key.
///
/// The payload is base64-encoded to hint that it shouldn't be manually constructed. There is no
/// technical measure to prevent API consumers for manually creating or modifying them, but
/// hopefully the base64 will be enough to convey that doing it is unsupported.
pub(crate) fn encode_seek<S: Serialize>(params: S) -> AppResult<String> {
    Ok(base64::encode_config(
        &serde_json::to_vec(&params)?,
        base64::URL_SAFE_NO_PAD,
    ))
}

/// Decode a list of params previously encoded with [`encode_seek`].
pub(crate) fn decode_seek<D: for<'a> Deserialize<'a>>(seek: &str) -> AppResult<D> {
    Ok(serde_json::from_slice(&base64::decode_config(
        seek,
        base64::URL_SAFE_NO_PAD,
    )?)?)
}

#[cfg(test)]
mod tests {
    use super::*;
    use conduit::StatusCode;
    use conduit_test::{MockRequest, ResponseExt};

    #[test]
    fn no_pagination_param() {
        let pagination = PaginationOptions::builder().gather(&mut mock("")).unwrap();
        assert_eq!(Page::Unspecified, pagination.page);
        assert_eq!(DEFAULT_PER_PAGE, pagination.per_page);
    }

    #[test]
    fn page_param_parsing() {
        let assert_error =
            |query, msg| assert_pagination_error(PaginationOptions::builder(), query, msg);

        assert_error("page=", "cannot parse integer from empty string");
        assert_error("page=not_a_number", "invalid digit found in string");
        assert_error("page=1.0", "invalid digit found in string");
        assert_error("page=0", "page indexing starts from 1, page 0 is invalid");

        let pagination = PaginationOptions::builder()
            .gather(&mut mock("page=5"))
            .unwrap();
        assert_eq!(Page::Numeric(5), pagination.page);
    }

    #[test]
    fn per_page_param_parsing() {
        let assert_error =
            |query, msg| assert_pagination_error(PaginationOptions::builder(), query, msg);

        assert_error("per_page=", "cannot parse integer from empty string");
        assert_error("per_page=not_a_number", "invalid digit found in string");
        assert_error("per_page=1.0", "invalid digit found in string");
        assert_error("per_page=101", "cannot request more than 100 items");

        let pagination = PaginationOptions::builder()
            .gather(&mut mock("per_page=5"))
            .unwrap();
        assert_eq!(5, pagination.per_page);
    }

    #[test]
    fn seek_param_parsing() {
        assert_pagination_error(
            PaginationOptions::builder(),
            "seek=OTg",
            "?seek= is not supported for this request",
        );

        let pagination = PaginationOptions::builder()
            .enable_seek(true)
            .gather(&mut mock("seek=OTg"))
            .unwrap();

        if let Page::Seek(raw) = pagination.page {
            assert_eq!(98, raw.decode::<i32>().unwrap());
        } else {
            panic!(
                "did not parse a seek page, parsed {:?} instead",
                pagination.page
            );
        }
    }

    #[test]
    fn both_page_and_seek() {
        assert_pagination_error(
            PaginationOptions::builder(),
            "page=1&seek=OTg",
            "providing both ?page= and ?seek= is unsupported",
        );
        assert_pagination_error(
            PaginationOptions::builder().enable_seek(true),
            "page=1&seek=OTg",
            "providing both ?page= and ?seek= is unsupported",
        );
    }

    #[test]
    fn disabled_pages() {
        assert_pagination_error(
            PaginationOptions::builder().enable_pages(false),
            "page=1",
            "?page= is not supported for this request",
        );
    }

    #[test]
    fn test_seek_encode_and_decode() {
        // Encoding produces the results we expect
        assert_eq!("OTg", encode_seek(98).unwrap());
        assert_eq!("WyJmb28iLDQyXQ", encode_seek(("foo", 42)).unwrap());

        // Encoded values can be then decoded.
        assert_eq!(98, decode_seek::<i32>(&encode_seek(98).unwrap()).unwrap(),);
        assert_eq!(
            ("foo".into(), 42),
            decode_seek::<(String, i32)>(&encode_seek(("foo", 42)).unwrap()).unwrap(),
        );
    }

    fn mock(query: &str) -> MockRequest {
        let mut req = MockRequest::new(http::Method::GET, "");
        req.with_query(query);
        req
    }

    fn assert_pagination_error(options: PaginationOptionsBuilder, query: &str, message: &str) {
        let response = options
            .gather(&mut mock(query))
            .unwrap_err()
            .response()
            .unwrap();

        assert_eq!(StatusCode::BAD_REQUEST, response.status());

        #[derive(Deserialize)]
        struct Parsed {
            errors: [ParsedError; 1],
        }
        #[derive(Deserialize)]
        struct ParsedError {
            detail: String,
        }

        let parsed: Parsed = serde_json::from_slice(&response.into_cow()).unwrap();
        assert_eq!(message, parsed.errors[0].detail);
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;

use kite::{Document, TermRef};
use kite::schema::FieldRef;
use byteorder::{BigEndian, WriteBytesExt};

use key_builder::KeyBuilder;

#[derive(Debug)]
pub struct SegmentBuilder {
    current_doc: u16,
    pub term_dictionary: HashMap<Vec<u8>, TermRef>,
    current_term_ref: u32,
    pub term_directories: HashMap<(FieldRef, TermRef), Vec<u16>>,
    pub statistics: HashMap<Vec<u8>, i64>,
    pub stored_field_values: HashMap<(FieldRef, u16, Vec<u8>), Vec<u8>>,
}

#[derive(Debug)]
pub enum DocumentInsertError {
    /// Segment couldn't hold any more docs
    SegmentFull,
}

impl SegmentBuilder {
    pub fn new() -> SegmentBuilder {
        SegmentBuilder {
            current_doc: 0,
            term_dictionary: HashMap::new(),
            current_term_ref: 0,
            term_directories: HashMap::new(),
            statistics: HashMap::new(),
            stored_field_values: HashMap::new(),
        }
    }

    // TODO: Need to translate field names to field refs and terms to term refs
    pub fn add_document(&mut self, doc: Document) -> Result<(), DocumentInsertError> {
        // Get document ord
        let doc_id = self.current_doc;
        self.current_doc += 1;
        try!(self.current_doc.checked_add(1).ok_or(DocumentInsertError::SegmentFull));

        // Insert indexed fields
        let mut term_frequencies = HashMap::new();
        for (field, tokens) in doc.indexed_fields.iter() {
            let mut field_token_count = 0;

            for token in tokens.iter() {
                field_token_count += 1;

                // Get term ref
                let term_bytes = token.term.to_bytes();
                let mut current_term_ref = self.current_term_ref;
                let term_ref = self.term_dictionary.entry(term_bytes).or_insert_with(|| {
                    let term_ref = TermRef::new(current_term_ref);
                    current_term_ref += 1;
                    term_ref
                });
                self.current_term_ref = current_term_ref;

                // Term frequency
                let mut term_frequency = term_frequencies.entry(*term_ref).or_insert(0);
                *term_frequency += 1;

                // Write directory list
                self.term_directories.entry((*field, *term_ref)).or_insert_with(Vec::new).push(doc_id);
            }

            // Term frequencies
            for (term_ref, frequency) in term_frequencies.drain() {
                // Write term frequency
                // 1 is by far the most common frequency. At search time, we interpret a missing
                // key as meaning there is a term frequency of 1
                if frequency != 1 {
                    let mut value_type = vec![b't', b'f'];
                    value_type.extend(term_ref.ord().to_string().as_bytes());

                    let mut frequency_bytes: Vec<u8> = Vec::new();
                    frequency_bytes.write_i64::<BigEndian>(frequency).unwrap();

                    self.stored_field_values.insert((*field, doc_id, value_type), frequency_bytes);
                }

                // Increment term document frequency
                let stat_name = KeyBuilder::segment_stat_term_doc_frequency_stat_name(field.ord(), term_ref.ord());
                let mut stat = self.statistics.entry(stat_name).or_insert(0);
                *stat += 1;
            }

            // Field length
            // Used by the BM25 similarity model
            let length = ((field_token_count as f64).sqrt() - 1.0) * 3.0;
            let length = if length > 255.0 { 255.0 } else { length } as u8;
            if length != 0 {
                self.stored_field_values.insert((*field, doc_id, b"len".to_vec()), vec![length]);
            }

            // Increment total field docs
            {
                let stat_name = KeyBuilder::segment_stat_total_field_docs_stat_name(field.ord());
                let mut stat = self.statistics.entry(stat_name).or_insert(0);
                *stat += 1;
            }

            // Increment total field tokens
            {
                let stat_name = KeyBuilder::segment_stat_total_field_tokens_stat_name(field.ord());
                let mut stat = self.statistics.entry(stat_name).or_insert(0);
                *stat += field_token_count;
            }
        }

        // Insert stored fields
        for (field, value) in doc.stored_fields.iter() {
            self.stored_field_values.insert((*field, doc_id, b"val".to_vec()), value.to_bytes());
        }

        // Increment total docs
        {
            let mut stat = self.statistics.entry(b"total_docs".to_vec()).or_insert(0);
            *stat += 1;
        }

        Ok(())
    }
}
<commit_msg>SegmentBuilder.add_document now returns id<commit_after>use std::collections::HashMap;

use kite::{Document, TermRef};
use kite::schema::FieldRef;
use byteorder::{BigEndian, WriteBytesExt};

use key_builder::KeyBuilder;

#[derive(Debug)]
pub struct SegmentBuilder {
    current_doc: u16,
    pub term_dictionary: HashMap<Vec<u8>, TermRef>,
    current_term_ref: u32,
    pub term_directories: HashMap<(FieldRef, TermRef), Vec<u16>>,
    pub statistics: HashMap<Vec<u8>, i64>,
    pub stored_field_values: HashMap<(FieldRef, u16, Vec<u8>), Vec<u8>>,
}

#[derive(Debug)]
pub enum DocumentInsertError {
    /// Segment couldn't hold any more docs
    SegmentFull,
}

impl SegmentBuilder {
    pub fn new() -> SegmentBuilder {
        SegmentBuilder {
            current_doc: 0,
            term_dictionary: HashMap::new(),
            current_term_ref: 0,
            term_directories: HashMap::new(),
            statistics: HashMap::new(),
            stored_field_values: HashMap::new(),
        }
    }

    // TODO: Need to translate field names to field refs and terms to term refs
    pub fn add_document(&mut self, doc: Document) -> Result<u16, DocumentInsertError> {
        // Get document ord
        let doc_id = self.current_doc;
        self.current_doc += 1;
        try!(self.current_doc.checked_add(1).ok_or(DocumentInsertError::SegmentFull));

        // Insert indexed fields
        let mut term_frequencies = HashMap::new();
        for (field, tokens) in doc.indexed_fields.iter() {
            let mut field_token_count = 0;

            for token in tokens.iter() {
                field_token_count += 1;

                // Get term ref
                let term_bytes = token.term.to_bytes();
                let mut current_term_ref = self.current_term_ref;
                let term_ref = self.term_dictionary.entry(term_bytes).or_insert_with(|| {
                    let term_ref = TermRef::new(current_term_ref);
                    current_term_ref += 1;
                    term_ref
                });
                self.current_term_ref = current_term_ref;

                // Term frequency
                let mut term_frequency = term_frequencies.entry(*term_ref).or_insert(0);
                *term_frequency += 1;

                // Write directory list
                self.term_directories.entry((*field, *term_ref)).or_insert_with(Vec::new).push(doc_id);
            }

            // Term frequencies
            for (term_ref, frequency) in term_frequencies.drain() {
                // Write term frequency
                // 1 is by far the most common frequency. At search time, we interpret a missing
                // key as meaning there is a term frequency of 1
                if frequency != 1 {
                    let mut value_type = vec![b't', b'f'];
                    value_type.extend(term_ref.ord().to_string().as_bytes());

                    let mut frequency_bytes: Vec<u8> = Vec::new();
                    frequency_bytes.write_i64::<BigEndian>(frequency).unwrap();

                    self.stored_field_values.insert((*field, doc_id, value_type), frequency_bytes);
                }

                // Increment term document frequency
                let stat_name = KeyBuilder::segment_stat_term_doc_frequency_stat_name(field.ord(), term_ref.ord());
                let mut stat = self.statistics.entry(stat_name).or_insert(0);
                *stat += 1;
            }

            // Field length
            // Used by the BM25 similarity model
            let length = ((field_token_count as f64).sqrt() - 1.0) * 3.0;
            let length = if length > 255.0 { 255.0 } else { length } as u8;
            if length != 0 {
                self.stored_field_values.insert((*field, doc_id, b"len".to_vec()), vec![length]);
            }

            // Increment total field docs
            {
                let stat_name = KeyBuilder::segment_stat_total_field_docs_stat_name(field.ord());
                let mut stat = self.statistics.entry(stat_name).or_insert(0);
                *stat += 1;
            }

            // Increment total field tokens
            {
                let stat_name = KeyBuilder::segment_stat_total_field_tokens_stat_name(field.ord());
                let mut stat = self.statistics.entry(stat_name).or_insert(0);
                *stat += field_token_count;
            }
        }

        // Insert stored fields
        for (field, value) in doc.stored_fields.iter() {
            self.stored_field_values.insert((*field, doc_id, b"val".to_vec()), value.to_bytes());
        }

        // Increment total docs
        {
            let mut stat = self.statistics.entry(b"total_docs".to_vec()).or_insert(0);
            *stat += 1;
        }

        Ok(doc_id)
    }
}
<|endoftext|>