<commit_before>//! # Libinput bindings for rust
//!
//! These bindings closely follow libinput's concepts and it's original API.
//! Please refer to the [libinput documentation](https://wayland.freedesktop.org/libinput/doc/latest/)
//! to understand the general structure and concepts.
//!
//! ## Differences to the C-Library:
//!
//! - Refcounting does not need to be done manually. Just call `clone` when you need an additional reference.
//! - Libinput logging cannot (currently) not be customized.
//!
//! ## Userdata handling
//!
//! Multiple types in the libinput library allow to attach a pointer of an arbitrary type, so called `userdata`.
//! Using this data is unsafe as there is no way to find out what type is stored in the libinput struct.
//! Additionally multiple references to the same libinput object may exist and userdata may be shared mutably.
//!
//! This is why using and setting userdata is an unsafe operation (except when creating an object).
//!
//! If you heavily rely on userdata, you should always stored them wrapped in a `Mutex` and use the same
//! type for every userdata access to further simplify usage.
//!
//! You need to be especially cautious when initializing libinput types from raw pointers, you obtained
//! from other libraries which may set their own userdata. If accessing their userdata make sure no shared
//! mutable access may happen and don't store something else instead, if the library does not explicitly
//! allow this.
//!
//! Generally usage of this api is error-prone and discouraged if not needed.
//!
//! ## Getting started
//! To get started check out the [`Libinput` struct](./struct.Libinput.html).
//!
//! Here's a small example that prints all events:
//! ```
//! extern crate input;
//! use input::{Libinput, LibinputInterface};
//! use std::fs::{File, OpenOptions};
//! use std::os::unix::fs::OpenOptionsExt;
//! use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd};
//! use std::path::Path;
//!
//! extern crate libc;
//! use libc::{O_RDONLY, O_RDWR, O_WRONLY};
//!
//! struct Interface;
//!
//! impl LibinputInterface for Interface {
//! 	fn open_restricted(&mut self, path: &Path, flags: i32) -> Result<RawFd, i32> {
//! 		OpenOptions::new()
//! 		    .custom_flags(flags)
//! 		    .read((flags & O_RDONLY != 0) | (flags & O_RDWR != 0))
//! 		    .write((flags & O_WRONLY != 0) | (flags & O_RDWR != 0))
//! 			.open(path)
//! 			.map(|file| file.into_raw_fd())
//! 			.map_err(|err| err.raw_os_error().unwrap())
//! 	}
//! 	fn close_restricted(&mut self, fd: RawFd) {
//! 		unsafe {
//! 			File::from_raw_fd(fd);
//! 		}
//! 	}
//! }
//!
//! fn main() {
//! 	let mut input = Libinput::new_with_udev(Interfacep());
//! 	input.udev_assign_seat("seat0").unwrap();
//! 	loop {
//! 		input.dispatch().unwrap();
//! 		for event in &mut input {
//! 			println!("Got event: {:?}", event);
//! 		}
//! 	}
//! }
//! ```

#![deny(missing_docs)]

extern crate input_sys;
extern crate libc;

#[macro_use]
extern crate bitflags;

#[cfg(feature = "udev")]
extern crate udev;

/// Unsafe c-api.
pub mod ffi {
    pub use input_sys::*;
}

/// Trait for types that allow to optain the underlying raw libinput pointer.
pub trait AsRaw<T> {
    /// Receive a raw pointer representing this type.
    fn as_raw(&self) -> *const T;

    #[doc(hidden)]
    fn as_raw_mut(&self) -> *mut T {
        self.as_raw() as *mut _
    }
}

/// Trait to receive the underlying context
pub trait Context {
    /// Returns the underlying libinput context
    fn context(&self) -> &Libinput;
}

/// Trait for types that allow to be initialized from a raw pointer
pub trait FromRaw<T> {
    /// Create a new instance of this type from a raw pointer and it's context.
    ///
    /// ## Warning
    ///
    /// If you make use of [`Userdata`](./trait.Userdata.html) make sure you use the correct types
    /// to allow receiving the set userdata. When dealing with raw pointers initialized by other
    /// libraries this must be done extra carefully to select a correct representation.
    ///
    /// If unsure using `()` is always a safe option..
    ///
    /// ## Unsafety
    ///
    /// If the pointer is pointing to a different struct, invalid memory or `NULL` the returned
    /// struct may panic on use or cause other undefined behavior.
    unsafe fn from_raw(ffi: *mut T, context: &context::Libinput) -> Self;
}

macro_rules! ffi_ref_struct {
    ($(#[$attr:meta])* struct $struct_name:ident, $ffi_name:path, $ref_fn:path, $unref_fn:path) => (
        #[derive(Eq)]
        $(#[$attr])*
        pub struct $struct_name
        {
            ffi: *mut $ffi_name,
            context: $crate::context::Libinput,
        }

        impl ::std::fmt::Debug for $struct_name {
            fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
                write!(f, "$struct_name @{:p}", self.as_raw())
            }
        }

        impl $crate::FromRaw<$ffi_name> for $struct_name
        {
            unsafe fn from_raw(ffi: *mut $ffi_name, context: &$crate::Libinput) -> Self {
                $struct_name {
                    ffi: $ref_fn(ffi),
                    context: context.clone(),
                }
            }
        }

        impl $crate::AsRaw<$ffi_name> for $struct_name
        {
            fn as_raw(&self) -> *const $ffi_name {
                self.ffi as *const _
            }
        }

        impl $crate::Context for $struct_name
        {
            fn context(&self) -> &$crate::Libinput {
                &self.context
            }
         }

        impl Clone for $struct_name {
            fn clone(&self) -> Self {
                unsafe { $struct_name::from_raw(self.as_raw_mut(), &self.context) }
            }
        }

        impl Drop for $struct_name
        {
            fn drop(&mut self) {
                unsafe {
                    $unref_fn(self.ffi);
                }
            }
        }

        impl PartialEq for $struct_name {
            fn eq(&self, other: &Self) -> bool {
                self.as_raw() == other.as_raw()
            }
        }

        impl ::std::hash::Hash for $struct_name {
            fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
                self.as_raw().hash(state);
            }
        }
    )
}

macro_rules! ffi_func {
    ($(#[$attr:meta])* fn $name:ident, $ffi_fn:path, bool) => (
        $(#[$attr])*
        fn $name(&self) -> bool {
            unsafe { $ffi_fn(self.as_raw_mut()) != 0 }
        }
    );
    ($(#[$attr:meta])* pub fn $name:ident, $ffi_fn:path, bool) => (
        $(#[$attr])*
        pub fn $name(&self) -> bool {
            unsafe { $ffi_fn(self.as_raw_mut()) != 0 }
        }
    );
    ($(#[$attr:meta])* fn $name:ident, $ffi_fn:path, $return_type:ty) => (
        $(#[$attr])*
        fn $name(&self) -> $return_type {
            unsafe { $ffi_fn(self.as_raw_mut()) as $return_type }
        }
    );
    ($(#[$attr:meta])* pub fn $name:ident, $ffi_fn:path, $return_type:ty) => (
        $(#[$attr])*
        pub fn $name(&self) -> $return_type {
            unsafe { $ffi_fn(self.as_raw_mut()) as $return_type }
        }
    );
}

mod context;
mod device;
pub mod event;
mod seat;

pub use context::*;
pub use device::*;
pub use event::Event;
pub use seat::*;
<commit_msg>Fix typo in documentation example<commit_after>//! # Libinput bindings for rust
//!
//! These bindings closely follow libinput's concepts and it's original API.
//! Please refer to the [libinput documentation](https://wayland.freedesktop.org/libinput/doc/latest/)
//! to understand the general structure and concepts.
//!
//! ## Differences to the C-Library:
//!
//! - Refcounting does not need to be done manually. Just call `clone` when you need an additional reference.
//! - Libinput logging cannot (currently) not be customized.
//!
//! ## Userdata handling
//!
//! Multiple types in the libinput library allow to attach a pointer of an arbitrary type, so called `userdata`.
//! Using this data is unsafe as there is no way to find out what type is stored in the libinput struct.
//! Additionally multiple references to the same libinput object may exist and userdata may be shared mutably.
//!
//! This is why using and setting userdata is an unsafe operation (except when creating an object).
//!
//! If you heavily rely on userdata, you should always stored them wrapped in a `Mutex` and use the same
//! type for every userdata access to further simplify usage.
//!
//! You need to be especially cautious when initializing libinput types from raw pointers, you obtained
//! from other libraries which may set their own userdata. If accessing their userdata make sure no shared
//! mutable access may happen and don't store something else instead, if the library does not explicitly
//! allow this.
//!
//! Generally usage of this api is error-prone and discouraged if not needed.
//!
//! ## Getting started
//! To get started check out the [`Libinput` struct](./struct.Libinput.html).
//!
//! Here's a small example that prints all events:
//! ```
//! extern crate input;
//! use input::{Libinput, LibinputInterface};
//! use std::fs::{File, OpenOptions};
//! use std::os::unix::fs::OpenOptionsExt;
//! use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd};
//! use std::path::Path;
//!
//! extern crate libc;
//! use libc::{O_RDONLY, O_RDWR, O_WRONLY};
//!
//! struct Interface;
//!
//! impl LibinputInterface for Interface {
//! 	fn open_restricted(&mut self, path: &Path, flags: i32) -> Result<RawFd, i32> {
//! 		OpenOptions::new()
//! 		    .custom_flags(flags)
//! 		    .read((flags & O_RDONLY != 0) | (flags & O_RDWR != 0))
//! 		    .write((flags & O_WRONLY != 0) | (flags & O_RDWR != 0))
//! 			.open(path)
//! 			.map(|file| file.into_raw_fd())
//! 			.map_err(|err| err.raw_os_error().unwrap())
//! 	}
//! 	fn close_restricted(&mut self, fd: RawFd) {
//! 		unsafe {
//! 			File::from_raw_fd(fd);
//! 		}
//! 	}
//! }
//!
//! fn main() {
//! 	let mut input = Libinput::new_with_udev(Interface{});
//! 	input.udev_assign_seat("seat0").unwrap();
//! 	loop {
//! 		input.dispatch().unwrap();
//! 		for event in &mut input {
//! 			println!("Got event: {:?}", event);
//! 		}
//! 	}
//! }
//! ```

#![deny(missing_docs)]

extern crate input_sys;
extern crate libc;

#[macro_use]
extern crate bitflags;

#[cfg(feature = "udev")]
extern crate udev;

/// Unsafe c-api.
pub mod ffi {
    pub use input_sys::*;
}

/// Trait for types that allow to optain the underlying raw libinput pointer.
pub trait AsRaw<T> {
    /// Receive a raw pointer representing this type.
    fn as_raw(&self) -> *const T;

    #[doc(hidden)]
    fn as_raw_mut(&self) -> *mut T {
        self.as_raw() as *mut _
    }
}

/// Trait to receive the underlying context
pub trait Context {
    /// Returns the underlying libinput context
    fn context(&self) -> &Libinput;
}

/// Trait for types that allow to be initialized from a raw pointer
pub trait FromRaw<T> {
    /// Create a new instance of this type from a raw pointer and it's context.
    ///
    /// ## Warning
    ///
    /// If you make use of [`Userdata`](./trait.Userdata.html) make sure you use the correct types
    /// to allow receiving the set userdata. When dealing with raw pointers initialized by other
    /// libraries this must be done extra carefully to select a correct representation.
    ///
    /// If unsure using `()` is always a safe option..
    ///
    /// ## Unsafety
    ///
    /// If the pointer is pointing to a different struct, invalid memory or `NULL` the returned
    /// struct may panic on use or cause other undefined behavior.
    unsafe fn from_raw(ffi: *mut T, context: &context::Libinput) -> Self;
}

macro_rules! ffi_ref_struct {
    ($(#[$attr:meta])* struct $struct_name:ident, $ffi_name:path, $ref_fn:path, $unref_fn:path) => (
        #[derive(Eq)]
        $(#[$attr])*
        pub struct $struct_name
        {
            ffi: *mut $ffi_name,
            context: $crate::context::Libinput,
        }

        impl ::std::fmt::Debug for $struct_name {
            fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
                write!(f, "$struct_name @{:p}", self.as_raw())
            }
        }

        impl $crate::FromRaw<$ffi_name> for $struct_name
        {
            unsafe fn from_raw(ffi: *mut $ffi_name, context: &$crate::Libinput) -> Self {
                $struct_name {
                    ffi: $ref_fn(ffi),
                    context: context.clone(),
                }
            }
        }

        impl $crate::AsRaw<$ffi_name> for $struct_name
        {
            fn as_raw(&self) -> *const $ffi_name {
                self.ffi as *const _
            }
        }

        impl $crate::Context for $struct_name
        {
            fn context(&self) -> &$crate::Libinput {
                &self.context
            }
         }

        impl Clone for $struct_name {
            fn clone(&self) -> Self {
                unsafe { $struct_name::from_raw(self.as_raw_mut(), &self.context) }
            }
        }

        impl Drop for $struct_name
        {
            fn drop(&mut self) {
                unsafe {
                    $unref_fn(self.ffi);
                }
            }
        }

        impl PartialEq for $struct_name {
            fn eq(&self, other: &Self) -> bool {
                self.as_raw() == other.as_raw()
            }
        }

        impl ::std::hash::Hash for $struct_name {
            fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
                self.as_raw().hash(state);
            }
        }
    )
}

macro_rules! ffi_func {
    ($(#[$attr:meta])* fn $name:ident, $ffi_fn:path, bool) => (
        $(#[$attr])*
        fn $name(&self) -> bool {
            unsafe { $ffi_fn(self.as_raw_mut()) != 0 }
        }
    );
    ($(#[$attr:meta])* pub fn $name:ident, $ffi_fn:path, bool) => (
        $(#[$attr])*
        pub fn $name(&self) -> bool {
            unsafe { $ffi_fn(self.as_raw_mut()) != 0 }
        }
    );
    ($(#[$attr:meta])* fn $name:ident, $ffi_fn:path, $return_type:ty) => (
        $(#[$attr])*
        fn $name(&self) -> $return_type {
            unsafe { $ffi_fn(self.as_raw_mut()) as $return_type }
        }
    );
    ($(#[$attr:meta])* pub fn $name:ident, $ffi_fn:path, $return_type:ty) => (
        $(#[$attr])*
        pub fn $name(&self) -> $return_type {
            unsafe { $ffi_fn(self.as_raw_mut()) as $return_type }
        }
    );
}

mod context;
mod device;
pub mod event;
mod seat;

pub use context::*;
pub use device::*;
pub use event::Event;
pub use seat::*;
<|endoftext|>


<commit_before>extern crate ogg_sys;
extern crate vorbis_sys;
extern crate vorbisfile_sys;
extern crate libc;

use std::io::{self, Read, Seek};

/// Allows you to decode a sound file stream into packets.
pub struct Decoder<R> where R: Read + Seek {
    // further informations are boxed so that a pointer can be passed to callbacks
    data: Box<DecoderData<R>>,
}

///
pub struct PacketsIter<'a, R: 'a + Read + Seek>(&'a mut DecoderData<R>);

/// Errors that can happen while decoding
#[derive(Debug)]
pub enum VorbisError {
    ReadError(io::Error),
    NotVorbis,
    VersionMismatch,
    BadHeader,
    InitialFileHeadersCorrupt,
    Hole,
}

impl std::error::Error for VorbisError {
    fn description(&self) -> &str {
        match self {
            &VorbisError::ReadError(_) => "A read from media returned an error",
            &VorbisError::NotVorbis => "Bitstream does not contain any Vorbis data",
            &VorbisError::VersionMismatch => "Vorbis version mismatch",
            &VorbisError::BadHeader => "Invalid Vorbis bitstream header",
            &VorbisError::InitialFileHeadersCorrupt => "Initial file headers are corrupt",
            &VorbisError::Hole => "Interruption of data",
        }
    }

    fn cause(&self) -> Option<&std::error::Error> {
        match self {
            &VorbisError::ReadError(ref err) => Some(err as &std::error::Error),
            _ => None
        }
    }
}

impl std::fmt::Display for VorbisError {
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
        write!(fmt, "{}", std::error::Error::description(self))
    }
}

impl From<io::Error> for VorbisError {
    fn from(err: io::Error) -> VorbisError {
        VorbisError::ReadError(err)
    }
}

struct DecoderData<R> where R: Read + Seek {
    vorbis: vorbisfile_sys::OggVorbis_File,
    reader: R,
    current_logical_bitstream: libc::c_int,
    read_error: Option<io::Error>,
}

unsafe impl<R: Read + Seek + Send> Send for DecoderData<R> {}

/// Packet of data.
///
/// Each sample is an `i16` ranging from I16_MIN to I16_MAX.
///
/// The channels are interleaved in the data. For example if you have two channels, you will
/// get a sample from channel 1, then a sample from channel 2, than a sample from channel 1, etc.
#[derive(Clone, Debug)]
pub struct Packet {
    pub data: Vec<i16>,
    pub channels: u16,
    pub rate: u64,
    pub bitrate_upper: u64,
    pub bitrate_nominal: u64,
    pub bitrate_lower: u64,
    pub bitrate_window: u64,
}

impl<R> Decoder<R> where R: Read + Seek {
    pub fn new(input: R) -> Result<Decoder<R>, VorbisError> {
        extern fn read_func<R>(ptr: *mut libc::c_void, size: libc::size_t, nmemb: libc::size_t,
            datasource: *mut libc::c_void) -> libc::size_t where R: Read + Seek
        {
            use std::slice;

            let mut ptr = ptr as *mut u8;

            let data: &mut DecoderData<R> = unsafe { std::mem::transmute(datasource) };

            loop {
                let buffer = ptr.clone();
                let buffer = unsafe { slice::from_raw_parts_mut(buffer, size as usize * nmemb as usize) };

                match data.reader.read(buffer) {
                    Ok(0) => return 0,
                    Ok(nb) => {
                        if buffer.len() == nb {
                            return nmemb;
                        } else {
                            unsafe { ptr = ptr.offset(nb as isize) };
                        }
                    },
                    Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {
                        return 0
                    },
                    Err(e) => {
                        data.read_error = Some(e);
                        return 0;
                    }
                };
            }
        }

        extern fn seek_func<R>(datasource: *mut libc::c_void, offset: ogg_sys::ogg_int64_t,
            whence: libc::c_int) -> libc::c_int where R: Read + Seek
        {
            let data: &mut DecoderData<R> = unsafe { std::mem::transmute(datasource) };

            let result = match whence {
                libc::SEEK_SET => data.reader.seek(io::SeekFrom::Start(offset as u64)),
                libc::SEEK_CUR => data.reader.seek(io::SeekFrom::Current(offset)),
                libc::SEEK_END => data.reader.seek(io::SeekFrom::End(offset)),
                _ => unreachable!()
            };

            match result {
                Ok(_) => 0,
                Err(_) => -1
            }
        }

        extern fn tell_func<R>(datasource: *mut libc::c_void) -> libc::c_long
            where R: Read + Seek
        {
            let data: &mut DecoderData<R> = unsafe { std::mem::transmute(datasource) };
            data.reader.seek(io::SeekFrom::Current(0)).map(|v| v as libc::c_long).unwrap_or(-1)
        }

        let callbacks = {
            let mut callbacks: vorbisfile_sys::ov_callbacks = unsafe { std::mem::zeroed() };
            callbacks.read_func = read_func::<R>;
            callbacks.seek_func = seek_func::<R>;
            callbacks.tell_func = tell_func::<R>;
            callbacks
        };

        let mut data = Box::new(DecoderData {
            vorbis: unsafe { std::mem::uninitialized() },
            reader: input,
            current_logical_bitstream: 0,
            read_error: None,
        });

        // initializing
        unsafe {
            let data_ptr = &mut *data as *mut DecoderData<R>;
            let data_ptr = data_ptr as *mut libc::c_void;
            try!(check_errors(vorbisfile_sys::ov_open_callbacks(data_ptr, &mut data.vorbis,
                std::ptr::null(), 0, callbacks)));
        }

        Ok(Decoder {
            data: data,
        })
    }

    pub fn time_seek(&mut self, s: f64) -> Result<(), VorbisError> {
        unsafe {
            check_errors(vorbisfile_sys::ov_time_seek(&mut self.data.vorbis, s))
        }
    }

    pub fn packets(&mut self) -> PacketsIter<R> {
        PacketsIter(&mut *self.data)
    }
}

impl<'a, R> Iterator for PacketsIter<'a, R> where R: 'a + Read + Seek {
    type Item = Result<Packet, VorbisError>;

    fn next(&mut self) -> Option<Result<Packet, VorbisError>> {
        let mut buffer = std::iter::repeat(0i16).take(2048).collect::<Vec<_>>();
        let buffer_len = buffer.len() * 2;

        match unsafe {
            vorbisfile_sys::ov_read(&mut self.0.vorbis, buffer.as_mut_ptr() as *mut i8,
                buffer_len as libc::c_int, 0, 2, 1, &mut self.0.current_logical_bitstream)
        } {
            0 => {
                match self.0.read_error.take() {
                    Some(err) => Some(Err(VorbisError::ReadError(err))),
                    None => None,
                }
            },

            err if err < 0 => {
                match check_errors(err as libc::c_int) {
                    Err(e) => Some(Err(e)),
                    Ok(_) => unreachable!()
                }
            },

            len => {
                buffer.truncate(len as usize / 2);

                let infos = unsafe { vorbisfile_sys::ov_info(&mut self.0.vorbis,
                    self.0.current_logical_bitstream) };

                let infos: &vorbis_sys::vorbis_info = unsafe { std::mem::transmute(infos) };

                Some(Ok(Packet {
                    data: buffer,
                    channels: infos.channels as u16,
                    rate: infos.rate as u64,
                    bitrate_upper: infos.bitrate_upper as u64,
                    bitrate_nominal: infos.bitrate_nominal as u64,
                    bitrate_lower: infos.bitrate_lower as u64,
                    bitrate_window: infos.bitrate_window as u64,
                }))
            }
        }
    }
}

impl<R> Drop for Decoder<R> where R: Read + Seek {
    fn drop(&mut self) {
        unsafe {
            vorbisfile_sys::ov_clear(&mut self.data.vorbis);
        }
    }
}

fn check_errors(code: libc::c_int) -> Result<(), VorbisError> {
    match code {
        0 => Ok(()),

        vorbis_sys::OV_ENOTVORBIS => Err(VorbisError::NotVorbis),
        vorbis_sys::OV_EVERSION => Err(VorbisError::VersionMismatch),
        vorbis_sys::OV_EBADHEADER => Err(VorbisError::BadHeader),
        vorbis_sys::OV_EINVAL => Err(VorbisError::InitialFileHeadersCorrupt),
        vorbis_sys::OV_HOLE => Err(VorbisError::Hole),

        vorbis_sys::OV_EREAD => unimplemented!(),

        // indicates a bug or heap/stack corruption
        vorbis_sys::OV_EFAULT => panic!("Internal libvorbis error"),
        _ => panic!("Unknown vorbis error")
    }
}
<commit_msg>Add support for tell<commit_after>extern crate ogg_sys;
extern crate vorbis_sys;
extern crate vorbisfile_sys;
extern crate libc;

use std::io::{self, Read, Seek};

/// Allows you to decode a sound file stream into packets.
pub struct Decoder<R> where R: Read + Seek {
    // further informations are boxed so that a pointer can be passed to callbacks
    data: Box<DecoderData<R>>,
}

///
pub struct PacketsIter<'a, R: 'a + Read + Seek>(&'a mut DecoderData<R>);

/// Errors that can happen while decoding
#[derive(Debug)]
pub enum VorbisError {
    ReadError(io::Error),
    NotVorbis,
    VersionMismatch,
    BadHeader,
    InitialFileHeadersCorrupt,
    Hole,
}

impl std::error::Error for VorbisError {
    fn description(&self) -> &str {
        match self {
            &VorbisError::ReadError(_) => "A read from media returned an error",
            &VorbisError::NotVorbis => "Bitstream does not contain any Vorbis data",
            &VorbisError::VersionMismatch => "Vorbis version mismatch",
            &VorbisError::BadHeader => "Invalid Vorbis bitstream header",
            &VorbisError::InitialFileHeadersCorrupt => "Initial file headers are corrupt",
            &VorbisError::Hole => "Interruption of data",
        }
    }

    fn cause(&self) -> Option<&std::error::Error> {
        match self {
            &VorbisError::ReadError(ref err) => Some(err as &std::error::Error),
            _ => None
        }
    }
}

impl std::fmt::Display for VorbisError {
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
        write!(fmt, "{}", std::error::Error::description(self))
    }
}

impl From<io::Error> for VorbisError {
    fn from(err: io::Error) -> VorbisError {
        VorbisError::ReadError(err)
    }
}

struct DecoderData<R> where R: Read + Seek {
    vorbis: vorbisfile_sys::OggVorbis_File,
    reader: R,
    current_logical_bitstream: libc::c_int,
    read_error: Option<io::Error>,
}

unsafe impl<R: Read + Seek + Send> Send for DecoderData<R> {}

/// Packet of data.
///
/// Each sample is an `i16` ranging from I16_MIN to I16_MAX.
///
/// The channels are interleaved in the data. For example if you have two channels, you will
/// get a sample from channel 1, then a sample from channel 2, than a sample from channel 1, etc.
#[derive(Clone, Debug)]
pub struct Packet {
    pub data: Vec<i16>,
    pub channels: u16,
    pub rate: u64,
    pub bitrate_upper: u64,
    pub bitrate_nominal: u64,
    pub bitrate_lower: u64,
    pub bitrate_window: u64,
}

impl<R> Decoder<R> where R: Read + Seek {
    pub fn new(input: R) -> Result<Decoder<R>, VorbisError> {
        extern fn read_func<R>(ptr: *mut libc::c_void, size: libc::size_t, nmemb: libc::size_t,
            datasource: *mut libc::c_void) -> libc::size_t where R: Read + Seek
        {
            use std::slice;

            let mut ptr = ptr as *mut u8;

            let data: &mut DecoderData<R> = unsafe { std::mem::transmute(datasource) };

            loop {
                let buffer = ptr.clone();
                let buffer = unsafe { slice::from_raw_parts_mut(buffer, size as usize * nmemb as usize) };

                match data.reader.read(buffer) {
                    Ok(0) => return 0,
                    Ok(nb) => {
                        if buffer.len() == nb {
                            return nmemb;
                        } else {
                            unsafe { ptr = ptr.offset(nb as isize) };
                        }
                    },
                    Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {
                        return 0
                    },
                    Err(e) => {
                        data.read_error = Some(e);
                        return 0;
                    }
                };
            }
        }

        extern fn seek_func<R>(datasource: *mut libc::c_void, offset: ogg_sys::ogg_int64_t,
            whence: libc::c_int) -> libc::c_int where R: Read + Seek
        {
            let data: &mut DecoderData<R> = unsafe { std::mem::transmute(datasource) };

            let result = match whence {
                libc::SEEK_SET => data.reader.seek(io::SeekFrom::Start(offset as u64)),
                libc::SEEK_CUR => data.reader.seek(io::SeekFrom::Current(offset)),
                libc::SEEK_END => data.reader.seek(io::SeekFrom::End(offset)),
                _ => unreachable!()
            };

            match result {
                Ok(_) => 0,
                Err(_) => -1
            }
        }

        extern fn tell_func<R>(datasource: *mut libc::c_void) -> libc::c_long
            where R: Read + Seek
        {
            let data: &mut DecoderData<R> = unsafe { std::mem::transmute(datasource) };
            data.reader.seek(io::SeekFrom::Current(0)).map(|v| v as libc::c_long).unwrap_or(-1)
        }

        let callbacks = {
            let mut callbacks: vorbisfile_sys::ov_callbacks = unsafe { std::mem::zeroed() };
            callbacks.read_func = read_func::<R>;
            callbacks.seek_func = seek_func::<R>;
            callbacks.tell_func = tell_func::<R>;
            callbacks
        };

        let mut data = Box::new(DecoderData {
            vorbis: unsafe { std::mem::uninitialized() },
            reader: input,
            current_logical_bitstream: 0,
            read_error: None,
        });

        // initializing
        unsafe {
            let data_ptr = &mut *data as *mut DecoderData<R>;
            let data_ptr = data_ptr as *mut libc::c_void;
            try!(check_errors(vorbisfile_sys::ov_open_callbacks(data_ptr, &mut data.vorbis,
                std::ptr::null(), 0, callbacks)));
        }

        Ok(Decoder {
            data: data,
        })
    }

    pub fn time_seek(&mut self, s: f64) -> Result<(), VorbisError> {
        unsafe {
            check_errors(vorbisfile_sys::ov_time_seek(&mut self.data.vorbis, s))
        }
    }

    pub fn time_tell(&mut self) -> Result<f64, VorbisError> {
        unsafe {
            Ok(vorbisfile_sys::ov_time_tell(&mut self.data.vorbis))
        }
    }

    pub fn packets(&mut self) -> PacketsIter<R> {
        PacketsIter(&mut *self.data)
    }
}

impl<'a, R> Iterator for PacketsIter<'a, R> where R: 'a + Read + Seek {
    type Item = Result<Packet, VorbisError>;

    fn next(&mut self) -> Option<Result<Packet, VorbisError>> {
        let mut buffer = std::iter::repeat(0i16).take(2048).collect::<Vec<_>>();
        let buffer_len = buffer.len() * 2;

        match unsafe {
            vorbisfile_sys::ov_read(&mut self.0.vorbis, buffer.as_mut_ptr() as *mut i8,
                buffer_len as libc::c_int, 0, 2, 1, &mut self.0.current_logical_bitstream)
        } {
            0 => {
                match self.0.read_error.take() {
                    Some(err) => Some(Err(VorbisError::ReadError(err))),
                    None => None,
                }
            },

            err if err < 0 => {
                match check_errors(err as libc::c_int) {
                    Err(e) => Some(Err(e)),
                    Ok(_) => unreachable!()
                }
            },

            len => {
                buffer.truncate(len as usize / 2);

                let infos = unsafe { vorbisfile_sys::ov_info(&mut self.0.vorbis,
                    self.0.current_logical_bitstream) };

                let infos: &vorbis_sys::vorbis_info = unsafe { std::mem::transmute(infos) };

                Some(Ok(Packet {
                    data: buffer,
                    channels: infos.channels as u16,
                    rate: infos.rate as u64,
                    bitrate_upper: infos.bitrate_upper as u64,
                    bitrate_nominal: infos.bitrate_nominal as u64,
                    bitrate_lower: infos.bitrate_lower as u64,
                    bitrate_window: infos.bitrate_window as u64,
                }))
            }
        }
    }
}

impl<R> Drop for Decoder<R> where R: Read + Seek {
    fn drop(&mut self) {
        unsafe {
            vorbisfile_sys::ov_clear(&mut self.data.vorbis);
        }
    }
}

fn check_errors(code: libc::c_int) -> Result<(), VorbisError> {
    match code {
        0 => Ok(()),

        vorbis_sys::OV_ENOTVORBIS => Err(VorbisError::NotVorbis),
        vorbis_sys::OV_EVERSION => Err(VorbisError::VersionMismatch),
        vorbis_sys::OV_EBADHEADER => Err(VorbisError::BadHeader),
        vorbis_sys::OV_EINVAL => Err(VorbisError::InitialFileHeadersCorrupt),
        vorbis_sys::OV_HOLE => Err(VorbisError::Hole),

        vorbis_sys::OV_EREAD => unimplemented!(),

        // indicates a bug or heap/stack corruption
        vorbis_sys::OV_EFAULT => panic!("Internal libvorbis error"),
        _ => panic!("Unknown vorbis error")
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#![crate_name = "azure"]
#![crate_type = "rlib"]
#![cfg_attr(feature = "plugins", feature(custom_derive, plugin, rustc_attrs, rustc_macro))]
#![cfg_attr(feature = "plugins", plugin(heapsize_plugin))]

extern crate libc;
#[cfg(feature = "plugins")]
extern crate serde;
#[cfg(feature = "plugins")]
#[macro_use]
extern crate serde_derive;
extern crate euclid;
extern crate skia;
#[macro_use]
#[cfg(feature = "plugins")]
extern crate heapsize;
#[cfg(target_os = "linux")]
extern crate x11;
#[cfg(target_os = "android")]
extern crate egl;
#[cfg(not(target_os = "macos"))]
extern crate freetype;
#[cfg(target_os = "macos")]
extern crate core_foundation;
#[cfg(target_os = "macos")]
extern crate core_graphics;
#[cfg(target_os = "macos")]
extern crate core_text;

pub use azure::{AzFontOptions, AzFloat, enum_AzSurfaceType, AZ_SURFACE_DATA,
                AZ_SURFACE_D2D1_BITMAP, AZ_SURFACE_D2D1_DRAWTARGET, AZ_SURFACE_CAIRO, AZ_SURFACE_CAIRO_IMAGE,
                AZ_SURFACE_COREGRAPHICS_IMAGE, AZ_SURFACE_COREGRAPHICS_CGCONTEXT, AZ_SURFACE_SKIA, AZ_SURFACE_DUAL_DT,
                enum_AzSurfaceFormat, AZ_FORMAT_B8G8R8A8, AZ_FORMAT_B8G8R8X8, AZ_FORMAT_R5G6B5,
                AZ_FORMAT_A8, AzSurfaceFormat, enum_AzBackendType, AZ_BACKEND_NONE,
                AZ_BACKEND_DIRECT2D, AZ_BACKEND_COREGRAPHICS, AZ_BACKEND_COREGRAPHICS_ACCELERATED, AZ_BACKEND_CAIRO,
                AZ_BACKEND_SKIA, AZ_BACKEND_RECORDING, AzBackendType, enum_AzFontType,
                AZ_FONT_DWRITE, AZ_FONT_GDI, AZ_FONT_MAC, AZ_FONT_SKIA,
                AZ_FONT_CAIRO, AZ_FONT_COREGRAPHICS, enum_AzNativeSurfaceType, AZ_NATIVE_SURFACE_D3D10_TEXTURE,
                AZ_NATIVE_SURFACE_CAIRO_SURFACE, AZ_NATIVE_SURFACE_CGCONTEXT, enum_AzNativeFontType, AZ_NATIVE_FONT_DWRITE_FONT_FACE,
                AZ_NATIVE_FONT_GDI_FONT_FACE, AZ_NATIVE_FONT_MAC_FONT_FACE, AZ_NATIVE_FONT_SKIA_FONT_FACE, AZ_NATIVE_FONT_CAIRO_FONT_FACE,
                enum_AzFontStyle, AZ_FONT_STYLE_NORMAL, AZ_FONT_STYLE_ITALIC, AZ_FONT_STYLE_BOLD,
                AZ_FONT_STYLE_BOLD_ITALIC, enum_AzCompositionOp, AZ_OP_OVER, AZ_OP_ADD,
                AZ_OP_ATOP, AZ_OP_OUT, AZ_OP_IN, AZ_OP_SOURCE,
                AZ_OP_DEST_IN, AZ_OP_DEST_OUT, AZ_OP_DEST_OVER, AZ_OP_DEST_ATOP,
                AZ_OP_XOR, AZ_OP_MULTIPLY, AZ_OP_SCREEN, AZ_OP_OVERLAY,
                AZ_OP_DARKEN, AZ_OP_LIGHTEN, AZ_OP_COLOR_DODGE, AZ_OP_COLOR_BURN,
                AZ_OP_HARD_LIGHT, AZ_OP_SOFT_LIGHT, AZ_OP_DIFFERENCE, AZ_OP_EXCLUSION,
                AZ_OP_HUE, AZ_OP_SATURATION, AZ_OP_COLOR, AZ_OP_LUMINOSITY,
                AZ_OP_COUNT, enum_AzExtendMode, AZ_EXTEND_CLAMP, AZ_EXTEND_REPEAT,
                AZ_EXTEND_REFLECT, enum_AzFillRule, AZ_FILL_WINDING, AZ_FILL_EVEN_ODD,
                enum_AzAntialiasMode, AZ_AA_NONE, AZ_AA_GRAY, AZ_AA_SUBPIXEL,
                enum_AzFilter, AZ_FILTER_LINEAR, AZ_FILTER_POINT, AzFilter, enum_AzPatternType,
                AZ_PATTERN_COLOR, AZ_PATTERN_SURFACE, AZ_PATTERN_LINEAR_GRADIENT, AZ_PATTERN_RADIAL_GRADIENT,
                enum_AzJoinStyle, AZ_JOIN_BEVEL, AZ_JOIN_ROUND, AZ_JOIN_MITER,
                AZ_JOIN_MITER_OR_BEVEL, enum_AzCapStyle, AZ_CAP_BUTT, AZ_CAP_ROUND,
                AZ_CAP_SQUARE, enum_AzSamplingBounds, AZ_SAMPLING_UNBOUNDED, AZ_SAMPLING_BOUNDED,
                enum_AzSide, AZ_eSideTop, AZ_eSideRight, AZ_eSideBottom,
                AZ_eSideLeft, struct__AzColor, AzColor, struct__AzGradientStop,
                AzGradientStop, struct__AzIntRect, AzIntRect, struct__AzRect,
                AzRect, struct__AzIntPoint, AzIntPoint, struct__AzPoint,
                AzPoint, struct__AzIntSize, AzIntSize, struct__AzSize,
                AzSize, struct__AzMatrix, AzMatrix, struct__AzDrawOptions,
                AzDrawOptions, struct__AzStrokeOptions, AzStrokeOptions, struct__AzDrawSurfaceOptions,
                AzDrawSurfaceOptions, struct__AzGlyph, AzGlyph, struct__AzGlyphBuffer,
                AzGlyphBuffer, struct__AzNativeFont, AzNativeFont, AzGradientStopsRef,
                AzDrawTargetRef, AzPatternRef, AzColorPatternRef, AzScaledFontRef,
                AzGlyphRenderingOptionsRef, AzSourceSurfaceRef, AzDataSourceSurfaceRef, AzDrawSurfaceOptionsRef,
                AzGLContext, AzGLContextMetadataRef,
                AzPathRef, AzPathBuilderRef, AzSanityCheck, AzCreateColorPattern,
                AzCreateDrawTarget, AzCreateDrawTargetForData, AzCreateDrawTargetSkiaWithGrContextAndFBO,
                AzRetainDrawTarget, AzReleaseDrawTarget, AzDrawTargetGetSize, AzDrawTargetFlush,
                AzDrawTargetClearRect, AzDrawTargetCopySurface, AzDrawTargetFillRect, AzDrawTargetStrokeRect, AzDrawTargetStrokeLine,
                AzDrawTargetFill, AzDrawTargetStroke, AzDrawTargetPushClip, AzDrawTargetPopClip, AzDrawTargetFillGlyphs,
                AzDrawTargetDrawSurface, AzDrawTargetGetSnapshot, AzDrawTargetCreateSourceSurfaceFromData, AzReleaseSourceSurface,
                AzSourceSurfaceGetSize, AzSourceSurfaceGetFormat, AzSourceSurfaceGetDataSurface, AzDataSourceSurfaceGetData,
                AzDataSourceSurfaceGetStride, AzCreateScaledFontForNativeFont, AzReleaseScaledFont, AzDrawTargetSetTransform,
                AzCreateFontOptionsForName, AzDestroyFontOptions, AzCreatePathBuilder, AzPathCopyToBuilder, AzPathContainsPoint,
                AzReleasePathBuilder, AzPathBuilderMoveTo, AzPathBuilderLineTo, AzPathBuilderFinish, AzReleasePath};

pub mod azure_hl;
pub mod scaled_font;

#[cfg(target_os = "linux")]
pub mod test;

pub mod linkhack;

pub mod azure;
<commit_msg>Auto merge of #246 - servo:derive, r=Manishearth<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#![crate_name = "azure"]
#![crate_type = "rlib"]
#![cfg_attr(feature = "plugins", feature(custom_derive, plugin, proc_macro, rustc_attrs))]
#![cfg_attr(feature = "plugins", plugin(heapsize_plugin))]

extern crate libc;
#[cfg(feature = "plugins")]
extern crate serde;
#[cfg(feature = "plugins")]
#[macro_use]
extern crate serde_derive;
extern crate euclid;
extern crate skia;
#[macro_use]
#[cfg(feature = "plugins")]
extern crate heapsize;
#[cfg(target_os = "linux")]
extern crate x11;
#[cfg(target_os = "android")]
extern crate egl;
#[cfg(not(target_os = "macos"))]
extern crate freetype;
#[cfg(target_os = "macos")]
extern crate core_foundation;
#[cfg(target_os = "macos")]
extern crate core_graphics;
#[cfg(target_os = "macos")]
extern crate core_text;

pub use azure::{AzFontOptions, AzFloat, enum_AzSurfaceType, AZ_SURFACE_DATA,
                AZ_SURFACE_D2D1_BITMAP, AZ_SURFACE_D2D1_DRAWTARGET, AZ_SURFACE_CAIRO, AZ_SURFACE_CAIRO_IMAGE,
                AZ_SURFACE_COREGRAPHICS_IMAGE, AZ_SURFACE_COREGRAPHICS_CGCONTEXT, AZ_SURFACE_SKIA, AZ_SURFACE_DUAL_DT,
                enum_AzSurfaceFormat, AZ_FORMAT_B8G8R8A8, AZ_FORMAT_B8G8R8X8, AZ_FORMAT_R5G6B5,
                AZ_FORMAT_A8, AzSurfaceFormat, enum_AzBackendType, AZ_BACKEND_NONE,
                AZ_BACKEND_DIRECT2D, AZ_BACKEND_COREGRAPHICS, AZ_BACKEND_COREGRAPHICS_ACCELERATED, AZ_BACKEND_CAIRO,
                AZ_BACKEND_SKIA, AZ_BACKEND_RECORDING, AzBackendType, enum_AzFontType,
                AZ_FONT_DWRITE, AZ_FONT_GDI, AZ_FONT_MAC, AZ_FONT_SKIA,
                AZ_FONT_CAIRO, AZ_FONT_COREGRAPHICS, enum_AzNativeSurfaceType, AZ_NATIVE_SURFACE_D3D10_TEXTURE,
                AZ_NATIVE_SURFACE_CAIRO_SURFACE, AZ_NATIVE_SURFACE_CGCONTEXT, enum_AzNativeFontType, AZ_NATIVE_FONT_DWRITE_FONT_FACE,
                AZ_NATIVE_FONT_GDI_FONT_FACE, AZ_NATIVE_FONT_MAC_FONT_FACE, AZ_NATIVE_FONT_SKIA_FONT_FACE, AZ_NATIVE_FONT_CAIRO_FONT_FACE,
                enum_AzFontStyle, AZ_FONT_STYLE_NORMAL, AZ_FONT_STYLE_ITALIC, AZ_FONT_STYLE_BOLD,
                AZ_FONT_STYLE_BOLD_ITALIC, enum_AzCompositionOp, AZ_OP_OVER, AZ_OP_ADD,
                AZ_OP_ATOP, AZ_OP_OUT, AZ_OP_IN, AZ_OP_SOURCE,
                AZ_OP_DEST_IN, AZ_OP_DEST_OUT, AZ_OP_DEST_OVER, AZ_OP_DEST_ATOP,
                AZ_OP_XOR, AZ_OP_MULTIPLY, AZ_OP_SCREEN, AZ_OP_OVERLAY,
                AZ_OP_DARKEN, AZ_OP_LIGHTEN, AZ_OP_COLOR_DODGE, AZ_OP_COLOR_BURN,
                AZ_OP_HARD_LIGHT, AZ_OP_SOFT_LIGHT, AZ_OP_DIFFERENCE, AZ_OP_EXCLUSION,
                AZ_OP_HUE, AZ_OP_SATURATION, AZ_OP_COLOR, AZ_OP_LUMINOSITY,
                AZ_OP_COUNT, enum_AzExtendMode, AZ_EXTEND_CLAMP, AZ_EXTEND_REPEAT,
                AZ_EXTEND_REFLECT, enum_AzFillRule, AZ_FILL_WINDING, AZ_FILL_EVEN_ODD,
                enum_AzAntialiasMode, AZ_AA_NONE, AZ_AA_GRAY, AZ_AA_SUBPIXEL,
                enum_AzFilter, AZ_FILTER_LINEAR, AZ_FILTER_POINT, AzFilter, enum_AzPatternType,
                AZ_PATTERN_COLOR, AZ_PATTERN_SURFACE, AZ_PATTERN_LINEAR_GRADIENT, AZ_PATTERN_RADIAL_GRADIENT,
                enum_AzJoinStyle, AZ_JOIN_BEVEL, AZ_JOIN_ROUND, AZ_JOIN_MITER,
                AZ_JOIN_MITER_OR_BEVEL, enum_AzCapStyle, AZ_CAP_BUTT, AZ_CAP_ROUND,
                AZ_CAP_SQUARE, enum_AzSamplingBounds, AZ_SAMPLING_UNBOUNDED, AZ_SAMPLING_BOUNDED,
                enum_AzSide, AZ_eSideTop, AZ_eSideRight, AZ_eSideBottom,
                AZ_eSideLeft, struct__AzColor, AzColor, struct__AzGradientStop,
                AzGradientStop, struct__AzIntRect, AzIntRect, struct__AzRect,
                AzRect, struct__AzIntPoint, AzIntPoint, struct__AzPoint,
                AzPoint, struct__AzIntSize, AzIntSize, struct__AzSize,
                AzSize, struct__AzMatrix, AzMatrix, struct__AzDrawOptions,
                AzDrawOptions, struct__AzStrokeOptions, AzStrokeOptions, struct__AzDrawSurfaceOptions,
                AzDrawSurfaceOptions, struct__AzGlyph, AzGlyph, struct__AzGlyphBuffer,
                AzGlyphBuffer, struct__AzNativeFont, AzNativeFont, AzGradientStopsRef,
                AzDrawTargetRef, AzPatternRef, AzColorPatternRef, AzScaledFontRef,
                AzGlyphRenderingOptionsRef, AzSourceSurfaceRef, AzDataSourceSurfaceRef, AzDrawSurfaceOptionsRef,
                AzGLContext, AzGLContextMetadataRef,
                AzPathRef, AzPathBuilderRef, AzSanityCheck, AzCreateColorPattern,
                AzCreateDrawTarget, AzCreateDrawTargetForData, AzCreateDrawTargetSkiaWithGrContextAndFBO,
                AzRetainDrawTarget, AzReleaseDrawTarget, AzDrawTargetGetSize, AzDrawTargetFlush,
                AzDrawTargetClearRect, AzDrawTargetCopySurface, AzDrawTargetFillRect, AzDrawTargetStrokeRect, AzDrawTargetStrokeLine,
                AzDrawTargetFill, AzDrawTargetStroke, AzDrawTargetPushClip, AzDrawTargetPopClip, AzDrawTargetFillGlyphs,
                AzDrawTargetDrawSurface, AzDrawTargetGetSnapshot, AzDrawTargetCreateSourceSurfaceFromData, AzReleaseSourceSurface,
                AzSourceSurfaceGetSize, AzSourceSurfaceGetFormat, AzSourceSurfaceGetDataSurface, AzDataSourceSurfaceGetData,
                AzDataSourceSurfaceGetStride, AzCreateScaledFontForNativeFont, AzReleaseScaledFont, AzDrawTargetSetTransform,
                AzCreateFontOptionsForName, AzDestroyFontOptions, AzCreatePathBuilder, AzPathCopyToBuilder, AzPathContainsPoint,
                AzReleasePathBuilder, AzPathBuilderMoveTo, AzPathBuilderLineTo, AzPathBuilderFinish, AzReleasePath};

pub mod azure_hl;
pub mod scaled_font;

#[cfg(target_os = "linux")]
pub mod test;

pub mod linkhack;

pub mod azure;
<|endoftext|>


<commit_before>#![allow(unsafe_code)]

use std::convert::TryFrom;
use std::ptr;
use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};

use crate::{
    atomic_shim::AtomicU64,
    debug_delay,
    dll::{DoublyLinkedList, Node},
    fastlock::FastLock,
    Guard, PageId,
};

#[cfg(any(test, feature = "lock_free_delays"))]
const MAX_QUEUE_ITEMS: usize = 4;

#[cfg(not(any(test, feature = "lock_free_delays")))]
const MAX_QUEUE_ITEMS: usize = 64;

#[cfg(any(test, feature = "lock_free_delays"))]
const N_SHARDS: usize = 2;

#[cfg(not(any(test, feature = "lock_free_delays")))]
const N_SHARDS: usize = 256;

struct AccessBlock {
    len: AtomicUsize,
    block: [AtomicU64; MAX_QUEUE_ITEMS],
    next: AtomicPtr<AccessBlock>,
}

impl Default for AccessBlock {
    fn default() -> AccessBlock {
        AccessBlock {
            len: AtomicUsize::new(0),
            block: array_init::array_init(|_| AtomicU64::default()),
            next: AtomicPtr::default(),
        }
    }
}

struct AccessQueue {
    writing: AtomicPtr<AccessBlock>,
    full_list: AtomicPtr<AccessBlock>,
}

impl Default for AccessQueue {
    fn default() -> AccessQueue {
        AccessQueue {
            writing: AtomicPtr::new(Box::into_raw(Box::new(
                AccessBlock::default(),
            ))),
            full_list: AtomicPtr::default(),
        }
    }
}

impl AccessQueue {
    fn push(&self, item: CacheAccess) -> bool {
        let mut filled = false;
        loop {
            debug_delay();
            let head = self.writing.load(Ordering::Acquire);
            let block = unsafe { &*head };

            debug_delay();
            let offset = block.len.fetch_add(1, Ordering::Release);

            if offset < MAX_QUEUE_ITEMS {
                debug_delay();
                unsafe {
                    block
                        .block
                        .get_unchecked(offset)
                        .store(item.0, Ordering::Release);
                }
                return filled;
            } else {
                // install new writer
                let new = Box::into_raw(Box::new(AccessBlock::default()));
                debug_delay();
                let prev =
                    self.writing.compare_and_swap(head, new, Ordering::Release);
                if prev != head {
                    // we lost the CAS, free the new item that was
                    // never published to other threads
                    unsafe {
                        drop(Box::from_raw(new));
                    }
                    continue;
                }

                // push the now-full item to the full list for future
                // consumption
                let mut ret;
                let mut full_list_ptr = self.full_list.load(Ordering::Acquire);
                while {
                    // we loop because maybe other threads are pushing stuff too
                    block.next.store(full_list_ptr, Ordering::Release);
                    debug_delay();
                    ret = self.full_list.compare_and_swap(
                        full_list_ptr,
                        head,
                        Ordering::Release,
                    );
                    ret != full_list_ptr
                } {
                    full_list_ptr = ret;
                }
                filled = true;
            }
        }
    }

    fn take<'a>(&self, guard: &'a Guard) -> CacheAccessIter<'a> {
        debug_delay();
        let ptr = self.full_list.swap(std::ptr::null_mut(), Ordering::Release);

        CacheAccessIter { guard, current_offset: 0, current_block: ptr }
    }
}

impl Drop for AccessQueue {
    fn drop(&mut self) {
        debug_delay();
        let writing = self.writing.load(Ordering::Acquire);
        unsafe {
            Box::from_raw(writing);
        }
        debug_delay();
        let mut head = self.full_list.load(Ordering::Acquire);
        while !head.is_null() {
            unsafe {
                debug_delay();
                let next =
                    (*head).next.swap(std::ptr::null_mut(), Ordering::Release);
                Box::from_raw(head);
                head = next;
            }
        }
    }
}

struct CacheAccessIter<'a> {
    guard: &'a Guard,
    current_offset: usize,
    current_block: *mut AccessBlock,
}

impl<'a> Iterator for CacheAccessIter<'a> {
    type Item = CacheAccess;

    fn next(&mut self) -> Option<CacheAccess> {
        while !self.current_block.is_null() {
            let current_block = unsafe { &*self.current_block };

            debug_delay();
            if self.current_offset >= MAX_QUEUE_ITEMS {
                let to_drop_ptr = self.current_block;
                debug_delay();
                self.current_block = current_block.next.load(Ordering::Acquire);
                self.current_offset = 0;
                debug_delay();
                let to_drop = unsafe { Box::from_raw(to_drop_ptr) };
                self.guard.defer(|| to_drop);
                continue;
            }

            let mut next = 0;
            while next == 0 {
                // we spin here because there's a race between bumping
                // the offset and setting the value to something other
                // than 0 (and 0 is an invalid value)
                debug_delay();
                next = current_block.block[self.current_offset]
                    .load(Ordering::Acquire);
            }
            self.current_offset += 1;
            return Some(CacheAccess(next));
        }

        None
    }
}

#[derive(Clone, Copy)]
struct CacheAccess(u64);

impl CacheAccess {
    fn new(pid: PageId, sz: u64) -> CacheAccess {
        let rounded_up_power_of_2 =
            u64::from(sz.next_power_of_two().trailing_zeros());

        assert!(rounded_up_power_of_2 < 256);

        CacheAccess(pid | (rounded_up_power_of_2 << 56))
    }

    const fn decompose(self) -> (PageId, u64) {
        let sz = 1 << (self.0 >> 56);
        let pid = self.0 << 8 >> 8;
        (pid, sz)
    }
}

/// A simple LRU cache.
pub struct Lru {
    shards: Vec<(AccessQueue, FastLock<Shard>)>,
}

unsafe impl Sync for Lru {}

impl Lru {
    /// Instantiates a new `Lru` cache.
    pub(crate) fn new(cache_capacity: u64) -> Self {
        assert!(
            cache_capacity >= 256,
            "Please configure the cache \
             capacity to be at least 256 bytes"
        );
        let shard_capacity = cache_capacity / N_SHARDS as u64;

        let mut shards = Vec::with_capacity(N_SHARDS);
        shards.resize_with(N_SHARDS, || {
            (AccessQueue::default(), FastLock::new(Shard::new(shard_capacity)))
        });

        Self { shards }
    }

    /// Called when an item is accessed. Returns a Vec of items to be
    /// evicted. Uses flat-combining to avoid blocking on what can
    /// be an asynchronous operation.
    ///
    /// layout:
    ///   items:   1 2 3 4 5 6 7 8 9 10
    ///   shards:  1 0 1 0 1 0 1 0 1 0
    ///   shard 0:   2   4   6   8   10
    ///   shard 1: 1   3   5   7   9
    pub(crate) fn accessed(
        &self,
        id: PageId,
        item_size: u64,
        guard: &Guard,
    ) -> Vec<PageId> {
        let mut ret = vec![];
        let shards = self.shards.len() as u64;
        let (shard_idx, item_pos) = (id % shards, id / shards);
        let (stack, shard_mu) = &self.shards[safe_usize(shard_idx)];

        let filled = stack.push(CacheAccess::new(item_pos, item_size));

        if filled {
            // only try to acquire this if
            if let Some(mut shard) = shard_mu.try_lock() {
                let accesses = stack.take(guard);
                for item in accesses {
                    let (item_pos, item_size) = item.decompose();
                    let to_evict =
                        shard.accessed(safe_usize(item_pos), item_size);
                    // map shard internal offsets to global items ids
                    for pos in to_evict {
                        let item = (pos * shards) + shard_idx;
                        ret.push(item);
                    }
                }
            }
        }
        ret
    }
}

#[derive(Clone)]
struct Entry {
    ptr: *mut Node,
    size: u64,
}

impl Default for Entry {
    fn default() -> Self {
        Self { ptr: ptr::null_mut(), size: 0 }
    }
}

struct Shard {
    list: DoublyLinkedList,
    entries: Vec<Entry>,
    capacity: u64,
    size: u64,
}

impl Shard {
    fn new(capacity: u64) -> Self {
        assert!(capacity > 0, "shard capacity must be non-zero");

        Self {
            list: DoublyLinkedList::default(),
            entries: vec![],
            capacity,
            size: 0,
        }
    }

    /// `PageId`s in the shard list are indexes of the entries.
    fn accessed(&mut self, pos: usize, size: u64) -> Vec<PageId> {
        if pos >= self.entries.len() {
            self.entries.resize(pos + 1, Entry::default());
        }

        {
            let entry = &mut self.entries[pos];

            self.size -= entry.size;
            entry.size = size;
            self.size += size;

            if entry.ptr.is_null() {
                entry.ptr = self.list.push_head(PageId::try_from(pos).unwrap());
            } else {
                entry.ptr = self.list.promote(entry.ptr);
            }
        }

        let mut to_evict = vec![];
        while self.size > self.capacity {
            if self.list.len() == 1 {
                // don't evict what we just added
                break;
            }

            let min_pid = self.list.pop_tail().unwrap();
            let min_pid_idx = safe_usize(min_pid);

            self.entries[min_pid_idx].ptr = ptr::null_mut();

            to_evict.push(min_pid);

            self.size -= self.entries[min_pid_idx].size;
            self.entries[min_pid_idx].size = 0;
        }

        to_evict
    }
}

#[inline]
fn safe_usize(value: PageId) -> usize {
    usize::try_from(value).unwrap()
}

#[test]
fn lru_smoke_test() {
    use crate::pin;

    let lru = Lru::new(256);
    for i in 0..1000 {
        let guard = pin();
        lru.accessed(i, 16, &guard);
    }
}
<commit_msg>Use AcqRel ordering instead of Release for swap to process the LRU access queue<commit_after>#![allow(unsafe_code)]

use std::convert::TryFrom;
use std::ptr;
use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};

use crate::{
    atomic_shim::AtomicU64,
    debug_delay,
    dll::{DoublyLinkedList, Node},
    fastlock::FastLock,
    Guard, PageId,
};

#[cfg(any(test, feature = "lock_free_delays"))]
const MAX_QUEUE_ITEMS: usize = 4;

#[cfg(not(any(test, feature = "lock_free_delays")))]
const MAX_QUEUE_ITEMS: usize = 64;

#[cfg(any(test, feature = "lock_free_delays"))]
const N_SHARDS: usize = 2;

#[cfg(not(any(test, feature = "lock_free_delays")))]
const N_SHARDS: usize = 256;

struct AccessBlock {
    len: AtomicUsize,
    block: [AtomicU64; MAX_QUEUE_ITEMS],
    next: AtomicPtr<AccessBlock>,
}

impl Default for AccessBlock {
    fn default() -> AccessBlock {
        AccessBlock {
            len: AtomicUsize::new(0),
            block: array_init::array_init(|_| AtomicU64::default()),
            next: AtomicPtr::default(),
        }
    }
}

struct AccessQueue {
    writing: AtomicPtr<AccessBlock>,
    full_list: AtomicPtr<AccessBlock>,
}

impl Default for AccessQueue {
    fn default() -> AccessQueue {
        AccessQueue {
            writing: AtomicPtr::new(Box::into_raw(Box::new(
                AccessBlock::default(),
            ))),
            full_list: AtomicPtr::default(),
        }
    }
}

impl AccessQueue {
    fn push(&self, item: CacheAccess) -> bool {
        let mut filled = false;
        loop {
            debug_delay();
            let head = self.writing.load(Ordering::Acquire);
            let block = unsafe { &*head };

            debug_delay();
            let offset = block.len.fetch_add(1, Ordering::Release);

            if offset < MAX_QUEUE_ITEMS {
                debug_delay();
                unsafe {
                    block
                        .block
                        .get_unchecked(offset)
                        .store(item.0, Ordering::Release);
                }
                return filled;
            } else {
                // install new writer
                let new = Box::into_raw(Box::new(AccessBlock::default()));
                debug_delay();
                let prev =
                    self.writing.compare_and_swap(head, new, Ordering::Release);
                if prev != head {
                    // we lost the CAS, free the new item that was
                    // never published to other threads
                    unsafe {
                        drop(Box::from_raw(new));
                    }
                    continue;
                }

                // push the now-full item to the full list for future
                // consumption
                let mut ret;
                let mut full_list_ptr = self.full_list.load(Ordering::Acquire);
                while {
                    // we loop because maybe other threads are pushing stuff too
                    block.next.store(full_list_ptr, Ordering::Release);
                    debug_delay();
                    ret = self.full_list.compare_and_swap(
                        full_list_ptr,
                        head,
                        Ordering::Release,
                    );
                    ret != full_list_ptr
                } {
                    full_list_ptr = ret;
                }
                filled = true;
            }
        }
    }

    fn take<'a>(&self, guard: &'a Guard) -> CacheAccessIter<'a> {
        debug_delay();
        let ptr = self.full_list.swap(std::ptr::null_mut(), Ordering::AcqRel);

        CacheAccessIter { guard, current_offset: 0, current_block: ptr }
    }
}

impl Drop for AccessQueue {
    fn drop(&mut self) {
        debug_delay();
        let writing = self.writing.load(Ordering::Acquire);
        unsafe {
            Box::from_raw(writing);
        }
        debug_delay();
        let mut head = self.full_list.load(Ordering::Acquire);
        while !head.is_null() {
            unsafe {
                debug_delay();
                let next =
                    (*head).next.swap(std::ptr::null_mut(), Ordering::Release);
                Box::from_raw(head);
                head = next;
            }
        }
    }
}

struct CacheAccessIter<'a> {
    guard: &'a Guard,
    current_offset: usize,
    current_block: *mut AccessBlock,
}

impl<'a> Iterator for CacheAccessIter<'a> {
    type Item = CacheAccess;

    fn next(&mut self) -> Option<CacheAccess> {
        while !self.current_block.is_null() {
            let current_block = unsafe { &*self.current_block };

            debug_delay();
            if self.current_offset >= MAX_QUEUE_ITEMS {
                let to_drop_ptr = self.current_block;
                debug_delay();
                self.current_block = current_block.next.load(Ordering::Acquire);
                self.current_offset = 0;
                debug_delay();
                let to_drop = unsafe { Box::from_raw(to_drop_ptr) };
                self.guard.defer(|| to_drop);
                continue;
            }

            let mut next = 0;
            while next == 0 {
                // we spin here because there's a race between bumping
                // the offset and setting the value to something other
                // than 0 (and 0 is an invalid value)
                debug_delay();
                next = current_block.block[self.current_offset]
                    .load(Ordering::Acquire);
            }
            self.current_offset += 1;
            return Some(CacheAccess(next));
        }

        None
    }
}

#[derive(Clone, Copy)]
struct CacheAccess(u64);

impl CacheAccess {
    fn new(pid: PageId, sz: u64) -> CacheAccess {
        let rounded_up_power_of_2 =
            u64::from(sz.next_power_of_two().trailing_zeros());

        assert!(rounded_up_power_of_2 < 256);

        CacheAccess(pid | (rounded_up_power_of_2 << 56))
    }

    const fn decompose(self) -> (PageId, u64) {
        let sz = 1 << (self.0 >> 56);
        let pid = self.0 << 8 >> 8;
        (pid, sz)
    }
}

/// A simple LRU cache.
pub struct Lru {
    shards: Vec<(AccessQueue, FastLock<Shard>)>,
}

unsafe impl Sync for Lru {}

impl Lru {
    /// Instantiates a new `Lru` cache.
    pub(crate) fn new(cache_capacity: u64) -> Self {
        assert!(
            cache_capacity >= 256,
            "Please configure the cache \
             capacity to be at least 256 bytes"
        );
        let shard_capacity = cache_capacity / N_SHARDS as u64;

        let mut shards = Vec::with_capacity(N_SHARDS);
        shards.resize_with(N_SHARDS, || {
            (AccessQueue::default(), FastLock::new(Shard::new(shard_capacity)))
        });

        Self { shards }
    }

    /// Called when an item is accessed. Returns a Vec of items to be
    /// evicted. Uses flat-combining to avoid blocking on what can
    /// be an asynchronous operation.
    ///
    /// layout:
    ///   items:   1 2 3 4 5 6 7 8 9 10
    ///   shards:  1 0 1 0 1 0 1 0 1 0
    ///   shard 0:   2   4   6   8   10
    ///   shard 1: 1   3   5   7   9
    pub(crate) fn accessed(
        &self,
        id: PageId,
        item_size: u64,
        guard: &Guard,
    ) -> Vec<PageId> {
        let mut ret = vec![];
        let shards = self.shards.len() as u64;
        let (shard_idx, item_pos) = (id % shards, id / shards);
        let (stack, shard_mu) = &self.shards[safe_usize(shard_idx)];

        let filled = stack.push(CacheAccess::new(item_pos, item_size));

        if filled {
            // only try to acquire this if
            if let Some(mut shard) = shard_mu.try_lock() {
                let accesses = stack.take(guard);
                for item in accesses {
                    let (item_pos, item_size) = item.decompose();
                    let to_evict =
                        shard.accessed(safe_usize(item_pos), item_size);
                    // map shard internal offsets to global items ids
                    for pos in to_evict {
                        let item = (pos * shards) + shard_idx;
                        ret.push(item);
                    }
                }
            }
        }
        ret
    }
}

#[derive(Clone)]
struct Entry {
    ptr: *mut Node,
    size: u64,
}

impl Default for Entry {
    fn default() -> Self {
        Self { ptr: ptr::null_mut(), size: 0 }
    }
}

struct Shard {
    list: DoublyLinkedList,
    entries: Vec<Entry>,
    capacity: u64,
    size: u64,
}

impl Shard {
    fn new(capacity: u64) -> Self {
        assert!(capacity > 0, "shard capacity must be non-zero");

        Self {
            list: DoublyLinkedList::default(),
            entries: vec![],
            capacity,
            size: 0,
        }
    }

    /// `PageId`s in the shard list are indexes of the entries.
    fn accessed(&mut self, pos: usize, size: u64) -> Vec<PageId> {
        if pos >= self.entries.len() {
            self.entries.resize(pos + 1, Entry::default());
        }

        {
            let entry = &mut self.entries[pos];

            self.size -= entry.size;
            entry.size = size;
            self.size += size;

            if entry.ptr.is_null() {
                entry.ptr = self.list.push_head(PageId::try_from(pos).unwrap());
            } else {
                entry.ptr = self.list.promote(entry.ptr);
            }
        }

        let mut to_evict = vec![];
        while self.size > self.capacity {
            if self.list.len() == 1 {
                // don't evict what we just added
                break;
            }

            let min_pid = self.list.pop_tail().unwrap();
            let min_pid_idx = safe_usize(min_pid);

            self.entries[min_pid_idx].ptr = ptr::null_mut();

            to_evict.push(min_pid);

            self.size -= self.entries[min_pid_idx].size;
            self.entries[min_pid_idx].size = 0;
        }

        to_evict
    }
}

#[inline]
fn safe_usize(value: PageId) -> usize {
    usize::try_from(value).unwrap()
}

#[test]
fn lru_smoke_test() {
    use crate::pin;

    let lru = Lru::new(256);
    for i in 0..1000 {
        let guard = pin();
        lru.accessed(i, 16, &guard);
    }
}
<|endoftext|>


<commit_before>use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::hash_map::Entry::*;
use std::hash::{Hash, Hasher};
use std::iter;
use std::str::CharIndices;
use std::ops;
use itertools::{GroupBy, Itertools};
use coalesce::Coalesce;

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Debug, Eq, Ord, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into() }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    // Documents are unique only upon their id
    fn hash<H>(&self, state: &mut H)
        where H: Hasher
    {
        self.id.hash(state);
    }
}

impl PartialEq for Document {
    // Documents are unique only upon their id
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

impl PartialOrd for Document {
    fn partial_cmp(&self, other: &Document) -> Option<::std::cmp::Ordering> {
        self.id.partial_cmp(&other.id)
    }
}

/// A SearchResult is the representation of a Document returned for a specific set of search
/// terms. It is unique upon the document and the vec of highlight indices. It also contains a
/// search score for use in ranking against the other search results
#[derive(Clone, Debug, RustcEncodable)]
pub struct SearchResult<'a> {
    doc: &'a Document,
    highlights: Vec<(usize, usize)>,
    score: f32,
}

impl<'a> SearchResult<'a> {
    fn new(doc: &'a Document, mut highlights: Vec<(usize, usize)>) -> SearchResult<'a> {
        SearchResult {
            score: highlights.iter()
                .map(|&(begin, end)| end - begin)
                .sum::<usize>() as f32 /
                   (doc.content.len() as f32).sqrt(),
            doc: doc,
            highlights: {
                highlights.sort();
                highlights
            },
        }
    }

    /// Returns the document
    pub fn doc(&self) -> &Document {
        &self.doc
    }

    /// Returns the highlighted indices.
    ///
    /// Each `(usize, usize)` indicates the start and end of a term in the document's content
    /// that should be highlighted.
    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    /// Returns the search result's score.
    ///
    /// Score is computed by the product of the summed length of the matching terms and the inverse
    /// square root of the length of the document. Taking the square root of the document's length
    /// helps to combat bias toward short content.
    pub fn score(&self) -> f32 {
        self.score
    }

    pub fn highlight(&self, before: &str, after: &str) -> String {
        let mut begin_idx = 0;
        let mut parts = String::new();
        for &(begin, end) in &self.highlights {
            parts.push_str(&self.doc.content[begin_idx..begin]);
            parts.push_str(before);
            parts.push_str(&self.doc.content[begin..end]);
            parts.push_str(after);
            begin_idx = end;
        }
        parts.push_str(&self.doc.content[begin_idx..]);
        parts
    }
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, BTreeMap<String, Vec<(usize, usize)>>>,
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex { index: BTreeMap::new(), docs: BTreeMap::new() }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let analyzed = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            let mut highlights = self.index
                                     .entry(ngram)
                                     .or_insert_with(BTreeMap::new)
                                     .entry(doc.id.clone())
                                     .or_insert_with(Vec::new);
            let coalesce_idx = highlights.binary_search(&highlighted).err().unwrap();
            highlights.coalesce(coalesce_idx, highlighted);
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        let map: HashMap<String, Vec<(usize, usize)>> = unique_terms.into_iter()
            .flat_map(|word| self.index.get(&word))
            .flat_map(BTreeMap::iter)
            .fold(HashMap::new(), |mut map, (doc_id, highlights)| {
                match map.entry(doc_id.clone()) {
                    Vacant(entry) => { entry.insert(highlights.clone()); },
                    Occupied(mut entry) => {
                        let entry: &mut Vec<(usize, usize)> = entry.get_mut();
                        for highlight in highlights {
                            let coalesce_idx = entry.binary_search(highlight).err().unwrap();
                            entry.coalesce(coalesce_idx, highlight.clone());
                        }
                    }
                }
                map
            });
        let mut results: Vec<_> = map.into_iter()
                                     .map(|(doc_id, index_map)| {
                                         SearchResult::new(&self.docs[&doc_id],
                                                           index_map.into_iter().collect())
                                     })
                                     .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

#[allow(unused_attributes)]
#[rustfmt_skip]
fn analyze_doc(doc: &str) ->
    iter::FlatMap<
        iter::Filter<
            GroupBy<bool, CharIndices, fn(&(usize, char)) -> bool>,
            fn(&(bool, Vec<(usize, char)>)) -> bool>,
        iter::Map<ops::Range<usize>, Ngrams>,
        fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![(6, 8), (25, 27)]),
                                        (doc2, vec![(13, 15)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2", "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
        let (begin, end) = highlights[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }
}
<commit_msg>Typedef for postings map<commit_after>use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::hash_map::Entry::*;
use std::hash::{Hash, Hasher};
use std::iter;
use std::str::CharIndices;
use std::ops;
use itertools::{GroupBy, Itertools};
use coalesce::Coalesce;

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Debug, Eq, Ord, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into() }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    // Documents are unique only upon their id
    fn hash<H>(&self, state: &mut H)
        where H: Hasher
    {
        self.id.hash(state);
    }
}

impl PartialEq for Document {
    // Documents are unique only upon their id
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

impl PartialOrd for Document {
    fn partial_cmp(&self, other: &Document) -> Option<::std::cmp::Ordering> {
        self.id.partial_cmp(&other.id)
    }
}

/// A SearchResult is the representation of a Document returned for a specific set of search
/// terms. It is unique upon the document and the vec of highlight indices. It also contains a
/// search score for use in ranking against the other search results
#[derive(Clone, Debug, RustcEncodable)]
pub struct SearchResult<'a> {
    doc: &'a Document,
    highlights: Vec<(usize, usize)>,
    score: f32,
}

impl<'a> SearchResult<'a> {
    fn new(doc: &'a Document, mut highlights: Vec<(usize, usize)>) -> SearchResult<'a> {
        SearchResult {
            score: highlights.iter()
                .map(|&(begin, end)| end - begin)
                .sum::<usize>() as f32 /
                   (doc.content.len() as f32).sqrt(),
            doc: doc,
            highlights: {
                highlights.sort();
                highlights
            },
        }
    }

    /// Returns the document
    pub fn doc(&self) -> &Document {
        &self.doc
    }

    /// Returns the highlighted indices.
    ///
    /// Each `(usize, usize)` indicates the start and end of a term in the document's content
    /// that should be highlighted.
    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    /// Returns the search result's score.
    ///
    /// Score is computed by the product of the summed length of the matching terms and the inverse
    /// square root of the length of the document. Taking the square root of the document's length
    /// helps to combat bias toward short content.
    pub fn score(&self) -> f32 {
        self.score
    }

    pub fn highlight(&self, before: &str, after: &str) -> String {
        let mut begin_idx = 0;
        let mut parts = String::new();
        for &(begin, end) in &self.highlights {
            parts.push_str(&self.doc.content[begin_idx..begin]);
            parts.push_str(before);
            parts.push_str(&self.doc.content[begin..end]);
            parts.push_str(after);
            begin_idx = end;
        }
        parts.push_str(&self.doc.content[begin_idx..]);
        parts
    }
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

/// A Postings map (doc id => highlights) for a single term.
/// Records which Documents contain the term, and at which locations in the documents.
pub type PostingsMap = BTreeMap<String, Vec<(usize, usize)>>;

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex { index: BTreeMap::new(), docs: BTreeMap::new() }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let analyzed = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            let mut highlights = self.index
                                     .entry(ngram)
                                     .or_insert_with(BTreeMap::new)
                                     .entry(doc.id.clone())
                                     .or_insert_with(Vec::new);
            let coalesce_idx = highlights.binary_search(&highlighted).err().unwrap();
            highlights.coalesce(coalesce_idx, highlighted);
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        let map: HashMap<String, Vec<(usize, usize)>> = unique_terms.into_iter()
            .flat_map(|word| self.index.get(&word))
            .flat_map(BTreeMap::iter)
            .fold(HashMap::new(), |mut map, (doc_id, highlights)| {
                match map.entry(doc_id.clone()) {
                    Vacant(entry) => { entry.insert(highlights.clone()); },
                    Occupied(mut entry) => {
                        let entry: &mut Vec<(usize, usize)> = entry.get_mut();
                        for highlight in highlights {
                            let coalesce_idx = entry.binary_search(highlight).err().unwrap();
                            entry.coalesce(coalesce_idx, highlight.clone());
                        }
                    }
                }
                map
            });
        let mut results: Vec<_> = map.into_iter()
                                     .map(|(doc_id, index_map)| {
                                         SearchResult::new(&self.docs[&doc_id],
                                                           index_map.into_iter().collect())
                                     })
                                     .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

#[allow(unused_attributes)]
#[rustfmt_skip]
fn analyze_doc(doc: &str) ->
    iter::FlatMap<
        iter::Filter<
            GroupBy<bool, CharIndices, fn(&(usize, char)) -> bool>,
            fn(&(bool, Vec<(usize, char)>)) -> bool>,
        iter::Map<ops::Range<usize>, Ngrams>,
        fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![(6, 8), (25, 27)]),
                                        (doc2, vec![(13, 15)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2", "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
        let (begin, end) = highlights[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }
}
<|endoftext|>