<commit_before>use alloc::sync::{Arc, Weak};
use alloc::collections::{BTreeMap, VecDeque};
use core::sync::atomic::{AtomicUsize, Ordering};
use spin::{Mutex, Once, RwLock, RwLockReadGuard, RwLockWriteGuard};

use event;
use scheme::{AtomicSchemeId, ATOMIC_SCHEMEID_INIT, SchemeId};
use sync::WaitCondition;
use syscall::error::{Error, Result, EAGAIN, EBADF, EINTR, EINVAL, EPIPE, ESPIPE};
use syscall::flag::{EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK, MODE_FIFO};
use syscall::scheme::Scheme;
use syscall::data::Stat;

/// Pipes list
pub static PIPE_SCHEME_ID: AtomicSchemeId = ATOMIC_SCHEMEID_INIT;
static PIPE_NEXT_ID: AtomicUsize = AtomicUsize::new(0);
static PIPES: Once<RwLock<(BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)>> = Once::new();

/// Initialize pipes, called if needed
fn init_pipes() -> RwLock<(BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)> {
    RwLock::new((BTreeMap::new(), BTreeMap::new()))
}

/// Get the global pipes list, const
fn pipes() -> RwLockReadGuard<'static, (BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)> {
    PIPES.call_once(init_pipes).read()
}

/// Get the global pipes list, mutable
fn pipes_mut() -> RwLockWriteGuard<'static, (BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)> {
    PIPES.call_once(init_pipes).write()
}

pub fn pipe(flags: usize) -> (usize, usize) {
    let mut pipes = pipes_mut();
    let scheme_id = PIPE_SCHEME_ID.load(Ordering::SeqCst);
    let read_id = PIPE_NEXT_ID.fetch_add(1, Ordering::SeqCst);
    let write_id = PIPE_NEXT_ID.fetch_add(1, Ordering::SeqCst);
    let read = PipeRead::new(scheme_id, read_id, flags);
    let write = PipeWrite::new(&read, flags);
    pipes.0.insert(read_id, Arc::new(read));
    pipes.1.insert(write_id, Arc::new(write));
    (read_id, write_id)
}

pub struct PipeScheme;

impl PipeScheme {
    pub fn new(scheme_id: SchemeId) -> PipeScheme {
        PIPE_SCHEME_ID.store(scheme_id, Ordering::SeqCst);
        PipeScheme
    }
}

impl Scheme for PipeScheme {
    fn read(&self, id: usize, buf: &mut [u8]) -> Result<usize> {
        // Clone to prevent deadlocks
        let pipe = {
            let pipes = pipes();
            pipes.0.get(&id).map(|pipe| pipe.clone()).ok_or(Error::new(EBADF))?
        };

        pipe.read(buf)
    }

    fn write(&self, id: usize, buf: &[u8]) -> Result<usize> {
        // Clone to prevent deadlocks
        let pipe = {
            let pipes = pipes();
            pipes.1.get(&id).map(|pipe| pipe.clone()).ok_or(Error::new(EBADF))?
        };

        pipe.write(buf)
    }

    fn fcntl(&self, id: usize, cmd: usize, arg: usize) -> Result<usize> {
        let pipes = pipes();

        if let Some(pipe) = pipes.0.get(&id) {
            return pipe.fcntl(cmd, arg);
        }

        if let Some(pipe) = pipes.1.get(&id) {
            return pipe.fcntl(cmd, arg);
        }

        Err(Error::new(EBADF))
    }

    fn fevent(&self, id: usize, flags: usize) -> Result<usize> {
        let pipes = pipes();

        if let Some(pipe) = pipes.0.get(&id) {
            return pipe.fevent(flags);
        }

        Err(Error::new(EBADF))
    }

    fn fpath(&self, _id: usize, buf: &mut [u8]) -> Result<usize> {
        let mut i = 0;
        let scheme_path = b"pipe:";
        while i < buf.len() && i < scheme_path.len() {
            buf[i] = scheme_path[i];
            i += 1;
        }
        Ok(i)
    }

    fn fstat(&self, _id: usize, stat: &mut Stat) -> Result<usize> {
        *stat = Stat {
            st_mode: MODE_FIFO | 0o666,
            ..Default::default()
        };

        Ok(0)
    }

    fn fsync(&self, _id: usize) -> Result<usize> {
        Ok(0)
    }

    fn close(&self, id: usize) -> Result<usize> {
        let mut pipes = pipes_mut();

        drop(pipes.0.remove(&id));
        drop(pipes.1.remove(&id));

        Ok(0)
    }

    fn seek(&self, _id: usize, _pos: usize, _whence: usize) -> Result<usize> {
        Err(Error::new(ESPIPE))
    }
}

/// Read side of a pipe
pub struct PipeRead {
    scheme_id: SchemeId,
    event_id: usize,
    flags: AtomicUsize,
    condition: Arc<WaitCondition>,
    vec: Arc<Mutex<VecDeque<u8>>>
}

impl PipeRead {
    pub fn new(scheme_id: SchemeId, event_id: usize, flags: usize) -> Self {
        PipeRead {
            scheme_id: scheme_id,
            event_id: event_id,
            flags: AtomicUsize::new(flags),
            condition: Arc::new(WaitCondition::new()),
            vec: Arc::new(Mutex::new(VecDeque::new())),
        }
    }

    fn fcntl(&self, cmd: usize, arg: usize) -> Result<usize> {
        match cmd {
            F_GETFL => Ok(self.flags.load(Ordering::SeqCst)),
            F_SETFL => {
                self.flags.store(arg & ! O_ACCMODE, Ordering::SeqCst);
                Ok(0)
            },
            _ => Err(Error::new(EINVAL))
        }
    }

    fn fevent(&self, _flags: usize) -> Result<usize> {
        Ok(0)
    }

    fn read(&self, buf: &mut [u8]) -> Result<usize> {
        loop {
            {
                let mut vec = self.vec.lock();

                let mut i = 0;
                while i < buf.len() {
                    if let Some(b) = vec.pop_front() {
                        buf[i] = b;
                        i += 1;
                    } else {
                        break;
                    }
                }

                if i > 0 {
                    return Ok(i);
                }
            }

            if Arc::weak_count(&self.vec) == 0 {
                return Ok(0);
            } else if self.flags.load(Ordering::SeqCst) & O_NONBLOCK == O_NONBLOCK {
                return Err(Error::new(EAGAIN));
            } else {
                if ! self.condition.wait() {
                    return Err(Error::new(EINTR));
                }
            }
        }
    }
}

/// Read side of a pipe
pub struct PipeWrite {
    scheme_id: SchemeId,
    event_id: usize,
    flags: AtomicUsize,
    condition: Arc<WaitCondition>,
    vec: Option<Weak<Mutex<VecDeque<u8>>>>
}

impl PipeWrite {
    pub fn new(read: &PipeRead, flags: usize) -> Self {
        PipeWrite {
            scheme_id: read.scheme_id,
            event_id: read.event_id,
            flags: AtomicUsize::new(flags),
            condition: read.condition.clone(),
            vec: Some(Arc::downgrade(&read.vec)),
        }
    }

    fn fcntl(&self, cmd: usize, arg: usize) -> Result<usize> {
        match cmd {
            F_GETFL => Ok(self.flags.load(Ordering::SeqCst)),
            F_SETFL => {
                self.flags.store(arg & ! O_ACCMODE, Ordering::SeqCst);
                Ok(0)
            },
            _ => Err(Error::new(EINVAL))
        }
    }

    fn write(&self, buf: &[u8]) -> Result<usize> {
        if let Some(ref vec_weak) = self.vec {
            if let Some(vec_lock) = vec_weak.upgrade() {
                {
                    let mut vec = vec_lock.lock();

                    for &b in buf.iter() {
                        vec.push_back(b);
                    }
                }

                event::trigger(self.scheme_id, self.event_id, EVENT_READ);
                self.condition.notify();

                Ok(buf.len())
            } else {
                Err(Error::new(EPIPE))
            }
        } else {
            panic!("PipeWrite dropped before write");
        }
    }
}

impl Drop for PipeWrite {
    fn drop(&mut self) {
        drop(self.vec.take());
        event::trigger(self.scheme_id, self.event_id, EVENT_READ);
        self.condition.notify();
    }
}
<commit_msg>Fix event logic for pipes<commit_after>use alloc::sync::{Arc, Weak};
use alloc::collections::{BTreeMap, VecDeque};
use core::sync::atomic::{AtomicUsize, Ordering};
use spin::{Mutex, Once, RwLock, RwLockReadGuard, RwLockWriteGuard};

use event;
use scheme::{AtomicSchemeId, ATOMIC_SCHEMEID_INIT, SchemeId};
use sync::WaitCondition;
use syscall::error::{Error, Result, EAGAIN, EBADF, EINTR, EINVAL, EPIPE, ESPIPE};
use syscall::flag::{EVENT_READ, EVENT_WRITE, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK, MODE_FIFO};
use syscall::scheme::Scheme;
use syscall::data::Stat;

/// Pipes list
pub static PIPE_SCHEME_ID: AtomicSchemeId = ATOMIC_SCHEMEID_INIT;
static PIPE_NEXT_ID: AtomicUsize = AtomicUsize::new(0);
static PIPES: Once<RwLock<(BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)>> = Once::new();

/// Initialize pipes, called if needed
fn init_pipes() -> RwLock<(BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)> {
    RwLock::new((BTreeMap::new(), BTreeMap::new()))
}

/// Get the global pipes list, const
fn pipes() -> RwLockReadGuard<'static, (BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)> {
    PIPES.call_once(init_pipes).read()
}

/// Get the global pipes list, mutable
fn pipes_mut() -> RwLockWriteGuard<'static, (BTreeMap<usize, Arc<PipeRead>>, BTreeMap<usize, Arc<PipeWrite>>)> {
    PIPES.call_once(init_pipes).write()
}

pub fn pipe(flags: usize) -> (usize, usize) {
    let mut pipes = pipes_mut();
    let scheme_id = PIPE_SCHEME_ID.load(Ordering::SeqCst);
    let read_id = PIPE_NEXT_ID.fetch_add(1, Ordering::SeqCst);
    let write_id = PIPE_NEXT_ID.fetch_add(1, Ordering::SeqCst);
    let read = PipeRead::new(scheme_id, write_id, flags);
    let write = PipeWrite::new(&read, read_id, flags);
    pipes.0.insert(read_id, Arc::new(read));
    pipes.1.insert(write_id, Arc::new(write));
    (read_id, write_id)
}

pub struct PipeScheme;

impl PipeScheme {
    pub fn new(scheme_id: SchemeId) -> PipeScheme {
        PIPE_SCHEME_ID.store(scheme_id, Ordering::SeqCst);
        PipeScheme
    }
}

impl Scheme for PipeScheme {
    fn read(&self, id: usize, buf: &mut [u8]) -> Result<usize> {
        // Clone to prevent deadlocks
        let pipe = {
            let pipes = pipes();
            pipes.0.get(&id).map(|pipe| pipe.clone()).ok_or(Error::new(EBADF))?
        };

        pipe.read(buf)
    }

    fn write(&self, id: usize, buf: &[u8]) -> Result<usize> {
        // Clone to prevent deadlocks
        let pipe = {
            let pipes = pipes();
            pipes.1.get(&id).map(|pipe| pipe.clone()).ok_or(Error::new(EBADF))?
        };

        pipe.write(buf)
    }

    fn fcntl(&self, id: usize, cmd: usize, arg: usize) -> Result<usize> {
        let pipes = pipes();

        if let Some(pipe) = pipes.0.get(&id) {
            return pipe.fcntl(cmd, arg);
        }

        if let Some(pipe) = pipes.1.get(&id) {
            return pipe.fcntl(cmd, arg);
        }

        Err(Error::new(EBADF))
    }

    fn fevent(&self, id: usize, flags: usize) -> Result<usize> {
        let pipes = pipes();

        if let Some(pipe) = pipes.0.get(&id) {
            if flags == EVENT_READ {
                // TODO: Return correct flags
                if pipe.vec.lock().is_empty() {
                    return Ok(0);
                } else {
                    return Ok(EVENT_READ);
                }
            }
        }

        if let Some(pipe) = pipes.1.get(&id) {
            if flags == EVENT_WRITE {
                return Ok(EVENT_WRITE);
            }
        }

        Err(Error::new(EBADF))
    }

    fn fpath(&self, _id: usize, buf: &mut [u8]) -> Result<usize> {
        let mut i = 0;
        let scheme_path = b"pipe:";
        while i < buf.len() && i < scheme_path.len() {
            buf[i] = scheme_path[i];
            i += 1;
        }
        Ok(i)
    }

    fn fstat(&self, _id: usize, stat: &mut Stat) -> Result<usize> {
        *stat = Stat {
            st_mode: MODE_FIFO | 0o666,
            ..Default::default()
        };

        Ok(0)
    }

    fn fsync(&self, _id: usize) -> Result<usize> {
        Ok(0)
    }

    fn close(&self, id: usize) -> Result<usize> {
        let mut pipes = pipes_mut();

        drop(pipes.0.remove(&id));
        drop(pipes.1.remove(&id));

        Ok(0)
    }

    fn seek(&self, _id: usize, _pos: usize, _whence: usize) -> Result<usize> {
        Err(Error::new(ESPIPE))
    }
}

/// Read side of a pipe
pub struct PipeRead {
    scheme_id: SchemeId,
    write_id: usize,
    flags: AtomicUsize,
    condition: Arc<WaitCondition>,
    vec: Arc<Mutex<VecDeque<u8>>>
}

impl PipeRead {
    pub fn new(scheme_id: SchemeId, write_id: usize, flags: usize) -> Self {
        PipeRead {
            scheme_id,
            write_id,
            flags: AtomicUsize::new(flags),
            condition: Arc::new(WaitCondition::new()),
            vec: Arc::new(Mutex::new(VecDeque::new())),
        }
    }

    fn fcntl(&self, cmd: usize, arg: usize) -> Result<usize> {
        match cmd {
            F_GETFL => Ok(self.flags.load(Ordering::SeqCst)),
            F_SETFL => {
                self.flags.store(arg & ! O_ACCMODE, Ordering::SeqCst);
                Ok(0)
            },
            _ => Err(Error::new(EINVAL))
        }
    }

    fn read(&self, buf: &mut [u8]) -> Result<usize> {
        loop {
            {
                let mut vec = self.vec.lock();

                let mut i = 0;
                while i < buf.len() {
                    if let Some(b) = vec.pop_front() {
                        buf[i] = b;
                        i += 1;
                    } else {
                        break;
                    }
                }

                if i > 0 {
                    event::trigger(self.scheme_id, self.write_id, EVENT_WRITE);

                    return Ok(i);
                }
            }

            if Arc::weak_count(&self.vec) == 0 {
                return Ok(0);
            } else if self.flags.load(Ordering::SeqCst) & O_NONBLOCK == O_NONBLOCK {
                return Err(Error::new(EAGAIN));
            } else {
                if ! self.condition.wait() {
                    return Err(Error::new(EINTR));
                }
            }
        }
    }
}

/// Read side of a pipe
pub struct PipeWrite {
    scheme_id: SchemeId,
    read_id: usize,
    flags: AtomicUsize,
    condition: Arc<WaitCondition>,
    vec: Option<Weak<Mutex<VecDeque<u8>>>>
}

impl PipeWrite {
    pub fn new(read: &PipeRead, read_id: usize, flags: usize) -> Self {
        PipeWrite {
            scheme_id: read.scheme_id,
            read_id,
            flags: AtomicUsize::new(flags),
            condition: read.condition.clone(),
            vec: Some(Arc::downgrade(&read.vec)),
        }
    }

    fn fcntl(&self, cmd: usize, arg: usize) -> Result<usize> {
        match cmd {
            F_GETFL => Ok(self.flags.load(Ordering::SeqCst)),
            F_SETFL => {
                self.flags.store(arg & ! O_ACCMODE, Ordering::SeqCst);
                Ok(0)
            },
            _ => Err(Error::new(EINVAL))
        }
    }

    fn write(&self, buf: &[u8]) -> Result<usize> {
        if let Some(ref vec_weak) = self.vec {
            if let Some(vec_lock) = vec_weak.upgrade() {
                {
                    let mut vec = vec_lock.lock();

                    for &b in buf.iter() {
                        vec.push_back(b);
                    }
                }

                event::trigger(self.scheme_id, self.read_id, EVENT_READ);
                self.condition.notify();

                Ok(buf.len())
            } else {
                Err(Error::new(EPIPE))
            }
        } else {
            panic!("PipeWrite dropped before write");
        }
    }
}

impl Drop for PipeWrite {
    fn drop(&mut self) {
        drop(self.vec.take());
        event::trigger(self.scheme_id, self.read_id, EVENT_READ);
        self.condition.notify();
    }
}
<|endoftext|>


<commit_before>use Decimal;
use num::FromPrimitive;
use serde;
use std::fmt;
use std::str::FromStr;
use serde::de::Unexpected;
use num::Zero;

impl<'de> serde::Deserialize<'de> for Decimal {
    fn deserialize<D>(deserializer: D) -> Result<Decimal, D::Error>
    where
        D: serde::de::Deserializer<'de>, {
        deserializer.deserialize_any(DecimalVisitor)
    }
}

struct DecimalVisitor;

impl<'de> serde::de::Visitor<'de> for DecimalVisitor {
    type Value = Decimal;

    fn visit_f64<E>(self, value: f64) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        Decimal::from_str(&value.to_string()).map_err(|_| E::invalid_value(Unexpected::Float(value), &self))
    }

    fn visit_i64<E>(self, value: i64) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        match Decimal::from_i64(value) {
            Some(s) => Ok(s),
            None => Err(E::invalid_value(Unexpected::Signed(value), &self)),
        }
    }

    fn visit_u64<E>(self, value: u64) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        match Decimal::from_u64(value) {
            Some(s) => Ok(s),
            None => Err(E::invalid_value(Unexpected::Unsigned(value), &self)),
        }
    }

    fn visit_str<E>(self, value: &str) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        Decimal::from_str(value).map_err(|_| E::invalid_value(Unexpected::Str(value), &self))
    }

    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(
            formatter,
            "a Decimal type representing a fixed-point number"
        )
    }
}

impl serde::Serialize for Decimal {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer, {
        serializer.serialize_str(&self.to_string())
    }
}

#[cfg(test)]
mod test {
    extern crate serde_json;

    use super::*;

    #[derive(Serialize, Deserialize, Debug)]
    struct Record {
        amount: Decimal
    }

    #[test]
    fn deserialize_valid_decimal() {
        let data = [
            ("{\"amount\":\"1.234\"}", "1.234"),
            ("{\"amount\":1234}", "1234"),
        ];
        for &(serialized,value) in data.iter() {
            let record : Record = serde_json::from_str(serialized).unwrap();
            assert_eq!(value, record.amount.to_string());
        }
    }

    #[test]
    #[should_panic]
    fn deserialize_invalid_decimal() {
        let serialized = "{\"amount\":\"foo\"}";
        let _ : Record = serde_json::from_str(serialized).unwrap();
    }

    #[test]
    fn serialize_decimal() {
        let record = Record { amount: Decimal::new(1234, 3) };
        let serialized = serde_json::to_string(&record).unwrap();
        assert_eq!("{\"amount\":\"1.234\"}", serialized);
    }
}
<commit_msg>Replace deserialize_any with deserialize_str for bincode support<commit_after>use Decimal;
use num::FromPrimitive;
use serde;
use std::fmt;
use std::str::FromStr;
use serde::de::Unexpected;

impl<'de> serde::Deserialize<'de> for Decimal {
    fn deserialize<D>(deserializer: D) -> Result<Decimal, D::Error>
    where
        D: serde::de::Deserializer<'de>, {
        deserializer.deserialize_str(DecimalVisitor)
    }
}

struct DecimalVisitor;

impl<'de> serde::de::Visitor<'de> for DecimalVisitor {
    type Value = Decimal;

    fn visit_f64<E>(self, value: f64) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        Decimal::from_str(&value.to_string()).map_err(|_| E::invalid_value(Unexpected::Float(value), &self))
    }

    fn visit_i64<E>(self, value: i64) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        match Decimal::from_i64(value) {
            Some(s) => Ok(s),
            None => Err(E::invalid_value(Unexpected::Signed(value), &self)),
        }
    }

    fn visit_u64<E>(self, value: u64) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        match Decimal::from_u64(value) {
            Some(s) => Ok(s),
            None => Err(E::invalid_value(Unexpected::Unsigned(value), &self)),
        }
    }

    fn visit_str<E>(self, value: &str) -> Result<Decimal, E>
        where E: serde::de::Error
    {
        Decimal::from_str(value).map_err(|_| E::invalid_value(Unexpected::Str(value), &self))
    }

    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(
            formatter,
            "a Decimal type representing a fixed-point number"
        )
    }
}

impl serde::Serialize for Decimal {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer, {
        serializer.serialize_str(&self.to_string())
    }
}

#[cfg(test)]
mod test {
    extern crate serde_json;

    use super::*;

    #[derive(Serialize, Deserialize, Debug)]
    struct Record {
        amount: Decimal
    }

    #[test]
    fn deserialize_valid_decimal() {
        let data = [
            ("{\"amount\":\"1.234\"}", "1.234"),
            ("{\"amount\":1234}", "1234"),
        ];
        for &(serialized,value) in data.iter() {
            let result = serde_json::from_str(serialized);
            assert_eq!(true, result.is_ok(), "expected successful deseralization for {}. Error: {:?}", serialized, result.err().unwrap());
            let record : Record = result.unwrap();
            assert_eq!(value, record.amount.to_string(), "expected: {}, actual: {}", value, record.amount.to_string());
        }
    }

    #[test]
    #[should_panic]
    fn deserialize_invalid_decimal() {
        let serialized = "{\"amount\":\"foo\"}";
        let _ : Record = serde_json::from_str(serialized).unwrap();
    }

    #[test]
    fn serialize_decimal() {
        let record = Record { amount: Decimal::new(1234, 3) };
        let serialized = serde_json::to_string(&record).unwrap();
        assert_eq!("{\"amount\":\"1.234\"}", serialized);
    }
}
<|endoftext|>


<commit_before>//! Panic runtime for Miri.
//!
//! The core pieces of the runtime are:
//! - An implementation of `__rust_maybe_catch_panic` that pushes the invoked stack frame with
//!   some extra metadata derived from the panic-catching arguments of `__rust_maybe_catch_panic`.
//! - A hack in `libpanic_unwind` that calls the `miri_start_panic` intrinsic instead of the
//!   target-native panic runtime. (This lives in the rustc repo.)
//! - An implementation of `miri_start_panic` that stores its argument (the panic payload), and then
//!   immediately returns, but on the *unwind* edge (not the normal return edge), thus initiating unwinding.
//! - A hook executed each time a frame is popped, such that if the frame pushed by `__rust_maybe_catch_panic`
//!   gets popped *during unwinding*, we take the panic payload and store it according to the extra
//!   metadata we remembered when pushing said frame.

use rustc::mir;
use crate::*;
use super::machine::FrameData;
use rustc_target::spec::PanicStrategy;
use crate::rustc_target::abi::LayoutOf;

/// Holds all of the relevant data for a call to
/// `__rust_maybe_catch_panic`.
///
/// If a panic occurs, we update this data with
/// the information from the panic site.
#[derive(Debug)]
pub struct CatchUnwindData<'tcx> {
    /// The dereferenced `data_ptr` argument passed to `__rust_maybe_catch_panic`.
    pub data_place: MPlaceTy<'tcx, Tag>,
    /// The dereferenced `vtable_ptr` argument passed to `__rust_maybe_catch_panic`.
    pub vtable_place: MPlaceTy<'tcx, Tag>,
    /// The `dest` from the original call to `__rust_maybe_catch_panic`.
    pub dest: PlaceTy<'tcx, Tag>,
}

impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {

    /// Handles the special "miri_start_panic" intrinsic, which is called
    /// by libpanic_unwind to delegate the actual unwinding process to Miri.
    #[inline(always)]
    fn handle_miri_start_panic(
        &mut self,
        args: &[OpTy<'tcx, Tag>],
        unwind: Option<mir::BasicBlock>
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        trace!("miri_start_panic: {:?}", this.frame().span);

        let scalar = this.read_immediate(args[0])?;
        assert!(this.machine.panic_payload.is_none(), "the panic runtime should avoid double-panics");
        this.machine.panic_payload = Some(scalar);

        // Jump to the unwind block to begin unwinding.
        // We don't use `goto_block` as that is just meant for normal returns.
        let next_frame = this.frame_mut();
        next_frame.block = unwind;
        next_frame.stmt = 0;
        return Ok(())
    }

    #[inline(always)]
    fn handle_catch_panic(
        &mut self,
        args: &[OpTy<'tcx, Tag>],
        dest: PlaceTy<'tcx, Tag>,
        ret: mir::BasicBlock,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let tcx = &{this.tcx.tcx};

        // fn __rust_maybe_catch_panic(
        //     f: fn(*mut u8),
        //     data: *mut u8,
        //     data_ptr: *mut usize,
        //     vtable_ptr: *mut usize,
        // ) -> u32

        // Get all the arguments.
        let f = this.read_scalar(args[0])?.not_undef()?;
        let f_arg = this.read_scalar(args[1])?.not_undef()?;
        let data_place = this.deref_operand(args[2])?;
        let vtable_place = this.deref_operand(args[3])?;

        // Now we make a function call, and pass `f_arg` as first and only argument.
        let f_instance = this.memory.get_fn(f)?.as_instance()?;
        trace!("__rust_maybe_catch_panic: {:?}", f_instance);
        // TODO: consider making this reusable? `InterpCx::step` does something similar
        // for the TLS destructors, and of course `eval_main`.
        let mir = this.load_mir(f_instance.def, None)?;
        let ret_place =
            MPlaceTy::dangling(this.layout_of(tcx.mk_unit())?, this).into();
        this.push_stack_frame(
            f_instance,
            mir.span,
            mir,
            Some(ret_place),
            // Directly return to caller.
            StackPopCleanup::Goto { ret: Some(ret), unwind: None },
        )?;

        let mut args = this.frame().body.args_iter();
        // First argument.
        let arg_local = args
            .next()
            .expect("Argument to __rust_maybe_catch_panic does not take enough arguments.");
        let arg_dest = this.local_place(arg_local)?;
        this.write_scalar(f_arg, arg_dest)?;
        // No more arguments.
        args.next().expect_none("__rust_maybe_catch_panic argument has more arguments than expected");

        // We ourselves will return `0`, eventually (will be overwritten if we catch a panic).
        this.write_null(dest)?;

        // In unwind mode, we tag this frame with some extra data.
        // This lets `handle_stack_pop` (below) know that we should stop unwinding
        // when we pop this frame.
        if this.tcx.tcx.sess.panic_strategy() == PanicStrategy::Unwind {
            this.frame_mut().extra.catch_panic = Some(CatchUnwindData {
                data_place,
                vtable_place,
                dest,
            })
        }

        return Ok(());
    }

    #[inline(always)]
    fn handle_stack_pop(
        &mut self,
        mut extra: FrameData<'tcx>,
        unwinding: bool
    ) -> InterpResult<'tcx, StackPopInfo> {
        let this = self.eval_context_mut();

        trace!("handle_stack_pop(extra = {:?}, unwinding = {})", extra, unwinding);

        // We only care about `catch_panic` if we're unwinding - if we're doing a normal
        // return, then we don't need to do anything special.
        let res = if let (true, Some(unwind_data)) = (unwinding, extra.catch_panic.take()) {
            // We've just popped a frame that was pushed by `__rust_maybe_catch_panic`,
            // and we are unwinding, so we should catch that.
            trace!("unwinding: found catch_panic frame during unwinding: {:?}", this.frame().span);

            // `panic_payload` now holds a `*mut (dyn Any + Send)`,
            // provided by the `miri_start_panic` intrinsic.
            // We want to split this into its consituient parts -
            // the data and vtable pointers - and store them according to
            // `unwind_data`, i.e., we store them where `__rust_maybe_catch_panic`
            // was told to put them.
            let payload = this.machine.panic_payload.take().unwrap();
            let payload = this.ref_to_mplace(payload)?;
            let payload_data_place = payload.ptr;
            let payload_vtable_place = payload.meta.expect("Expected fat pointer");

            this.write_scalar(payload_data_place, unwind_data.data_place.into())?;
            this.write_scalar(payload_vtable_place, unwind_data.vtable_place.into())?;

            // We set the return value of `__rust_maybe_catch_panic` to 1,
            // since there was a panic.
            let dest = unwind_data.dest;
            this.write_scalar(Scalar::from_int(1, dest.layout.size), dest)?;

            StackPopInfo::StopUnwinding
        } else {
            StackPopInfo::Normal
        };
        this.memory.extra.stacked_borrows.borrow_mut().end_call(extra.call_id);
        Ok(res)
    }
}
<commit_msg>Re-add comment<commit_after>//! Panic runtime for Miri.
//!
//! The core pieces of the runtime are:
//! - An implementation of `__rust_maybe_catch_panic` that pushes the invoked stack frame with
//!   some extra metadata derived from the panic-catching arguments of `__rust_maybe_catch_panic`.
//! - A hack in `libpanic_unwind` that calls the `miri_start_panic` intrinsic instead of the
//!   target-native panic runtime. (This lives in the rustc repo.)
//! - An implementation of `miri_start_panic` that stores its argument (the panic payload), and then
//!   immediately returns, but on the *unwind* edge (not the normal return edge), thus initiating unwinding.
//! - A hook executed each time a frame is popped, such that if the frame pushed by `__rust_maybe_catch_panic`
//!   gets popped *during unwinding*, we take the panic payload and store it according to the extra
//!   metadata we remembered when pushing said frame.

use rustc::mir;
use crate::*;
use super::machine::FrameData;
use rustc_target::spec::PanicStrategy;
use crate::rustc_target::abi::LayoutOf;

/// Holds all of the relevant data for a call to
/// `__rust_maybe_catch_panic`.
///
/// If a panic occurs, we update this data with
/// the information from the panic site.
#[derive(Debug)]
pub struct CatchUnwindData<'tcx> {
    /// The dereferenced `data_ptr` argument passed to `__rust_maybe_catch_panic`.
    pub data_place: MPlaceTy<'tcx, Tag>,
    /// The dereferenced `vtable_ptr` argument passed to `__rust_maybe_catch_panic`.
    pub vtable_place: MPlaceTy<'tcx, Tag>,
    /// The `dest` from the original call to `__rust_maybe_catch_panic`.
    pub dest: PlaceTy<'tcx, Tag>,
}

impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {

    /// Handles the special "miri_start_panic" intrinsic, which is called
    /// by libpanic_unwind to delegate the actual unwinding process to Miri.
    #[inline(always)]
    fn handle_miri_start_panic(
        &mut self,
        args: &[OpTy<'tcx, Tag>],
        unwind: Option<mir::BasicBlock>
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        trace!("miri_start_panic: {:?}", this.frame().span);

        // Get the raw pointer stored in arg[0] (the panic payload).
        let scalar = this.read_immediate(args[0])?;
        assert!(this.machine.panic_payload.is_none(), "the panic runtime should avoid double-panics");
        this.machine.panic_payload = Some(scalar);

        // Jump to the unwind block to begin unwinding.
        // We don't use `goto_block` as that is just meant for normal returns.
        let next_frame = this.frame_mut();
        next_frame.block = unwind;
        next_frame.stmt = 0;
        return Ok(())
    }

    #[inline(always)]
    fn handle_catch_panic(
        &mut self,
        args: &[OpTy<'tcx, Tag>],
        dest: PlaceTy<'tcx, Tag>,
        ret: mir::BasicBlock,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        let tcx = &{this.tcx.tcx};

        // fn __rust_maybe_catch_panic(
        //     f: fn(*mut u8),
        //     data: *mut u8,
        //     data_ptr: *mut usize,
        //     vtable_ptr: *mut usize,
        // ) -> u32

        // Get all the arguments.
        let f = this.read_scalar(args[0])?.not_undef()?;
        let f_arg = this.read_scalar(args[1])?.not_undef()?;
        let data_place = this.deref_operand(args[2])?;
        let vtable_place = this.deref_operand(args[3])?;

        // Now we make a function call, and pass `f_arg` as first and only argument.
        let f_instance = this.memory.get_fn(f)?.as_instance()?;
        trace!("__rust_maybe_catch_panic: {:?}", f_instance);
        // TODO: consider making this reusable? `InterpCx::step` does something similar
        // for the TLS destructors, and of course `eval_main`.
        let mir = this.load_mir(f_instance.def, None)?;
        let ret_place =
            MPlaceTy::dangling(this.layout_of(tcx.mk_unit())?, this).into();
        this.push_stack_frame(
            f_instance,
            mir.span,
            mir,
            Some(ret_place),
            // Directly return to caller.
            StackPopCleanup::Goto { ret: Some(ret), unwind: None },
        )?;

        let mut args = this.frame().body.args_iter();
        // First argument.
        let arg_local = args
            .next()
            .expect("Argument to __rust_maybe_catch_panic does not take enough arguments.");
        let arg_dest = this.local_place(arg_local)?;
        this.write_scalar(f_arg, arg_dest)?;
        // No more arguments.
        args.next().expect_none("__rust_maybe_catch_panic argument has more arguments than expected");

        // We ourselves will return `0`, eventually (will be overwritten if we catch a panic).
        this.write_null(dest)?;

        // In unwind mode, we tag this frame with some extra data.
        // This lets `handle_stack_pop` (below) know that we should stop unwinding
        // when we pop this frame.
        if this.tcx.tcx.sess.panic_strategy() == PanicStrategy::Unwind {
            this.frame_mut().extra.catch_panic = Some(CatchUnwindData {
                data_place,
                vtable_place,
                dest,
            })
        }

        return Ok(());
    }

    #[inline(always)]
    fn handle_stack_pop(
        &mut self,
        mut extra: FrameData<'tcx>,
        unwinding: bool
    ) -> InterpResult<'tcx, StackPopInfo> {
        let this = self.eval_context_mut();

        trace!("handle_stack_pop(extra = {:?}, unwinding = {})", extra, unwinding);

        // We only care about `catch_panic` if we're unwinding - if we're doing a normal
        // return, then we don't need to do anything special.
        let res = if let (true, Some(unwind_data)) = (unwinding, extra.catch_panic.take()) {
            // We've just popped a frame that was pushed by `__rust_maybe_catch_panic`,
            // and we are unwinding, so we should catch that.
            trace!("unwinding: found catch_panic frame during unwinding: {:?}", this.frame().span);

            // `panic_payload` now holds a `*mut (dyn Any + Send)`,
            // provided by the `miri_start_panic` intrinsic.
            // We want to split this into its consituient parts -
            // the data and vtable pointers - and store them according to
            // `unwind_data`, i.e., we store them where `__rust_maybe_catch_panic`
            // was told to put them.
            let payload = this.machine.panic_payload.take().unwrap();
            let payload = this.ref_to_mplace(payload)?;
            let payload_data_place = payload.ptr;
            let payload_vtable_place = payload.meta.expect("Expected fat pointer");

            this.write_scalar(payload_data_place, unwind_data.data_place.into())?;
            this.write_scalar(payload_vtable_place, unwind_data.vtable_place.into())?;

            // We set the return value of `__rust_maybe_catch_panic` to 1,
            // since there was a panic.
            let dest = unwind_data.dest;
            this.write_scalar(Scalar::from_int(1, dest.layout.size), dest)?;

            StackPopInfo::StopUnwinding
        } else {
            StackPopInfo::Normal
        };
        this.memory.extra.stacked_borrows.borrow_mut().end_call(extra.call_id);
        Ok(res)
    }
}
<|endoftext|>


<commit_before>use tokenizer::{TokenStream, Tokenizer, Token};
use std::collections::BTreeMap;
use Term;
use Document;
use Index;
use schema::FieldValue;
use schema::Value;
use tokenizer::BoxedTokenizer;
use htmlescape::encode_minimal;

#[derive(Debug)]
pub struct HighlightSection {
    start: usize,
    stop: usize,
}

impl HighlightSection {
    fn new(start: usize, stop: usize) -> HighlightSection {
        HighlightSection {
            start,
            stop
        }
    }
}

#[derive(Debug)]
pub struct FragmentCandidate {
    score: f32,
    start_offset: usize,
    stop_offset: usize,
    num_chars: usize,
    highlighted: Vec<HighlightSection>,
}

impl FragmentCandidate {

    fn new(start_offset: usize, end_offset: usize) -> FragmentCandidate {
        FragmentCandidate{score: 0.0,
                          start_offset: start_offset,
                          stop_offset: end_offset,
                          num_chars: 0,
                          highlighted: vec![]}
    }

    /// Updates `score` and `highlighted` fields of the objects.
    ///
    ///
    fn calculate_score(&mut self, token: &Token, terms: &BTreeMap<String, f32>) {
        if let Some(score) = terms.get(&token.text.to_lowercase()) {
            self.score += score;
            self.highlighted.push(HighlightSection{start: token.offset_from,
                                                   stop: token.offset_to});
        }
    }
}

#[derive(Debug)]
pub struct Snippet {
    fragments: String,
    highlighted: Vec<HighlightSection>,
}

const HIGHLIGHTEN_PREFIX:&str = "";
const HIGHLIGHTEN_POSTFIX:&str = "";

impl Snippet {

    /// Returns a hignlightned html from the `Snippet`.
    pub fn to_html(&self) -> String {
        let mut html = String::new();
        let mut start_from: usize = 0;

        for item in self.highlighted.iter() {
            html.push_str(&encode_minimal(&self.fragments[start_from..item.start]));
            html.push_str(HIGHLIGHTEN_PREFIX);
            html.push_str(&encode_minimal(&self.fragments[item.start..item.stop]));
            html.push_str(HIGHLIGHTEN_POSTFIX);
            start_from = item.stop;
        }
        html.push_str(&encode_minimal(&self.fragments[start_from..self.fragments.len()]));
        html
    }
}

/// Returns a non-empty list of "good" fragments.
///
/// If no target term is within the text, then the function
/// should return an empty Vec.
///
/// If a target term is within the text, then the returned
/// list is required to be non-empty.
///
/// The returned list is non-empty and contain less
/// than 12 possibly overlapping fragments.
///
/// All fragments should contain at least one target term
/// and have at most `max_num_chars` characters (not bytes).
///
/// It is ok to emit non-overlapping fragments, for instance,
/// one short and one long containing the same keyword, in order
/// to leave optimization opportunity to the fragment selector
/// upstream.
///
/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
/// has to be a valid string.
fn search_fragments<'a>(
    tokenizer: Box<BoxedTokenizer>,
    text: &'a str,
    terms: BTreeMap<String, f32>,
    max_num_chars: usize) -> Vec<FragmentCandidate> {
    let mut token_stream = tokenizer.token_stream(text);
    let mut fragment = FragmentCandidate::new(0, 0);
    let mut fragments:Vec<FragmentCandidate> = vec![];

    while let Some(next) = token_stream.next() {
        if (next.offset_to - fragment.start_offset) > max_num_chars {
            if fragment.score > 0.0 {
                fragments.push(fragment)
            };
            fragment = FragmentCandidate::new(next.offset_from, next.offset_to);
        } else {
            fragment.calculate_score(next, &terms);
            fragment.stop_offset = next.offset_to;
        }
    }
    if fragment.score > 0.0 {
        fragments.push(fragment)
    }

    fragments
}

/// Returns a Snippet
///
/// Takes a vector of `FragmentCandidate`s and the text.
/// Figures out the best fragment from it and creates a snippet.
fn select_best_fragment_combination<'a>(fragments: Vec<FragmentCandidate>,
                                        text: &'a str,) -> Snippet {
    if let Some(init) = fragments.iter().nth(0) {
        let fragment = fragments.iter().skip(1).fold(init, |acc, item| {
            if item.score > acc.score { item } else { acc }
        });
        let fragment_text = &text[fragment.start_offset..fragment.stop_offset];
        let highlighted = fragment.highlighted.iter().map(|item| {
            HighlightSection{start: item.start-fragment.start_offset,
                             stop: item.stop-fragment.start_offset}
        }).collect();
        Snippet{fragments: fragment_text.to_owned(),
                highlighted: highlighted}
    } else {
        // when there no fragments to chose from,
        // for now create a empty snippet
        Snippet{fragments: String::new(),
                highlighted: vec![]}
    }
}

pub fn generate_snippet<'a>(
    doc: &'a [FieldValue],
    index: &Index,
    terms: Vec<Term>,
    max_num_chars: usize) -> Snippet {
    unimplemented!();
}

#[cfg(test)]
mod tests {
    use tokenizer::{SimpleTokenizer, box_tokenizer};
    use std::iter::Iterator;
    use std::collections::BTreeMap;
    use super::{search_fragments, select_best_fragment_combination};

    #[test]
    fn test_snippet() {
        let tokenizer = SimpleTokenizer;

        let t = box_tokenizer(tokenizer);

        let text = "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide better memory safety while still maintaining performance.

Rust is free and open-source software, released under an MIT License, or Apache License 2.0. Its designers have refined the language through the experiences of writing the Servo web browser layout engine[14] and the Rust compiler. A large proportion of current commits to the project are from community members.[15]

Rust won first place for \"most loved programming language\" in the Stack Overflow Developer Survey in 2016, 2017, and 2018.
";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("rust"), 1.0);
        terms.insert(String::from("language"), 0.9);

        let fragments = search_fragments(t, &text, terms, 100);
        assert_eq!(fragments.len(), 7);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.9);
            assert_eq!(first.stop_offset, 89);
        }
        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe".to_owned());
        assert_eq!(snippet.to_html(), "Rust is a systems programming language sponsored by Mozilla which describes it as a "safe".to_owned())
    }
}
<commit_msg>update `calculate_score` to `try_add_token`<commit_after>use tokenizer::{TokenStream, Tokenizer, Token};
use std::collections::BTreeMap;
use Term;
use Document;
use Index;
use schema::FieldValue;
use schema::Value;
use tokenizer::BoxedTokenizer;
use htmlescape::encode_minimal;

#[derive(Debug)]
pub struct HighlightSection {
    start: usize,
    stop: usize,
}

impl HighlightSection {
    fn new(start: usize, stop: usize) -> HighlightSection {
        HighlightSection {
            start,
            stop
        }
    }
}

#[derive(Debug)]
pub struct FragmentCandidate {
    score: f32,
    start_offset: usize,
    stop_offset: usize,
    num_chars: usize,
    highlighted: Vec<HighlightSection>,
}

impl FragmentCandidate {

    /// Create a basic `FragmentCandidate`
    ///
    /// `score`, `num_chars` are set to 0
    /// and `highlighted` is set to empty vec
    /// stop_offset is set to start_offset, which is taken as a param.
    fn new(start_offset: usize) -> FragmentCandidate {
        FragmentCandidate{score: 0.0,
                          start_offset: start_offset,
                          stop_offset: start_offset,
                          num_chars: 0,
                          highlighted: vec![]}
    }

    /// Updates `score` and `highlighted` fields of the objects.
    ///
    /// taking the token and terms, the token is added to the fragment.
    /// if the token is one of the terms, the score
    /// and highlighted fields are updated in the fragment.
    fn try_add_token(&mut self, token: &Token, terms: &BTreeMap<String, f32>) {
        self.stop_offset = token.offset_to;

        if let Some(score) = terms.get(&token.text.to_lowercase()) {
            self.score += score;
            self.highlighted.push(HighlightSection{start: token.offset_from,
                                                   stop: token.offset_to});
        }
    }
}

#[derive(Debug)]
pub struct Snippet {
    fragments: String,
    highlighted: Vec<HighlightSection>,
}

const HIGHLIGHTEN_PREFIX:&str = "";
const HIGHLIGHTEN_POSTFIX:&str = "";

impl Snippet {

    /// Returns a hignlightned html from the `Snippet`.
    pub fn to_html(&self) -> String {
        let mut html = String::new();
        let mut start_from: usize = 0;

        for item in self.highlighted.iter() {
            html.push_str(&encode_minimal(&self.fragments[start_from..item.start]));
            html.push_str(HIGHLIGHTEN_PREFIX);
            html.push_str(&encode_minimal(&self.fragments[item.start..item.stop]));
            html.push_str(HIGHLIGHTEN_POSTFIX);
            start_from = item.stop;
        }
        html.push_str(&encode_minimal(&self.fragments[start_from..self.fragments.len()]));
        html
    }
}

/// Returns a non-empty list of "good" fragments.
///
/// If no target term is within the text, then the function
/// should return an empty Vec.
///
/// If a target term is within the text, then the returned
/// list is required to be non-empty.
///
/// The returned list is non-empty and contain less
/// than 12 possibly overlapping fragments.
///
/// All fragments should contain at least one target term
/// and have at most `max_num_chars` characters (not bytes).
///
/// It is ok to emit non-overlapping fragments, for instance,
/// one short and one long containing the same keyword, in order
/// to leave optimization opportunity to the fragment selector
/// upstream.
///
/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
/// has to be a valid string.
fn search_fragments<'a>(
    tokenizer: Box<BoxedTokenizer>,
    text: &'a str,
    terms: BTreeMap<String, f32>,
    max_num_chars: usize) -> Vec<FragmentCandidate> {
    let mut token_stream = tokenizer.token_stream(text);
    let mut fragment = FragmentCandidate::new(0);
    let mut fragments:Vec<FragmentCandidate> = vec![];

    while let Some(next) = token_stream.next() {
        if (next.offset_to - fragment.start_offset) > max_num_chars {
            if fragment.score > 0.0 {
                fragments.push(fragment)
            };
            fragment = FragmentCandidate::new(next.offset_from);
        } else {
            fragment.try_add_token(next, &terms);
        }
    }
    if fragment.score > 0.0 {
        fragments.push(fragment)
    }

    fragments
}

/// Returns a Snippet
///
/// Takes a vector of `FragmentCandidate`s and the text.
/// Figures out the best fragment from it and creates a snippet.
fn select_best_fragment_combination<'a>(fragments: Vec<FragmentCandidate>,
                                        text: &'a str,) -> Snippet {
    if let Some(init) = fragments.iter().nth(0) {
        let fragment = fragments.iter().skip(1).fold(init, |acc, item| {
            if item.score > acc.score { item } else { acc }
        });
        let fragment_text = &text[fragment.start_offset..fragment.stop_offset];
        let highlighted = fragment.highlighted.iter().map(|item| {
            HighlightSection{start: item.start-fragment.start_offset,
                             stop: item.stop-fragment.start_offset}
        }).collect();
        Snippet{fragments: fragment_text.to_owned(),
                highlighted: highlighted}
    } else {
        // when there no fragments to chose from,
        // for now create a empty snippet
        Snippet{fragments: String::new(),
                highlighted: vec![]}
    }
}

pub fn generate_snippet<'a>(
    doc: &'a [FieldValue],
    index: &Index,
    terms: Vec<Term>,
    max_num_chars: usize) -> Snippet {
    unimplemented!();
}

#[cfg(test)]
mod tests {
    use tokenizer::{SimpleTokenizer, box_tokenizer};
    use std::iter::Iterator;
    use std::collections::BTreeMap;
    use super::{search_fragments, select_best_fragment_combination};

    #[test]
    fn test_snippet() {
        let tokenizer = SimpleTokenizer;

        let t = box_tokenizer(tokenizer);

        let text = "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide better memory safety while still maintaining performance.

Rust is free and open-source software, released under an MIT License, or Apache License 2.0. Its designers have refined the language through the experiences of writing the Servo web browser layout engine[14] and the Rust compiler. A large proportion of current commits to the project are from community members.[15]

Rust won first place for \"most loved programming language\" in the Stack Overflow Developer Survey in 2016, 2017, and 2018.
";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("rust"), 1.0);
        terms.insert(String::from("language"), 0.9);

        let fragments = search_fragments(t, &text, terms, 100);
        assert_eq!(fragments.len(), 7);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.9);
            assert_eq!(first.stop_offset, 89);
        }
        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe".to_owned());
        assert_eq!(snippet.to_html(), "Rust is a systems programming language sponsored by Mozilla which describes it as a "safe".to_owned())
    }
}
<|endoftext|>


<commit_before>use std::io::fs::{PathExtensions};
use std::io::{File};
use std::collections::{BTreeMap, HashMap};
use std::collections::btree_map::{Keys};
use std::str::FromStr;
use opengl_graphics::{Texture};
use graphics::{Image};
use toml::{Parser, Value, Table};

/// Sprite sheets must have an associated configuration file (in TOML).
pub struct SpriteSheet {
    pub texture: Texture,
    pub sprites: Vec<Sprite>,
    pub tile_width: u32,
    pub tile_height: u32,
    /// each spritesheet must have a corresponding toml file that
    /// allows us to retrieve sprite tiles by name.
    pub mapping: BTreeMap<String, Value>
}

enum SpriteType {
    Unique(String, String),
    Sequence(String, u32)
}

struct Sprite {
    t: SpriteType,
    h: u32,
    w: u32,
    x: u32,
    y: u32
}

const default_tile_size: i64 = 16;

type Coords = (u32, u32);

struct TomlConfig;

impl TomlConfig {

    /// the spritesheet configuration file must have the same base
    /// file name as the spritesheet itself.
    fn location(path: &Path) -> Option<Path> {
        let mut new_path = path.clone();
        new_path.set_extension("toml");
        if new_path.exists() && new_path.is_file() {
            Some(new_path)
        } else {
            None
        }
    }

    /// override the default value if the given table location exists.
    fn defaults(what: &Table, attribute: &str, default: i64) -> i64 {
        let wrapped_default = Value::Integer(default);
        let attr = what.get(attribute).unwrap_or(&wrapped_default);
        attr.as_integer().expect(format!("`{}` must be an integer", attribute).as_slice())
    }

    /// given a TOML configuration file, extract the relevant
    /// spritesheet information. returns a vector of `Sprite`s.
    fn process(toml_path: &Path) {
        let mut toml_file = File::open(toml_path);
        match toml_file.read_to_string() {
            Err(why) => panic!("Could not read configuration file {}: {}", toml_path.display(), why),
            Ok(contents) => {
                let value = Parser::new(contents.as_slice()).parse().expect("Configuration file is not valid TOML.");
                let sprites = value.get("sprites").expect("Configuration file does not have `sprites` entry.");
                let sprites_table = sprites.as_table().expect("`sprites` entry is not a TOML table.");
                let tile_width = TomlConfig::defaults(sprites_table, "tile_width", default_tile_size);
                let tile_height = TomlConfig::defaults(sprites_table, "tile_height", default_tile_size);
                let mut attributes = sprites_table.iter()
                    .filter(|&(_, v)| v.type_str() == "table")
                    .map(|(k, v)| (k, v.as_table().unwrap()));
                for (name, values) in attributes {
                    // look for a local tile_width or tile_height
                    let tile_width = TomlConfig::defaults(values, "tile_width", tile_width);
                    let tile_height = TomlConfig::defaults(values, "tile_height", tile_height);
                    TomlConfig::process_sprite(name, values, tile_width as u32, tile_height as u32);
                }
            }
        }
    }

    /// auxiliary function. given a TOML table, convert it to a
    /// straight up HashMap of tile_name, coords.
    fn to_coords(what: &Table) -> HashMap<&String, Coords> {
        let mut result: HashMap<&String, Coords> = HashMap::new();
        let mut filtered = what
            .iter()
            .filter(|&(_, v)| v.type_str() == "array")
            .map(|(k, v)| (k, v.as_slice().unwrap()));
        for (k, v) in filtered {
            if v.len() != 2 {
                panic!("attribute {:?} is not a coordinate.", v);
            }
            let x = v[0].as_integer().expect("`x` must be an integer") as u32;
            let y = v[1].as_integer().expect("`y` must be an integer") as u32;
            result.insert(k, (x, y));
        }
        result
    }

    // determine what kind of sprite we have.
    fn process_sprite(name: &String, ids: &Table, w: u32, h: u32) -> Vec<Sprite> {
        let info = TomlConfig::to_coords(ids);
        if info.len() == 0 {
            println!("Warning: `{}` has no valid sprite information.", name);
            vec![]
        } else {
            let first = info.keys().next().unwrap();
            let seq: Option<u32> = FromStr::from_str(first.as_slice());
            match seq {
                Some(_) => {
                    info.iter().map(|(k, &(x, y))| {
                        let id = FromStr::from_str(k.as_slice()).unwrap();
                        Sprite { t: SpriteType::Sequence(name.clone(), id), x: x, y: y, w: w, h: h }
                    }).collect()
                },
                _ => {
                    info.iter().map(|(&k, &(x, y))| {
                        Sprite { t: SpriteType::Unique(name.clone(), k.clone()), x: x, y: y, w: w, h: h }
                    }).collect()
                }
            }
        }
    }
}

impl SpriteSheet {

    pub fn new(path: &str, tile_width: u32, tile_height: u32) -> SpriteSheet {
        let filepath = Path::new(path);
        let texture = Texture::from_path(&filepath).unwrap();

        // obtain the mapping from the corresponding toml file.
        let toml_filepath = TomlConfig::location(&filepath).expect("No spritesheet configuration file.");
        TomlConfig::process(&toml_filepath);
        println!("passed parsing 1");

        let mut toml_file = File::open(&toml_filepath);
        let contents = String::from_utf8(toml_file.read_to_end().unwrap()).unwrap();
        let value = Parser::new(contents.as_slice()).parse().expect("parsing");
        let spritesheet = value.get("sprites").expect("No sprites entry?");
        let mapping = spritesheet.as_table().expect("No table value?");
        println!("passed parsing");

        // TODO pre-process toml.

        SpriteSheet {
            texture: texture,
            tile_width: tile_width,
            tile_height: tile_height,
            sprites: vec![],
            mapping: mapping.clone()
        }
    }

    pub fn get_sprite(&self, name: &str) -> Option<Image> {
        match self.mapping.get(name) {
            Some(&ref val) => {
                let coords = &val.as_slice().expect("coords");
                let ref x = coords[0].as_integer().expect("x") as i32;
                let ref y = coords[1].as_integer().expect("y") as i32;
                let w = self.tile_width as i32;
                let h = self.tile_height as i32;
                Some(Image {
                    color: None,
                    rectangle: None,
                    source_rectangle: Some([x * w, y * h, w, h])
                })
            }
            _ => None
        }
    }
}
<commit_msg>Pre-process TOML, part 3.<commit_after>use std::io::{File};
use std::io::fs::{PathExtensions};
use std::collections::{HashMap};
use std::str::{FromStr};
use opengl_graphics::{Texture};
use graphics::{Image};
use toml::{Parser, Value, Table};

pub struct SpriteSheet {
    pub texture: Texture,
    pub sprites: HashMap<String, SpriteCategory>
}

/// sprite categories are equivalent to TOML blocks.
#[derive(Clone, Debug)]
enum SpriteCategory {
    Unique(HashMap<String, SpriteRect>),
    Sequence(Vec<SpriteRect>)
}

/// the sprite "area" on the texture.
#[derive(Clone, Debug)]
struct SpriteRect {
    h: u32,
    w: u32,
    x: u32,
    y: u32
}

const default_tile_size: i64 = 16;

type Coords = (u32, u32);

struct TomlConfig;

impl TomlConfig {

    /// the spritesheet configuration file must have the same base
    /// file name as the spritesheet itself.
    fn location(path: &Path) -> Option<Path> {
        let mut new_path = path.clone();
        new_path.set_extension("toml");
        if new_path.exists() && new_path.is_file() {
            Some(new_path)
        } else {
            None
        }
    }

    /// override the default value if the given table location exists.
    fn defaults(what: &Table, attribute: &str, default: i64) -> i64 {
        let wrapped_default = Value::Integer(default);
        let attr = what.get(attribute).unwrap_or(&wrapped_default);
        attr.as_integer().expect(format!("`{}` must be an integer", attribute).as_slice())
    }

    /// given a TOML configuration file, extract the relevant
    /// spritesheet information. returns a hashmap of `Sprite`s.
    fn process(toml_path: &Path) -> HashMap<String, SpriteCategory> {
        let mut toml_file = File::open(toml_path);
        match toml_file.read_to_string() {
            Err(why) => panic!("Could not read configuration file {}: {}", toml_path.display(), why),
            Ok(contents) => {
                let value = Parser::new(contents.as_slice()).parse().expect("Configuration file is not valid TOML.");
                let sprites = value.get("sprites").expect("Configuration file does not have `sprites` entry.");
                let sprites_table = sprites.as_table().expect("`sprites` entry is not a TOML table.");
                let tile_width = TomlConfig::defaults(sprites_table, "tile_width", default_tile_size);
                let tile_height = TomlConfig::defaults(sprites_table, "tile_height", default_tile_size);
                let mut attributes = sprites_table.iter()
                    .filter(|&(_, v)| v.type_str() == "table")
                    .map(|(k, v)| (k, v.as_table().unwrap()));
                let mut sprites = HashMap::new();
                for (name, values) in attributes {
                    // look for a local tile_width or tile_height
                    let tile_width = TomlConfig::defaults(values, "tile_width", tile_width);
                    let tile_height = TomlConfig::defaults(values, "tile_height", tile_height);
                    let sprite = TomlConfig::process_sprite(name, values, tile_width as u32, tile_height as u32);
                    sprites.insert(name.clone(), sprite);
                }
                sprites
            }
        }
    }

    /// auxiliary function. given a TOML table, convert it to a
    /// straight up HashMap of tile_name, coords.
    fn to_coords(what: &Table) -> HashMap<&String, Coords> {
        let mut result: HashMap<&String, Coords> = HashMap::new();
        let mut filtered = what
            .iter()
            .filter(|&(_, v)| v.type_str() == "array")
            .map(|(k, v)| (k, v.as_slice().unwrap()));
        for (k, v) in filtered {
            if v.len() != 2 {
                panic!("attribute {:?} is not a coordinate.", v);
            }
            let x = v[0].as_integer().expect("`x` must be an integer") as u32;
            let y = v[1].as_integer().expect("`y` must be an integer") as u32;
            result.insert(k, (x, y));
        }
        result
    }

    // determine what kind of sprite we have.
    fn process_sprite(name: &String, ids: &Table, w: u32, h: u32) -> SpriteCategory {
        let info = TomlConfig::to_coords(ids);
        if info.len() == 0 {
            println!("Warning: `{}` has no valid sprite information.", name);
            SpriteCategory::Sequence(vec![])
        } else {
            let first = info.keys().next().unwrap();
            let seq: Option<u32> = FromStr::from_str(first.as_slice());
            match seq {
                Some(_) => {
                    // TODO: order by id.
                    let seq = info.iter().map(|(k, &(x, y))| {
                        let id: u32 = FromStr::from_str(k.as_slice()).unwrap();
                        let rect = SpriteRect { x: x, y: y, w: w, h: h };
                        rect
                        }).collect();
                    SpriteCategory::Sequence(seq)
                },
                _ => {
                    let map = info.iter().map(|(&k, &(x, y))| {
                        let rect = SpriteRect { x: x, y: y, w: w, h: h };
                        (k.clone(), rect)
                    }).collect();
                    SpriteCategory::Unique(map)
                }
            }
        }
    }
}

impl SpriteSheet {

    pub fn new(path: &str, tile_width: u32, tile_height: u32) -> SpriteSheet {
        let filepath = Path::new(path);
        let texture = Texture::from_path(&filepath).unwrap();
        let toml_filepath = TomlConfig::location(&filepath).expect("No spritesheet configuration file.");
        let sprites = TomlConfig::process(&toml_filepath);
        SpriteSheet {
            texture: texture,
            sprites: sprites
        }
    }

    pub fn get_sprite(&self, name: &str) -> Option<Image> {
        None
        // TODO: implement.
        // match self.mapping.get(name) {
        //     Some(&ref val) => {
        //         let coords = &val.as_slice().expect("coords");
        //         let ref x = coords[0].as_integer().expect("x") as i32;
        //         let ref y = coords[1].as_integer().expect("y") as i32;
        //         let w = self.tile_width as i32;
        //         let h = self.tile_height as i32;
        //         Some(Image {
        //             color: None,
        //             rectangle: None,
        //             source_rectangle: Some([x * w, y * h, w, h])
        //         })
        //     }
        //     _ => None
        // }
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! HTML Escaping
//!
//! This module contains one unit-struct which can be used to HTML-escape a
//! string of text (for use in a format string).

use std::fmt;

/// Wrapper struct which will emit the HTML-escaped version of the contained
/// string when passed to a format string.
pub struct Escape<'a>(pub &'a str);

impl<'a> fmt::Display for Escape<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        // Because the internet is always right, turns out there's not that many
        // characters to escape: http://stackoverflow.com/questions/7381974
        let Escape(s) = *self;
        let pile_o_bits = s;
        let mut last = 0;
        for (i, ch) in s.bytes().enumerate() {
            match ch as char {
                '<' | '>' | '&' | '\'' | '"' => {
                    fmt.write_str(&pile_o_bits[last..i])?;
                    let s = match ch as char {
                        '>' => ">",
                        '<' => "<",
                        '&' => "&",
                        '\'' => "'",
                        '"' => """,
                        _ => unreachable!(),
                    };
                    fmt.write_str(s)?;
                    last = i + 1;
                }
                _ => {}
            }
        }

        if last < s.len() {
            fmt.write_str(&pile_o_bits[last..])?;
        }
        Ok(())
    }
}
<commit_msg>Update copyright notice<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! HTML Escaping
//!
//! This module contains one unit-struct which can be used to HTML-escape a
//! string of text (for use in a format string).

use std::fmt;

/// Wrapper struct which will emit the HTML-escaped version of the contained
/// string when passed to a format string.
pub struct Escape<'a>(pub &'a str);

impl<'a> fmt::Display for Escape<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        // Because the internet is always right, turns out there's not that many
        // characters to escape: http://stackoverflow.com/questions/7381974
        let Escape(s) = *self;
        let pile_o_bits = s;
        let mut last = 0;
        for (i, ch) in s.bytes().enumerate() {
            match ch as char {
                '<' | '>' | '&' | '\'' | '"' => {
                    fmt.write_str(&pile_o_bits[last..i])?;
                    let s = match ch as char {
                        '>' => ">",
                        '<' => "<",
                        '&' => "&",
                        '\'' => "'",
                        '"' => """,
                        _ => unreachable!(),
                    };
                    fmt.write_str(s)?;
                    last = i + 1;
                }
                _ => {}
            }
        }

        if last < s.len() {
            fmt.write_str(&pile_o_bits[last..])?;
        }
        Ok(())
    }
}
<|endoftext|>


<commit_before>//! Implements a database that stores results of previously performed
//! searches.
//!
//! Using such a database (called "transposition table") is a way to
//! greatly reduce the search space of a chess tree with little
//! negative impact. Chess programs, during their brute-force search,
//! encounter the same positions again and again, but from different
//! sequences of moves, which is called a transposition.
//!
//! When the search encounters a transposition, it is beneficial to
//! "remember" what was determined last time the position was
//! examined, rather than redoing the entire search again. For this
//! reason, chess programs have a transposition table, which is a
//! large hash table storing information about positions previously
//! searched, how deeply they were searched, and what we concluded
//! about them.

use std;
use std::cell::{UnsafeCell, Cell};
use std::mem::transmute;
use std::num::Wrapping;
use basetypes::Value;
use chess_move::MoveDigest;

/// `BOUND_EXACT`, `BOUND_LOWER`, `BOUND_UPPER`, or `BOUND_NONE`.
///
/// * `BOUND_EXACT` means that the evaluation is exact.
///
/// * `BOUND_LOWER` means that the real value is greater or equal to
///    the evaluation.
///
/// * `BOUND_UPPER` means that the real value is lesser or equal to
///   the evaluation.
///
/// * `BOUND_NONE` means that the real value might be lesser, equal,
///   or grater than the evaluation.
pub type BoundType = u8;

/// The evaluation is not bounded.
pub const BOUND_NONE: BoundType = 0;

/// The evaluation is an upper bound.
pub const BOUND_UPPER: BoundType = 0b10;

/// The evaluation is a lower bound.
pub const BOUND_LOWER: BoundType = 0b01;

/// The evaluation is exact.
pub const BOUND_EXACT: BoundType = BOUND_UPPER | BOUND_LOWER;

/// Stores information about a particular position.
#[derive(Copy, Clone)]
pub struct EntryData {
    move16: MoveDigest,
    value: Value,
    eval_value: Value,
    gen_bound: u8,
    depth: u8,
}

impl EntryData {
    /// Creates a new instance.
    ///
    /// * `value` -- the value assigned to the position;
    ///
    /// * `bound` -- the meaning of the assigned value;
    ///
    /// * `depth` -- the depth of search;
    ///
    /// * `move16` -- best or refutation move, or `0` if no move is
    /// available;
    ///
    /// * `eval_value` -- the calculated static evaluation for the
    ///   position.
    pub fn new(value: Value,
               bound: BoundType,
               depth: u8,
               move16: MoveDigest,
               eval_value: Value)
               -> EntryData {
        assert!(bound <= 0b11);
        assert!(depth < 127);
        EntryData {
            move16: move16,
            value: value,
            eval_value: eval_value,
            gen_bound: bound, // Stores the entry's generation and the bound.
            depth: depth,
        }
    }

    #[inline(always)]
    pub fn value(&self) -> Value {
        self.value
    }

    #[inline(always)]
    pub fn bound(&self) -> BoundType {
        self.gen_bound & 0b11
    }

    #[inline(always)]
    pub fn depth(&self) -> u8 {
        self.depth
    }

    #[inline(always)]
    pub fn move16(&self) -> MoveDigest {
        self.move16
    }

    #[inline(always)]
    pub fn eval_value(&self) -> Value {
        self.eval_value
    }
}

// Represents transposition table entry.
//
// It is 16 bytes, defined as below:
//
// * key        64 bit
// * move16     16 bit
// * value      16 bit
// * eval value 16 bit
// * generation  6 bit
// * bound type  2 bit
// * depth       8 bit
#[derive(Copy, Clone)]
struct Entry {
    key: u64,
    data: EntryData,
}

impl Default for Entry {
    fn default() -> Entry {
        Entry {
            key: 0,
            data: unsafe { transmute(0u64) },
        }
    }
}

impl Entry {
    // Returns the whole contained data as one `u64` value.
    //
    // This is needed for implementing the lock-less probing and
    // storing.
    #[inline(always)]
    fn data_u64(&self) -> u64 {
        unsafe { transmute(self.data) }
    }

    // Returns entry's generation.
    #[inline(always)]
    fn generation(&self) -> u8 {
        self.data.gen_bound & 0b11111100
    }

    // Updates entry's generation.
    //
    // Since the `key` is saved xored with the data, when we change
    // the data, we have to change the stored `key` as well.
    #[inline(always)]
    fn update_generation(&mut self, generation: u8) {
        assert_eq!(generation & 0b11, 0);
        let old_data_u64 = self.data_u64();
        self.data.gen_bound = generation | self.data.bound();
        self.key ^= old_data_u64 ^ self.data_u64();
    }
}

/// A large hash-table that stores results of previously performed
/// searches.
pub struct TranspositionTable {
    generation: Cell<u8>,
    cluster_count: usize,
    table: UnsafeCell<Vec<[Entry; 4]>>,
}

impl TranspositionTable {
    /// Creates a new transposition table.
    ///
    /// The newly created table has the minimum possible size. Before
    /// using the new table for anything, `resize()` should be called
    /// on it, specifying the desired size.
    pub fn new() -> TranspositionTable {
        TranspositionTable {
            generation: Cell::new(0),
            cluster_count: 1,
            table: UnsafeCell::new(vec![Default::default()]),
        }
    }

    /// Resizes the transpositon table. All entries in the table will
    /// be lost.
    ///
    /// `size_mb` is the desired new size in Mbytes. If `size_mb` is
    /// not in the form "2**n", the new size of the transposition
    /// table will be as close as possible, but less than `size_mb`.
    pub fn resize(&mut self, size_mb: usize) {
        // The cluster count should be in the form "2**n", so the best
        // we can do is to ensure that the new cluster count will be
        // as close as possible but no greater than the requested one.
        let requested_cluster_count = (size_mb * 1024 * 1024) / std::mem::size_of::<[Entry; 4]>();

        // First, make sure `requested_cluster_count` is exceeded.
        let mut new_cluster_count = 1;
        while requested_cluster_count >= new_cluster_count {
            new_cluster_count <<= 1;
        }

        // Then make one step back, being careful so that the new
        // cluster count is not zero.
        if new_cluster_count > 1 {
            new_cluster_count >>= 1;
        } else {
            new_cluster_count = 1;
        }

        // Finally, reallocate the vector of clusters.
        if new_cluster_count != self.cluster_count {
            self.cluster_count = new_cluster_count;
            self.table = UnsafeCell::new(vec![Default::default(); new_cluster_count]);
        }
    }

    /// Returns the size of the transposition table in Mbytes.
    pub fn size(&self) -> usize {
        unsafe { &*self.table.get() }.len() * std::mem::size_of::<[Entry; 4]>() / 1024 / 1024
    }

    /// Removes all entries in the table.
    pub fn clear(&self) {
        unsafe {
            let table = self.table.get().as_mut().unwrap();
            for cluster in table {
                for entry in cluster.iter_mut() {
                    entry.key = 0;
                    entry.data = transmute(0u64);
                }
            }
        }
    }

    /// Signals that a new search is about to begin.
    ///
    /// Internally, the transposition table maintains a generation
    /// counter that is used to implement an efficient replacement
    /// strategy. This method increases this counter.
    pub fn new_search(&self) {
        // The lowest 2 bits of the `generation` field are used by
        // bound type.
        self.generation.set((Wrapping(self.generation.get()) + Wrapping(0b100)).0);
        assert_eq!(self.generation.get() & 0b11, 0);
    }

    /// Probes for data by a specific key.
    #[inline]
    pub fn probe(&self, key: u64) -> Option<EntryData> {
        let cluster = unsafe { self.cluster_mut(key) };
        for entry in cluster.iter_mut() {
            if entry.key ^ entry.data_u64() == key {
                // If `key` and `data` were written simultaneously by
                // different search instances with different keys,
                // this will yield in a mismatch of the above
                // comparison (except for the rare and inherent key
                // collisions).
                entry.update_generation(self.generation.get());
                return Some(entry.data);
            }
        }
        None
    }

    /// Stores data by a specific key.
    ///
    /// After being stored, the data might be retrieved by
    /// `probe(key)`. This is not guaranteed though, because in the
    /// meantime it might have been overwritten.
    #[inline]
    pub fn store(&self, key: u64, mut data: EntryData) {
        // `store` and `probe` jointly implement a clever lock-less
        // hashing method. Rather than to store two disjoint items,
        // the key is stored xored with data, while data is stored
        // additionally as usual.

        data.gen_bound |= self.generation.get();  // Sets the generation.
        let mut cluster = unsafe { self.cluster_mut(key) };
        let mut replace_index = 0;
        let mut replace_score = 0xff;
        for (i, entry) in cluster.iter_mut().enumerate() {
            // Check if this is an empty slot, or an old entry for the
            // same key. If this this is the case we will use this
            // slot for the new entry.
            if entry.key == 0 || entry.key ^ entry.data_u64() == key {
                if data.move16 == 0 {
                    data.move16 = entry.data.move16;  // Preserve any existing move.
                }
                replace_index = i;
                break;
            }
            // If we can not find empty/old slot, the replaced entry
            // will be the entry with the lowest score.
            let entry_score = self.calc_score(entry);
            if entry_score < replace_score {
                replace_index = i;
                replace_score = entry_score;
            }
        }
        unsafe {
            cluster.get_unchecked_mut(replace_index).key = key ^ transmute::<EntryData, u64>(data);
            cluster.get_unchecked_mut(replace_index).data = data;
        }
    }

    // A helper method for `store`.
    //
    // Implements our replacement strategy. Should return higher
    // values for the entries that are move likely to save CPU work in
    // the future.
    #[inline]
    fn calc_score(&self, entry: &Entry) -> u8 {
        // Positions from the current generation are always scored
        // higher than positions from older generations.
        (if entry.generation() == self.generation.get() {
            128
        } else {
            0
        })

        // Positions with higher search depth are scored higher.
        + entry.data.depth()

        // Positions with exact evaluation are given slight advantage.
        + (if entry.data.bound() == BOUND_EXACT {
            1
        } else {
            0
        })
    }

    // A helper method for `probe` and `store`.
    #[inline]
    unsafe fn cluster_mut(&self, key: u64) -> &mut [Entry; 4] {
        let cluster_index = (key & (self.cluster_count - 1) as u64) as usize;
        self.table.get().as_mut().unwrap().get_unchecked_mut(cluster_index)
    }
}

unsafe impl Sync for TranspositionTable {}

#[cfg(test)]
mod tests {
    use super::*;
    use super::Entry;
    use std;

    #[test]
    fn test_cluster_size() {
        assert_eq!(std::mem::size_of::<[Entry; 4]>(), 64);
        assert_eq!(std::mem::size_of::<Entry>(), 16);
    }

    #[test]
    fn test_tt_resize() {
        let mut tt = TranspositionTable::new();
        assert_eq!(unsafe { &*tt.table.get() }.capacity(), 1);
        tt.resize(1);
        assert_eq!(tt.size(), 1);
        tt.clear();
    }

    #[test]
    fn test_store_and_probe() {
        let tt = TranspositionTable::new();
        assert!(tt.probe(1).is_none());
        let data = EntryData::new(0, 0, 100, 666, 0);
        assert_eq!(data.depth(), 100);
        assert_eq!(data.move16(), 666);
        tt.store(1, data);
        assert_eq!(tt.probe(1).unwrap().depth(), 100);
        tt.store(1, EntryData::new(0, 0, 100, 666, 0));
        assert_eq!(tt.probe(1).unwrap().depth(), 100);
        assert_eq!(tt.probe(1).unwrap().move16(), 666);
        for i in 2..100 {
            tt.store(i, EntryData::new(i as i16, 0, i as u8, i as u16, i as i16));
        }
        assert_eq!(tt.probe(1).unwrap().depth(), 100);
        assert_eq!(tt.probe(99).unwrap().depth(), 99);
        assert_eq!(tt.probe(98).unwrap().depth(), 98);
        assert_eq!(tt.probe(97).unwrap().depth(), 97);
        tt.clear();
        assert!(tt.probe(1).is_none());
        tt.store(1, data);
        tt.new_search();
        tt.probe(1);
        assert!(tt.probe(1).is_some());
    }

    #[test]
    fn test_new_search() {
        let tt = TranspositionTable::new();
        assert_eq!(tt.generation.get(), 0 << 2);
        tt.new_search();
        assert_eq!(tt.generation.get(), 1 << 2);
        for _ in 0..64 {
            tt.new_search();
        }
        assert_eq!(tt.generation.get(), 1 << 2);
    }
}
<commit_msg>cleaner clear()<commit_after>//! Implements a database that stores results of previously performed
//! searches.
//!
//! Using such a database (called "transposition table") is a way to
//! greatly reduce the search space of a chess tree with little
//! negative impact. Chess programs, during their brute-force search,
//! encounter the same positions again and again, but from different
//! sequences of moves, which is called a transposition.
//!
//! When the search encounters a transposition, it is beneficial to
//! "remember" what was determined last time the position was
//! examined, rather than redoing the entire search again. For this
//! reason, chess programs have a transposition table, which is a
//! large hash table storing information about positions previously
//! searched, how deeply they were searched, and what we concluded
//! about them.

use std;
use std::cell::{UnsafeCell, Cell};
use std::mem::transmute;
use std::num::Wrapping;
use basetypes::Value;
use chess_move::MoveDigest;

/// `BOUND_EXACT`, `BOUND_LOWER`, `BOUND_UPPER`, or `BOUND_NONE`.
///
/// * `BOUND_EXACT` means that the evaluation is exact.
///
/// * `BOUND_LOWER` means that the real value is greater or equal to
///    the evaluation.
///
/// * `BOUND_UPPER` means that the real value is lesser or equal to
///   the evaluation.
///
/// * `BOUND_NONE` means that the real value might be lesser, equal,
///   or grater than the evaluation.
pub type BoundType = u8;

/// The evaluation is not bounded.
pub const BOUND_NONE: BoundType = 0;

/// The evaluation is an upper bound.
pub const BOUND_UPPER: BoundType = 0b10;

/// The evaluation is a lower bound.
pub const BOUND_LOWER: BoundType = 0b01;

/// The evaluation is exact.
pub const BOUND_EXACT: BoundType = BOUND_UPPER | BOUND_LOWER;

/// Stores information about a particular position.
#[derive(Copy, Clone)]
pub struct EntryData {
    move16: MoveDigest,
    value: Value,
    eval_value: Value,
    gen_bound: u8,
    depth: u8,
}

impl EntryData {
    /// Creates a new instance.
    ///
    /// * `value` -- the value assigned to the position;
    ///
    /// * `bound` -- the meaning of the assigned value;
    ///
    /// * `depth` -- the depth of search;
    ///
    /// * `move16` -- best or refutation move, or `0` if no move is
    /// available;
    ///
    /// * `eval_value` -- the calculated static evaluation for the
    ///   position.
    pub fn new(value: Value,
               bound: BoundType,
               depth: u8,
               move16: MoveDigest,
               eval_value: Value)
               -> EntryData {
        assert!(bound <= 0b11);
        assert!(depth < 127);
        EntryData {
            move16: move16,
            value: value,
            eval_value: eval_value,
            gen_bound: bound, // Stores the entry's generation and the bound.
            depth: depth,
        }
    }

    #[inline(always)]
    pub fn value(&self) -> Value {
        self.value
    }

    #[inline(always)]
    pub fn bound(&self) -> BoundType {
        self.gen_bound & 0b11
    }

    #[inline(always)]
    pub fn depth(&self) -> u8 {
        self.depth
    }

    #[inline(always)]
    pub fn move16(&self) -> MoveDigest {
        self.move16
    }

    #[inline(always)]
    pub fn eval_value(&self) -> Value {
        self.eval_value
    }
}

// Represents transposition table entry.
//
// It is 16 bytes, defined as below:
//
// * key        64 bit
// * move16     16 bit
// * value      16 bit
// * eval value 16 bit
// * generation  6 bit
// * bound type  2 bit
// * depth       8 bit
#[derive(Copy, Clone)]
struct Entry {
    key: u64,
    data: EntryData,
}

impl Default for Entry {
    fn default() -> Entry {
        Entry {
            key: 0,
            data: unsafe { transmute(0u64) },
        }
    }
}

impl Entry {
    // Returns the whole contained data as one `u64` value.
    //
    // This is needed for implementing the lock-less probing and
    // storing.
    #[inline(always)]
    fn data_u64(&self) -> u64 {
        unsafe { transmute(self.data) }
    }

    // Returns entry's generation.
    #[inline(always)]
    fn generation(&self) -> u8 {
        self.data.gen_bound & 0b11111100
    }

    // Updates entry's generation.
    //
    // Since the `key` is saved xored with the data, when we change
    // the data, we have to change the stored `key` as well.
    #[inline(always)]
    fn update_generation(&mut self, generation: u8) {
        assert_eq!(generation & 0b11, 0);
        let old_data_u64 = self.data_u64();
        self.data.gen_bound = generation | self.data.bound();
        self.key ^= old_data_u64 ^ self.data_u64();
    }
}

/// A large hash-table that stores results of previously performed
/// searches.
pub struct TranspositionTable {
    generation: Cell<u8>,
    cluster_count: usize,
    table: UnsafeCell<Vec<[Entry; 4]>>,
}

impl TranspositionTable {
    /// Creates a new transposition table.
    ///
    /// The newly created table has the minimum possible size. Before
    /// using the new table for anything, `resize()` should be called
    /// on it, specifying the desired size.
    pub fn new() -> TranspositionTable {
        TranspositionTable {
            generation: Cell::new(0),
            cluster_count: 1,
            table: UnsafeCell::new(vec![Default::default()]),
        }
    }

    /// Resizes the transpositon table. All entries in the table will
    /// be lost.
    ///
    /// `size_mb` is the desired new size in Mbytes. If `size_mb` is
    /// not in the form "2**n", the new size of the transposition
    /// table will be as close as possible, but less than `size_mb`.
    pub fn resize(&mut self, size_mb: usize) {
        // The cluster count should be in the form "2**n", so the best
        // we can do is to ensure that the new cluster count will be
        // as close as possible but no greater than the requested one.
        let requested_cluster_count = (size_mb * 1024 * 1024) / std::mem::size_of::<[Entry; 4]>();

        // First, make sure `requested_cluster_count` is exceeded.
        let mut new_cluster_count = 1;
        while requested_cluster_count >= new_cluster_count {
            new_cluster_count <<= 1;
        }

        // Then make one step back, being careful so that the new
        // cluster count is not zero.
        if new_cluster_count > 1 {
            new_cluster_count >>= 1;
        } else {
            new_cluster_count = 1;
        }

        // Finally, reallocate the vector of clusters.
        if new_cluster_count != self.cluster_count {
            self.cluster_count = new_cluster_count;
            self.table = UnsafeCell::new(vec![Default::default(); new_cluster_count]);
        }
    }

    /// Returns the size of the transposition table in Mbytes.
    pub fn size(&self) -> usize {
        unsafe { &*self.table.get() }.len() * std::mem::size_of::<[Entry; 4]>() / 1024 / 1024
    }

    /// Removes all entries in the table.
    pub fn clear(&self) {
        let table = unsafe { self.table.get().as_mut().unwrap() };
        for cluster in table {
            for entry in cluster.iter_mut() {
                *entry = Default::default();
            }
        }
    }

    /// Signals that a new search is about to begin.
    ///
    /// Internally, the transposition table maintains a generation
    /// counter that is used to implement an efficient replacement
    /// strategy. This method increases this counter.
    pub fn new_search(&self) {
        // The lowest 2 bits of the `generation` field are used by
        // bound type.
        self.generation.set((Wrapping(self.generation.get()) + Wrapping(0b100)).0);
        assert_eq!(self.generation.get() & 0b11, 0);
    }

    /// Probes for data by a specific key.
    #[inline]
    pub fn probe(&self, key: u64) -> Option<EntryData> {
        let cluster = unsafe { self.cluster_mut(key) };
        for entry in cluster.iter_mut() {
            if entry.key ^ entry.data_u64() == key {
                // If `key` and `data` were written simultaneously by
                // different search instances with different keys,
                // this will yield in a mismatch of the above
                // comparison (except for the rare and inherent key
                // collisions).
                entry.update_generation(self.generation.get());
                return Some(entry.data);
            }
        }
        None
    }

    /// Stores data by a specific key.
    ///
    /// After being stored, the data might be retrieved by
    /// `probe(key)`. This is not guaranteed though, because in the
    /// meantime it might have been overwritten.
    #[inline]
    pub fn store(&self, key: u64, mut data: EntryData) {
        // `store` and `probe` jointly implement a clever lock-less
        // hashing method. Rather than to store two disjoint items,
        // the key is stored xored with data, while data is stored
        // additionally as usual.

        data.gen_bound |= self.generation.get();  // Sets the generation.
        let mut cluster = unsafe { self.cluster_mut(key) };
        let mut replace_index = 0;
        let mut replace_score = 0xff;
        for (i, entry) in cluster.iter_mut().enumerate() {
            // Check if this is an empty slot, or an old entry for the
            // same key. If this this is the case we will use this
            // slot for the new entry.
            if entry.key == 0 || entry.key ^ entry.data_u64() == key {
                if data.move16 == 0 {
                    data.move16 = entry.data.move16;  // Preserve any existing move.
                }
                replace_index = i;
                break;
            }
            // If we can not find empty/old slot, the replaced entry
            // will be the entry with the lowest score.
            let entry_score = self.calc_score(entry);
            if entry_score < replace_score {
                replace_index = i;
                replace_score = entry_score;
            }
        }
        unsafe {
            cluster.get_unchecked_mut(replace_index).key = key ^ transmute::<EntryData, u64>(data);
            cluster.get_unchecked_mut(replace_index).data = data;
        }
    }

    // A helper method for `store`.
    //
    // Implements our replacement strategy. Should return higher
    // values for the entries that are move likely to save CPU work in
    // the future.
    #[inline]
    fn calc_score(&self, entry: &Entry) -> u8 {
        // Positions from the current generation are always scored
        // higher than positions from older generations.
        (if entry.generation() == self.generation.get() {
            128
        } else {
            0
        })

        // Positions with higher search depth are scored higher.
        + entry.data.depth()

        // Positions with exact evaluation are given slight advantage.
        + (if entry.data.bound() == BOUND_EXACT {
            1
        } else {
            0
        })
    }

    // A helper method for `probe` and `store`.
    #[inline]
    unsafe fn cluster_mut(&self, key: u64) -> &mut [Entry; 4] {
        let cluster_index = (key & (self.cluster_count - 1) as u64) as usize;
        self.table.get().as_mut().unwrap().get_unchecked_mut(cluster_index)
    }
}

unsafe impl Sync for TranspositionTable {}

#[cfg(test)]
mod tests {
    use super::*;
    use super::Entry;
    use std;

    #[test]
    fn test_cluster_size() {
        assert_eq!(std::mem::size_of::<[Entry; 4]>(), 64);
        assert_eq!(std::mem::size_of::<Entry>(), 16);
    }

    #[test]
    fn test_tt_resize() {
        let mut tt = TranspositionTable::new();
        assert_eq!(unsafe { &*tt.table.get() }.capacity(), 1);
        tt.resize(1);
        assert_eq!(tt.size(), 1);
        tt.clear();
    }

    #[test]
    fn test_store_and_probe() {
        let tt = TranspositionTable::new();
        assert!(tt.probe(1).is_none());
        let data = EntryData::new(0, 0, 100, 666, 0);
        assert_eq!(data.depth(), 100);
        assert_eq!(data.move16(), 666);
        tt.store(1, data);
        assert_eq!(tt.probe(1).unwrap().depth(), 100);
        tt.store(1, EntryData::new(0, 0, 100, 666, 0));
        assert_eq!(tt.probe(1).unwrap().depth(), 100);
        assert_eq!(tt.probe(1).unwrap().move16(), 666);
        for i in 2..100 {
            tt.store(i, EntryData::new(i as i16, 0, i as u8, i as u16, i as i16));
        }
        assert_eq!(tt.probe(1).unwrap().depth(), 100);
        assert_eq!(tt.probe(99).unwrap().depth(), 99);
        assert_eq!(tt.probe(98).unwrap().depth(), 98);
        assert_eq!(tt.probe(97).unwrap().depth(), 97);
        tt.clear();
        assert!(tt.probe(1).is_none());
        tt.store(1, data);
        tt.new_search();
        tt.probe(1);
        assert!(tt.probe(1).is_some());
    }

    #[test]
    fn test_new_search() {
        let tt = TranspositionTable::new();
        assert_eq!(tt.generation.get(), 0 << 2);
        tt.new_search();
        assert_eq!(tt.generation.get(), 1 << 2);
        for _ in 0..64 {
            tt.new_search();
        }
        assert_eq!(tt.generation.get(), 1 << 2);
    }
}
<|endoftext|>


<commit_before>pub fn ffs(i: ::int_t) -> ::int_t {
    extern { fn ffs(i: ::int_t) -> ::int_t; }
    unsafe { ffs(i) }
}

extern {
    pub fn strcasecmp(s1: *const ::char_t, s2: *const ::char_t) -> ::int_t;
    pub fn strcasecmp_l(s1: *const ::char_t, s2: *const ::char_t,
                        loc: ::locale::locale_t) -> ::int_t;
    pub fn strncasecmp(s1: *const ::char_t, s2: *const ::char_t, n: ::size_t) -> ::int_t;
    pub fn strncasecmp_l(s1: *const ::char_t, s2: *const ::char_t, n: ::size_t,
                         loc: ::locale::locale_t) -> ::int_t;
}
<commit_msg>strings.h<commit_after>use {int_t, size_t, char_t};
use locale::{locale_t};

pub fn ffs(i: int_t) -> int_t {
    extern { fn ffs(i: int_t) -> int_t; }
    unsafe { ffs(i) }
}

extern {
    pub fn strcasecmp(s1: *const char_t, s2: *const char_t) -> int_t;
    pub fn strcasecmp_l(s1: *const char_t, s2: *const char_t,
                        loc: locale_t) -> int_t;
    pub fn strncasecmp(s1: *const char_t, s2: *const char_t, n: size_t) -> int_t;
    pub fn strncasecmp_l(s1: *const char_t, s2: *const char_t, n: size_t,
                         loc: locale_t) -> int_t;
}
<|endoftext|>