<commit_before>use unicode_width::UnicodeWidthStr;

use crate::buffer::Buffer;
use crate::layout::Rect;
use crate::style::Style;
use crate::symbols::line;
use crate::widgets::{Block, Widget};

/// A widget to display available tabs in a multiple panels context.
///
/// # Examples
///
/// ```
/// # use tui::widgets::{Block, Borders, Tabs};
/// # use tui::style::{Style, Color};
/// # use tui::symbols::{DOT};
/// # fn main() {
/// Tabs::default()
///     .block(Block::default().title("Tabs").borders(Borders::ALL))
///     .titles(&["Tab1", "Tab2", "Tab3", "Tab4"])
///     .style(Style::default().fg(Color::White))
///     .highlight_style(Style::default().fg(Color::Yellow))
///     .divider(DOT);
/// # }
/// ```
pub struct Tabs<'a, T>
where
    T: AsRef<str> + 'a,
{
    /// A block to wrap this widget in if necessary
    block: Option<Block<'a>>,
    /// One title for each tab
    titles: &'a [T],
    /// The index of the selected tabs
    selected: usize,
    /// The style used to draw the text
    style: Style,
    /// The style used to display the selected item
    highlight_style: Style,
    /// Tab divider
    divider: &'a str,
}

impl<'a, T> Default for Tabs<'a, T>
where
    T: AsRef<str>,
{
    fn default() -> Tabs<'a, T> {
        Tabs {
            block: None,
            titles: &[],
            selected: 0,
            style: Default::default(),
            highlight_style: Default::default(),
            divider: line::VERTICAL,
        }
    }
}

impl<'a, T> Tabs<'a, T>
where
    T: AsRef<str>,
{
    pub fn block(mut self, block: Block<'a>) -> Tabs<'a, T> {
        self.block = Some(block);
        self
    }

    pub fn titles(mut self, titles: &'a [T]) -> Tabs<'a, T> {
        self.titles = titles;
        self
    }

    pub fn select(mut self, selected: usize) -> Tabs<'a, T> {
        self.selected = selected;
        self
    }

    pub fn style(mut self, style: Style) -> Tabs<'a, T> {
        self.style = style;
        self
    }

    pub fn highlight_style(mut self, style: Style) -> Tabs<'a, T> {
        self.highlight_style = style;
        self
    }

    pub fn divider(mut self, divider: &'a str) -> Tabs<'a, T> {
        self.divider = divider;
        self
    }
}

impl<'a, T> Widget for Tabs<'a, T>
where
    T: AsRef<str>,
{
    fn draw(&mut self, area: Rect, buf: &mut Buffer) {
        let tabs_area = match self.block {
            Some(ref mut b) => {
                b.draw(area, buf);
                b.inner(area)
            }
            None => area,
        };

        if tabs_area.height < 1 {
            return;
        }

        self.background(tabs_area, buf, self.style.bg);

        let mut x = tabs_area.left();
        let titles_length = self.titles.len();
        let divider_width = self.divider.width() as u16;
        for (title, style, last_title) in self.titles.iter().enumerate().map(|(i, t)| {
            let lt = i + 1 == titles_length;
            if i == self.selected {
                (t, self.highlight_style, lt)
            } else {
                (t, self.style, lt)
            }
        }) {
            x += 1;
            if x > tabs_area.right() {
                break;
            } else {
                buf.set_string(x, tabs_area.top(), title.as_ref(), style);
                x += title.as_ref().width() as u16 + 1;
                if x >= tabs_area.right() || last_title {
                    break;
                } else {
                    buf.set_string(x, tabs_area.top(), self.divider, style);
                    x += divider_width;
                }
            }
        }
    }
}
<commit_msg>Don't highlight Tabs separator behind selection<commit_after>use unicode_width::UnicodeWidthStr;

use crate::buffer::Buffer;
use crate::layout::Rect;
use crate::style::Style;
use crate::symbols::line;
use crate::widgets::{Block, Widget};

/// A widget to display available tabs in a multiple panels context.
///
/// # Examples
///
/// ```
/// # use tui::widgets::{Block, Borders, Tabs};
/// # use tui::style::{Style, Color};
/// # use tui::symbols::{DOT};
/// # fn main() {
/// Tabs::default()
///     .block(Block::default().title("Tabs").borders(Borders::ALL))
///     .titles(&["Tab1", "Tab2", "Tab3", "Tab4"])
///     .style(Style::default().fg(Color::White))
///     .highlight_style(Style::default().fg(Color::Yellow))
///     .divider(DOT);
/// # }
/// ```
pub struct Tabs<'a, T>
where
    T: AsRef<str> + 'a,
{
    /// A block to wrap this widget in if necessary
    block: Option<Block<'a>>,
    /// One title for each tab
    titles: &'a [T],
    /// The index of the selected tabs
    selected: usize,
    /// The style used to draw the text
    style: Style,
    /// The style used to display the selected item
    highlight_style: Style,
    /// Tab divider
    divider: &'a str,
}

impl<'a, T> Default for Tabs<'a, T>
where
    T: AsRef<str>,
{
    fn default() -> Tabs<'a, T> {
        Tabs {
            block: None,
            titles: &[],
            selected: 0,
            style: Default::default(),
            highlight_style: Default::default(),
            divider: line::VERTICAL,
        }
    }
}

impl<'a, T> Tabs<'a, T>
where
    T: AsRef<str>,
{
    pub fn block(mut self, block: Block<'a>) -> Tabs<'a, T> {
        self.block = Some(block);
        self
    }

    pub fn titles(mut self, titles: &'a [T]) -> Tabs<'a, T> {
        self.titles = titles;
        self
    }

    pub fn select(mut self, selected: usize) -> Tabs<'a, T> {
        self.selected = selected;
        self
    }

    pub fn style(mut self, style: Style) -> Tabs<'a, T> {
        self.style = style;
        self
    }

    pub fn highlight_style(mut self, style: Style) -> Tabs<'a, T> {
        self.highlight_style = style;
        self
    }

    pub fn divider(mut self, divider: &'a str) -> Tabs<'a, T> {
        self.divider = divider;
        self
    }
}

impl<'a, T> Widget for Tabs<'a, T>
where
    T: AsRef<str>,
{
    fn draw(&mut self, area: Rect, buf: &mut Buffer) {
        let tabs_area = match self.block {
            Some(ref mut b) => {
                b.draw(area, buf);
                b.inner(area)
            }
            None => area,
        };

        if tabs_area.height < 1 {
            return;
        }

        self.background(tabs_area, buf, self.style.bg);

        let mut x = tabs_area.left();
        let titles_length = self.titles.len();
        let divider_width = self.divider.width() as u16;
        for (title, style, last_title) in self.titles.iter().enumerate().map(|(i, t)| {
            let lt = i + 1 == titles_length;
            if i == self.selected {
                (t, self.highlight_style, lt)
            } else {
                (t, self.style, lt)
            }
        }) {
            x += 1;
            if x > tabs_area.right() {
                break;
            } else {
                buf.set_string(x, tabs_area.top(), title.as_ref(), style);
                x += title.as_ref().width() as u16 + 1;
                if x >= tabs_area.right() || last_title {
                    break;
                } else {
                    buf.set_string(x, tabs_area.top(), self.divider, self.style);
                    x += divider_width;
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use super::scheduler_thread::*;
use super::job_queue::*;
use super::queue_state::*;
use super::wake_queue::*;

use std::sync::*;
use std::collections::vec_deque::*;

use futures::task;
use futures::task::{Context};

///
/// The scheduler core contains the internal data used by the scheduler
///
pub (super) struct SchedulerCore {
    /// The queues that are active in the scheduler
    pub (super) schedule: Arc<Mutex<VecDeque<Arc<JobQueue>>>>,

    /// Active threads and whether or not they're busy
    pub (super) threads: Mutex<Vec<(Arc<Mutex<bool>>, SchedulerThread)>>,

    /// The maximum number of threads permitted in this scheduler
    pub (super) max_threads: Mutex<usize>
}

impl SchedulerCore {
    ///
    /// Wakes a thread to run a dormant queue. Returns true if a thread was woken up
    ///
    pub (super) fn schedule_thread(&self, core: Arc<SchedulerCore>) -> bool {
        // Find a dormant thread and activate it
        let schedule = self.schedule.clone();

        // Schedule work on this dormant thread
        let work_core   = Arc::clone(&core);
        let do_work     = move |work: Arc<JobQueue>| {
            let waker       = Arc::new(WakeQueue(Arc::clone(&work), Arc::clone(&work_core)));
            let waker       = task::waker_ref(&waker);
            let mut context = Context::from_waker(&waker);

            work.drain(&mut context)
        };

        if !self.schedule_dormant(move || Self::next_to_run(&schedule), do_work) {
            // Try to create a new thread
            if self.spawn_thread_if_less_than_maximum() {
                // Try harder to schedule this task if a thread was created
                self.schedule_thread(core)
            } else {
                // Couldn't schedule on an existing thread or create a new one
                false
            }
        } else {
            // Successfully scheduled
            true
        }
    }

    ///
    /// If a queue is idle and has pending jobs, places it in the schedule
    ///
    pub (super) fn reschedule_queue(&self, queue: &Arc<JobQueue>, core: Arc<SchedulerCore>) {
        let reschedule = {
            let mut core = queue.core.lock().expect("JobQueue core lock");

            match core.state {
                QueueState::Idle => {
                    // Schedule a thread to restart the queue if more things were queued
                    if core.queue.len() > 0 {
                        // Need to schedule the queue after this event
                        core.state = QueueState::Pending;
                        true
                    } else {
                        // Queue is empty and can go back to idle
                        core.state = QueueState::Idle;
                        false
                    }
                },

                QueueState::WaitingForPoll(_) => {
                    // If the target thread gets stuck and stops draining the queue, race it to reschedule it on one of our threads if we can
                    true
                },

                _ => {
                    // Not scheduled
                    false
                }
            }
        };

        if reschedule {
            self.schedule.lock().expect("Schedule lock").push_back(queue.clone());
            self.schedule_thread(core);
        }
    }

    ///
    /// Finds the next queue that should be run. If this returns successfully, the queue will
    /// be marked as running.
    ///
    pub (super) fn next_to_run(schedule: &Arc<Mutex<VecDeque<Arc<JobQueue>>>>) -> Option<Arc<JobQueue>> {
        // Search the queues...
        let mut schedule = schedule.lock().expect("Schedule lock");

        // Find a queue where the state is pending
        while let Some(q) = schedule.pop_front() {
            let mut core = q.core.lock().expect("JobQueue core lock");

            match core.state {
                QueueState::Pending |
                QueueState::WaitingForPoll(_) => {
                    // Queue is ready to run. Mark it as running and return it
                    core.state = QueueState::Running;
                    return Some(q.clone());
                }

                _ => {
                    // Move to the next queue in the schedule
                }
            }
        }

        None
    }

    ///
    /// If any of the scheduler threads have finished (which generally means they panicked), despawn them
    ///
    pub (super) fn remove_finished_threads(&self) {
        let mut dead_threads = vec![];

        // Collate the dead threads into a vec
        {
            // Find busy threads
            let mut threads = self.threads.lock().expect("Scheduler threads lock");

            // TODO: drain_filter in nightly would be better than this (but it's in nightly)
            let mut thread_num = 0;
            while thread_num < threads.len() {
                let (_, thread) = &threads[thread_num];

                if thread.is_finished() {
                    // Remove the thread and queue it for later despawning
                    let (is_busy, dead_thread) = threads.remove(thread_num);
                    dead_threads.push((is_busy, dead_thread));
                } else {
                    // Thread still running
                    thread_num += 1;
                }
            }
        }

        // Despawn the dead threads (which might panic a bit)
        for (is_busy, dead_thread) in dead_threads {
            // Join with the thread in case it's mid-panic
            let maybe_panic = dead_thread.despawn().join();

            if let Ok(()) = maybe_panic {
                // The busy flag should always be unset after a thread is despawned
                let busy = is_busy.lock().unwrap();
                if *busy {
                    panic!("Thread despawned while busy");
                }
            } else {
                // Panics are relayed via the desync that failed
            }
        }
    }

    ///
    /// Attempts to schedule a task on a dormant thread
    ///
    pub (super) fn schedule_dormant<NextJob, RunJob, JobData>(&self, next_job: NextJob, job: RunJob) -> bool
    where
        RunJob:     'static + Send + Fn(JobData) -> (),
        NextJob:    'static + Send + Fn() -> Option<JobData>,
    {
        // Try to despawn any threads that have finished since the last time we were called
        self.remove_finished_threads();

        // Find busy threads
        let threads = self.threads.lock().expect("Scheduler threads lock");

        // Find the first thread that is not marked as busy and schedule this task on it
        for &(ref busy_rc, ref thread) in threads.iter() {
            if let Ok(mut busy) = busy_rc.try_lock() {
                // If the busy lock is held, then we consider the thread to be busy
                if !*busy {
                    // Clone the busy mutex so we can return this thread to readiness
                    let also_busy =  busy_rc.clone();

                    // This thread is busy
                    *busy = true;
                    thread.run(move || {
                        let mut done = false;

                        while !done {
                            // Obtain the next job. The thread is not busy once there are no longer any jobs
                            // We hold the mutex while this is going on to avoid a race condition when a thread is going dormant
                            let job_data = {
                                let mut busy = also_busy.lock().expect("Thread busy lock");
                                let job_data = next_job();

                                // If there's no next job, then this thread is no longer busy
                                if job_data.is_none() {
                                    *busy = false;
                                }

                                job_data
                            };

                            // Run the job if there is one, stop the thread if there is not
                            if let Some(job_data) = job_data {
                                job(job_data);
                            } else {
                                done = true;
                            }
                        }
                    });

                    return true;
                }
            }
        }

        // No dormant threads were found
        false
    }

    ///
    /// If we're running fewer than the maximum number of threads, try to spawn a new one
    ///
    pub (super) fn spawn_thread_if_less_than_maximum(&self) -> bool {
        let max_threads = { *self.max_threads.lock().expect("Max threads lock") };
        let mut threads = self.threads.lock().expect("Scheduler threads lock");

        if threads.len() < max_threads {
            // Create a new thread
            let is_busy     = Arc::new(Mutex::new(false));
            let new_thread  = SchedulerThread::new();
            threads.push((is_busy, new_thread));

            true
        } else {
            // Can't spawn a new thread
            false
        }
    }
}
<commit_msg>Signal the condition variables if there are any when rescheduling a queue<commit_after>use super::scheduler_thread::*;
use super::job_queue::*;
use super::queue_state::*;
use super::wake_queue::*;

use std::sync::*;
use std::collections::vec_deque::*;

use futures::task;
use futures::task::{Context};

///
/// The scheduler core contains the internal data used by the scheduler
///
pub (super) struct SchedulerCore {
    /// The queues that are active in the scheduler
    pub (super) schedule: Arc<Mutex<VecDeque<Arc<JobQueue>>>>,

    /// Active threads and whether or not they're busy
    pub (super) threads: Mutex<Vec<(Arc<Mutex<bool>>, SchedulerThread)>>,

    /// The maximum number of threads permitted in this scheduler
    pub (super) max_threads: Mutex<usize>
}

impl SchedulerCore {
    ///
    /// Wakes a thread to run a dormant queue. Returns true if a thread was woken up
    ///
    pub (super) fn schedule_thread(&self, core: Arc<SchedulerCore>) -> bool {
        // Find a dormant thread and activate it
        let schedule = self.schedule.clone();

        // Schedule work on this dormant thread
        let work_core   = Arc::clone(&core);
        let do_work     = move |work: Arc<JobQueue>| {
            let waker       = Arc::new(WakeQueue(Arc::clone(&work), Arc::clone(&work_core)));
            let waker       = task::waker_ref(&waker);
            let mut context = Context::from_waker(&waker);

            work.drain(&mut context)
        };

        if !self.schedule_dormant(move || Self::next_to_run(&schedule), do_work) {
            // Try to create a new thread
            if self.spawn_thread_if_less_than_maximum() {
                // Try harder to schedule this task if a thread was created
                self.schedule_thread(core)
            } else {
                // Couldn't schedule on an existing thread or create a new one
                false
            }
        } else {
            // Successfully scheduled
            true
        }
    }

    ///
    /// If a queue is idle and has pending jobs, places it in the schedule
    ///
    pub (super) fn reschedule_queue(&self, queue: &Arc<JobQueue>, core: Arc<SchedulerCore>) {
        let reschedule = {
            let mut core = queue.core.lock().expect("JobQueue core lock");

            // Signal any waiting condition variables
            core.wake_blocked.iter_mut()
                .for_each(|cond_var| {
                    if let Some(cond_var) = cond_var.upgrade() {
                        cond_var.notify_one();
                    }
                });
            core.wake_blocked.retain(|cond_var| cond_var.strong_count() > 0);

            match core.state {
                QueueState::Idle => {
                    // Schedule a thread to restart the queue if more things were queued
                    if core.queue.len() > 0 {
                        // Need to schedule the queue after this event
                        core.state = QueueState::Pending;
                        true
                    } else {
                        // Queue is empty and can go back to idle
                        core.state = QueueState::Idle;
                        false
                    }
                },

                QueueState::WaitingForPoll(_) => {
                    // If the target thread gets stuck and stops draining the queue, race it to reschedule it on one of our threads if we can
                    true
                },

                _ => {
                    // Not scheduled
                    false
                }
            }
        };

        if reschedule {
            self.schedule.lock().expect("Schedule lock").push_back(queue.clone());
            self.schedule_thread(core);
        }
    }

    ///
    /// Finds the next queue that should be run. If this returns successfully, the queue will
    /// be marked as running.
    ///
    pub (super) fn next_to_run(schedule: &Arc<Mutex<VecDeque<Arc<JobQueue>>>>) -> Option<Arc<JobQueue>> {
        // Search the queues...
        let mut schedule = schedule.lock().expect("Schedule lock");

        // Find a queue where the state is pending
        while let Some(q) = schedule.pop_front() {
            let mut core = q.core.lock().expect("JobQueue core lock");

            match core.state {
                QueueState::Pending |
                QueueState::WaitingForPoll(_) => {
                    // Queue is ready to run. Mark it as running and return it
                    core.state = QueueState::Running;
                    return Some(q.clone());
                }

                _ => {
                    // Move to the next queue in the schedule
                }
            }
        }

        None
    }

    ///
    /// If any of the scheduler threads have finished (which generally means they panicked), despawn them
    ///
    pub (super) fn remove_finished_threads(&self) {
        let mut dead_threads = vec![];

        // Collate the dead threads into a vec
        {
            // Find busy threads
            let mut threads = self.threads.lock().expect("Scheduler threads lock");

            // TODO: drain_filter in nightly would be better than this (but it's in nightly)
            let mut thread_num = 0;
            while thread_num < threads.len() {
                let (_, thread) = &threads[thread_num];

                if thread.is_finished() {
                    // Remove the thread and queue it for later despawning
                    let (is_busy, dead_thread) = threads.remove(thread_num);
                    dead_threads.push((is_busy, dead_thread));
                } else {
                    // Thread still running
                    thread_num += 1;
                }
            }
        }

        // Despawn the dead threads (which might panic a bit)
        for (is_busy, dead_thread) in dead_threads {
            // Join with the thread in case it's mid-panic
            let maybe_panic = dead_thread.despawn().join();

            if let Ok(()) = maybe_panic {
                // The busy flag should always be unset after a thread is despawned
                let busy = is_busy.lock().unwrap();
                if *busy {
                    panic!("Thread despawned while busy");
                }
            } else {
                // Panics are relayed via the desync that failed
            }
        }
    }

    ///
    /// Attempts to schedule a task on a dormant thread
    ///
    pub (super) fn schedule_dormant<NextJob, RunJob, JobData>(&self, next_job: NextJob, job: RunJob) -> bool
    where
        RunJob:     'static + Send + Fn(JobData) -> (),
        NextJob:    'static + Send + Fn() -> Option<JobData>,
    {
        // Try to despawn any threads that have finished since the last time we were called
        self.remove_finished_threads();

        // Find busy threads
        let threads = self.threads.lock().expect("Scheduler threads lock");

        // Find the first thread that is not marked as busy and schedule this task on it
        for &(ref busy_rc, ref thread) in threads.iter() {
            if let Ok(mut busy) = busy_rc.try_lock() {
                // If the busy lock is held, then we consider the thread to be busy
                if !*busy {
                    // Clone the busy mutex so we can return this thread to readiness
                    let also_busy =  busy_rc.clone();

                    // This thread is busy
                    *busy = true;
                    thread.run(move || {
                        let mut done = false;

                        while !done {
                            // Obtain the next job. The thread is not busy once there are no longer any jobs
                            // We hold the mutex while this is going on to avoid a race condition when a thread is going dormant
                            let job_data = {
                                let mut busy = also_busy.lock().expect("Thread busy lock");
                                let job_data = next_job();

                                // If there's no next job, then this thread is no longer busy
                                if job_data.is_none() {
                                    *busy = false;
                                }

                                job_data
                            };

                            // Run the job if there is one, stop the thread if there is not
                            if let Some(job_data) = job_data {
                                job(job_data);
                            } else {
                                done = true;
                            }
                        }
                    });

                    return true;
                }
            }
        }

        // No dormant threads were found
        false
    }

    ///
    /// If we're running fewer than the maximum number of threads, try to spawn a new one
    ///
    pub (super) fn spawn_thread_if_less_than_maximum(&self) -> bool {
        let max_threads = { *self.max_threads.lock().expect("Max threads lock") };
        let mut threads = self.threads.lock().expect("Scheduler threads lock");

        if threads.len() < max_threads {
            // Create a new thread
            let is_busy     = Arc::new(Mutex::new(false));
            let new_thread  = SchedulerThread::new();
            threads.push((is_busy, new_thread));

            true
        } else {
            // Can't spawn a new thread
            false
        }
    }
}
<|endoftext|>


<commit_before>use std::{
    collections::{BTreeMap, HashMap},
    convert::TryFrom,
    fs::File,
    io,
    iter::FromIterator,
    ops::Deref,
    path::{Path, PathBuf},
    sync::atomic::{AtomicUsize, Ordering},
};

#[cfg(unix)]
use std::os::unix::prelude::FileExt;

use serde::{Deserialize, Serialize};
use uuid::Uuid;

/// In-memory address for a record.
#[derive(PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)]
pub(super) struct MemAddr(u64);

impl From<u64> for MemAddr {
    fn from(addr: u64) -> Self {
        MemAddr(addr)
    }
}

impl Deref for MemAddr {
    type Target = u64;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

/// File addresss from which to fetch data from (based on the length of the record).
struct DiskAddr(u64);

impl From<u64> for DiskAddr {
    fn from(addr: u64) -> Self {
        DiskAddr(addr)
    }
}

impl Deref for DiskAddr {
    type Target = u64;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

/// The amount of data to be fetched from a given position.
type RecordLength = u64;

/// TypeId of the record being stored, used to deserialize the data later.
#[repr(u8)]
#[derive(Clone, Copy)]
pub(super) enum TypeId {
    LogSent = 0,
    GrMemb = 1,
    GrFunc = 2,
}

pub(super) trait ToBinaryObject
where
    Self: Sized,
{
    fn destruct(self) -> (MemAddr, Vec<u8>);
    fn get_type_id() -> TypeId;
    fn build<T>(address: MemAddr, key: &str, data: &T) -> bincode::Result<Self>
    where
        T: Serialize;
}

macro_rules! binary_storage {
    ( struct $bin:ident -> $type:expr) => {
        pub(super) struct $bin {
            /// the current physical address, used to preserve relationships
            address: MemAddr,
            /// the actual binary serialized content of the record:
            // [key_size: u64 as [u8], key as [u8], data as [u8]]
            record: Vec<u8>,
        }

        impl ToBinaryObject for $bin {
            fn build<T: Serialize>(address: MemAddr, key: &str, data: &T) -> bincode::Result<Self> {
                // TODO: optimize this if possible to avoid copying
                let bin_key = bincode::serialize(key)?;
                let mut record: Vec<u8> = (bin_key.len() as u64).to_le_bytes().to_vec();
                record.extend(bin_key);
                let record_data = &mut bincode::serialize::<T>(&data)?;
                record.append(record_data);
                Ok($bin { address, record })
            }

            fn destruct(self) -> (MemAddr, Vec<u8>) {
                (self.address, self.record)
            }

            fn get_type_id() -> TypeId {
                $type
            }
        }
    };
}

binary_storage!(struct BinGrFuncRecord -> TypeId::GrFunc);
binary_storage!(struct BinGrMembRecord -> TypeId::GrMemb);
binary_storage!(struct BinLogSentRecord -> TypeId::LogSent);

/// A handle in memory of the on-disk storage for a given Representation.
/// Allows for rebuilding of an agent from a binary blob as well as keeping the copy of data
/// in the storage layer up to date. Each representation has it's own file for storing their
/// own objects.
///
/// Accurately rebuilds any shared objects, maintaining the underlying memory relationship
/// and shared memory space.
pub(super) struct ReprStorageManager {
    /// C0 in-memory tree, buffer for data waiting to be flushed to disk.
    /// This is only for record data, meaning records which hold a stable address in memory
    /// which is used as a key.
    level0_map: BTreeMap<MemAddr, Vec<u8>>,
    buffer_size: u64,
    file_size: u64,
    page_manager: PageManager,
}

impl ReprStorageManager {
    /// Create a new ReprStorage for the given representation, will be stored at the given directory
    /// if provided or in a temporary directory otherwise.
    pub fn new(id: Uuid, path: Option<&Path>) -> Result<Self, io::Error> {
        let file_dir = if let Some(path) = path {
            let md = path.metadata()?;
            if !md.is_dir() {
                return Err(io::ErrorKind::InvalidInput.into());
            }
            if md.permissions().readonly() {
                return Err(io::ErrorKind::PermissionDenied.into());
            }
            path.into()
        } else {
            std::env::temp_dir().join(format!("simag-{}", id))
        };

        // let file = OpenOptions::new()
        //     .read(true)
        //     .write(true)
        //     .truncate(false)
        //     .create(true)
        //     .open(&location)?;

        Ok(ReprStorageManager {
            level0_map: BTreeMap::new(),
            buffer_size: 0,
            file_size: 0,
            page_manager: PageManager::new(file_dir),
        })
    }

    /// Inserts a new record, if it exists the new record will be updated.
    ///
    /// Internally this does not flush to disk, but only stores the data in the record buffer.
    pub fn insert_rec<O>(&mut self, rec: O)
    where
        O: ToBinaryObject,
    {
        let (mem_addr, data) = rec.destruct();
        if self.buffer_size + data.len() as u64 > PageManager::PAGE_SIZE {
            // should flush first
            self.flush::<O>();
            self.buffer_size = 0;
        }
        self.buffer_size += data.len() as u64;
        self.level0_map.insert(mem_addr, data);
    }

    pub fn flush<O: ToBinaryObject>(&mut self) {
        let mut to_flush = BTreeMap::new();
        let type_id = O::get_type_id() as u8;
        std::mem::swap(&mut to_flush, &mut self.level0_map);
        let mut buffer = Vec::with_capacity(PageManager::PAGE_SIZE as usize);
        //key_addr: BTreeMap<MemAddr, (DiskAddr, RecordLength)>
        let mut table_data = HashMap::new();
        let mut addr = 0u64;
        for (mem_addr, rec) in to_flush.into_iter() {
            let len = rec.len() as u64;
            // add [type_id: u8, mem_addr: u64 as [u8], size: u64 as [u8]], total 17 bytes header
            buffer.push(type_id);
            buffer.extend((&*mem_addr).to_le_bytes().as_ref());
            buffer.extend(&len.to_le_bytes());
            // add the value itself
            buffer.extend(rec);
            table_data.insert(
                mem_addr,
                DiskRecord {
                    type_id: O::get_type_id(),
                    addr: DiskAddr::from(addr),
                    len,
                },
            );
            addr += len;
        }

        if (buffer.len() as u64) < PageManager::PAGE_SIZE {
            // Fill with zeros the remaining until the page is full.
            let remainder = PageManager::PAGE_SIZE - (buffer.len() as u64);
            let remaining = vec![0; remainder as usize];
            buffer.extend(remaining);
        }
        debug_assert_eq!(buffer.len() as u64, PageManager::PAGE_SIZE);

        // self.page_manager;
        // #[cfg(unix)]
        // {
        //     self.file.write_at(&buffer, self.file_size).unwrap();
        // }
        self.file_size = buffer.len() as u64;
    }
}

impl TryFrom<&Path> for ReprStorageManager {
    type Error = bincode::Error;

    fn try_from(path: &Path) -> Result<Self, Self::Error> {
        let file = File::open(path)?;

        todo!()
    }
}

struct DiskRecord {
    type_id: TypeId,
    addr: DiskAddr,
    len: RecordLength,
}

impl Into<[u8; 17]> for DiskRecord {
    fn into(self) -> [u8; 17] {
        let mut bytes = [0u8; 17];
        bytes[0] = self.type_id as u8;
        let (_, pos_part) = bytes.split_at_mut(1);
        pos_part.copy_from_slice((&*self.addr).to_le_bytes().as_ref());
        let (_, len_part) = bytes.split_at_mut(9);
        len_part.copy_from_slice(self.len.to_le_bytes().as_ref());
        bytes
    }
}

static NEXT_PAGE_ID: AtomicUsize = AtomicUsize::new(0);
type PageId = usize;

struct PageMeta {
    /// Current size of page.
    size: u64,
    /// In representations a lot of records map well to key-values tuples, where the key is
    /// the shared memory address pointer and the value the underlying data.
    /// On pages we append the record to the log, and later on we can merge the data
    /// using the address as a key, fetching from the registered file disk address.
    key_addr: BTreeMap<MemAddr, (DiskAddr, RecordLength)>,
}

enum MapLevel {
    Level0,
    Level1,
    Level2,
    Level3,
}

struct PageManager {
    file_dir: PathBuf,
    levels: [BTreeMap<PageId, PageMeta>; 3],
}

impl PageManager {
    /// Modern disks sectors are usually 4KB bytes wide, and are optimized for such size writes/reads.
    /// All the pages must be then a multiple of this so they are written to block effitiently.
    const PAGE_SIZE: u64 = 1024 * 4;

    /// The LEVEL0 tree is maintained in-memory.
    /// Max size for all the pages
    const LEVEL1_SIZE: u64 = 1024 * 1024 * 10; // 10MB
    const LEVEL1_NUM_PAGES: u64 = Self::LEVEL1_SIZE / Self::PAGE_SIZE;
    const LEVEL2_SIZE: u64 = Self::LEVEL1_SIZE ^ 10; // 100 MB
    const LEVEL2_NUM_PAGES: u64 = Self::LEVEL2_SIZE / Self::PAGE_SIZE;
    // LEVEL3 size is unbounded
    const LEVEL3_MAX_FILE_SIZE: u64 = Self::LEVEL2_SIZE;

    fn new(file_dir: PathBuf) -> PageManager {
        let level1_map = BTreeMap::from_iter((0..Self::LEVEL1_NUM_PAGES).map(|_| {
            (
                NEXT_PAGE_ID.load(Ordering::SeqCst),
                PageMeta {
                    size: 0,
                    key_addr: BTreeMap::new(),
                },
            )
        }));
        debug_assert_eq!(level1_map.len() as u64, Self::LEVEL1_NUM_PAGES);

        let level2_map = BTreeMap::from_iter((0..Self::LEVEL2_NUM_PAGES).map(|_| {
            (
                NEXT_PAGE_ID.load(Ordering::SeqCst),
                PageMeta {
                    size: 0,
                    key_addr: BTreeMap::new(),
                },
            )
        }));
        debug_assert_eq!(level2_map.len() as u64, Self::LEVEL2_NUM_PAGES);

        PageManager {
            file_dir,
            levels: [level1_map, level2_map, BTreeMap::new()],
        }
    }

    #[inline(always)]
    fn spill_to_disk(&mut self, data: Vec<u8>) {
        let level1_map = &mut self.levels[0];
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn flush_data() -> Result<(), std::io::Error> {
        let mut rep = ReprStorageManager::new(Uuid::nil(), None)?;
        let record = String::from("test_data").into_bytes();
        let rec = BinGrMembRecord {
            address: 0x1234u64.into(),
            record,
        };

        rep.insert_rec(rec);
        rep.flush::<BinGrMembRecord>();

        Ok(())
    }
}
<commit_msg>page manager insert buffered records in c0 log<commit_after>use std::{
    collections::BTreeMap,
    convert::TryFrom,
    fs::{File, OpenOptions},
    io,
    iter::FromIterator,
    ops::Deref,
    path::{Path, PathBuf},
    sync::atomic::{AtomicUsize, Ordering},
};

#[cfg(unix)]
use std::os::unix::prelude::FileExt;

use serde::{Deserialize, Serialize};
use uuid::Uuid;

/// In-memory address for a record.
#[derive(PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash, Clone, Copy)]
pub(super) struct MemAddr(u64);

impl From<u64> for MemAddr {
    fn from(addr: u64) -> Self {
        MemAddr(addr)
    }
}

impl Deref for MemAddr {
    type Target = u64;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

/// File addresss from which to fetch data from (based on the length of the record).
#[derive(Clone, Copy)]
struct DiscAddr(u64);

impl From<u64> for DiscAddr {
    fn from(addr: u64) -> Self {
        DiscAddr(addr)
    }
}

impl Deref for DiscAddr {
    type Target = u64;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

/// The amount of data to be fetched from a given position.
type RecordLength = u64;

/// TypeId of the record being stored, used to deserialize the data later.
#[repr(u8)]
#[derive(Clone, Copy)]
pub(super) enum TypeId {
    LogSent = 0,
    GrMemb = 1,
    GrFunc = 2,
}

pub(super) trait ToBinaryObject
where
    Self: Sized,
{
    fn destruct(self) -> (MemAddr, Vec<u8>);
    fn get_type_id() -> TypeId;
    fn build<T>(address: MemAddr, key: &str, data: &T) -> bincode::Result<Self>
    where
        T: Serialize;
}

macro_rules! binary_storage {
    ( struct $bin:ident -> $type:expr) => {
        #[derive(Clone)]
        pub(super) struct $bin {
            /// the current physical address, used to preserve relationships
            address: MemAddr,
            /// the actual binary serialized content of the record:
            // [key_size: u64 as [u8], key as [u8], data as [u8]]
            record: Vec<u8>,
        }

        impl ToBinaryObject for $bin {
            fn build<T: Serialize>(address: MemAddr, key: &str, data: &T) -> bincode::Result<Self> {
                // TODO: optimize this if possible to avoid copying
                let bin_key = bincode::serialize(key)?;
                let mut record: Vec<u8> = (bin_key.len() as u64).to_le_bytes().to_vec();
                record.extend(bin_key);
                let record_data = &mut bincode::serialize::<T>(&data)?;
                record.append(record_data);
                Ok($bin { address, record })
            }

            fn destruct(self) -> (MemAddr, Vec<u8>) {
                (self.address, self.record)
            }

            fn get_type_id() -> TypeId {
                $type
            }
        }
    };
}

binary_storage!(struct BinGrFuncRecord -> TypeId::GrFunc);
binary_storage!(struct BinGrMembRecord -> TypeId::GrMemb);
binary_storage!(struct BinLogSentRecord -> TypeId::LogSent);

/// A handle in memory of the on-disk storage for a given Representation.
/// Allows for rebuilding of an agent from a binary blob as well as keeping the copy of data
/// in the storage layer up to date. Each representation has it's own file for storing their
/// own objects.
///
/// Accurately rebuilds any shared objects, maintaining the underlying memory relationship
/// and shared memory space.
pub(super) struct ReprStorageManager {
    /// C0 in-memory tree, buffer for data waiting to be flushed to disk.
    /// This is only for record data, meaning records which hold a stable address in memory
    /// which is used as a key.
    level0_map: BTreeMap<MemAddr, Record>,
    buffer_size: u64,
    page_manager: PageManager,
}

impl ReprStorageManager {
    /// Create a new ReprStorage for the given representation, will be stored at the given directory
    /// if provided or in a temporary directory otherwise.
    pub fn new(id: Uuid, path: Option<&Path>) -> io::Result<Self> {
        let file_dir = if let Some(path) = path {
            let md = path.metadata()?;
            if !md.is_dir() {
                return Err(io::ErrorKind::InvalidInput.into());
            }
            if md.permissions().readonly() {
                return Err(io::ErrorKind::PermissionDenied.into());
            }
            path.into()
        } else {
            let p = std::env::temp_dir().join(format!("simag-{}", id));
            if !p.exists() {
                std::fs::create_dir(&p)?;
            }
            p
        };

        Ok(ReprStorageManager {
            level0_map: BTreeMap::new(),
            buffer_size: 0,
            page_manager: PageManager::new(file_dir, id),
        })
    }

    /// Inserts a new record, if it exists the new record will be updated.
    ///
    /// Internally this does not flush to disk, but only stores the data in the record buffer.
    pub fn insert_rec<O>(&mut self, rec: O) -> io::Result<()>
    where
        O: ToBinaryObject,
    {
        let (mem_addr, data) = rec.destruct();
        let rec = Record::new(O::get_type_id(), mem_addr, data);
        if self.buffer_size + rec.len() as u64 > PageManager::PAGE_SIZE {
            // should flush first
            self.flush()?;
            self.buffer_size = 0;
        }
        self.buffer_size += rec.len() as u64;
        self.level0_map.insert(mem_addr, rec);
        Ok(())
    }

    pub fn flush(&mut self) -> io::Result<()> {
        let mut to_flush = BTreeMap::new();
        std::mem::swap(&mut to_flush, &mut self.level0_map);
        let mut buffer = Vec::with_capacity(PageManager::PAGE_SIZE as usize);
        let mut table_data = BTreeMap::new();
        let mut addr = 0u64;
        for (mem_addr, rec) in to_flush.into_iter() {
            let len = rec.len() as u64;
            let type_id = rec.type_id();
            rec.append_to_buf(&mut buffer);
            table_data.insert(
                mem_addr,
                DiscRecordRef {
                    type_id,
                    addr: DiscAddr::from(addr),
                    len,
                },
            );
            addr += len;
        }
        self.page_manager.spill_to_disk(buffer, table_data)
    }
}

impl TryFrom<&Path> for ReprStorageManager {
    type Error = bincode::Error;

    fn try_from(path: &Path) -> Result<Self, Self::Error> {
        let _file = File::open(path)?;
        todo!()
    }
}

/// A record prepared to be stored on disk.
struct Record {
    type_id: TypeId,
    mem_addr: MemAddr,
    data: Vec<u8>,
    /// computed lenth after encoding to bytes
    length: u64,
}

impl Record {
    fn new(type_id: TypeId, mem_addr: MemAddr, data: Vec<u8>) -> Self {
        let length = data.len() as u64 + 17;
        Record {
            type_id,
            mem_addr,
            data,
            length,
        }
    }

    #[inline]
    fn len(&self) -> u64 {
        self.length
    }

    fn append_to_buf(self, buffer: &mut Vec<u8>) {
        let Record {
            type_id,
            mem_addr,
            data,
            ..
        } = self;
        let header = Self::make_header(type_id, mem_addr, data.len() as u64);
        buffer.extend(&header);
        buffer.extend(data);
    }

    fn type_id(&self) -> TypeId {
        self.type_id
    }

    #[inline(always)]
    fn make_header(type_id: TypeId, mem_addr: MemAddr, len: u64) -> [u8; 17] {
        let mut header = [0u8; 17];
        let addr_part = &mut header[0..8];
        addr_part.copy_from_slice((&*mem_addr).to_le_bytes().as_ref());
        header[8] = type_id as u8;
        let len_part = &mut header[9..];
        len_part.copy_from_slice(len.to_le_bytes().as_ref());
        header
    }
}

struct DiscRecordRef {
    type_id: TypeId,
    addr: DiscAddr,
    len: RecordLength,
}

static NEXT_PAGE_ID: AtomicUsize = AtomicUsize::new(0);
type PageId = usize;

struct PageMeta {
    id: PageId,
    offset: u64,
    /// Current free space of page.
    free_space: u64,
    /// In representations a lot of records map well to key-values tuples, where the key is
    /// the shared memory address pointer and the value the underlying data.
    /// On pages we append the record to the log, and later on we can merge the data
    /// using the address as a key, fetching from the registered file disk address.
    key_addr: BTreeMap<MemAddr, DiscRecordRef>,
}

/// Manages pages and file synchronization with memory data for log record-like data.
struct PageManager {
    id: Uuid,
    file_dir: PathBuf,
    levels: [Vec<PageMeta>; 3],
}

impl PageManager {
    /// Modern disks blocks are usually 4KB bytes wide, and are optimized for such size writes/reads.
    /// All the pages must be then a multiple of this so they are written to block effitiently.
    const PAGE_SIZE: u64 = 1024 * 4;

    /// The LEVEL0 tree is maintained in-memory.
    /// Max size for all the pages
    const LEVEL1_SIZE: u64 = 1024 * 1024 * 10; // 10MB
    const LEVEL1_NUM_PAGES: u64 = Self::LEVEL1_SIZE / Self::PAGE_SIZE;
    const LEVEL2_SIZE: u64 = Self::LEVEL1_SIZE * 10; // 100 MB
    const LEVEL2_NUM_PAGES: u64 = Self::LEVEL2_SIZE / Self::PAGE_SIZE;
    // LEVEL3 size is unbounded
    const LEVEL3_MAX_FILE_SIZE: u64 = Self::LEVEL2_SIZE;

    fn new(file_dir: PathBuf, id: Uuid) -> PageManager {
        let mut lvl1_offset = 0;
        let lvl1_pages = Vec::from_iter((0..Self::LEVEL1_NUM_PAGES).map(|_| {
            let p = PageMeta {
                id: NEXT_PAGE_ID.load(Ordering::SeqCst),
                offset: lvl1_offset,
                free_space: Self::PAGE_SIZE,
                key_addr: BTreeMap::new(),
            };
            lvl1_offset += Self::PAGE_SIZE;
            p
        }));
        debug_assert_eq!(lvl1_pages.len() as u64, Self::LEVEL1_NUM_PAGES);

        let mut lvl2_offset = 0;
        let lvl2_pages = Vec::from_iter((0..Self::LEVEL2_NUM_PAGES).map(|_| {
            let p = PageMeta {
                id: NEXT_PAGE_ID.load(Ordering::SeqCst),
                offset: lvl2_offset,
                free_space: Self::PAGE_SIZE * 10,
                key_addr: BTreeMap::new(),
            };
            lvl2_offset += Self::PAGE_SIZE;
            p
        }));
        debug_assert_eq!(lvl2_pages.len() as u64, Self::LEVEL2_NUM_PAGES);

        PageManager {
            id,
            file_dir,
            levels: [lvl1_pages, lvl2_pages, Vec::new()],
        }
    }

    #[inline(always)]
    fn spill_to_disk(
        &mut self,
        mut buffer: Vec<u8>,
        table_map: BTreeMap<MemAddr, DiscRecordRef>,
    ) -> io::Result<()> {
        let p = self.file_dir.join("data0.dat");

        let lvl1_pages = &mut self.levels[0];
        lvl1_pages.sort_unstable_by_key(|e| e.free_space);
        if lvl1_pages[0].free_space >= buffer.len() as u64 {
            let page = &mut lvl1_pages[0];
            let buf_len = buffer.len() as u64;
            if buf_len < page.free_space {
                // Fill with zeros the remaining until the page is full.
                let remainder = page.free_space - buf_len;
                let remaining = vec![0; remainder as usize];
                buffer.extend(remaining);
            }

            let file = OpenOptions::new()
                .write(true)
                .truncate(false)
                .create(true)
                .open(&p)?;

            let offset = page.offset + (Self::PAGE_SIZE - page.free_space);
            #[cfg(unix)]
            {
                file.write_at(&buffer, offset)?;
            }

            for (addr, rec) in table_map {
                page.key_addr.insert(addr, rec);
            }
            page.free_space -= buf_len;
        }

        Ok(())
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn flush_data() -> io::Result<()> {
        let mut rep = ReprStorageManager::new(Uuid::nil(), None)?;
        let record = String::from("test_data").into_bytes();
        let rec = BinGrMembRecord {
            address: 0x1234u64.into(),
            record,
        };
        for _ in 0..3 {
            rep.insert_rec(rec.clone())?;
            rep.flush()?;
        }
        rep.insert_rec(rec.clone())?;
        rep.insert_rec(rec)?;
        rep.flush()?;
        Ok(())
    }
}
<|endoftext|>


<commit_before>// xfail-test

// Broken due to arena API problems.

// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern mod extra;
use extra::arena;

enum tree<'self> {
    nil,
    node(&'self tree<'self>, &'self tree<'self>, int),
}

fn item_check(t: &tree) -> int {
    match *t {
      nil => { return 0; }
      node(left, right, item) => {
        return item + item_check(left) - item_check(right);
      }
    }
}

fn bottom_up_tree<'r>(arena: &'r mut arena::Arena, item: int, depth: int)
                   -> &'r tree<'r> {
    if depth > 0 {
        return arena.alloc(
            || node(bottom_up_tree(arena, 2 * item - 1, depth - 1),
                    bottom_up_tree(arena, 2 * item, depth - 1),
                    item));
    }
    return arena.alloc(|| nil);
}

fn main() {
    use std::os;
    use std::int;
    let args = std::os::args();
    let args = if os::getenv(~"RUST_BENCH").is_some() {
        ~[~"", ~"17"]
    } else if args.len() <= 1u {
        ~[~"", ~"8"]
    } else {
        args
    };

    let n = int::from_str(args[1]).get();
    let min_depth = 4;
    let mut max_depth;
    if min_depth + 2 > n {
        max_depth = min_depth + 2;
    } else {
        max_depth = n;
    }

    let mut stretch_arena = arena::Arena();
    let stretch_depth = max_depth + 1;
    let stretch_tree = bottom_up_tree(&mut stretch_arena, 0, stretch_depth);

    println(fmt!("stretch tree of depth %d\t check: %d",
                          stretch_depth,
                          item_check(stretch_tree)));

    let mut long_lived_arena = arena::Arena();
    let long_lived_tree = bottom_up_tree(&mut long_lived_arena, 0, max_depth);
    let mut depth = min_depth;
    while depth <= max_depth {
        let iterations = int::pow(2, (max_depth - depth + min_depth) as uint);
        let mut chk = 0;
        let mut i = 1;
        while i <= iterations {
            let mut temp_tree = bottom_up_tree(&mut long_lived_arena, i, depth);
            chk += item_check(temp_tree);
            temp_tree = bottom_up_tree(&mut long_lived_arena, -i, depth);
            chk += item_check(temp_tree);
            i += 1;
        }
        println(fmt!("%d\t trees of depth %d\t check: %d",
                         iterations * 2, depth,
                         chk));
        depth += 2;
    }
    println(fmt!("long lived tree of depth %d\t check: %d",
                     max_depth,
                     item_check(long_lived_tree)));
}
<commit_msg>Updated copyright year on shootout-binarytrees.rs<commit_after>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern mod extra;
use extra::arena;

enum Tree<'self> {
    Nil,
    Node(&'self Tree<'self>, &'self Tree<'self>, int),
}

fn item_check(t: &Tree) -> int {
    match *t {
      Nil => { return 0; }
      Node(left, right, item) => {
        return item + item_check(left) - item_check(right);
      }
    }
}

fn bottom_up_tree<'r>(arena: &'r arena::Arena, item: int, depth: int)
                   -> &'r Tree<'r> {
    if depth > 0 {
        return arena.alloc(
            || Node(bottom_up_tree(arena, 2 * item - 1, depth - 1),
                    bottom_up_tree(arena, 2 * item, depth - 1),
                    item));
    }
    return arena.alloc(|| Nil);
}

fn main() {
    use std::os;
    use std::int;
    let args = std::os::args();
    let args = if os::getenv(~"RUST_BENCH").is_some() {
        ~[~"", ~"17"]
    } else if args.len() <= 1u {
        ~[~"", ~"8"]
    } else {
        args
    };

    let n = int::from_str(args[1]).get();
    let min_depth = 4;
    let mut max_depth;
    if min_depth + 2 > n {
        max_depth = min_depth + 2;
    } else {
        max_depth = n;
    }

    let stretch_arena = arena::Arena();
    let stretch_depth = max_depth + 1;
    let stretch_tree = bottom_up_tree(&stretch_arena, 0, stretch_depth);

    println(fmt!("stretch tree of depth %d\t check: %d",
                          stretch_depth,
                          item_check(stretch_tree)));

    let long_lived_arena = arena::Arena();
    let long_lived_tree = bottom_up_tree(&long_lived_arena, 0, max_depth);
    let mut depth = min_depth;
    while depth <= max_depth {
        let iterations = int::pow(2, (max_depth - depth + min_depth) as uint);
        let mut chk = 0;
        let mut i = 1;
        while i <= iterations {
            let mut temp_tree = bottom_up_tree(&long_lived_arena, i, depth);
            chk += item_check(temp_tree);
            temp_tree = bottom_up_tree(&long_lived_arena, -i, depth);
            chk += item_check(temp_tree);
            i += 1;
        }
        println(fmt!("%d\t trees of depth %d\t check: %d",
                         iterations * 2, depth,
                         chk));
        depth += 2;
    }
    println(fmt!("long lived tree of depth %d\t check: %d",
                     max_depth,
                     item_check(long_lived_tree)));
}
<|endoftext|>


<commit_before>/// 2-headed, Copy-on-Write B+Tree map
/// Supports MVCC up to 2 revisions
/// Lives entirely within the slice that is given to it.
/// Keys and Values are byte slices.

use std::mem;
use self::NodeType::*;
use allocator::*;

pub const N: usize = 2;
pub const B: usize = 100;

#[repr(u8)]
enum NodeType {
    Meta,
    Root,
    Internal,
    Leaf,
}

/// Maps arbitrary [u8] to [u8].
/// One value per key
pub struct BTree<'a> {
    page_pool: Pool,
    current_root: &'a NodeHeader,
    roots: [&'a NodeHeader; 2],
}

/// Public API
impl <'a> BTree<'a> {
    pub fn new(buf: &'a mut [u8]) -> Result<BTree, &'static str> {
        let mut page_pool = Pool::new(buf);
        let root_a_i = try!(page_pool.alloc());
        let root_b_i = try!(page_pool.alloc());

        Err("Not implemented yet")
    }

    pub fn open() {

    }
}

/// Internal Functions
impl <'a> BTree<'a> {

}

/// The structure of a tree is a series of Nodes.
/// Each node is made up of at least 1 page.
/// The first page is interpreted as a NodeHeader + data
/// subsequent pages are interpreted as data based on the
/// NodeType defined by the header.
#[repr(C)]
struct NodeHeader {
    node_type: NodeType,
    tx_id: usize,
    keys: [EntryLocation; B],
    children: [EntryLocation; B],
}

impl NodeHeader {
    fn new_in_page(p: &mut Page) -> &NodeHeader {
        p.transmute_page::<NodeHeader>()
    }
}

#[repr(C)]
struct EntryLocation {
    page_index: usize,
    offset: usize,
}

#[repr(u8)]
enum EntryType {
    Alias,
    Entry,
    Deleted,
}

#[repr(C)]
struct ByteStringEntryAlias {
    entry_type: EntryType,
    num_segments: usize,
    // sizeof(EntryLocation) * num_segments
}

#[repr(C)]
struct ByteStringEntry {
    entry_type: EntryType,
    data_size: usize,
    // data_size bytes of data
}
<commit_msg>Cleaned up structure of allocator (I know I'm progastinating)<commit_after>/// 2-headed, Copy-on-Write B+Tree map
/// Supports MVCC up to 2 revisions
/// Lives entirely within the slice that is given to it.
/// Keys and Values are byte slices.

use std::mem;
use self::NodeType::*;
use allocator::*;

pub const N: usize = 2;
pub const B: usize = 100;

#[repr(u8)]
enum NodeType {
    Meta,
    Root,
    Internal,
    Leaf,
}

/// Maps arbitrary [u8] to [u8].
/// One value per key
pub struct BTree<'a> {
    page_pool: Pool,
    current_root: &'a NodeHeader,
    roots: Vec<&'a NodeHeader>,
}

/// Public API
impl <'a> BTree<'a> {
    pub fn new(buf: &'a mut [u8]) -> Result<BTree, &'static str> {
        let mut page_pool = Pool::new(buf);
        let mut roots = vec![];
        for _ in 0..N {
            let p = try!(page_pool.alloc());
            roots.push(NodeHeader::new_in_page(p));
        }

        Err("Not implemented yet")
    }

    pub fn open() {

    }
}

/// Internal Functions
impl <'a> BTree<'a> {

}

/// The structure of a tree is a series of Nodes.
/// Each node is made up of at least 1 page.
/// The first page is interpreted as a NodeHeader + data
/// subsequent pages are interpreted as data based on the
/// NodeType defined by the header.
#[repr(C)]
struct NodeHeader {
    node_type: NodeType,
    tx_id: usize,
    keys: [EntryLocation; B],
    children: [EntryLocation; B],
}

impl NodeHeader {
    fn new_in_page(p: &mut Page) -> &NodeHeader {
        p.transmute_page::<NodeHeader>()
    }
}

#[repr(C)]
struct EntryLocation {
    page_index: usize,
    offset: usize,
}

#[repr(u8)]
enum EntryType {
    Alias,
    Entry,
    Deleted,
}

#[repr(C)]
struct ByteStringEntryAlias {
    entry_type: EntryType,
    num_segments: usize,
    // sizeof(EntryLocation) * num_segments
}

#[repr(C)]
struct ByteStringEntry {
    entry_type: EntryType,
    data_size: usize,
    // data_size bytes of data
}
<|endoftext|>


<commit_before>// Copyright 2014-2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Handling log-probabilities.

use std::mem;
use std::f64;
use std::iter;
use std::ops::{Add, Sub, Mul, Div};
use num::traits::cast;
use num::NumCast;

use itertools::linspace;
use itertools::Itertools;
use itertools::misc::ToFloat;

pub use stats::{Prob, LogProb};

/// A factor to convert log-probabilities to PHRED-scale (phred = p * `LOG_TO_PHRED_FACTOR`).
const LOG_TO_PHRED_FACTOR: f64 = -4.3429448190325175; // -10 * 1 / ln(10)

/// A factor to convert PHRED-scale to log-probabilities (p = phred * `PHRED_TO_LOG_FACTOR`).
const PHRED_TO_LOG_FACTOR: f64 = -0.23025850929940456; // 1 / (-10 * log10(e))

/// Calculate log(1 - p) with p given in log space without loss of precision as described in
/// http://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf.
pub fn ln_1m_exp(p: LogProb) -> LogProb {
    assert!(p <= 0.0);
    if p < -0.693 {
        (-p.exp()).ln_1p()
    } else {
        (-p.exp_m1()).ln()
    }
}

/// Convert log scale probability to PHRED scale.
pub fn log_to_phred(p: LogProb) -> f64 {
    p * LOG_TO_PHRED_FACTOR
}

/// Convert PHRED scale probability to log scale.
pub fn phred_to_log(p: f64) -> LogProb {
    p * PHRED_TO_LOG_FACTOR
}

/// Calculate the sum of the given probabilities in a numerically stable way (Durbin 1998).
pub fn sum(probs: &[LogProb]) -> LogProb {
    if probs.is_empty() {
        f64::NEG_INFINITY
    } else {
        let mut pmax = probs[0];
        let mut imax = 0;
        for (i, &p) in probs.iter().enumerate().skip(1) {
            if p > pmax {
                pmax = p;
                imax = i;
            }
        }
        if pmax == f64::NEG_INFINITY {
            f64::NEG_INFINITY
        } else if pmax == f64::INFINITY {
            f64::INFINITY
        } else {
            // TODO use sum() once it has been stabilized: .sum::<usize>()
            pmax +
            (probs.iter()
                  .enumerate()
                  .filter_map(|(i, p)| {
                      if i == imax {
                          None
                      } else {
                          Some((p - pmax).exp())
                      }
                  })
                  .fold(0.0, |s, e| s + e))
                .ln_1p()
        }
    }
}

/// Add the given probabilities in a numerically stable way (Durbin 1998).
pub fn add(mut p0: LogProb, mut p1: LogProb) -> LogProb {
    if p1 > p0 {
        mem::swap(&mut p0, &mut p1);
    }
    if p0 == f64::NEG_INFINITY {
        f64::NEG_INFINITY
    } else if p0 == f64::INFINITY {
        f64::INFINITY
    } else {
        p0 + (p1 - p0).exp().ln_1p()
    }
}

/// Subtract the given probabilities in a numerically stable way (Durbin 1998).
pub fn sub(p0: LogProb, p1: LogProb) -> LogProb {
    assert!(p0 >= p1,
            "Subtraction would lead to negative probability, which is undefined in log space.");
    if (p0 - p1).abs() <= f64::EPSILON || p0 == f64::NEG_INFINITY {
        // the first case leads to zero,
        // in the second case p0 and p1 are -inf, which is fine
        f64::NEG_INFINITY
    } else if p0 == f64::INFINITY {
        f64::INFINITY
    } else {
        p0 + ln_1m_exp(p1 - p0)
    }
}

fn scan_add(s: &mut LogProb, p: LogProb) -> Option<LogProb> {
    *s = add(*s, p);
    Some(*s)
}

/// Iterator returned by scans over logprobs.
pub type ScanIter<I> = iter::Scan<I, LogProb, fn(&mut LogProb, LogProb) -> Option<LogProb>>;

/// Calculate the cumulative sum of the given probabilities in a numerically stable way (Durbin 1998).
pub fn cumsum<I: Iterator<Item = LogProb>>(probs: I) -> ScanIter<I> {
    probs.scan(f64::NEG_INFINITY, scan_add)
}

/// Integrate numerically stable over given log-space density in the interval [a, b]. Uses the trapezoidal rule with n grid points.
pub fn integrate<T, D>(density: D, a: T, b: T, n: usize) -> LogProb
    where T: NumCast + Copy + Add<Output=T> + Sub<Output=T> + Div<Output=T> + Mul<Output=T>, D: Fn(T) -> LogProb, usize: ToFloat<T>
{
    let mut probs = linspace(a, b, n).dropping(1).dropping_back(1).map(|v| 2.0f64.ln() + density(v)).collect_vec();
    probs.push(density(a));
    probs.push(density(b));
    let width: f64 = cast(b - a).unwrap();

    width.ln() - (2.0 * (n - 1) as f64).ln() + sum(&probs)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::f64;
    use itertools::Itertools;

    #[test]
    fn test_sum() {
        let probs = [f64::NEG_INFINITY, 0.0, f64::NEG_INFINITY];
        assert_eq!(sum(&probs), 0.0);
    }

    #[test]
    fn test_empty_sum() {
        assert_eq!(sum(&[]), f64::NEG_INFINITY);
    }

    #[test]
    fn test_cumsum() {
        let probs = vec![0.0f64.ln(), 0.01f64.ln(), 0.001f64.ln()];
        assert_eq!(cumsum(probs.into_iter()).collect_vec(),
                   [0.0f64.ln(), 0.01f64.ln(), 0.011f64.ln()]);
    }

    #[test]
    fn test_sub() {
        assert_eq!(sub(0.0f64.ln(), 0.0f64.ln()), f64::NEG_INFINITY);
        assert_relative_eq!(sub(1.0f64.ln(), 0.5f64.ln()), 0.5f64.ln());
    }

    #[test]
    fn test_ln_1m_exp() {
        assert_eq!(ln_1m_exp(f64::NEG_INFINITY), 0.0);
        assert_eq!(ln_1m_exp(-0.0), f64::NEG_INFINITY);
    }

    #[test]
    fn test_integrate() {
        let density = |_| 0.1f64.ln();
        let prob = integrate(density, 0.0, 10.0, 5);
        assert_relative_eq!(prob.exp(), 1.0, epsilon=0.0000001);
    }
}
<commit_msg>Take reference to density function.<commit_after>// Copyright 2014-2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Handling log-probabilities.

use std::mem;
use std::f64;
use std::iter;
use std::ops::{Add, Sub, Mul, Div};
use num::traits::cast;
use num::NumCast;

use itertools::linspace;
use itertools::Itertools;
use itertools::misc::ToFloat;

pub use stats::{Prob, LogProb};

/// A factor to convert log-probabilities to PHRED-scale (phred = p * `LOG_TO_PHRED_FACTOR`).
const LOG_TO_PHRED_FACTOR: f64 = -4.3429448190325175; // -10 * 1 / ln(10)

/// A factor to convert PHRED-scale to log-probabilities (p = phred * `PHRED_TO_LOG_FACTOR`).
const PHRED_TO_LOG_FACTOR: f64 = -0.23025850929940456; // 1 / (-10 * log10(e))

/// Calculate log(1 - p) with p given in log space without loss of precision as described in
/// http://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf.
pub fn ln_1m_exp(p: LogProb) -> LogProb {
    assert!(p <= 0.0);
    if p < -0.693 {
        (-p.exp()).ln_1p()
    } else {
        (-p.exp_m1()).ln()
    }
}

/// Convert log scale probability to PHRED scale.
pub fn log_to_phred(p: LogProb) -> f64 {
    p * LOG_TO_PHRED_FACTOR
}

/// Convert PHRED scale probability to log scale.
pub fn phred_to_log(p: f64) -> LogProb {
    p * PHRED_TO_LOG_FACTOR
}

/// Calculate the sum of the given probabilities in a numerically stable way (Durbin 1998).
pub fn sum(probs: &[LogProb]) -> LogProb {
    if probs.is_empty() {
        f64::NEG_INFINITY
    } else {
        let mut pmax = probs[0];
        let mut imax = 0;
        for (i, &p) in probs.iter().enumerate().skip(1) {
            if p > pmax {
                pmax = p;
                imax = i;
            }
        }
        if pmax == f64::NEG_INFINITY {
            f64::NEG_INFINITY
        } else if pmax == f64::INFINITY {
            f64::INFINITY
        } else {
            // TODO use sum() once it has been stabilized: .sum::<usize>()
            pmax +
            (probs.iter()
                  .enumerate()
                  .filter_map(|(i, p)| {
                      if i == imax {
                          None
                      } else {
                          Some((p - pmax).exp())
                      }
                  })
                  .fold(0.0, |s, e| s + e))
                .ln_1p()
        }
    }
}

/// Add the given probabilities in a numerically stable way (Durbin 1998).
pub fn add(mut p0: LogProb, mut p1: LogProb) -> LogProb {
    if p1 > p0 {
        mem::swap(&mut p0, &mut p1);
    }
    if p0 == f64::NEG_INFINITY {
        f64::NEG_INFINITY
    } else if p0 == f64::INFINITY {
        f64::INFINITY
    } else {
        p0 + (p1 - p0).exp().ln_1p()
    }
}

/// Subtract the given probabilities in a numerically stable way (Durbin 1998).
pub fn sub(p0: LogProb, p1: LogProb) -> LogProb {
    assert!(p0 >= p1,
            "Subtraction would lead to negative probability, which is undefined in log space.");
    if (p0 - p1).abs() <= f64::EPSILON || p0 == f64::NEG_INFINITY {
        // the first case leads to zero,
        // in the second case p0 and p1 are -inf, which is fine
        f64::NEG_INFINITY
    } else if p0 == f64::INFINITY {
        f64::INFINITY
    } else {
        p0 + ln_1m_exp(p1 - p0)
    }
}

fn scan_add(s: &mut LogProb, p: LogProb) -> Option<LogProb> {
    *s = add(*s, p);
    Some(*s)
}

/// Iterator returned by scans over logprobs.
pub type ScanIter<I> = iter::Scan<I, LogProb, fn(&mut LogProb, LogProb) -> Option<LogProb>>;

/// Calculate the cumulative sum of the given probabilities in a numerically stable way (Durbin 1998).
pub fn cumsum<I: Iterator<Item = LogProb>>(probs: I) -> ScanIter<I> {
    probs.scan(f64::NEG_INFINITY, scan_add)
}

/// Integrate numerically stable over given log-space density in the interval [a, b]. Uses the trapezoidal rule with n grid points.
pub fn integrate<T, D>(density: &D, a: T, b: T, n: usize) -> LogProb
    where T: NumCast + Copy + Add<Output=T> + Sub<Output=T> + Div<Output=T> + Mul<Output=T>, D: Fn(T) -> LogProb, usize: ToFloat<T>
{
    let mut probs = linspace(a, b, n).dropping(1).dropping_back(1).map(|v| 2.0f64.ln() + density(v)).collect_vec();
    probs.push(density(a));
    probs.push(density(b));
    let width: f64 = cast(b - a).unwrap();

    width.ln() - (2.0 * (n - 1) as f64).ln() + sum(&probs)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::f64;
    use itertools::Itertools;

    #[test]
    fn test_sum() {
        let probs = [f64::NEG_INFINITY, 0.0, f64::NEG_INFINITY];
        assert_eq!(sum(&probs), 0.0);
    }

    #[test]
    fn test_empty_sum() {
        assert_eq!(sum(&[]), f64::NEG_INFINITY);
    }

    #[test]
    fn test_cumsum() {
        let probs = vec![0.0f64.ln(), 0.01f64.ln(), 0.001f64.ln()];
        assert_eq!(cumsum(probs.into_iter()).collect_vec(),
                   [0.0f64.ln(), 0.01f64.ln(), 0.011f64.ln()]);
    }

    #[test]
    fn test_sub() {
        assert_eq!(sub(0.0f64.ln(), 0.0f64.ln()), f64::NEG_INFINITY);
        assert_relative_eq!(sub(1.0f64.ln(), 0.5f64.ln()), 0.5f64.ln());
    }

    #[test]
    fn test_ln_1m_exp() {
        assert_eq!(ln_1m_exp(f64::NEG_INFINITY), 0.0);
        assert_eq!(ln_1m_exp(-0.0), f64::NEG_INFINITY);
    }

    #[test]
    fn test_integrate() {
        let density = |_| 0.1f64.ln();
        let prob = integrate(&density, 0.0, 10.0, 5);
        assert_relative_eq!(prob.exp(), 1.0, epsilon=0.0000001);
    }
}
<|endoftext|>


<commit_before>use super::types::Types;

pub struct Column {
    col_type: Types
}

impl Column {
    pub fn new(c_type: Types) -> Self {
        Column { col_type: c_type }
    }

    pub fn size(&self) -> u32 {
        self.col_type.size()
    }
}
<commit_msg>added column and table<commit_after>use super::types::Types;

pub struct Column {
    name: String,
    col_type: Types
}

impl Column {
    pub fn new(name: &str, c_type: Types) -> Self {
        Column {
            name: name.to_string(),
            col_type: c_type
        }
    }

    pub fn size(&self) -> u32 {
        self.col_type.size()
    }

    pub fn get_name(&self) -> String {
        self.name.clone()
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
use std::fmt;

use regex::Regex;

use super::file::header::data::FileHeaderData;

pub struct ParserError {
    summary: String,
    parsertext: String,
    index: i32,
    explanation: Option<String>,
    caused_by: Option<Box<Error>>,
}

impl ParserError {
    pub fn new(sum: &'static str, text: String, idx: i32, expl: &'static str) -> ParserError {
        ParserError {
            summary: String::from(sum),
            parsertext: text,
            index: idx,
            explanation: Some(String::from(expl)),
            caused_by: None,
        }
    }

    pub fn short(sum: &str, text: String, idx: i32) -> ParserError {
        ParserError {
            summary: String::from(sum),
            parsertext: text,
            index: idx,
            explanation: None,
            caused_by: None,
        }
    }

    pub fn with_cause(mut self, e: Box<Error>) -> ParserError {
        self.caused_by = Some(e);
        self
    }

}

impl Error for ParserError {

    fn description(&self) -> &str {
        &self.summary[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Debug for ParserError {

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "ParserError: {}\n\n", self.summary);

        if let Some(ref e) = self.explanation {
            write!(fmt, "{}\n\n", e);
        }

        write!(fmt, "On position {}\nin\n{}", self.index, self.parsertext);
        Ok(())
    }

}

impl Display for ParserError {

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "ParserError: {}", self.summary);

        if let Some(ref e) = self.explanation {
            write!(fmt, "\n\n{}", e);
        }

        Ok(())
    }

}

pub trait FileHeaderParser : Sized + Debug + Display {
    fn read(&self, string: Option<String>) -> Result<FileHeaderData, ParserError>;
    fn write(&self, data: &FileHeaderData) -> Result<String, ParserError>;
}

type TextTpl = (Option<String>, Option<String>);

pub struct Parser<HP>
{
    headerp : HP,
}

impl<HP> Parser<HP> where
    HP: FileHeaderParser,
{

    pub fn new(headerp: HP) -> Parser<HP> {
        Parser {
            headerp: headerp,
        }
    }

    pub fn read(&self, s: String) -> Result<(FileHeaderData, String), ParserError>
    {
        debug!("Reading into internal datastructure: '{}'", s);
        let divided = self.divide_text(&s);

        if divided.is_err() {
            debug!("Error reading into internal datastructure");
            let mut p = ParserError::new("Dividing text failed",
                                         s, 0,
                    "Dividing text with divide_text() failed");
            return Err(p.with_cause(Box::new(divided.err().unwrap())));
        }

        let (header, data) = divided.ok().unwrap();
        debug!("Header = '{:?}'", header);
        debug!("Data   = '{:?}'", data);

        let h_parseres = try!(self.headerp.read(header));
        debug!("Success parsing header");

        Ok((h_parseres, data.unwrap_or(String::new())))
    }

    pub fn write(&self, tpl : (FileHeaderData, String)) -> Result<String, ParserError>
    {
        debug!("Parsing internal datastructure to String");
        let (header, data) = tpl;
        let h_text = try!(self.headerp.write(&header));
        debug!("Success translating header");

        let text = format!("---\n{}\n---\n{}", h_text, data);
        Ok(text)
    }

    fn divide_text(&self, text: &String) -> Result<TextTpl, ParserError> {
        let re = Regex::new(r"(?sm)^---$(.*)^---$(.*)").unwrap();

        debug!("Splitting: '{}'", text);
        debug!("   regex = {:?}", re);

        re.captures(text).map(|captures| {

            if captures.len() != 3 {
                debug!("Unexpected amount of captures");
                return Err(ParserError::new("Unexpected Regex output",
                                            text.clone(), 0,
                                            "The regex to divide text into header and content had an unexpected output."))
            }

            let header  = captures.at(1).map(|s| String::from(s));
            let content = captures.at(2).map(|s| String::from(s));

            debug!("Splitted, Header = '{:?}'", header.clone().unwrap_or("NONE".into()));
            debug!("Splitted, Data   = '{:?}'", content.clone().unwrap_or("NONE".into()));
            Ok((header, content))
        }).or_else(|| {
            debug!("Cannot capture from text");
            let e = ParserError::new("Cannot run regex on text",
                                     text.clone(), 0,
                                     "Cannot run regex on text to divide it into header and content.");
            Some(Err(e))
        }).unwrap()
    }

}

<commit_msg>Impl Debug for Parser<commit_after>use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
use std::fmt;

use regex::Regex;

use super::file::header::data::FileHeaderData;

pub struct ParserError {
    summary: String,
    parsertext: String,
    index: i32,
    explanation: Option<String>,
    caused_by: Option<Box<Error>>,
}

impl ParserError {
    pub fn new(sum: &'static str, text: String, idx: i32, expl: &'static str) -> ParserError {
        ParserError {
            summary: String::from(sum),
            parsertext: text,
            index: idx,
            explanation: Some(String::from(expl)),
            caused_by: None,
        }
    }

    pub fn short(sum: &str, text: String, idx: i32) -> ParserError {
        ParserError {
            summary: String::from(sum),
            parsertext: text,
            index: idx,
            explanation: None,
            caused_by: None,
        }
    }

    pub fn with_cause(mut self, e: Box<Error>) -> ParserError {
        self.caused_by = Some(e);
        self
    }

}

impl Error for ParserError {

    fn description(&self) -> &str {
        &self.summary[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Debug for ParserError {

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "ParserError: {}\n\n", self.summary);

        if let Some(ref e) = self.explanation {
            write!(fmt, "{}\n\n", e);
        }

        write!(fmt, "On position {}\nin\n{}", self.index, self.parsertext);
        Ok(())
    }

}

impl Display for ParserError {

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "ParserError: {}", self.summary);

        if let Some(ref e) = self.explanation {
            write!(fmt, "\n\n{}", e);
        }

        Ok(())
    }

}

pub trait FileHeaderParser : Sized + Debug + Display {
    fn read(&self, string: Option<String>) -> Result<FileHeaderData, ParserError>;
    fn write(&self, data: &FileHeaderData) -> Result<String, ParserError>;
}

type TextTpl = (Option<String>, Option<String>);

pub struct Parser<HP>
{
    headerp : HP,
}

impl<HP> Parser<HP> where
    HP: FileHeaderParser,
{

    pub fn new(headerp: HP) -> Parser<HP> {
        Parser {
            headerp: headerp,
        }
    }

    pub fn read(&self, s: String) -> Result<(FileHeaderData, String), ParserError>
    {
        debug!("Reading into internal datastructure: '{}'", s);
        let divided = self.divide_text(&s);

        if divided.is_err() {
            debug!("Error reading into internal datastructure");
            let mut p = ParserError::new("Dividing text failed",
                                         s, 0,
                    "Dividing text with divide_text() failed");
            return Err(p.with_cause(Box::new(divided.err().unwrap())));
        }

        let (header, data) = divided.ok().unwrap();
        debug!("Header = '{:?}'", header);
        debug!("Data   = '{:?}'", data);

        let h_parseres = try!(self.headerp.read(header));
        debug!("Success parsing header");

        Ok((h_parseres, data.unwrap_or(String::new())))
    }

    pub fn write(&self, tpl : (FileHeaderData, String)) -> Result<String, ParserError>
    {
        debug!("Parsing internal datastructure to String");
        let (header, data) = tpl;
        let h_text = try!(self.headerp.write(&header));
        debug!("Success translating header");

        let text = format!("---\n{}\n---\n{}", h_text, data);
        Ok(text)
    }

    fn divide_text(&self, text: &String) -> Result<TextTpl, ParserError> {
        let re = Regex::new(r"(?sm)^---$(.*)^---$(.*)").unwrap();

        debug!("Splitting: '{}'", text);
        debug!("   regex = {:?}", re);

        re.captures(text).map(|captures| {

            if captures.len() != 3 {
                debug!("Unexpected amount of captures");
                return Err(ParserError::new("Unexpected Regex output",
                                            text.clone(), 0,
                                            "The regex to divide text into header and content had an unexpected output."))
            }

            let header  = captures.at(1).map(|s| String::from(s));
            let content = captures.at(2).map(|s| String::from(s));

            debug!("Splitted, Header = '{:?}'", header.clone().unwrap_or("NONE".into()));
            debug!("Splitted, Data   = '{:?}'", content.clone().unwrap_or("NONE".into()));
            Ok((header, content))
        }).or_else(|| {
            debug!("Cannot capture from text");
            let e = ParserError::new("Cannot run regex on text",
                                     text.clone(), 0,
                                     "Cannot run regex on text to divide it into header and content.");
            Some(Err(e))
        }).unwrap()
    }

}

impl<HP> Debug for Parser<HP>
    where HP: FileHeaderParser
{

    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "Parser<{:?}>", self.headerp);

        Ok(())
    }

}
<|endoftext|>


<commit_before>/// A type for which the layout is determined by its C++ definition.
///
/// This trait serves the following two related purposes.
///
///

///
/// ## Safely unifying occurrences of the same extern type
///
/// `ExternType` makes it possible for CXX to safely share a consistent Rust
/// type across multiple #\[cxx::bridge\] invocations that refer to a common
/// extern C++ type.
///
/// In the following snippet, two #\[cxx::bridge\] invocations in different
/// files (possibly different crates) both contain function signatures involving
/// the same C++ type `example::Demo`. If both were written just containing
/// `type Demo;`, then both macro expansions would produce their own separate
/// Rust type called `Demo` and thus the compiler wouldn't allow us to take the
/// `Demo` returned by `file1::ffi::create_demo` and pass it as the `Demo`
/// argument accepted by `file2::ffi::take_ref_demo`. Instead, one of the two
/// `Demo`s has been defined as an extern type alias of the other, making them
/// the same type in Rust. The CXX code generator will use an automatically
/// generated `ExternType` impl emitted in file1 to statically verify that in
/// file2 `crate::file1::ffi::Demo` really does refer to the C++ type
/// `example::Demo` as expected in file2.
///
/// ```no_run
/// // file1.rs
/// # mod file1 {
/// #[cxx::bridge(namespace = example)]
/// pub mod ffi {
///     extern "C" {
///         type Demo;
///
///         fn create_demo() -> UniquePtr<Demo>;
///     }
/// }
/// # }
///
/// // file2.rs
/// #[cxx::bridge(namespace = example)]
/// pub mod ffi {
///     extern "C" {
///         type Demo = crate::file1::ffi::Demo;
///
///         fn take_ref_demo(demo: &Demo);
///     }
/// }
/// #
/// # fn main() {}
/// ```
///
///

///
/// ## Integrating with bindgen-generated types
///
/// Handwritten `ExternType` impls make it possible to plug in a data structure
/// emitted by bindgen as the definition of a C++ type emitted by CXX.
///
/// By writing the unsafe `ExternType` impl, the programmer asserts that the C++
/// namespace and type name given in the type id refers to a C++ type that is
/// equivalent to Rust type that is the `Self` type of the impl.
///
/// ```no_run
/// # const _: &str = stringify! {
/// mod folly_sys;  // the bindgen-generated bindings
/// # };
/// # mod folly_sys {
/// #     #[repr(transparent)]
/// #     pub struct StringPiece([usize; 2]);
/// # }
///
/// use cxx::{type_id, ExternType};
///
/// unsafe impl ExternType for folly_sys::StringPiece {
///     type Id = type_id!("folly::StringPiece");
///     type Kind = cxx::kind::Opaque;
/// }
///
/// #[cxx::bridge(namespace = folly)]
/// pub mod ffi {
///     extern "C" {
///         include!("rust_cxx_bindings.h");
///
///         type StringPiece = crate::folly_sys::StringPiece;
///
///         fn print_string_piece(s: &StringPiece);
///     }
/// }
///
/// // Now if we construct a StringPiece or obtain one through one
/// // of the bindgen-generated signatures, we are able to pass it
/// // along to ffi::print_string_piece.
/// #
/// # fn main() {}
/// ```
pub unsafe trait ExternType {
    /// A type-level representation of the type's C++ namespace and type name.
    ///
    /// This will always be defined using `type_id!` in the following form:
    ///
    /// ```
    /// # struct TypeName;
    /// # unsafe impl cxx::ExternType for TypeName {
    /// type Id = cxx::type_id!("name::space::of::TypeName");
    /// #     type Kind = cxx::kind::Opaque;
    /// # }
    /// ```
    type Id;

    /// Either [`cxx::kind::Opaque`] or [`cxx::kind::Trivial`].
    ///
    /// [`cxx::kind::Opaque`]: kind::Opaque
    /// [`cxx::kind::Trivial`]: kind::Trivial
    ///
    /// A C++ type is only okay to hold and pass around by value in Rust if its
    /// [move constructor is trivial] and it has no destructor. In CXX, these
    /// are called Trivial extern C++ types, while types with nontrivial move
    /// behavior or a destructor must be considered Opaque and handled by Rust
    /// only behind an indirection, such as a reference or UniquePtr.
    ///
    /// [move constructor is trivial]: https://en.cppreference.com/w/cpp/types/is_move_constructible
    ///
    /// If you believe your C++ type reflected by this ExternType impl is indeed
    /// trivial, you can specify:
    ///
    /// ```
    /// # struct TypeName;
    /// # unsafe impl cxx::ExternType for TypeName {
    /// #     type Id = cxx::type_id!("name::space::of::TypeName");
    /// type Kind = cxx::kind::Trivial;
    /// # }
    /// ```
    ///
    /// which will enable you to pass it into C++ functions by value, return it
    /// by value, and include it in `struct`s that you have declared to
    /// `cxx::bridge`. Your claim about the triviality of the C++ type will be
    /// checked by a `static_assert` in the generated C++ side of the binding.
    type Kind;
}

/// Marker types identifying Rust's knowledge about an extern C++ type.
///
/// These markers are used in the [`Kind`][ExternType::Kind] associated type in
/// impls of the `ExternType` trait. Refer to the documentation of `Kind` for an
/// overview of their purpose.
pub mod kind {
    /// An opaque type which cannot be passed or held by value within Rust.
    ///
    /// Rust's move semantics are such that every move is equivalent to a
    /// memcpy. This is incompatible in general with C++'s constructor-based
    /// move semantics, so a C++ type which has a destructor or nontrivial move
    /// constructor must never exist by value in Rust. In CXX, such types are
    /// called opaque C++ types.
    ///
    /// When passed across an FFI boundary, an opaque C++ type must be behind an
    /// indirection such as a reference or UniquePtr.
    pub struct Opaque;

    /// A type with trivial move constructor and no destructor, which can
    /// therefore be owned and moved around in Rust code without requiring
    /// indirection.
    pub struct Trivial;
}

#[doc(hidden)]
pub fn verify_extern_type<T: ExternType<Id = Id>, Id>() {}

#[doc(hidden)]
pub fn verify_extern_kind<T: ExternType<Kind = Kind>, Kind>() {}
<commit_msg>Make kind::Opaque and kind::Trivial impossible to instantiate<commit_after>/// A type for which the layout is determined by its C++ definition.
///
/// This trait serves the following two related purposes.
///
///

///
/// ## Safely unifying occurrences of the same extern type
///
/// `ExternType` makes it possible for CXX to safely share a consistent Rust
/// type across multiple #\[cxx::bridge\] invocations that refer to a common
/// extern C++ type.
///
/// In the following snippet, two #\[cxx::bridge\] invocations in different
/// files (possibly different crates) both contain function signatures involving
/// the same C++ type `example::Demo`. If both were written just containing
/// `type Demo;`, then both macro expansions would produce their own separate
/// Rust type called `Demo` and thus the compiler wouldn't allow us to take the
/// `Demo` returned by `file1::ffi::create_demo` and pass it as the `Demo`
/// argument accepted by `file2::ffi::take_ref_demo`. Instead, one of the two
/// `Demo`s has been defined as an extern type alias of the other, making them
/// the same type in Rust. The CXX code generator will use an automatically
/// generated `ExternType` impl emitted in file1 to statically verify that in
/// file2 `crate::file1::ffi::Demo` really does refer to the C++ type
/// `example::Demo` as expected in file2.
///
/// ```no_run
/// // file1.rs
/// # mod file1 {
/// #[cxx::bridge(namespace = example)]
/// pub mod ffi {
///     extern "C" {
///         type Demo;
///
///         fn create_demo() -> UniquePtr<Demo>;
///     }
/// }
/// # }
///
/// // file2.rs
/// #[cxx::bridge(namespace = example)]
/// pub mod ffi {
///     extern "C" {
///         type Demo = crate::file1::ffi::Demo;
///
///         fn take_ref_demo(demo: &Demo);
///     }
/// }
/// #
/// # fn main() {}
/// ```
///
///

///
/// ## Integrating with bindgen-generated types
///
/// Handwritten `ExternType` impls make it possible to plug in a data structure
/// emitted by bindgen as the definition of a C++ type emitted by CXX.
///
/// By writing the unsafe `ExternType` impl, the programmer asserts that the C++
/// namespace and type name given in the type id refers to a C++ type that is
/// equivalent to Rust type that is the `Self` type of the impl.
///
/// ```no_run
/// # const _: &str = stringify! {
/// mod folly_sys;  // the bindgen-generated bindings
/// # };
/// # mod folly_sys {
/// #     #[repr(transparent)]
/// #     pub struct StringPiece([usize; 2]);
/// # }
///
/// use cxx::{type_id, ExternType};
///
/// unsafe impl ExternType for folly_sys::StringPiece {
///     type Id = type_id!("folly::StringPiece");
///     type Kind = cxx::kind::Opaque;
/// }
///
/// #[cxx::bridge(namespace = folly)]
/// pub mod ffi {
///     extern "C" {
///         include!("rust_cxx_bindings.h");
///
///         type StringPiece = crate::folly_sys::StringPiece;
///
///         fn print_string_piece(s: &StringPiece);
///     }
/// }
///
/// // Now if we construct a StringPiece or obtain one through one
/// // of the bindgen-generated signatures, we are able to pass it
/// // along to ffi::print_string_piece.
/// #
/// # fn main() {}
/// ```
pub unsafe trait ExternType {
    /// A type-level representation of the type's C++ namespace and type name.
    ///
    /// This will always be defined using `type_id!` in the following form:
    ///
    /// ```
    /// # struct TypeName;
    /// # unsafe impl cxx::ExternType for TypeName {
    /// type Id = cxx::type_id!("name::space::of::TypeName");
    /// #     type Kind = cxx::kind::Opaque;
    /// # }
    /// ```
    type Id;

    /// Either [`cxx::kind::Opaque`] or [`cxx::kind::Trivial`].
    ///
    /// [`cxx::kind::Opaque`]: kind::Opaque
    /// [`cxx::kind::Trivial`]: kind::Trivial
    ///
    /// A C++ type is only okay to hold and pass around by value in Rust if its
    /// [move constructor is trivial] and it has no destructor. In CXX, these
    /// are called Trivial extern C++ types, while types with nontrivial move
    /// behavior or a destructor must be considered Opaque and handled by Rust
    /// only behind an indirection, such as a reference or UniquePtr.
    ///
    /// [move constructor is trivial]: https://en.cppreference.com/w/cpp/types/is_move_constructible
    ///
    /// If you believe your C++ type reflected by this ExternType impl is indeed
    /// trivial, you can specify:
    ///
    /// ```
    /// # struct TypeName;
    /// # unsafe impl cxx::ExternType for TypeName {
    /// #     type Id = cxx::type_id!("name::space::of::TypeName");
    /// type Kind = cxx::kind::Trivial;
    /// # }
    /// ```
    ///
    /// which will enable you to pass it into C++ functions by value, return it
    /// by value, and include it in `struct`s that you have declared to
    /// `cxx::bridge`. Your claim about the triviality of the C++ type will be
    /// checked by a `static_assert` in the generated C++ side of the binding.
    type Kind;
}

/// Marker types identifying Rust's knowledge about an extern C++ type.
///
/// These markers are used in the [`Kind`][ExternType::Kind] associated type in
/// impls of the `ExternType` trait. Refer to the documentation of `Kind` for an
/// overview of their purpose.
pub mod kind {
    /// An opaque type which cannot be passed or held by value within Rust.
    ///
    /// Rust's move semantics are such that every move is equivalent to a
    /// memcpy. This is incompatible in general with C++'s constructor-based
    /// move semantics, so a C++ type which has a destructor or nontrivial move
    /// constructor must never exist by value in Rust. In CXX, such types are
    /// called opaque C++ types.
    ///
    /// When passed across an FFI boundary, an opaque C++ type must be behind an
    /// indirection such as a reference or UniquePtr.
    pub enum Opaque {}

    /// A type with trivial move constructor and no destructor, which can
    /// therefore be owned and moved around in Rust code without requiring
    /// indirection.
    pub enum Trivial {}
}

#[doc(hidden)]
pub fn verify_extern_type<T: ExternType<Id = Id>, Id>() {}

#[doc(hidden)]
pub fn verify_extern_kind<T: ExternType<Kind = Kind>, Kind>() {}
<|endoftext|>


<commit_before>use std::io::{self, Read};
use std::fs::File;
use std::path::Path;

use response::{Response, ResponseWriter};

pub struct FileLoader {
    ///The size, in bytes, of the file chunks. Default is 1024.
    pub chunk_size: usize
}

impl FileLoader {
    pub fn new() -> FileLoader {
        FileLoader {
            chunk_size: 1024
        }
    }

    pub fn send_file<'a, 'b, P: AsRef<Path>>(&self, path: P, response: Response<'a, 'b>) -> Result<(), Error<'a, 'b>> {
        let mut file = match File::open(path) {
            Ok(file) => file,
            Err(e) => return Err(Error::Open(e, response))
        };

        let mut writer = response.into_writer();
        let mut buffer = vec![0; self.chunk_size];
        loop {
            match file.read(&mut buffer) {
                Ok(len) if len == 0 => break,
                Ok(len) => writer.send(&buffer[..len]),
                Err(e) => return Err(Error::Read(e, writer))
            }
        }

        Ok(())
    }
}

pub enum Error<'a, 'b> {
    Open(io::Error, Response<'a, 'b>),
    Read(io::Error, ResponseWriter<'a, 'b>)
}

impl<'a, 'b> Error<'a, 'b> {
    pub fn io_error(&self) -> &io::Error {
        match *self {
            Error::Open(ref e, _) => e,
            Error::Read(ref e, _) => e
        }
    }
}<commit_msg>Increase file chunk size in FileLoader<commit_after>use std::io::{self, Read};
use std::fs::File;
use std::path::Path;

use response::{Response, ResponseWriter};

pub struct FileLoader {
    ///The size, in bytes, of the file chunks. Default is 1048576 (1 megabyte).
    pub chunk_size: usize
}

impl FileLoader {
    pub fn new() -> FileLoader {
        FileLoader {
            chunk_size: 1048576
        }
    }

    pub fn send_file<'a, 'b, P: AsRef<Path>>(&self, path: P, response: Response<'a, 'b>) -> Result<(), Error<'a, 'b>> {
        let mut file = match File::open(path) {
            Ok(file) => file,
            Err(e) => return Err(Error::Open(e, response))
        };

        let mut writer = response.into_writer();
        let mut buffer = vec![0; self.chunk_size];
        loop {
            match file.read(&mut buffer) {
                Ok(len) if len == 0 => break,
                Ok(len) => writer.send(&buffer[..len]),
                Err(e) => return Err(Error::Read(e, writer))
            }
        }

        Ok(())
    }
}

pub enum Error<'a, 'b> {
    Open(io::Error, Response<'a, 'b>),
    Read(io::Error, ResponseWriter<'a, 'b>)
}

impl<'a, 'b> Error<'a, 'b> {
    pub fn io_error(&self) -> &io::Error {
        match *self {
            Error::Open(ref e, _) => e,
            Error::Read(ref e, _) => e
        }
    }
}<|endoftext|>