<commit_before>#![cfg(target_os = "linux")]

pub mod ffi;

use std::ptr;
use std::ffi::CString;

use error::{RuntimeError, ErrorKind};

use super::xcb::Connection;

pub struct Display {
   pub ptr: *mut ffi::Display
}

impl Display {
   pub fn default() -> Result<Self, RuntimeError> {
      let display_ptr = unsafe {
         ffi::XOpenDisplay(ptr::null())
      };

      if display_ptr.is_null() {
         return Err(RuntimeError::new(
            ErrorKind::Xlib,
            "Opening default X display failed".to_string()
         ));
      }

      Ok(
         Display {
            ptr: display_ptr
         }
      )
   }

   pub fn new(name: &str) -> Result<Self, RuntimeError> {
      let c_name = match CString::new(name){
         Ok(c_name) => c_name,
         Err(e) => {
            let description = format!(
               "Opening X display with bad display name '{}'", name
            );
            return Err(RuntimeError::new(
               ErrorKind::Xlib,
               description
            ));
         }
      };

      let display_ptr = unsafe {
         ffi::XOpenDisplay(c_name.as_ptr())
      };

      if display_ptr.is_null() {
         let description = format!(
            "Opening X display '{}' failed", name
         );
         return Err(RuntimeError::new(
            ErrorKind::Xlib,
            description
         ));
      }

      Ok(
         Display {
            ptr: display_ptr
         }
      )
   }

   pub fn xcb_connection(&self) -> Result<Connection, RuntimeError> {
      let connection = unsafe {
         ffi::XGetXCBConnection(self.ptr)
      };

      if connection.is_null() {
         return Err(RuntimeError::new(
            ErrorKind::Xlib,
            "Getting XCB connection failed".to_string()
         ));
      }

      Ok(
         Connection::new(connection)
      )
   }

   pub fn xcb_own_event_queue(&self) {
      unsafe {
         ffi::XSetEventQueueOwner(
            self.ptr,
            ffi::XCBOwnsEventQueue
         )
      };
   }

   #[inline]
   pub fn default_screen(&self) -> ScreenID {
      let screen = unsafe {
         (*(self.ptr as ffi::_XPrivDisplay)).default_screen
      };

      ScreenID {
         screen: screen
      }
   }
}

impl Drop for Display {
   fn drop (&mut self) {
      unsafe {
         ffi::XCloseDisplay(self.ptr);
      }
   }
}

pub struct ScreenID {
   pub screen: ffi::c_int
}

#[cfg(test)]
mod test {
   use std::env;
   use super::Display;

   #[test]
   fn test_create_display() {
      assert!(Display::default().is_ok());
   }

   #[test]
   fn test_create_display_from_env() {
      let name = env::var("DISPLAY").unwrap();
      assert!(Display::new(name.as_ref()).is_ok());
   }

   #[test]
   fn test_create_display_error_str() {
      assert!(Display::new("ErrorStringHere").is_err());
   }

   #[test]
   fn test_create_display_nul_str() {
      assert!(Display::new("one\0two").is_err());
   }

   #[test]
   fn test_create_get_xcb_connection() {
      assert!(Display::default().unwrap().xcb_connection().is_ok());
   }
}

<commit_msg>Fixing unused variable warning<commit_after>#![cfg(target_os = "linux")]

pub mod ffi;

use std::ptr;
use std::ffi::CString;

use error::{RuntimeError, ErrorKind};

use super::xcb::Connection;

pub struct Display {
   pub ptr: *mut ffi::Display
}

impl Display {
   pub fn default() -> Result<Self, RuntimeError> {
      let display_ptr = unsafe {
         ffi::XOpenDisplay(ptr::null())
      };

      if display_ptr.is_null() {
         return Err(RuntimeError::new(
            ErrorKind::Xlib,
            "Opening default X display failed".to_string()
         ));
      }

      Ok(
         Display {
            ptr: display_ptr
         }
      )
   }

   pub fn new(name: &str) -> Result<Self, RuntimeError> {
      let c_name = match CString::new(name){
         Ok(c_name) => c_name,
         Err(_) => {
            let description = format!(
               "Opening X display with bad display name '{}'", name
            );
            return Err(RuntimeError::new(
               ErrorKind::Xlib,
               description
            ));
         }
      };

      let display_ptr = unsafe {
         ffi::XOpenDisplay(c_name.as_ptr())
      };

      if display_ptr.is_null() {
         let description = format!(
            "Opening X display '{}' failed", name
         );
         return Err(RuntimeError::new(
            ErrorKind::Xlib,
            description
         ));
      }

      Ok(
         Display {
            ptr: display_ptr
         }
      )
   }

   pub fn xcb_connection(&self) -> Result<Connection, RuntimeError> {
      let connection = unsafe {
         ffi::XGetXCBConnection(self.ptr)
      };

      if connection.is_null() {
         return Err(RuntimeError::new(
            ErrorKind::Xlib,
            "Getting XCB connection failed".to_string()
         ));
      }

      Ok(
         Connection::new(connection)
      )
   }

   pub fn xcb_own_event_queue(&self) {
      unsafe {
         ffi::XSetEventQueueOwner(
            self.ptr,
            ffi::XCBOwnsEventQueue
         )
      };
   }

   #[inline]
   pub fn default_screen(&self) -> ScreenID {
      let screen = unsafe {
         (*(self.ptr as ffi::_XPrivDisplay)).default_screen
      };

      ScreenID {
         screen: screen
      }
   }
}

impl Drop for Display {
   fn drop (&mut self) {
      unsafe {
         ffi::XCloseDisplay(self.ptr);
      }
   }
}

pub struct ScreenID {
   pub screen: ffi::c_int
}

#[cfg(test)]
mod test {
   use std::env;
   use super::Display;

   #[test]
   fn test_create_display() {
      assert!(Display::default().is_ok());
   }

   #[test]
   fn test_create_display_from_env() {
      let name = env::var("DISPLAY").unwrap();
      assert!(Display::new(name.as_ref()).is_ok());
   }

   #[test]
   fn test_create_display_error_str() {
      assert!(Display::new("ErrorStringHere").is_err());
   }

   #[test]
   fn test_create_display_nul_str() {
      assert!(Display::new("one\0two").is_err());
   }

   #[test]
   fn test_create_get_xcb_connection() {
      assert!(Display::default().unwrap().xcb_connection().is_ok());
   }
}

<|endoftext|>


<commit_before>
use std::collections::HashMap;
use std::io::Error;
use serde::{Serialize, Serializer};
use sysinfo::{NetworkExt, System, SystemExt, Disk, Process, Component, Processor};
use hostname::get_hostname;

/// `sysinfo` with extended features
pub struct SysinfoExt<'a> {
    process_list: HashMap<i32, Process>,
    processor_list: &'a [Processor],
    components_list: &'a [Component],
    disks: &'a [Disk],
    memory: [u64; 6],
    hostname: String,
    uptime: String,
    pub bandwith: (u64, u64),
}

impl<'a> SysinfoExt<'a> {
    pub fn new(system: &'a System) -> Self {
        let network = system.get_network();
        SysinfoExt {
            process_list: system.get_process_list().to_owned(),
            processor_list: system.get_processor_list(),
            components_list: system.get_components_list(),
            disks: system.get_disks(),
            memory: [system.get_total_memory(),
                     system.get_used_memory(),
                     system.get_free_memory(),
                     system.get_total_swap(),
                     system.get_used_swap(),
                     system.get_free_swap()],
            hostname: get_hostname().unwrap_or("Unknown".to_owned()),
            uptime: get_uptime().unwrap_or("Unknown".to_owned()),
            bandwith: (network.get_income(), network.get_outcome()),
        }
    }
}

impl<'a> Serialize for SysinfoExt<'a> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer
    {
        use serde::ser::SerializeMap;
        use sysinfo_serde::Ser;
        let mut map = serializer.serialize_map(None)?;
        map.serialize_entry("processor_list",
                             &self.processor_list
                                 .iter()
                                 .map(|p| Ser::new(p))
                                 .collect::<Vec<Ser<Processor>>>())?;
        map.serialize_entry("process_list", &Ser::new(&self.process_list))?;
        map.serialize_entry("components_list",
                             &self.components_list
                                 .iter()
                                 .map(|c| Ser::new(c))
                                 .collect::<Vec<Ser<Component>>>())?;
        map.serialize_entry("disks",
                             &self.disks
                                 .iter()
                                 .map(|d| Ser::new(d))
                                 .collect::<Vec<Ser<Disk>>>())?;
        map.serialize_entry("memory", &self.memory)?;
        map.serialize_entry("hostname", &self.hostname)?;
        map.serialize_entry("uptime", &self.uptime)?;
        map.serialize_entry("bandwith", &self.bandwith)?;
        map.end()
    }
}

/// Gets uptime from /proc/uptime and converts it to human readable String
fn get_uptime() -> Result<String, Error> {
    use std::process::Command;
    let output = Command::new("uptime").output()?;
    Ok(String::from_utf8(output.stdout).unwrap_or("Unknown".to_owned()))
}
<commit_msg>Remove unnamed kernel threads<commit_after>
use std::collections::HashMap;
use std::io::Error;
use serde::{Serialize, Serializer};
use sysinfo::{NetworkExt, System, SystemExt, Disk, Process, ProcessExt, Component, Processor};
use hostname::get_hostname;

/// `sysinfo` with extended features
pub struct SysinfoExt<'a> {
    process_list: HashMap<i32, Process>,
    processor_list: &'a [Processor],
    components_list: &'a [Component],
    disks: &'a [Disk],
    memory: [u64; 6],
    hostname: String,
    uptime: String,
    pub bandwith: (u64, u64),
}

impl<'a> SysinfoExt<'a> {
    pub fn new(system: &'a System) -> Self {
        let network = system.get_network();
        SysinfoExt {
            process_list: system.get_process_list()
                // filter unnamed kernel threads
                .iter().filter(|p| !p.1.name().is_empty())
                .map(|p| (*p.0, p.1.clone()))
                .collect(),
            processor_list: system.get_processor_list(),
            components_list: system.get_components_list(),
            disks: system.get_disks(),
            memory: [system.get_total_memory(),
                     system.get_used_memory(),
                     system.get_free_memory(),
                     system.get_total_swap(),
                     system.get_used_swap(),
                     system.get_free_swap()],
            hostname: get_hostname().unwrap_or("Unknown".to_owned()),
            uptime: get_uptime().unwrap_or("Unknown".to_owned()),
            bandwith: (network.get_income(), network.get_outcome()),
        }
    }
}

impl<'a> Serialize for SysinfoExt<'a> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer
    {
        use serde::ser::SerializeMap;
        use sysinfo_serde::Ser;
        let mut map = serializer.serialize_map(None)?;
        map.serialize_entry("processor_list",
                             &self.processor_list
                                 .iter()
                                 .map(|p| Ser::new(p))
                                 .collect::<Vec<Ser<Processor>>>())?;
        map.serialize_entry("process_list", &Ser::new(&self.process_list))?;
        map.serialize_entry("components_list",
                             &self.components_list
                                 .iter()
                                 .map(|c| Ser::new(c))
                                 .collect::<Vec<Ser<Component>>>())?;
        map.serialize_entry("disks",
                             &self.disks
                                 .iter()
                                 .map(|d| Ser::new(d))
                                 .collect::<Vec<Ser<Disk>>>())?;
        map.serialize_entry("memory", &self.memory)?;
        map.serialize_entry("hostname", &self.hostname)?;
        map.serialize_entry("uptime", &self.uptime)?;
        map.serialize_entry("bandwith", &self.bandwith)?;
        map.end()
    }
}

/// Gets uptime from /proc/uptime and converts it to human readable String
fn get_uptime() -> Result<String, Error> {
    use std::process::Command;
    let output = Command::new("uptime").output()?;
    Ok(String::from_utf8(output.stdout).unwrap_or("Unknown".to_owned()))
}
<|endoftext|>


<commit_before>use Configuration;
use deque;
use deque::{Worker, Stealer, Stolen};
use job::Job;
use latch::{Latch, LockLatch, SpinLatch};
#[allow(unused_imports)]
use log::Event::*;
use rand;
use std::cell::Cell;
use std::sync::{Arc, Condvar, Mutex, Once, ONCE_INIT};
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
use util::leak;
use num_cpus;

///////////////////////////////////////////////////////////////////////////

pub struct Registry {
    thread_infos: Vec<ThreadInfo>,
    state: Mutex<RegistryState>,
    work_available: Condvar,
    terminate: AtomicBool,
}

struct RegistryState {
    threads_at_work: usize,
    injected_jobs: Vec<*mut Job<LockLatch>>,
}

unsafe impl Send for Registry { }
unsafe impl Sync for Registry { }

///////////////////////////////////////////////////////////////////////////
// Initialization

static mut THE_REGISTRY: Option<&'static Registry> = None;
static THE_REGISTRY_SET: Once = ONCE_INIT;

/// Starts the worker threads (if that has not already happened). If
/// initialization has not already occurred, use the default
/// configuration.
pub fn get_registry() -> &'static Registry {
    THE_REGISTRY_SET.call_once(|| unsafe { init_registry(Configuration::new()) });
    unsafe { THE_REGISTRY.unwrap() }
}

/// Starts the worker threads (if that has not already happened) with
/// the given configuration.
pub fn get_registry_with_config(config: Configuration) -> &'static Registry {
    THE_REGISTRY_SET.call_once(|| unsafe { init_registry(config) });
    unsafe { THE_REGISTRY.unwrap() }
}

/// Initializes the global registry with the given configuration.
/// Meant to be called from within the `THE_REGISTRY_SET` once
/// function. Declared `unsafe` because it writes to `THE_REGISTRY` in
/// an unsynchronized fashion.
unsafe fn init_registry(config: Configuration) {
    let registry = leak(Registry::new(config.num_threads()));
    THE_REGISTRY = Some(registry);
}

impl Registry {
    pub fn new(num_threads: Option<usize>) -> Arc<Registry> {
        let limit_value = match num_threads {
            Some(value) => value,
            None => num_cpus::get()
        };
        let registry = Arc::new(Registry {
            thread_infos: (0..limit_value).map(|_| ThreadInfo::new())
                                          .collect(),
            state: Mutex::new(RegistryState::new()),
            work_available: Condvar::new(),
            terminate: AtomicBool::new(false),
        });

        for index in 0 .. limit_value {
            let registry = registry.clone();
            thread::spawn(move || unsafe { main_loop(registry, index) });
        }

        registry
    }

    pub fn num_threads(&self) -> usize {
        self.thread_infos.len()
    }

    /// Waits for the worker threads to get up and running.  This is
    /// meant to be used for benchmarking purposes, primarily, so that
    /// you can get more consistent numbers by having everything
    /// "ready to go".
    pub fn wait_until_primed(&self) {
        for info in &self.thread_infos {
            info.primed.wait();
        }
    }

    ///////////////////////////////////////////////////////////////////////////
    // MAIN LOOP
    //
    // So long as all of the worker threads are hanging out in their
    // top-level loop, there is no work to be done.

    fn start_working(&self, index: usize) {
        log!(StartWorking { index: index });
        let mut state = self.state.lock().unwrap();
        state.threads_at_work += 1;
        self.work_available.notify_all();
    }

    pub unsafe fn inject(&self, injected_jobs: &[*mut Job<LockLatch>]) {
        log!(InjectJobs { count: injected_jobs.len() });
        let mut state = self.state.lock().unwrap();
        state.injected_jobs.extend(injected_jobs);
        self.work_available.notify_all();
    }

    fn wait_for_work(&self, _worker: usize, was_active: bool) -> Option<*mut Job<LockLatch>> {
        log!(WaitForWork { worker: _worker, was_active: was_active });

        let mut state = self.state.lock().unwrap();

        if was_active {
            state.threads_at_work -= 1;
        }

        loop {
            // Otherwise, if anything was injected from outside,
            // return that.  Note that this gives preference to
            // injected items over stealing from others, which is a
            // bit dubious, but then so is the opposite.
            if let Some(job) = state.injected_jobs.pop() {
                state.threads_at_work += 1;
                self.work_available.notify_all();
                return Some(job);
            }

            // If any of the threads are running a job, we should spin
            // up, since they may generate subworkitems.
            if state.threads_at_work > 0 {
                return None;
            }

            state = self.work_available.wait(state).unwrap();
        }
    }

    pub fn terminate(&self) {
        self.terminate.store(true, Ordering::SeqCst);
    }
}

impl RegistryState {
    pub fn new() -> RegistryState {
        RegistryState {
            threads_at_work: 0,
            injected_jobs: Vec::new(),
        }
    }
}

struct ThreadInfo {
    // latch is set once thread has started and we are entering into
    // the main loop
    primed: LockLatch,
    worker: Worker<JobRef>,
    stealer: Stealer<JobRef>,
}

impl ThreadInfo {
    fn new() -> ThreadInfo {
        let (worker, stealer) = deque::new();
        ThreadInfo {
            primed: LockLatch::new(),
            worker: worker,
            stealer: stealer,
        }
    }
}

///////////////////////////////////////////////////////////////////////////
// WorkerThread identifiers

pub struct WorkerThread {
    registry: Arc<Registry>,
    index: usize
}

// This is a bit sketchy, but basically: the WorkerThread is
// allocated on the stack of the worker on entry and stored into this
// thread local variable. So it will remain valid at least until the
// worker is fully unwound. Using an unsafe pointer avoids the need
// for a RefCell<T> etc.
thread_local! {
    static WORKER_THREAD_STATE: Cell<*const WorkerThread> =
        Cell::new(0 as *const WorkerThread)
}

impl WorkerThread {
    /// Gets the `WorkerThread` index for the current thread; returns
    /// NULL if this is not a worker thread. This pointer is valid
    /// anywhere on the current thread.
    #[inline]
    pub unsafe fn current() -> *const WorkerThread {
        WORKER_THREAD_STATE.with(|t| t.get())
    }

    /// Sets `self` as the worker thread index for the current thread.
    /// This is done during worker thread startup.
    unsafe fn set_current(&self) {
        WORKER_THREAD_STATE.with(|t| {
            assert!(t.get().is_null());
            t.set(self);
        });
    }

    #[inline]
    pub fn index(&self) -> usize {
        self.index
    }

    #[inline]
    fn thread_info(&self) -> &ThreadInfo {
        &self.registry.thread_infos[self.index]
    }

    #[inline]
    pub unsafe fn push(&self, job: *mut Job<SpinLatch>) {
        self.thread_info().worker.push(JobRef { ptr: job });
    }

    /// Pop `job` from top of stack, returning `false` if it has been
    /// stolen.
    #[inline]
    pub unsafe fn pop(&self) -> bool {
        self.thread_info().worker.pop().is_some()
    }

    pub unsafe fn steal_until(&self, latch: &SpinLatch) {
        while !latch.probe() {
            if let Some(job) = steal_work(&self.registry, self.index) {
                (*job).execute();
            } else {
                thread::yield_now();
            }
        }
    }
}

///////////////////////////////////////////////////////////////////////////

unsafe fn main_loop(registry: Arc<Registry>, index: usize) {
    let worker_thread = WorkerThread {
        registry: registry.clone(),
        index: index,
    };
    worker_thread.set_current();

    // let registry know we are ready to do work
    registry.thread_infos[index].primed.set();

    let mut was_active = false;
    while !registry.terminate.load(Ordering::SeqCst) {
        if let Some(injected_job) = registry.wait_for_work(index, was_active) {
            (*injected_job).execute();
            was_active = true;
        } else if let Some(stolen_job) = steal_work(&registry, index) {
            log!(StoleWork { worker: index, job: stolen_job });
            registry.start_working(index);
            (*stolen_job).execute();
            was_active = true;
        } else {
            was_active = false;
        }
    }
}

unsafe fn steal_work(registry: &Registry, index: usize) -> Option<*mut Job<SpinLatch>> {
    let num_threads = registry.num_threads();
    let start = rand::random::<usize>() % num_threads;
    (start .. num_threads)
        .chain(0 .. start)
        .filter(|&i| i != index)
        .flat_map(|i| match registry.thread_infos[i].stealer.steal() {
            Stolen::Empty => None,
            Stolen::Abort => None, // loop?
            Stolen::Data(v) => Some(v.ptr),
        })
        .next()
}

///////////////////////////////////////////////////////////////////////////

pub struct JobRef {
    pub ptr: *mut Job<SpinLatch>
}

unsafe impl Send for JobRef { }
unsafe impl Sync for JobRef { }
<commit_msg>Terminate the thread pool if a worker thread panicks<commit_after>use Configuration;
use deque;
use deque::{Worker, Stealer, Stolen};
use job::Job;
use latch::{Latch, LockLatch, SpinLatch};
#[allow(unused_imports)]
use log::Event::*;
use rand;
use std::cell::Cell;
use std::sync::{Arc, Condvar, Mutex, Once, ONCE_INIT};
use std::thread;
use util::leak;
use num_cpus;

///////////////////////////////////////////////////////////////////////////

pub struct Registry {
    thread_infos: Vec<ThreadInfo>,
    state: Mutex<RegistryState>,
    work_available: Condvar,
}

struct RegistryState {
    terminate: bool,
    threads_at_work: usize,
    injected_jobs: Vec<*mut Job<LockLatch>>,
}

unsafe impl Send for Registry { }
unsafe impl Sync for Registry { }

///////////////////////////////////////////////////////////////////////////
// Initialization

static mut THE_REGISTRY: Option<&'static Registry> = None;
static THE_REGISTRY_SET: Once = ONCE_INIT;

/// Starts the worker threads (if that has not already happened). If
/// initialization has not already occurred, use the default
/// configuration.
pub fn get_registry() -> &'static Registry {
    THE_REGISTRY_SET.call_once(|| unsafe { init_registry(Configuration::new()) });
    unsafe { THE_REGISTRY.unwrap() }
}

/// Starts the worker threads (if that has not already happened) with
/// the given configuration.
pub fn get_registry_with_config(config: Configuration) -> &'static Registry {
    THE_REGISTRY_SET.call_once(|| unsafe { init_registry(config) });
    unsafe { THE_REGISTRY.unwrap() }
}

/// Initializes the global registry with the given configuration.
/// Meant to be called from within the `THE_REGISTRY_SET` once
/// function. Declared `unsafe` because it writes to `THE_REGISTRY` in
/// an unsynchronized fashion.
unsafe fn init_registry(config: Configuration) {
    let registry = leak(Registry::new(config.num_threads()));
    THE_REGISTRY = Some(registry);
}

enum Work {
    None,
    Job(*mut Job<LockLatch>),
    Terminate,
}

impl Registry {
    pub fn new(num_threads: Option<usize>) -> Arc<Registry> {
        let limit_value = match num_threads {
            Some(value) => value,
            None => num_cpus::get()
        };
        let registry = Arc::new(Registry {
            thread_infos: (0..limit_value).map(|_| ThreadInfo::new())
                                          .collect(),
            state: Mutex::new(RegistryState::new()),
            work_available: Condvar::new(),
        });

        for index in 0 .. limit_value {
            let registry = registry.clone();
            thread::spawn(move || unsafe { main_loop(registry, index) });
        }

        registry
    }

    pub fn num_threads(&self) -> usize {
        self.thread_infos.len()
    }

    /// Waits for the worker threads to get up and running.  This is
    /// meant to be used for benchmarking purposes, primarily, so that
    /// you can get more consistent numbers by having everything
    /// "ready to go".
    pub fn wait_until_primed(&self) {
        for info in &self.thread_infos {
            info.primed.wait();
        }
    }

    ///////////////////////////////////////////////////////////////////////////
    // MAIN LOOP
    //
    // So long as all of the worker threads are hanging out in their
    // top-level loop, there is no work to be done.

    fn start_working(&self, index: usize) {
        log!(StartWorking { index: index });
        let mut state = self.state.lock().unwrap();
        state.threads_at_work += 1;
        self.work_available.notify_all();
    }

    pub unsafe fn inject(&self, injected_jobs: &[*mut Job<LockLatch>]) {
        log!(InjectJobs { count: injected_jobs.len() });
        let mut state = self.state.lock().unwrap();
        if state.terminate {
            panic!("attempted to inject a job after thread pool shut down");
        }
        state.injected_jobs.extend(injected_jobs);
        self.work_available.notify_all();
    }

    fn wait_for_work(&self, _worker: usize, was_active: bool) -> Work {
        log!(WaitForWork { worker: _worker, was_active: was_active });

        let mut state = self.state.lock().unwrap();

        if was_active {
            state.threads_at_work -= 1;
        }

        loop {
            // Check if we need to terminate
            if state.terminate {
                return Work::Terminate;
            }

            // Otherwise, if anything was injected from outside,
            // return that.  Note that this gives preference to
            // injected items over stealing from others, which is a
            // bit dubious, but then so is the opposite.
            if let Some(job) = state.injected_jobs.pop() {
                state.threads_at_work += 1;
                self.work_available.notify_all();
                return Work::Job(job);
            }

            // If any of the threads are running a job, we should spin
            // up, since they may generate subworkitems.
            if state.threads_at_work > 0 {
                return Work::None;
            }

            state = self.work_available.wait(state).unwrap();
        }
    }

    pub fn terminate(&self) {
        let mut state = self.state.lock().unwrap();
        state.terminate = true;
        self.work_available.notify_all();
    }
}

impl RegistryState {
    pub fn new() -> RegistryState {
        RegistryState {
            threads_at_work: 0,
            injected_jobs: Vec::new(),
            terminate: false,
        }
    }
}

struct ThreadInfo {
    // latch is set once thread has started and we are entering into
    // the main loop
    primed: LockLatch,
    worker: Worker<JobRef>,
    stealer: Stealer<JobRef>,
}

impl ThreadInfo {
    fn new() -> ThreadInfo {
        let (worker, stealer) = deque::new();
        ThreadInfo {
            primed: LockLatch::new(),
            worker: worker,
            stealer: stealer,
        }
    }
}

///////////////////////////////////////////////////////////////////////////
// WorkerThread identifiers

pub struct WorkerThread {
    registry: Arc<Registry>,
    index: usize
}

// This is a bit sketchy, but basically: the WorkerThread is
// allocated on the stack of the worker on entry and stored into this
// thread local variable. So it will remain valid at least until the
// worker is fully unwound. Using an unsafe pointer avoids the need
// for a RefCell<T> etc.
thread_local! {
    static WORKER_THREAD_STATE: Cell<*const WorkerThread> =
        Cell::new(0 as *const WorkerThread)
}

impl WorkerThread {
    /// Gets the `WorkerThread` index for the current thread; returns
    /// NULL if this is not a worker thread. This pointer is valid
    /// anywhere on the current thread.
    #[inline]
    pub unsafe fn current() -> *const WorkerThread {
        WORKER_THREAD_STATE.with(|t| t.get())
    }

    /// Sets `self` as the worker thread index for the current thread.
    /// This is done during worker thread startup.
    unsafe fn set_current(&self) {
        WORKER_THREAD_STATE.with(|t| {
            assert!(t.get().is_null());
            t.set(self);
        });
    }

    #[inline]
    pub fn index(&self) -> usize {
        self.index
    }

    #[inline]
    fn thread_info(&self) -> &ThreadInfo {
        &self.registry.thread_infos[self.index]
    }

    #[inline]
    pub unsafe fn push(&self, job: *mut Job<SpinLatch>) {
        self.thread_info().worker.push(JobRef { ptr: job });
    }

    /// Pop `job` from top of stack, returning `false` if it has been
    /// stolen.
    #[inline]
    pub unsafe fn pop(&self) -> bool {
        self.thread_info().worker.pop().is_some()
    }

    pub unsafe fn steal_until(&self, latch: &SpinLatch) {
        while !latch.probe() {
            if let Some(job) = steal_work(&self.registry, self.index) {
                (*job).execute();
            } else {
                thread::yield_now();
            }
        }
    }
}

///////////////////////////////////////////////////////////////////////////

#[allow(unused_variables)]
unsafe fn main_loop(registry: Arc<Registry>, index: usize) {
    let worker_thread = WorkerThread {
        registry: registry.clone(),
        index: index,
    };
    worker_thread.set_current();

    // let registry know we are ready to do work
    registry.thread_infos[index].primed.set();

    // If a worker thread panics, bring down the entire thread pool
    struct PanicGuard;
    impl Drop for PanicGuard {
        fn drop(&mut self) {
            unsafe {
                (*WorkerThread::current()).registry.terminate();
            }
        }
    }
    let guard = PanicGuard;

    let mut was_active = false;
    loop {
        match registry.wait_for_work(index, was_active) {
            Work::Job(injected_job) => {
                (*injected_job).execute();
                was_active = true;
                continue;
            }
            Work::Terminate => break,
            Work::None => {},
        }

        if let Some(stolen_job) = steal_work(&registry, index) {
            log!(StoleWork { worker: index, job: stolen_job });
            registry.start_working(index);
            (*stolen_job).execute();
            was_active = true;
        } else {
            was_active = false;
        }
    }
}

unsafe fn steal_work(registry: &Registry, index: usize) -> Option<*mut Job<SpinLatch>> {
    let num_threads = registry.num_threads();
    let start = rand::random::<usize>() % num_threads;
    (start .. num_threads)
        .chain(0 .. start)
        .filter(|&i| i != index)
        .flat_map(|i| match registry.thread_infos[i].stealer.steal() {
            Stolen::Empty => None,
            Stolen::Abort => None, // loop?
            Stolen::Data(v) => Some(v.ptr),
        })
        .next()
}

///////////////////////////////////////////////////////////////////////////

pub struct JobRef {
    pub ptr: *mut Job<SpinLatch>
}

unsafe impl Send for JobRef { }
unsafe impl Sync for JobRef { }
<|endoftext|>


<commit_before>use super::gl;
use std::mem;
use std::raw;

pub trait VertexData: Copy { }

#[allow(non_camel_case_types)]
#[derive(Debug, Clone, Copy)]
pub enum DataType {
    BYTE = gl::BYTE as isize,
    UNSIGNED_BYTE = gl::UNSIGNED_BYTE as isize,
    SHORT = gl::SHORT as isize,
    UNSIGNED_SHORT = gl::UNSIGNED_SHORT as isize,
    FIXED = gl::FIXED as isize,
    FLOAT = gl::FLOAT as isize
}
pub use self::DataType::*;

pub trait VertexBytes {
    fn vertex_bytes(&self) -> &[u8];
}

impl<T> VertexBytes for T where T: VertexData {
    fn vertex_bytes(&self) -> &[u8] {
        unsafe {
            mem::transmute(raw::Slice::<Self> {
                data: self,
                len: mem::size_of::<Self>()
            })
        }
    }
}

#[macro_export]
macro_rules! vertex_data {
    (struct $name:ident {
        $($field_name:ident: $field_type:ty),*
    }) => {
        #[repr(C)]
        #[derive(Debug, Clone, Copy)]
        struct $name {
            $($field_name: $field_type),*
        }

        impl $crate::VertexData for $name { }
    }
}
<commit_msg>Refactor `vertex_data!` macro<commit_after>use super::gl;
use std::mem;
use std::raw;

pub trait VertexData: Copy {
}

#[allow(non_camel_case_types)]
#[derive(Debug, Clone, Copy)]
pub enum DataType {
    BYTE = gl::BYTE as isize,
    UNSIGNED_BYTE = gl::UNSIGNED_BYTE as isize,
    SHORT = gl::SHORT as isize,
    UNSIGNED_SHORT = gl::UNSIGNED_SHORT as isize,
    FIXED = gl::FIXED as isize,
    FLOAT = gl::FLOAT as isize
}
pub use self::DataType::*;

pub trait VertexBytes {
    fn vertex_bytes(&self) -> &[u8];
}

impl<T> VertexBytes for T where T: VertexData {
    fn vertex_bytes(&self) -> &[u8] {
        unsafe {
            mem::transmute(raw::Slice::<Self> {
                data: self,
                len: mem::size_of::<Self>()
            })
        }
    }
}

#[macro_export]
macro_rules! vertex_data {
    (struct $name:ident {
        $($field_name:ident: $field_type:ty),*
    }) => {
        mod _glitter_vertex_data {
            #[allow(non_snake_case)]
            pub mod $name {
                #[repr(C)]
                #[derive(Debug, Clone, Copy)]
                pub struct $name {
                    $($field_name: $field_type),*
                }

                impl $crate::VertexData for $name {

                }
            }
        }

        #[allow(unused_imports)]
        use self::_glitter_vertex_data::$name::$name;
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

use std::io;
use std::collections::VecDeque;
use std::rc::Rc;
use std::cell::RefCell;
use futures::{self, task, Async, Future, Poll};
use futures::sync::oneshot;

use capnp::{Error};

use serialize::{self, AsOutputSegments};

enum State<W, M> where W: io::Write, M: AsOutputSegments {
    Writing(serialize::Write<W, M>, oneshot::Sender<M>),
    BetweenWrites(W),
    Empty,
}

/// A queue of messages being written.
pub struct WriteQueue<W, M> where W: io::Write, M: AsOutputSegments {
    inner: Rc<RefCell<Inner<M>>>,
    state: State<W, M>,
}

struct Inner<M> {
    queue: VecDeque<(M, oneshot::Sender<M>)>,
    sender_count: usize,
    task: Option<task::Task>,

    // If set, then the queue has been requested to end, and we should complete the oneshot once
    // the queue has been emptied.
    end_notifier: Option<oneshot::Sender<()>>
}

/// A handle that allows message to be sent to a `WriteQueue`.
pub struct Sender<M> where M: AsOutputSegments {
    inner: Rc<RefCell<Inner<M>>>,
}

impl <M> Clone for Sender<M> where M: AsOutputSegments {
    fn clone(&self) -> Sender<M> {
        self.inner.borrow_mut().sender_count += 1;
        Sender { inner: self.inner.clone() }
    }
}

impl <M> Drop for Sender<M> where M: AsOutputSegments {
    fn drop(&mut self) {
        self.inner.borrow_mut().sender_count -= 1;
    }
}

pub fn write_queue<W, M>(writer: W) -> (Sender<M>, WriteQueue<W, M>)
    where W: io::Write, M: AsOutputSegments
{
    let inner = Rc::new(RefCell::new(Inner {
        queue: VecDeque::new(),
        task: None,
        sender_count: 1,
        end_notifier: None,
    }));

    let queue = WriteQueue {
        inner: inner.clone(),
        state: State::BetweenWrites(writer),
    };

    let sender = Sender { inner: inner };

    (sender, queue)
}

impl <M> Sender<M> where M: AsOutputSegments {
    /// Enqueues a message to be written.
    pub fn send(&mut self, message: M) -> oneshot::Receiver<M> {
        let (complete, oneshot) = futures::oneshot();

        if self.inner.borrow().end_notifier.is_some() {
            drop(complete)
        } else {
            self.inner.borrow_mut().queue.push_back((message, complete));
        }

        match self.inner.borrow_mut().task.take() {
            Some(t) => t.unpark(),
            None => (),
        }

        oneshot
    }

    /// Returns the number of messages queued to be written, not including any in-progress write.
    pub fn len(&mut self) -> usize {
        self.inner.borrow().queue.len()
    }

    /// Commands the queue to stop writing messages once it is empty. After this method has been called,
    /// any new calls to `send()` will return a future that immediate resolves to an error.
    pub fn end(&mut self) -> oneshot::Receiver<()> {
        let (complete, receiver) = futures::oneshot();

        // TODO: what if end_notifier is already full? Maybe it should be a vector?
        self.inner.borrow_mut().end_notifier = Some(complete);
        receiver
    }
}

enum IntermediateState<W, M> where W: io::Write, M: AsOutputSegments {
    WriteDone(M, W),
    StartWrite(M, oneshot::Sender<M>),
    Resolve,
}

impl <W, M> Future for WriteQueue<W, M> where W: io::Write, M: AsOutputSegments {
    type Item = W; // Resolves when all senders have been dropped and all messages written.
    type Error = Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        loop {
            let next = match self.state {
                State::Writing(ref mut write, ref mut _complete) => {
                    let (w, m) = try_ready!(Future::poll(write));
                    IntermediateState::WriteDone(m, w)
                }
                State::BetweenWrites(ref mut _writer) => {
                    let front = self.inner.borrow_mut().queue.pop_front();
                    match front {
                        Some((m, complete)) => {
                            IntermediateState::StartWrite(m, complete)
                        }
                        None => {
                            let count = self.inner.borrow().sender_count;
                            let ended = self.inner.borrow().end_notifier.is_some();
                            if count == 0 || ended {
                                IntermediateState::Resolve
                            } else {
                                self.inner.borrow_mut().task = Some(task::park());
                                return Ok(Async::NotReady)
                            }
                        }
                    }
                }
                State::Empty => unreachable!(),
            };

            match next {
                IntermediateState::WriteDone(m, w) => {
                    match ::std::mem::replace(&mut self.state, State::BetweenWrites(w)) {
                        State::Writing(_, complete) => {
                            complete.complete(m)
                        }
                        _ => unreachable!(),
                    }
                }
                IntermediateState::StartWrite(m, c) => {
                    let new_state = match ::std::mem::replace(&mut self.state, State::Empty) {
                        State::BetweenWrites(w) => {
                            State::Writing(::serialize::write_message(w, m), c)
                        }
                        _ => unreachable!(),
                    };
                    self.state = new_state;
                }
                IntermediateState::Resolve => {
                    let ender = self.inner.borrow_mut().end_notifier.take();
                    match ender {
                        None => (),
                        Some(complete) => complete.complete(()),
                    }
                    match ::std::mem::replace(&mut self.state, State::Empty) {
                        State::BetweenWrites(w) => {
                            return Ok(Async::Ready(w))
                        }
                        _ => unreachable!(),
                    }
                }
            }
        }
    }
}
<commit_msg>unpark on end()<commit_after>// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

use std::io;
use std::collections::VecDeque;
use std::rc::Rc;
use std::cell::RefCell;
use futures::{self, task, Async, Future, Poll};
use futures::sync::oneshot;

use capnp::{Error};

use serialize::{self, AsOutputSegments};

enum State<W, M> where W: io::Write, M: AsOutputSegments {
    Writing(serialize::Write<W, M>, oneshot::Sender<M>),
    BetweenWrites(W),
    Empty,
}

/// A queue of messages being written.
pub struct WriteQueue<W, M> where W: io::Write, M: AsOutputSegments {
    inner: Rc<RefCell<Inner<M>>>,
    state: State<W, M>,
}

struct Inner<M> {
    queue: VecDeque<(M, oneshot::Sender<M>)>,
    sender_count: usize,
    task: Option<task::Task>,

    // If set, then the queue has been requested to end, and we should complete the oneshot once
    // the queue has been emptied.
    end_notifier: Option<oneshot::Sender<()>>
}

/// A handle that allows message to be sent to a `WriteQueue`.
pub struct Sender<M> where M: AsOutputSegments {
    inner: Rc<RefCell<Inner<M>>>,
}

impl <M> Clone for Sender<M> where M: AsOutputSegments {
    fn clone(&self) -> Sender<M> {
        self.inner.borrow_mut().sender_count += 1;
        Sender { inner: self.inner.clone() }
    }
}

impl <M> Drop for Sender<M> where M: AsOutputSegments {
    fn drop(&mut self) {
        self.inner.borrow_mut().sender_count -= 1;
    }
}

pub fn write_queue<W, M>(writer: W) -> (Sender<M>, WriteQueue<W, M>)
    where W: io::Write, M: AsOutputSegments
{
    let inner = Rc::new(RefCell::new(Inner {
        queue: VecDeque::new(),
        task: None,
        sender_count: 1,
        end_notifier: None,
    }));

    let queue = WriteQueue {
        inner: inner.clone(),
        state: State::BetweenWrites(writer),
    };

    let sender = Sender { inner: inner };

    (sender, queue)
}

impl <M> Sender<M> where M: AsOutputSegments {
    /// Enqueues a message to be written.
    pub fn send(&mut self, message: M) -> oneshot::Receiver<M> {
        let (complete, oneshot) = futures::oneshot();

        if self.inner.borrow().end_notifier.is_some() {
            drop(complete)
        } else {
            self.inner.borrow_mut().queue.push_back((message, complete));
        }

        match self.inner.borrow_mut().task.take() {
            Some(t) => t.unpark(),
            None => (),
        }

        oneshot
    }

    /// Returns the number of messages queued to be written, not including any in-progress write.
    pub fn len(&mut self) -> usize {
        self.inner.borrow().queue.len()
    }

    /// Commands the queue to stop writing messages once it is empty. After this method has been called,
    /// any new calls to `send()` will return a future that immediately resolves to an error.
    pub fn end(&mut self) -> oneshot::Receiver<()> {
        let (complete, receiver) = futures::oneshot();

        // TODO: what if end_notifier is already full? Maybe it should be a vector?
        self.inner.borrow_mut().end_notifier = Some(complete);

        match self.inner.borrow_mut().task.take() {
            Some(t) => t.unpark(),
            None => (),
        }

        receiver
    }
}

enum IntermediateState<W, M> where W: io::Write, M: AsOutputSegments {
    WriteDone(M, W),
    StartWrite(M, oneshot::Sender<M>),
    Resolve,
}

impl <W, M> Future for WriteQueue<W, M> where W: io::Write, M: AsOutputSegments {
    type Item = W; // Resolves when all senders have been dropped and all messages written.
    type Error = Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        loop {
            let next = match self.state {
                State::Writing(ref mut write, ref mut _complete) => {
                    let (w, m) = try_ready!(Future::poll(write));
                    IntermediateState::WriteDone(m, w)
                }
                State::BetweenWrites(ref mut _writer) => {
                    let front = self.inner.borrow_mut().queue.pop_front();
                    match front {
                        Some((m, complete)) => {
                            IntermediateState::StartWrite(m, complete)
                        }
                        None => {
                            let count = self.inner.borrow().sender_count;
                            let ended = self.inner.borrow().end_notifier.is_some();
                            if count == 0 || ended {
                                IntermediateState::Resolve
                            } else {
                                self.inner.borrow_mut().task = Some(task::park());
                                return Ok(Async::NotReady)
                            }
                        }
                    }
                }
                State::Empty => unreachable!(),
            };

            match next {
                IntermediateState::WriteDone(m, w) => {
                    match ::std::mem::replace(&mut self.state, State::BetweenWrites(w)) {
                        State::Writing(_, complete) => {
                            complete.complete(m)
                        }
                        _ => unreachable!(),
                    }
                }
                IntermediateState::StartWrite(m, c) => {
                    let new_state = match ::std::mem::replace(&mut self.state, State::Empty) {
                        State::BetweenWrites(w) => {
                            State::Writing(::serialize::write_message(w, m), c)
                        }
                        _ => unreachable!(),
                    };
                    self.state = new_state;
                }
                IntermediateState::Resolve => {
                    let ender = self.inner.borrow_mut().end_notifier.take();
                    match ender {
                        None => (),
                        Some(complete) => complete.complete(()),
                    }
                    match ::std::mem::replace(&mut self.state, State::Empty) {
                        State::BetweenWrites(w) => {
                            return Ok(Async::Ready(w))
                        }
                        _ => unreachable!(),
                    }
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>//! The codemap
//!
//! # Motivation
//!
//! The parser annotates Nodes with `Span`s that map into the source file.
//! This is handy for error reporting, e.g. mentioning the source line
//! or showing a snippet of the offending code. The codemap allows to
//! get the source location of a Span by keeping track of all newlines.
//!
//! Should RusTiny get support for modules, the codemap would be responsible
//! to resolve the file name, too.
//!
//! # Implementation notes
//!
//! The implementation is currently a bit messey. It's not really clear to me
//! what exactly `Codemap::lines` stores and which offset is 0-based and
//! which one is 1-based. But hey, it works!

// TODO: Let the codemap own the source string

use std::cell::RefCell;
use std::ops::{Add, Sub};

#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub struct BytePos(pub u32);

impl BytePos {
    pub fn as_int(&self) -> u32 {
        let BytePos(i) = *self;
        i
    }
}

impl Add for BytePos {
    type Output = BytePos;

    fn add(self, rhs: BytePos) -> BytePos {
        BytePos((self.as_int() + rhs.as_int()) as u32)
    }
}

impl Add<u32> for BytePos {
    type Output = BytePos;

    fn add(self, rhs: u32) -> BytePos {
        BytePos((self.as_int() + rhs) as u32)
    }
}

impl Sub for BytePos {
    type Output = BytePos;

    fn sub(self, rhs: BytePos) -> BytePos {
        BytePos((self.as_int() - rhs.as_int()) as u32)
    }
}

/// A source location (used for error reporting)
#[derive(Copy)]
pub struct Loc {
    /// The (1-based) line number
    pub line: u32,
    /// The (1-based) column offset
    pub col: u32
}

pub struct Codemap {
    /// Mapping of the line number to the start index
    lines: RefCell<Vec<BytePos>>
}

impl Codemap {
    /// Create a new Codemap instance
    pub fn new() -> Codemap {
        let mut lines = Vec::new();
        lines.push(BytePos(0));

        Codemap { lines: RefCell::new(lines) }
    }

    /// Register the beginning of a new line at a given offset
    pub fn new_line(&self, pos: BytePos) {
        let mut lines = self.lines.borrow_mut();
        let line_count = lines.len();

        assert!(line_count == 0 || (lines[line_count - 1] < pos));
        lines.push(pos)
    }

    /// Get the source location of an offset
    pub fn resolve(&self, mut pos: BytePos) -> Loc {
        let lines = self.lines.borrow();

        debug!("pos: {:?}", pos);
        debug!("lines: {:?}", lines);

        // Binary search for the index at which the offset is <= pos
        let mut lower = 0;
        let mut upper = lines.len();

        while upper - lower > 1 {
            let mid = (lower + upper) / 2;
            let offset = lines[mid];
            if offset > pos {
                // Continue on the left
                upper = mid;
            } else {
                // Continue on the right
                lower = mid;
            }
        }

        let line = lower;
        let offset = lines[line];

        // FIXME: Why is this needed?
        if line == 0 {
            pos = pos + 1;
        }

        debug!("line: {:?}", line);
        debug!("offset: {:?}", offset);

        debug_assert!(pos >= offset,
                      "pos ({:?}) >= offset ({:?})\nlines: {:?}\nline:{:?}",
                      pos, offset, lines, line);

        Loc {
            line: line as u32 + 1,
            col: (pos - offset).as_int()
        }
    }
}<commit_msg>Fix warnings<commit_after>//! The codemap
//!
//! # Motivation
//!
//! The parser annotates Nodes with `Span`s that map into the source file.
//! This is handy for error reporting, e.g. mentioning the source line
//! or showing a snippet of the offending code. The codemap allows to
//! get the source location of a Span by keeping track of all newlines.
//!
//! Should RusTiny get support for modules, the codemap would be responsible
//! to resolve the file name, too.
//!
//! # Implementation notes
//!
//! The implementation is currently a bit messey. It's not really clear to me
//! what exactly `Codemap::lines` stores and which offset is 0-based and
//! which one is 1-based. But hey, it works!

// TODO: Let the codemap own the source string

use std::cell::RefCell;
use std::ops::{Add, Sub};

#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub struct BytePos(pub u32);

impl BytePos {
    pub fn as_int(&self) -> u32 {
        let BytePos(i) = *self;
        i
    }
}

impl Add for BytePos {
    type Output = BytePos;

    fn add(self, rhs: BytePos) -> BytePos {
        BytePos(self.as_int() + rhs.as_int())
    }
}

impl Add<u32> for BytePos {
    type Output = BytePos;

    fn add(self, rhs: u32) -> BytePos {
        BytePos(self.as_int() + rhs)
    }
}

impl Sub for BytePos {
    type Output = BytePos;

    fn sub(self, rhs: BytePos) -> BytePos {
        BytePos(self.as_int() - rhs.as_int())
    }
}

/// A source location (used for error reporting)
#[derive(Copy)]
pub struct Loc {
    /// The (1-based) line number
    pub line: u32,
    /// The (1-based) column offset
    pub col: u32
}

pub struct Codemap {
    /// Mapping of the line number to the start index
    lines: RefCell<Vec<BytePos>>
}

impl Codemap {
    /// Create a new Codemap instance
    pub fn new() -> Codemap {
        let mut lines = Vec::new();
        lines.push(BytePos(0));

        Codemap { lines: RefCell::new(lines) }
    }

    /// Register the beginning of a new line at a given offset
    pub fn new_line(&self, pos: BytePos) {
        let mut lines = self.lines.borrow_mut();
        let line_count = lines.len();

        assert!(line_count == 0 || (lines[line_count - 1] < pos));
        lines.push(pos)
    }

    /// Get the source location of an offset
    pub fn resolve(&self, mut pos: BytePos) -> Loc {
        let lines = self.lines.borrow();

        debug!("pos: {:?}", pos);
        debug!("lines: {:?}", lines);

        // Binary search for the index at which the offset is <= pos
        let mut lower = 0;
        let mut upper = lines.len();

        while upper - lower > 1 {
            let mid = (lower + upper) / 2;
            let offset = lines[mid];
            if offset > pos {
                // Continue on the left
                upper = mid;
            } else {
                // Continue on the right
                lower = mid;
            }
        }

        let line = lower;
        let offset = lines[line];

        // FIXME: Why is this needed?
        if line == 0 {
            pos = pos + 1;
        }

        debug!("line: {:?}", line);
        debug!("offset: {:?}", offset);

        debug_assert!(pos >= offset,
                      "pos ({:?}) >= offset ({:?})\nlines: {:?}\nline:{:?}",
                      pos, offset, lines, line);

        Loc {
            line: line as u32 + 1,
            col: (pos - offset).as_int()
        }
    }
}<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::io;
use std::io::{Read, Write};

use basic::Compression as CodecType;
use errors::{Result, ParquetError};
use snap::{Decoder, Encoder, decompress_len};
use brotli;
use flate2::Compression;
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;

pub trait Codec {
  /// Compress data stored in slice `input_buf` and return a new
  /// vector with the compressed data.
  /// TODO: it's better to pass in vec here (e.g., allow reuse),
  ///   but flate2 api doesn't support this.
  fn compress(&mut self, input_buf: &[u8]) -> Result<Vec<u8>>;

  /// Decompress data stored in slice `input_buf` and write output
  /// to `output_buf`. Return the total number of bytes written.
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize>;
}

pub fn create_codec(codec: CodecType) -> Result<Box<Codec>> {
  match codec {
    CodecType::BROTLI => Ok(Box::new(BrotliCodec::new())),
    CodecType::GZIP => Ok(Box::new(GZipCodec::new())),
    CodecType::SNAPPY => Ok(Box::new(SnappyCodec::new())),
    _ => Err(unsupported_err!("The codec type {} is not supported yet", codec))
  }
}

pub struct SnappyCodec {
  decoder: Decoder,
  encoder: Encoder
}

impl SnappyCodec {
  fn new() -> Self {
    Self { decoder: Decoder::new(), encoder: Encoder::new() }
  }
}

impl Codec for SnappyCodec {
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
    let len = decompress_len(input_buf)?;
    output_buf.resize(len, 0);
    self.decoder.decompress(input_buf, output_buf)
      .map_err(|e| io_err!(io::Error::from(e), "Error when decompressing using Snappy"))
  }

  fn compress(&mut self, input_buf: &[u8]) -> Result<Vec<u8>> {
    self.encoder.compress_vec(input_buf)
      .map_err(|e| io_err!(io::Error::from(e), "Error when compressing using Snappy"))
  }
}

pub struct GZipCodec {
}

impl GZipCodec {
  fn new() -> Self {
    Self { }
  }
}

impl Codec for GZipCodec {
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
    let mut decoder = GzDecoder::new(input_buf)?;
    decoder.read_to_end(output_buf).map_err(|e| io_err!(e, "Error when decompressing using GZip"))
  }

  fn compress(&mut self, input_buf: &[u8]) -> Result<Vec<u8>> {
    let mut encoder = GzEncoder::new(Vec::new(), Compression::Default);
    encoder.write_all(input_buf)?;
    encoder.finish().map_err(|e| io_err!(e, "Error when compressing using GZip"))
  }
}

pub struct BrotliCodec {
}

impl BrotliCodec {
  fn new() -> Self {
    Self { }
  }
}

impl Codec for BrotliCodec {
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize>{
    brotli::Decompressor::new(input_buf, 4096).read(output_buf)
      .map_err(|e| io_err!(io::Error::from(e), "Error when decompressing using Brotli"))
  }

  fn compress(&mut self, _: &[u8]) -> Result<Vec<u8>> {
    // TODO: add this once rust-brotli implements the functionality
    unimplemented!()
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use util::test_common::*;

  fn test_roundtrip(c: CodecType, data: &Vec<u8>) {
    let mut c1 = create_codec(c).unwrap();
    let mut c2 = create_codec(c).unwrap();

    // compress with c1
    let mut decompressed = Vec::new();
    let mut compressed_res = c1.compress(data.as_slice());
    assert!(compressed_res.is_ok());
    let mut compressed = compressed_res.unwrap();

    // decompress with c2
    let mut decompressed_size = c2.decompress(compressed.as_slice(), &mut decompressed);
    assert!(decompressed_size.is_ok());
    decompressed.truncate(decompressed_size.unwrap());
    assert!(*data == decompressed);

    // compress with c2
    compressed_res = c2.compress(data.as_slice());
    assert!(compressed_res.is_ok());
    compressed = compressed_res.unwrap();

    // decompress with c1
    decompressed_size = c1.decompress(compressed.as_slice(), &mut decompressed);
    assert!(decompressed_size.is_ok());
    decompressed.truncate(decompressed_size.unwrap());
    assert!(*data == decompressed);
  }

  fn test_codec(c: CodecType) {
    let sizes = vec![100, 10000, 100000];
    for size in sizes {
      let mut data = Vec::new();
      random_bytes(size, &mut data);
      test_roundtrip(c, &mut data);
    }
  }

  #[test]
  fn test_codec_snappy() {
    test_codec(CodecType::SNAPPY);
  }

  #[test]
  fn test_codec_gzip() {
    test_codec(CodecType::GZIP);
  }
}
<commit_msg>create_codec() should handle UNCOMPRESSED type<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::io;
use std::io::{Read, Write};

use basic::Compression as CodecType;
use errors::{Result, ParquetError};
use snap::{Decoder, Encoder, decompress_len};
use brotli;
use flate2::Compression;
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;

pub trait Codec {
  /// Compress data stored in slice `input_buf` and return a new
  /// vector with the compressed data.
  /// TODO: it's better to pass in vec here (e.g., allow reuse),
  ///   but flate2 api doesn't support this.
  fn compress(&mut self, input_buf: &[u8]) -> Result<Vec<u8>>;

  /// Decompress data stored in slice `input_buf` and write output
  /// to `output_buf`. Return the total number of bytes written.
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize>;
}

/// Given the compression type `codec`, return a codec used to
/// compress & decompress bytes for the compression type.
/// This returns `None` if the codec type is `UNCOMPRESSED`.
pub fn create_codec(codec: CodecType) -> Result<Option<Box<Codec>>> {
  match codec {
    CodecType::BROTLI => Ok(Some(Box::new(BrotliCodec::new()))),
    CodecType::GZIP => Ok(Some(Box::new(GZipCodec::new()))),
    CodecType::SNAPPY => Ok(Some(Box::new(SnappyCodec::new()))),
    CodecType::UNCOMPRESSED => Ok(None),
    _ => Err(unsupported_err!("The codec type {} is not supported yet", codec))
  }
}

pub struct SnappyCodec {
  decoder: Decoder,
  encoder: Encoder
}

impl SnappyCodec {
  fn new() -> Self {
    Self { decoder: Decoder::new(), encoder: Encoder::new() }
  }
}

impl Codec for SnappyCodec {
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
    let len = decompress_len(input_buf)?;
    output_buf.resize(len, 0);
    self.decoder.decompress(input_buf, output_buf)
      .map_err(|e| io_err!(io::Error::from(e), "Error when decompressing using Snappy"))
  }

  fn compress(&mut self, input_buf: &[u8]) -> Result<Vec<u8>> {
    self.encoder.compress_vec(input_buf)
      .map_err(|e| io_err!(io::Error::from(e), "Error when compressing using Snappy"))
  }
}

pub struct GZipCodec {
}

impl GZipCodec {
  fn new() -> Self {
    Self { }
  }
}

impl Codec for GZipCodec {
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize> {
    let mut decoder = GzDecoder::new(input_buf)?;
    decoder.read_to_end(output_buf).map_err(|e| io_err!(e, "Error when decompressing using GZip"))
  }

  fn compress(&mut self, input_buf: &[u8]) -> Result<Vec<u8>> {
    let mut encoder = GzEncoder::new(Vec::new(), Compression::Default);
    encoder.write_all(input_buf)?;
    encoder.finish().map_err(|e| io_err!(e, "Error when compressing using GZip"))
  }
}

pub struct BrotliCodec {
}

impl BrotliCodec {
  fn new() -> Self {
    Self { }
  }
}

impl Codec for BrotliCodec {
  fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<usize>{
    brotli::Decompressor::new(input_buf, 4096).read(output_buf)
      .map_err(|e| io_err!(io::Error::from(e), "Error when decompressing using Brotli"))
  }

  fn compress(&mut self, _: &[u8]) -> Result<Vec<u8>> {
    // TODO: add this once rust-brotli implements the functionality
    unimplemented!()
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use util::test_common::*;

  fn test_roundtrip(c: CodecType, data: &Vec<u8>) {
    let mut c1 = create_codec(c).unwrap().unwrap();
    let mut c2 = create_codec(c).unwrap().unwrap();

    // compress with c1
    let mut decompressed = Vec::new();
    let mut compressed_res = c1.compress(data.as_slice());
    assert!(compressed_res.is_ok());
    let mut compressed = compressed_res.unwrap();

    // decompress with c2
    let mut decompressed_size = c2.decompress(compressed.as_slice(), &mut decompressed);
    assert!(decompressed_size.is_ok());
    decompressed.truncate(decompressed_size.unwrap());
    assert!(*data == decompressed);

    // compress with c2
    compressed_res = c2.compress(data.as_slice());
    assert!(compressed_res.is_ok());
    compressed = compressed_res.unwrap();

    // decompress with c1
    decompressed_size = c1.decompress(compressed.as_slice(), &mut decompressed);
    assert!(decompressed_size.is_ok());
    decompressed.truncate(decompressed_size.unwrap());
    assert!(*data == decompressed);
  }

  fn test_codec(c: CodecType) {
    let sizes = vec![100, 10000, 100000];
    for size in sizes {
      let mut data = Vec::new();
      random_bytes(size, &mut data);
      test_roundtrip(c, &mut data);
    }
  }

  #[test]
  fn test_codec_snappy() {
    test_codec(CodecType::SNAPPY);
  }

  #[test]
  fn test_codec_gzip() {
    test_codec(CodecType::GZIP);
  }
}
<|endoftext|>


<commit_before>//
//use std::fs::File;
//use std::io::prelude::*;

use core::bufferlog::BufferLog;

use core::mapped_file::MappedFile;
//use core::mapped_file::MappedFileIterator;
use core::mapped_file::FileHandle;

//
pub type Id = u64;
pub type Offset = u64;
pub type PageSize = usize;

//
#[derive(Debug)]
pub enum OpenMode {
    ReadOnly = 0,
    ReadWrite = 1,
}

/// The **Buffer** represent a linear array of bytes.

/// it can be in memory only or backed by an on disk file.

/// The editor **Modes** use this api to read/modify the content
/// of the file at the byte level
#[derive(Debug)]
pub struct Buffer<'a> {
    pub id: Id,
    pub file_name: String,
    pub size: usize,
    pub nr_changes: u64, // number of changes since last save
    mode: OpenMode,
    pub data: FileHandle<'a>,
    pub buffer_log: BufferLog,
    // phantom: PhantomData<&'a u8>,
}

impl<'a> Buffer<'a> {
    /// Creates a new `Buffer`.
    ///
    /// file_name param[in] path to the file we want to load in the buffer,
    /// use "/dev/null" to create empty buffer
    /// this function allocate a buffer
    /// if file_name is null the content will be stored in heap
    /// if file_name is non null the the content will be read from the file
    /// if document_name is null , file_name will be used to give a name to the buffer
    /// mode = 0 : read only , mode 1 : read_write
    /// the allocated_bid pointer will be filled on successfull open operation
    pub fn new(file_name: String, mode: OpenMode) -> Option<Buffer<'a>> {
        // TODO: check permission
        // TODO: check file's type => ignore directory (for now)
        println!("-- mapping file {}", file_name);

        let page_size = 4096 * 256 * 2;
        let file = match MappedFile::new(file_name.clone(), page_size) {
            Some(file) => file,
            None => {
                eprintln!("cannot map file '{}'", file_name);
                return None;
            }
        };

        let size = file.as_ref().borrow().size() as usize;

        println!("'{}' opened size '{}'", file_name, size);

        Some(Buffer {
            id: 0,
            file_name: file_name.clone(),
            mode,
            size,
            nr_changes: 0,
            data: file,
            buffer_log: BufferLog::new(),
        })
    }

    /// close a previously opened buffer see buffer_open
    pub fn close(&mut self) -> bool {
        false
    }

    /// returns the name of the file associated to the buffer
    pub fn get_file_name(&self) -> String {
        self.file_name.clone()
    }

    /// change the on disk target file
    pub fn set_file_name(&mut self, name: String) -> bool {
        self.file_name = name;
        true
    }

    /// returns the number of bytes a given buffer contains
    pub fn size(&self) -> usize {
        self.size
    }

    /// returns the number of changes sine the last save

    ///     0  => the no change since last save

    ///     >0 => the number of changes since last save

    pub fn nr_changes(&self) -> u64 {
        self.nr_changes
    }

    /// copy the content of the buffer up to 'nr_bytes' into the data Vec
    /// the read bytes are appended to the data Vec
    /// return XXX on error (use ioresult)
    pub fn read(&self, offset: u64, nr_bytes: usize, mut data: &mut Vec<u8>) -> usize {
        let mut it = MappedFile::iter_from(&self.data, offset);
        MappedFile::read(&mut it, nr_bytes, &mut data)
    }

    /// insert the 'data' Vec content in the buffer up to 'nr_bytes'
    /// return the number of written bytes (TODO: use io::Result)
    pub fn insert(&mut self, offset: u64, nr_bytes: usize, data: &[u8]) -> usize {
        let mut it = MappedFile::iter_from(&self.data, offset);
        MappedFile::insert(&mut it, &data);

        self.size += nr_bytes;
        self.nr_changes += 1;

        nr_bytes
    }

    /// remove up to 'nr_bytes' from the buffer starting at offset
    /// if removed_data is provided will call self.read(offset, nr_bytes, data)
    /// before remove the bytes
    pub fn remove(
        &mut self,
        offset: u64,
        nr_bytes: usize,
        removed_data: Option<&mut Vec<u8>>,
    ) -> usize {
        let start_offset = ::std::cmp::min(offset as usize, self.size);
        let end_offset = ::std::cmp::min(start_offset + nr_bytes, self.size);
        let nr_bytes_removed = (end_offset - start_offset) as usize;

        // copy removed data
        if let Some(v) = removed_data {
            self.read(offset, nr_bytes_removed, v);
        }

        let mut it = MappedFile::iter_from(&self.data, start_offset as u64);
        MappedFile::remove(&mut it, nr_bytes_removed);

        self.size -= nr_bytes_removed;
        self.nr_changes += 1;

        nr_bytes_removed
    }

    /// can be used to know the number of blocks that compose the buffer,
    /// api to be used by indexer etc...
    pub fn nr_pages(&self) -> u64 {
        1
    }

    /*
    /// returns the position and size of a given page
    pub fn get_page_info(&self, page_index: u64) -> (Offset, PageSize) {
        // if page_index > 0 {
        //    (0, self.size)
        // } else {
        //    (0, self.size)
        // }
        (0, self.size)
    }
*/

    pub fn sync_to_disk(&self, _tmp_file_name: &str) -> ::std::io::Result<()> {
        panic!("TODO: do not save now, the editor is broken");

        // use std::fs;
        // use std::fs::File;
        // use std::io::prelude::*;

        // let mut f = File::create(&tmp_file_name)?;

        // f.write_all(&self.data)?;

        // f.sync_all()?;
        // fs::rename(&tmp_file_name, &self.file_name)
    }
}

#[test]
fn test_buffer() {
    let mut bb = Buffer::new("/dev/null".to_owned(), OpenMode::ReadWrite).unwrap();

    let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];

    bb.insert(0, 10, &data);
    let mut rdata = Vec::new();
    let nread = bb.read(0, bb.size(), &mut rdata);
    assert_eq!(rdata, data);
    assert_eq!(nread, bb.size());

    let data = vec![0, 1, 2, 6, 7, 8, 9];
    let rm_expect = vec![3, 4, 5];

    let mut rm = vec![];
    let n = bb.remove(3, 3, Some(&mut rm));
    assert_eq!(n, 3);
    assert_eq!(rm, rm_expect);

    rdata.clear();
    let nread = bb.read(0, bb.size(), &mut rdata);
    assert_eq!(rdata, data);

    println!("rm {:?}", rm);
    println!("rm_expect {:?}", rm_expect);
}
<commit_msg>(re)enable simple save to disk, no checks done (permission, available space, etc)<commit_after>//
//use std::fs::File;
//use std::io::prelude::*;

use core::bufferlog::BufferLog;

use core::mapped_file::MappedFile;
//use core::mapped_file::MappedFileIterator;
use core::mapped_file::FileHandle;

//
pub type Id = u64;
pub type Offset = u64;
pub type PageSize = usize;

//
#[derive(Debug)]
pub enum OpenMode {
    ReadOnly = 0,
    ReadWrite = 1,
}

/// The **Buffer** represent a linear array of bytes.

/// it can be in memory only or backed by an on disk file.

/// The editor **Modes** use this api to read/modify the content
/// of the file at the byte level
#[derive(Debug)]
pub struct Buffer<'a> {
    pub id: Id,
    pub file_name: String,
    pub size: usize,
    pub nr_changes: u64, // number of changes since last save
    mode: OpenMode,
    pub data: FileHandle<'a>,
    pub buffer_log: BufferLog,
    // phantom: PhantomData<&'a u8>,
}

impl<'a> Buffer<'a> {
    /// Creates a new `Buffer`.
    ///
    /// file_name param[in] path to the file we want to load in the buffer,
    /// use "/dev/null" to create empty buffer
    /// this function allocate a buffer
    /// if file_name is null the content will be stored in heap
    /// if file_name is non null the the content will be read from the file
    /// if document_name is null , file_name will be used to give a name to the buffer
    /// mode = 0 : read only , mode 1 : read_write
    /// the allocated_bid pointer will be filled on successfull open operation
    pub fn new(file_name: String, mode: OpenMode) -> Option<Buffer<'a>> {
        // TODO: check permission
        // TODO: check file's type => ignore directory (for now)
        println!("-- mapping file {}", file_name);

        let page_size = 4096 * 256 * 2;
        let file = match MappedFile::new(file_name.clone(), page_size) {
            Some(file) => file,
            None => {
                eprintln!("cannot map file '{}'", file_name);
                return None;
            }
        };

        let size = file.as_ref().borrow().size() as usize;

        println!("'{}' opened size '{}'", file_name, size);

        Some(Buffer {
            id: 0,
            file_name: file_name.clone(),
            mode,
            size,
            nr_changes: 0,
            data: file,
            buffer_log: BufferLog::new(),
        })
    }

    /// close a previously opened buffer see buffer_open
    pub fn close(&mut self) -> bool {
        false
    }

    /// returns the name of the file associated to the buffer
    pub fn get_file_name(&self) -> String {
        self.file_name.clone()
    }

    /// change the on disk target file
    pub fn set_file_name(&mut self, name: String) -> bool {
        self.file_name = name;
        true
    }

    /// returns the number of bytes a given buffer contains
    pub fn size(&self) -> usize {
        self.size
    }

    /// returns the number of changes sine the last save

    ///     0  => the no change since last save

    ///     >0 => the number of changes since last save

    pub fn nr_changes(&self) -> u64 {
        self.nr_changes
    }

    /// copy the content of the buffer up to 'nr_bytes' into the data Vec
    /// the read bytes are appended to the data Vec
    /// return XXX on error (use ioresult)
    pub fn read(&self, offset: u64, nr_bytes: usize, mut data: &mut Vec<u8>) -> usize {
        let mut it = MappedFile::iter_from(&self.data, offset);
        MappedFile::read(&mut it, nr_bytes, &mut data)
    }

    /// insert the 'data' Vec content in the buffer up to 'nr_bytes'
    /// return the number of written bytes (TODO: use io::Result)
    pub fn insert(&mut self, offset: u64, nr_bytes: usize, data: &[u8]) -> usize {
        let mut it = MappedFile::iter_from(&self.data, offset);
        MappedFile::insert(&mut it, &data);

        self.size += nr_bytes;
        self.nr_changes += 1;

        nr_bytes
    }

    /// remove up to 'nr_bytes' from the buffer starting at offset
    /// if removed_data is provided will call self.read(offset, nr_bytes, data)
    /// before remove the bytes
    pub fn remove(
        &mut self,
        offset: u64,
        nr_bytes: usize,
        removed_data: Option<&mut Vec<u8>>,
    ) -> usize {
        let start_offset = ::std::cmp::min(offset as usize, self.size);
        let end_offset = ::std::cmp::min(start_offset + nr_bytes, self.size);
        let nr_bytes_removed = (end_offset - start_offset) as usize;

        // copy removed data
        if let Some(v) = removed_data {
            self.read(offset, nr_bytes_removed, v);
        }

        let mut it = MappedFile::iter_from(&self.data, start_offset as u64);
        MappedFile::remove(&mut it, nr_bytes_removed);

        self.size -= nr_bytes_removed;
        self.nr_changes += 1;

        nr_bytes_removed
    }

    /// can be used to know the number of blocks that compose the buffer,
    /// api to be used by indexer etc...
    pub fn nr_pages(&self) -> u64 {
        1
    }

    /*
    /// returns the position and size of a given page
    pub fn get_page_info(&self, page_index: u64) -> (Offset, PageSize) {
        // if page_index > 0 {
        //    (0, self.size)
        // } else {
        //    (0, self.size)
        // }
        (0, self.size)
    }
*/

    pub fn sync_to_disk(&self, tmp_file_name: &str) -> ::std::io::Result<()> {

        use std::fs;
        use std::fs::File;
        // use std::io::prelude::*;
        use std::io::Write;

        let mut f = File::create(&tmp_file_name)?;

        let mut offset : u64 = 0;
        while offset != self.size as u64 {

            let nr_bytes= 4096*32;
            let mut data = Vec::with_capacity(nr_bytes);
            let read_size = self.read(offset, nr_bytes, &mut data);
            f.write(&data)?;
            offset += read_size as u64;
        }
        f.sync_all()?;
        fs::rename(&tmp_file_name, &self.file_name);
        Ok(())
    }
}

#[test]
fn test_buffer() {
    let mut bb = Buffer::new("/dev/null".to_owned(), OpenMode::ReadWrite).unwrap();

    let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];

    bb.insert(0, 10, &data);
    let mut rdata = Vec::new();
    let nread = bb.read(0, bb.size(), &mut rdata);
    assert_eq!(rdata, data);
    assert_eq!(nread, bb.size());

    let data = vec![0, 1, 2, 6, 7, 8, 9];
    let rm_expect = vec![3, 4, 5];

    let mut rm = vec![];
    let n = bb.remove(3, 3, Some(&mut rm));
    assert_eq!(n, 3);
    assert_eq!(rm, rm_expect);

    rdata.clear();
    let nread = bb.read(0, bb.size(), &mut rdata);
    assert_eq!(rdata, data);

    println!("rm {:?}", rm);
    println!("rm_expect {:?}", rm_expect);
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

mod byvalue_checker;

use std::collections::HashSet;

use autocxx_parser::TypeConfig;
use byvalue_checker::ByValueChecker;
use syn::{Item, ItemStruct};

use crate::{
    conversion::{
        api::{Api, ApiAnalysis, ApiDetail, TypeKind, UnanalyzedApi},
        codegen_rs::make_non_pod,
        parse::type_converter::TypeConverter,
        ConvertError,
    },
    types::{Namespace, TypeName},
};

pub(crate) struct PodAnalysis;

impl ApiAnalysis for PodAnalysis {
    type TypeAnalysis = TypeKind;
    type FunAnalysis = ();
}

/// In our set of APIs, work out which ones are safe to represent
/// by value in Rust (e.g. they don't have a destructor) and record
/// as such. Return a set of APIs annotated with extra metadata,
/// and an object which can be used to query the POD status of any
/// type whether or not it's one of the [Api]s.
pub(crate) fn analyze_pod_apis(
    apis: Vec<UnanalyzedApi>,
    type_config: &TypeConfig,
    type_converter: &mut TypeConverter,
) -> Result<Vec<Api<PodAnalysis>>, ConvertError> {
    let byvalue_checker = ByValueChecker::new_from_apis(&apis, type_config)?;
    let mut extra_apis = Vec::new();
    let mut results: Vec<_> = apis
        .into_iter()
        .map(|api| analyze_pod_api(api, &byvalue_checker, type_converter, &mut extra_apis))
        .collect::<Result<Vec<_>, ConvertError>>()?;
    // Conceivably, the process of POD-analysing the first set of APIs could result
    // in us creating new APIs to concretize generic types.
    let mut more_extra_apis = Vec::new();
    let mut more_results = extra_apis
        .into_iter()
        .map(|api| analyze_pod_api(api, &byvalue_checker, type_converter, &mut more_extra_apis))
        .collect::<Result<Vec<_>, ConvertError>>()?;
    assert!(more_extra_apis.is_empty());
    results.append(&mut more_results);
    Ok(results)
}

fn analyze_pod_api(
    api: UnanalyzedApi,
    byvalue_checker: &ByValueChecker,
    type_converter: &mut TypeConverter,
    extra_apis: &mut Vec<UnanalyzedApi>,
) -> Result<Api<PodAnalysis>, ConvertError> {
    let ty_id = api.typename();
    let mut new_deps = api.deps;
    let api_detail = match api.detail {
        // No changes to any of these...
        ApiDetail::ConcreteType {
            ty_details,
            additional_cpp,
        } => ApiDetail::ConcreteType {
            ty_details,
            additional_cpp,
        },
        ApiDetail::StringConstructor => ApiDetail::StringConstructor,
        ApiDetail::Function { fun, analysis } => ApiDetail::Function { fun, analysis },
        ApiDetail::Const { const_item } => ApiDetail::Const { const_item },
        ApiDetail::Typedef { type_item } => ApiDetail::Typedef { type_item },
        ApiDetail::CType { typename } => ApiDetail::CType { typename },
        // Just changes to this one...
        ApiDetail::Type {
            ty_details,
            for_extern_c_ts,
            is_forward_declaration,
            mut bindgen_mod_item,
            analysis: _,
        } => {
            let type_kind = if is_forward_declaration {
                TypeKind::ForwardDeclaration
            } else if byvalue_checker.is_pod(&ty_id) {
                // It's POD so let's mark dependencies on things in its field
                if let Some(Item::Struct(ref s)) = bindgen_mod_item {
                    get_struct_field_types(type_converter, &api.ns, &s, &mut new_deps, extra_apis)?;
                } // otherwise might be an enum, etc.
                TypeKind::Pod
            } else {
                // It's non-POD. So also, make the fields opaque...
                if let Some(Item::Struct(ref mut s)) = bindgen_mod_item {
                    make_non_pod(s);
                } // otherwise might be an enum, etc.
                  // ... and say we don't depend on other types.
                new_deps.clear();
                TypeKind::NonPod
            };
            ApiDetail::Type {
                ty_details,
                for_extern_c_ts,
                is_forward_declaration,
                bindgen_mod_item,
                analysis: type_kind,
            }
        }
        ApiDetail::OpaqueTypedef => ApiDetail::OpaqueTypedef,
    };
    Ok(Api {
        ns: api.ns,
        id: api.id,
        deps: new_deps,
        detail: api_detail,
    })
}

fn get_struct_field_types(
    type_converter: &mut TypeConverter,
    ns: &Namespace,
    s: &ItemStruct,
    deps: &mut HashSet<TypeName>,
    extra_apis: &mut Vec<UnanalyzedApi>,
) -> Result<(), ConvertError> {
    for f in &s.fields {
        let annotated = type_converter.convert_type(f.ty.clone(), ns, false)?;
        extra_apis.extend(annotated.extra_apis);
        deps.extend(annotated.types_encountered);
    }
    Ok(())
}
<commit_msg>Add comment.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

mod byvalue_checker;

use std::collections::HashSet;

use autocxx_parser::TypeConfig;
use byvalue_checker::ByValueChecker;
use syn::{Item, ItemStruct};

use crate::{
    conversion::{
        api::{Api, ApiAnalysis, ApiDetail, TypeKind, UnanalyzedApi},
        codegen_rs::make_non_pod,
        parse::type_converter::TypeConverter,
        ConvertError,
    },
    types::{Namespace, TypeName},
};

pub(crate) struct PodAnalysis;

impl ApiAnalysis for PodAnalysis {
    type TypeAnalysis = TypeKind;
    type FunAnalysis = ();
}

/// In our set of APIs, work out which ones are safe to represent
/// by value in Rust (e.g. they don't have a destructor) and record
/// as such. Return a set of APIs annotated with extra metadata,
/// and an object which can be used to query the POD status of any
/// type whether or not it's one of the [Api]s.
pub(crate) fn analyze_pod_apis(
    apis: Vec<UnanalyzedApi>,
    type_config: &TypeConfig,
    type_converter: &mut TypeConverter,
) -> Result<Vec<Api<PodAnalysis>>, ConvertError> {
    // This next line will return an error if any of the 'generate_pod'
    // directives from the user can't be met because, for instance,
    // a type contains a std::string or some other type which can't be
    // held safely by value in Rust.
    let byvalue_checker = ByValueChecker::new_from_apis(&apis, type_config)?;
    let mut extra_apis = Vec::new();
    let mut results: Vec<_> = apis
        .into_iter()
        .map(|api| analyze_pod_api(api, &byvalue_checker, type_converter, &mut extra_apis))
        .collect::<Result<Vec<_>, ConvertError>>()?;
    // Conceivably, the process of POD-analysing the first set of APIs could result
    // in us creating new APIs to concretize generic types.
    let mut more_extra_apis = Vec::new();
    let mut more_results = extra_apis
        .into_iter()
        .map(|api| analyze_pod_api(api, &byvalue_checker, type_converter, &mut more_extra_apis))
        .collect::<Result<Vec<_>, ConvertError>>()?;
    assert!(more_extra_apis.is_empty());
    results.append(&mut more_results);
    Ok(results)
}

fn analyze_pod_api(
    api: UnanalyzedApi,
    byvalue_checker: &ByValueChecker,
    type_converter: &mut TypeConverter,
    extra_apis: &mut Vec<UnanalyzedApi>,
) -> Result<Api<PodAnalysis>, ConvertError> {
    let ty_id = api.typename();
    let mut new_deps = api.deps;
    let api_detail = match api.detail {
        // No changes to any of these...
        ApiDetail::ConcreteType {
            ty_details,
            additional_cpp,
        } => ApiDetail::ConcreteType {
            ty_details,
            additional_cpp,
        },
        ApiDetail::StringConstructor => ApiDetail::StringConstructor,
        ApiDetail::Function { fun, analysis } => ApiDetail::Function { fun, analysis },
        ApiDetail::Const { const_item } => ApiDetail::Const { const_item },
        ApiDetail::Typedef { type_item } => ApiDetail::Typedef { type_item },
        ApiDetail::CType { typename } => ApiDetail::CType { typename },
        // Just changes to this one...
        ApiDetail::Type {
            ty_details,
            for_extern_c_ts,
            is_forward_declaration,
            mut bindgen_mod_item,
            analysis: _,
        } => {
            let type_kind = if is_forward_declaration {
                TypeKind::ForwardDeclaration
            } else if byvalue_checker.is_pod(&ty_id) {
                // It's POD so let's mark dependencies on things in its field
                if let Some(Item::Struct(ref s)) = bindgen_mod_item {
                    get_struct_field_types(type_converter, &api.ns, &s, &mut new_deps, extra_apis)?;
                } // otherwise might be an enum, etc.
                TypeKind::Pod
            } else {
                // It's non-POD. So also, make the fields opaque...
                if let Some(Item::Struct(ref mut s)) = bindgen_mod_item {
                    make_non_pod(s);
                } // otherwise might be an enum, etc.
                  // ... and say we don't depend on other types.
                new_deps.clear();
                TypeKind::NonPod
            };
            ApiDetail::Type {
                ty_details,
                for_extern_c_ts,
                is_forward_declaration,
                bindgen_mod_item,
                analysis: type_kind,
            }
        }
        ApiDetail::OpaqueTypedef => ApiDetail::OpaqueTypedef,
    };
    Ok(Api {
        ns: api.ns,
        id: api.id,
        deps: new_deps,
        detail: api_detail,
    })
}

fn get_struct_field_types(
    type_converter: &mut TypeConverter,
    ns: &Namespace,
    s: &ItemStruct,
    deps: &mut HashSet<TypeName>,
    extra_apis: &mut Vec<UnanalyzedApi>,
) -> Result<(), ConvertError> {
    for f in &s.fields {
        let annotated = type_converter.convert_type(f.ty.clone(), ns, false)?;
        extra_apis.extend(annotated.extra_apis);
        deps.extend(annotated.types_encountered);
    }
    Ok(())
}
<|endoftext|>