<commit_before>//
// FlightVars
// Copyright (c) 2015, 2016 Alvaro Polo
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::time;

use domain::Command;
use domain::notify::*;

const POLLING_DELAY_MS: u64 = 20;

pub trait Handler {
    fn command(&mut self, cmd: Command);
    fn poll(&mut self);
}

pub struct Worker {
    tx: NotifySender<Envelope>,
    rx: NotifyReceiver<Envelope>,
    run: bool,
}

impl Worker {
    pub fn new() -> Worker {
        let (tx, rx) = notification();
        Worker {
            tx: tx,
            rx: rx,
            run: true,
        }
    }

    pub fn sender(&self) -> NotifySender<Envelope> {
        self.tx.clone()
    }

    pub fn run<H: Handler>(&mut self, handler: &mut H) {
        self.run = true;
        let timeout = time::Duration::from_millis(POLLING_DELAY_MS);
        while self.run {
            match self.rx.recv(timeout) {
                Ok(Some(Envelope::Shutdown)) => self.shutdown(),
                Ok(Some(Envelope::Cmd(cmd))) => handler.command(cmd),
                Ok(None) => handler.poll(),
                _ => {},
            }
        }
    }

    pub fn shutdown(&mut self) {
        self.run = false;
    }
}

#[derive(Debug)]
pub enum Envelope {
    Cmd(Command),
    Shutdown
}

#[cfg(test)]
mod tests {
    use std::sync::mpsc;
    use std::thread;

    use domain::{Command, Var, Value};

    use super::*;

    #[test]
    fn should_shutdown() {
        let mut worker = Worker::new();
        let tx = worker.sender();
        let (polling_tx, _) = mpsc::channel();
        let (command_tx, _) = mpsc::channel();
        let child = thread::spawn(move || {
            let mut handler = MockHandler {
                pollings: polling_tx,
                commands: command_tx,
            };
            worker.run(&mut handler);
        });
        assert!(tx.send(Envelope::Shutdown).is_ok());
        assert!(child.join().is_ok());
    }

    #[test]
    fn should_tick_polling() {
        let mut worker = Worker::new();
        let tx = worker.sender();
        let (polling_tx, polling_rx) = mpsc::channel();
        let (command_tx, _) = mpsc::channel();
        let child = thread::spawn(move || {
            let mut handler = MockHandler {
                pollings: polling_tx,
                commands: command_tx,
            };
            worker.run(&mut handler);
        });
        assert!(polling_rx.recv().is_ok());
        assert!(tx.send(Envelope::Shutdown).is_ok());
        assert!(child.join().is_ok());
    }

    #[test]
    fn should_process_msg() {
        let mut worker = Worker::new();
        let tx = worker.sender();
        let (polling_tx, _polling_rx) = mpsc::channel();
        let (command_tx, command_rx) = mpsc::channel();
        let child = thread::spawn(move || {
            let mut handler = MockHandler {
                pollings: polling_tx,
                commands: command_tx,
            };
            worker.run(&mut handler);
        });
        let cmd = Command::Write(Var::lvar("foobar"), Value::Bool(true));
        assert!(tx.send(Envelope::Cmd(cmd.clone())).is_ok());
        assert_eq!(command_rx.recv().unwrap(), cmd);
        assert!(tx.send(Envelope::Shutdown).is_ok());
        assert!(child.join().is_ok());
    }

    struct MockHandler {
        pollings: mpsc::Sender<()>,
        commands: mpsc::Sender<Command>,
    }

    impl Handler for MockHandler {
        fn command(&mut self, cmd: Command) {
            self.commands.send(cmd).unwrap();
        }

        fn poll(&mut self) {
            self.pollings.send(()).unwrap();
        }
    }
}
<commit_msg>Introduce `domain::WorkerStub`<commit_after>//
// FlightVars
// Copyright (c) 2015, 2016 Alvaro Polo
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::thread;
use std::time;

use domain::Command;
use domain::notify::*;
use util::Consume;

const POLLING_DELAY_MS: u64 = 20;

pub trait Handle {
    fn command(&mut self, cmd: Command);
    fn poll(&mut self);
}

pub struct Worker {
    tx: NotifySender<Envelope>,
    rx: NotifyReceiver<Envelope>,
    run: bool,
}

impl Worker {
    pub fn new() -> Worker {
        let (tx, rx) = notification();
        Worker {
            tx: tx,
            rx: rx,
            run: true,
        }
    }

    pub fn sender(&self) -> NotifySender<Envelope> {
        self.tx.clone()
    }

    pub fn run<H: Handle>(&mut self, handler: &mut H) {
        self.run = true;
        let timeout = time::Duration::from_millis(POLLING_DELAY_MS);
        while self.run {
            match self.rx.recv(timeout) {
                Ok(Some(Envelope::Shutdown)) => self.shutdown(),
                Ok(Some(Envelope::Cmd(cmd))) => handler.command(cmd),
                Ok(None) => handler.poll(),
                _ => {},
            }
        }
    }

    pub fn shutdown(&mut self) {
        self.run = false;
    }
}

#[derive(Debug)]
pub enum Envelope {
    Cmd(Command),
    Shutdown
}

pub struct WorkerStub {
    sender: NotifySender<Envelope>,
    child: thread::JoinHandle<()>,
}

impl WorkerStub {
    pub fn consumer(&self) -> Consumer {
        Consumer {
            sender: self.sender.clone(),
        }
    }

    pub fn shutdown(self) {
        if let Err(e) = self.sender.send(Envelope::Shutdown) {
            error!("unexpected error while shutting down domain worker: {:?}", e);
        }
        if let Err(e) = self.child.join() {
            error!("unexpected error while joining to domain worker thread: {:?}", e);
        }
    }
}

#[derive(Clone)]
pub struct Consumer {
    sender: NotifySender<Envelope>,
}

impl Consume for Consumer {
    type Item = Command;
    type Error = NotifyError;
    fn consume(&mut self, cmd: Command) -> Result<(), NotifyError> {
        self.sender.send(Envelope::Cmd(cmd))
    }
}

pub fn spawn_worker<H: Handle + Send + 'static>(handler: H) -> WorkerStub {
    let worker = Worker::new();
    let sender = worker.sender();
    let child = thread::spawn(move || {
        let mut handler = handler;
        let mut worker = worker;
        worker.run(&mut handler);
    });
    WorkerStub {
        sender: sender,
        child: child,
    }
}

#[cfg(test)]
mod tests {
    use std::sync::mpsc;
    use std::thread;

    use domain::{Command, Var, Value};

    use super::*;

    #[test]
    fn should_shutdown() {
        let mut worker = Worker::new();
        let tx = worker.sender();
        let (polling_tx, _) = mpsc::channel();
        let (command_tx, _) = mpsc::channel();
        let child = thread::spawn(move || {
            let mut handler = MockHandle {
                pollings: polling_tx,
                commands: command_tx,
            };
            worker.run(&mut handler);
        });
        assert!(tx.send(Envelope::Shutdown).is_ok());
        assert!(child.join().is_ok());
    }

    #[test]
    fn should_tick_polling() {
        let mut worker = Worker::new();
        let tx = worker.sender();
        let (polling_tx, polling_rx) = mpsc::channel();
        let (command_tx, _) = mpsc::channel();
        let child = thread::spawn(move || {
            let mut handler = MockHandle {
                pollings: polling_tx,
                commands: command_tx,
            };
            worker.run(&mut handler);
        });
        assert!(polling_rx.recv().is_ok());
        assert!(tx.send(Envelope::Shutdown).is_ok());
        assert!(child.join().is_ok());
    }

    #[test]
    fn should_process_msg() {
        let mut worker = Worker::new();
        let tx = worker.sender();
        let (polling_tx, _polling_rx) = mpsc::channel();
        let (command_tx, command_rx) = mpsc::channel();
        let child = thread::spawn(move || {
            let mut handler = MockHandle {
                pollings: polling_tx,
                commands: command_tx,
            };
            worker.run(&mut handler);
        });
        let cmd = Command::Write(Var::lvar("foobar"), Value::Bool(true));
        assert!(tx.send(Envelope::Cmd(cmd.clone())).is_ok());
        assert_eq!(command_rx.recv().unwrap(), cmd);
        assert!(tx.send(Envelope::Shutdown).is_ok());
        assert!(child.join().is_ok());
    }

    struct MockHandle {
        pollings: mpsc::Sender<()>,
        commands: mpsc::Sender<Command>,
    }

    impl Handle for MockHandle {
        fn command(&mut self, cmd: Command) {
            self.commands.send(cmd).unwrap();
        }

        fn poll(&mut self) {
            self.pollings.send(()).unwrap();
        }
    }
}
<|endoftext|>


<commit_before>use bifrost::utils::fut_exec::wait;
use client::AsyncClient;
use dovahkiin::types::custom_types::bytes::SmallBytes;
use dovahkiin::types::custom_types::id::Id;
use dovahkiin::types::custom_types::map::Map;
use dovahkiin::types::type_id_of;
use dovahkiin::types::value::ToValue;
use futures::prelude::*;
use index::btree::Ordering;
use index::EntryKey;
use index::Slice;
use index::{id_from_key, key_with_id};
use itertools::Itertools;
use parking_lot::Mutex;
use parking_lot::RwLock;
use ram::cell::Cell;
use ram::schema::Field;
use ram::schema::Schema;
use ram::types::*;
use std::cell::RefCell;
use std::collections::btree_map::Range;
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, BTreeSet, LinkedList};
use std::fmt::Debug;
use std::marker::PhantomData;
use std::mem;
use std::sync::Arc;
use utils::lru_cache::LRUCache;

// LevelTree items cannot been added or removed individually
// Items must been merged from higher level in bulk
// Deletion will be performed when merging to or from higher level tree
// Because tree update will not to be performed in parallel. Unlike memtable, a single r/w lock
// should be sufficient. Thus concurrency control will be simple and efficient.

type Tombstones = BTreeSet<EntryKey>;
type PageCache<S> = Arc<Mutex<LRUCache<Id, Arc<SSPage<S>>>>>;

const PAGE_SCHEMA: &'static str = "NEB_SSTABLE_PAGE";

lazy_static! {
    static ref PAGE_SCHEMA_ID: u32 = key_hash(PAGE_SCHEMA) as u32;
}

pub struct LevelTree<S>
    where
        S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    neb_client: Arc<AsyncClient>,
    index: BTreeMap<EntryKey, Id>,
    tombstones: Tombstones,
    pages: PageCache<S>,
    marker: PhantomData<S>,
}
struct SSPage<S>
    where
        S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    id: Id,
    slice: S,
    len: usize,
}

impl<S> SSPage<S>
    where
        S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn from_cell(cell: Cell) -> Self {
        let mut slice = S::init();
        let slice_len = slice.as_slice().len();
        let cell_id = cell.id();
        let keys = cell.data;
        let keys_len = keys.len().unwrap();
        let keys_array = if let Value::PrimArray(PrimitiveArray::SmallBytes(ref array)) = keys {
            array
        } else {
            panic!()
        };
        for (i, val) in keys_array.iter().enumerate() {
            slice.as_slice()[i] = EntryKey::from(val.as_slice());
        }
        Self {
            id: cell_id,
            slice,
            len: keys_array.len(),
        }
    }

    pub fn to_cell(&self) -> Cell {
        let value = self
            .slice
            .as_slice_immute()
            .iter()
            .map(|key| SmallBytes::from_vec(key.as_slice().to_vec()))
            .take(self.len)
            .collect_vec()
            .value();
        Cell::new_with_id(*PAGE_SCHEMA_ID, &self.id, value)
    }

    pub fn new() -> Self {
        SSPage {
            slice: S::init(),
            id: Id::unit_id(),
            len: 0,
        }
    }
}

impl<S> LevelTree<S>
    where
        S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn new(neb_client: &Arc<AsyncClient>) -> Self {
        let client = neb_client.clone();
        let mut index = BTreeMap::new();
        Self {
            neb_client: neb_client.clone(),
            index,
            tombstones: Tombstones::new(),
            // TODO: carefully set the capacity
            pages: Arc::new(Mutex::new(LRUCache::new(
                100,
                move |id| {
                    client
                        .read_cell(*id)
                        .wait()
                        .unwrap()
                        .map(|cell| Arc::new(SSPage::from_cell(cell)))
                        .ok()
                },
                |_, _| {
                    // will do nothing on evict for pages will be write to back storage once updated
                },
            ))),
            marker: PhantomData {},
        }
    }

    pub fn seek(&self, key: &EntryKey, ordering: Ordering) -> RTCursor<S> {
        debug!("Seeking for {:?}, in index len {}", key, self.index.len());
        let mut range = match ordering {
            Ordering::Forward => self.index.range::<EntryKey, _>(key..),
            Ordering::Backward => self.index.range::<EntryKey, _>(..=key),
        };
        let first = match ordering {
            Ordering::Forward => range.next(),
            Ordering::Backward => range.next_back(),
        }
            .map(|(_, page_id)| {
                debug!("First page id is {:?}", page_id);
                self.pages.lock().get_or_fetch(page_id).unwrap().clone()
            });

        let (pos, page_len) = if let Some(ref page) = first {
            debug!("First page has {} elements", page.len);
            (
                page.slice.as_slice_immute()[..page.len]
                    .binary_search(key)
                    .unwrap_or_else(|i| i),
                page.len,
            )
        } else {
            (0, 0)
        };

        debug!("First page search pos is {}", pos);

        let mut cursor = RTCursor {
            ordering,
            range,
            page_len,
            current: first,
            index: pos,
            cache: self.pages.clone(),
        };

        if cursor.current.is_some() {
            match ordering {
                Ordering::Forward => {
                    if pos >= page_len {
                        cursor.next();
                    }
                }
                Ordering::Backward => {}
            }
        }

        cursor
    }

    pub fn merge(&mut self, mut source: Vec<EntryKey>, tombstones: &mut Tombstones) {
        let mut pages_cache = self.pages.lock();
        let (keys_to_removed, mut merged, mut ids_to_reuse) = {
            let (target_keys, target_pages): (Vec<_>, Vec<_>) = {
                let first = source.first().unwrap();
                let last = source.last().unwrap();
                self.index
                    .range::<EntryKey, _>(first..=last)
                    .map(|(k, id)| (k, pages_cache.get_or_fetch(id).unwrap().clone()))
                    .unzip()
            };
            let mut merged: Vec<SSPage<S>> = vec![SSPage::new()];
            let page_size = merged[0].slice.len();
            let target_page_ids = target_pages.iter().map(|p| p.id).collect::<LinkedList<_>>();
            let mut target = target_pages
                .iter()
                .map(|page| &page.slice.as_slice_immute()[..page.len])
                .flat_map(|x| x)
                .map(|x| (*x).clone())
                .collect_vec();

            let src_len = source.len();
            let tgt_len = target.len();

            debug!("Src len {}, target len {}", src_len, tgt_len);

            let mut srci = 0;
            let mut tgti = 0;

            while srci < src_len || tgti < tgt_len {
                let mut from_source = false;
                let mut lowest: &mut EntryKey = if srci >= src_len {
                    // source overflowed
                    tgti += 1;
                    &mut target[tgti - 1]
                } else if tgti >= tgt_len {
                    // target overflowed
                    srci += 1;
                    from_source = true;
                    &mut source[srci - 1]
                } else {
                    // normal case, compare
                    let t = &mut target[tgti];
                    let s = &mut source[srci];
                    if t <= s {
                        tgti += 1;
                        t
                    } else {
                        srci += 1;
                        from_source = true;
                        s
                    }
                };

                if from_source {
                    if tombstones.remove(lowest) {
                        continue;
                    }
                } else {
                    if self.tombstones.remove(lowest) {
                        continue;
                    }
                }

                let merging_page_len = {
                    let merging_page = merged.last_mut().unwrap();
                    let page_len = merging_page.len;
                    debug!("i is {}, picking {:?}", page_len, lowest);
                    mem::swap(&mut merging_page.slice.as_slice()[page_len], lowest);
                    merging_page.len += 1;
                    merging_page.len
                };

                if merging_page_len >= page_size {
                    merged.push(SSPage::new());
                }
            }
            if merged.last().unwrap().len == 0 {
                let merged_len = merged.len();
                merged.remove(merged_len - 1);
            }
            (
                target_keys.into_iter().map(|x| x.clone()).collect_vec(),
                merged,
                target_page_ids,
            )
        };

        debug!("Merged {} pages", merged.len());

        for k in keys_to_removed {
            debug!("Remove stale page index {:?}", k);
            self.index.remove(&k).unwrap();
        }

        for id in &ids_to_reuse {
            debug!("Remove stale page cache {:?}", id);
            pages_cache.remove(id);
        }

        for mut page in merged {
            let reuse_id = ids_to_reuse.pop_back();
            let id = reuse_id.unwrap_or_else(|| Id::rand());
            let index_key = page.slice.as_slice_immute()[0].clone();
            page.id = id;
            let cell = page.to_cell();
            if reuse_id.is_some() {
                debug!("Reuse page id by updating {:?}", cell.id());
                self.neb_client.update_cell(cell).wait().unwrap();
            } else {
                debug!("Inserting page with id {:?}", cell.id());
                self.neb_client.write_cell(cell).wait().unwrap();
            }

            debug!("Insert page index key {:?}, id {:?}", index_key, id);
            self.index.insert(index_key, id);
        }
    }
}

pub trait SortableEntrySlice: Sized + Slice<Item = EntryKey> {}

pub struct RTCursor<'a, S>
    where
        S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    index: usize,
    ordering: Ordering,
    page_len: usize,
    range: Range<'a, EntryKey, Id>,
    current: Option<Arc<SSPage<S>>>,
    cache: PageCache<S>,
}

impl<'a, S> RTCursor<'a, S>
    where
        S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn next(&mut self) -> bool {
        if self.current.is_some() {
            match self.ordering {
                Ordering::Forward => {
                    if self.index + 1 >= self.page_len {
                        if let Some((_, page_id)) = self.range.next() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = 0;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index += 1;
                    }
                }
                Ordering::Backward => {
                    if self.index == 0 {
                        if let Some((_, page_id)) = self.range.next_back() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = self.page_len - 1;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index -= 1;
                    }
                }
            }
            true
        } else {
            false
        }
    }

    pub fn current(&self) -> Option<&EntryKey> {
        if self.index < 0 || self.index >= self.page_len {
            return None;
        }
        if let Some(ref page) = self.current {
            Some(&page.slice.as_slice_immute()[self.index])
        } else {
            None
        }
    }

    fn get_page(&self, id: &Id) -> Arc<SSPage<S>> {
        self.cache.lock().get_or_fetch(id).unwrap().clone()
    }
}

pub fn get_schema() -> Schema {
    Schema {
        id: *PAGE_SCHEMA_ID,
        name: String::from(PAGE_SCHEMA),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new("*", type_id_of(Type::SmallBytes), false, true, None),
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use client;
    use futures::prelude::*;
    use index::sstable::LevelTree;
    use server::NebServer;
    use server::ServerOptions;
    use std::ptr;
    use std::sync::Arc;

    type SmallPage = [EntryKey; 5];

    impl_sspage_slice!(SmallPage, EntryKey, 5);

    #[test]
    pub fn init() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5200");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 16 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(super::get_schema()).wait();

        let mut tree: LevelTree<SmallPage> = LevelTree::new(&client);
        let id = Id::unit_id();
        let key = smallvec![1, 2, 3, 4, 5, 6];
        let mut tombstones = BTreeSet::new();
        info!("test insertion");

        let mut key_id = key.clone();
        key_with_id(&mut key_id, &id);

        tree.merge(vec![key_id], &mut tombstones);
        let mut cursor = tree.seek(&key, Ordering::Forward);
        assert_eq!(id_from_key(cursor.current().unwrap()), id);
    }
}
<commit_msg>cursor trait<commit_after>use bifrost::utils::fut_exec::wait;
use client::AsyncClient;
use dovahkiin::types::custom_types::bytes::SmallBytes;
use dovahkiin::types::custom_types::id::Id;
use dovahkiin::types::custom_types::map::Map;
use dovahkiin::types::type_id_of;
use dovahkiin::types::value::ToValue;
use futures::prelude::*;
use index::btree::Ordering;
use index::EntryKey;
use index::Slice;
use index::{id_from_key, key_with_id};
use itertools::Itertools;
use parking_lot::Mutex;
use parking_lot::RwLock;
use ram::cell::Cell;
use ram::schema::Field;
use ram::schema::Schema;
use ram::types::*;
use std::cell::RefCell;
use std::collections::btree_map::Range;
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, BTreeSet, LinkedList};
use std::fmt::Debug;
use std::marker::PhantomData;
use std::mem;
use std::sync::Arc;
use utils::lru_cache::LRUCache;

// LevelTree items cannot been added or removed individually
// Items must been merged from higher level in bulk
// Deletion will be performed when merging to or from higher level tree
// Because tree update will not to be performed in parallel. Unlike memtable, a single r/w lock
// should be sufficient. Thus concurrency control will be simple and efficient.

type Tombstones = BTreeSet<EntryKey>;
type PageCache<S> = Arc<Mutex<LRUCache<Id, Arc<SSPage<S>>>>>;

const PAGE_SCHEMA: &'static str = "NEB_SSTABLE_PAGE";

lazy_static! {
    static ref PAGE_SCHEMA_ID: u32 = key_hash(PAGE_SCHEMA) as u32;
}

pub struct LevelTree<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    neb_client: Arc<AsyncClient>,
    index: BTreeMap<EntryKey, Id>,
    tombstones: Tombstones,
    pages: PageCache<S>,
    marker: PhantomData<S>,
}
struct SSPage<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    id: Id,
    slice: S,
    len: usize,
}

impl<S> SSPage<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn from_cell(cell: Cell) -> Self {
        let mut slice = S::init();
        let slice_len = slice.as_slice().len();
        let cell_id = cell.id();
        let keys = cell.data;
        let keys_len = keys.len().unwrap();
        let keys_array = if let Value::PrimArray(PrimitiveArray::SmallBytes(ref array)) = keys {
            array
        } else {
            panic!()
        };
        for (i, val) in keys_array.iter().enumerate() {
            slice.as_slice()[i] = EntryKey::from(val.as_slice());
        }
        Self {
            id: cell_id,
            slice,
            len: keys_array.len(),
        }
    }

    pub fn to_cell(&self) -> Cell {
        let value = self
            .slice
            .as_slice_immute()
            .iter()
            .map(|key| SmallBytes::from_vec(key.as_slice().to_vec()))
            .take(self.len)
            .collect_vec()
            .value();
        Cell::new_with_id(*PAGE_SCHEMA_ID, &self.id, value)
    }

    pub fn new() -> Self {
        SSPage {
            slice: S::init(),
            id: Id::unit_id(),
            len: 0,
        }
    }
}

impl<S> LevelTree<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn new(neb_client: &Arc<AsyncClient>) -> Self {
        let client = neb_client.clone();
        let mut index = BTreeMap::new();
        Self {
            neb_client: neb_client.clone(),
            index,
            tombstones: Tombstones::new(),
            // TODO: carefully set the capacity
            pages: Arc::new(Mutex::new(LRUCache::new(
                100,
                move |id| {
                    client
                        .read_cell(*id)
                        .wait()
                        .unwrap()
                        .map(|cell| Arc::new(SSPage::from_cell(cell)))
                        .ok()
                },
                |_, _| {
                    // will do nothing on evict for pages will be write to back storage once updated
                },
            ))),
            marker: PhantomData {},
        }
    }

    pub fn seek(&self, key: &EntryKey, ordering: Ordering) -> impl Cursor {
        debug!("Seeking for {:?}, in index len {}", key, self.index.len());
        let mut range = match ordering {
            Ordering::Forward => self.index.range::<EntryKey, _>(key..),
            Ordering::Backward => self.index.range::<EntryKey, _>(..=key),
        };
        let first = match ordering {
            Ordering::Forward => range.next(),
            Ordering::Backward => range.next_back(),
        }
        .map(|(_, page_id)| {
            debug!("First page id is {:?}", page_id);
            self.pages.lock().get_or_fetch(page_id).unwrap().clone()
        });

        let (pos, page_len) = if let Some(ref page) = first {
            debug!("First page has {} elements", page.len);
            (
                page.slice.as_slice_immute()[..page.len]
                    .binary_search(key)
                    .unwrap_or_else(|i| i),
                page.len,
            )
        } else {
            (0, 0)
        };

        debug!("First page search pos is {}", pos);

        let mut cursor = RTCursor {
            ordering,
            range,
            page_len,
            current: first,
            index: pos,
            cache: self.pages.clone(),
        };

        if cursor.current.is_some() {
            match ordering {
                Ordering::Forward => {
                    if pos >= page_len {
                        cursor.next();
                    }
                }
                Ordering::Backward => {}
            }
        }

        cursor
    }

    pub fn merge(&mut self, mut source: Vec<EntryKey>, tombstones: &mut Tombstones) {
        let mut pages_cache = self.pages.lock();
        let (keys_to_removed, mut merged, mut ids_to_reuse) = {
            let (target_keys, target_pages): (Vec<_>, Vec<_>) = {
                let first = source.first().unwrap();
                let last = source.last().unwrap();
                self.index
                    .range::<EntryKey, _>(first..=last)
                    .map(|(k, id)| (k, pages_cache.get_or_fetch(id).unwrap().clone()))
                    .unzip()
            };
            let mut merged: Vec<SSPage<S>> = vec![SSPage::new()];
            let page_size = merged[0].slice.len();
            let target_page_ids = target_pages.iter().map(|p| p.id).collect::<LinkedList<_>>();
            let mut target = target_pages
                .iter()
                .map(|page| &page.slice.as_slice_immute()[..page.len])
                .flat_map(|x| x)
                .map(|x| (*x).clone())
                .collect_vec();

            let src_len = source.len();
            let tgt_len = target.len();

            debug!("Src len {}, target len {}", src_len, tgt_len);

            let mut srci = 0;
            let mut tgti = 0;

            while srci < src_len || tgti < tgt_len {
                let mut from_source = false;
                let mut lowest: &mut EntryKey = if srci >= src_len {
                    // source overflowed
                    tgti += 1;
                    &mut target[tgti - 1]
                } else if tgti >= tgt_len {
                    // target overflowed
                    srci += 1;
                    from_source = true;
                    &mut source[srci - 1]
                } else {
                    // normal case, compare
                    let t = &mut target[tgti];
                    let s = &mut source[srci];
                    if t <= s {
                        tgti += 1;
                        t
                    } else {
                        srci += 1;
                        from_source = true;
                        s
                    }
                };

                if from_source {
                    if tombstones.remove(lowest) {
                        continue;
                    }
                } else {
                    if self.tombstones.remove(lowest) {
                        continue;
                    }
                }

                let merging_page_len = {
                    let merging_page = merged.last_mut().unwrap();
                    let page_len = merging_page.len;
                    debug!("i is {}, picking {:?}", page_len, lowest);
                    mem::swap(&mut merging_page.slice.as_slice()[page_len], lowest);
                    merging_page.len += 1;
                    merging_page.len
                };

                if merging_page_len >= page_size {
                    merged.push(SSPage::new());
                }
            }
            if merged.last().unwrap().len == 0 {
                let merged_len = merged.len();
                merged.remove(merged_len - 1);
            }
            (
                target_keys.into_iter().map(|x| x.clone()).collect_vec(),
                merged,
                target_page_ids,
            )
        };

        debug!("Merged {} pages", merged.len());

        for k in keys_to_removed {
            debug!("Remove stale page index {:?}", k);
            self.index.remove(&k).unwrap();
        }

        for id in &ids_to_reuse {
            debug!("Remove stale page cache {:?}", id);
            pages_cache.remove(id);
        }

        for mut page in merged {
            let reuse_id = ids_to_reuse.pop_back();
            let id = reuse_id.unwrap_or_else(|| Id::rand());
            let index_key = page.slice.as_slice_immute()[0].clone();
            page.id = id;
            let cell = page.to_cell();
            if reuse_id.is_some() {
                debug!("Reuse page id by updating {:?}", cell.id());
                self.neb_client.update_cell(cell).wait().unwrap();
            } else {
                debug!("Inserting page with id {:?}", cell.id());
                self.neb_client.write_cell(cell).wait().unwrap();
            }

            debug!("Insert page index key {:?}, id {:?}", index_key, id);
            self.index.insert(index_key, id);
        }
    }
}

pub trait SortableEntrySlice: Sized + Slice<Item = EntryKey> {}

pub trait Cursor<'a> {
    fn next(&mut self) -> bool;
    fn current(&self) -> Option<&EntryKey>;
}

pub struct RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    index: usize,
    ordering: Ordering,
    page_len: usize,
    range: Range<'a, EntryKey, Id>,
    current: Option<Arc<SSPage<S>>>,
    cache: PageCache<S>,
}

impl<'a, S> Cursor<'a> for RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    fn next(&mut self) -> bool {
        if self.current.is_some() {
            match self.ordering {
                Ordering::Forward => {
                    if self.index + 1 >= self.page_len {
                        if let Some((_, page_id)) = self.range.next() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = 0;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index += 1;
                    }
                }
                Ordering::Backward => {
                    if self.index == 0 {
                        if let Some((_, page_id)) = self.range.next_back() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = self.page_len - 1;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index -= 1;
                    }
                }
            }
            true
        } else {
            false
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        if self.index < 0 || self.index >= self.page_len {
            return None;
        }
        if let Some(ref page) = self.current {
            Some(&page.slice.as_slice_immute()[self.index])
        } else {
            None
        }
    }
}

impl<'a, S> RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    fn get_page(&self, id: &Id) -> Arc<SSPage<S>> {
        self.cache.lock().get_or_fetch(id).unwrap().clone()
    }
}

pub fn get_schema() -> Schema {
    Schema {
        id: *PAGE_SCHEMA_ID,
        name: String::from(PAGE_SCHEMA),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new("*", type_id_of(Type::SmallBytes), false, true, None),
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use client;
    use futures::prelude::*;
    use index::sstable::LevelTree;
    use server::NebServer;
    use server::ServerOptions;
    use std::ptr;
    use std::sync::Arc;

    type SmallPage = [EntryKey; 5];

    impl_sspage_slice!(SmallPage, EntryKey, 5);

    #[test]
    pub fn init() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5200");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 16 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(super::get_schema()).wait();

        let mut tree: LevelTree<SmallPage> = LevelTree::new(&client);
        let id = Id::unit_id();
        let key = smallvec![1, 2, 3, 4, 5, 6];
        let mut tombstones = BTreeSet::new();
        info!("test insertion");

        let mut key_id = key.clone();
        key_with_id(&mut key_id, &id);

        tree.merge(vec![key_id], &mut tombstones);
        let mut cursor = tree.seek(&key, Ordering::Forward);
        assert_eq!(id_from_key(cursor.current().unwrap()), id);
    }
}
<|endoftext|>


<commit_before>// #![allow(dead_code)]

use std::os::raw::{
	c_char,
	c_int,
	c_void,
};

// type without an instance
pub enum DNSServiceT {}
pub type DNSServiceRef = *mut DNSServiceT;

// type without an instance
pub enum DNSRecordT {}
pub type DNSRecordRef = *mut DNSRecordT;

pub type DNSServiceFlags = u32;
// pub const FLAGS_NONE                 : DNSServiceFlags = 0x0;
pub const FLAGS_MORE_COMING: DNSServiceFlags = 0x1;
pub const FLAGS_ADD: DNSServiceFlags = 0x2;
pub const FLAGS_DEFAULT: DNSServiceFlags = 0x4;
pub const FLAGS_NO_AUTO_RENAME: DNSServiceFlags = 0x8;
pub const FLAGS_SHARED: DNSServiceFlags = 0x10;
pub const FLAGS_UNIQUE: DNSServiceFlags = 0x20;
pub const FLAGS_BROWSE_DOMAINS: DNSServiceFlags = 0x40;
pub const FLAGS_REGISTRATION_DOMAINS: DNSServiceFlags = 0x80;
// unix only?
#[cfg(unix)]
pub const FLAGS_LONG_LIVED_QUERY: DNSServiceFlags = 0x100;
#[cfg(not(unix))]
pub const FLAGS_LONG_LIVED_QUERY: DNSServiceFlags = 0;
// avahi only?
// pub const FLAGS_ALLOW_REMOTE_QUERY: DNSServiceFlags = 0x200;
// pub const FLAGS_FORCE_MULTICAS: DNSServiceFlags = 0x400;
// pub const FLAGS_RETURN_CNAME: DNSServiceFlags = 0x800;

/// Maximum length of full name including trailing dot and terminating NULL
///
/// See [`kDNSServiceMaxDomainName`](https://developer.apple.com/documentation/dnssd/kdnsservicemaxdomainname)
pub const MAX_DOMAIN_NAME: usize = 1009;

pub const INTERFACE_INDEX_ANY: u32 = 0;
pub const INTERFACE_INDEX_LOCAL_ONLY: u32 = !0;
pub const INTERFACE_INDEX_UNICAST: u32 = !1;
pub const INTERFACE_INDEX_P2P: u32 = !2;

macro_rules! c_api_enum {
	($(#[$m:meta])* $name:ident : $ty:tt => $($case:ident = $val:expr,)* ) => (
		#[derive(Clone,Copy,Eq,PartialEq,Ord,PartialOrd,Hash,Debug)]
		#[repr($ty)]
		$(#[$m])*
		pub enum $name {
			$($case = $val,)*
		}
		impl $name {
			pub fn try_from(value: $ty) -> Option<$name> {
				$(if value == $val {
					Some($name::$case)
				} else)* {
					None
				}
			}
		}
	)
}

pub type DNSServiceErrorType = i32;
c_api_enum! {DNSServiceNoError: i32 =>
	NoError               = 0,
	// windows "TCP Connection Status"
	ConnectionPending     = -65570,
	ConnectionFailed      = -65571,
	ConnectionEstablished = -65572,
	// windows "Non-error values"
	GrowCache             = -65790,
	ConfigChanged         = -65791,
	MemFree               = -65792,
}
c_api_enum! {
/// Known error codes
///
/// See [`DNSServiceErrorType`](https://developer.apple.com/documentation/dnssd/1823426-anonymous)
DNSServiceError: i32 =>
	Unknown               = -65537,
	NoSuchName            = -65538,
	NoMemory              = -65539,
	BadParam              = -65540,
	BadReference          = -65541,
	BadState              = -65542,
	BadFlags              = -65543,
	Unsupported           = -65544,
	NotInitialized        = -65545,
	NoCache               = -65546,
	AlreadyRegistered     = -65547,
	NameConflict          = -65548,
	Invalid               = -65549,
	Incompatible          = -65551,
	BadInterfaceIndex     = -65552,
	Refused               = -65553,
	NoSuchRecord          = -65554,
	NoAuth                = -65555,
	NoSuchKey             = -65556,
	NoValue               = -65557,
	BufferTooSmall        = -65558,
}

pub type DNSServiceDomainEnumReply = Option<
	extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		reply_domain: *const c_char,
		context: *mut c_void,
	),
>;
pub type DNSServiceRegisterReply = Option<
	extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		error_code: DNSServiceErrorType,
		name: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
		context: *mut c_void,
	),
>;
pub type DNSServiceBrowseReply = Option<
	extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		service_name: *const c_char,
		reg_type: *const c_char,
		reply_domain: *const c_char,
		context: *mut c_void,
	),
>;
pub type DNSServiceResolveReply = Option<
	extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		fullname: *const c_char,
		host_target: *const c_char,
		port: u16,
		txt_len: u16,
		txt_record: *const u8,
		context: *mut c_void,
	),
>;
pub type DNSServiceRegisterRecordReply = Option<
	extern "C" fn(
		sd_ref: DNSServiceRef,
		record_ref: DNSRecordRef,
		flags: DNSServiceFlags,
		error_code: DNSServiceErrorType,
		context: *mut c_void,
	),
>;
pub type DNSServiceQueryRecordReply = Option<
	extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
		context: *mut c_void,
	),
>;

extern "C" {
	pub fn DNSServiceRefSockFD(sd_ref: DNSServiceRef) -> c_int;
	pub fn DNSServiceProcessResult(sd_ref: DNSServiceRef) -> DNSServiceErrorType;
	pub fn DNSServiceRefDeallocate(sd_ref: DNSServiceRef);
	pub fn DNSServiceEnumerateDomains(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		callback: DNSServiceDomainEnumReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceRegister(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		name: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
		host: *const c_char,
		port: u16,
		txt_len: u16,
		txt_record: *const u8,
		callback: DNSServiceRegisterReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceAddRecord(
		sd_ref: DNSServiceRef,
		record_ref: *mut DNSRecordRef,
		flags: DNSServiceFlags,
		rr_type: u16,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
	) -> DNSServiceErrorType;
	pub fn DNSServiceUpdateRecord(
		sd_ref: DNSServiceRef,
		record_ref: DNSRecordRef,
		flags: DNSServiceFlags,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
	) -> DNSServiceErrorType;
	pub fn DNSServiceRemoveRecord(
		sd_ref: DNSServiceRef,
		record_ref: DNSRecordRef,
		flags: DNSServiceFlags,
	) -> DNSServiceErrorType;
	pub fn DNSServiceBrowse(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		reg_type: *const c_char,
		domain: *const c_char,
		callback: DNSServiceBrowseReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceResolve(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		name: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
		callback: DNSServiceResolveReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceCreateConnection(sd_ref: *mut DNSServiceRef) -> DNSServiceErrorType;
	pub fn DNSServiceRegisterRecord(
		sd_ref: DNSServiceRef,
		record_ref: *mut DNSRecordRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
		callback: DNSServiceRegisterRecordReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceQueryRecord(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		callback: DNSServiceQueryRecordReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceReconfirmRecord(
		flags: DNSServiceFlags,
		interface_index: u32,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		rd_len: u16,
		rdata: *const u8,
	) -> DNSServiceErrorType;
	pub fn DNSServiceConstructFullName(
		fullName: *mut c_char,
		service: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
	) -> c_int;
}

// TXTRecordRef utils not wrapped - should be easy enough to implement
// in pure rust

/* Not used so far:
#[cfg(windows)]
mod ffi_windows {
	use super::DNSServiceErrorType;
	use std::os::raw::{
		c_int,
		c_void,
	};

	pub type DNSServiceInitializeFlags = u32;
	pub const INITIALIZE_FLAGS_NONE: DNSServiceInitializeFlags = 0x0;
	pub const INITIALIZE_FLAGS_ADVERTISE: DNSServiceInitializeFlags = 0x1;
	pub const INITIALIZE_FLAGS_NO_SERVER_CHECK: DNSServiceInitializeFlags = 0x2;

	pub type DNSPropertyCode = u32;

	pub const PROPERTY_CODE_VERSION: DNSPropertyCode = 0x76657273;
	#[repr(C)]
	pub struct DnsPropertyVersion {
		pub code: DNSPropertyCode,

		pub client_current_version: u32,
		pub client_oldest_server_version: u32,
		pub server_current_version: u32,
		pub server_oldest_client_version: u32,
	}

	extern "C" {
		pub fn DNSServiceInitialize(
			inFlags: DNSServiceInitializeFlags,
			inCacheEntryCount: c_int,
		) -> DNSServiceErrorType;
		pub fn DNSServiceFinalize();
		pub fn DNSServiceCheckVersion() -> DNSServiceErrorType;

		// TODO? DNSPropertyData on windows.
		// the c API uses a union... - using void instead here
		pub fn DNSServiceCopyProperty(
			inCode: DNSPropertyCode,
			outData: *mut c_void,
		) -> DNSServiceErrorType;
		pub fn DNSServiceReleaseProperty(
			inData: *mut c_void,
		) -> DNSServiceErrorType;
	}
}
#[cfg(windows)]
pub use self::ffi_windows::*;
*/
<commit_msg>ffi: allow unsafe callback functions<commit_after>// #![allow(dead_code)]

use std::os::raw::{
	c_char,
	c_int,
	c_void,
};

// type without an instance
pub enum DNSServiceT {}
pub type DNSServiceRef = *mut DNSServiceT;

// type without an instance
pub enum DNSRecordT {}
pub type DNSRecordRef = *mut DNSRecordT;

pub type DNSServiceFlags = u32;
// pub const FLAGS_NONE                 : DNSServiceFlags = 0x0;
pub const FLAGS_MORE_COMING: DNSServiceFlags = 0x1;
pub const FLAGS_ADD: DNSServiceFlags = 0x2;
pub const FLAGS_DEFAULT: DNSServiceFlags = 0x4;
pub const FLAGS_NO_AUTO_RENAME: DNSServiceFlags = 0x8;
pub const FLAGS_SHARED: DNSServiceFlags = 0x10;
pub const FLAGS_UNIQUE: DNSServiceFlags = 0x20;
pub const FLAGS_BROWSE_DOMAINS: DNSServiceFlags = 0x40;
pub const FLAGS_REGISTRATION_DOMAINS: DNSServiceFlags = 0x80;
// unix only?
#[cfg(unix)]
pub const FLAGS_LONG_LIVED_QUERY: DNSServiceFlags = 0x100;
#[cfg(not(unix))]
pub const FLAGS_LONG_LIVED_QUERY: DNSServiceFlags = 0;
// avahi only?
// pub const FLAGS_ALLOW_REMOTE_QUERY: DNSServiceFlags = 0x200;
// pub const FLAGS_FORCE_MULTICAS: DNSServiceFlags = 0x400;
// pub const FLAGS_RETURN_CNAME: DNSServiceFlags = 0x800;

/// Maximum length of full name including trailing dot and terminating NULL
///
/// See [`kDNSServiceMaxDomainName`](https://developer.apple.com/documentation/dnssd/kdnsservicemaxdomainname)
pub const MAX_DOMAIN_NAME: usize = 1009;

pub const INTERFACE_INDEX_ANY: u32 = 0;
pub const INTERFACE_INDEX_LOCAL_ONLY: u32 = !0;
pub const INTERFACE_INDEX_UNICAST: u32 = !1;
pub const INTERFACE_INDEX_P2P: u32 = !2;

macro_rules! c_api_enum {
	($(#[$m:meta])* $name:ident : $ty:tt => $($case:ident = $val:expr,)* ) => (
		#[derive(Clone,Copy,Eq,PartialEq,Ord,PartialOrd,Hash,Debug)]
		#[repr($ty)]
		$(#[$m])*
		pub enum $name {
			$($case = $val,)*
		}
		impl $name {
			pub fn try_from(value: $ty) -> Option<$name> {
				$(if value == $val {
					Some($name::$case)
				} else)* {
					None
				}
			}
		}
	)
}

pub type DNSServiceErrorType = i32;
c_api_enum! {DNSServiceNoError: i32 =>
	NoError               = 0,
	// windows "TCP Connection Status"
	ConnectionPending     = -65570,
	ConnectionFailed      = -65571,
	ConnectionEstablished = -65572,
	// windows "Non-error values"
	GrowCache             = -65790,
	ConfigChanged         = -65791,
	MemFree               = -65792,
}
c_api_enum! {
/// Known error codes
///
/// See [`DNSServiceErrorType`](https://developer.apple.com/documentation/dnssd/1823426-anonymous)
DNSServiceError: i32 =>
	Unknown               = -65537,
	NoSuchName            = -65538,
	NoMemory              = -65539,
	BadParam              = -65540,
	BadReference          = -65541,
	BadState              = -65542,
	BadFlags              = -65543,
	Unsupported           = -65544,
	NotInitialized        = -65545,
	NoCache               = -65546,
	AlreadyRegistered     = -65547,
	NameConflict          = -65548,
	Invalid               = -65549,
	Incompatible          = -65551,
	BadInterfaceIndex     = -65552,
	Refused               = -65553,
	NoSuchRecord          = -65554,
	NoAuth                = -65555,
	NoSuchKey             = -65556,
	NoValue               = -65557,
	BufferTooSmall        = -65558,
}

pub type DNSServiceDomainEnumReply = Option<
	unsafe extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		reply_domain: *const c_char,
		context: *mut c_void,
	),
>;
pub type DNSServiceRegisterReply = Option<
	unsafe extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		error_code: DNSServiceErrorType,
		name: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
		context: *mut c_void,
	),
>;
pub type DNSServiceBrowseReply = Option<
	unsafe extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		service_name: *const c_char,
		reg_type: *const c_char,
		reply_domain: *const c_char,
		context: *mut c_void,
	),
>;
pub type DNSServiceResolveReply = Option<
	unsafe extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		fullname: *const c_char,
		host_target: *const c_char,
		port: u16,
		txt_len: u16,
		txt_record: *const u8,
		context: *mut c_void,
	),
>;
pub type DNSServiceRegisterRecordReply = Option<
	unsafe extern "C" fn(
		sd_ref: DNSServiceRef,
		record_ref: DNSRecordRef,
		flags: DNSServiceFlags,
		error_code: DNSServiceErrorType,
		context: *mut c_void,
	),
>;
pub type DNSServiceQueryRecordReply = Option<
	unsafe extern "C" fn(
		sd_ref: DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		error_code: DNSServiceErrorType,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
		context: *mut c_void,
	),
>;

extern "C" {
	pub fn DNSServiceRefSockFD(sd_ref: DNSServiceRef) -> c_int;
	pub fn DNSServiceProcessResult(sd_ref: DNSServiceRef) -> DNSServiceErrorType;
	pub fn DNSServiceRefDeallocate(sd_ref: DNSServiceRef);
	pub fn DNSServiceEnumerateDomains(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		callback: DNSServiceDomainEnumReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceRegister(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		name: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
		host: *const c_char,
		port: u16,
		txt_len: u16,
		txt_record: *const u8,
		callback: DNSServiceRegisterReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceAddRecord(
		sd_ref: DNSServiceRef,
		record_ref: *mut DNSRecordRef,
		flags: DNSServiceFlags,
		rr_type: u16,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
	) -> DNSServiceErrorType;
	pub fn DNSServiceUpdateRecord(
		sd_ref: DNSServiceRef,
		record_ref: DNSRecordRef,
		flags: DNSServiceFlags,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
	) -> DNSServiceErrorType;
	pub fn DNSServiceRemoveRecord(
		sd_ref: DNSServiceRef,
		record_ref: DNSRecordRef,
		flags: DNSServiceFlags,
	) -> DNSServiceErrorType;
	pub fn DNSServiceBrowse(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		reg_type: *const c_char,
		domain: *const c_char,
		callback: DNSServiceBrowseReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceResolve(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		name: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
		callback: DNSServiceResolveReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceCreateConnection(sd_ref: *mut DNSServiceRef) -> DNSServiceErrorType;
	pub fn DNSServiceRegisterRecord(
		sd_ref: DNSServiceRef,
		record_ref: *mut DNSRecordRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		rd_len: u16,
		rdata: *const u8,
		ttl: u32,
		callback: DNSServiceRegisterRecordReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceQueryRecord(
		sd_ref: *mut DNSServiceRef,
		flags: DNSServiceFlags,
		interface_index: u32,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		callback: DNSServiceQueryRecordReply,
		context: *mut c_void,
	) -> DNSServiceErrorType;
	pub fn DNSServiceReconfirmRecord(
		flags: DNSServiceFlags,
		interface_index: u32,
		fullname: *const c_char,
		rr_type: u16,
		rr_class: u16,
		rd_len: u16,
		rdata: *const u8,
	) -> DNSServiceErrorType;
	pub fn DNSServiceConstructFullName(
		fullName: *mut c_char,
		service: *const c_char,
		reg_type: *const c_char,
		domain: *const c_char,
	) -> c_int;
}

// TXTRecordRef utils not wrapped - should be easy enough to implement
// in pure rust

/* Not used so far:
#[cfg(windows)]
mod ffi_windows {
	use super::DNSServiceErrorType;
	use std::os::raw::{
		c_int,
		c_void,
	};

	pub type DNSServiceInitializeFlags = u32;
	pub const INITIALIZE_FLAGS_NONE: DNSServiceInitializeFlags = 0x0;
	pub const INITIALIZE_FLAGS_ADVERTISE: DNSServiceInitializeFlags = 0x1;
	pub const INITIALIZE_FLAGS_NO_SERVER_CHECK: DNSServiceInitializeFlags = 0x2;

	pub type DNSPropertyCode = u32;

	pub const PROPERTY_CODE_VERSION: DNSPropertyCode = 0x76657273;
	#[repr(C)]
	pub struct DnsPropertyVersion {
		pub code: DNSPropertyCode,

		pub client_current_version: u32,
		pub client_oldest_server_version: u32,
		pub server_current_version: u32,
		pub server_oldest_client_version: u32,
	}

	extern "C" {
		pub fn DNSServiceInitialize(
			inFlags: DNSServiceInitializeFlags,
			inCacheEntryCount: c_int,
		) -> DNSServiceErrorType;
		pub fn DNSServiceFinalize();
		pub fn DNSServiceCheckVersion() -> DNSServiceErrorType;

		// TODO? DNSPropertyData on windows.
		// the c API uses a union... - using void instead here
		pub fn DNSServiceCopyProperty(
			inCode: DNSPropertyCode,
			outData: *mut c_void,
		) -> DNSServiceErrorType;
		pub fn DNSServiceReleaseProperty(
			inData: *mut c_void,
		) -> DNSServiceErrorType;
	}
}
#[cfg(windows)]
pub use self::ffi_windows::*;
*/
<|endoftext|>


<commit_before>//! Tests for various types of file (video, image, compressed, etc).
//!
//! Currently this is dependent on the file’s name and extension, because
//! those are the only metadata that we have access to without reading the
//! file’s contents.

use ansi_term::Style;

use fs::File;
use output::file_name::FileColours;

#[derive(Debug, Default, PartialEq)]
pub struct FileExtensions;

impl FileExtensions {

    /// An “immediate” file is something that can be run or activated somehow
    /// in order to kick off the build of a project. It’s usually only present
    /// in directories full of source code.
    fn is_immediate(&self, file: &File) -> bool {
        file.name.starts_with("README") || file.name_is_one_of( &[
            "Makefile", "Cargo.toml", "SConstruct", "CMakeLists.txt",
            "build.gradle", "Rakefile", "Gruntfile.js",
            "Gruntfile.coffee", "BUILD", "WORKSPACE", "build.xml"
        ])
    }

    fn is_image(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "png", "jpeg", "jpg", "gif", "bmp", "tiff", "tif",
            "ppm", "pgm", "pbm", "pnm", "webp", "raw", "arw",
            "svg", "stl", "eps", "dvi", "ps", "cbr",
            "cbz", "xpm", "ico", "cr2", "orf", "nef",
        ])
    }

    fn is_video(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "avi", "flv", "m2v", "mkv", "mov", "mp4", "mpeg",
            "mpg", "ogm", "ogv", "vob", "wmv", "webm", "m2ts",
            "ts",
        ])
    }

    fn is_music(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "aac", "m4a", "mp3", "ogg", "wma", "mka", "opus",
        ])
    }

    // Lossless music, rather than any other kind of data...
    fn is_lossless(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "alac", "ape", "flac", "wav",
        ])
    }

    fn is_crypto(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "asc", "enc", "gpg", "pgp", "sig", "signature", "pfx", "p12",
        ])
    }

    fn is_document(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "djvu", "doc", "docx", "dvi", "eml", "eps", "fotd",
            "odp", "odt", "pdf", "ppt", "pptx", "rtf",
            "xls", "xlsx",
        ])
    }

    fn is_compressed(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "zip", "tar", "Z", "z", "gz", "bz2", "a", "ar", "7z",
            "iso", "dmg", "tc", "rar", "par", "tgz", "xz", "txz",
            "lzma", "deb", "rpm", "zst",
        ])
    }

    fn is_temp(&self, file: &File) -> bool {
        file.name.ends_with('~')
            || (file.name.starts_with('#') && file.name.ends_with('#'))
            || file.extension_is_one_of( &[ "tmp", "swp", "swo", "swn", "bak", "bk" ])
    }

    fn is_compiled(&self, file: &File) -> bool {
        if file.extension_is_one_of( &[ "class", "elc", "hi", "o", "pyc" ]) {
            true
        }
        else if let Some(dir) = file.parent_dir {
            file.get_source_files().iter().any(|path| dir.contains(path))
        }
        else {
            false
        }
    }
}

impl FileColours for FileExtensions {
    fn colour_file(&self, file: &File) -> Option<Style> {
        use ansi_term::Colour::*;

        Some(match file {
            f if self.is_immediate(f)   => Yellow.bold().underline(),
            f if self.is_image(f)       => Fixed(133).normal(),
            f if self.is_video(f)       => Fixed(135).normal(),
            f if self.is_music(f)       => Fixed(92).normal(),
            f if self.is_lossless(f)    => Fixed(93).normal(),
            f if self.is_crypto(f)      => Fixed(109).normal(),
            f if self.is_document(f)    => Fixed(105).normal(),
            f if self.is_compressed(f)  => Red.normal(),
            f if self.is_temp(f)        => Fixed(244).normal(),
            f if self.is_compiled(f)    => Fixed(137).normal(),
            _                           => return None,
        })
    }
}
<commit_msg>Don't treat .ts as a video file extension<commit_after>//! Tests for various types of file (video, image, compressed, etc).
//!
//! Currently this is dependent on the file’s name and extension, because
//! those are the only metadata that we have access to without reading the
//! file’s contents.

use ansi_term::Style;

use fs::File;
use output::file_name::FileColours;

#[derive(Debug, Default, PartialEq)]
pub struct FileExtensions;

impl FileExtensions {

    /// An “immediate” file is something that can be run or activated somehow
    /// in order to kick off the build of a project. It’s usually only present
    /// in directories full of source code.
    fn is_immediate(&self, file: &File) -> bool {
        file.name.starts_with("README") || file.name_is_one_of( &[
            "Makefile", "Cargo.toml", "SConstruct", "CMakeLists.txt",
            "build.gradle", "Rakefile", "Gruntfile.js",
            "Gruntfile.coffee", "BUILD", "WORKSPACE", "build.xml"
        ])
    }

    fn is_image(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "png", "jpeg", "jpg", "gif", "bmp", "tiff", "tif",
            "ppm", "pgm", "pbm", "pnm", "webp", "raw", "arw",
            "svg", "stl", "eps", "dvi", "ps", "cbr",
            "cbz", "xpm", "ico", "cr2", "orf", "nef",
        ])
    }

    fn is_video(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "avi", "flv", "m2v", "mkv", "mov", "mp4", "mpeg",
            "mpg", "ogm", "ogv", "vob", "wmv", "webm", "m2ts",
        ])
    }

    fn is_music(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "aac", "m4a", "mp3", "ogg", "wma", "mka", "opus",
        ])
    }

    // Lossless music, rather than any other kind of data...
    fn is_lossless(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "alac", "ape", "flac", "wav",
        ])
    }

    fn is_crypto(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "asc", "enc", "gpg", "pgp", "sig", "signature", "pfx", "p12",
        ])
    }

    fn is_document(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "djvu", "doc", "docx", "dvi", "eml", "eps", "fotd",
            "odp", "odt", "pdf", "ppt", "pptx", "rtf",
            "xls", "xlsx",
        ])
    }

    fn is_compressed(&self, file: &File) -> bool {
        file.extension_is_one_of( &[
            "zip", "tar", "Z", "z", "gz", "bz2", "a", "ar", "7z",
            "iso", "dmg", "tc", "rar", "par", "tgz", "xz", "txz",
            "lzma", "deb", "rpm", "zst",
        ])
    }

    fn is_temp(&self, file: &File) -> bool {
        file.name.ends_with('~')
            || (file.name.starts_with('#') && file.name.ends_with('#'))
            || file.extension_is_one_of( &[ "tmp", "swp", "swo", "swn", "bak", "bk" ])
    }

    fn is_compiled(&self, file: &File) -> bool {
        if file.extension_is_one_of( &[ "class", "elc", "hi", "o", "pyc" ]) {
            true
        }
        else if let Some(dir) = file.parent_dir {
            file.get_source_files().iter().any(|path| dir.contains(path))
        }
        else {
            false
        }
    }
}

impl FileColours for FileExtensions {
    fn colour_file(&self, file: &File) -> Option<Style> {
        use ansi_term::Colour::*;

        Some(match file {
            f if self.is_immediate(f)   => Yellow.bold().underline(),
            f if self.is_image(f)       => Fixed(133).normal(),
            f if self.is_video(f)       => Fixed(135).normal(),
            f if self.is_music(f)       => Fixed(92).normal(),
            f if self.is_lossless(f)    => Fixed(93).normal(),
            f if self.is_crypto(f)      => Fixed(109).normal(),
            f if self.is_document(f)    => Fixed(105).normal(),
            f if self.is_compressed(f)  => Red.normal(),
            f if self.is_temp(f)        => Fixed(244).normal(),
            f if self.is_compiled(f)    => Fixed(137).normal(),
            _                           => return None,
        })
    }
}
<|endoftext|>


<commit_before>use anyhow::Result;
use std::io;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread;
use tree_sitter::Parser;

#[cfg(unix)]
use std::path::PathBuf;
#[cfg(unix)]
use std::process::{Child, ChildStdin, Command, Stdio};

#[cfg(unix)]
const HTML_HEADER: &[u8] = b"<!DOCTYPE html>\n<style>svg { width: 100%; }</style>\n\n";

pub fn cancel_on_stdin() -> Arc<AtomicUsize> {
    let result = Arc::new(AtomicUsize::new(0));
    if atty::is(atty::Stream::Stdin) {
        thread::spawn({
            let flag = result.clone();
            move || {
                let mut line = String::new();
                io::stdin().read_line(&mut line).unwrap();
                flag.store(1, Ordering::Relaxed);
            }
        });
    }
    result
}
#[cfg(windows)]
pub struct LogSession();

#[cfg(unix)]
pub struct LogSession(PathBuf, Option<Child>, Option<ChildStdin>);

#[cfg(windows)]
pub fn log_graphs(_parser: &mut Parser, _path: &str) -> Result<LogSession> {
    Ok(LogSession())
}

#[cfg(unix)]
pub fn log_graphs(parser: &mut Parser, path: &str) -> Result<LogSession> {
    use std::io::Write;

    let mut dot_file = std::fs::File::create(path)?;
    dot_file.write(HTML_HEADER)?;
    let mut dot_process = Command::new("dot")
        .arg("-Tsvg")
        .stdin(Stdio::piped())
        .stdout(dot_file)
        .spawn()
        .with_context(|| "Failed to run the `dot` command. Check that graphviz is installed.")?;
    let dot_stdin = dot_process
        .stdin
        .take()
        .ok_or_else(|| anyhow!("Failed to open stdin for `dot` process."))?;
    parser.print_dot_graphs(&dot_stdin);
    Ok(LogSession(
        PathBuf::from(path),
        Some(dot_process),
        Some(dot_stdin),
    ))
}

#[cfg(unix)]
impl Drop for LogSession {
    fn drop(&mut self) {
        use std::fs;

        drop(self.2.take().unwrap());
        let output = self.1.take().unwrap().wait_with_output().unwrap();
        if output.status.success() {
            if cfg!(target_os = "macos")
                && fs::metadata(&self.0).unwrap().len() > HTML_HEADER.len() as u64
            {
                Command::new("open").arg(&self.0).output().unwrap();
            }
        } else {
            eprintln!(
                "Dot failed: {} {}",
                String::from_utf8_lossy(&output.stdout),
                String::from_utf8_lossy(&output.stderr)
            );
        }
    }
}
<commit_msg>Fix linux build<commit_after>use anyhow::Result;
use std::io;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread;
use tree_sitter::Parser;

#[cfg(unix)]
use anyhow::{anyhow, Context};
#[cfg(unix)]
use std::path::PathBuf;
#[cfg(unix)]
use std::process::{Child, ChildStdin, Command, Stdio};

#[cfg(unix)]
const HTML_HEADER: &[u8] = b"<!DOCTYPE html>\n<style>svg { width: 100%; }</style>\n\n";

pub fn cancel_on_stdin() -> Arc<AtomicUsize> {
    let result = Arc::new(AtomicUsize::new(0));
    if atty::is(atty::Stream::Stdin) {
        thread::spawn({
            let flag = result.clone();
            move || {
                let mut line = String::new();
                io::stdin().read_line(&mut line).unwrap();
                flag.store(1, Ordering::Relaxed);
            }
        });
    }
    result
}
#[cfg(windows)]
pub struct LogSession();

#[cfg(unix)]
pub struct LogSession(PathBuf, Option<Child>, Option<ChildStdin>);

#[cfg(windows)]
pub fn log_graphs(_parser: &mut Parser, _path: &str) -> Result<LogSession> {
    Ok(LogSession())
}

#[cfg(unix)]
pub fn log_graphs(parser: &mut Parser, path: &str) -> Result<LogSession> {
    use std::io::Write;

    let mut dot_file = std::fs::File::create(path)?;
    dot_file.write(HTML_HEADER)?;
    let mut dot_process = Command::new("dot")
        .arg("-Tsvg")
        .stdin(Stdio::piped())
        .stdout(dot_file)
        .spawn()
        .with_context(|| "Failed to run the `dot` command. Check that graphviz is installed.")?;
    let dot_stdin = dot_process
        .stdin
        .take()
        .ok_or_else(|| anyhow!("Failed to open stdin for `dot` process."))?;
    parser.print_dot_graphs(&dot_stdin);
    Ok(LogSession(
        PathBuf::from(path),
        Some(dot_process),
        Some(dot_stdin),
    ))
}

#[cfg(unix)]
impl Drop for LogSession {
    fn drop(&mut self) {
        use std::fs;

        drop(self.2.take().unwrap());
        let output = self.1.take().unwrap().wait_with_output().unwrap();
        if output.status.success() {
            if cfg!(target_os = "macos")
                && fs::metadata(&self.0).unwrap().len() > HTML_HEADER.len() as u64
            {
                Command::new("open").arg(&self.0).output().unwrap();
            }
        } else {
            eprintln!(
                "Dot failed: {} {}",
                String::from_utf8_lossy(&output.stdout),
                String::from_utf8_lossy(&output.stderr)
            );
        }
    }
}
<|endoftext|>


<commit_before>use super::error::{Error, Result};
use super::generate::parse_grammar::GrammarJSON;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::path::Path;
use std::process::Command;

pub fn get_grammar_name(src_dir: &Path) -> Result<String> {
    let grammar_json_path = src_dir.join("grammar.json");
    let grammar_json = fs::read_to_string(&grammar_json_path).map_err(Error::wrap(|| {
        format!("Failed to read grammar file {:?}", grammar_json_path)
    }))?;
    let grammar: GrammarJSON = serde_json::from_str(&grammar_json).map_err(Error::wrap(|| {
        format!("Failed to parse grammar file {:?}", grammar_json_path)
    }))?;
    Ok(grammar.name)
}

pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Result<()> {
    let src_dir = language_dir.join("src");
    let grammar_name = get_grammar_name(&src_dir)?;
    let output_filename = format!("tree-sitter-{}.wasm", grammar_name);

    let mut command;
    if !force_docker && Command::new("emcc").output().is_ok() {
        command = Command::new("emcc");
        command.current_dir(&language_dir);
    } else {
        command = Command::new("docker");
        command.args(&["run", "--rm"]);

        // Mount the parser directory as a volume
        let mut volume_string;
        if let (Some(parent), Some(filename)) = (language_dir.parent(), language_dir.file_name()) {
            volume_string = OsString::from(parent);
            volume_string.push(":/src");
            command.arg("--workdir");
            command.arg(&Path::new("/src").join(filename));
        } else {
            volume_string = OsString::from(language_dir);
            volume_string.push(":/src");
            command.args(&["--workdir", "/src"]);
        }

        command.args(&[OsStr::new("--volume"), &volume_string]);

        // Get the current user id so that files created in the docker container will have
        // the same owner.
        let user_id_output = Command::new("id")
            .arg("-u")
            .output()
            .map_err(Error::wrap(|| "Failed to get get current user id {}"))?;
        let user_id = String::from_utf8_lossy(&user_id_output.stdout);
        let user_id = user_id.trim();
        command.args(&["--user", user_id]);

        // Run `emcc` in a container using the `emscripten-slim` image
        command.args(&["trzeci/emscripten-slim", "emcc"]);
    }

    command.args(&[
        "-o",
        &output_filename,
        "-Os",
        "-s",
        "WASM=1",
        "-s",
        "SIDE_MODULE=1",
        "-s",
        "TOTAL_MEMORY=33554432",
        "-s",
        &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar_name),
        "-fno-exceptions",
        "-I",
        "src",
    ]);

    // Find source files to pass to emscripten
    let src_entries = fs::read_dir(&src_dir).map_err(Error::wrap(|| {
        format!("Failed to read source directory {:?}", src_dir)
    }))?;

    for entry in src_entries {
        let entry = entry?;
        let file_name = entry.file_name();

        // Do not compile the node.js binding file.
        if file_name
            .to_str()
            .map_or(false, |s| s.starts_with("binding"))
        {
            continue;
        }

        // Compile any .c, .cc, or .cpp files
        if let Some(extension) = Path::new(&file_name).extension().and_then(|s| s.to_str()) {
            if extension == "c" || extension == "cc" || extension == "cpp" {
                command.arg(Path::new("src").join(entry.file_name()));
            }
        }
    }

    let output = command
        .output()
        .map_err(Error::wrap(|| "Failed to run emcc command"))?;
    if !output.status.success() {
        return Err(Error::from(format!(
            "emcc command failed - {}",
            String::from_utf8_lossy(&output.stderr)
        )));
    }

    // Move the created `.wasm` file into the current working directory.
    fs::rename(&language_dir.join(&output_filename), &output_filename).map_err(Error::wrap(
        || format!("Couldn't find output file {:?}", output_filename),
    ))?;

    Ok(())
}
<commit_msg>Fix problems with build-wasm command<commit_after>use super::error::{Error, Result};
use super::generate::parse_grammar::GrammarJSON;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::path::Path;
use std::process::Command;

pub fn get_grammar_name(src_dir: &Path) -> Result<String> {
    let grammar_json_path = src_dir.join("grammar.json");
    let grammar_json = fs::read_to_string(&grammar_json_path).map_err(Error::wrap(|| {
        format!("Failed to read grammar file {:?}", grammar_json_path)
    }))?;
    let grammar: GrammarJSON = serde_json::from_str(&grammar_json).map_err(Error::wrap(|| {
        format!("Failed to parse grammar file {:?}", grammar_json_path)
    }))?;
    Ok(grammar.name)
}

pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Result<()> {
    let src_dir = language_dir.join("src");
    let grammar_name = get_grammar_name(&src_dir)?;
    let output_filename = format!("tree-sitter-{}.wasm", grammar_name);

    let mut command;
    if !force_docker && Command::new("emcc").output().is_ok() {
        command = Command::new("emcc");
        command.current_dir(&language_dir);
    } else if Command::new("docker").output().is_ok() {
        command = Command::new("docker");
        command.args(&["run", "--rm"]);

        // Mount the parser directory as a volume
        let mut volume_string;
        if let (Some(parent), Some(filename)) = (language_dir.parent(), language_dir.file_name()) {
            volume_string = OsString::from(parent);
            volume_string.push(":/src");
            command.arg("--workdir");
            command.arg(&Path::new("/src").join(filename));
        } else {
            volume_string = OsString::from(language_dir);
            volume_string.push(":/src");
            command.args(&["--workdir", "/src"]);
        }

        command.args(&[OsStr::new("--volume"), &volume_string]);

        // Get the current user id so that files created in the docker container will have
        // the same owner.
        if cfg!(unix) {
            let user_id_output = Command::new("id")
                .arg("-u")
                .output()
                .map_err(Error::wrap(|| "Failed to get get current user id"))?;
            let user_id = String::from_utf8_lossy(&user_id_output.stdout);
            let user_id = user_id.trim();
            command.args(&["--user", user_id]);
        }

        // Run `emcc` in a container using the `emscripten-slim` image
        command.args(&["trzeci/emscripten-slim", "emcc"]);
    } else {
        return Error::err("You must have either emcc or docker on your PATH to run this command".to_string());
    }

    command.args(&[
        "-o",
        &output_filename,
        "-Os",
        "-s",
        "WASM=1",
        "-s",
        "SIDE_MODULE=1",
        "-s",
        "TOTAL_MEMORY=33554432",
        "-s",
        &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar_name),
        "-fno-exceptions",
        "-I",
        "src",
    ]);

    // Find source files to pass to emscripten
    let src_entries = fs::read_dir(&src_dir).map_err(Error::wrap(|| {
        format!("Failed to read source directory {:?}", src_dir)
    }))?;

    for entry in src_entries {
        let entry = entry?;
        let file_name = entry.file_name();

        // Do not compile the node.js binding file.
        if file_name
            .to_str()
            .map_or(false, |s| s.starts_with("binding"))
        {
            continue;
        }

        // Compile any .c, .cc, or .cpp files
        if let Some(extension) = Path::new(&file_name).extension().and_then(|s| s.to_str()) {
            if extension == "c" || extension == "cc" || extension == "cpp" {
                command.arg(Path::new("src").join(entry.file_name()));
            }
        }
    }

    let output = command
        .output()
        .map_err(Error::wrap(|| "Failed to run emcc command"))?;
    if !output.status.success() {
        return Err(Error::from(format!(
            "emcc command failed - {}",
            String::from_utf8_lossy(&output.stderr)
        )));
    }

    // Move the created `.wasm` file into the current working directory.
    fs::rename(&language_dir.join(&output_filename), &output_filename).map_err(Error::wrap(
        || format!("Couldn't find output file {:?}", output_filename),
    ))?;

    Ok(())
}
<|endoftext|>


<commit_before>//! An SDL2 backend for [Glium](https://github.com/tomaka/glium) - a high-level
//! OpenGL wrapper for the Rust language.
//!
//! # Example
//! ```no_run
//! # #[macro_use] extern crate glium;
//! # extern crate glium_sdl2;
//! # extern crate sdl2;
//! # fn main() {
//! use glium_sdl2::DisplayBuild;
//!
//! let mut sdl_context = sdl2::init().video().unwrap();
//!
//! let display = sdl_context.window("My window", 800, 600)
//!     .resizable()
//!     .build_glium()
//!     .unwrap();
//!
//! let mut running = true;
//!
//! while running {
//!     let mut target = display.draw();
//!     // do drawing here...
//!     target.finish();
//!
//!     // Event loop: includes all windows
//!
//!     for event in sdl_context.event_pump().poll_iter() {
//!         use sdl2::event::Event;
//!
//!         match event {
//!             Event::Quit { .. } => {
//!                 running = false;
//!             },
//!             _ => ()
//!         }
//!     }
//! }
//! # }
//! ```

extern crate libc;

extern crate glium;
extern crate sdl2;

use std::mem;
use std::cell::UnsafeCell;
use std::rc::Rc;

use glium::backend::{Backend, Context, Facade};
use sdl2::Sdl;
use sdl2::SdlResult;

pub type Display = SDL2Facade;

/// Facade implementation for an SDL2 window.
pub struct SDL2Facade {
    // contains everything related to the current context and its state
    context: Rc<Context>,

    backend: Rc<SDL2WindowBackend>,
}

impl Facade for SDL2Facade {
    fn get_context(&self) -> &Rc<Context> { &self.context }
}

impl SDL2Facade {
    /// Get the SDL Window ID.
    pub fn window_id(&self) -> u32 {
        self.backend.window().get_id()
    }

    /// Accesses the Window properties, such as the position, size and title of a Window.
    pub fn properties<'a>(&'a mut self, sdl: &'a Sdl) -> sdl2::video::WindowProperties<'a> {
        self.backend.window_mut().properties(sdl)
    }

    /// Immutably accesses the Window properties, such as the position, size and title of a Window.
    pub fn properties_getters(&self) -> sdl2::video::WindowPropertiesGetters {
        self.backend.window().properties_getters()
    }

    /// Start drawing on the backbuffer.
    ///
    /// This function returns a `Frame`, which can be used to draw on it.
    /// When the `Frame` is destroyed, the buffers are swapped.
    ///
    /// Note that destroying a `Frame` is immediate, even if vsync is enabled.
    pub fn draw(&self) -> glium::Frame {
        glium::Frame::new(self.context.clone(), self.backend.get_framebuffer_dimensions())
    }
}

/// An object that can build a facade object.
///
/// This trait is different from `glium::DisplayBuild` because Rust doesn't allow trait
/// implementations on types from external crates, unless the trait is in the same crate as the impl.
pub trait DisplayBuild {
    /// Build a context and a facade to draw on it.
    fn build_glium(self) -> Result<SDL2Facade, glium::GliumCreationError<String>>;

    /// Build a context and a facade to draw on it
    ///
    /// This function does the same as `build_glium`, except that the resulting context will assume
    /// that the current OpenGL context will never change.
    unsafe fn build_glium_unchecked(self) -> Result<SDL2Facade, glium::GliumCreationError<String>>;
}

impl<'a> DisplayBuild for &'a mut sdl2::video::WindowBuilder {
    fn build_glium(self) -> Result<SDL2Facade, glium::GliumCreationError<String>> {
        let backend = Rc::new(try!(SDL2WindowBackend::new(self)));
        let context = try!(unsafe { Context::new(backend.clone(), true) });

        let display = SDL2Facade {
            context: context,
            backend: backend
        };

        Ok(display)
    }

    unsafe fn build_glium_unchecked(self) -> Result<SDL2Facade, glium::GliumCreationError<String>> {
        let backend = Rc::new(try!(SDL2WindowBackend::new(self)));
        let context = try!(Context::new(backend.clone(), false));

        let display = SDL2Facade {
            context: context,
            backend: backend
        };

        Ok(display)
    }
}

pub struct SDL2WindowBackend {
    window: UnsafeCell<sdl2::video::Window>,
    context: sdl2::video::GLContext
}

impl SDL2WindowBackend {
    fn window(&self) -> &sdl2::video::Window {
        let ptr = self.window.get();
        unsafe { mem::transmute(ptr) }
    }

    fn window_mut(&self) -> &mut sdl2::video::Window {
        let ptr = self.window.get();
        unsafe { mem::transmute(ptr) }
    }

    pub fn new(window_builder: &mut sdl2::video::WindowBuilder) -> SdlResult<SDL2WindowBackend> {
        let window = try!(window_builder.opengl().build());
        let context = try!(window.gl_create_context());

        Ok(SDL2WindowBackend {
            window: UnsafeCell::new(window),
            context: context
        })
    }
}

unsafe impl Backend for SDL2WindowBackend {
    fn swap_buffers(&self) {
        self.window().gl_swap_window();
    }

    unsafe fn get_proc_address(&self, symbol: &str) -> *const libc::c_void {
        // Assumes the appropriate context for the window has been set before this call.

        sdl2::video::gl_get_proc_address(symbol)
    }

    fn get_framebuffer_dimensions(&self) -> (u32, u32) {
        let (width, height) = self.window().properties_getters().get_drawable_size();
        (width as u32, height as u32)
    }

    fn is_current(&self) -> bool {
        self.context.is_current()
    }

    unsafe fn make_current(&self) {
        self.window().gl_make_current(&self.context).unwrap()
    }
}
<commit_msg>Add an impl to make SDL2Facade Deref to Context<commit_after>//! An SDL2 backend for [Glium](https://github.com/tomaka/glium) - a high-level
//! OpenGL wrapper for the Rust language.
//!
//! # Example
//! ```no_run
//! # #[macro_use] extern crate glium;
//! # extern crate glium_sdl2;
//! # extern crate sdl2;
//! # fn main() {
//! use glium_sdl2::DisplayBuild;
//!
//! let mut sdl_context = sdl2::init().video().unwrap();
//!
//! let display = sdl_context.window("My window", 800, 600)
//!     .resizable()
//!     .build_glium()
//!     .unwrap();
//!
//! let mut running = true;
//!
//! while running {
//!     let mut target = display.draw();
//!     // do drawing here...
//!     target.finish();
//!
//!     // Event loop: includes all windows
//!
//!     for event in sdl_context.event_pump().poll_iter() {
//!         use sdl2::event::Event;
//!
//!         match event {
//!             Event::Quit { .. } => {
//!                 running = false;
//!             },
//!             _ => ()
//!         }
//!     }
//! }
//! # }
//! ```

extern crate libc;

extern crate glium;
extern crate sdl2;

use std::mem;
use std::cell::UnsafeCell;
use std::ops::Deref;
use std::rc::Rc;

use glium::backend::{Backend, Context, Facade};
use sdl2::Sdl;
use sdl2::SdlResult;

pub type Display = SDL2Facade;

/// Facade implementation for an SDL2 window.
pub struct SDL2Facade {
    // contains everything related to the current context and its state
    context: Rc<Context>,

    backend: Rc<SDL2WindowBackend>,
}

impl Facade for SDL2Facade {
    fn get_context(&self) -> &Rc<Context> { &self.context }
}

impl Deref for SDL2Facade {
    type Target = Context;

    fn deref(&self) -> &Context { &self.context }
}

impl SDL2Facade {
    /// Get the SDL Window ID.
    pub fn window_id(&self) -> u32 {
        self.backend.window().get_id()
    }

    /// Accesses the Window properties, such as the position, size and title of a Window.
    pub fn properties<'a>(&'a mut self, sdl: &'a Sdl) -> sdl2::video::WindowProperties<'a> {
        self.backend.window_mut().properties(sdl)
    }

    /// Immutably accesses the Window properties, such as the position, size and title of a Window.
    pub fn properties_getters(&self) -> sdl2::video::WindowPropertiesGetters {
        self.backend.window().properties_getters()
    }

    /// Start drawing on the backbuffer.
    ///
    /// This function returns a `Frame`, which can be used to draw on it.
    /// When the `Frame` is destroyed, the buffers are swapped.
    ///
    /// Note that destroying a `Frame` is immediate, even if vsync is enabled.
    pub fn draw(&self) -> glium::Frame {
        glium::Frame::new(self.context.clone(), self.backend.get_framebuffer_dimensions())
    }
}

/// An object that can build a facade object.
///
/// This trait is different from `glium::DisplayBuild` because Rust doesn't allow trait
/// implementations on types from external crates, unless the trait is in the same crate as the impl.
pub trait DisplayBuild {
    /// Build a context and a facade to draw on it.
    fn build_glium(self) -> Result<SDL2Facade, glium::GliumCreationError<String>>;

    /// Build a context and a facade to draw on it
    ///
    /// This function does the same as `build_glium`, except that the resulting context will assume
    /// that the current OpenGL context will never change.
    unsafe fn build_glium_unchecked(self) -> Result<SDL2Facade, glium::GliumCreationError<String>>;
}

impl<'a> DisplayBuild for &'a mut sdl2::video::WindowBuilder {
    fn build_glium(self) -> Result<SDL2Facade, glium::GliumCreationError<String>> {
        let backend = Rc::new(try!(SDL2WindowBackend::new(self)));
        let context = try!(unsafe { Context::new(backend.clone(), true) });

        let display = SDL2Facade {
            context: context,
            backend: backend
        };

        Ok(display)
    }

    unsafe fn build_glium_unchecked(self) -> Result<SDL2Facade, glium::GliumCreationError<String>> {
        let backend = Rc::new(try!(SDL2WindowBackend::new(self)));
        let context = try!(Context::new(backend.clone(), false));

        let display = SDL2Facade {
            context: context,
            backend: backend
        };

        Ok(display)
    }
}

pub struct SDL2WindowBackend {
    window: UnsafeCell<sdl2::video::Window>,
    context: sdl2::video::GLContext
}

impl SDL2WindowBackend {
    fn window(&self) -> &sdl2::video::Window {
        let ptr = self.window.get();
        unsafe { mem::transmute(ptr) }
    }

    fn window_mut(&self) -> &mut sdl2::video::Window {
        let ptr = self.window.get();
        unsafe { mem::transmute(ptr) }
    }

    pub fn new(window_builder: &mut sdl2::video::WindowBuilder) -> SdlResult<SDL2WindowBackend> {
        let window = try!(window_builder.opengl().build());
        let context = try!(window.gl_create_context());

        Ok(SDL2WindowBackend {
            window: UnsafeCell::new(window),
            context: context
        })
    }
}

unsafe impl Backend for SDL2WindowBackend {
    fn swap_buffers(&self) {
        self.window().gl_swap_window();
    }

    unsafe fn get_proc_address(&self, symbol: &str) -> *const libc::c_void {
        // Assumes the appropriate context for the window has been set before this call.

        sdl2::video::gl_get_proc_address(symbol)
    }

    fn get_framebuffer_dimensions(&self) -> (u32, u32) {
        let (width, height) = self.window().properties_getters().get_drawable_size();
        (width as u32, height as u32)
    }

    fn is_current(&self) -> bool {
        self.context.is_current()
    }

    unsafe fn make_current(&self) {
        self.window().gl_make_current(&self.context).unwrap()
    }
}
<|endoftext|>