<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Local state for known hashes and their external location (blob reference).

use blob;
use db;

use errors::{DieselError, RetryError};

use libsodium_sys;
use std::sync::{Arc, Mutex, MutexGuard};
use tags;
use util::UniquePriorityQueue;

pub mod tree;

#[cfg(test)]
mod tests;
#[cfg(all(test, feature = "benchmarks"))]
mod benchmarks;

pub struct HashIndex(InternalHashIndex);

/// A wrapper around Hash digests.
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
pub struct Hash {
    pub bytes: Vec<u8>,
}

impl Hash {
    /// Computes `hash(text)` and stores this digest as the `bytes` field in a new `Hash` structure.
    pub fn new(text: &[u8]) -> Hash {
        let digest_len = libsodium_sys::crypto_generichash_blake2b_BYTES_MAX;
        let mut digest = vec![0; digest_len];
        unsafe {
            libsodium_sys::crypto_generichash_blake2b(digest.as_mut_ptr(),
                                                      digest_len,
                                                      text.as_ptr(),
                                                      text.len() as u64,
                                                      vec![].as_ptr(),
                                                      0);
        }
        Hash { bytes: digest }
    }
}

/// An entry that can be inserted into the hash index.
#[derive(Clone)]
pub struct Entry {
    /// The hash of this entry (unique among all entries in the index).
    pub hash: Hash,

    /// The level in a hash tree that this entry is from. Level `0` represents `leaf`s, i.e. entries
    /// that represent user-data, where levels `1` and up represents `branches` of the tree,
    /// i.e. internal meta-data.
    pub level: i64,

    /// An optional list of child hash ids.
    pub childs: Option<Vec<i64>>,

    /// A reference to a location in the external persistent storage (a chunk reference) that
    /// contains the data for this entry (e.g. an object-name and a byte range).
    pub persistent_ref: Option<blob::ChunkRef>,
}

pub enum ReserveResult {
    HashKnown(i64),
    ReserveOk(i64),
}

type Queue = UniquePriorityQueue<i64, Vec<u8>, db::QueueEntry>;

pub struct InternalHashIndex {
    index: Arc<db::Index>,
    queue: Mutex<Queue>,
}

impl InternalHashIndex {
    fn new(index: Arc<db::Index>) -> Result<InternalHashIndex, DieselError> {
        Ok(InternalHashIndex {
            index: index,
            queue: Mutex::new(UniquePriorityQueue::new()),
        })
    }

    pub fn queue_lock(&self) -> MutexGuard<Queue> {
        self.queue.lock().expect("Hash queue mutex poisoned")
    }

    pub fn lock(&self) -> (MutexGuard<Queue>, db::IndexGuard) {
        let queue = self.queue_lock();
        let index = self.index.lock();
        (queue, index)
    }

    fn locate(&self,
              hash: &Hash,
              queue: &MutexGuard<Queue>,
              mut index: &mut db::IndexGuard)
              -> Option<db::QueueEntry> {
        let result_opt = queue.find_value_of_key(&hash.bytes).cloned();
        result_opt.or_else(|| index.hash_locate(hash))
    }

    fn reserve(&self,
               hash_entry: &Entry,
               mut queue: &mut MutexGuard<Queue>,
               mut index: &mut db::IndexGuard)
               -> i64 {
        index.maybe_flush();

        let Entry { ref hash, ref level, ref childs, ref persistent_ref } = *hash_entry;
        assert!(!hash.bytes.is_empty());

        let my_id = index.hash_next_id();
        assert!(queue.put_value(my_id,
                       hash.bytes.clone(),
                       db::QueueEntry {
                           id: my_id,
                           level: level.clone(),
                           childs: childs.clone(),
                           tag: None,
                           persistent_ref: persistent_ref.clone(),
                       })
            .is_ok());
        my_id
    }

    fn reserved_id(&self, hash_entry: &Entry, queue: &MutexGuard<Queue>) -> Option<i64> {
        queue.find_key(&hash_entry.hash.bytes).cloned()
    }

    fn update_reserved(&self,
                       hash_entry: Entry,
                       mut queue: &mut MutexGuard<Queue>,
                       mut index: &mut db::IndexGuard) {
        let id_opt = self.reserved_id(&hash_entry, queue);
        let Entry { hash, level, childs, persistent_ref } = hash_entry;
        assert!(!hash.bytes.is_empty());
        let old_entry = self.locate(&hash, queue, index).expect("hash was reserved");

        // If we didn't already commit and pop() the hash, update it:
        if id_opt.is_some() {
            assert_eq!(id_opt, Some(old_entry.id));
            queue.update_value(&hash.bytes, |qe| {
                qe.level = level;
                qe.childs = childs;
                qe.persistent_ref = persistent_ref;
            });
        }
    }

    fn insert_completed_in_order(&self,
                                 mut queue: &mut MutexGuard<Queue>,
                                 mut index: &mut db::IndexGuard) {
        loop {
            match queue.pop_min_if_complete() {
                None => break,
                Some((id_, hash_bytes, queue_entry)) => {
                    assert_eq!(id_, queue_entry.id);
                    index.hash_insert_new(id_, hash_bytes, queue_entry);
                }
            }
        }
        index.maybe_flush();
    }

    fn commit(&self,
              hash: &Hash,
              chunk_ref: blob::ChunkRef,
              mut queue: &mut MutexGuard<Queue>,
              mut index: &mut db::IndexGuard) {
        // Update persistent reference for ready hash
        let queue_entry = self.locate(hash, queue, index).expect("hash was committed");
        queue.update_value(&hash.bytes, |old_qe| {
            old_qe.persistent_ref = Some(chunk_ref);
        });
        queue.set_ready(&queue_entry.id);
        self.insert_completed_in_order(&mut queue, &mut index);
    }
}

impl HashIndex {
    pub fn new(index: Arc<db::Index>) -> Result<HashIndex, DieselError> {
        Ok(HashIndex(try!(InternalHashIndex::new(index))))
    }

    /// Locate the local ID of this hash.
    pub fn get_id(&self, hash: &Hash) -> Option<i64> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|entry| entry.id)
    }

    /// Locate hash entry from its ID.
    pub fn get_hash(&self, id: i64) -> Option<db::Entry> {
        self.0.index.lock().hash_locate_by_id(id)
    }

    /// Check whether this `Hash` already exists in the system.
    pub fn hash_exists(&self, hash: &Hash) -> bool {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).is_some()
    }

    /// Locate the local childs of the `Hash`.
    pub fn fetch_childs(&self, hash: &Hash) -> Option<Option<Vec<i64>>> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|queue_entry| queue_entry.childs)
    }

    /// Locate the persistent reference (external blob reference) for this `Hash`.
    pub fn fetch_persistent_ref(&self, hash: &Hash) -> Result<Option<blob::ChunkRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => Ok(Some(queue_entry.persistent_ref.expect("persistent_ref"))),
            None => Ok(None),
        }
    }

    /// Locate the hash reference (including persistent blob reference) for this `Hash~.
    pub fn fetch_hash_ref(&self, hash: &Hash) -> Result<Option<tree::HashRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => {
                Ok(Some(tree::HashRef {
                    hash: hash.clone(),
                    kind: blob::node_from_height(queue_entry.level),
                    persistent_ref: queue_entry.persistent_ref.expect("persistent_ref"),
                }))
            }
            None => Ok(None),
        }
    }

    /// Reserve a `Hash` in the index, while sending its content to external storage.
    /// This is used to ensure that each `Hash` is stored only once.
    pub fn reserve(&self, hash_entry: &Entry) -> ReserveResult {
        assert!(!hash_entry.hash.bytes.is_empty());
        // To avoid unused IO, we store entries in-memory until committed to persistent
        // storage. This allows us to continue after a crash without needing to scan
        // through and delete uncommitted entries.
        let (mut queue, mut index) = self.0.lock();
        match self.0.locate(&hash_entry.hash, &queue, &mut index) {
            Some(entry) => ReserveResult::HashKnown(entry.id),
            None => {
                let id = self.0.reserve(hash_entry, &mut queue, &mut index);
                ReserveResult::ReserveOk(id)
            }
        }
    }

    /// Check whether an entry was previously reserved.
    pub fn reserved_id(&self, hash_entry: &Entry) -> Option<i64> {
        let queue = self.0.queue_lock();
        self.0.reserved_id(hash_entry, &queue)
    }

    /// Update the info for a reserved `Hash`. The `Hash` remains reserved. This is used to update
    /// the persistent reference (external blob reference) as soon as it is available (to allow new
    /// references to the `Hash` to be created before it is committed).
    pub fn update_reserved(&self, hash_entry: Entry) {
        assert!(!hash_entry.hash.bytes.is_empty());
        let (mut queue, mut index) = self.0.lock();
        self.0.update_reserved(hash_entry, &mut queue, &mut index);
    }

    /// A `Hash` is committed when it has been `finalized` in the external storage. `Commit`
    /// includes the persistent reference that the content is available at.
    pub fn commit(&self, hash: &Hash, persistent_ref: blob::ChunkRef) {
        assert!(!hash.bytes.is_empty());
        let (mut queue, mut index) = self.0.lock();
        self.0.commit(hash, persistent_ref, &mut queue, &mut index);
    }

    /// List all hash entries.
    pub fn list(&self) -> Vec<db::Entry> {
        self.0.index.lock().hash_list()
    }

    /// Permanently delete hash by its ID.
    pub fn delete(&self, id: i64) {
        self.0.index.lock().hash_delete(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_tag(&self, id: i64, tag: tags::Tag) {
        {
            if let Some(ref mut q) = self.0.queue_lock().find_mut_value_of_priority(&id) {
                q.tag = Some(tag);
                return;
            }
        }
        self.0.index.lock().hash_set_tag(Some(id), tag);
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_all_tags(&self, tag: tags::Tag) {
        self.0.index.lock().hash_set_tag(None, tag)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_tag(&self, id: i64) -> Option<tags::Tag> {
        {
            if let Some(ref q) = self.0.queue_lock().find_mut_value_of_priority(&id) {
                return q.tag;
            }
        }
        self.0.index.lock().hash_get_tag(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_ids_by_tag(&self, tag: i64) -> Vec<i64> {
        self.0.index.lock().hash_list_ids_by_tag(tag)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn read_gc_data(&self, hash_id: i64, family_id: i64) -> db::GcData {
        self.0.index.lock().hash_read_gc_data(hash_id, family_id)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_gc_data<F: db::UpdateFn>(&self,
                                           hash_id: i64,
                                           family_id: i64,
                                           update_fn: F)
                                           -> db::GcData {
        self.0.index.lock().hash_update_gc_data(hash_id, family_id, update_fn)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_family_gc_data<F: db::UpdateFn, I: Iterator<Item = F>>(&self,
                                                                         family_id: i64,
                                                                         update_fns: I) {
        self.0.index.lock().hash_update_family_gc_data(family_id, update_fns)
    }

    /// Manual commit. This also disables automatic periodic commit.
    pub fn manual_commit(&self) {
        let mut guard = self.0.index.lock();
        guard.flush();
        guard.set_auto_flush(false);
    }

    /// Flush the hash index to clear internal buffers and commit the underlying database.
    pub fn flush(&self) {
        self.0.index.lock().flush()
    }
}
<commit_msg>Minor: Simplify<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Local state for known hashes and their external location (blob reference).

use blob;
use db;

use errors::{DieselError, RetryError};

use libsodium_sys;
use std::sync::{Arc, Mutex, MutexGuard};
use tags;
use util::UniquePriorityQueue;

pub mod tree;

#[cfg(test)]
mod tests;
#[cfg(all(test, feature = "benchmarks"))]
mod benchmarks;

pub struct HashIndex(InternalHashIndex);

/// A wrapper around Hash digests.
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
pub struct Hash {
    pub bytes: Vec<u8>,
}

impl Hash {
    /// Computes `hash(text)` and stores this digest as the `bytes` field in a new `Hash` structure.
    pub fn new(text: &[u8]) -> Hash {
        let digest_len = libsodium_sys::crypto_generichash_blake2b_BYTES_MAX;
        let mut digest = vec![0; digest_len];
        unsafe {
            libsodium_sys::crypto_generichash_blake2b(digest.as_mut_ptr(),
                                                      digest_len,
                                                      text.as_ptr(),
                                                      text.len() as u64,
                                                      vec![].as_ptr(),
                                                      0);
        }
        Hash { bytes: digest }
    }
}

/// An entry that can be inserted into the hash index.
#[derive(Clone)]
pub struct Entry {
    /// The hash of this entry (unique among all entries in the index).
    pub hash: Hash,

    /// The level in a hash tree that this entry is from. Level `0` represents `leaf`s, i.e. entries
    /// that represent user-data, where levels `1` and up represents `branches` of the tree,
    /// i.e. internal meta-data.
    pub level: i64,

    /// An optional list of child hash ids.
    pub childs: Option<Vec<i64>>,

    /// A reference to a location in the external persistent storage (a chunk reference) that
    /// contains the data for this entry (e.g. an object-name and a byte range).
    pub persistent_ref: Option<blob::ChunkRef>,
}

pub enum ReserveResult {
    HashKnown(i64),
    ReserveOk(i64),
}

type Queue = UniquePriorityQueue<i64, Vec<u8>, db::QueueEntry>;

pub struct InternalHashIndex {
    index: Arc<db::Index>,
    queue: Mutex<Queue>,
}

impl InternalHashIndex {
    fn new(index: Arc<db::Index>) -> Result<InternalHashIndex, DieselError> {
        Ok(InternalHashIndex {
            index: index,
            queue: Mutex::new(UniquePriorityQueue::new()),
        })
    }

    pub fn queue_lock(&self) -> MutexGuard<Queue> {
        self.queue.lock().expect("Hash queue mutex poisoned")
    }

    pub fn lock(&self) -> (MutexGuard<Queue>, db::IndexGuard) {
        let queue = self.queue_lock();
        let index = self.index.lock();
        (queue, index)
    }

    fn locate(&self,
              hash: &Hash,
              queue: &MutexGuard<Queue>,
              mut index: &mut db::IndexGuard)
              -> Option<db::QueueEntry> {
        let result_opt = queue.find_value_of_key(&hash.bytes).cloned();
        result_opt.or_else(|| index.hash_locate(hash))
    }

    fn reserve(&self,
               hash_entry: &Entry,
               mut queue: &mut MutexGuard<Queue>,
               mut index: &mut db::IndexGuard)
               -> i64 {
        index.maybe_flush();

        let Entry { ref hash, ref level, ref childs, ref persistent_ref } = *hash_entry;
        assert!(!hash.bytes.is_empty());

        let my_id = index.hash_next_id();
        assert!(queue.put_value(my_id,
                       hash.bytes.clone(),
                       db::QueueEntry {
                           id: my_id,
                           level: level.clone(),
                           childs: childs.clone(),
                           tag: None,
                           persistent_ref: persistent_ref.clone(),
                       })
            .is_ok());
        my_id
    }

    fn reserved_id(&self, hash_entry: &Entry, queue: &MutexGuard<Queue>) -> Option<i64> {
        queue.find_key(&hash_entry.hash.bytes).cloned()
    }

    fn update_reserved(&self,
                       hash_entry: Entry,
                       mut queue: &mut MutexGuard<Queue>,
                       mut index: &mut db::IndexGuard) {
        let id_opt = self.reserved_id(&hash_entry, queue);
        let Entry { hash, level, childs, persistent_ref } = hash_entry;
        assert!(!hash.bytes.is_empty());
        let old_entry = self.locate(&hash, queue, index).expect("hash was reserved");

        // If we didn't already commit and pop() the hash, update it:
        if id_opt.is_some() {
            assert_eq!(id_opt, Some(old_entry.id));
            queue.update_value(&hash.bytes, |qe| {
                qe.level = level;
                qe.childs = childs;
                qe.persistent_ref = persistent_ref;
            });
        }
    }

    fn insert_completed_in_order(&self,
                                 mut queue: &mut MutexGuard<Queue>,
                                 mut index: &mut db::IndexGuard) {
        while let Some((id_, hash_bytes, queue_entry)) = queue.pop_min_if_complete() {
            assert_eq!(id_, queue_entry.id);
            index.hash_insert_new(id_, hash_bytes, queue_entry);
        }
        index.maybe_flush();
    }

    fn commit(&self,
              hash: &Hash,
              chunk_ref: blob::ChunkRef,
              mut queue: &mut MutexGuard<Queue>,
              mut index: &mut db::IndexGuard) {
        // Update persistent reference for ready hash
        let queue_entry = self.locate(hash, queue, index).expect("hash was committed");
        queue.update_value(&hash.bytes, |old_qe| {
            old_qe.persistent_ref = Some(chunk_ref);
        });
        queue.set_ready(&queue_entry.id);
        self.insert_completed_in_order(&mut queue, &mut index);
    }
}

impl HashIndex {
    pub fn new(index: Arc<db::Index>) -> Result<HashIndex, DieselError> {
        Ok(HashIndex(try!(InternalHashIndex::new(index))))
    }

    /// Locate the local ID of this hash.
    pub fn get_id(&self, hash: &Hash) -> Option<i64> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|entry| entry.id)
    }

    /// Locate hash entry from its ID.
    pub fn get_hash(&self, id: i64) -> Option<db::Entry> {
        self.0.index.lock().hash_locate_by_id(id)
    }

    /// Check whether this `Hash` already exists in the system.
    pub fn hash_exists(&self, hash: &Hash) -> bool {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).is_some()
    }

    /// Locate the local childs of the `Hash`.
    pub fn fetch_childs(&self, hash: &Hash) -> Option<Option<Vec<i64>>> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|queue_entry| queue_entry.childs)
    }

    /// Locate the persistent reference (external blob reference) for this `Hash`.
    pub fn fetch_persistent_ref(&self, hash: &Hash) -> Result<Option<blob::ChunkRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => Ok(Some(queue_entry.persistent_ref.expect("persistent_ref"))),
            None => Ok(None),
        }
    }

    /// Locate the hash reference (including persistent blob reference) for this `Hash~.
    pub fn fetch_hash_ref(&self, hash: &Hash) -> Result<Option<tree::HashRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => {
                Ok(Some(tree::HashRef {
                    hash: hash.clone(),
                    kind: blob::node_from_height(queue_entry.level),
                    persistent_ref: queue_entry.persistent_ref.expect("persistent_ref"),
                }))
            }
            None => Ok(None),
        }
    }

    /// Reserve a `Hash` in the index, while sending its content to external storage.
    /// This is used to ensure that each `Hash` is stored only once.
    pub fn reserve(&self, hash_entry: &Entry) -> ReserveResult {
        assert!(!hash_entry.hash.bytes.is_empty());
        // To avoid unused IO, we store entries in-memory until committed to persistent
        // storage. This allows us to continue after a crash without needing to scan
        // through and delete uncommitted entries.
        let (mut queue, mut index) = self.0.lock();
        match self.0.locate(&hash_entry.hash, &queue, &mut index) {
            Some(entry) => ReserveResult::HashKnown(entry.id),
            None => {
                let id = self.0.reserve(hash_entry, &mut queue, &mut index);
                ReserveResult::ReserveOk(id)
            }
        }
    }

    /// Check whether an entry was previously reserved.
    pub fn reserved_id(&self, hash_entry: &Entry) -> Option<i64> {
        let queue = self.0.queue_lock();
        self.0.reserved_id(hash_entry, &queue)
    }

    /// Update the info for a reserved `Hash`. The `Hash` remains reserved. This is used to update
    /// the persistent reference (external blob reference) as soon as it is available (to allow new
    /// references to the `Hash` to be created before it is committed).
    pub fn update_reserved(&self, hash_entry: Entry) {
        assert!(!hash_entry.hash.bytes.is_empty());
        let (mut queue, mut index) = self.0.lock();
        self.0.update_reserved(hash_entry, &mut queue, &mut index);
    }

    /// A `Hash` is committed when it has been `finalized` in the external storage. `Commit`
    /// includes the persistent reference that the content is available at.
    pub fn commit(&self, hash: &Hash, persistent_ref: blob::ChunkRef) {
        assert!(!hash.bytes.is_empty());
        let (mut queue, mut index) = self.0.lock();
        self.0.commit(hash, persistent_ref, &mut queue, &mut index);
    }

    /// List all hash entries.
    pub fn list(&self) -> Vec<db::Entry> {
        self.0.index.lock().hash_list()
    }

    /// Permanently delete hash by its ID.
    pub fn delete(&self, id: i64) {
        self.0.index.lock().hash_delete(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_tag(&self, id: i64, tag: tags::Tag) {
        {
            if let Some(ref mut q) = self.0.queue_lock().find_mut_value_of_priority(&id) {
                q.tag = Some(tag);
                return;
            }
        }
        self.0.index.lock().hash_set_tag(Some(id), tag);
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_all_tags(&self, tag: tags::Tag) {
        self.0.index.lock().hash_set_tag(None, tag)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_tag(&self, id: i64) -> Option<tags::Tag> {
        {
            if let Some(ref q) = self.0.queue_lock().find_mut_value_of_priority(&id) {
                return q.tag;
            }
        }
        self.0.index.lock().hash_get_tag(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_ids_by_tag(&self, tag: i64) -> Vec<i64> {
        self.0.index.lock().hash_list_ids_by_tag(tag)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn read_gc_data(&self, hash_id: i64, family_id: i64) -> db::GcData {
        self.0.index.lock().hash_read_gc_data(hash_id, family_id)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_gc_data<F: db::UpdateFn>(&self,
                                           hash_id: i64,
                                           family_id: i64,
                                           update_fn: F)
                                           -> db::GcData {
        self.0.index.lock().hash_update_gc_data(hash_id, family_id, update_fn)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_family_gc_data<F: db::UpdateFn, I: Iterator<Item = F>>(&self,
                                                                         family_id: i64,
                                                                         update_fns: I) {
        self.0.index.lock().hash_update_family_gc_data(family_id, update_fns)
    }

    /// Manual commit. This also disables automatic periodic commit.
    pub fn manual_commit(&self) {
        let mut guard = self.0.index.lock();
        guard.flush();
        guard.set_auto_flush(false);
    }

    /// Flush the hash index to clear internal buffers and commit the underlying database.
    pub fn flush(&self) {
        self.0.index.lock().flush()
    }
}
<|endoftext|>


<commit_before>use {Delay, Network, NeuronId, Num, SynapseId, Timestep};

// XXX: Make configurable
const STDP_FIRE_RESET: Num = 0.1;
const STDP_DECAY: Num = 0.95;

pub struct Simulator {
    current_time_step: Timestep,

    // We use a cyclic buffer with (time_step % max_delay) as index into the futures_spike array
    future_spikes: Vec<Vec<SynapseId>>,
}

// NOTE: This must be a power of two
const MAX_DELAY: usize = 64;

#[inline(always)]
fn timeslot(at: Timestep) -> usize {
    (at & (MAX_DELAY - 1)) as usize
}

#[inline(always)]
fn timeslot_in_future(at: Timestep, delay: Delay) -> usize {
    timeslot(at + delay as usize)
}

impl Simulator {
    pub fn new(max_delay: usize) -> Simulator {
        assert!(max_delay < MAX_DELAY);
        Simulator {
            current_time_step: 0,
            future_spikes: (0..MAX_DELAY).map(|_| Vec::new()).collect(),
        }
    }

    pub fn current_time_step(&self) -> Timestep {
        self.current_time_step
    }

    /// External input currents have to be set manually by calling `set_external_input`.

    pub fn step<F>(&mut self, network: &mut Network, fired_callback: &mut F)
    where
        F: FnMut(NeuronId, Timestep),
    {
        let time_step = self.current_time_step;

        // Clear all input currents
        network.reset_all_input_currents();

        // get all synapse input
        {
            let spikes = &mut self.future_spikes[timeslot(time_step)];
            network.process_firing_synapses(spikes);
            spikes.clear();
        }

        network.update_state(
            STDP_FIRE_RESET,
            STDP_DECAY,
            &mut |syn_id, delay| {
                self.future_spikes[timeslot_in_future(time_step, delay)].push(syn_id);
            },
            &mut |neuron_id| {
                fired_callback(neuron_id, time_step);
            },
        );

        self.current_time_step += 1;
    }
}
<commit_msg>Get rid of constant MAX_DELAY. This is now being determined at Simulator startup time.<commit_after>use {Delay, Network, NeuronId, Num, SynapseId, Timestep};

// XXX: Make configurable
const STDP_FIRE_RESET: Num = 0.1;
const STDP_DECAY: Num = 0.95;

pub struct Simulator {
    current_time_step: Timestep,

    // We use a cyclic buffer with (time_step % max_delay) as index into the futures_spike array
    future_spikes: Vec<Vec<SynapseId>>,

    /// The next power of two of `max_delay` - 1. This is used as
    /// bit-wise AND mask, to avoid using the expensive modulo (%)
    /// operator.
    max_delay_bitwise_and_mask: usize,
}

impl Simulator {
    pub fn new(max_delay: usize) -> Simulator {
        assert!(max_delay > 1);
        let next_power_of_two = max_delay.checked_next_power_of_two().unwrap();
        assert!(next_power_of_two >= max_delay);
        let max_delay_bitwise_and_mask = next_power_of_two - 1;

        Simulator {
            current_time_step: 0,
            future_spikes: (0..next_power_of_two).map(|_| Vec::new()).collect(),
            max_delay_bitwise_and_mask,
        }
    }

    #[inline(always)]
    fn timeslot(&self, at: Timestep) -> usize {
        (at as usize) & self.max_delay_bitwise_and_mask
    }

    #[inline(always)]
    fn timeslot_in_future(&self, at: Timestep, delay: Delay) -> usize {
        self.timeslot(at + delay as usize)
    }

    pub fn current_time_step(&self) -> Timestep {
        self.current_time_step
    }

    /// External input currents have to be set manually by calling `set_external_input`.

    pub fn step<F>(&mut self, network: &mut Network, fired_callback: &mut F)
    where
        F: FnMut(NeuronId, Timestep),
    {
        let time_step = self.current_time_step;

        // Clear all input currents
        network.reset_all_input_currents();

        // get all synapse input
        {
            let idx = self.timeslot(time_step);
            let spikes = &mut self.future_spikes[idx];
            network.process_firing_synapses(spikes);
            spikes.clear();
        }

        network.update_state(
            STDP_FIRE_RESET,
            STDP_DECAY,
            &mut |syn_id, delay| {
                let idx = self.timeslot_in_future(time_step, delay);
                self.future_spikes[idx].push(syn_id);
            },
            &mut |neuron_id| {
                fired_callback(neuron_id, time_step);
            },
        );

        self.current_time_step += 1;
    }
}
<|endoftext|>


<commit_before>use {Errno, Result};
use libc::c_int;
use std::os::unix::io::RawFd;

mod ffi {
    use libc::{c_int};
    use super::EpollEvent;

    extern {
        pub fn epoll_create(size: c_int) -> c_int;
        pub fn epoll_ctl(epfd: c_int, op: c_int, fd: c_int, event: *const EpollEvent) -> c_int;
        pub fn epoll_wait(epfd: c_int, events: *mut EpollEvent, max_events: c_int, timeout: c_int) -> c_int;
    }
}

bitflags!(
    #[repr(C)]
    flags EpollEventKind: u32 {
        const EPOLLIN = 0x001,
        const EPOLLPRI = 0x002,
        const EPOLLOUT = 0x004,
        const EPOLLRDNORM = 0x040,
        const EPOLLRDBAND = 0x080,
        const EPOLLWRNORM = 0x100,
        const EPOLLWRBAND = 0x200,
        const EPOLLMSG = 0x400,
        const EPOLLERR = 0x008,
        const EPOLLHUP = 0x010,
        const EPOLLRDHUP = 0x2000,
        const EPOLLEXCLUSIVE = 1 << 28,
        const EPOLLWAKEUP = 1 << 29,
        const EPOLLONESHOT = 1 << 30,
        const EPOLLET = 1 << 31
    }
);

#[derive(Clone, Copy)]
#[repr(C)]
pub enum EpollOp {
    EpollCtlAdd = 1,
    EpollCtlDel = 2,
    EpollCtlMod = 3
}

#[cfg(not(target_arch = "x86_64"))]
#[derive(Clone, Copy)]
#[repr(C)]
pub struct EpollEvent {
    pub events: EpollEventKind,
    pub data: u64
}

#[cfg(target_arch = "x86_64")]
#[derive(Clone, Copy)]
#[repr(C, packed)]
pub struct EpollEvent {
    pub events: EpollEventKind,
    pub data: u64
}

#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[test]
fn test_epoll_event_size() {
    use std::mem::size_of;
    assert_eq!(size_of::<EpollEvent>(), 12);
}

#[cfg(target_arch = "arm")]
#[test]
fn test_epoll_event_size() {
    use std::mem::size_of;
    assert_eq!(size_of::<EpollEvent>(), 16);
}

#[inline]
pub fn epoll_create() -> Result<RawFd> {
    let res = unsafe { ffi::epoll_create(1024) };

    Errno::result(res)
}

#[inline]
pub fn epoll_ctl(epfd: RawFd, op: EpollOp, fd: RawFd, event: &EpollEvent) -> Result<()> {
    let res = unsafe { ffi::epoll_ctl(epfd, op as c_int, fd, event as *const EpollEvent) };

    Errno::result(res).map(drop)
}

#[inline]
pub fn epoll_wait(epfd: RawFd, events: &mut [EpollEvent], timeout_ms: isize) -> Result<usize> {
    let res = unsafe {
        ffi::epoll_wait(epfd, events.as_mut_ptr(), events.len() as c_int, timeout_ms as c_int)
    };

    Errno::result(res).map(|r| r as usize)
}
<commit_msg>Add epoll_create1 and set EPOLL_CLOEXEC flag by default.<commit_after>use {Errno, Result};
use libc::c_int;
use std::os::unix::io::RawFd;

mod ffi {
    use libc::{c_int};
    use super::EpollEvent;

    extern {
        pub fn epoll_create(size: c_int) -> c_int;
        pub fn epoll_create1(flags: c_int) -> c_int;
        pub fn epoll_ctl(epfd: c_int, op: c_int, fd: c_int, event: *const EpollEvent) -> c_int;
        pub fn epoll_wait(epfd: c_int, events: *mut EpollEvent, max_events: c_int, timeout: c_int) -> c_int;
    }
}

bitflags!(
    flags EpollFdFlag: c_int {
        const EPOLL_NONBLOCK = 0x800,
        const EPOLL_CLOEXEC = 0x80000
    }
);

bitflags!(
    #[repr(C)]
    flags EpollEventKind: u32 {
        const EPOLLIN = 0x001,
        const EPOLLPRI = 0x002,
        const EPOLLOUT = 0x004,
        const EPOLLRDNORM = 0x040,
        const EPOLLRDBAND = 0x080,
        const EPOLLWRNORM = 0x100,
        const EPOLLWRBAND = 0x200,
        const EPOLLMSG = 0x400,
        const EPOLLERR = 0x008,
        const EPOLLHUP = 0x010,
        const EPOLLRDHUP = 0x2000,
        const EPOLLEXCLUSIVE = 1 << 28,
        const EPOLLWAKEUP = 1 << 29,
        const EPOLLONESHOT = 1 << 30,
        const EPOLLET = 1 << 31
    }
);

#[derive(Clone, Copy)]
#[repr(C)]
pub enum EpollOp {
    EpollCtlAdd = 1,
    EpollCtlDel = 2,
    EpollCtlMod = 3
}

#[cfg(not(target_arch = "x86_64"))]
#[derive(Clone, Copy)]
#[repr(C)]
pub struct EpollEvent {
    pub events: EpollEventKind,
    pub data: u64
}

#[cfg(target_arch = "x86_64")]
#[derive(Clone, Copy)]
#[repr(C, packed)]
pub struct EpollEvent {
    pub events: EpollEventKind,
    pub data: u64
}

#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[test]
fn test_epoll_event_size() {
    use std::mem::size_of;
    assert_eq!(size_of::<EpollEvent>(), 12);
}

#[cfg(target_arch = "arm")]
#[test]
fn test_epoll_event_size() {
    use std::mem::size_of;
    assert_eq!(size_of::<EpollEvent>(), 16);
}

#[inline]
pub fn epoll_create() -> Result<RawFd> {
    let res = unsafe { ffi::epoll_create(1024) };

    Errno::result(res)
}

#[inline]
pub fn epoll_create1(flags: EpollFdFlag) -> Result<RawFd> {
    let res = unsafe { ffi::epoll_create1(flags.bits() | EPOLL_CLOEXEC.bits) };

    Errno::result(res)
}

#[inline]
pub fn epoll_ctl(epfd: RawFd, op: EpollOp, fd: RawFd, event: &EpollEvent) -> Result<()> {
    let res = unsafe { ffi::epoll_ctl(epfd, op as c_int, fd, event as *const EpollEvent) };

    Errno::result(res).map(drop)
}

#[inline]
pub fn epoll_wait(epfd: RawFd, events: &mut [EpollEvent], timeout_ms: isize) -> Result<usize> {
    let res = unsafe {
        ffi::epoll_wait(epfd, events.as_mut_ptr(), events.len() as c_int, timeout_ms as c_int)
    };

    Errno::result(res).map(|r| r as usize)
}
<|endoftext|>


<commit_before>/* TOOD: Implement for other kqueue based systems
 */

use {Error, Result};
use errno::Errno;
use libc::{timespec, time_t, c_int, c_long, uintptr_t};
use std::fmt;
use std::os::unix::io::RawFd;

pub use self::ffi::kevent as KEvent;

mod ffi {
    pub use libc::{c_int, c_void, uintptr_t, intptr_t, timespec};
    use super::{EventFilter, EventFlag, FilterFlag};

    #[derive(Clone, Copy)]
    #[repr(C)]
    pub struct kevent {
        pub ident: uintptr_t,       // 8
        pub filter: EventFilter,    // 2
        pub flags: EventFlag,       // 2
        pub fflags: FilterFlag,     // 4
        pub data: intptr_t,         // 8
        pub udata: usize             // 8
    }

    // Bug in rustc, cannot determine that kevent is #[repr(C)]
    #[allow(improper_ctypes)]
    extern {
        pub fn kqueue() -> c_int;

        pub fn kevent(
            kq: c_int,
            changelist: *const kevent,
            nchanges: c_int,
            eventlist: *mut kevent,
            nevents: c_int,
            timeout: *const timespec) -> c_int;
    }
}

#[repr(i16)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum EventFilter {
    EVFILT_READ = -1,
    EVFILT_WRITE = -2,
    EVFILT_AIO = -3,
    EVFILT_VNODE = -4,
    EVFILT_PROC = -5,
    EVFILT_SIGNAL = -6,
    EVFILT_TIMER = -7,
    EVFILT_MACHPORT = -8,
    EVFILT_FS = -9,
    EVFILT_USER = -10,
    // -11: unused
    EVFILT_VM = -12,
    EVFILT_SYSCOUNT = 13
}

bitflags!(
    flags EventFlag: u16 {
        const EV_ADD       = 0x0001,
        const EV_DELETE    = 0x0002,
        const EV_ENABLE    = 0x0004,
        const EV_DISABLE   = 0x0008,
        const EV_RECEIPT   = 0x0040,
        const EV_ONESHOT   = 0x0010,
        const EV_CLEAR     = 0x0020,
        const EV_DISPATCH  = 0x0080,
        const EV_SYSFLAGS  = 0xF000,
        const EV_FLAG0     = 0x1000,
        const EV_FLAG1     = 0x2000,
        const EV_EOF       = 0x8000,
        const EV_ERROR     = 0x4000
    }
);

impl fmt::Debug for EventFlag {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let mut one = false;
        let flags = [
            (EV_ADD, "EV_ADD"),
            (EV_DELETE, "EV_DELETE"),
            (EV_ENABLE, "EV_ENABLE"),
            (EV_DISABLE, "EV_DISABLE"),
            (EV_RECEIPT, "EV_RECEIPT"),
            (EV_ONESHOT, "EV_ONESHOT"),
            (EV_CLEAR, "EV_CLEAR"),
            (EV_DISPATCH, "EV_DISPATCH"),
            (EV_SYSFLAGS, "EV_SYSFLAGS"),
            (EV_FLAG0, "EV_FLAG0"),
            (EV_FLAG1, "EV_FLAG1"),
            (EV_EOF, "EV_EOF")];

        for &(flag, msg) in flags.iter() {
            if self.contains(flag) {
                if one { try!(write!(fmt, " | ")) }
                try!(write!(fmt, "{}", msg));

                one = true
            }
        }

        if !one {
            try!(write!(fmt, "<None>"));
        }

        Ok(())
    }
}

bitflags!(
    flags FilterFlag: u32 {
        const NOTE_TRIGGER                         = 0x01000000,
        const NOTE_FFNOP                           = 0x00000000,
        const NOTE_FFAND                           = 0x40000000,
        const NOTE_FFOR                            = 0x80000000,
        const NOTE_FFCOPY                          = 0xc0000000,
        const NOTE_FFCTRLMASK                      = 0xc0000000,
        const NOTE_FFLAGSMASK                      = 0x00ffffff,
        const NOTE_LOWAT                           = 0x00000001,
        const NOTE_DELETE                          = 0x00000001,
        const NOTE_WRITE                           = 0x00000002,
        const NOTE_EXTEND                          = 0x00000004,
        const NOTE_ATTRIB                          = 0x00000008,
        const NOTE_LINK                            = 0x00000010,
        const NOTE_RENAME                          = 0x00000020,
        const NOTE_REVOKE                          = 0x00000040,
        const NOTE_NONE                            = 0x00000080,
        const NOTE_EXIT                            = 0x80000000,
        const NOTE_FORK                            = 0x40000000,
        const NOTE_EXEC                            = 0x20000000,
        const NOTE_REAP                            = 0x10000000,
        const NOTE_SIGNAL                          = 0x08000000,
        const NOTE_EXITSTATUS                      = 0x04000000,
        const NOTE_RESOURCEEND                     = 0x02000000,
        const NOTE_APPACTIVE                       = 0x00800000,
        const NOTE_APPBACKGROUND                   = 0x00400000,
        const NOTE_APPNONUI                        = 0x00200000,
        const NOTE_APPINACTIVE                     = 0x00100000,
        const NOTE_APPALLSTATES                    = 0x00f00000,
        const NOTE_PDATAMASK                       = 0x000fffff,
        const NOTE_PCTRLMASK                       = 0xfff00000,
        const NOTE_EXIT_REPARENTED                 = 0x00080000,
        const NOTE_VM_PRESSURE                     = 0x80000000,
        const NOTE_VM_PRESSURE_TERMINATE           = 0x40000000,
        const NOTE_VM_PRESSURE_SUDDEN_TERMINATE    = 0x20000000,
        const NOTE_VM_ERROR                        = 0x10000000,
        const NOTE_SECONDS                         = 0x00000001,
        const NOTE_USECONDS                        = 0x00000002,
        const NOTE_NSECONDS                        = 0x00000004,
        const NOTE_ABSOLUTE                        = 0x00000008,
        const NOTE_TRACK                           = 0x00000001,
        const NOTE_TRACKERR                        = 0x00000002,
        const NOTE_CHILD                           = 0x00000004
    }
);

pub const EV_POLL: EventFlag = EV_FLAG0;
pub const EV_OOBAND: EventFlag = EV_FLAG1;

pub fn kqueue() -> Result<RawFd> {
    let res = unsafe { ffi::kqueue() };

    if res < 0 {
        return Err(Error::Sys(Errno::last()));
    }

    Ok(res)
}

pub fn kevent(kq: RawFd,
              changelist: &[KEvent],
              eventlist: &mut [KEvent],
              timeout_ms: usize) -> Result<usize> {

    // Convert ms to timespec
    let timeout = timespec {
        tv_sec: (timeout_ms / 1000) as time_t,
        tv_nsec: ((timeout_ms % 1000) * 1_000_000) as c_long
    };

    let res = unsafe {
        ffi::kevent(
            kq,
            changelist.as_ptr(),
            changelist.len() as c_int,
            eventlist.as_mut_ptr(),
            eventlist.len() as c_int,
            &timeout as *const timespec)
    };

    if res < 0 {
        return Err(Error::Sys(Errno::last()));
    }

    return Ok(res as usize)
}

#[inline]
pub fn ev_set(ev: &mut KEvent,
              ident: usize,
              filter: EventFilter,
              flags: EventFlag,
              fflags: FilterFlag,
              udata: usize) {

    ev.ident  = ident as uintptr_t;
    ev.filter = filter;
    ev.flags  = flags;
    ev.fflags = fflags;
    ev.data   = 0;
    ev.udata  = udata;
}
<commit_msg>remove osx-specific Debug impl<commit_after>/* TOOD: Implement for other kqueue based systems
 */

use {Error, Result};
use errno::Errno;
use libc::{timespec, time_t, c_int, c_long, uintptr_t};
use std::os::unix::io::RawFd;

pub use self::ffi::kevent as KEvent;

mod ffi {
    pub use libc::{c_int, c_void, uintptr_t, intptr_t, timespec};
    use super::{EventFilter, EventFlag, FilterFlag};

    #[derive(Clone, Copy)]
    #[repr(C)]
    pub struct kevent {
        pub ident: uintptr_t,       // 8
        pub filter: EventFilter,    // 2
        pub flags: EventFlag,       // 2
        pub fflags: FilterFlag,     // 4
        pub data: intptr_t,         // 8
        pub udata: usize             // 8
    }

    // Bug in rustc, cannot determine that kevent is #[repr(C)]
    #[allow(improper_ctypes)]
    extern {
        pub fn kqueue() -> c_int;

        pub fn kevent(
            kq: c_int,
            changelist: *const kevent,
            nchanges: c_int,
            eventlist: *mut kevent,
            nevents: c_int,
            timeout: *const timespec) -> c_int;
    }
}

#[repr(i16)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum EventFilter {
    EVFILT_READ = -1,
    EVFILT_WRITE = -2,
    EVFILT_AIO = -3,
    EVFILT_VNODE = -4,
    EVFILT_PROC = -5,
    EVFILT_SIGNAL = -6,
    EVFILT_TIMER = -7,
    EVFILT_MACHPORT = -8,
    EVFILT_FS = -9,
    EVFILT_USER = -10,
    // -11: unused
    EVFILT_VM = -12,
    EVFILT_SYSCOUNT = 13
}

bitflags!(
    flags EventFlag: u16 {
        const EV_ADD       = 0x0001,
        const EV_DELETE    = 0x0002,
        const EV_ENABLE    = 0x0004,
        const EV_DISABLE   = 0x0008,
        const EV_RECEIPT   = 0x0040,
        const EV_ONESHOT   = 0x0010,
        const EV_CLEAR     = 0x0020,
        const EV_DISPATCH  = 0x0080,
        const EV_SYSFLAGS  = 0xF000,
        const EV_FLAG0     = 0x1000,
        const EV_FLAG1     = 0x2000,
        const EV_EOF       = 0x8000,
        const EV_ERROR     = 0x4000
    }
);

bitflags!(
    flags FilterFlag: u32 {
        const NOTE_TRIGGER                         = 0x01000000,
        const NOTE_FFNOP                           = 0x00000000,
        const NOTE_FFAND                           = 0x40000000,
        const NOTE_FFOR                            = 0x80000000,
        const NOTE_FFCOPY                          = 0xc0000000,
        const NOTE_FFCTRLMASK                      = 0xc0000000,
        const NOTE_FFLAGSMASK                      = 0x00ffffff,
        const NOTE_LOWAT                           = 0x00000001,
        const NOTE_DELETE                          = 0x00000001,
        const NOTE_WRITE                           = 0x00000002,
        const NOTE_EXTEND                          = 0x00000004,
        const NOTE_ATTRIB                          = 0x00000008,
        const NOTE_LINK                            = 0x00000010,
        const NOTE_RENAME                          = 0x00000020,
        const NOTE_REVOKE                          = 0x00000040,
        const NOTE_NONE                            = 0x00000080,
        const NOTE_EXIT                            = 0x80000000,
        const NOTE_FORK                            = 0x40000000,
        const NOTE_EXEC                            = 0x20000000,
        const NOTE_REAP                            = 0x10000000,
        const NOTE_SIGNAL                          = 0x08000000,
        const NOTE_EXITSTATUS                      = 0x04000000,
        const NOTE_RESOURCEEND                     = 0x02000000,
        const NOTE_APPACTIVE                       = 0x00800000,
        const NOTE_APPBACKGROUND                   = 0x00400000,
        const NOTE_APPNONUI                        = 0x00200000,
        const NOTE_APPINACTIVE                     = 0x00100000,
        const NOTE_APPALLSTATES                    = 0x00f00000,
        const NOTE_PDATAMASK                       = 0x000fffff,
        const NOTE_PCTRLMASK                       = 0xfff00000,
        const NOTE_EXIT_REPARENTED                 = 0x00080000,
        const NOTE_VM_PRESSURE                     = 0x80000000,
        const NOTE_VM_PRESSURE_TERMINATE           = 0x40000000,
        const NOTE_VM_PRESSURE_SUDDEN_TERMINATE    = 0x20000000,
        const NOTE_VM_ERROR                        = 0x10000000,
        const NOTE_SECONDS                         = 0x00000001,
        const NOTE_USECONDS                        = 0x00000002,
        const NOTE_NSECONDS                        = 0x00000004,
        const NOTE_ABSOLUTE                        = 0x00000008,
        const NOTE_TRACK                           = 0x00000001,
        const NOTE_TRACKERR                        = 0x00000002,
        const NOTE_CHILD                           = 0x00000004
    }
);

pub const EV_POLL: EventFlag = EV_FLAG0;
pub const EV_OOBAND: EventFlag = EV_FLAG1;

pub fn kqueue() -> Result<RawFd> {
    let res = unsafe { ffi::kqueue() };

    if res < 0 {
        return Err(Error::Sys(Errno::last()));
    }

    Ok(res)
}

pub fn kevent(kq: RawFd,
              changelist: &[KEvent],
              eventlist: &mut [KEvent],
              timeout_ms: usize) -> Result<usize> {

    // Convert ms to timespec
    let timeout = timespec {
        tv_sec: (timeout_ms / 1000) as time_t,
        tv_nsec: ((timeout_ms % 1000) * 1_000_000) as c_long
    };

    let res = unsafe {
        ffi::kevent(
            kq,
            changelist.as_ptr(),
            changelist.len() as c_int,
            eventlist.as_mut_ptr(),
            eventlist.len() as c_int,
            &timeout as *const timespec)
    };

    if res < 0 {
        return Err(Error::Sys(Errno::last()));
    }

    return Ok(res as usize)
}

#[inline]
pub fn ev_set(ev: &mut KEvent,
              ident: usize,
              filter: EventFilter,
              flags: EventFlag,
              fflags: FilterFlag,
              udata: usize) {

    ev.ident  = ident as uintptr_t;
    ev.filter = filter;
    ev.flags  = flags;
    ev.fflags = fflags;
    ev.data   = 0;
    ev.udata  = udata;
}
<|endoftext|>


<commit_before>use std::io::{TcpListener, TcpStream};
use std::io::{Acceptor, Listener};
use std::string::String;
use data::Data;

pub struct Server
{
    port: String,
    ip: String
}

pub trait ServerFunction
{
    fn new(new_port: String) -> Server;
    fn start_server<T: Data>(&self, mut data_object: T);
    fn format_ip(&self) -> String;
    fn get_port(&self) -> String;
    fn get_ip(&self) -> String;
}

impl ServerFunction for Server
{
    fn new(new_port: String) -> Server
    {
        return Server{port: new_port, ip: String::from_str("localhost")};
    }

    fn start_server<T: Data>(&self, mut data_object: T)
    {
        let ip = self.format_ip();
        let listener = TcpListener::bind(ip.as_slice());
        let mut acceptor = listener.listen().unwrap();
        for opt_stream in acceptor.incoming()
        {
            let mut stream = opt_stream.unwrap();
            data_object.process_request_data(stream);
        }
    }

    fn format_ip(&self) -> String
    {
        let mut complete_ip = String::new();
        complete_ip.push_str(self.ip.as_slice());
        complete_ip.push_str(":");
        complete_ip.push_str(self.port.as_slice());
        return complete_ip;
    }

    fn get_port(&self) -> String
    {
        let mut port = String::new();
        port.push_str(self.port.as_slice());
        return port;
    }

    fn get_ip(&self) -> String
    {
        let mut ip = String::new();
        ip.push_str(self.ip.as_slice());
        return ip;
    }
}
<commit_msg>You can now change the serving ip<commit_after>use std::io::{TcpListener, TcpStream};
use std::io::{Acceptor, Listener};
use std::string::String;
use data::Data;

pub struct Server
{
    port: String,
    ip: String
}

pub trait ServerFunction
{
    fn new(new_port: String, new_ip: String) -> Server;
    fn start_server<T: Data>(&self, mut data_object: T);
    fn format_ip(&self) -> String;
    fn get_port(&self) -> String;
    fn get_ip(&self) -> String;
}

impl ServerFunction for Server
{
    fn new(new_port: String, new_ip: String) -> Server
    {
        return Server{port: new_port, ip: new_ip};
    }

    fn start_server<T: Data>(&self, mut data_object: T)
    {
        let ip = self.format_ip();
        let listener = TcpListener::bind(ip.as_slice());
        let mut acceptor = listener.listen().unwrap();
        for opt_stream in acceptor.incoming()
        {
            let mut stream = opt_stream.unwrap();
            data_object.process_request_data(stream);
        }
    }

    fn format_ip(&self) -> String
    {
        let mut complete_ip = String::new();
        complete_ip.push_str(self.ip.as_slice());
        complete_ip.push_str(":");
        complete_ip.push_str(self.port.as_slice());
        return complete_ip;
    }

    fn get_port(&self) -> String
    {
        let mut port = String::new();
        port.push_str(self.port.as_slice());
        return port;
    }

    fn get_ip(&self) -> String
    {
        let mut ip = String::new();
        ip.push_str(self.ip.as_slice());
        return ip;
    }
}
<|endoftext|>


<commit_before>use std::env;
use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};

use super::actor::Actor;
use super::{Result, expand, collapse};

lazy_static! {
    static ref CURRENT_DIR: PathBuf = {
        let path = fs::canonicalize(Path::new(file!()).parent().unwrap()).unwrap();
        println!("setting current dir to {}", path.display());
        env::set_current_dir(&path).unwrap();
        path
    };
}

macro_rules! test {
    ($name:ident $body:block) => {
        #[test]
        fn $name() {
            let _ = &*CURRENT_DIR;
            clear();
            $body
            clear();
        }
    }
}

fn clear() {
    for entry in fs::read_dir(".").unwrap().map(|e| e.unwrap()) {
        let path = entry.path();
        let path_str = path.to_str().unwrap();
        if path_str.contains('_') && path_str.ends_with(".m") {
            fs::remove_file(&path).unwrap();
        }
    }
}

fn with_backup<P1: AsRef<Path>, P2: AsRef<Path>, F: FnOnce()>(orig: P1, bak: P2, f: F) {
    println!("backing up {} to {}", orig.as_ref().display(), bak.as_ref().display());
    fs::copy(orig.as_ref(), bak.as_ref()).unwrap();
    f();
    println!("restoring {} to {}", bak.as_ref().display(), orig.as_ref().display());
    fs::rename(bak.as_ref(), orig.as_ref()).unwrap();
}

fn compare<P1: AsRef<Path>, P2: AsRef<Path>>(a: P1, b: P2, replace_dir: bool) {
    println!("comparing {:?} to {:?}", a.as_ref(), b.as_ref());
    let mut v1 = String::new();
    let mut v2 = String::new();
    File::open(a).unwrap().read_to_string(&mut v1).unwrap();
    File::open(b).unwrap().read_to_string(&mut v2).unwrap();
    if replace_dir {
        v1 = v1.replace(CURRENT_DIR.to_str().unwrap(), "{{dir}}");
    }
    assert_eq!(v1, v2);
}

fn try<T>(res: Result<T>) {
    match res {
        Ok(_) => {},
        Err(e) => panic!("{:?}", e),
    }
}

test!(expanding {
    try(expand(Actor::new(), "script.m", false));
    for part in &["_0", "a_1", "b_2", "if_true_3", "gen"] {
        compare(format!("script_{}.m", part),
                format!("script{}.m", part.replace('_', "")),
                true);
    }
});

test!(collapsing {
    with_backup("script.m", "script_backup.m", || {
        try(expand(Actor::new(), "script.m", false));
        try(collapse(Actor::new(), "script.m"));
        compare("script.m", "script_backup.m", false);
    });
});

<commit_msg>serialize tests<commit_after>use std::env;
use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::Mutex;

use super::actor::Actor;
use super::{Result, expand, collapse};

lazy_static! {
    static ref CURRENT_DIR: PathBuf = {
        let path = fs::canonicalize(Path::new(file!()).parent().unwrap()).unwrap();
        println!("setting current dir to {}", path.display());
        env::set_current_dir(&path).unwrap();
        path
    };
    static ref GIL: Mutex<()> = Mutex::new(());
}

macro_rules! test {
    ($name:ident $body:block) => {
        #[test]
        fn $name() {
            let _ = &*CURRENT_DIR;
            let _g = GIL.lock().unwrap();
            clear();
            $body
            clear();
        }
    }
}

fn clear() {
    for entry in fs::read_dir(".").unwrap().map(|e| e.unwrap()) {
        let path = entry.path();
        let path_str = path.to_str().unwrap();
        if path_str.contains('_') && path_str.ends_with(".m") {
            fs::remove_file(&path).unwrap();
        }
    }
}

fn with_backup<P1: AsRef<Path>, P2: AsRef<Path>, F: FnOnce()>(orig: P1, bak: P2, f: F) {
    println!("backing up {} to {}", orig.as_ref().display(), bak.as_ref().display());
    fs::copy(orig.as_ref(), bak.as_ref()).unwrap();
    f();
    println!("restoring {} to {}", bak.as_ref().display(), orig.as_ref().display());
    fs::rename(bak.as_ref(), orig.as_ref()).unwrap();
}

fn compare<P1: AsRef<Path>, P2: AsRef<Path>>(a: P1, b: P2, replace_dir: bool) {
    println!("comparing {:?} to {:?}", a.as_ref(), b.as_ref());
    let mut v1 = String::new();
    let mut v2 = String::new();
    File::open(a).unwrap().read_to_string(&mut v1).unwrap();
    File::open(b).unwrap().read_to_string(&mut v2).unwrap();
    if replace_dir {
        v1 = v1.replace(CURRENT_DIR.to_str().unwrap(), "{{dir}}");
    }
    assert_eq!(v1, v2);
}

fn try<T>(res: Result<T>) {
    match res {
        Ok(_) => {},
        Err(e) => panic!("{:?}", e),
    }
}

test!(expanding {
    try(expand(Actor::new(), "script.m", false));
    for part in &["_0", "a_1", "b_2", "if_true_3", "gen"] {
        compare(format!("script_{}.m", part),
                format!("script{}.m", part.replace('_', "")),
                true);
    }
});

test!(collapsing {
    with_backup("script.m", "script_backup.m", || {
        try(expand(Actor::new(), "script.m", false));
        try(collapse(Actor::new(), "script.m"));
        compare("script.m", "script_backup.m", false);
    });
});

<|endoftext|>


<commit_before>// FIXME: The doctest doesn't run on `cargo test`, why?
/// Splits string into a vector of words. A word is any sequence of alphanumeric
/// characters. Non-words get into the vector one and one.
///
/// ```rust
/// assert_eq!(tokenize("Adam, Bea"), ["Adam", ", ", " ", "Bea"]);
/// ```
pub fn tokenize(input: &str) -> Vec<&str> {
    let mut first_alphanumeric_index = 0;
    let mut last_was_alphanumeric = false;
    let mut result: Vec<&str> = Vec::new();
    for (index, character) in input.chars().enumerate() {
        let current_is_alphanumeric = character.is_alphanumeric();
        if current_is_alphanumeric {
            if last_was_alphanumeric {
                // Do nothing, keep working on the current word.
            } else {
                // Last was something else, we just started a new word.
                first_alphanumeric_index = index;
            }
        } else {
            // Current is something else
            if last_was_alphanumeric {
                // Push the word that just ended
                result.push(&input[first_alphanumeric_index..index]);
            }

            // Push current char
            result.push(&input[index..index + 1]);
        }

        last_was_alphanumeric = current_is_alphanumeric;
    }

    if last_was_alphanumeric {
        // Push the ending word
        result.push(&input[first_alphanumeric_index..]);
    }

    return result;
}

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(test)]
    use pretty_assertions::assert_eq;

    #[test]
    fn test_empty() {
        let no_strings: Vec<String> = Vec::new();
        assert_eq!(tokenize(""), no_strings);
    }

    #[test]
    fn test_words() {
        assert_eq!(tokenize("word"), ["word"]);
        assert_eq!(tokenize("Adam Bea"), ["Adam", " ", "Bea"]);
    }

    #[test]
    fn test_numbers() {
        assert_eq!(tokenize("123"), ["123"]);
        assert_eq!(tokenize("123 456"), ["123", " ", "456"]);
    }

    #[test]
    fn test_alphanumeric() {
        assert_eq!(tokenize("0xC0deCafe"), ["0xC0deCafe"]);
    }

    #[test]
    fn test_others() {
        assert_eq!(tokenize("+!,"), ["+", "!", ","]);
    }
}
<commit_msg>Test non breaking space<commit_after>// FIXME: The doctest doesn't run on `cargo test`, why?
/// Splits string into a vector of words. A word is any sequence of alphanumeric
/// characters. Non-words get into the vector one and one.
///
/// ```rust
/// assert_eq!(tokenize("Adam, Bea"), ["Adam", ", ", " ", "Bea"]);
/// ```
pub fn tokenize(input: &str) -> Vec<&str> {
    let mut first_alphanumeric_index = 0;
    let mut last_was_alphanumeric = false;
    let mut result: Vec<&str> = Vec::new();
    for (index, character) in input.chars().enumerate() {
        let current_is_alphanumeric = character.is_alphanumeric();
        if current_is_alphanumeric {
            if last_was_alphanumeric {
                // Do nothing, keep working on the current word.
            } else {
                // Last was something else, we just started a new word.
                first_alphanumeric_index = index;
            }
        } else {
            // Current is something else
            if last_was_alphanumeric {
                // Push the word that just ended
                result.push(&input[first_alphanumeric_index..index]);
            }

            // Push current char
            result.push(&input[index..index + 1]);
        }

        last_was_alphanumeric = current_is_alphanumeric;
    }

    if last_was_alphanumeric {
        // Push the ending word
        result.push(&input[first_alphanumeric_index..]);
    }

    return result;
}

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(test)]
    use pretty_assertions::assert_eq;

    #[test]
    fn test_empty() {
        let no_strings: Vec<String> = Vec::new();
        assert_eq!(tokenize(""), no_strings);
    }

    #[test]
    fn test_words() {
        assert_eq!(tokenize("word"), ["word"]);
        assert_eq!(tokenize("Adam Bea"), ["Adam", " ", "Bea"]);
    }

    #[test]
    fn test_numbers() {
        assert_eq!(tokenize("123"), ["123"]);
        assert_eq!(tokenize("123 456"), ["123", " ", "456"]);
    }

    #[test]
    fn test_alphanumeric() {
        assert_eq!(tokenize("0xC0deCafe"), ["0xC0deCafe"]);
    }

    #[test]
    fn test_others() {
        assert_eq!(tokenize("+!,"), ["+", "!", ","]);
    }

    #[test]
    fn test_non_breaking_space() {
        assert_eq!(tokenize("\u{00a0}"), ["\u{00a0}"]);
    }
}
<|endoftext|>


<commit_before>//! Utility module for lexer implementations,
//! providing types to help manage states and tokens.

use std::str::Chars;
use super::token::Token;
use super::token::Category;

/// A recursive function type used by lexers to manage their state.
/// Based on Rob Pike's "Lexical Scanning in Go" talk, these functions are
/// invoked in a call/return loop (letting the current function determine
/// the next) until a `None` value is returned, after which lexing is complete.
///
/// See the `lexers` module for examples.
pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store tokens for lexers.
pub struct Tokenizer<'a> {
    data: Chars<'a>,
    current_token: String,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let tokenizer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.chars(),
      current_token: String::new(),
      tokens: vec![],
      states: vec![]
    }
}

impl<'a> Tokenizer<'a> {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// tokenizer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Ensure that we're at the first character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the first character.
    /// tokenizer.advance();
    ///
    /// // Ensure that we're at the next character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        match self.data.next() {
            Some(c) => self.current_token.push(c),
            None => ()
        }
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("l");
    ///
    /// // Ensure that the current character is correct.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the last bit of data.
    /// tokenizer.advance();
    ///
    /// // Ensure that there is no current character.
    /// assert_eq!(tokenizer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        match self.data.clone().peekable().peek() {
            Some(c) => Some(c.clone()),
            None => None
        }
    }

    /// Whether or not the remaining data starts with the specified prefix.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("lex");
    ///
    /// assert!(tokenizer.has_prefix("le"));
    /// ```
    pub fn has_prefix(&self, prefix: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let mut data_iter = self.data.clone();

        // Check that the subject is a prefix, character by character.
        // This is much faster than building a string of equal length from
        // self.data and deferring to a straight string comparison using ==.
        prefix.chars().all(|c| {
            match data_iter.next() {
                Some(d) => c == d,
                None => false
            }
        })
    }

    /// Whether or not the remaining data starts with the specified lexeme.
    /// Ensures that the specified lexeme is not just a prefix by checking that
    /// the data that follows it is a newline, space, comma, or nothing at all.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("lex\nluthor lib,rary");
    ///
    /// // Prefixes don't count.
    /// assert!(!tokenizer.starts_with_lexeme("le"));
    ///
    /// // Newlines delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lex"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // Spaces delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("luthor"));
    ///
    /// // Consume 7 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(7, Category::Text);
    ///
    /// // Commas delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lib"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // End of string delineates lexemes.
    /// assert!(tokenizer.starts_with_lexeme("rary"));
    /// ```
    pub fn starts_with_lexeme(&self, lexeme: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let data_iter = self.data.clone();

        self.has_prefix(lexeme) && match data_iter.skip(lexeme.len()).next() {
            Some(' ') | Some('\n') | Some(',') => true,
            None => true,
            _ => false
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume two characters and then tokenize them.
    /// tokenizer.advance();
    /// tokenizer.advance();
    /// tokenizer.tokenize(Category::Text);
    ///
    /// // Ensure that we have a correctly-categorized token.
    /// assert_eq!(tokenizer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if !self.current_token.is_empty() {
            let token = Token{
                lexeme: self.current_token.clone(),
                category: category,
            };
            self.tokens.push(token);
            self.current_token = String::new();
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// `Category::Text` category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume one character, and then tokenize the next 5.
    /// tokenizer.advance();
    /// tokenizer.tokenize_next(5, Category::Keyword);
    ///
    /// // Ensure that we have two properly-categorized tokens.
    /// assert_eq!(
    ///     tokenizer.tokens()[0],
    ///     Token{ lexeme: "l".to_string(), category: Category::Text }
    /// );
    /// assert_eq!(
    ///     tokenizer.tokens()[1],
    ///     Token{ lexeme: "uthor".to_string(), category: Category::Keyword }
    /// );
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        // If there's any data that has yet
        // to be tokenized, take care of that.
        self.tokenize(Category::Text);

        // Mark the next series of characters.
        for _ in 0..amount { self.advance(); }

        // Tokenize the marked characters.
        self.tokenize(category);
    }

    /// Consumes consecutive whitespace characters as a single token.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// let mut tokenizer = luthor::tokenizer::new("  \nluthor");
    /// tokenizer.consume_whitespace();
    ///
    /// assert_eq!(
    ///     tokenizer.tokens()[0],
    ///     Token{ lexeme: "  \n".to_string(), category: Category::Whitespace }
    /// );
    /// ```
    pub fn consume_whitespace(&mut self) {
        let mut found_whitespace = false;
        loop {
            match self.current_char() {
                Some(' ') | Some('\n') => {
                    if !found_whitespace {
                        self.tokenize(Category::Text);
                        found_whitespace = true;
                    }

                    self.advance();
                },
                _ => {
                    if found_whitespace {
                        self.tokenize(Category::Whitespace);
                    }
                    return
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    #[test]
    fn current_char_returns_the_char_at_head() {
        let data = "él";
        let tokenizer = new(data);

        assert_eq!(tokenizer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let data = "él";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();

        assert_eq!(tokenizer.current_char(), None);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize(Category::Text);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.tokenize(Category::Text);

        assert_eq!(tokenizer.tokens.len(), 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(1, Category::Keyword);

        let token = tokenizer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(5, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(15, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn consume_whitespace_handles_preexisting_noncategorized_chars() {
        let data = "e  ";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.consume_whitespace();

        assert_eq!(
            tokenizer.tokens()[0],
            Token{ lexeme: "e".to_string(), category: Category::Text }
        );
        assert_eq!(
            tokenizer.tokens()[1],
            Token{ lexeme: "  ".to_string(), category: Category::Whitespace }
        );
    }
}
<commit_msg>Update tokens to append in-progress or remaining data as a text token.<commit_after>//! Utility module for lexer implementations,
//! providing types to help manage states and tokens.

use std::str::Chars;
use super::token::Token;
use super::token::Category;

/// A recursive function type used by lexers to manage their state.
/// Based on Rob Pike's "Lexical Scanning in Go" talk, these functions are
/// invoked in a call/return loop (letting the current function determine
/// the next) until a `None` value is returned, after which lexing is complete.
///
/// See the `lexers` module for examples.
pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store tokens for lexers.
pub struct Tokenizer<'a> {
    data: Chars<'a>,
    current_token: String,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let tokenizer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.chars(),
      current_token: String::new(),
      tokens: vec![],
      states: vec![]
    }
}

impl<'a> Tokenizer<'a> {
    /// Returns a copy of the tokens processed to date, in addition to any
    /// in-progress or remaining data appended as a text-category token.
    /// As a result, the returned tokens always produce the original dataset
    /// when their lexemes are concatenated.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::{Category, Token};
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.tokenize_next(2, Category::Keyword);
    ///
    /// assert_eq!(
    ///     tokenizer.tokens(),
    ///     vec![
    ///         Token{ lexeme: "lu".to_string(), category: Category::Keyword },
    ///         Token{ lexeme: "thor".to_string(), category: Category::Text }
    ///     ]
    /// );
    ///
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        let mut tokens = self.tokens.clone();

        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let data_iter = self.data.clone();

        // Append any remaining data to the in-progress token.
        let mut remaining_data = self.current_token.clone();
        for c in data_iter {
            remaining_data.push(c);
        }

        // If there was any remaining or in-progress data, add it as a text token.
        if !remaining_data.is_empty() {
            tokens.push(Token{ lexeme: remaining_data, category: Category::Text});
        }

        tokens
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Ensure that we're at the first character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the first character.
    /// tokenizer.advance();
    ///
    /// // Ensure that we're at the next character.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        match self.data.next() {
            Some(c) => self.current_token.push(c),
            None => ()
        }
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("l");
    ///
    /// // Ensure that the current character is correct.
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    ///
    /// // Consume the last bit of data.
    /// tokenizer.advance();
    ///
    /// // Ensure that there is no current character.
    /// assert_eq!(tokenizer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        match self.data.clone().peekable().peek() {
            Some(c) => Some(c.clone()),
            None => None
        }
    }

    /// Whether or not the remaining data starts with the specified prefix.
    ///
    /// # Examples
    ///
    /// ```
    /// // Set up a new tokenizer.
    /// let tokenizer = luthor::tokenizer::new("lex");
    ///
    /// assert!(tokenizer.has_prefix("le"));
    /// ```
    pub fn has_prefix(&self, prefix: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let mut data_iter = self.data.clone();

        // Check that the subject is a prefix, character by character.
        // This is much faster than building a string of equal length from
        // self.data and deferring to a straight string comparison using ==.
        prefix.chars().all(|c| {
            match data_iter.next() {
                Some(d) => c == d,
                None => false
            }
        })
    }

    /// Whether or not the remaining data starts with the specified lexeme.
    /// Ensures that the specified lexeme is not just a prefix by checking that
    /// the data that follows it is a newline, space, comma, or nothing at all.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("lex\nluthor lib,rary");
    ///
    /// // Prefixes don't count.
    /// assert!(!tokenizer.starts_with_lexeme("le"));
    ///
    /// // Newlines delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lex"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // Spaces delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("luthor"));
    ///
    /// // Consume 7 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(7, Category::Text);
    ///
    /// // Commas delineate lexemes.
    /// assert!(tokenizer.starts_with_lexeme("lib"));
    ///
    /// // Consume 4 characters, advancing to the next lexeme.
    /// tokenizer.tokenize_next(4, Category::Text);
    ///
    /// // End of string delineates lexemes.
    /// assert!(tokenizer.starts_with_lexeme("rary"));
    /// ```
    pub fn starts_with_lexeme(&self, lexeme: &str) -> bool {
        // Duplicate the tokenizer's character iterator so that we can
        // advance it to check for equality without affecting the original.
        let data_iter = self.data.clone();

        self.has_prefix(lexeme) && match data_iter.skip(lexeme.len()).next() {
            Some(' ') | Some('\n') | Some(',') => true,
            None => true,
            _ => false
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume two characters and then tokenize them.
    /// tokenizer.advance();
    /// tokenizer.advance();
    /// tokenizer.tokenize(Category::Text);
    ///
    /// // Ensure that we have a correctly-categorized token.
    /// assert_eq!(tokenizer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if !self.current_token.is_empty() {
            let token = Token{
                lexeme: self.current_token.clone(),
                category: category,
            };
            self.tokens.push(token);
            self.current_token = String::new();
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// `Category::Text` category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// // Set up a new tokenizer.
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    ///
    /// // Consume one character, and then tokenize the next 5.
    /// tokenizer.advance();
    /// tokenizer.tokenize_next(5, Category::Keyword);
    ///
    /// // Ensure that we have two properly-categorized tokens.
    /// assert_eq!(
    ///     tokenizer.tokens()[0],
    ///     Token{ lexeme: "l".to_string(), category: Category::Text }
    /// );
    /// assert_eq!(
    ///     tokenizer.tokens()[1],
    ///     Token{ lexeme: "uthor".to_string(), category: Category::Keyword }
    /// );
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        // If there's any data that has yet
        // to be tokenized, take care of that.
        self.tokenize(Category::Text);

        // Mark the next series of characters.
        for _ in 0..amount { self.advance(); }

        // Tokenize the marked characters.
        self.tokenize(category);
    }

    /// Consumes consecutive whitespace characters as a single token.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// let mut tokenizer = luthor::tokenizer::new("  \nluthor");
    /// tokenizer.consume_whitespace();
    ///
    /// assert_eq!(
    ///     tokenizer.tokens()[0],
    ///     Token{ lexeme: "  \n".to_string(), category: Category::Whitespace }
    /// );
    /// ```
    pub fn consume_whitespace(&mut self) {
        let mut found_whitespace = false;
        loop {
            match self.current_char() {
                Some(' ') | Some('\n') => {
                    if !found_whitespace {
                        self.tokenize(Category::Text);
                        found_whitespace = true;
                    }

                    self.advance();
                },
                _ => {
                    if found_whitespace {
                        self.tokenize(Category::Whitespace);
                    }
                    return
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    #[test]
    fn current_char_returns_the_char_at_head() {
        let data = "él";
        let tokenizer = new(data);

        assert_eq!(tokenizer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let data = "él";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();

        assert_eq!(tokenizer.current_char(), None);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize(Category::Text);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.tokenize(Category::Text);

        assert_eq!(tokenizer.tokens.len(), 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(1, Category::Keyword);

        let token = tokenizer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(5, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(15, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn consume_whitespace_handles_preexisting_noncategorized_chars() {
        let data = "e  ";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.consume_whitespace();

        assert_eq!(
            tokenizer.tokens()[0],
            Token{ lexeme: "e".to_string(), category: Category::Text }
        );
        assert_eq!(
            tokenizer.tokens()[1],
            Token{ lexeme: "  ".to_string(), category: Category::Whitespace }
        );
    }

    #[test]
    fn tokens_returns_unprocessed_data_as_text_token() {
        let tokenizer = new("luthor");

        assert_eq!(
            tokenizer.tokens()[0],
            Token{ lexeme: "luthor".to_string(), category: Category::Text }
        );
    }

    #[test]
    fn tokens_joins_advanced_data_with_unprocessed_data_as_text_token() {
        let mut tokenizer = new("luthor");
        tokenizer.advance();

        assert_eq!(
            tokenizer.tokens()[0],
            Token{ lexeme: "luthor".to_string(), category: Category::Text }
        );
    }
}
<|endoftext|>