<commit_before>use std::sync::atomic::{AtomicUsize, Ordering};
use std::ptr;

use ctxt::SemContext;
use gc::Address;
use gc::root::IndirectObj;
use gc::swiper::{CARD_SIZE, PROMOTION_AGE, Region};
use gc::swiper::card::CardTable;
use gc::swiper::crossing::CrossingMap;
use gc::swiper::old::OldGen;
use object::Obj;
use timer::{in_ms, Timer};

pub struct YoungGen {
    // bounds of from- & to-space
    total: Region,

    // address that separates from & to-space
    separator: Address,

    // address of next free memory
    free: AtomicUsize,

    // end of free memory (either separator or total.end)
    end: AtomicUsize,
}

impl YoungGen {
    pub fn new(young_start: Address, young_end: Address) -> YoungGen {
        let half_size = (young_end.to_usize() - young_start.to_usize()) / 2;
        let half_address = young_start.offset(half_size);

        YoungGen {
            total: Region::new(young_start, young_end),
            separator: half_address,
            free: AtomicUsize::new(young_start.to_usize()),
            end: AtomicUsize::new(half_address.to_usize()),
        }
    }

    pub fn alloc(&self, size: usize) -> *const u8 {
        let mut old = self.free.load(Ordering::Relaxed);
        let mut new;

        loop {
            new = old + size;

            if new >= self.end.load(Ordering::Relaxed) {
                return ptr::null();
            }

            let res =
                self.free
                    .compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed);

            match res {
                Ok(_) => break,
                Err(x) => old = x,
            }
        }

        old as *const u8
    }

    pub fn collect(
        &self,
        ctxt: &SemContext,
        rootset: Vec<IndirectObj>,
        card_table: &CardTable,
        crossing_map: &CrossingMap,
        old: &OldGen,
    ) {
        let mut timer = Timer::new(ctxt.args.flag_gc_events);

        let mut scan;
        let scan_end: Address = self.free.load(Ordering::SeqCst).into();

        let mut free;
        let new_end;

        if self.end.load(Ordering::Relaxed) == self.separator.to_usize() {
            scan = self.total.start;
            free = self.separator;
            new_end = self.total.end;

        } else {
            scan = self.separator;
            free = self.total.start;
            new_end = self.separator;
        }

        for &root in &rootset {
            let root_ptr = root.get();

            if self.total.includes(Address::from_ptr(root_ptr)) {
                root.set(copy(root_ptr, &mut free, old));
            }
        }

        // copy objects for old -> young references
        card_table.visit_dirty(|card| {
            let crossing_entry = crossing_map.get(card);

            // card contains: any data but no references, then first object
            if crossing_entry.is_first_object() {
                let card_address = crossing_map.address_of_card(card);

                // first_object() returns number of words before end of card
                let offset_from_end = crossing_entry.first_object() as usize * 8;
                let ptr = card_address.offset(CARD_SIZE - offset_from_end);

                let end = card_address.offset(CARD_SIZE);

                // copy all objects from this card
                copy_card(ptr, end, &mut free, &self.total, old);

            // card contains: references, then first object
            } else if crossing_entry.is_references_at_start() {
                let number_references = crossing_entry.references_at_start() as usize;
                let card_address = crossing_map.address_of_card(card);
                let mut ptr = card_address;

                for _ in 0..number_references {
                    let ind_ptr = IndirectObj::from_address(ptr);
                    let dir_ptr = ind_ptr.get();

                    if self.total.includes(Address::from_ptr(dir_ptr)) {
                        ind_ptr.set(copy(dir_ptr, &mut free, old));
                    }

                    ptr = ptr.offset(8);
                }

                let end = card_address.offset(CARD_SIZE);

                // copy all objects from this card
                copy_card(ptr, end, &mut free, &self.total, old);

            // object spans multiple cards
            } else if crossing_entry.is_previous_card() {
                unimplemented!();

            } else {
                assert!(crossing_entry.is_no_references());
                panic!("dirty card without references: can this ever happen?");
            }
        });

        while scan < scan_end {
            let object = unsafe { &mut *scan.to_mut_ptr::<Obj>() };

            object.visit_reference_fields(|child| {
                let child_ptr = child.get();

                if self.total.includes(Address::from_ptr(child_ptr)) {
                    child.set(copy(child_ptr, &mut free, old));
                }
            });

            scan = scan.offset(object.size());
        }

        // swap spaces
        self.end.store(new_end.to_usize(), Ordering::Relaxed);

        // update start of free memory
        self.free.store(free.to_usize(), Ordering::SeqCst);

        // memset from-space to garbage data for debug builds
        // makes sure that no pointer into from-space is left
        if cfg!(debug_assertions) {
            let start = if new_end == self.separator {
                self.total.start
            } else {
                self.separator
            };

            let size = self.separator.to_usize() - self.total.start.to_usize();

            unsafe {
                ptr::write_bytes(start.to_mut_ptr::<u8>(), 0xcc, size);
            }
        }

        timer.stop_with(|dur| {
            if ctxt.args.flag_gc_events {
                println!("GC minor: collect garbage ({} ms)", in_ms(dur));
            }
        });
    }
}

pub fn copy_card(mut ptr: Address, end: Address, mut free: &mut Address, young: &Region, old: &OldGen) {
    while ptr < end {
        let object = unsafe { &mut *ptr.to_mut_ptr::<Obj>() };

        object.visit_reference_fields(|child| {
            let child_ptr = child.get();

            if young.includes(Address::from_ptr(child_ptr)) {
                child.set(copy(child_ptr, &mut free, old));
            }
        });

        ptr = ptr.offset(object.size());
    }
}

pub fn copy(obj: *mut Obj, free: &mut Address, old: &OldGen) -> *mut Obj {
    let obj = unsafe { &mut *obj };
    let addr = get_forwarding_address(obj);

    if is_forwarding_address(addr) {
        unmark_forwarding_address(addr).to_mut_ptr::<Obj>()
    } else {
        let obj_size = obj.size();
        let age = obj.header_mut().increase_age();

        let addr: Address;

        // if object is old enough we copy it into the old generation
        if age >= PROMOTION_AGE {
            addr = Address::from_ptr(old.alloc(obj_size));
            copy_object(obj, addr, obj_size);

        // otherwise the object remains in the young generation for now
        } else {
            addr = *free;
            copy_object(obj, addr, obj_size);
            *free = addr.offset(obj_size);
        }

        set_forwarding_address(obj, addr);

        addr.to_mut_ptr::<Obj>()
    }
}

fn copy_object(obj: &Obj, addr: Address, size: usize) {
    unsafe {
        ptr::copy_nonoverlapping(obj as *const Obj as *const u8, addr.to_mut_ptr::<u8>(), size);
    }
}

pub fn is_forwarding_address(obj: Address) -> bool {
    (obj.to_usize() & 1) == 1
}

pub fn mark_forwarding_address(obj: Address) -> Address {
    (obj.to_usize() | 1).into()
}

pub fn unmark_forwarding_address(obj: Address) -> Address {
    (obj.to_usize() & !1).into()
}

pub fn get_forwarding_address(obj: &Obj) -> Address {
    unsafe { *(obj as *const Obj as *const Address) }
}

pub fn set_forwarding_address(obj: &mut Obj, addr: Address) {
    unsafe {
        *(obj as *mut Obj as *mut Address) = mark_forwarding_address(addr);
    }
}<commit_msg>added copy_dirty_cards<commit_after>use std::sync::atomic::{AtomicUsize, Ordering};
use std::ptr;

use ctxt::SemContext;
use gc::Address;
use gc::root::IndirectObj;
use gc::swiper::{CARD_SIZE, PROMOTION_AGE, Region};
use gc::swiper::card::CardTable;
use gc::swiper::crossing::CrossingMap;
use gc::swiper::old::OldGen;
use object::Obj;
use timer::{in_ms, Timer};

pub struct YoungGen {
    // bounds of from- & to-space
    total: Region,

    // address that separates from & to-space
    separator: Address,

    // address of next free memory
    free: AtomicUsize,

    // end of free memory (either separator or total.end)
    end: AtomicUsize,
}

impl YoungGen {
    pub fn new(young_start: Address, young_end: Address) -> YoungGen {
        let half_size = (young_end.to_usize() - young_start.to_usize()) / 2;
        let half_address = young_start.offset(half_size);

        YoungGen {
            total: Region::new(young_start, young_end),
            separator: half_address,
            free: AtomicUsize::new(young_start.to_usize()),
            end: AtomicUsize::new(half_address.to_usize()),
        }
    }

    pub fn alloc(&self, size: usize) -> *const u8 {
        let mut old = self.free.load(Ordering::Relaxed);
        let mut new;

        loop {
            new = old + size;

            if new >= self.end.load(Ordering::Relaxed) {
                return ptr::null();
            }

            let res =
                self.free
                    .compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed);

            match res {
                Ok(_) => break,
                Err(x) => old = x,
            }
        }

        old as *const u8
    }

    pub fn collect(
        &self,
        ctxt: &SemContext,
        rootset: Vec<IndirectObj>,
        card_table: &CardTable,
        crossing_map: &CrossingMap,
        old: &OldGen,
    ) {
        let mut timer = Timer::new(ctxt.args.flag_gc_events);

        let mut scan;
        let scan_end: Address = self.free.load(Ordering::SeqCst).into();

        let mut free;
        let new_end;

        if self.end.load(Ordering::Relaxed) == self.separator.to_usize() {
            scan = self.total.start;
            free = self.separator;
            new_end = self.total.end;

        } else {
            scan = self.separator;
            free = self.total.start;
            new_end = self.separator;
        }

        for &root in &rootset {
            let root_ptr = root.get();

            if self.total.includes(Address::from_ptr(root_ptr)) {
                root.set(copy(root_ptr, &mut free, old));
            }
        }

        copy_dirty_cards(card_table, crossing_map, &mut free, &self.total, old);

        while scan < scan_end {
            let object = unsafe { &mut *scan.to_mut_ptr::<Obj>() };

            object.visit_reference_fields(|child| {
                let child_ptr = child.get();

                if self.total.includes(Address::from_ptr(child_ptr)) {
                    child.set(copy(child_ptr, &mut free, old));
                }
            });

            scan = scan.offset(object.size());
        }

        // swap spaces
        self.end.store(new_end.to_usize(), Ordering::Relaxed);

        // update start of free memory
        self.free.store(free.to_usize(), Ordering::SeqCst);

        // memset from-space to garbage data for debug builds
        // makes sure that no pointer into from-space is left
        if cfg!(debug_assertions) {
            let start = if new_end == self.separator {
                self.total.start
            } else {
                self.separator
            };

            let size = self.separator.to_usize() - self.total.start.to_usize();

            unsafe {
                ptr::write_bytes(start.to_mut_ptr::<u8>(), 0xcc, size);
            }
        }

        timer.stop_with(|dur| {
            if ctxt.args.flag_gc_events {
                println!("GC minor: collect garbage ({} ms)", in_ms(dur));
            }
        });
    }
}

fn copy_dirty_cards(card_table: &CardTable, crossing_map: &CrossingMap, mut free: &mut Address, young: &Region, old: &OldGen) {
    // copy objects for old -> young references
    card_table.visit_dirty(|card| {
        let crossing_entry = crossing_map.get(card);

        // card contains: any data but no references, then first object
        if crossing_entry.is_first_object() {
            let card_address = crossing_map.address_of_card(card);

            // first_object() returns number of words before end of card
            let offset_from_end = crossing_entry.first_object() as usize * 8;
            let ptr = card_address.offset(CARD_SIZE - offset_from_end);

            let end = card_address.offset(CARD_SIZE);

            // copy all objects from this card
            copy_card(ptr, end, free, young, old);

        // card contains: references, then first object
        } else if crossing_entry.is_references_at_start() {
            let number_references = crossing_entry.references_at_start() as usize;
            let card_address = crossing_map.address_of_card(card);
            let mut ptr = card_address;

            for _ in 0..number_references {
                let ind_ptr = IndirectObj::from_address(ptr);
                let dir_ptr = ind_ptr.get();

                if young.includes(Address::from_ptr(dir_ptr)) {
                    ind_ptr.set(copy(dir_ptr, &mut free, old));
                }

                ptr = ptr.offset(8);
            }

            let end = card_address.offset(CARD_SIZE);

            // copy all objects from this card
            copy_card(ptr, end, free, young, old);

        // object spans multiple cards
        } else if crossing_entry.is_previous_card() {
            unimplemented!();

        } else {
            assert!(crossing_entry.is_no_references());
            panic!("dirty card without references: can this ever happen?");
        }
    });
}

fn copy_card(mut ptr: Address, end: Address, mut free: &mut Address, young: &Region, old: &OldGen) {
    while ptr < end {
        let object = unsafe { &mut *ptr.to_mut_ptr::<Obj>() };

        object.visit_reference_fields(|child| {
            let child_ptr = child.get();

            if young.includes(Address::from_ptr(child_ptr)) {
                child.set(copy(child_ptr, &mut free, old));
            }
        });

        ptr = ptr.offset(object.size());
    }
}

pub fn copy(obj: *mut Obj, free: &mut Address, old: &OldGen) -> *mut Obj {
    let obj = unsafe { &mut *obj };
    let addr = get_forwarding_address(obj);

    if is_forwarding_address(addr) {
        unmark_forwarding_address(addr).to_mut_ptr::<Obj>()
    } else {
        let obj_size = obj.size();
        let age = obj.header_mut().increase_age();

        let addr: Address;

        // if object is old enough we copy it into the old generation
        if age >= PROMOTION_AGE {
            addr = Address::from_ptr(old.alloc(obj_size));
            copy_object(obj, addr, obj_size);

        // otherwise the object remains in the young generation for now
        } else {
            addr = *free;
            copy_object(obj, addr, obj_size);
            *free = addr.offset(obj_size);
        }

        set_forwarding_address(obj, addr);

        addr.to_mut_ptr::<Obj>()
    }
}

fn copy_object(obj: &Obj, addr: Address, size: usize) {
    unsafe {
        ptr::copy_nonoverlapping(obj as *const Obj as *const u8, addr.to_mut_ptr::<u8>(), size);
    }
}

pub fn is_forwarding_address(obj: Address) -> bool {
    (obj.to_usize() & 1) == 1
}

pub fn mark_forwarding_address(obj: Address) -> Address {
    (obj.to_usize() | 1).into()
}

pub fn unmark_forwarding_address(obj: Address) -> Address {
    (obj.to_usize() & !1).into()
}

pub fn get_forwarding_address(obj: &Obj) -> Address {
    unsafe { *(obj as *const Obj as *const Address) }
}

pub fn set_forwarding_address(obj: &mut Obj, addr: Address) {
    unsafe {
        *(obj as *mut Obj as *mut Address) = mark_forwarding_address(addr);
    }
}<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 Robert Clipsham <robert@octarineparrot.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Support for sending and receiving data link layer packets using the netmap library

#![allow(bad_style)]

extern crate netmap_sys;
extern crate libc;

use self::netmap_sys::netmap_user::{NETMAP_BUF, NETMAP_FD, NETMAP_TXRING, nm_close, nm_desc,
                                    nm_nextpkt, nm_open, nm_pkthdr, nm_ring_next};
use self::netmap_sys::netmap::{netmap_slot, nm_ring_empty};

use std::ffi::CString;
use std::path::Path;
use std::fs::File;
use std::io;
use std::io::Read;
use std::mem;
use std::ptr;
use std::slice;
use std::sync::Arc;

use datalink::{self, NetworkInterface};
use datalink::Channel::Ethernet;
use datalink::{EthernetDataLinkChannelIterator, EthernetDataLinkReceiver, EthernetDataLinkSender};
use packet::Packet;
use packet::ethernet::{EthernetPacket, MutableEthernetPacket};

#[cfg(any(target_os = "linux", target_os = "freebsd"))]
#[repr(C)]
struct pollfd {
    fd: libc::c_int,
    events: libc::c_short,
    revents: libc::c_short,
}

#[cfg(any(target_os = "linux", target_os = "freebsd"))]
const POLLIN: libc::c_short = 0x0001;
#[cfg(any(target_os = "linux", target_os = "freebsd"))]
const POLLOUT: libc::c_short = 0x0004;

#[cfg(target_os = "freebsd")]
type nfds_t = libc::c_uint;
#[cfg(target_os = "linux")]
type nfds_t = libc::c_ulong;

extern {
    fn poll(fds: *mut pollfd, nfds: nfds_t, timeout: libc::c_int) -> libc::c_int;
    fn ppoll(fds: *mut pollfd, nfds: nfds_t, timeout: *const libc::timespec, newsigmask: *const libc::sigset_t) -> libc::c_int;
}

struct NmDesc {
    desc: *mut nm_desc,
    buf_size: libc::c_uint,
}

unsafe impl Send for NmDesc {}
unsafe impl Sync for NmDesc {}

impl NmDesc {
    fn new(iface: &NetworkInterface) -> io::Result<NmDesc> {
        let ifname = CString::new(("netmap:".to_owned() + &iface.name[..]).as_bytes());
        let desc = unsafe { nm_open(ifname.unwrap().as_ptr(), ptr::null(), 0, ptr::null()) };

        if desc.is_null() {
            Err(io::Error::last_os_error())
        } else {
            let mut f = try!(File::open(&Path::new("/sys/module/netmap/parameters/buf_size")));
            let mut num_str = String::new();
            try!(f.read_to_string(&mut num_str));
            let buf_size = num_str.trim_right().parse().unwrap();

            Ok(NmDesc {
                desc: desc,
                buf_size: buf_size,
            })
        }
    }
}

impl Drop for NmDesc {
    fn drop(&mut self) {
        unsafe {
            nm_close(self.desc);
        }
    }
}

/// Netmap specific configuration
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Config {
    /// The read timeout. Defaults to None.
    pub read_timeout: Option<Duration>,

    /// The write timeout. Defaults to None.
    pub write_timeout: Option<Duration>,
}

impl<'a> From<&'a datalink::Config> for Config {
    fn from(_config: &datalink::Config) -> Config {
        Config {
            read_timeout: config.read_timeout,
            write_timeout: config.write_timeout,
        }
    }
}

impl Default for Config {
    fn default() -> Config {
        Config {
            read_timeout: None,
            write_timeout: None,
        }
    }
}

#[inline]
fn get_timeout(to: Option<Duration>) -> *const libc::timespec {
    match to {
        Some(to) => &libc::timespec {
            tv_sec: to.as_secs() as libc::time_t,
            tv_nsec: to.subsec_nanos() as libc::c_long
        } as *const libc::timespec,
        None => ptr::null()
    }
}

/// Create a datalink channel using the netmap library
#[inline]
pub fn channel(network_interface: &NetworkInterface, config: Config)
    -> io::Result<datalink::Channel> {
    // FIXME probably want one for each of send/recv
    let desc = NmDesc::new(network_interface);
    match desc {
        Ok(desc) => {
            let arc = Arc::new(desc);

            Ok(Ethernet(
                Box::new(DataLinkSenderImpl {
                    desc: arc.clone(),
                    timeout: get_timeout(config.write_timeout)
                }),
                Box::new(DataLinkReceiverImpl {
                    desc: arc,
                    timeout: get_timeout(config.read_timeout)
                })))
        }
        Err(e) => Err(e),
    }
}

struct DataLinkSenderImpl {
    desc: Arc<NmDesc>,
    timeout: *const libc::timespec,
}

impl EthernetDataLinkSender for DataLinkSenderImpl {
    #[inline]
    fn build_and_send(&mut self,
                      num_packets: usize,
                      packet_size: usize,
                      func: &mut FnMut(MutableEthernetPacket))
        -> Option<io::Result<()>> {
        assert!(packet_size <= self.desc.buf_size as usize);
        let desc = self.desc.desc;
        let mut fds = pollfd {
            fd: unsafe { NETMAP_FD(desc) },
            events: POLLOUT,
            revents: 0,
        };
        let mut packet_idx = 0usize;
        while packet_idx < num_packets {
            unsafe {
                if ppoll(&mut fds, 1, self.timeout, ptr::null()) < 0 {
                    return Some(Err(io::Error::last_os_error()));
                }
                let ring = NETMAP_TXRING((*desc).nifp, 0);
                while !nm_ring_empty(ring) && packet_idx < num_packets {
                    let i = (*ring).cur;
                    let slot_ptr: *mut netmap_slot = mem::transmute(&mut (*ring).slot);
                    let buf = NETMAP_BUF(ring, (*slot_ptr.offset(i as isize)).buf_idx as isize);
                    let slice = slice::from_raw_parts_mut(buf as *mut u8, packet_size);
                    let meh = MutableEthernetPacket::new(slice).unwrap();
                    (*slot_ptr.offset(i as isize)).len = packet_size as u16;
                    func(meh);
                    let next = nm_ring_next(ring, i);
                    (*ring).head = next;
                    (*ring).cur = next;
                    packet_idx += 1;
                }
            }
        }

        Some(Ok(()))
    }

    #[inline]
    fn send_to(&mut self,
               packet: &EthernetPacket,
               _dst: Option<NetworkInterface>)
        -> Option<io::Result<()>> {
        use packet::MutablePacket;
        self.build_and_send(1,
                            packet.packet().len(),
                            &mut |mut eh: MutableEthernetPacket| {
                                eh.clone_from(packet);
                            })
    }
}

struct DataLinkReceiverImpl {
    desc: Arc<NmDesc>,
    timeout: *const libc::timespec,
}

impl EthernetDataLinkReceiver for DataLinkReceiverImpl {
    // FIXME Layer 3
    fn iter<'a>(&'a mut self) -> Box<EthernetDataLinkChannelIterator + 'a> {
        Box::new(DataLinkChannelIteratorImpl { pc: self })
    }
}

struct DataLinkChannelIteratorImpl<'a> {
    pc: &'a mut DataLinkReceiverImpl,
}

impl<'a> EthernetDataLinkChannelIterator<'a> for DataLinkChannelIteratorImpl<'a> {
    fn next(&mut self) -> io::Result<EthernetPacket> {
        let desc = self.pc.desc.desc;
        let mut h: nm_pkthdr = unsafe { mem::uninitialized() };
        let mut buf = unsafe { nm_nextpkt(desc, &mut h) };
        if buf.is_null() {
            let mut fds = pollfd {
                fd: unsafe { NETMAP_FD(desc) },
                events: POLLIN,
                revents: 0,
            };
            if unsafe { ppoll(&mut fds, 1, self.timeout, ptr::null()) } < 0 {
                return Err(io::Error::last_os_error());
            }
            buf = unsafe { nm_nextpkt(desc, &mut h) };
        }
        Ok(EthernetPacket::new(unsafe {
            slice::from_raw_parts(buf, h.len as usize)
        }).unwrap())
    }
}

/// Get a list of available network interfaces for the current machine.
pub fn interfaces() -> Vec<NetworkInterface> {
    #[path = "unix_interfaces.rs"]
    mod interfaces;
    interfaces::interfaces()
}
<commit_msg>Auto merge of #220 - mbudde:fix-netmap-timeout, r=mrmonday<commit_after>// Copyright (c) 2015-2016 Robert Clipsham <robert@octarineparrot.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Support for sending and receiving data link layer packets using the netmap library

#![allow(bad_style)]

extern crate netmap_sys;
extern crate libc;

use self::netmap_sys::netmap_user::{NETMAP_BUF, NETMAP_FD, NETMAP_TXRING, nm_close, nm_desc,
                                    nm_nextpkt, nm_open, nm_pkthdr, nm_ring_next};
use self::netmap_sys::netmap::{netmap_slot, nm_ring_empty};

use std::ffi::CString;
use std::path::Path;
use std::fs::File;
use std::io;
use std::io::Read;
use std::mem;
use std::ptr;
use std::slice;
use std::sync::Arc;

use datalink::{self, NetworkInterface};
use datalink::Channel::Ethernet;
use datalink::{EthernetDataLinkChannelIterator, EthernetDataLinkReceiver, EthernetDataLinkSender};
use packet::Packet;
use packet::ethernet::{EthernetPacket, MutableEthernetPacket};

#[cfg(any(target_os = "linux", target_os = "freebsd"))]
#[repr(C)]
struct pollfd {
    fd: libc::c_int,
    events: libc::c_short,
    revents: libc::c_short,
}

#[cfg(any(target_os = "linux", target_os = "freebsd"))]
const POLLIN: libc::c_short = 0x0001;
#[cfg(any(target_os = "linux", target_os = "freebsd"))]
const POLLOUT: libc::c_short = 0x0004;

#[cfg(target_os = "freebsd")]
type nfds_t = libc::c_uint;
#[cfg(target_os = "linux")]
type nfds_t = libc::c_ulong;

extern {
    fn poll(fds: *mut pollfd, nfds: nfds_t, timeout: libc::c_int) -> libc::c_int;
    fn ppoll(fds: *mut pollfd, nfds: nfds_t, timeout: *const libc::timespec, newsigmask: *const libc::sigset_t) -> libc::c_int;
}

struct NmDesc {
    desc: *mut nm_desc,
    buf_size: libc::c_uint,
}

unsafe impl Send for NmDesc {}
unsafe impl Sync for NmDesc {}

impl NmDesc {
    fn new(iface: &NetworkInterface) -> io::Result<NmDesc> {
        let ifname = CString::new(("netmap:".to_owned() + &iface.name[..]).as_bytes());
        let desc = unsafe { nm_open(ifname.unwrap().as_ptr(), ptr::null(), 0, ptr::null()) };

        if desc.is_null() {
            Err(io::Error::last_os_error())
        } else {
            let mut f = try!(File::open(&Path::new("/sys/module/netmap/parameters/buf_size")));
            let mut num_str = String::new();
            try!(f.read_to_string(&mut num_str));
            let buf_size = num_str.trim_right().parse().unwrap();

            Ok(NmDesc {
                desc: desc,
                buf_size: buf_size,
            })
        }
    }
}

impl Drop for NmDesc {
    fn drop(&mut self) {
        unsafe {
            nm_close(self.desc);
        }
    }
}

/// Netmap specific configuration
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Config {
    /// The read timeout. Defaults to None.
    pub read_timeout: Option<Duration>,

    /// The write timeout. Defaults to None.
    pub write_timeout: Option<Duration>,
}

impl<'a> From<&'a datalink::Config> for Config {
    fn from(_config: &datalink::Config) -> Config {
        Config {
            read_timeout: config.read_timeout,
            write_timeout: config.write_timeout,
        }
    }
}

impl Default for Config {
    fn default() -> Config {
        Config {
            read_timeout: None,
            write_timeout: None,
        }
    }
}

#[inline]
fn get_timeout(to: Option<Duration>) -> Option<libc::timespec> {
    to.map(|dur| {
        libc::timespec {
            tv_sec: dur.as_secs() as libc::time_t,
            tv_nsec: dur.subsec_nanos() as libc::c_long
        }
    })
}

/// Create a datalink channel using the netmap library
#[inline]
pub fn channel(network_interface: &NetworkInterface, config: Config)
    -> io::Result<datalink::Channel> {
    // FIXME probably want one for each of send/recv
    let desc = NmDesc::new(network_interface);
    match desc {
        Ok(desc) => {
            let arc = Arc::new(desc);

            Ok(Ethernet(
                Box::new(DataLinkSenderImpl {
                    desc: arc.clone(),
                    timeout: get_timeout(config.write_timeout)
                }),
                Box::new(DataLinkReceiverImpl {
                    desc: arc,
                    timeout: get_timeout(config.read_timeout)
                })))
        }
        Err(e) => Err(e),
    }
}

struct DataLinkSenderImpl {
    desc: Arc<NmDesc>,
    timeout: Option<libc::timespec>,
}

impl EthernetDataLinkSender for DataLinkSenderImpl {
    #[inline]
    fn build_and_send(&mut self,
                      num_packets: usize,
                      packet_size: usize,
                      func: &mut FnMut(MutableEthernetPacket))
        -> Option<io::Result<()>> {
        assert!(packet_size <= self.desc.buf_size as usize);
        let desc = self.desc.desc;
        let mut fds = pollfd {
            fd: unsafe { NETMAP_FD(desc) },
            events: POLLOUT,
            revents: 0,
        };
        let mut packet_idx = 0usize;
        while packet_idx < num_packets {
            unsafe {
                let timespec = self.timeout.as_ref().map(|ts| ts as *const _).unwrap_or(ptr::null());
                if ppoll(&mut fds, 1, timespec, ptr::null()) < 0 {
                    return Some(Err(io::Error::last_os_error()));
                }
                let ring = NETMAP_TXRING((*desc).nifp, 0);
                while !nm_ring_empty(ring) && packet_idx < num_packets {
                    let i = (*ring).cur;
                    let slot_ptr: *mut netmap_slot = mem::transmute(&mut (*ring).slot);
                    let buf = NETMAP_BUF(ring, (*slot_ptr.offset(i as isize)).buf_idx as isize);
                    let slice = slice::from_raw_parts_mut(buf as *mut u8, packet_size);
                    let meh = MutableEthernetPacket::new(slice).unwrap();
                    (*slot_ptr.offset(i as isize)).len = packet_size as u16;
                    func(meh);
                    let next = nm_ring_next(ring, i);
                    (*ring).head = next;
                    (*ring).cur = next;
                    packet_idx += 1;
                }
            }
        }

        Some(Ok(()))
    }

    #[inline]
    fn send_to(&mut self,
               packet: &EthernetPacket,
               _dst: Option<NetworkInterface>)
        -> Option<io::Result<()>> {
        use packet::MutablePacket;
        self.build_and_send(1,
                            packet.packet().len(),
                            &mut |mut eh: MutableEthernetPacket| {
                                eh.clone_from(packet);
                            })
    }
}

struct DataLinkReceiverImpl {
    desc: Arc<NmDesc>,
    timeout: Option<libc::timespec>,
}

impl EthernetDataLinkReceiver for DataLinkReceiverImpl {
    // FIXME Layer 3
    fn iter<'a>(&'a mut self) -> Box<EthernetDataLinkChannelIterator + 'a> {
        Box::new(DataLinkChannelIteratorImpl { pc: self })
    }
}

struct DataLinkChannelIteratorImpl<'a> {
    pc: &'a mut DataLinkReceiverImpl,
}

impl<'a> EthernetDataLinkChannelIterator<'a> for DataLinkChannelIteratorImpl<'a> {
    fn next(&mut self) -> io::Result<EthernetPacket> {
        let desc = self.pc.desc.desc;
        let mut h: nm_pkthdr = unsafe { mem::uninitialized() };
        let mut buf = unsafe { nm_nextpkt(desc, &mut h) };
        if buf.is_null() {
            let mut fds = pollfd {
                fd: unsafe { NETMAP_FD(desc) },
                events: POLLIN,
                revents: 0,
            };
            let timespec = self.timeout.as_ref().map(|ts| ts as *const _).unwrap_or(ptr::null());
            if unsafe { ppoll(&mut fds, 1, timespec, ptr::null()) } < 0 {
                return Err(io::Error::last_os_error());
            }
            buf = unsafe { nm_nextpkt(desc, &mut h) };
        }
        Ok(EthernetPacket::new(unsafe {
            slice::from_raw_parts(buf, h.len as usize)
        }).unwrap())
    }
}

/// Get a list of available network interfaces for the current machine.
pub fn interfaces() -> Vec<NetworkInterface> {
    #[path = "unix_interfaces.rs"]
    mod interfaces;
    interfaces::interfaces()
}
<|endoftext|>


<commit_before>use db::row::Row;
// mis info about what's on each page.
const HEADER_SIZE_IN_BYTES : usize = 64;
pub const PAGE_SIZE : usize = 4096;

pub struct Page {
    header: Header,
    data: [u8; PAGE_SIZE - HEADER_SIZE_IN_BYTES],
    bytes_used: usize, // tracking where we are in the current page

                                      // if we go past the page boundary on a write we fault
}

enum PageError {
    Full
}

type PageResult<T> = Result<T, PageError>;

struct Header;

// page deals with bytes, and has zero knowledge of a Row
// storage engine needs to serialize
impl Page {
    pub fn new() -> Page {
        Page{header: Header::new(),
             data: [0; 4096 - HEADER_SIZE_IN_BYTES],
             bytes_used: HEADER_SIZE_IN_BYTES,
             }
    }

    // attempt to write to a page
    // can fail if the page is already full
    // if it fails, the storage engine will have to flush this page
    // then allocate a new page
    fn write(&mut self, bytes: &[u8]) -> PageResult<()> {
        Err(PageError::Full)
    }

    fn space_available(self) -> usize {
        PAGE_SIZE - self.bytes_used
    }

}

impl Header {
    fn new() -> Header {
        Header{}
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn test_page_insert_ok() {

    }

    #[test]
    fn test_page_fault() {

    }
}

<commit_msg>finishing up the Page<commit_after>use db::row::Row;
// mis info about what's on each page.
const HEADER_SIZE_IN_BYTES : usize = 64;
pub const PAGE_SIZE : usize = 4096;

pub struct Page {
    header: Header,
    data: [u8; PAGE_SIZE - HEADER_SIZE_IN_BYTES],
    bytes_used: usize, // tracking where we are in the current page

                                      // if we go past the page boundary on a write we fault
}

#[derive(Debug)]
enum PageError {
    Full
}

type PageResult<T> = Result<T, PageError>;

struct Header;

// page deals with bytes, and has zero knowledge of a Row
// storage engine needs to serialize
impl Page {
    pub fn new() -> Page {
        Page{header: Header::new(),
             data: [0; 4096 - HEADER_SIZE_IN_BYTES],
             bytes_used: HEADER_SIZE_IN_BYTES,
             }
    }

    // attempt to write to a page
    // can fail if the page is already full
    // if it fails, the storage engine will have to flush this page
    // then allocate a new page
    fn write(&mut self, bytes: &[u8]) -> PageResult<()> {
        Err(PageError::Full)
    }

    fn space_available(self) -> usize {
        PAGE_SIZE - self.bytes_used
    }

}

impl Header {
    fn new() -> Header {
        Header{}
    }
}

#[cfg(test)]
mod tests {
    use super::{Page, HEADER_SIZE_IN_BYTES};
    #[test]
    fn test_page_insert_ok() {
        let mut p = Page::new();
        let data: [u8; 16] = [0; 16];
        p.write(&data).expect("Data written");
        assert_eq!(p.bytes_used, HEADER_SIZE_IN_BYTES + 16);
    }

    #[test]
    fn test_page_fault() {

    }
}

<|endoftext|>


<commit_before>// Implements http://rosettacode.org/wiki/Stack

#[deriving(Show)]
struct Stack<T> {
    // We use a vector because of simplicity
    vec: Vec<T>
}

impl<T> Stack<T> {
    fn new() -> Stack<T> {
        Stack { vec: Vec::new() }
    }

    // Adds an element at the top of the stack
    fn push(&mut self, elem: T) {
        self.vec.push(elem);
    }

    // Removes and returns the element at the top of the stack
    fn pop(&mut self) -> Option<T> {
        self.vec.pop()
    }

    // Returns a reference of the element at the top of the stack
    fn peek<'r>(&'r self) -> Option<&'r T> {
        self.vec.last()
    }

    // Returns true if the stack is empty
    fn empty(&self) -> bool {
        self.vec.len() == 0
    }
}

#[cfg(not(test))]
fn main() {
    let mut stack = Stack::new();

    // Fill the stack
    stack.push(5);
    stack.push(8);
    stack.push(9);

    // Show the element at the top
    println!("{}", stack.peek().unwrap());
}

#[test]
fn test_basic() {
    let mut stack = Stack::new();

    // The stack is empty
    assert!(stack.empty());

    // Fill the stack
    stack.push(5);
    stack.push(8);
    stack.push(9);

    // The stack is not empty
    assert!(!stack.empty());

    // The element at the top is 9
    assert!(stack.peek().unwrap() == &9);

    // Remove one element
    stack.pop();

    // The element at the top is now 8
    assert!(stack.peek().unwrap() == &8);
}<commit_msg>add usage to fn main in stack.rs<commit_after>// Implements http://rosettacode.org/wiki/Stack

#[deriving(Show)]
struct Stack<T> {
    // We use a vector because of simplicity
    vec: Vec<T>
}

impl<T> Stack<T> {
    fn new() -> Stack<T> {
        Stack { vec: Vec::new() }
    }

    // Adds an element at the top of the stack
    fn push(&mut self, elem: T) {
        self.vec.push(elem);
    }

    // Removes and returns the element at the top of the stack
    fn pop(&mut self) -> Option<T> {
        self.vec.pop()
    }

    // Returns a reference of the element at the top of the stack
    fn peek<'r>(&'r self) -> Option<&'r T> {
        self.vec.last()
    }

    // Returns true if the stack is empty
    fn empty(&self) -> bool {
        self.vec.len() == 0
    }
}

#[cfg(not(test))]
fn main() {
    let mut stack = Stack::new();

    // Fill the stack
    stack.push(5);
    stack.push(8);
    stack.push(9);

    // Show the element at the top
    println!("{}", stack.peek().unwrap());
    // Show the element we popped
    println!("{}", stack.pop().unwrap());
    if stack.empty() {
        println!("The stack is empty.")
    } else {
        println!("The stack is not empty.")
    }
}

#[test]
fn test_basic() {
    let mut stack = Stack::new();

    // The stack is empty
    assert!(stack.empty());

    // Fill the stack
    stack.push(5);
    stack.push(8);
    stack.push(9);

    // The stack is not empty
    assert!(!stack.empty());

    // The element at the top is 9
    assert!(stack.peek().unwrap() == &9);

    // Remove one element
    stack.pop();

    // The element at the top is now 8
    assert!(stack.peek().unwrap() == &8);
}
<|endoftext|>


<commit_before>
use std::default::Default;

use types::{Int, ForthCell};
use error::{ForthResult, ForthError};
use dictionary::Dictionary;
use statement::{Statement, StatementExt};

mod builtins;

pub type Stack = Vec<Int>;

#[derive(Default, Debug)]
pub struct Interpreter {
    pub dictionary: Dictionary,
    pub parameter_stack: Stack,
    pub return_stack: Stack,
    pub statement: Statement,
    pub statement_index: usize,
}

// TODO: Think about refactoring this to make it less... meh.
impl Interpreter {
    pub fn new() -> Interpreter {
        let mut dict = Dictionary::new();

        builtins::insert_builtins(&mut dict);

        Interpreter {
            dictionary: dict,
            ..Default::default()
        }
    }

    pub fn parse(&mut self, string: &str) -> ForthResult<()> {
        self.statement = try!(Statement::parse(self, string));
        self.statement_index = 0;
        self.parameter_stack.clear();
        self.return_stack.clear();
        Ok(())
    }

    fn exec_current(&mut self) -> ForthResult<()> {
        match self.get_current().unwrap().clone() {
            ForthCell::Number(n) => { self.parameter_stack.push(n) },
            ForthCell::Word(ref w) => { try!(w.code.run(self)) }
        }
        Ok(())
    }

    fn get_current(&mut self) -> Option<&ForthCell> {
        self.statement.get(self.statement_index)
    }

    pub fn has_next(&self) -> bool {
        self.statement_index < self.statement.len() - 1
    }

    pub fn has_prev(&self) -> bool {
        self.statement_index > 0
    }

    pub fn next(&mut self) -> ForthResult<()> {
        if self.has_next() {
            self.statement_index += 1;
        } else {
            return Err(ForthError::InvalidJump("No next statement".to_string()));
        }

        self.exec_current()
    }

    pub fn prev(&mut self) -> ForthResult<()> {
        if self.has_prev() {
            self.statement_index -= 1;
        } else {
            return Err(ForthError::InvalidJump("No previous statement".to_string()));
        }

        self.exec_current()
    }

    pub fn jump(&mut self, offset: i32) -> ForthResult<()> {
        let target = self.statement_index as isize + offset as isize;
        if target < self.statement.len() as isize && target >= 0 {
            self.statement_index = target as usize;
        } else {
            return Err(ForthError::InvalidJump(format!("Invalid jump offset {} to index {}", offset, target).to_string()));
        }

        Ok(())
    }

    pub fn run(&mut self) -> ForthResult<()> {
        try!(self.exec_current());
        while self.has_next() {
            try!(self.next());
        }
        Ok(())
    }

    pub fn exec(&mut self, string: &str) -> ForthResult<()> {
        try!(self.parse(string));
        self.run()

        // let mut stmt = Statement::new();
        // while let Some(word) = words.next() { match word {
        //     "(" => while let Some(w) = words.next() { if w == ")" { break; } },
        //     "\\" => while let Some(w) = words.next() { if w == "\n" { break; } },
        //     ":" => {
        //         let mut comp = Statement::new();
        //         let name = match words.next() {
        //             Some(w) => w,
        //             None => return Err(ForthError::WordNameNotFound),
        //         };
        //         while let Some(w) = words.next() {
        //             if w == ";" {
        //                 self.dictionary.insert_entry(Entry::from_statement(name, comp));
        //                 return Ok(());
        //             }
        //             match w.parse::<i32>() {
        //                 Ok(n) => comp.push_back(ForthCell::Number(n)),
        //                 Err(_) => comp.push_back(ForthCell::Word(.unwrap().clone())),
        //             }
        //         }
        //         return Err(ForthError::UnterminatedWordDefinition);
        //     },
        //     _ => match word.parse::<i32>() {
        //         Ok(n) => {
        //             stmt.push_back(ForthCell::Number(n));
        //         },
        //         Err(_) => {
        //             if let Some(elem) = self.dictionary.get(word) {
        //                 stmt.push_back(ForthCell::Word(elem.clone()));
        //             } else {
        //                 return Err(ForthError::WordNotFound)
        //             }
        //         },
        //     },

        // }}

        // match stmt.run(self) {
        //     Err(e) => {
        //         self.last_result = None;
        //         Err(e)
        //     },
        //     Ok(s) => {
        //         self.last_result = Some(s);
        //         Ok(())
        //     }
        // }
    }
}<commit_msg>Minor formatting.<commit_after>
use std::default::Default;

use types::{Int, ForthCell};
use error::{ForthResult, ForthError};
use dictionary::Dictionary;
use statement::{Statement, StatementExt};

mod builtins;

pub type Stack = Vec<Int>;

#[derive(Default, Debug)]
pub struct Interpreter {
    pub dictionary: Dictionary,
    pub parameter_stack: Stack,
    pub return_stack: Stack,
    pub statement: Statement,
    pub statement_index: usize,
}

// TODO: Think about refactoring this to make it less... meh.
impl Interpreter {
    pub fn new() -> Interpreter {
        let mut dict = Dictionary::new();

        builtins::insert_builtins(&mut dict);

        Interpreter {
            dictionary: dict,
            ..Default::default()
        }
    }

    pub fn parse(&mut self, string: &str) -> ForthResult<()> {
        self.statement = try!(Statement::parse(self, string));
        self.statement_index = 0;
        self.parameter_stack.clear();
        self.return_stack.clear();
        Ok(())
    }

    fn exec_current(&mut self) -> ForthResult<()> {
        match self.get_current().unwrap().clone() {
            ForthCell::Number(n) => { self.parameter_stack.push(n) },
            ForthCell::Word(ref w) => { try!(w.code.run(self)) }
        }
        Ok(())
    }

    fn get_current(&mut self) -> Option<&ForthCell> {
        self.statement.get(self.statement_index)
    }

    pub fn has_next(&self) -> bool {
        self.statement_index < self.statement.len() - 1
    }

    pub fn has_prev(&self) -> bool {
        self.statement_index > 0
    }

    pub fn next(&mut self) -> ForthResult<()> {
        if self.has_next() {
            self.statement_index += 1;
        } else {
            return Err(ForthError::InvalidJump("No next statement".to_string()));
        }

        self.exec_current()
    }

    pub fn prev(&mut self) -> ForthResult<()> {
        if self.has_prev() {
            self.statement_index -= 1;
        } else {
            return Err(ForthError::InvalidJump("No previous statement".to_string()));
        }

        self.exec_current()
    }

    pub fn jump(&mut self, offset: i32) -> ForthResult<()> {
        let target = self.statement_index as isize + offset as isize;
        if target < self.statement.len() as isize && target >= 0 {
            self.statement_index = target as usize;
        } else {
            return Err(ForthError::InvalidJump(format!("Invalid jump offset {} to index {}", offset, target).to_string()));
        }

        Ok(())
    }

    pub fn run(&mut self) -> ForthResult<()> {
        try!(self.exec_current());
        while self.has_next() {
            try!(self.next());
        }

        Ok(())
    }

    pub fn exec(&mut self, string: &str) -> ForthResult<()> {
        try!(self.parse(string));
        self.run()
    }
}<|endoftext|>


<commit_before>use context::Context;
use get;
use std::default::Default;
use types::Type;
use types::kind::*;
macro_rules! test_compile(
    ($ty:ty, $test_name:ident, $kind:expr) => (
        #[test]
        fn $test_name() {
            let default_value:$ty = Default::default();
            Context::new().build_func(get::<fn() -> $ty>(), |func| {
                let ref val = func.insn_of(&default_value);
                assert!(val.get_type().get_kind() == $kind);
                func.insn_return(val);
            }).with::<(), $ty>(|func| {
                assert_eq!(func(()), default_value);
            });
        }
    );
)
#[test]
fn test_sqrt() {
    Context::new().build_func(get::<fn(uint) -> uint>(), |func| {
        let ref arg = func[0];
        assert_eq!(arg.get_type(), get::<uint>());
        let sqrt_arg = func.insn_sqrt(arg);
        let sqrt_arg_ui = func.insn_convert(&sqrt_arg, get::<uint>(), false);
        func.insn_return(&sqrt_arg_ui);
    }).with::<uint, uint>(|sqrt| {
        assert_eq!(sqrt(64), 8);
        assert_eq!(sqrt(16), 4);
        assert_eq!(sqrt(9), 3);
        assert_eq!(sqrt(4), 2);
        assert_eq!(sqrt(1), 1);
    });
}

#[test]
fn test_struct() {
    ::init();
    let double_float_t = jit!(struct {
        "first": f64,
        "second": f64
    });
    assert_eq!(double_float_t.find_name("first").index, 0);
    assert_eq!(double_float_t.find_name("second").index, 1);
    let fields:Vec<String> = double_float_t.fields().map(|field| field.get_name().unwrap()).collect();
    assert_eq!(fields[], [
        "first".into_string(),
        "second".into_string()
    ][]);
    assert_eq!(double_float_t.fields().count(), 2);
    assert!({
        let field:Option<String> = double_float_t.fields().nth(1).unwrap().get_name();
        field.unwrap().as_slice() == "second"
    });
}

test_compile!((), test_compile_void, Void)
test_compile!(f64, test_compile_f64, Float64)
test_compile!(f32, test_compile_f32, Float32)
test_compile!(int, test_compile_int, NInt)
test_compile!(uint, test_compile_uint, NUInt)
test_compile!(i32, test_compile_i32, Int)
test_compile!(u32, test_compile_u32, UInt)
test_compile!(i16, test_compile_i16, Short)
test_compile!(u16, test_compile_u16, UShort)
test_compile!(i8, test_compile_i8, SByte)
test_compile!(u8, test_compile_u8, UByte)
<commit_msg>Simplify struct test<commit_after>use context::Context;
use get;
use std::default::Default;
use types::Type;
use types::kind::*;
macro_rules! test_compile(
    ($ty:ty, $test_name:ident, $kind:expr) => (
        #[test]
        fn $test_name() {
            let default_value:$ty = Default::default();
            Context::new().build_func(get::<fn() -> $ty>(), |func| {
                let ref val = func.insn_of(&default_value);
                assert!(val.get_type().get_kind() == $kind);
                func.insn_return(val);
            }).with::<(), $ty>(|func| {
                assert_eq!(func(()), default_value);
            });
        }
    );
)
#[test]
fn test_sqrt() {
    Context::new().build_func(get::<fn(uint) -> uint>(), |func| {
        let ref arg = func[0];
        assert_eq!(arg.get_type(), get::<uint>());
        let sqrt_arg = func.insn_sqrt(arg);
        let sqrt_arg_ui = func.insn_convert(&sqrt_arg, get::<uint>(), false);
        func.insn_return(&sqrt_arg_ui);
    }).with::<uint, uint>(|sqrt| {
        assert_eq!(sqrt(64), 8);
        assert_eq!(sqrt(16), 4);
        assert_eq!(sqrt(9), 3);
        assert_eq!(sqrt(4), 2);
        assert_eq!(sqrt(1), 1);
    });
}
#[test]
fn test_struct() {
    ::init();
    let pos_t = jit!(struct {
        "x": f64,
        "y": f64
    });
    for (i, field) in pos_t.fields().enumerate() {
        assert_eq!(field.get_type(), get::<f64>());
        assert_eq!(field.get_name().unwrap()[], match i {
            0 => "x",
            1 => "y",
            _ => unimplemented!()
        })
    }
}

test_compile!((), test_compile_void, Void)
test_compile!(f64, test_compile_f64, Float64)
test_compile!(f32, test_compile_f32, Float32)
test_compile!(int, test_compile_int, NInt)
test_compile!(uint, test_compile_uint, NUInt)
test_compile!(i32, test_compile_i32, Int)
test_compile!(u32, test_compile_u32, UInt)
test_compile!(i16, test_compile_i16, Short)
test_compile!(u16, test_compile_u16, UShort)
test_compile!(i8, test_compile_i8, SByte)
test_compile!(u8, test_compile_u8, UByte)
<|endoftext|>


<commit_before>use variables::*;
use variables::LpExpression::*;
use problem::*;
use std::rc::Rc;

#[test]
fn expressions_creation() {
    let ref a = LpInteger::new("a")
        .lower_bound(10.0)
        .upper_bound(20.0);
    let ref b = LpInteger::new("b");

    assert_eq!(a + b, AddExpr(Rc::new(ConsInt(a.clone())), Rc::new(ConsInt(b.clone()))));
}

#[test]
fn expressions_to_string() {
    let ref a = LpInteger::new("a");
    let ref b = LpInteger::new("b");
    let ref c = LpInteger::new("c");

    assert_eq!((a + 2*b + c).to_string(), "a + 2 b + c");
    assert_eq!((a + b*2 + c).to_string(), "a + 2 b + c");
}

<commit_msg>Unit tests for expressions to string<commit_after>use variables::*;
use variables::LpExpression::*;
use problem::*;
use std::rc::Rc;

#[test]
fn expressions_creation() {
    let ref a = LpInteger::new("a")
        .lower_bound(10.0)
        .upper_bound(20.0);
    let ref b = LpInteger::new("b");

    assert_eq!(a + b, AddExpr(Rc::new(ConsInt(a.clone())), Rc::new(ConsInt(b.clone()))));
}

#[test]
fn expressions_to_string() {
    let ref a = LpInteger::new("a");
    let ref b = LpInteger::new("b");
    let ref c = LpInteger::new("c");

    assert_eq!((a + 2*b + c).to_string(), "a + 2 b + c");
    assert_eq!((a + b*2 + c).to_string(), "a + 2 b + c");
    assert_eq!((a + b*2 + 3 * 2 * c).to_string(), "a + 2 b + 6 c");
    assert_eq!((a + 2).to_string(), "a + 2");
    assert_eq!((2*a + 2*b -4*c).to_string(), "2 a + 2 b - 4 c");
    assert_eq!((-2*a).to_string(), "-2 a");

}

<|endoftext|>


<commit_before>//! Timing and measurement functions.
//!
//! ggez does not try to do any framerate limitation by default. If
//! you want to run at anything other than full-bore max speed all the
//! time, calling `thread::yield_now()` (or `timer::yield_now()` which
//! does the same thing) yield to the OS
//! so it has a chance to breathe before continuing with your game.
//! This should prevent it from using 100% CPU unless it really needs
//! to.  Enabling vsync by setting `vsync` in your `Conf` object is
//! generally the best way to cap your displayed framerate.
//!
//! For a more detailed tutorial in how to handle frame timings in games,
//! see <http://gafferongames.com/game-physics/fix-your-timestep/>

use context::Context;

use std::cmp;
use std::time;
use std::thread;
use std::f64;

/// A simple buffer that fills
/// up to a limit and then holds the last
/// N items that have been inserted into it,
/// overwriting old ones in a round-robin fashion.
///
/// It's not quite a ring buffer 'cause you can't
/// remove items from it, it just holds the last N
/// things.
#[derive(Debug, Clone)]
struct LogBuffer<T>
where
    T: Clone,
{
    head: usize,
    size: usize,
    contents: Vec<T>,
}

impl<T> LogBuffer<T>
where
    T: Clone + Copy,
{
    fn new(size: usize, init_val: T) -> LogBuffer<T> {
        let mut v = Vec::with_capacity(size);
        v.resize(size, init_val);
        LogBuffer {
            head: 0,
            size: size,
            contents: v,
        }
    }

    /// Pushes a new item into the logbuffer, overwriting
    /// the oldest item in it.
    fn push(&mut self, item: T) {
        self.head = (self.head + 1) % self.contents.len();
        self.contents[self.head] = item;
        self.size = cmp::min(self.size + 1, self.contents.len());
    }

    /// Returns a slice pointing at the contents of the buffer.
    /// They are in *no particular order*, and if not all the
    /// slots are filled, the empty slots will be present but
    /// contain the initial value given to `new()`
    ///
    /// We're only using this to log FPS for a short time,
    /// so we don't care for the second or so when it's inaccurate.
    fn contents(&self) -> &[T] {
        &self.contents
    }

    /// Returns the most recent value in the buffer.
    fn latest(&self) -> T {
        self.contents[self.head]
    }
}

/// A structure that contains our time-tracking state.
#[derive(Debug)]
pub struct TimeContext {
    init_instant: time::Instant,
    last_instant: time::Instant,
    frame_durations: LogBuffer<time::Duration>,
    residual_update_dt: time::Duration,
    frame_count: usize,
}

// How many frames we log update times for.
const TIME_LOG_FRAMES: usize = 200;

impl TimeContext {
    /// Creates a new `TimeContext` and initializes the start to this instant.
    pub fn new() -> TimeContext {
        TimeContext {
            init_instant: time::Instant::now(),
            last_instant: time::Instant::now(),
            frame_durations: LogBuffer::new(TIME_LOG_FRAMES, time::Duration::new(0, 0)),
            residual_update_dt: time::Duration::from_secs(0),
            frame_count: 0,
        }
    }

    /// Update the state of the TimeContext to record that
    /// another frame has taken place.  Necessary for the FPS
    /// tracking and `check_update_time()` functions to work.
    ///
    /// It's usually not necessary to call this function yourself,
    /// `EventHandler::run()` will do it for you.
    pub fn tick(&mut self) {
        let now = time::Instant::now();
        let time_since_last = now - self.last_instant;
        self.frame_durations.push(time_since_last);
        self.last_instant = now;
        self.frame_count += 1;

        self.residual_update_dt += time_since_last;
    }
}

impl Default for TimeContext {
    fn default() -> Self {
        Self::new()
    }
}

/// Get the time between the start of the last frame and the current one;
/// in other words, the length of the last frame.
pub fn get_delta(ctx: &Context) -> time::Duration {
    let tc = &ctx.timer_context;
    tc.frame_durations.latest()
}

/// Gets the average time of a frame, averaged
/// over the last 200 frames.
pub fn get_average_delta(ctx: &Context) -> time::Duration {
    let tc = &ctx.timer_context;
    let init = time::Duration::new(0, 0);
    let sum = tc.frame_durations
        .contents()
        .iter()
        .fold(init, |d1, d2| d1 + *d2);
    sum / (tc.frame_durations.size as u32)
}

/// A convenience function to convert a Rust `Duration` type
/// to a (less precise but more useful) f64.
///
/// Does not make sure that the `Duration` is within the bounds
/// of the `f64`.
pub fn duration_to_f64(d: time::Duration) -> f64 {
    let seconds = d.as_secs() as f64;
    let nanos = f64::from(d.subsec_nanos());
    seconds + (nanos * 1e-9)
}

/// A convenience function to create a Rust `Duration` type
/// from a (less precise but more useful) f64.
///
/// Only handles positive numbers correctly.
pub fn f64_to_duration(t: f64) -> time::Duration {
    debug_assert!(t >= 0.0, "f64_to_duration passed a negative number!");
    let seconds = t.trunc();
    let nanos = t.fract() * 1e9;
    time::Duration::new(seconds as u64, nanos as u32)
}

/// Returns a `Duration` representing how long each
/// frame should be to match the given fps.
///
/// Approximately.
fn fps_as_duration(fps: u32) -> time::Duration {
    let target_dt_seconds = 1.0 / f64::from(fps);
    f64_to_duration(target_dt_seconds)
}

/// Gets the FPS of the game, averaged over the last
/// 200 frames.
pub fn get_fps(ctx: &Context) -> f64 {
    let duration_per_frame = get_average_delta(ctx);
    let seconds_per_frame = duration_to_f64(duration_per_frame);
    1.0 / seconds_per_frame
}

/// Returns the time since the game was initialized,
/// as reported by the system clock.
pub fn get_time_since_start(ctx: &Context) -> time::Duration {
    let tc = &ctx.timer_context;
    time::Instant::now() - tc.init_instant
}

/// This function will return true if the time since the
/// last `update()` call has been equal to or greater to
/// the update FPS indicated by the `target_fps`.
/// It keeps track of fractional frames, so if you want
/// 60 fps (16.67 ms/frame) and the game stutters so that
/// there is 40 ms between `update()` calls, this will return
/// `true` twice, and take the remaining 6.67 ms into account
/// in the next frame.
///
/// The intention is to for it to be called in a while loop
/// in your `update()` callback:
///
/// ```rust,ignore
/// fn update(&mut self, ctx: &mut Context) -> GameResult<()>
///     while(timer::get_time_since_start(ctx, 60)) {
///         update_game_physics()?;
///     }
///     Ok(())
/// }
/// ```
pub fn check_update_time(ctx: &mut Context, target_fps: u32) -> bool {
    let timedata = &mut ctx.timer_context;

    let target_dt = fps_as_duration(target_fps);
    if timedata.residual_update_dt > target_dt {
        timedata.residual_update_dt -= target_dt;
        true
    } else {
        false
    }
}

/// Pauses the current thread for the target duration.
/// Just calls `std::thread::sleep()` so it's as accurate
/// as that is (which is usually not very).
pub fn sleep(duration: time::Duration) {
    thread::sleep(duration);
}

/// Yields the current timeslice to the OS.
///
/// This just calls `std::thread::yield_now()` but it's
/// handy to have here.
pub fn yield_now() {
    thread::yield_now();
}

/// Gets the number of times the game has gone through its event loop.
///
/// Specifically, the number of times that `TimeContext::tick()` has been
/// called by it.
pub fn get_ticks(ctx: &Context) -> usize {
    ctx.timer_context.frame_count
}
<commit_msg>Fix example in documentation of timer::check_update_time<commit_after>//! Timing and measurement functions.
//!
//! ggez does not try to do any framerate limitation by default. If
//! you want to run at anything other than full-bore max speed all the
//! time, calling `thread::yield_now()` (or `timer::yield_now()` which
//! does the same thing) yield to the OS
//! so it has a chance to breathe before continuing with your game.
//! This should prevent it from using 100% CPU unless it really needs
//! to.  Enabling vsync by setting `vsync` in your `Conf` object is
//! generally the best way to cap your displayed framerate.
//!
//! For a more detailed tutorial in how to handle frame timings in games,
//! see <http://gafferongames.com/game-physics/fix-your-timestep/>

use context::Context;

use std::cmp;
use std::time;
use std::thread;
use std::f64;

/// A simple buffer that fills
/// up to a limit and then holds the last
/// N items that have been inserted into it,
/// overwriting old ones in a round-robin fashion.
///
/// It's not quite a ring buffer 'cause you can't
/// remove items from it, it just holds the last N
/// things.
#[derive(Debug, Clone)]
struct LogBuffer<T>
where
    T: Clone,
{
    head: usize,
    size: usize,
    contents: Vec<T>,
}

impl<T> LogBuffer<T>
where
    T: Clone + Copy,
{
    fn new(size: usize, init_val: T) -> LogBuffer<T> {
        let mut v = Vec::with_capacity(size);
        v.resize(size, init_val);
        LogBuffer {
            head: 0,
            size: size,
            contents: v,
        }
    }

    /// Pushes a new item into the logbuffer, overwriting
    /// the oldest item in it.
    fn push(&mut self, item: T) {
        self.head = (self.head + 1) % self.contents.len();
        self.contents[self.head] = item;
        self.size = cmp::min(self.size + 1, self.contents.len());
    }

    /// Returns a slice pointing at the contents of the buffer.
    /// They are in *no particular order*, and if not all the
    /// slots are filled, the empty slots will be present but
    /// contain the initial value given to `new()`
    ///
    /// We're only using this to log FPS for a short time,
    /// so we don't care for the second or so when it's inaccurate.
    fn contents(&self) -> &[T] {
        &self.contents
    }

    /// Returns the most recent value in the buffer.
    fn latest(&self) -> T {
        self.contents[self.head]
    }
}

/// A structure that contains our time-tracking state.
#[derive(Debug)]
pub struct TimeContext {
    init_instant: time::Instant,
    last_instant: time::Instant,
    frame_durations: LogBuffer<time::Duration>,
    residual_update_dt: time::Duration,
    frame_count: usize,
}

// How many frames we log update times for.
const TIME_LOG_FRAMES: usize = 200;

impl TimeContext {
    /// Creates a new `TimeContext` and initializes the start to this instant.
    pub fn new() -> TimeContext {
        TimeContext {
            init_instant: time::Instant::now(),
            last_instant: time::Instant::now(),
            frame_durations: LogBuffer::new(TIME_LOG_FRAMES, time::Duration::new(0, 0)),
            residual_update_dt: time::Duration::from_secs(0),
            frame_count: 0,
        }
    }

    /// Update the state of the TimeContext to record that
    /// another frame has taken place.  Necessary for the FPS
    /// tracking and `check_update_time()` functions to work.
    ///
    /// It's usually not necessary to call this function yourself,
    /// `EventHandler::run()` will do it for you.
    pub fn tick(&mut self) {
        let now = time::Instant::now();
        let time_since_last = now - self.last_instant;
        self.frame_durations.push(time_since_last);
        self.last_instant = now;
        self.frame_count += 1;

        self.residual_update_dt += time_since_last;
    }
}

impl Default for TimeContext {
    fn default() -> Self {
        Self::new()
    }
}

/// Get the time between the start of the last frame and the current one;
/// in other words, the length of the last frame.
pub fn get_delta(ctx: &Context) -> time::Duration {
    let tc = &ctx.timer_context;
    tc.frame_durations.latest()
}

/// Gets the average time of a frame, averaged
/// over the last 200 frames.
pub fn get_average_delta(ctx: &Context) -> time::Duration {
    let tc = &ctx.timer_context;
    let init = time::Duration::new(0, 0);
    let sum = tc.frame_durations
        .contents()
        .iter()
        .fold(init, |d1, d2| d1 + *d2);
    sum / (tc.frame_durations.size as u32)
}

/// A convenience function to convert a Rust `Duration` type
/// to a (less precise but more useful) f64.
///
/// Does not make sure that the `Duration` is within the bounds
/// of the `f64`.
pub fn duration_to_f64(d: time::Duration) -> f64 {
    let seconds = d.as_secs() as f64;
    let nanos = f64::from(d.subsec_nanos());
    seconds + (nanos * 1e-9)
}

/// A convenience function to create a Rust `Duration` type
/// from a (less precise but more useful) f64.
///
/// Only handles positive numbers correctly.
pub fn f64_to_duration(t: f64) -> time::Duration {
    debug_assert!(t >= 0.0, "f64_to_duration passed a negative number!");
    let seconds = t.trunc();
    let nanos = t.fract() * 1e9;
    time::Duration::new(seconds as u64, nanos as u32)
}

/// Returns a `Duration` representing how long each
/// frame should be to match the given fps.
///
/// Approximately.
fn fps_as_duration(fps: u32) -> time::Duration {
    let target_dt_seconds = 1.0 / f64::from(fps);
    f64_to_duration(target_dt_seconds)
}

/// Gets the FPS of the game, averaged over the last
/// 200 frames.
pub fn get_fps(ctx: &Context) -> f64 {
    let duration_per_frame = get_average_delta(ctx);
    let seconds_per_frame = duration_to_f64(duration_per_frame);
    1.0 / seconds_per_frame
}

/// Returns the time since the game was initialized,
/// as reported by the system clock.
pub fn get_time_since_start(ctx: &Context) -> time::Duration {
    let tc = &ctx.timer_context;
    time::Instant::now() - tc.init_instant
}

/// This function will return true if the time since the
/// last `update()` call has been equal to or greater to
/// the update FPS indicated by the `target_fps`.
/// It keeps track of fractional frames, so if you want
/// 60 fps (16.67 ms/frame) and the game stutters so that
/// there is 40 ms between `update()` calls, this will return
/// `true` twice, and take the remaining 6.67 ms into account
/// in the next frame.
///
/// The intention is to for it to be called in a while loop
/// in your `update()` callback:
///
/// ```rust,ignore
/// fn update(&mut self, ctx: &mut Context) -> GameResult<()>
///     while(timer::check_update_time(ctx, 60)) {
///         update_game_physics()?;
///     }
///     Ok(())
/// }
/// ```
pub fn check_update_time(ctx: &mut Context, target_fps: u32) -> bool {
    let timedata = &mut ctx.timer_context;

    let target_dt = fps_as_duration(target_fps);
    if timedata.residual_update_dt > target_dt {
        timedata.residual_update_dt -= target_dt;
        true
    } else {
        false
    }
}

/// Pauses the current thread for the target duration.
/// Just calls `std::thread::sleep()` so it's as accurate
/// as that is (which is usually not very).
pub fn sleep(duration: time::Duration) {
    thread::sleep(duration);
}

/// Yields the current timeslice to the OS.
///
/// This just calls `std::thread::yield_now()` but it's
/// handy to have here.
pub fn yield_now() {
    thread::yield_now();
}

/// Gets the number of times the game has gone through its event loop.
///
/// Specifically, the number of times that `TimeContext::tick()` has been
/// called by it.
pub fn get_ticks(ctx: &Context) -> usize {
    ctx.timer_context.frame_count
}
<|endoftext|>


<commit_before>//! Tokens in the dependency graph.

use std::collections::BTreeMap;
use std::fmt;
use std::fmt::{Debug, Display};
use std::iter::FromIterator;
use std::mem;
use std::ops::{Deref, DerefMut};

use itertools::Itertools;

pub const EMPTY_TOKEN: &str = "_";

/// A builder for `Token`s.
///
/// The `Token` type stores a CoNLL-X token. However, since this format
/// permits a large number of fields, construction of a token can get
/// tedious. This builder provides a fluent interface for creating `Token`s.
pub struct TokenBuilder {
    token: Token,
}

impl TokenBuilder {
    /// Create a `Token` builder with all non-form fields set to absent.
    pub fn new(form: impl Into<String>) -> TokenBuilder {
        TokenBuilder {
            token: Token::new(form),
        }
    }

    /// Set the word form or punctuation symbol.
    pub fn form(mut self, form: impl Into<String>) -> TokenBuilder {
        self.token.set_form(form);
        self
    }

    /// Set the lemma or stem of the word form.
    pub fn lemma(mut self, lemma: impl Into<String>) -> TokenBuilder {
        self.token.set_lemma(Some(lemma));
        self
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn cpos(mut self, cpos: impl Into<String>) -> TokenBuilder {
        self.token.set_cpos(Some(cpos));
        self
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn pos(mut self, pos: impl Into<String>) -> TokenBuilder {
        self.token.set_pos(Some(pos));
        self
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn features(mut self, features: Features) -> TokenBuilder {
        self.token.set_features(Some(features));
        self
    }
}

impl From<Token> for TokenBuilder {
    fn from(token: Token) -> Self {
        TokenBuilder { token }
    }
}

impl From<TokenBuilder> for Token {
    fn from(builder: TokenBuilder) -> Self {
        builder.token
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    form: String,
    lemma: Option<String>,
    cpos: Option<String>,
    pos: Option<String>,
    features: Option<Features>,
}

impl Token {
    /// Create a new token where all the non-form fields are absent.
    pub fn new(form: impl Into<String>) -> Token {
        Token {
            form: form.into(),
            lemma: None,
            cpos: None,
            pos: None,
            features: None,
        }
    }

    /// Get the word form or punctuation symbol.
    pub fn form(&self) -> &str {
        self.form.as_ref()
    }

    /// Get the lemma or stem of the word form.
    pub fn lemma(&self) -> Option<&str> {
        self.lemma.as_ref().map(String::as_ref)
    }

    /// Get the coarse-grained part-of-speech tag.
    pub fn cpos(&self) -> Option<&str> {
        self.cpos.as_ref().map(String::as_ref)
    }

    /// Get the fine-grained part-of-speech tag.
    pub fn pos(&self) -> Option<&str> {
        self.pos.as_ref().map(String::as_ref)
    }

    /// Get the syntactic and/or morphological features of the token.
    pub fn features(&self) -> Option<&Features> {
        self.features.as_ref()
    }

    /// Get the syntactic and/or morphological features of the token.
    ///
    /// Returns a mutable reference, so that the features can be updated.
    pub fn features_mut(&mut self) -> Option<&mut Features> {
        self.features.as_mut()
    }

    /// Set the word form or punctuation symbol.
    ///
    /// Returns the form that is replaced.
    pub fn set_form(&mut self, form: impl Into<String>) -> String {
        mem::replace(&mut self.form, form.into())
    }

    /// Set the lemma or stem of the word form.
    ///
    /// Returns the lemma that is replaced.
    pub fn set_lemma<S>(&mut self, lemma: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.lemma, lemma.map(Into::into))
    }

    /// Set the coarse-grained part-of-speech tag.
    ///
    /// Returns the coarse-grained part-of-speech tag that is replaced.
    pub fn set_cpos<S>(&mut self, cpos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.cpos, cpos.map(Into::into))
    }

    /// Set the fine-grained part-of-speech tag.
    ///
    /// Returns the fine-grained part-of-speech tag that is replaced.
    pub fn set_pos<S>(&mut self, pos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.pos, pos.map(Into::into))
    }

    /// Set the syntactic and/or morphological features of the token.
    ///
    /// Returns the features that are replaced.
    pub fn set_features(&mut self, features: Option<Features>) -> Option<Features> {
        mem::replace(&mut self.features, features)
    }
}

/// Token features.
///
/// In the CoNLL-X specification, these are morphological features of the
/// token. Typically, the features are a list or a key-value mapping.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Features {
    inner: BTreeMap<String, Option<String>>,
}

impl Features {
    /// Unwrap the contained feature map.
    pub fn into_inner(self) -> BTreeMap<String, Option<String>> {
        self.inner
    }

    fn parse_features(feature_string: impl AsRef<str>) -> BTreeMap<String, Option<String>> {
        let mut features = BTreeMap::new();

        for fv in feature_string.as_ref().split('|') {
            let fv: &str = fv;
            let (k, v) = fv
                .find(':')
                .map(|idx| (fv[..idx].to_owned(), Some(fv[idx + 1..].to_owned())))
                .unwrap_or_else(|| (fv.to_owned(), None));
            features.insert(k, v);
        }

        features
    }
}

impl Deref for Features {
    type Target = BTreeMap<String, Option<String>>;

    fn deref(&self) -> &Self::Target {
        &self.inner
    }
}

impl DerefMut for Features {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.inner
    }
}

impl Display for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let feature_str: String = self.into();
        f.write_str(&feature_str)
    }
}

impl From<&str> for Features {
    fn from(feature_string: &str) -> Self {
        Features {
            inner: Features::parse_features(feature_string),
        }
    }
}

impl<S, T> FromIterator<(S, Option<T>)> for Features
where
    S: Into<String>,
    T: Into<String>,
{
    fn from_iter<I>(iter: I) -> Self
    where
        I: IntoIterator<Item = (S, Option<T>)>,
    {
        let features =
            BTreeMap::from_iter(iter.into_iter().map(|(k, v)| (k.into(), v.map(Into::into))));

        Features { inner: features }
    }
}

impl From<Features> for String {
    fn from(features: Features) -> Self {
        (&features).into()
    }
}

impl From<&Features> for String {
    fn from(features: &Features) -> Self {
        features
            .inner
            .iter()
            .map(|(k, v)| match *v {
                Some(ref v) => format!("{}:{}", k, v),
                None => k.to_owned(),
            })
            .join("|")
    }
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::iter::FromIterator;

    use maplit::btreemap;
    use quickcheck::quickcheck;

    use super::{Features, Token, TokenBuilder};

    quickcheck! {
        fn features_from_iter(feature_map: BTreeMap<String, Option<String>>) -> bool{
            feature_map == *Features::from_iter(feature_map.clone())
        }
    }

    #[test]
    fn features_from_iter_as_string() {
        let feature_map = btreemap! {
            "feature2" => Some("y"),
            "feature3" => None,
            "feature1" => Some("x")
        };

        let features = Features::from_iter(feature_map);
        let features_string: String = features.into();

        assert_eq!(features_string, "feature1:x|feature2:y|feature3");
    }

    #[test]
    fn features_with_colons() {
        let f = "Some:feature:with|additional:colons|feature";
        let features = Features::from(f);
        let some = features.get("Some").unwrap().as_ref().map(String::as_str);
        assert_eq!(some, Some("feature:with"));
        let additional = features
            .get("additional")
            .unwrap()
            .as_ref()
            .map(String::as_str);
        assert_eq!(additional, Some("colons"));
        let feature = features
            .get("feature")
            .unwrap()
            .as_ref()
            .map(String::as_str);
        assert_eq!(feature, None);
    }

    #[test]
    fn features() {
        let tokens = token_with_features();
        let features = features_correct();

        for (token, correct) in tokens.iter().zip(features) {
            let kv = &**token.features().unwrap();
            assert_eq!(&correct, kv);
        }
    }

    fn token_with_features() -> Vec<Token> {
        vec![
            TokenBuilder::new("Gilles")
                .lemma("Gilles")
                .cpos("N")
                .pos("NE")
                .features(Features::from(
                    "case:nominative|number:singular|gender:masculine",
                ))
                .into(),
            TokenBuilder::new("Deleuze")
                .lemma("Deleuze")
                .cpos("N")
                .pos("NE")
                .features(Features::from("nominative|singular|masculine"))
                .into(),
        ]
    }

    fn features_correct() -> Vec<BTreeMap<String, Option<String>>> {
        let mut correct1 = BTreeMap::new();
        correct1.insert("case".to_owned(), Some("nominative".to_owned()));
        correct1.insert("number".to_owned(), Some("singular".to_owned()));
        correct1.insert("gender".to_owned(), Some("masculine".to_owned()));

        let mut correct2 = BTreeMap::new();
        correct2.insert("nominative".to_owned(), None);
        correct2.insert("singular".to_owned(), None);
        correct2.insert("masculine".to_owned(), None);

        vec![correct1, correct2]
    }

    #[test]
    fn eq_features_is_order_insensitive() {
        let token1: Token = TokenBuilder::new("a")
            .features(Features::from("a|b:c"))
            .into();
        let token2 = TokenBuilder::new("a")
            .features(Features::from("b:c|a"))
            .into();
        let token3: Token = TokenBuilder::new("a")
            .features(Features::from("b|a:c"))
            .into();

        assert_eq!(token1, token2);
        assert_ne!(token1, token3);
        assert_ne!(token2, token3);
    }
}
<commit_msg>Remove unused import<commit_after>//! Tokens in the dependency graph.

use std::collections::BTreeMap;
use std::fmt;
use std::fmt::Display;
use std::iter::FromIterator;
use std::mem;
use std::ops::{Deref, DerefMut};

use itertools::Itertools;

pub const EMPTY_TOKEN: &str = "_";

/// A builder for `Token`s.
///
/// The `Token` type stores a CoNLL-X token. However, since this format
/// permits a large number of fields, construction of a token can get
/// tedious. This builder provides a fluent interface for creating `Token`s.
pub struct TokenBuilder {
    token: Token,
}

impl TokenBuilder {
    /// Create a `Token` builder with all non-form fields set to absent.
    pub fn new(form: impl Into<String>) -> TokenBuilder {
        TokenBuilder {
            token: Token::new(form),
        }
    }

    /// Set the word form or punctuation symbol.
    pub fn form(mut self, form: impl Into<String>) -> TokenBuilder {
        self.token.set_form(form);
        self
    }

    /// Set the lemma or stem of the word form.
    pub fn lemma(mut self, lemma: impl Into<String>) -> TokenBuilder {
        self.token.set_lemma(Some(lemma));
        self
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn cpos(mut self, cpos: impl Into<String>) -> TokenBuilder {
        self.token.set_cpos(Some(cpos));
        self
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn pos(mut self, pos: impl Into<String>) -> TokenBuilder {
        self.token.set_pos(Some(pos));
        self
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn features(mut self, features: Features) -> TokenBuilder {
        self.token.set_features(Some(features));
        self
    }
}

impl From<Token> for TokenBuilder {
    fn from(token: Token) -> Self {
        TokenBuilder { token }
    }
}

impl From<TokenBuilder> for Token {
    fn from(builder: TokenBuilder) -> Self {
        builder.token
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    form: String,
    lemma: Option<String>,
    cpos: Option<String>,
    pos: Option<String>,
    features: Option<Features>,
}

impl Token {
    /// Create a new token where all the non-form fields are absent.
    pub fn new(form: impl Into<String>) -> Token {
        Token {
            form: form.into(),
            lemma: None,
            cpos: None,
            pos: None,
            features: None,
        }
    }

    /// Get the word form or punctuation symbol.
    pub fn form(&self) -> &str {
        self.form.as_ref()
    }

    /// Get the lemma or stem of the word form.
    pub fn lemma(&self) -> Option<&str> {
        self.lemma.as_ref().map(String::as_ref)
    }

    /// Get the coarse-grained part-of-speech tag.
    pub fn cpos(&self) -> Option<&str> {
        self.cpos.as_ref().map(String::as_ref)
    }

    /// Get the fine-grained part-of-speech tag.
    pub fn pos(&self) -> Option<&str> {
        self.pos.as_ref().map(String::as_ref)
    }

    /// Get the syntactic and/or morphological features of the token.
    pub fn features(&self) -> Option<&Features> {
        self.features.as_ref()
    }

    /// Get the syntactic and/or morphological features of the token.
    ///
    /// Returns a mutable reference, so that the features can be updated.
    pub fn features_mut(&mut self) -> Option<&mut Features> {
        self.features.as_mut()
    }

    /// Set the word form or punctuation symbol.
    ///
    /// Returns the form that is replaced.
    pub fn set_form(&mut self, form: impl Into<String>) -> String {
        mem::replace(&mut self.form, form.into())
    }

    /// Set the lemma or stem of the word form.
    ///
    /// Returns the lemma that is replaced.
    pub fn set_lemma<S>(&mut self, lemma: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.lemma, lemma.map(Into::into))
    }

    /// Set the coarse-grained part-of-speech tag.
    ///
    /// Returns the coarse-grained part-of-speech tag that is replaced.
    pub fn set_cpos<S>(&mut self, cpos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.cpos, cpos.map(Into::into))
    }

    /// Set the fine-grained part-of-speech tag.
    ///
    /// Returns the fine-grained part-of-speech tag that is replaced.
    pub fn set_pos<S>(&mut self, pos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.pos, pos.map(Into::into))
    }

    /// Set the syntactic and/or morphological features of the token.
    ///
    /// Returns the features that are replaced.
    pub fn set_features(&mut self, features: Option<Features>) -> Option<Features> {
        mem::replace(&mut self.features, features)
    }
}

/// Token features.
///
/// In the CoNLL-X specification, these are morphological features of the
/// token. Typically, the features are a list or a key-value mapping.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Features {
    inner: BTreeMap<String, Option<String>>,
}

impl Features {
    /// Unwrap the contained feature map.
    pub fn into_inner(self) -> BTreeMap<String, Option<String>> {
        self.inner
    }

    fn parse_features(feature_string: impl AsRef<str>) -> BTreeMap<String, Option<String>> {
        let mut features = BTreeMap::new();

        for fv in feature_string.as_ref().split('|') {
            let fv: &str = fv;
            let (k, v) = fv
                .find(':')
                .map(|idx| (fv[..idx].to_owned(), Some(fv[idx + 1..].to_owned())))
                .unwrap_or_else(|| (fv.to_owned(), None));
            features.insert(k, v);
        }

        features
    }
}

impl Deref for Features {
    type Target = BTreeMap<String, Option<String>>;

    fn deref(&self) -> &Self::Target {
        &self.inner
    }
}

impl DerefMut for Features {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.inner
    }
}

impl Display for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let feature_str: String = self.into();
        f.write_str(&feature_str)
    }
}

impl From<&str> for Features {
    fn from(feature_string: &str) -> Self {
        Features {
            inner: Features::parse_features(feature_string),
        }
    }
}

impl<S, T> FromIterator<(S, Option<T>)> for Features
where
    S: Into<String>,
    T: Into<String>,
{
    fn from_iter<I>(iter: I) -> Self
    where
        I: IntoIterator<Item = (S, Option<T>)>,
    {
        let features =
            BTreeMap::from_iter(iter.into_iter().map(|(k, v)| (k.into(), v.map(Into::into))));

        Features { inner: features }
    }
}

impl From<Features> for String {
    fn from(features: Features) -> Self {
        (&features).into()
    }
}

impl From<&Features> for String {
    fn from(features: &Features) -> Self {
        features
            .inner
            .iter()
            .map(|(k, v)| match *v {
                Some(ref v) => format!("{}:{}", k, v),
                None => k.to_owned(),
            })
            .join("|")
    }
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::iter::FromIterator;

    use maplit::btreemap;
    use quickcheck::quickcheck;

    use super::{Features, Token, TokenBuilder};

    quickcheck! {
        fn features_from_iter(feature_map: BTreeMap<String, Option<String>>) -> bool{
            feature_map == *Features::from_iter(feature_map.clone())
        }
    }

    #[test]
    fn features_from_iter_as_string() {
        let feature_map = btreemap! {
            "feature2" => Some("y"),
            "feature3" => None,
            "feature1" => Some("x")
        };

        let features = Features::from_iter(feature_map);
        let features_string: String = features.into();

        assert_eq!(features_string, "feature1:x|feature2:y|feature3");
    }

    #[test]
    fn features_with_colons() {
        let f = "Some:feature:with|additional:colons|feature";
        let features = Features::from(f);
        let some = features.get("Some").unwrap().as_ref().map(String::as_str);
        assert_eq!(some, Some("feature:with"));
        let additional = features
            .get("additional")
            .unwrap()
            .as_ref()
            .map(String::as_str);
        assert_eq!(additional, Some("colons"));
        let feature = features
            .get("feature")
            .unwrap()
            .as_ref()
            .map(String::as_str);
        assert_eq!(feature, None);
    }

    #[test]
    fn features() {
        let tokens = token_with_features();
        let features = features_correct();

        for (token, correct) in tokens.iter().zip(features) {
            let kv = &**token.features().unwrap();
            assert_eq!(&correct, kv);
        }
    }

    fn token_with_features() -> Vec<Token> {
        vec![
            TokenBuilder::new("Gilles")
                .lemma("Gilles")
                .cpos("N")
                .pos("NE")
                .features(Features::from(
                    "case:nominative|number:singular|gender:masculine",
                ))
                .into(),
            TokenBuilder::new("Deleuze")
                .lemma("Deleuze")
                .cpos("N")
                .pos("NE")
                .features(Features::from("nominative|singular|masculine"))
                .into(),
        ]
    }

    fn features_correct() -> Vec<BTreeMap<String, Option<String>>> {
        let mut correct1 = BTreeMap::new();
        correct1.insert("case".to_owned(), Some("nominative".to_owned()));
        correct1.insert("number".to_owned(), Some("singular".to_owned()));
        correct1.insert("gender".to_owned(), Some("masculine".to_owned()));

        let mut correct2 = BTreeMap::new();
        correct2.insert("nominative".to_owned(), None);
        correct2.insert("singular".to_owned(), None);
        correct2.insert("masculine".to_owned(), None);

        vec![correct1, correct2]
    }

    #[test]
    fn eq_features_is_order_insensitive() {
        let token1: Token = TokenBuilder::new("a")
            .features(Features::from("a|b:c"))
            .into();
        let token2 = TokenBuilder::new("a")
            .features(Features::from("b:c|a"))
            .into();
        let token3: Token = TokenBuilder::new("a")
            .features(Features::from("b|a:c"))
            .into();

        assert_eq!(token1, token2);
        assert_ne!(token1, token3);
        assert_ne!(token2, token3);
    }
}
<|endoftext|>