<commit_before>use std::error::Error;
use std::fmt;

/// Basic type for errors returned from the server.
#[derive(Debug, EnumConvertFromInt, EnumError, PartialEq)]
#[EnumConvertFromIntFallback = "Unimplemented"]
pub enum ZkError {
    /// This code is never returned from the server. It should not be used other than to indicate a
    /// range. Specifically error codes greater than this value are API errors (while values less
    /// than this indicate a system error.
    APIError = -100,
    /// Client authentication failed.
    AuthFailed = -115,
    /// Invalid arguments.
    BadArguments = -8,
    /// Version conflict in `set` operation. In case of reconfiguration: reconfig requested from
    /// config version X but last seen config has a different version Y.
    BadVersion = -103,
    /// Connection to the server has been lost.
    ConnectionLoss = -4,
    /// A data inconsistency was found.
    DataInconsistency = -3,
    /// Attempt to create ephemeral node on a local session.
    EphemeralOnLocalSession = -120,
    /// Invalid `Acl` specified.
    InvalidACL = -114,
    /// Invalid callback specified.
    InvalidCallback = -113,
    /// Error while marshalling or unmarshalling data.
    MarshallingError = -5,
    /// Not authenticated.
    NoAuth = -102,
    /// Ephemeral nodes may not have children.
    NoChildrenForEphemerals = -108,
    /// Request to create node that already exists.
    NodeExists = -110,
    /// Attempted to read a node that does not exist.
    NoNode = -101,
    /// The node has children.
    NotEmpty = -111,
    /// State-changing request is passed to read-only server.
    NotReadOnly = -119,
    /// Attempt to remove a non-existing watcher.
    NoWatcher = -121,
    /// Operation timeout.
    OperationTimeout = -7,
    /// A runtime inconsistency was found.
    RuntimeInconsistency = -2,
    /// The session has been expired by the server.
    SessionExpired = -112,
    /// Session moved to another server, so operation is ignored.
    SessionMoved = -118,
    /// System and server-side errors. This is never thrown by the server, it shouldn't be used
    /// other than to indicate a range. Specifically error codes greater than this value, but lesser
    /// than `APIError`, are system errors.
    SystemError = -1,
    /// Operation is unimplemented.
    Unimplemented = -6
}

impl fmt::Display for ZkError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Zookeeper Error: {}", self.description())
    }
}

/// CreateMode value determines how the znode is created on ZooKeeper.
#[derive(Clone, Debug, PartialEq)]
pub enum CreateMode {
    /// The znode will not be automatically deleted upon client's disconnect.
    Persistent = 0,
    /// The znode will be deleted upon the client's disconnect.
    Ephemeral = 1,
    /// The name of the znode will be appended with a monotonically increasing number. The actual
    /// path name of a sequential node will be the given path plus a suffix `"i"` where *i* is the
    /// current sequential number of the node. The sequence number is always fixed length of 10
    /// digits, 0 padded. Once such a node is created, the sequential number will be incremented by
    /// one.
    PersistentSequential = 2,
    /// The znode will be deleted upon the client's disconnect, and its name will be appended with a
    /// monotonically increasing number.
    EphemeralSequential = 3,
    /// Container nodes are special purpose nodes useful for recipes such as leader, lock, etc. When
    /// the last child of a container is deleted, the container becomes a candidate to be deleted by
    /// the server at some point in the future. Given this property, you should be prepared to get
    /// `ZkError::NoNode` when creating children inside of this container node.
    Container = 4,
}

/// Enumeration of states the client may be at a Watcher Event. It represents the state of the
/// server at the time the event was generated.
#[derive(Clone, Debug, EnumConvertFromInt, PartialEq)]
pub enum KeeperState {
    /// The client is in the disconnected state - it is not connected to any server in the ensemble.
    Disconnected = 0,
    /// The client is in the connected state - it is connected to a server in the ensemble (one of
    /// the servers specified in the host connection parameter during ZooKeeper client creation).
    SyncConnected = 3,
    /// Authentication has failed -- connection requires a new `ZooKeeper` instance.
    AuthFailed = 4,
    /// The client is connected to a read-only server, that is the server which is not currently
    /// connected to the majority. The only operations allowed after receiving this state is read
    /// operations. This state is generated for read-only clients only since read/write clients
    /// aren't allowed to connect to read-only servers.
    ConnectedReadOnly = 5,
    /// Used to notify clients that they are SASL-authenticated, so that they can perform ZooKeeper
    /// actions with their SASL-authorized permissions.
    SaslAuthenticated = 6,
    /// The serving cluster has expired this session. The ZooKeeper client connection (the session)
    /// is no longer valid. You must create a new client connection (instantiate a new `ZooKeeper`
    /// instance) if you with to access the ensemble.
    Expired = -112
}

/// Enumeration of types of events that may occur on the znode.
#[derive(Clone, Debug, EnumConvertFromInt)]
pub enum WatchedEventType {
    /// Nothing known has occurred on the znode. This value is issued as part of a `WatchedEvent`
    /// when the `KeeperState` changes.
    None = -1,
    /// Issued when a znode at a given path is created.
    NodeCreated = 1,
    /// Issued when a znode at a given path is deleted.
    NodeDeleted = 2,
    /// Issued when the data of a watched znode are altered. This event value is issued whenever a
    /// *set* operation occurs without an actual contents check, so there is no guarantee the data
    /// actually changed.
    NodeDataChanged = 3,
    /// Issued when the children of a watched znode are created or deleted. This event is not issued
    /// when the data within children is altered.
    NodeChildrenChanged = 4,
    /// Issued when the client removes a data watcher.
    DataWatchRemoved = 5,
    /// Issued when the client removes a child watcher.
    ChildWatchRemoved = 6
}

/// Enumeration of states the client may be at any time.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ZkState {
    /// Previously used to represent a state between connection (as in connected to a server) and
    /// authenticated. This is no longer used.
    #[deprecated]
    Associating,
    /// Authentication has failed. Operations will return `ZkError::AuthFailed`.
    AuthFailed,
    /// The session has ended. Operations will return `ZkError::SessionExpired`.
    Closed,
    /// Session has been fully established. Operations will proceed as normal.
    Connected,
    /// Connected to a read-only server. See `KeeperState::ConnectedReadOnly`.
    ConnectedReadOnly,
    /// Currently attempting to connect with an ensemble member. Operations are queued until a
    /// session is established.
    Connecting,
    /// Theoretically used as a special state to represent `ZkError::ConnectionLoss` for expected
    /// reasons (ensemble reconfiguration), but `Closed` has proven less error-prone. This is no
    /// longer used.
    #[deprecated]
    NotConnected,
}
<commit_msg>add Copy trait to trivially copyable types<commit_after>use std::error::Error;
use std::fmt;

/// Basic type for errors returned from the server.
#[derive(Clone, Copy, Debug, EnumConvertFromInt, EnumError, PartialEq)]
#[EnumConvertFromIntFallback = "Unimplemented"]
pub enum ZkError {
    /// This code is never returned from the server. It should not be used other than to indicate a
    /// range. Specifically error codes greater than this value are API errors (while values less
    /// than this indicate a system error.
    APIError = -100,
    /// Client authentication failed.
    AuthFailed = -115,
    /// Invalid arguments.
    BadArguments = -8,
    /// Version conflict in `set` operation. In case of reconfiguration: reconfig requested from
    /// config version X but last seen config has a different version Y.
    BadVersion = -103,
    /// Connection to the server has been lost.
    ConnectionLoss = -4,
    /// A data inconsistency was found.
    DataInconsistency = -3,
    /// Attempt to create ephemeral node on a local session.
    EphemeralOnLocalSession = -120,
    /// Invalid `Acl` specified.
    InvalidACL = -114,
    /// Invalid callback specified.
    InvalidCallback = -113,
    /// Error while marshalling or unmarshalling data.
    MarshallingError = -5,
    /// Not authenticated.
    NoAuth = -102,
    /// Ephemeral nodes may not have children.
    NoChildrenForEphemerals = -108,
    /// Request to create node that already exists.
    NodeExists = -110,
    /// Attempted to read a node that does not exist.
    NoNode = -101,
    /// The node has children.
    NotEmpty = -111,
    /// State-changing request is passed to read-only server.
    NotReadOnly = -119,
    /// Attempt to remove a non-existing watcher.
    NoWatcher = -121,
    /// Operation timeout.
    OperationTimeout = -7,
    /// A runtime inconsistency was found.
    RuntimeInconsistency = -2,
    /// The session has been expired by the server.
    SessionExpired = -112,
    /// Session moved to another server, so operation is ignored.
    SessionMoved = -118,
    /// System and server-side errors. This is never thrown by the server, it shouldn't be used
    /// other than to indicate a range. Specifically error codes greater than this value, but lesser
    /// than `APIError`, are system errors.
    SystemError = -1,
    /// Operation is unimplemented.
    Unimplemented = -6
}

impl fmt::Display for ZkError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Zookeeper Error: {}", self.description())
    }
}

/// CreateMode value determines how the znode is created on ZooKeeper.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum CreateMode {
    /// The znode will not be automatically deleted upon client's disconnect.
    Persistent = 0,
    /// The znode will be deleted upon the client's disconnect.
    Ephemeral = 1,
    /// The name of the znode will be appended with a monotonically increasing number. The actual
    /// path name of a sequential node will be the given path plus a suffix `"i"` where *i* is the
    /// current sequential number of the node. The sequence number is always fixed length of 10
    /// digits, 0 padded. Once such a node is created, the sequential number will be incremented by
    /// one.
    PersistentSequential = 2,
    /// The znode will be deleted upon the client's disconnect, and its name will be appended with a
    /// monotonically increasing number.
    EphemeralSequential = 3,
    /// Container nodes are special purpose nodes useful for recipes such as leader, lock, etc. When
    /// the last child of a container is deleted, the container becomes a candidate to be deleted by
    /// the server at some point in the future. Given this property, you should be prepared to get
    /// `ZkError::NoNode` when creating children inside of this container node.
    Container = 4,
}

/// Enumeration of states the client may be at a Watcher Event. It represents the state of the
/// server at the time the event was generated.
#[derive(Clone, Copy, Debug, EnumConvertFromInt, PartialEq)]
pub enum KeeperState {
    /// The client is in the disconnected state - it is not connected to any server in the ensemble.
    Disconnected = 0,
    /// The client is in the connected state - it is connected to a server in the ensemble (one of
    /// the servers specified in the host connection parameter during ZooKeeper client creation).
    SyncConnected = 3,
    /// Authentication has failed -- connection requires a new `ZooKeeper` instance.
    AuthFailed = 4,
    /// The client is connected to a read-only server, that is the server which is not currently
    /// connected to the majority. The only operations allowed after receiving this state is read
    /// operations. This state is generated for read-only clients only since read/write clients
    /// aren't allowed to connect to read-only servers.
    ConnectedReadOnly = 5,
    /// Used to notify clients that they are SASL-authenticated, so that they can perform ZooKeeper
    /// actions with their SASL-authorized permissions.
    SaslAuthenticated = 6,
    /// The serving cluster has expired this session. The ZooKeeper client connection (the session)
    /// is no longer valid. You must create a new client connection (instantiate a new `ZooKeeper`
    /// instance) if you with to access the ensemble.
    Expired = -112
}

/// Enumeration of types of events that may occur on the znode.
#[derive(Clone, Copy, Debug, EnumConvertFromInt)]
pub enum WatchedEventType {
    /// Nothing known has occurred on the znode. This value is issued as part of a `WatchedEvent`
    /// when the `KeeperState` changes.
    None = -1,
    /// Issued when a znode at a given path is created.
    NodeCreated = 1,
    /// Issued when a znode at a given path is deleted.
    NodeDeleted = 2,
    /// Issued when the data of a watched znode are altered. This event value is issued whenever a
    /// *set* operation occurs without an actual contents check, so there is no guarantee the data
    /// actually changed.
    NodeDataChanged = 3,
    /// Issued when the children of a watched znode are created or deleted. This event is not issued
    /// when the data within children is altered.
    NodeChildrenChanged = 4,
    /// Issued when the client removes a data watcher.
    DataWatchRemoved = 5,
    /// Issued when the client removes a child watcher.
    ChildWatchRemoved = 6
}

/// Enumeration of states the client may be at any time.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ZkState {
    /// Previously used to represent a state between connection (as in connected to a server) and
    /// authenticated. This is no longer used.
    #[deprecated]
    Associating,
    /// Authentication has failed. Operations will return `ZkError::AuthFailed`.
    AuthFailed,
    /// The session has ended. Operations will return `ZkError::SessionExpired`.
    Closed,
    /// Session has been fully established. Operations will proceed as normal.
    Connected,
    /// Connected to a read-only server. See `KeeperState::ConnectedReadOnly`.
    ConnectedReadOnly,
    /// Currently attempting to connect with an ensemble member. Operations are queued until a
    /// session is established.
    Connecting,
    /// Theoretically used as a special state to represent `ZkError::ConnectionLoss` for expected
    /// reasons (ensemble reconfiguration), but `Closed` has proven less error-prone. This is no
    /// longer used.
    #[deprecated]
    NotConnected,
}
<|endoftext|>


<commit_before>//!
//!	This module contains the code to sum up n-mer aligned read counts based on a
//! SeqTable instance.
//!
use htslib::bam;
use htslib::bam::Read;
use htslib::bam::Reader;
use htslib::bam::Records;
use std::fs::File;
use std::io::Read as ioRead;
use std::io::Seek;
use std::process::exit;
use std::iter::Peekable;
use seqtable::SeqTable;

fn process_bam_seq<R: ioRead+Seek>(counts: &mut Vec<(u64, u64, u64, u64)>, table: &mut SeqTable<R>, bamrecs: &mut Peekable<Records<Reader>>, tid: &mut i32, map: &Vec<usize>, minqual: u8) -> bool {
    let mut rdr = table.get_sequence_by_idx(map[*tid as usize]).ok().expect("read sequence");

    loop {
        // check if we changed sequence
        match bamrecs.peek() {
            Some(&Ok(ref rec)) => if rec.tid() != *tid { *tid = rec.tid(); return true; },
            Some(&Err(_)) => return false,
            None => return false,
        }

        // if not count position
        if let Some(Ok(record)) = bamrecs.next() {
            if record.mapq() >= minqual {
                let pair = rdr.get(record.pos() as u64).unwrap();

                if record.is_reverse() {
                    counts[pair.1 as usize].3 += 1;
                } else {
                    counts[pair.0 as usize].2 += 1;
                }
            }
        }
    }
}

pub fn tabulate(seqfile: &str, bamfile: Option<String>, minqual: u8) {
    // read
    let file = File::open(seqfile).ok().expect("read file");
    let mut table = SeqTable::open(file).ok().expect("read store");

    // allocate counts table
    let mut counts: Vec<(u64, u64, u64, u64)> = Vec::new();
    let nmer_count = 4u64.pow(table.params.cut_length as u32) + 1;

    for _ in 0..nmer_count {
        counts.push((0,0,0,0));
    }

    let seqinfos = table.sequences();

    for idx in 0..seqinfos.len() {
        let length = seqinfos[idx].length;
        let mut rdr = table.get_sequence_by_idx(idx).ok().expect("read sequence");

        for i in 0..length {
            let pair = rdr.get(i).unwrap();

            counts[pair.0 as usize].0 += 1;
            counts[pair.1 as usize].1 += 1;
        }
    }

    // if we received a BAM file, parse it
    if let Some(bamfilename) = bamfile {
        let bam = bam::Reader::new(&bamfilename).ok().expect("Error opening bam.");
	    let names = bam.header.target_names();
        let mut cur_tid = 0;

        // map BAM tid's to SeqTable idx's
        let map : Vec<usize> = names.iter().map(|&id| {
            let chrom = String::from_utf8_lossy(id);
            for i in 0..seqinfos.len() {
                if seqinfos[i].name.eq(&chrom) {
                    return i;
                }
            }
            println!("Error: Unknown sequence name in BAM: {}", chrom);
            exit(1);
        }
        ).collect();

        // reads
        let mut iter = bam.records().peekable();

        while process_bam_seq(&mut counts, &mut table, &mut iter, &mut cur_tid, &map, minqual) {}

        // TODO: tmp: output table
        for i in 1..counts.len() {
            let (plus, minus, bam_plus, bam_minus) = counts[i];
            println!("{}\t{}\t{}\t{}\t{}", i, plus, minus, bam_plus, bam_minus);
        }
    } else {

        // TODO: tmp: output table
        for i in 1..counts.len() {
            let (plus, minus, _, _) = counts[i];
            println!("{}\t{}\t{}", i, plus, minus);
        }
    }
}<commit_msg>Add filter on read length and "mapped" status<commit_after>//!
//!	This module contains the code to sum up n-mer aligned read counts based on a
//! SeqTable instance.
//!
use htslib::bam;
use htslib::bam::Read;
use htslib::bam::Reader;
use htslib::bam::Records;
use std::fs::File;
use std::io::Read as ioRead;
use std::io::Seek;
use std::process::exit;
use std::iter::Peekable;
use seqtable::SeqTable;

fn process_bam_seq<R: ioRead+Seek>(counts: &mut Vec<(u64, u64, u64, u64)>, table: &mut SeqTable<R>, bamrecs: &mut Peekable<Records<Reader>>, tid: &mut i32, map: &Vec<usize>, rlen: usize, minqual: u8) -> bool {
    let mut rdr = table.get_sequence_by_idx(map[*tid as usize]).ok().expect("read sequence");

    loop {
        // check if we changed sequence
        match bamrecs.peek() {
            Some(&Ok(ref rec)) => if rec.tid() != *tid { *tid = rec.tid(); return true; },
            Some(&Err(_)) => return false,
            None => return false,
        }

        // if not count position
        if let Some(Ok(record)) = bamrecs.next() {
            if !record.is_unmapped() && record.seq().len() == rlen && record.mapq() >= minqual {
                let pair = rdr.get(record.pos() as u64).unwrap();

                if record.is_reverse() {
                    counts[pair.1 as usize].3 += 1;
                } else {
                    counts[pair.0 as usize].2 += 1;
                }
            }
        }
    }
}

pub fn tabulate(seqfile: &str, bamfile: Option<String>, minqual: u8) {
    // read
    let file = File::open(seqfile).ok().expect("read file");
    let mut table = SeqTable::open(file).ok().expect("read store");

    // allocate counts table
    let mut counts: Vec<(u64, u64, u64, u64)> = Vec::new();
    let nmer_count = 4u64.pow(table.params.cut_length as u32) + 1;

    for _ in 0..nmer_count {
        counts.push((0,0,0,0));
    }

    let rlen = table.params.read_length as usize;
    let seqinfos = table.sequences();

    for idx in 0..seqinfos.len() {
        let length = seqinfos[idx].length;
        let mut rdr = table.get_sequence_by_idx(idx).ok().expect("read sequence");

        for i in 0..length {
            let pair = rdr.get(i).unwrap();

            counts[pair.0 as usize].0 += 1;
            counts[pair.1 as usize].1 += 1;
        }
    }

    // if we received a BAM file, parse it
    if let Some(bamfilename) = bamfile {
        let bam = bam::Reader::new(&bamfilename).ok().expect("Error opening bam.");
	    let names = bam.header.target_names();
        let mut cur_tid = 0;

        // map BAM tid's to SeqTable idx's
        let map : Vec<usize> = names.iter().map(|&id| {
            let chrom = String::from_utf8_lossy(id);
            for i in 0..seqinfos.len() {
                if seqinfos[i].name.eq(&chrom) {
                    return i;
                }
            }
            println!("Error: Unknown sequence name in BAM: {}", chrom);
            exit(1);
        }
        ).collect();

        // reads
        let mut iter = bam.records().peekable();

        while process_bam_seq(&mut counts, &mut table, &mut iter, &mut cur_tid, &map, rlen, minqual) {}

        // TODO: tmp: output table
        for i in 1..counts.len() {
            let (plus, minus, bam_plus, bam_minus) = counts[i];
            println!("{}\t{}\t{}\t{}\t{}", i, plus, minus, bam_plus, bam_minus);
        }
    } else {

        // TODO: tmp: output table
        for i in 1..counts.len() {
            let (plus, minus, _, _) = counts[i];
            println!("{}\t{}\t{}", i, plus, minus);
        }
    }
}<|endoftext|>


<commit_before>use std;

fn main() {
    std::io::println("
 Mr. Goodkat: The reason I'm in town, in case you're wondering, is because of a
              Kansas City Shuffle.
        Nick: What's a Kansas City Shuffle?
 Mr. Goodkat: A Kansas City Shuffle is when everybody looks right, you go left.
        Nick: Never heard of it.
 Mr. Goodkat: It's not something people hear about. Falls on deaf ears mostly.
              This particular one has been over twenty years in the making.
        Nick: Twenty years, huh?
 Mr. Goodkat: No small matter. Requires a lot of planning. Involves a lot of
              people. People connected only by the slightest of events. Like
              whispers in the night, in that place that never forgets, even
              when those people do.
 Mr. Goodkat: It starts with a horse.");
}
<commit_msg>Update dummy .rs file<commit_after>fn main() {
    io::println("
 Mr. Goodkat: The reason I'm in town, in case you're wondering, is because of a
              Kansas City Shuffle.
        Nick: What's a Kansas City Shuffle?
 Mr. Goodkat: A Kansas City Shuffle is when everybody looks right, you go left.
        Nick: Never heard of it.
 Mr. Goodkat: It's not something people hear about. Falls on deaf ears mostly.
              This particular one has been over twenty years in the making.
        Nick: Twenty years, huh?
 Mr. Goodkat: No small matter. Requires a lot of planning. Involves a lot of
              people. People connected only by the slightest of events. Like
              whispers in the night, in that place that never forgets, even
              when those people do.
 Mr. Goodkat: It starts with a horse.");
}
<|endoftext|>


<commit_before>use super::scheduler::*;

use std::sync::Arc;
use futures::sync::oneshot;
use futures::future::Future;

///
/// A data storage structure used to govern synchronous and asynchronous access to an underlying object.
///
pub struct Desync<T: Send> {
    /// Queue used for scheduling runtime for this object
    queue:  Arc<JobQueue>,

    /// Data for this object. Boxed so the pointer remains the same through the lifetime of the object.
    data:   Box<T>
}

// Rust actually derives this anyway at the moment
unsafe impl<T: Send> Send for Desync<T> {}

// True iff queue: Sync
unsafe impl<T: Send> Sync for Desync<T> {}

///
/// Used for passing the data pointer through to the queue
///
/// 'Safe' because the queue is synchronised during drop, so we can never use the pointer
/// if the object does not exist.
///
struct DataRef<T: Send>(*const T);
unsafe impl<T: Send> Send for DataRef<T> {}

// TODO: we can change DataRef to Shared (https://doc.rust-lang.org/std/ptr/struct.Shared.html in the future)

// TODO: T does not need to be static as we know that its lifetime is at least the lifetime of Desync<T> and hence the queue
impl<T: 'static+Send> Desync<T> {
    ///
    /// Creates a new Desync object
    ///
    pub fn new(data: T) -> Desync<T> {
        let queue = queue();

        Desync {
            queue:  queue,
            data:   Box::new(data)
        }
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    pub fn async<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        unsafe {
            // As drop() is the last thing called, we know that this object will still exist at the point where
            let data = DataRef(&*self.data);

            async(&self.queue, move || {
                let data = data.0 as *mut T;
                job(&mut *data);
            })
        }
    }

    ///
    /// Performs an operation synchronously on this item. This will be queued with any other
    /// jobs that this item may be performing, and this function will not return until the
    /// job is complete and the result is available.
    ///
    pub fn sync<TFn, Result>(&self, job: TFn) -> Result
    where TFn: Send+FnOnce(&mut T) -> Result, Result: Send {
        let result = unsafe {
            // As drop() is the last thing called, we know that this object will still exist at the point where
            let data = DataRef(&*self.data);

            sync(&self.queue, move || {
                let data = data.0 as *mut T;
                job(&mut *data)
            })
        };

        result
    }

    ///
    /// Performs an operation asynchronously on this thread, returning the result via a future.
    ///
    pub fn future<TFn, Item: 'static+Send>(&self, job: TFn) -> Box<Future<Item=Item, Error=oneshot::Canceled>>
    where TFn: 'static+Send+FnOnce(&mut T) -> Item {
        let (send, receive) = oneshot::channel();

        self.async(|data| {
            let result = job(data);

            if let Err(e) = send.send(result) {
                panic!(e);
            }
        });

        Box::new(receive)
    }
}

impl<T: Send> Drop for Desync<T> {
    fn drop(&mut self) {
        // Ensure that everything on the queue has committed by queueing a last synchronous event
        // (Not synchronising the queue would make this unsafe as we would hold on to a pointer to
        // the internal data structure)
        sync(&self.queue, || {});
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::thread::*;
    use std::time::*;
    use super::super::scheduler::test::timeout;

    struct TestData {
        val: u32
    }

    #[test]
    fn can_retrieve_data_synchronously() {
        let desynced = Desync::new(TestData { val: 0 });

        assert!(desynced.sync(|data| data.val) == 0);
    }

    #[test]
    fn can_update_data_asynchronously() {
        let desynced = Desync::new(TestData { val: 0 });

        desynced.async(|data| {
            sleep(Duration::from_millis(100));
            data.val = 42;
        });

        assert!(desynced.sync(|data| data.val) == 42);
    }

    #[test]
    fn can_update_data_asynchronously_1000_times() {
        for _i in 0..1000 {
            timeout(|| {
                let desynced = Desync::new(TestData { val: 0 });

                desynced.async(|data| {
                    sleep(Duration::from_millis(1));
                    data.val = 42;
                });

                assert!(desynced.sync(|data| data.val) == 42);
            }, 500);
        }
    }

    #[test]
    fn can_update_data_with_future() {
        timeout(|| {
            use futures::executor;

            let desynced = Desync::new(TestData { val: 0 });

            desynced.async(|data| {
                sleep(Duration::from_millis(100));
                data.val = 42;
            });

            let mut future = executor::spawn(desynced.future(|data| {
                data.val
            }));

            assert!(future.wait_future().unwrap() == 42);
        }, 500);
    }

    #[test]
    fn can_update_data_with_future_1000_times() {
        // Seems to timeout fairly reliably after signalling the future
        use futures::executor;

        for _i in 0..1000 {
            timeout(|| {
                let desynced = Desync::new(TestData { val: 0 });

                desynced.async(|data| {
                    sleep(Duration::from_millis(1));
                    data.val = 42;
                });

                let mut future = executor::spawn(desynced.future(|data| data.val));

                assert!(future.wait_future().unwrap() == 42);
            }, 500);
        }
    }

    #[test]
    fn dropping_while_running_isnt_obviously_bad() {
        let desynced = Desync::new(TestData { val: 0 });

        desynced.async(|data| {
            sleep(Duration::from_millis(100));
            data.val = 42;
        });
        desynced.async(|data| {
            sleep(Duration::from_millis(100));
            data.val = 42;
        });
    }
}
<commit_msg>Add change that seems to reliably cause the version without using futures to fail with a timeout similarly<commit_after>use super::scheduler::*;

use std::sync::Arc;
use futures::sync::oneshot;
use futures::future::Future;

///
/// A data storage structure used to govern synchronous and asynchronous access to an underlying object.
///
pub struct Desync<T: Send> {
    /// Queue used for scheduling runtime for this object
    queue:  Arc<JobQueue>,

    /// Data for this object. Boxed so the pointer remains the same through the lifetime of the object.
    data:   Box<T>
}

// Rust actually derives this anyway at the moment
unsafe impl<T: Send> Send for Desync<T> {}

// True iff queue: Sync
unsafe impl<T: Send> Sync for Desync<T> {}

///
/// Used for passing the data pointer through to the queue
///
/// 'Safe' because the queue is synchronised during drop, so we can never use the pointer
/// if the object does not exist.
///
struct DataRef<T: Send>(*const T);
unsafe impl<T: Send> Send for DataRef<T> {}

// TODO: we can change DataRef to Shared (https://doc.rust-lang.org/std/ptr/struct.Shared.html in the future)

// TODO: T does not need to be static as we know that its lifetime is at least the lifetime of Desync<T> and hence the queue
impl<T: 'static+Send> Desync<T> {
    ///
    /// Creates a new Desync object
    ///
    pub fn new(data: T) -> Desync<T> {
        let queue = queue();

        Desync {
            queue:  queue,
            data:   Box::new(data)
        }
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    pub fn async<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        unsafe {
            // As drop() is the last thing called, we know that this object will still exist at the point where
            let data = DataRef(&*self.data);

            async(&self.queue, move || {
                let data = data.0 as *mut T;
                job(&mut *data);
            })
        }
    }

    ///
    /// Performs an operation synchronously on this item. This will be queued with any other
    /// jobs that this item may be performing, and this function will not return until the
    /// job is complete and the result is available.
    ///
    pub fn sync<TFn, Result>(&self, job: TFn) -> Result
    where TFn: Send+FnOnce(&mut T) -> Result, Result: Send {
        let result = unsafe {
            // As drop() is the last thing called, we know that this object will still exist at the point where
            let data = DataRef(&*self.data);

            sync(&self.queue, move || {
                let data = data.0 as *mut T;
                job(&mut *data)
            })
        };

        result
    }

    ///
    /// Performs an operation asynchronously on this thread, returning the result via a future.
    ///
    pub fn future<TFn, Item: 'static+Send>(&self, job: TFn) -> Box<Future<Item=Item, Error=oneshot::Canceled>>
    where TFn: 'static+Send+FnOnce(&mut T) -> Item {
        let (send, receive) = oneshot::channel();

        self.async(|data| {
            let result = job(data);

            if let Err(e) = send.send(result) {
                panic!(e);
            }
        });

        Box::new(receive)
    }
}

impl<T: Send> Drop for Desync<T> {
    fn drop(&mut self) {
        // Ensure that everything on the queue has committed by queueing a last synchronous event
        // (Not synchronising the queue would make this unsafe as we would hold on to a pointer to
        // the internal data structure)
        sync(&self.queue, || {});
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::thread::*;
    use std::time::*;
    use super::super::scheduler::test::timeout;

    struct TestData {
        val: u32
    }

    #[test]
    fn can_retrieve_data_synchronously() {
        let desynced = Desync::new(TestData { val: 0 });

        assert!(desynced.sync(|data| data.val) == 0);
    }

    #[test]
    fn can_update_data_asynchronously() {
        let desynced = Desync::new(TestData { val: 0 });

        desynced.async(|data| {
            sleep(Duration::from_millis(100));
            data.val = 42;
        });

        assert!(desynced.sync(|data| data.val) == 42);
    }

    #[test]
    fn can_update_data_asynchronously_1000_times() {
        for _i in 0..1000 {
            timeout(|| {
                let desynced = Desync::new(TestData { val: 0 });

                desynced.async(|data| {
                    data.val = 42;
                });
                desynced.async(|data| {
                    data.val = 43;
                });

                assert!(desynced.sync(|data| data.val) == 43);
            }, 500);
        }
    }

    #[test]
    fn can_update_data_with_future() {
        timeout(|| {
            use futures::executor;

            let desynced = Desync::new(TestData { val: 0 });

            desynced.async(|data| {
                sleep(Duration::from_millis(100));
                data.val = 42;
            });

            let mut future = executor::spawn(desynced.future(|data| {
                data.val
            }));

            assert!(future.wait_future().unwrap() == 42);
        }, 500);
    }

    #[test]
    fn can_update_data_with_future_1000_times() {
        // Seems to timeout fairly reliably after signalling the future
        use futures::executor;

        for _i in 0..1000 {
            timeout(|| {
                let desynced = Desync::new(TestData { val: 0 });

                desynced.async(|data| {
                    sleep(Duration::from_millis(1));
                    data.val = 42;
                });

                let mut future = executor::spawn(desynced.future(|data| data.val));

                assert!(future.wait_future().unwrap() == 42);
            }, 500);
        }
    }

    #[test]
    fn dropping_while_running_isnt_obviously_bad() {
        let desynced = Desync::new(TestData { val: 0 });

        desynced.async(|data| {
            sleep(Duration::from_millis(100));
            data.val = 42;
        });
        desynced.async(|data| {
            sleep(Duration::from_millis(100));
            data.val = 42;
        });
    }
}
<|endoftext|>


<commit_before>use crate::fastfield::DeleteBitSet;
use crate::DocId;
use std::borrow::Borrow;
use std::borrow::BorrowMut;

/// Sentinel value returned when a DocSet has been entirely consumed.
///
/// This is not u32::MAX as one would have expected, due to the lack of SSE2 instructions
/// to compare [u32; 4].
pub const TERMINATED: DocId = std::i32::MAX as u32;

/// Represents an iterable set of sorted doc ids.
pub trait DocSet {
    /// Goes to the next element.
    ///
    /// The DocId of the next element is returned.
    /// In other words we should always have :
    /// ```ignore
    /// let doc = docset.advance();
    /// assert_eq!(doc, docset.doc());
    /// ```
    ///
    /// If we reached the end of the DocSet, TERMINATED should be returned.
    ///
    /// Calling `.advance()` on a terminated DocSet should be supported, and TERMINATED should
    /// be returned.
    /// TODO Test existing docsets.
    fn advance(&mut self) -> DocId;

    /// Advances the DocSet forward until reaching the target, or going to the
    /// lowest DocId greater than the target.
    ///
    /// If the end of the DocSet is reached, TERMINATED is returned.
    ///
    /// Calling `.seek(target)` on a terminated DocSet is legal. Implementation
    /// of DocSet should support it.
    ///
    /// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a DocSet.
    fn seek(&mut self, target: DocId) -> DocId {
        let mut doc = self.doc();
        debug_assert!(doc <= target);
        while doc < target {
            doc = self.advance();
        }
        doc
    }

    /// Fills a given mutable buffer with the next doc ids from the
    /// `DocSet`
    ///
    /// If that many `DocId`s are available, the method should
    /// fill the entire buffer and return the length of the buffer.
    ///
    /// If we reach the end of the `DocSet` before filling
    /// it entirely, then the buffer is filled up to this point, and
    /// return value is the number of elements that were filled.
    ///
    /// # Warning
    ///
    /// This method is only here for specific high-performance
    /// use case where batching. The normal way to
    /// go through the `DocId`'s is to call `.advance()`.
    fn fill_buffer(&mut self, buffer: &mut [DocId]) -> usize {
        if self.doc() == TERMINATED {
            return 0;
        }
        for (i, buffer_val) in buffer.iter_mut().enumerate() {
            *buffer_val = self.doc();
            if self.advance() == TERMINATED {
                return i + 1;
            }
        }
        buffer.len()
    }

    /// Returns the current document
    /// Right after creating a new DocSet, the docset points to the first document.
    ///
    /// If the DocSet is empty, .doc() should return `TERMINATED`.
    fn doc(&self) -> DocId;

    /// Returns a best-effort hint of the
    /// length of the docset.
    fn size_hint(&self) -> u32;

    /// Returns the number documents matching.
    /// Calling this method consumes the `DocSet`.
    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
        let mut count = 0u32;
        let mut doc = self.doc();
        while doc != TERMINATED {
            if !delete_bitset.is_deleted(doc) {
                count += 1u32;
            }
            doc = self.advance();
        }
        count
    }

    /// Returns the count of documents, deleted or not.
    /// Calling this method consumes the `DocSet`.
    ///
    /// Of course, the result is an upper bound of the result
    /// given by `count()`.
    fn count_including_deleted(&mut self) -> u32 {
        let mut count = 0u32;
        let mut doc = self.doc();
        while doc != TERMINATED {
            count += 1u32;
            doc = self.advance();
        }
        count
    }
}

impl<'a> DocSet for &'a mut dyn DocSet {
    fn advance(&mut self) -> u32 {
        (**self).advance()
    }

    fn seek(&mut self, target: DocId) -> DocId {
        (**self).seek(target)
    }

    fn doc(&self) -> u32 {
        (**self).doc()
    }

    fn size_hint(&self) -> u32 {
        (**self).size_hint()
    }
}

impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
    fn advance(&mut self) -> DocId {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.advance()
    }

    fn seek(&mut self, target: DocId) -> DocId {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.seek(target)
    }

    fn doc(&self) -> DocId {
        let unboxed: &TDocSet = self.borrow();
        unboxed.doc()
    }

    fn size_hint(&self) -> u32 {
        let unboxed: &TDocSet = self.borrow();
        unboxed.size_hint()
    }

    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.count(delete_bitset)
    }

    fn count_including_deleted(&mut self) -> u32 {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.count_including_deleted()
    }
}
<commit_msg>Added specialized implementation for count/count_including... in &mut DocSet<commit_after>use crate::fastfield::DeleteBitSet;
use crate::DocId;
use std::borrow::Borrow;
use std::borrow::BorrowMut;

/// Sentinel value returned when a DocSet has been entirely consumed.
///
/// This is not u32::MAX as one would have expected, due to the lack of SSE2 instructions
/// to compare [u32; 4].
pub const TERMINATED: DocId = std::i32::MAX as u32;

/// Represents an iterable set of sorted doc ids.
pub trait DocSet {
    /// Goes to the next element.
    ///
    /// The DocId of the next element is returned.
    /// In other words we should always have :
    /// ```ignore
    /// let doc = docset.advance();
    /// assert_eq!(doc, docset.doc());
    /// ```
    ///
    /// If we reached the end of the DocSet, TERMINATED should be returned.
    ///
    /// Calling `.advance()` on a terminated DocSet should be supported, and TERMINATED should
    /// be returned.
    /// TODO Test existing docsets.
    fn advance(&mut self) -> DocId;

    /// Advances the DocSet forward until reaching the target, or going to the
    /// lowest DocId greater than the target.
    ///
    /// If the end of the DocSet is reached, TERMINATED is returned.
    ///
    /// Calling `.seek(target)` on a terminated DocSet is legal. Implementation
    /// of DocSet should support it.
    ///
    /// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a DocSet.
    fn seek(&mut self, target: DocId) -> DocId {
        let mut doc = self.doc();
        debug_assert!(doc <= target);
        while doc < target {
            doc = self.advance();
        }
        doc
    }

    /// Fills a given mutable buffer with the next doc ids from the
    /// `DocSet`
    ///
    /// If that many `DocId`s are available, the method should
    /// fill the entire buffer and return the length of the buffer.
    ///
    /// If we reach the end of the `DocSet` before filling
    /// it entirely, then the buffer is filled up to this point, and
    /// return value is the number of elements that were filled.
    ///
    /// # Warning
    ///
    /// This method is only here for specific high-performance
    /// use case where batching. The normal way to
    /// go through the `DocId`'s is to call `.advance()`.
    fn fill_buffer(&mut self, buffer: &mut [DocId]) -> usize {
        if self.doc() == TERMINATED {
            return 0;
        }
        for (i, buffer_val) in buffer.iter_mut().enumerate() {
            *buffer_val = self.doc();
            if self.advance() == TERMINATED {
                return i + 1;
            }
        }
        buffer.len()
    }

    /// Returns the current document
    /// Right after creating a new DocSet, the docset points to the first document.
    ///
    /// If the DocSet is empty, .doc() should return `TERMINATED`.
    fn doc(&self) -> DocId;

    /// Returns a best-effort hint of the
    /// length of the docset.
    fn size_hint(&self) -> u32;

    /// Returns the number documents matching.
    /// Calling this method consumes the `DocSet`.
    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
        let mut count = 0u32;
        let mut doc = self.doc();
        while doc != TERMINATED {
            if !delete_bitset.is_deleted(doc) {
                count += 1u32;
            }
            doc = self.advance();
        }
        count
    }

    /// Returns the count of documents, deleted or not.
    /// Calling this method consumes the `DocSet`.
    ///
    /// Of course, the result is an upper bound of the result
    /// given by `count()`.
    fn count_including_deleted(&mut self) -> u32 {
        let mut count = 0u32;
        let mut doc = self.doc();
        while doc != TERMINATED {
            count += 1u32;
            doc = self.advance();
        }
        count
    }
}

impl<'a> DocSet for &'a mut dyn DocSet {
    fn advance(&mut self) -> u32 {
        (**self).advance()
    }

    fn seek(&mut self, target: DocId) -> DocId {
        (**self).seek(target)
    }

    fn doc(&self) -> u32 {
        (**self).doc()
    }

    fn size_hint(&self) -> u32 {
        (**self).size_hint()
    }

    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
        (**self).count(delete_bitset)
    }

    fn count_including_deleted(&mut self) -> u32 {
        (**self).count_including_deleted()
    }
}

impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
    fn advance(&mut self) -> DocId {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.advance()
    }

    fn seek(&mut self, target: DocId) -> DocId {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.seek(target)
    }

    fn doc(&self) -> DocId {
        let unboxed: &TDocSet = self.borrow();
        unboxed.doc()
    }

    fn size_hint(&self) -> u32 {
        let unboxed: &TDocSet = self.borrow();
        unboxed.size_hint()
    }

    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.count(delete_bitset)
    }

    fn count_including_deleted(&mut self) -> u32 {
        let unboxed: &mut TDocSet = self.borrow_mut();
        unboxed.count_including_deleted()
    }
}
<|endoftext|>


<commit_before>use rand::Rng;
use domination::{Dominate, Domination};
use multi_objective::MultiObjective;
use population::{UnratedPopulation, SelectedPopulation};
use time;

pub struct DriverConfig {
    pub mu: usize,
    pub lambda: usize,
    pub k: usize,
    pub ngen: usize,
    pub num_objectives: usize,
}

pub trait Driver<I, F>: Sync
where I: Clone + Sync,
      F: Dominate + MultiObjective + Clone + Send
{
    fn random_individual<R: Rng>(&self, rng: &mut R) -> I;
    fn random_population<R: Rng>(&self, rng: &mut R, n: usize) -> UnratedPopulation<I> {
        (0..n).map(|_| self.random_individual(rng)).collect()
    }
    fn fitness(&self, ind: &I) -> F;
    fn mate<R: Rng>(&self, rng: &mut R, parent1: &I, parent2: &I) -> I;
    fn is_solution(&self, _ind: &I, _fit: &F) -> bool {
        false
    }

    fn run<R, D, L>(&self,
                    rng: &mut R,
                    config: &DriverConfig,
                    weight: f64,
                    domination: &mut D,
                    logger: &L)
                    -> SelectedPopulation<I, F>
        where R: Rng,
              D: Domination<F>,
              L: Fn(usize, u64, usize, &SelectedPopulation<I, F>)
    {
        // this is generation 0. it's empty
        let mut time_last = time::precise_time_ns();
        let mut parents = SelectedPopulation::<I, F>::new();
        let mut offspring = self.random_population(rng, config.mu);
        let mut gen: usize = 0;

        loop {
            let rated_offspring = offspring.rate_in_parallel(&|ind| self.fitness(ind), weight);
            let next_generation = parents.merge(rated_offspring);
            parents = next_generation.select(config.mu, config.num_objectives, domination);

            let mut found_solutions = 0;
            parents.all(&mut |ind, fit| {
                if self.is_solution(ind, fit) {
                    found_solutions += 1;
                }
            });

            let now = time::precise_time_ns();
            let duration = now - time_last;
            time_last = now;

            logger(gen, duration, found_solutions, &parents);

            if found_solutions > 0 {
                break;
            }

            gen += 1;
            if gen > config.ngen {
                break;
            }

            offspring = parents.reproduce(rng,
                                          config.lambda,
                                          config.k,
                                          &|rng, p1, p2| self.mate(rng, p1, p2));
        }

        return parents;
    }
}
<commit_msg>Add Driver#population_metric() which updates the objectives of the current population<commit_after>use rand::Rng;
use domination::{Dominate, Domination};
use multi_objective::MultiObjective;
use population::{UnratedPopulation, RatedPopulation, SelectedPopulation};
use time;

pub struct DriverConfig {
    pub mu: usize,
    pub lambda: usize,
    pub k: usize,
    pub ngen: usize,
    pub num_objectives: usize,
}

pub trait Driver<I, F>: Sync
where I: Clone + Sync,
      F: Dominate + MultiObjective + Clone + Send
{
    fn random_individual<R: Rng>(&self, rng: &mut R) -> I;
    fn random_population<R: Rng>(&self, rng: &mut R, n: usize) -> UnratedPopulation<I> {
        (0..n).map(|_| self.random_individual(rng)).collect()
    }
    fn fitness(&self, ind: &I) -> F;
    fn mate<R: Rng>(&self, rng: &mut R, parent1: &I, parent2: &I) -> I;
    fn is_solution(&self, _ind: &I, _fit: &F) -> bool {
        false
    }

    /// This can be used to update certain objectives in relation to the whole population.
    fn population_metric(&self, _population: &mut RatedPopulation<I, F>) {
    }

    fn run<R, D, L>(&self,
                    rng: &mut R,
                    config: &DriverConfig,
                    weight: f64,
                    domination: &mut D,
                    logger: &L)
                    -> SelectedPopulation<I, F>
        where R: Rng,
              D: Domination<F>,
              L: Fn(usize, u64, usize, &SelectedPopulation<I, F>)
    {
        // this is generation 0. it's empty
        let mut time_last = time::precise_time_ns();
        let mut parents = SelectedPopulation::<I, F>::new();
        let mut offspring = self.random_population(rng, config.mu);
        let mut gen: usize = 0;

        loop {
            let rated_offspring = offspring.rate_in_parallel(&|ind| self.fitness(ind), weight);
            let mut next_generation = parents.merge(rated_offspring);
            // apply a population metric on the whole population
            self.population_metric(&mut next_generation);
            parents = next_generation.select(config.mu, config.num_objectives, domination);

            let mut found_solutions = 0;
            parents.all(&mut |ind, fit| {
                if self.is_solution(ind, fit) {
                    found_solutions += 1;
                }
            });

            let now = time::precise_time_ns();
            let duration = now - time_last;
            time_last = now;

            logger(gen, duration, found_solutions, &parents);

            if found_solutions > 0 {
                break;
            }

            gen += 1;
            if gen > config.ngen {
                break;
            }

            offspring = parents.reproduce(rng,
                                          config.lambda,
                                          config.k,
                                          &|rng, p1, p2| self.mate(rng, p1, p2));
        }

        return parents;
    }
}
<|endoftext|>


<commit_before>use bincode;
pub use failure::ResultExt;
use failure::{Context, Fail};
use std::path::{Path, PathBuf};
use std::{fmt, io, result};

pub type Result<T> = result::Result<T, failure::Error>;

#[derive(Debug, Fail)]
pub struct Error {
    ctx: Context<ErrorKind>,
}

impl Error {
    pub fn kind(&self) -> &ErrorKind {
        self.ctx.get_context()
    }

    pub fn io(err: io::Error) -> Error {
        Error::from(ErrorKind::IO(err.to_string()))
    }

    pub fn path<P: AsRef<Path>>(path: P) -> Error {
        let kind = ErrorKind::Path(path.as_ref().to_path_buf());
        Error::from(kind)
    }

    pub fn config() -> Error {
        let kind = ErrorKind::Config;
        Error::from(kind)
    }

    pub fn not_found(key: String) -> Error {
        let kind = ErrorKind::NotFound(key);
        Error::from(kind)
    }

    pub fn bincode(err: bincode::Error) -> Error {
        let kind = ErrorKind::Bincode(err.to_string());
        Error::from(kind)
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.ctx.fmt(f)
    }
}

impl From<ErrorKind> for Error {
    fn from(kind: ErrorKind) -> Error {
        Error::from(Context::new(kind))
    }
}

impl From<Context<ErrorKind>> for Error {
    fn from(ctx: Context<ErrorKind>) -> Error {
        Error { ctx }
    }
}

#[derive(Debug, Fail)]
pub enum ErrorKind {
    Path(PathBuf),
    IO(String),
    NotFound(String),
    Config,
    Bincode(String),
}

impl fmt::Display for ErrorKind {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ErrorKind::Path(ref path) => write!(f, "path: {}", path.display()),
            ErrorKind::IO(ref msg) => write!(f, "IO error: {}", msg),
            ErrorKind::NotFound(ref bookmark) => write!(f, "bookmark not found: {}", bookmark),
            ErrorKind::Config => write!(f, "failed to locat config"),
            ErrorKind::Bincode(ref msg) => write!(f, "bincode error: {}", msg),
        }
    }
}

pub fn pretty_error(err: &failure::Error) -> String {
    let mut pretty = format!("\n => {}", err.to_string());
    let mut prev = err.as_fail();
    while let Some(next) = prev.cause() {
        pretty.push_str("\n => ");
        pretty.push_str(&next.to_string());
        prev = next;
    }

    pretty.push_str(&format!("\n\n{}", err.backtrace()));

    pretty
}
<commit_msg>remove unused error<commit_after>use bincode;
pub use failure::ResultExt;
use failure::{Context, Fail};
use std::path::{Path, PathBuf};
use std::{fmt, io, result};

pub type Result<T> = result::Result<T, failure::Error>;

#[derive(Debug, Fail)]
pub struct Error {
    ctx: Context<ErrorKind>,
}

impl Error {
    pub fn io(err: io::Error) -> Error {
        Error::from(ErrorKind::IO(err.to_string()))
    }

    pub fn path<P: AsRef<Path>>(path: P) -> Error {
        let kind = ErrorKind::Path(path.as_ref().to_path_buf());
        Error::from(kind)
    }

    pub fn not_found(key: String) -> Error {
        let kind = ErrorKind::NotFound(key);
        Error::from(kind)
    }

    pub fn bincode(err: bincode::Error) -> Error {
        let kind = ErrorKind::Bincode(err.to_string());
        Error::from(kind)
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.ctx.fmt(f)
    }
}

impl From<ErrorKind> for Error {
    fn from(kind: ErrorKind) -> Error {
        Error::from(Context::new(kind))
    }
}

impl From<Context<ErrorKind>> for Error {
    fn from(ctx: Context<ErrorKind>) -> Error {
        Error { ctx }
    }
}

#[derive(Debug, Fail)]
pub enum ErrorKind {
    Path(PathBuf),
    IO(String),
    NotFound(String),
    Bincode(String),
}

impl fmt::Display for ErrorKind {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ErrorKind::Path(ref path) => write!(f, "path: {}", path.display()),
            ErrorKind::IO(ref msg) => write!(f, "IO error: {}", msg),
            ErrorKind::NotFound(ref bookmark) => write!(f, "bookmark not found: {}", bookmark),
            ErrorKind::Bincode(ref msg) => write!(f, "bincode error: {}", msg),
        }
    }
}

pub fn pretty_error(err: &failure::Error) -> String {
    let mut pretty = format!("\n => {}", err.to_string());
    let mut prev = err.as_fail();
    while let Some(next) = prev.cause() {
        pretty.push_str("\n => ");
        pretty.push_str(&next.to_string());
        prev = next;
    }

    pretty.push_str(&format!("\n\n{}", err.backtrace()));

    pretty
}
<|endoftext|>


<commit_before>use libc;
use std::ptr::{self, write_bytes};

use ctxt::{Context, get_ctxt};
use gc::copy::{minor_collect, SemiSpace};
use gc::root::{get_rootset, IndirectObj};
use gc::space::{Space, SpaceConfig};
use object::Obj;
use os;
use timer::{in_ms, Timer};

pub mod copy;
pub mod root;
pub mod space;

const INITIAL_THRESHOLD: usize = 128;
const USED_RATIO: f64 = 0.75;

const INITIAL_SIZE: usize = 64 * 1024;
const LARGE_OBJECT_SIZE: usize = 64 * 1024;

const CHUNK_SIZE: usize = 8 * 1024;
const CODE_SPACE_LIMIT: usize = 128 * 1024;
const PERM_SPACE_LIMIT: usize = 16 * 1024;

pub struct Gc {
    obj_start: *mut Obj,
    obj_end: *mut Obj,
    bytes_allocated: usize,
    threshold: usize,
    pub cur_marked: bool,

    pub collect_duration: u64,
    pub total_allocated: u64,
    pub collections: u64,
    pub allocations: u64,

    from_space: SemiSpace,
    to_space: SemiSpace,

    code_space: Space,
    perm_space: Space,
}

impl Gc {
    pub fn new() -> Gc {
        let code_config = SpaceConfig {
            prot: os::Executable,
            chunk_size: CHUNK_SIZE,
            limit: CODE_SPACE_LIMIT,
            align: 64,
        };

        let perm_config = SpaceConfig {
            prot: os::Writable,
            chunk_size: CHUNK_SIZE,
            limit: PERM_SPACE_LIMIT,
            align: 8,
        };

        Gc {
            obj_start: ptr::null_mut(),
            obj_end: ptr::null_mut(),
            cur_marked: true,
            bytes_allocated: 0,
            threshold: INITIAL_THRESHOLD,

            collect_duration: 0,
            total_allocated: 0,
            collections: 0,
            allocations: 0,

            from_space: SemiSpace::new(INITIAL_SIZE),
            to_space: SemiSpace::new(INITIAL_SIZE),
            code_space: Space::new(code_config, "code"),
            perm_space: Space::new(perm_config, "perm"),
        }
    }

    pub fn alloc_code(&mut self, size: usize) -> *mut u8 {
        self.code_space.alloc(size)
    }

    pub fn alloc_perm(&mut self, size: usize) -> *mut u8 {
        self.perm_space.alloc(size)
    }

    pub fn alloc_copy(&mut self, ctxt: &Context, size: usize) -> *mut u8 {
        if ctxt.args.flag_gc_stress {
            let rootset = get_rootset(ctxt);
            minor_collect(ctxt, &mut self.from_space, &mut self.to_space, rootset);
        }

        if size > LARGE_OBJECT_SIZE {
            panic!("large object heap not implemented yet.");
        }

        let mut ptr = self.to_space.allocate(size);

        if ptr.is_null() {
            let rootset = get_rootset(ctxt);
            minor_collect(ctxt, &mut self.from_space, &mut self.to_space, rootset);

            ptr = self.to_space.allocate(size);

            if ptr.is_null() {
                panic!("out of memory");
            }
        }

        ptr as *mut u8
    }

    pub fn alloc(&mut self, size: usize) -> *mut Obj {
        let ctxt = get_ctxt();

        if ctxt.args.flag_gc_stress {
            // with --gc-stress collect garbage at every allocation
            // useful for testing
            self.collect(ctxt);

            // do we pass threshold with this allocation?
        } else if self.bytes_allocated + size > self.threshold {
            // collect garbage
            self.collect(ctxt);

            // if still more memory than USED_RATIO % of the threshold,
            // we need to increase the threshold
            if (self.bytes_allocated + size) as f64 > self.threshold as f64 * USED_RATIO {
                let saved_threshold = self.threshold;
                self.threshold = (self.threshold as f64 / USED_RATIO) as usize;

                if ctxt.args.flag_gc_dump {
                    println!("GC: increase threshold from {} to {}",
                             saved_threshold,
                             self.threshold);
                }
            }
        }

        let ptr = unsafe { libc::malloc(size) as *mut Obj };
        unsafe {
            write_bytes(ptr as *mut u8, 0, size);
        }

        {
            let ptr = ptr as *mut Obj;

            if self.obj_end.is_null() {
                assert!(self.obj_start.is_null());

                self.obj_start = ptr;
                self.obj_end = ptr;

            } else {
                assert!(!self.obj_start.is_null());

                let obj_end = unsafe { &mut *self.obj_end };
                obj_end.header_mut().set_succ(ptr);

                self.obj_end = ptr;
            }
        }

        self.bytes_allocated += size;
        self.total_allocated += size as u64;
        self.allocations += 1;

        ptr
    }

    pub fn collect(&mut self, ctxt: &Context) {
        let active_timer = ctxt.args.flag_gc_dump || ctxt.args.flag_gc_stats;
        let mut timer = Timer::new(active_timer);
        let rootset = get_rootset(ctxt);

        mark_rootset(ctxt, rootset, self.cur_marked);

        let cur_marked = self.cur_marked;
        sweep(self, cur_marked);

        // switch cur_marked value, so that we don't have to unmark all
        // objects in the beginning of the next collection
        self.cur_marked = !self.cur_marked;
        self.collections += 1;

        timer.stop_with(|dur| {
            self.collect_duration += dur;

            if ctxt.args.flag_gc_dump {
                println!("GC: collect garbage ({} ms)", in_ms(dur));
            }
        });
    }
}

impl Drop for Gc {
    fn drop(&mut self) {
        let mut obj = self.obj_start;

        while !obj.is_null() {
            let curr = unsafe { &mut *obj };
            obj = unsafe { &mut *obj }.header().succ();

            unsafe {
                libc::free(curr as *mut _ as *mut libc::c_void);
            }
        }
    }
}

fn mark_rootset(ctxt: &Context, rootset: Vec<IndirectObj>, cur_marked: bool) {
    for root in rootset {
        mark_recursive(ctxt, root.get(), cur_marked);
    }
}

fn mark_recursive(ctxt: &Context, obj: *mut Obj, cur_marked: bool) {
    if obj.is_null() {
        return;
    }

    let mut elements: Vec<*mut Obj> = vec![obj];

    while !elements.is_empty() {
        let ptr = elements.pop().unwrap();
        let obj = unsafe { &mut *ptr };

        if obj.header().marked() != cur_marked {
            obj.header_mut().set_mark(cur_marked);
            let mut classid = obj.header().vtbl().class().id;

            loop {
                let cls = ctxt.classes[classid].borrow();

                for field in &cls.fields {
                    if field.ty.reference_type() {
                        let addr = ptr as isize + field.offset as isize;
                        let obj = unsafe { *(addr as *const usize) } as *mut Obj;

                        if obj.is_null() {
                            continue;
                        }

                        elements.push(obj);
                    }
                }

                if let Some(parent_class) = cls.parent_class {
                    classid = parent_class;

                } else {
                    break;
                }
            }
        }
    }
}

fn sweep(gc: &mut Gc, cur_marked: bool) {
    let mut obj = gc.obj_start;
    let mut last: *mut Obj = ptr::null_mut();

    while !obj.is_null() {
        let curr = unsafe { &mut *obj };
        let succ = curr.header().succ();

        if curr.header().marked() != cur_marked {
            let size = curr.size();

            unsafe {
                // overwrite memory for better detection of gc bugs
                write_bytes(obj as *mut u8, 0xcc, size);

                // free unused memory
                libc::free(obj as *mut libc::c_void);
            }

            gc.bytes_allocated -= size;

        } else {
            if last.is_null() {
                gc.obj_start = obj;
            } else {
                let last = unsafe { &mut *last };
                last.header_mut().set_succ(obj);
            }

            // last survived object
            last = obj;
        }

        // set next handled object
        obj = succ;
    }

    // if no objects left, obj_start needs to be reset to null
    if last.is_null() {
        gc.obj_start = ptr::null_mut();
    }

    // last survived object is now end of list
    gc.obj_end = last;
}
<commit_msg>use visit_reference_fields in gc<commit_after>use libc;
use std::ptr::{self, write_bytes};

use ctxt::{Context, get_ctxt};
use gc::copy::{minor_collect, SemiSpace};
use gc::root::{get_rootset, IndirectObj};
use gc::space::{Space, SpaceConfig};
use object::Obj;
use os;
use timer::{in_ms, Timer};

pub mod copy;
pub mod root;
pub mod space;

const INITIAL_THRESHOLD: usize = 128;
const USED_RATIO: f64 = 0.75;

const INITIAL_SIZE: usize = 64 * 1024;
const LARGE_OBJECT_SIZE: usize = 64 * 1024;

const CHUNK_SIZE: usize = 8 * 1024;
const CODE_SPACE_LIMIT: usize = 128 * 1024;
const PERM_SPACE_LIMIT: usize = 16 * 1024;

pub struct Gc {
    obj_start: *mut Obj,
    obj_end: *mut Obj,
    bytes_allocated: usize,
    threshold: usize,
    pub cur_marked: bool,

    pub collect_duration: u64,
    pub total_allocated: u64,
    pub collections: u64,
    pub allocations: u64,

    from_space: SemiSpace,
    to_space: SemiSpace,

    code_space: Space,
    perm_space: Space,
}

impl Gc {
    pub fn new() -> Gc {
        let code_config = SpaceConfig {
            prot: os::Executable,
            chunk_size: CHUNK_SIZE,
            limit: CODE_SPACE_LIMIT,
            align: 64,
        };

        let perm_config = SpaceConfig {
            prot: os::Writable,
            chunk_size: CHUNK_SIZE,
            limit: PERM_SPACE_LIMIT,
            align: 8,
        };

        Gc {
            obj_start: ptr::null_mut(),
            obj_end: ptr::null_mut(),
            cur_marked: true,
            bytes_allocated: 0,
            threshold: INITIAL_THRESHOLD,

            collect_duration: 0,
            total_allocated: 0,
            collections: 0,
            allocations: 0,

            from_space: SemiSpace::new(INITIAL_SIZE),
            to_space: SemiSpace::new(INITIAL_SIZE),
            code_space: Space::new(code_config, "code"),
            perm_space: Space::new(perm_config, "perm"),
        }
    }

    pub fn alloc_code(&mut self, size: usize) -> *mut u8 {
        self.code_space.alloc(size)
    }

    pub fn alloc_perm(&mut self, size: usize) -> *mut u8 {
        self.perm_space.alloc(size)
    }

    pub fn alloc_copy(&mut self, ctxt: &Context, size: usize) -> *mut u8 {
        if ctxt.args.flag_gc_stress {
            let rootset = get_rootset(ctxt);
            minor_collect(ctxt, &mut self.from_space, &mut self.to_space, rootset);
        }

        if size > LARGE_OBJECT_SIZE {
            panic!("large object heap not implemented yet.");
        }

        let mut ptr = self.to_space.allocate(size);

        if ptr.is_null() {
            let rootset = get_rootset(ctxt);
            minor_collect(ctxt, &mut self.from_space, &mut self.to_space, rootset);

            ptr = self.to_space.allocate(size);

            if ptr.is_null() {
                panic!("out of memory");
            }
        }

        ptr as *mut u8
    }

    pub fn alloc(&mut self, size: usize) -> *mut Obj {
        let ctxt = get_ctxt();

        if ctxt.args.flag_gc_stress {
            // with --gc-stress collect garbage at every allocation
            // useful for testing
            self.collect(ctxt);

            // do we pass threshold with this allocation?
        } else if self.bytes_allocated + size > self.threshold {
            // collect garbage
            self.collect(ctxt);

            // if still more memory than USED_RATIO % of the threshold,
            // we need to increase the threshold
            if (self.bytes_allocated + size) as f64 > self.threshold as f64 * USED_RATIO {
                let saved_threshold = self.threshold;
                self.threshold = (self.threshold as f64 / USED_RATIO) as usize;

                if ctxt.args.flag_gc_dump {
                    println!("GC: increase threshold from {} to {}",
                             saved_threshold,
                             self.threshold);
                }
            }
        }

        let ptr = unsafe { libc::malloc(size) as *mut Obj };
        unsafe {
            write_bytes(ptr as *mut u8, 0, size);
        }

        {
            let ptr = ptr as *mut Obj;

            if self.obj_end.is_null() {
                assert!(self.obj_start.is_null());

                self.obj_start = ptr;
                self.obj_end = ptr;

            } else {
                assert!(!self.obj_start.is_null());

                let obj_end = unsafe { &mut *self.obj_end };
                obj_end.header_mut().set_succ(ptr);

                self.obj_end = ptr;
            }
        }

        self.bytes_allocated += size;
        self.total_allocated += size as u64;
        self.allocations += 1;

        ptr
    }

    pub fn collect(&mut self, ctxt: &Context) {
        let active_timer = ctxt.args.flag_gc_dump || ctxt.args.flag_gc_stats;
        let mut timer = Timer::new(active_timer);
        let rootset = get_rootset(ctxt);

        mark_rootset(rootset, self.cur_marked);

        let cur_marked = self.cur_marked;
        sweep(self, cur_marked);

        // switch cur_marked value, so that we don't have to unmark all
        // objects in the beginning of the next collection
        self.cur_marked = !self.cur_marked;
        self.collections += 1;

        timer.stop_with(|dur| {
            self.collect_duration += dur;

            if ctxt.args.flag_gc_dump {
                println!("GC: collect garbage ({} ms)", in_ms(dur));
            }
        });
    }
}

impl Drop for Gc {
    fn drop(&mut self) {
        let mut obj = self.obj_start;

        while !obj.is_null() {
            let curr = unsafe { &mut *obj };
            obj = unsafe { &mut *obj }.header().succ();

            unsafe {
                libc::free(curr as *mut _ as *mut libc::c_void);
            }
        }
    }
}

fn mark_rootset(rootset: Vec<IndirectObj>, cur_marked: bool) {
    for root in rootset {
        mark_recursive(root.get(), cur_marked);
    }
}

fn mark_recursive(obj: *mut Obj, cur_marked: bool) {
    if obj.is_null() {
        return;
    }

    let mut elements: Vec<*mut Obj> = vec![obj];

    while !elements.is_empty() {
        let ptr = elements.pop().unwrap();
        let obj = unsafe { &mut *ptr };

        if obj.header().marked() != cur_marked {
            obj.header_mut().set_mark(cur_marked);

            obj.visit_reference_fields(|child| {
                let child = child.get();

                if !child.is_null() {
                    elements.push(child);
                }
            });
        }
    }
}

fn sweep(gc: &mut Gc, cur_marked: bool) {
    let mut obj = gc.obj_start;
    let mut last: *mut Obj = ptr::null_mut();

    while !obj.is_null() {
        let curr = unsafe { &mut *obj };
        let succ = curr.header().succ();

        if curr.header().marked() != cur_marked {
            let size = curr.size();

            unsafe {
                // overwrite memory for better detection of gc bugs
                write_bytes(obj as *mut u8, 0xcc, size);

                // free unused memory
                libc::free(obj as *mut libc::c_void);
            }

            gc.bytes_allocated -= size;

        } else {
            if last.is_null() {
                gc.obj_start = obj;
            } else {
                let last = unsafe { &mut *last };
                last.header_mut().set_succ(obj);
            }

            // last survived object
            last = obj;
        }

        // set next handled object
        obj = succ;
    }

    // if no objects left, obj_start needs to be reset to null
    if last.is_null() {
        gc.obj_start = ptr::null_mut();
    }

    // last survived object is now end of list
    gc.obj_end = last;
}
<|endoftext|>