<commit_before>//! Futures-powered synchronization primitives.
#![allow(unused)]

use futures_core::future::Future;
use futures_core::task::{Context, Poll, Waker};
use core::cell::UnsafeCell;
use core::fmt;
use core::mem;
use core::ops::{Deref, DerefMut};
use core::pin::Pin;
use core::sync::atomic::AtomicUsize;
use core::sync::atomic::Ordering::SeqCst;
use alloc::boxed::Box;
use alloc::sync::Arc;
#[cfg(feature = "std")]
use std::any::Any;
#[cfg(feature = "std")]
use std::error::Error;

/// A type of futures-powered synchronization primitive which is a mutex between
/// two possible owners.
///
/// This primitive is not as generic as a full-blown mutex but is sufficient for
/// many use cases where there are only two possible owners of a resource. The
/// implementation of `BiLock` can be more optimized for just the two possible
/// owners.
///
/// Note that it's possible to use this lock through a poll-style interface with
/// the `poll_lock` method but you can also use it as a future with the `lock`
/// method that consumes a `BiLock` and returns a future that will resolve when
/// it's locked.
///
/// A `BiLock` is typically used for "split" operations where data which serves
/// two purposes wants to be split into two to be worked with separately. For
/// example a TCP stream could be both a reader and a writer or a framing layer
/// could be both a stream and a sink for messages. A `BiLock` enables splitting
/// these two and then using each independently in a futures-powered fashion.
#[derive(Debug)]
pub struct BiLock<T> {
    arc: Arc<Inner<T>>,
}

#[derive(Debug)]
struct Inner<T> {
    state: AtomicUsize,
    value: Option<UnsafeCell<T>>,
}

unsafe impl<T: Send> Send for Inner<T> {}
unsafe impl<T: Send> Sync for Inner<T> {}

impl<T> BiLock<T> {
    /// Creates a new `BiLock` protecting the provided data.
    ///
    /// Two handles to the lock are returned, and these are the only two handles
    /// that will ever be available to the lock. These can then be sent to separate
    /// tasks to be managed there.
    ///
    /// The data behind the bilock is considered to be pinned, which allows `Pin`
    /// references to locked data. However, this means that the locked value
    /// will only be available through `Pin<&mut T>` (not `&mut T`) unless `T` is `Unpin`.
    /// Similarly, reuniting the lock and extracting the inner value is only
    /// possible when `T` is `Unpin`.
    pub fn new(t: T) -> (BiLock<T>, BiLock<T>) {
        let arc = Arc::new(Inner {
            state: AtomicUsize::new(0),
            value: Some(UnsafeCell::new(t)),
        });

        (BiLock { arc: arc.clone() }, BiLock { arc })
    }

    /// Attempt to acquire this lock, returning `Pending` if it can't be
    /// acquired.
    ///
    /// This function will acquire the lock in a nonblocking fashion, returning
    /// immediately if the lock is already held. If the lock is successfully
    /// acquired then `Poll::Ready` is returned with a value that represents
    /// the locked value (and can be used to access the protected data). The
    /// lock is unlocked when the returned `BiLockGuard` is dropped.
    ///
    /// If the lock is already held then this function will return
    /// `Poll::Pending`. In this case the current task will also be scheduled
    /// to receive a notification when the lock would otherwise become
    /// available.
    ///
    /// # Panics
    ///
    /// This function will panic if called outside the context of a future's
    /// task.
    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<BiLockGuard<'_, T>> {
        loop {
            match self.arc.state.swap(1, SeqCst) {
                // Woohoo, we grabbed the lock!
                0 => return Poll::Ready(BiLockGuard { bilock: self }),

                // Oops, someone else has locked the lock
                1 => {}

                // A task was previously blocked on this lock, likely our task,
                // so we need to update that task.
                n => unsafe {
                    drop(Box::from_raw(n as *mut Waker));
                }
            }

            // type ascription for safety's sake!
            let me: Box<Waker> = Box::new(cx.waker().clone());
            let me = Box::into_raw(me) as usize;

            match self.arc.state.compare_exchange(1, me, SeqCst, SeqCst) {
                // The lock is still locked, but we've now parked ourselves, so
                // just report that we're scheduled to receive a notification.
                Ok(_) => return Poll::Pending,

                // Oops, looks like the lock was unlocked after our swap above
                // and before the compare_exchange. Deallocate what we just
                // allocated and go through the loop again.
                Err(0) => unsafe {
                    drop(Box::from_raw(me as *mut Waker));
                },

                // The top of this loop set the previous state to 1, so if we
                // failed the CAS above then it's because the previous value was
                // *not* zero or one. This indicates that a task was blocked,
                // but we're trying to acquire the lock and there's only one
                // other reference of the lock, so it should be impossible for
                // that task to ever block itself.
                Err(n) => panic!("invalid state: {}", n),
            }
        }
    }

    /// Perform a "blocking lock" of this lock, consuming this lock handle and
    /// returning a future to the acquired lock.
    ///
    /// This function consumes the `BiLock<T>` and returns a sentinel future,
    /// `BiLockAcquire<T>`. The returned future will resolve to
    /// `BiLockAcquired<T>` which represents a locked lock similarly to
    /// `BiLockGuard<T>`.
    ///
    /// Note that the returned future will never resolve to an error.
    pub fn lock(&self) -> BiLockAcquire<'_, T> {
        BiLockAcquire {
            bilock: self,
        }
    }

    /// Attempts to put the two "halves" of a `BiLock<T>` back together and
    /// recover the original value. Succeeds only if the two `BiLock<T>`s
    /// originated from the same call to `BiLock::new`.
    pub fn reunite(self, other: Self) -> Result<T, ReuniteError<T>>
    where
        T: Unpin,
    {
        if &*self.arc as *const _ == &*other.arc as *const _ {
            drop(other);
            let inner = Arc::try_unwrap(self.arc)
                .ok()
                .expect("futures: try_unwrap failed in BiLock<T>::reunite");
            Ok(unsafe { inner.into_value() })
        } else {
            Err(ReuniteError(self, other))
        }
    }

    fn unlock(&self) {
        match self.arc.state.swap(0, SeqCst) {
            // we've locked the lock, shouldn't be possible for us to see an
            // unlocked lock.
            0 => panic!("invalid unlocked state"),

            // Ok, no one else tried to get the lock, we're done.
            1 => {}

            // Another task has parked themselves on this lock, let's wake them
            // up as its now their turn.
            n => unsafe {
                Box::from_raw(n as *mut Waker).wake();
            }
        }
    }
}

impl<T: Unpin> Inner<T> {
    unsafe fn into_value(mut self) -> T {
        mem::replace(&mut self.value, None).unwrap().into_inner()
    }
}

impl<T> Drop for Inner<T> {
    fn drop(&mut self) {
        assert_eq!(self.state.load(SeqCst), 0);
    }
}

/// Error indicating two `BiLock<T>`s were not two halves of a whole, and
/// thus could not be `reunite`d.
pub struct ReuniteError<T>(pub BiLock<T>, pub BiLock<T>);

impl<T> fmt::Debug for ReuniteError<T> {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        fmt.debug_tuple("ReuniteError")
            .field(&"...")
            .finish()
    }
}

impl<T> fmt::Display for ReuniteError<T> {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(fmt, "tried to reunite two BiLocks that don't form a pair")
    }
}

#[cfg(feature = "std")]
impl<T: Any> Error for ReuniteError<T> {}

/// Returned RAII guard from the `poll_lock` method.
///
/// This structure acts as a sentinel to the data in the `BiLock<T>` itself,
/// implementing `Deref` and `DerefMut` to `T`. When dropped, the lock will be
/// unlocked.
#[derive(Debug)]
pub struct BiLockGuard<'a, T> {
    bilock: &'a BiLock<T>,
}

impl<T> Deref for BiLockGuard<'_, T> {
    type Target = T;
    fn deref(&self) -> &T {
        unsafe { &*self.bilock.arc.value.as_ref().unwrap().get() }
    }
}

impl<T: Unpin> DerefMut for BiLockGuard<'_, T> {
    fn deref_mut(&mut self) -> &mut T {
        unsafe { &mut *self.bilock.arc.value.as_ref().unwrap().get() }
    }
}

impl<T> BiLockGuard<'_, T> {
    /// Get a mutable pinned reference to the locked value.
    pub fn as_pin_mut(&mut self) -> Pin<&mut T> {
        // Safety: we never allow moving a !Unpin value out of a bilock, nor
        // allow mutable access to it
        unsafe { Pin::new_unchecked(&mut *self.bilock.arc.value.as_ref().unwrap().get()) }
    }
}

impl<T> Drop for BiLockGuard<'_, T> {
    fn drop(&mut self) {
        self.bilock.unlock();
    }
}

/// Future returned by `BiLock::lock` which will resolve when the lock is
/// acquired.
#[must_use = "futures do nothing unless you `.await` or poll them"]
#[derive(Debug)]
pub struct BiLockAcquire<'a, T> {
    bilock: &'a BiLock<T>,
}

// Pinning is never projected to fields
impl<T> Unpin for BiLockAcquire<'_, T> {}

impl<'a, T> Future for BiLockAcquire<'a, T> {
    type Output = BiLockGuard<'a, T>;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        self.bilock.poll_lock(cx)
    }
}
<commit_msg>Replace mem::replace(Option, None) with Option::take()<commit_after>//! Futures-powered synchronization primitives.
#![allow(unused)]

use futures_core::future::Future;
use futures_core::task::{Context, Poll, Waker};
use core::cell::UnsafeCell;
use core::fmt;
use core::mem;
use core::ops::{Deref, DerefMut};
use core::pin::Pin;
use core::sync::atomic::AtomicUsize;
use core::sync::atomic::Ordering::SeqCst;
use alloc::boxed::Box;
use alloc::sync::Arc;
#[cfg(feature = "std")]
use std::any::Any;
#[cfg(feature = "std")]
use std::error::Error;

/// A type of futures-powered synchronization primitive which is a mutex between
/// two possible owners.
///
/// This primitive is not as generic as a full-blown mutex but is sufficient for
/// many use cases where there are only two possible owners of a resource. The
/// implementation of `BiLock` can be more optimized for just the two possible
/// owners.
///
/// Note that it's possible to use this lock through a poll-style interface with
/// the `poll_lock` method but you can also use it as a future with the `lock`
/// method that consumes a `BiLock` and returns a future that will resolve when
/// it's locked.
///
/// A `BiLock` is typically used for "split" operations where data which serves
/// two purposes wants to be split into two to be worked with separately. For
/// example a TCP stream could be both a reader and a writer or a framing layer
/// could be both a stream and a sink for messages. A `BiLock` enables splitting
/// these two and then using each independently in a futures-powered fashion.
#[derive(Debug)]
pub struct BiLock<T> {
    arc: Arc<Inner<T>>,
}

#[derive(Debug)]
struct Inner<T> {
    state: AtomicUsize,
    value: Option<UnsafeCell<T>>,
}

unsafe impl<T: Send> Send for Inner<T> {}
unsafe impl<T: Send> Sync for Inner<T> {}

impl<T> BiLock<T> {
    /// Creates a new `BiLock` protecting the provided data.
    ///
    /// Two handles to the lock are returned, and these are the only two handles
    /// that will ever be available to the lock. These can then be sent to separate
    /// tasks to be managed there.
    ///
    /// The data behind the bilock is considered to be pinned, which allows `Pin`
    /// references to locked data. However, this means that the locked value
    /// will only be available through `Pin<&mut T>` (not `&mut T`) unless `T` is `Unpin`.
    /// Similarly, reuniting the lock and extracting the inner value is only
    /// possible when `T` is `Unpin`.
    pub fn new(t: T) -> (BiLock<T>, BiLock<T>) {
        let arc = Arc::new(Inner {
            state: AtomicUsize::new(0),
            value: Some(UnsafeCell::new(t)),
        });

        (BiLock { arc: arc.clone() }, BiLock { arc })
    }

    /// Attempt to acquire this lock, returning `Pending` if it can't be
    /// acquired.
    ///
    /// This function will acquire the lock in a nonblocking fashion, returning
    /// immediately if the lock is already held. If the lock is successfully
    /// acquired then `Poll::Ready` is returned with a value that represents
    /// the locked value (and can be used to access the protected data). The
    /// lock is unlocked when the returned `BiLockGuard` is dropped.
    ///
    /// If the lock is already held then this function will return
    /// `Poll::Pending`. In this case the current task will also be scheduled
    /// to receive a notification when the lock would otherwise become
    /// available.
    ///
    /// # Panics
    ///
    /// This function will panic if called outside the context of a future's
    /// task.
    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<BiLockGuard<'_, T>> {
        loop {
            match self.arc.state.swap(1, SeqCst) {
                // Woohoo, we grabbed the lock!
                0 => return Poll::Ready(BiLockGuard { bilock: self }),

                // Oops, someone else has locked the lock
                1 => {}

                // A task was previously blocked on this lock, likely our task,
                // so we need to update that task.
                n => unsafe {
                    drop(Box::from_raw(n as *mut Waker));
                }
            }

            // type ascription for safety's sake!
            let me: Box<Waker> = Box::new(cx.waker().clone());
            let me = Box::into_raw(me) as usize;

            match self.arc.state.compare_exchange(1, me, SeqCst, SeqCst) {
                // The lock is still locked, but we've now parked ourselves, so
                // just report that we're scheduled to receive a notification.
                Ok(_) => return Poll::Pending,

                // Oops, looks like the lock was unlocked after our swap above
                // and before the compare_exchange. Deallocate what we just
                // allocated and go through the loop again.
                Err(0) => unsafe {
                    drop(Box::from_raw(me as *mut Waker));
                },

                // The top of this loop set the previous state to 1, so if we
                // failed the CAS above then it's because the previous value was
                // *not* zero or one. This indicates that a task was blocked,
                // but we're trying to acquire the lock and there's only one
                // other reference of the lock, so it should be impossible for
                // that task to ever block itself.
                Err(n) => panic!("invalid state: {}", n),
            }
        }
    }

    /// Perform a "blocking lock" of this lock, consuming this lock handle and
    /// returning a future to the acquired lock.
    ///
    /// This function consumes the `BiLock<T>` and returns a sentinel future,
    /// `BiLockAcquire<T>`. The returned future will resolve to
    /// `BiLockAcquired<T>` which represents a locked lock similarly to
    /// `BiLockGuard<T>`.
    ///
    /// Note that the returned future will never resolve to an error.
    pub fn lock(&self) -> BiLockAcquire<'_, T> {
        BiLockAcquire {
            bilock: self,
        }
    }

    /// Attempts to put the two "halves" of a `BiLock<T>` back together and
    /// recover the original value. Succeeds only if the two `BiLock<T>`s
    /// originated from the same call to `BiLock::new`.
    pub fn reunite(self, other: Self) -> Result<T, ReuniteError<T>>
    where
        T: Unpin,
    {
        if &*self.arc as *const _ == &*other.arc as *const _ {
            drop(other);
            let inner = Arc::try_unwrap(self.arc)
                .ok()
                .expect("futures: try_unwrap failed in BiLock<T>::reunite");
            Ok(unsafe { inner.into_value() })
        } else {
            Err(ReuniteError(self, other))
        }
    }

    fn unlock(&self) {
        match self.arc.state.swap(0, SeqCst) {
            // we've locked the lock, shouldn't be possible for us to see an
            // unlocked lock.
            0 => panic!("invalid unlocked state"),

            // Ok, no one else tried to get the lock, we're done.
            1 => {}

            // Another task has parked themselves on this lock, let's wake them
            // up as its now their turn.
            n => unsafe {
                Box::from_raw(n as *mut Waker).wake();
            }
        }
    }
}

impl<T: Unpin> Inner<T> {
    unsafe fn into_value(mut self) -> T {
        self.value.take().unwrap().into_inner()
    }
}

impl<T> Drop for Inner<T> {
    fn drop(&mut self) {
        assert_eq!(self.state.load(SeqCst), 0);
    }
}

/// Error indicating two `BiLock<T>`s were not two halves of a whole, and
/// thus could not be `reunite`d.
pub struct ReuniteError<T>(pub BiLock<T>, pub BiLock<T>);

impl<T> fmt::Debug for ReuniteError<T> {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        fmt.debug_tuple("ReuniteError")
            .field(&"...")
            .finish()
    }
}

impl<T> fmt::Display for ReuniteError<T> {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(fmt, "tried to reunite two BiLocks that don't form a pair")
    }
}

#[cfg(feature = "std")]
impl<T: Any> Error for ReuniteError<T> {}

/// Returned RAII guard from the `poll_lock` method.
///
/// This structure acts as a sentinel to the data in the `BiLock<T>` itself,
/// implementing `Deref` and `DerefMut` to `T`. When dropped, the lock will be
/// unlocked.
#[derive(Debug)]
pub struct BiLockGuard<'a, T> {
    bilock: &'a BiLock<T>,
}

impl<T> Deref for BiLockGuard<'_, T> {
    type Target = T;
    fn deref(&self) -> &T {
        unsafe { &*self.bilock.arc.value.as_ref().unwrap().get() }
    }
}

impl<T: Unpin> DerefMut for BiLockGuard<'_, T> {
    fn deref_mut(&mut self) -> &mut T {
        unsafe { &mut *self.bilock.arc.value.as_ref().unwrap().get() }
    }
}

impl<T> BiLockGuard<'_, T> {
    /// Get a mutable pinned reference to the locked value.
    pub fn as_pin_mut(&mut self) -> Pin<&mut T> {
        // Safety: we never allow moving a !Unpin value out of a bilock, nor
        // allow mutable access to it
        unsafe { Pin::new_unchecked(&mut *self.bilock.arc.value.as_ref().unwrap().get()) }
    }
}

impl<T> Drop for BiLockGuard<'_, T> {
    fn drop(&mut self) {
        self.bilock.unlock();
    }
}

/// Future returned by `BiLock::lock` which will resolve when the lock is
/// acquired.
#[must_use = "futures do nothing unless you `.await` or poll them"]
#[derive(Debug)]
pub struct BiLockAcquire<'a, T> {
    bilock: &'a BiLock<T>,
}

// Pinning is never projected to fields
impl<T> Unpin for BiLockAcquire<'_, T> {}

impl<'a, T> Future for BiLockAcquire<'a, T> {
    type Output = BiLockGuard<'a, T>;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        self.bilock.poll_lock(cx)
    }
}
<|endoftext|>


<commit_before>use bio::io::fastq;
use rust_htslib::bam;
use rust_htslib::bam::Read;
use std::fs;
use std::process::Command;

/// Compare an output file to the expected output and delete the output file.
fn test_output(result: &str, expected: &str) {
    assert!(Command::new("cmp")
        .arg(result)
        .arg(expected)
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    fs::remove_file(result).unwrap();
}

/// Compare two fastq files, ignoring the name lines
/// Reads are sorted by their sequence, which is not 100% robust
/// if mutations/ sequencing errors are considered.
fn compare_fastq(result: &str, expected: &str) {
    let result_reader = fastq::Reader::from_file(result).unwrap();
    let mut result_recs: Vec<fastq::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().to_owned());
    let expected_reader = fastq::Reader::from_file(expected).unwrap();
    let mut expected_recs: Vec<fastq::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().to_owned());

    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq(), expected.seq());
        assert_eq!(result.qual(), expected.qual());
    }
}

fn compare_bam(result: &str, expected: &str) {
    let mut result_reader = bam::Reader::from_path(result).unwrap();
    let mut result_recs: Vec<bam::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    let mut expected_reader = bam::Reader::from_path(expected).unwrap();
    let mut expected_recs: Vec<bam::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq().as_bytes(), expected.seq().as_bytes());
        assert_eq!(result.qual(), expected.qual());
    }
}

#[test]
fn fastq_split() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt fastq-split tests/A.fastq tests/B.fastq < tests/test.fastq")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/A.fastq", "tests/expected/A.fastq");
    test_output("tests/B.fastq", "tests/expected/B.fastq");
}

#[test]
fn fastq_filter() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg(
            "target/debug/rbt fastq-filter tests/ids.txt < tests/test.fastq > tests/filtered.fastq"
        )
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/filtered.fastq", "tests/expected/B.fastq");
}

#[test]
fn bam_depth() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt bam-depth tests/test.bam < tests/pos.txt > tests/depth.txt")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/depth.txt", "tests/expected/depth.txt");
}

#[test]
fn vcf_to_txt() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-to-txt --genotypes --fmt S --info T X SOMATIC < tests/test.vcf > tests/variant-table.txt")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/variant-table.txt",
        "tests/expected/variant-table.txt",
    );
}

#[test]
fn vcf_match() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test3.vcf < tests/test2.vcf > tests/matching.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output("tests/matching.bcf", "tests/expected/matching.bcf");
}

#[test]
fn vcf_match_same() {
    assert!(Command::new("bash").arg("-c")
                                .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test4.vcf < tests/test4.vcf > tests/matching-same.bcf")
                                .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/matching-same.bcf",
        "tests/expected/matching-same.bcf",
    );
}

#[test]
fn vcf_baf() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt vcf-baf < tests/test-freebayes.vcf > tests/baf.bcf")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/baf.bcf", "tests/expected/baf.bcf");
}

#[test]
fn test_call_consensus_reads_two_cluster() {
    assert!(
        Command::new("bash")
                .arg("-c")
                .arg("target/debug/rbt call-consensus-reads fastq --umi-len 3 -u --max-umi-dist 0 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus.1.fastq /tmp/test-consensus.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus.1.fastq",
        "tests/expected/test-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus.2.fastq",
        "tests/expected/test-consensus.2.fastq",
    );
}

#[test]
fn test_call_consensus_reads_single_cluster() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt call-consensus-reads fastq --umi-len 3 -u --max-umi-dist 2 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus_single.1.fastq /tmp/test-consensus_single.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus_single.1.fastq",
        "tests/expected/test-consensus_single.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus_single.2.fastq",
        "tests/expected/test-consensus_single.2.fastq",
    );
}

#[test]
fn test_call_overlapping_consensus_reads() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt call-consensus-reads fastq --umi-len 10 --max-umi-dist 0 --max-seq-dist 8 --insert-size 450 --std-dev 50  tests/overlapping-consensus.1.fastq tests/overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.1.fastq /tmp/test_overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.3.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test_overlapping-consensus.1.fastq",
        "tests/expected/test_overlapping-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.2.fastq",
        "tests/expected/test_overlapping-consensus.2.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.3.fastq",
        "tests/expected/test_overlapping-consensus.3.fastq",
    );
}

#[test]
fn test_call_consensus_from_bam() {
    assert!(
    Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt call-consensus-reads bam --max-seq-dist 8 tests/overlapping_consensus_marked.bam /tmp/overlapping_consensus_marked.bam")
        .spawn().unwrap().wait().unwrap().success());
    compare_bam(
        "/tmp/overlapping_consensus_marked.bam",
        "tests/expected/overlapping_consensus_marked.bam",
    );
}
<commit_msg>Added test case<commit_after>use bio::io::fastq;
use rust_htslib::bam;
use rust_htslib::bam::Read;
use std::fs;
use std::process::Command;

/// Compare an output file to the expected output and delete the output file.
fn test_output(result: &str, expected: &str) {
    assert!(Command::new("cmp")
        .arg(result)
        .arg(expected)
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    fs::remove_file(result).unwrap();
}

/// Compare two fastq files, ignoring the name lines
/// Reads are sorted by their sequence, which is not 100% robust
/// if mutations/ sequencing errors are considered.
fn compare_fastq(result: &str, expected: &str) {
    let result_reader = fastq::Reader::from_file(result).unwrap();
    let mut result_recs: Vec<fastq::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().to_owned());
    let expected_reader = fastq::Reader::from_file(expected).unwrap();
    let mut expected_recs: Vec<fastq::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().to_owned());

    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq(), expected.seq());
        assert_eq!(result.qual(), expected.qual());
    }
}

fn compare_bam(result: &str, expected: &str) {
    let mut result_reader = bam::Reader::from_path(result).unwrap();
    let mut result_recs: Vec<bam::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    let mut expected_reader = bam::Reader::from_path(expected).unwrap();
    let mut expected_recs: Vec<bam::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq().as_bytes(), expected.seq().as_bytes());
        assert_eq!(result.qual(), expected.qual());
    }
}

#[test]
fn fastq_split() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt fastq-split tests/A.fastq tests/B.fastq < tests/test.fastq")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/A.fastq", "tests/expected/A.fastq");
    test_output("tests/B.fastq", "tests/expected/B.fastq");
}

#[test]
fn fastq_filter() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg(
            "target/debug/rbt fastq-filter tests/ids.txt < tests/test.fastq > tests/filtered.fastq"
        )
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/filtered.fastq", "tests/expected/B.fastq");
}

#[test]
fn bam_depth() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt bam-depth tests/test.bam < tests/pos.txt > tests/depth.txt")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/depth.txt", "tests/expected/depth.txt");
}

#[test]
fn vcf_to_txt() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-to-txt --genotypes --fmt S --info T X SOMATIC < tests/test.vcf > tests/variant-table.txt")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/variant-table.txt",
        "tests/expected/variant-table.txt",
    );
}

#[test]
fn vcf_match() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test3.vcf < tests/test2.vcf > tests/matching.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output("tests/matching.bcf", "tests/expected/matching.bcf");
}

#[test]
fn vcf_match_same() {
    assert!(Command::new("bash").arg("-c")
                                .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test4.vcf < tests/test4.vcf > tests/matching-same.bcf")
                                .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/matching-same.bcf",
        "tests/expected/matching-same.bcf",
    );
}

#[test]
fn vcf_baf() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt vcf-baf < tests/test-freebayes.vcf > tests/baf.bcf")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/baf.bcf", "tests/expected/baf.bcf");
}

#[test]
fn test_call_consensus_reads_two_cluster() {
    assert!(
        Command::new("bash")
                .arg("-c")
                .arg("target/debug/rbt call-consensus-reads fastq --umi-len 3 -u --max-umi-dist 0 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus.1.fastq /tmp/test-consensus.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus.1.fastq",
        "tests/expected/test-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus.2.fastq",
        "tests/expected/test-consensus.2.fastq",
    );
}

#[test]
fn test_call_consensus_reads_single_cluster() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt call-consensus-reads fastq --umi-len 3 -u --max-umi-dist 2 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus_single.1.fastq /tmp/test-consensus_single.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus_single.1.fastq",
        "tests/expected/test-consensus_single.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus_single.2.fastq",
        "tests/expected/test-consensus_single.2.fastq",
    );
}

#[test]
fn test_call_overlapping_consensus_reads() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt call-consensus-reads fastq --umi-len 10 --max-umi-dist 0 --max-seq-dist 8 --insert-size 450 --std-dev 50  tests/overlapping-consensus.1.fastq tests/overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.1.fastq /tmp/test_overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.3.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test_overlapping-consensus.1.fastq",
        "tests/expected/test_overlapping-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.2.fastq",
        "tests/expected/test_overlapping-consensus.2.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.3.fastq",
        "tests/expected/test_overlapping-consensus.3.fastq",
    );
}

#[test]
fn test_call_consensus_from_bam() {
    assert!(
    Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt call-consensus-reads bam --max-seq-dist 8 tests/overlapping_consensus_marked.bam /tmp/overlapping_consensus_marked.bam")
        .spawn().unwrap().wait().unwrap().success());
    compare_bam(
        "/tmp/overlapping_consensus_marked.bam",
        "tests/expected/overlapping_consensus_marked.bam",
    );
}

#[test]
fn test_vcf_annotate_dgidb() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-annotate-dgidb tests/annotate_dgidb_test.vcf > /tmp/annotate_dgidb_test.vcf")
            .spawn().unwrap().wait().unwrap().success());
    compare_bam(
        "/tmp/annotate_dgidb_test.vcf",
        "tests/expected/annotate_dgidb_test.vcf",
    );
}<|endoftext|>


<commit_before>// TODO: COPYRIGHT, USE & AUTHORS
// TODO: RUST DOCS!

extern crate rust_monster;
#[macro_use]
extern crate log;
extern crate env_logger;
#[cfg(test)]
mod tests
{
    use rust_monster::ga::ga_core::{GeneticAlgorithm, GASolution, GAFactory, DEBUG_FLAG};
    use rust_monster::ga::ga_population::{GAPopulation, GAPopulationSortOrder, GAPopulationSortBasis};
    use rust_monster::ga::ga_simple::{SimpleGeneticAlgorithm, SimpleGeneticAlgorithmCfg};
    use rust_monster::ga::ga_selectors::{GASelector, GARankSelector, GAUniformSelector};

    use env_logger;
    use std::sync::{Once, ONCE_INIT};

    static INIT: Once = ONCE_INIT;
    const VAL: f32 = 3.14159;

    struct TestSolution
    {
        fitness: f32
    }
    impl GASolution for TestSolution
    {
        fn new() -> TestSolution
        {
            TestSolution{ fitness: VAL }
        }

        fn clone(&self) -> Self { TestSolution::new() }
        fn evaluate(&mut self) -> f32 { self.fitness }
    #[allow(unused_variables)]
        fn crossover(&self, other : &Self) -> Self { TestSolution::new() }
    #[allow(unused_variables)]
        fn mutate(&mut self, pm : f32) {}
        fn fitness(&self) -> f32 { (1.0 / self.fitness) }
        fn score(&self) -> f32 { self.fitness }
    }

    #[allow(dead_code)]
    struct TestFactory
    {
        starting_fitness: f32
    }
    impl GAFactory<TestSolution> for TestFactory
    {
        fn initial_population(&mut self) -> GAPopulation<TestSolution> {
            GAPopulation::new(vec![TestSolution { fitness: self.starting_fitness }], GAPopulationSortOrder::HighIsBest)
        }
    }

////////////////////////////////////////
// Utility functions

    fn ga_test_setup()
    {
        // This is only called once, works nicely as a Setup mechanism
        // TODO: Can we do somethign similar for teardown?
        INIT.call_once(||{
            env_logger::init().unwrap();
        });
    }

    fn simple_ga_validation(sga:&mut SimpleGeneticAlgorithm<TestSolution>)
    {
        sga.initialize();
        assert_eq!(sga.step(), 1);
        assert_eq!(sga.done(), false);
        assert_eq!(sga.population().size(), 1);
        assert_eq!(sga.population().best().score(), VAL);
    }

////////////////////////////////////////
// Tests

    #[test]
    fn init_test_with_initial_population()
    {
        ga_test_setup();
        let initial_population = GAPopulation::new(vec![TestSolution { fitness: VAL}], GAPopulationSortOrder::HighIsBest);
        let mut ga : SimpleGeneticAlgorithm<TestSolution> =
                     SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                                   d_seed : 1,
                                                   flags : DEBUG_FLAG,
                                                   max_generations: 100,
                                                   ..Default::default()
                                                 },
                                                 None,
                                                 Some(initial_population)
                                                 );
        simple_ga_validation(&mut ga);
    }

    #[test]
    fn init_test_with_factory()
    {
        ga_test_setup();
        let mut factory = TestFactory { starting_fitness: VAL };
        let mut ga : SimpleGeneticAlgorithm<TestSolution> =
                     SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                                   d_seed : 1,
                                                   flags : DEBUG_FLAG,
                                                   max_generations: 100,
                                                   ..Default::default()
                                                 },
                                                 Some(&mut factory as &mut GAFactory<TestSolution>),
                                                 None
                                                 );
        simple_ga_validation(&mut ga);
    }

    #[test]
    #[should_panic]
    #[allow(unused_variables)]
    fn init_test_missing_args()
    {
        ga_test_setup();
        let ga : SimpleGeneticAlgorithm<TestSolution> =
                 SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                               d_seed : 1,
                                               flags : DEBUG_FLAG,
                                               max_generations: 100,
                                               ..Default::default()
                                             },
                                             None,
                                             None
                                             );
        // Not reached
    }

    #[test]
    #[should_panic]
    fn init_test_empty_initial_pop()
    {
        ga_test_setup();
        let empty_initial_population : GAPopulation<TestSolution> = GAPopulation::new(vec![], GAPopulationSortOrder::HighIsBest);
        let mut ga : SimpleGeneticAlgorithm<TestSolution> =
                     SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                                   d_seed : 1,
                                                   flags : DEBUG_FLAG,
                                                   max_generations: 100,
                                                   ..Default::default()
                                                 },
                                                 None,
                                                 Some(empty_initial_population)
                                                 );
        ga.initialize()
        //Not reached
    }

    // This test should live in the ga_population file directly,
    // but since it requires GASolution objects it is here for now.
    // Maybe we could use a Mocking library.
    // TODO: This doesn't go here.
    #[test]
    fn test_sort_population()
    {
        let f = VAL;
        let f_m = VAL - 1.0;
        let i_f = 1.0 / f;
        let i_f_m = 1.0 / f_m;

        let mut population = GAPopulation::new(vec![TestSolution { fitness: f }, TestSolution { fitness: f_m }], GAPopulationSortOrder::HighIsBest);
        population.sort();

        //TestSolution's Fitness is the inverse of the Score (F = 1/S)
        assert_eq!(population.individual(0, GAPopulationSortBasis::Raw).score(), f);
        assert_eq!(population.individual(1, GAPopulationSortBasis::Raw).score(), f_m);
        assert_eq!(population.individual(0, GAPopulationSortBasis::Scaled).fitness(), i_f_m);
        assert_eq!(population.individual(1, GAPopulationSortBasis::Scaled).fitness(), i_f);
    }

    #[test]
    #[allow(unused_variables)]
    fn test_rank_selector()
    {
        let f = VAL;
        let f_m = VAL - 1.0;
        let i_f = 1.0 / f;
        let i_f_m = 1.0 / f_m;

        let mut population
          = GAPopulation::new(vec![TestSolution { fitness: f },
                                   TestSolution { fitness: f_m }],
                              GAPopulationSortOrder::HighIsBest);

        {
            let mut raw_rank_selector = GARankSelector::new(&mut population, GAPopulationSortBasis::Raw);

            // Best Raw score is that of 1st solution.
            assert_eq!(raw_rank_selector.select().score(), f);
        }

        {
            let mut scaled_rank_selector = GARankSelector::new(&mut population, GAPopulationSortBasis::Scaled);

            // Best Scaled score is that of 2nd solution (because fitness is inverse of score). Weird. In this case, LowIsBest.
            assert_eq!(scaled_rank_selector.select().fitness(), i_f_m);
        }
    }

    #[test]
    fn test_uniform_selector()
    {
        let f = VAL;
        let f_m = VAL - 1.0;

        let mut population
          = GAPopulation::new(vec![TestSolution { fitness: f },
                                   TestSolution { fitness: f_m }],
                              GAPopulationSortOrder::HighIsBest);

        let mut uniform_selector = GAUniformSelector::new(&mut population);

        let selected_individual = uniform_selector.select();
        assert!(selected_individual.score() == f || selected_individual.score() == f_m);
    }
}
<commit_msg>Adapt GARankSelector test based on addition of GAScoreTypeBasedSelection.<commit_after>// TODO: COPYRIGHT, USE & AUTHORS
// TODO: RUST DOCS!

extern crate rust_monster;
#[macro_use]
extern crate log;
extern crate env_logger;
#[cfg(test)]
mod tests
{
    use rust_monster::ga::ga_core::{GeneticAlgorithm, GASolution, GAFactory, DEBUG_FLAG};
    use rust_monster::ga::ga_population::{GAPopulation, GAPopulationSortOrder, GAPopulationSortBasis};
    use rust_monster::ga::ga_simple::{SimpleGeneticAlgorithm, SimpleGeneticAlgorithmCfg};
    use rust_monster::ga::ga_selectors::{GASelector, GARankSelector, GAUniformSelector, GARawScoreBasedSelection, GAScaledScoreBasedSelection};

    use env_logger;
    use std::sync::{Once, ONCE_INIT};

    static INIT: Once = ONCE_INIT;
    const VAL: f32 = 3.14159;

    struct TestSolution
    {
        fitness: f32
    }
    impl GASolution for TestSolution
    {
        fn new() -> TestSolution
        {
            TestSolution{ fitness: VAL }
        }

        fn clone(&self) -> Self { TestSolution::new() }
        fn evaluate(&mut self) -> f32 { self.fitness }
    #[allow(unused_variables)]
        fn crossover(&self, other : &Self) -> Self { TestSolution::new() }
    #[allow(unused_variables)]
        fn mutate(&mut self, pm : f32) {}
        fn fitness(&self) -> f32 { (1.0 / self.fitness) }
        fn score(&self) -> f32 { self.fitness }
    }

    #[allow(dead_code)]
    struct TestFactory
    {
        starting_fitness: f32
    }
    impl GAFactory<TestSolution> for TestFactory
    {
        fn initial_population(&mut self) -> GAPopulation<TestSolution> {
            GAPopulation::new(vec![TestSolution { fitness: self.starting_fitness }], GAPopulationSortOrder::HighIsBest)
        }
    }

////////////////////////////////////////
// Utility functions

    fn ga_test_setup()
    {
        // This is only called once, works nicely as a Setup mechanism
        // TODO: Can we do somethign similar for teardown?
        INIT.call_once(||{
            env_logger::init().unwrap();
        });
    }

    fn simple_ga_validation(sga:&mut SimpleGeneticAlgorithm<TestSolution>)
    {
        sga.initialize();
        assert_eq!(sga.step(), 1);
        assert_eq!(sga.done(), false);
        assert_eq!(sga.population().size(), 1);
        assert_eq!(sga.population().best().score(), VAL);
    }

////////////////////////////////////////
// Tests

    #[test]
    fn init_test_with_initial_population()
    {
        ga_test_setup();
        let initial_population = GAPopulation::new(vec![TestSolution { fitness: VAL}], GAPopulationSortOrder::HighIsBest);
        let mut ga : SimpleGeneticAlgorithm<TestSolution> =
                     SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                                   d_seed : 1,
                                                   flags : DEBUG_FLAG,
                                                   max_generations: 100,
                                                   ..Default::default()
                                                 },
                                                 None,
                                                 Some(initial_population)
                                                 );
        simple_ga_validation(&mut ga);
    }

    #[test]
    fn init_test_with_factory()
    {
        ga_test_setup();
        let mut factory = TestFactory { starting_fitness: VAL };
        let mut ga : SimpleGeneticAlgorithm<TestSolution> =
                     SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                                   d_seed : 1,
                                                   flags : DEBUG_FLAG,
                                                   max_generations: 100,
                                                   ..Default::default()
                                                 },
                                                 Some(&mut factory as &mut GAFactory<TestSolution>),
                                                 None
                                                 );
        simple_ga_validation(&mut ga);
    }

    #[test]
    #[should_panic]
    #[allow(unused_variables)]
    fn init_test_missing_args()
    {
        ga_test_setup();
        let ga : SimpleGeneticAlgorithm<TestSolution> =
                 SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                               d_seed : 1,
                                               flags : DEBUG_FLAG,
                                               max_generations: 100,
                                               ..Default::default()
                                             },
                                             None,
                                             None
                                             );
        // Not reached
    }

    #[test]
    #[should_panic]
    fn init_test_empty_initial_pop()
    {
        ga_test_setup();
        let empty_initial_population : GAPopulation<TestSolution> = GAPopulation::new(vec![], GAPopulationSortOrder::HighIsBest);
        let mut ga : SimpleGeneticAlgorithm<TestSolution> =
                     SimpleGeneticAlgorithm::new(SimpleGeneticAlgorithmCfg {
                                                   d_seed : 1,
                                                   flags : DEBUG_FLAG,
                                                   max_generations: 100,
                                                   ..Default::default()
                                                 },
                                                 None,
                                                 Some(empty_initial_population)
                                                 );
        ga.initialize()
        //Not reached
    }

    // This test should live in the ga_population file directly,
    // but since it requires GASolution objects it is here for now.
    // Maybe we could use a Mocking library.
    // TODO: This doesn't go here.
    #[test]
    fn test_sort_population()
    {
        let f = VAL;
        let f_m = VAL - 1.0;
        let i_f = 1.0 / f;
        let i_f_m = 1.0 / f_m;

        let mut population = GAPopulation::new(vec![TestSolution { fitness: f }, TestSolution { fitness: f_m }], GAPopulationSortOrder::HighIsBest);
        population.sort();

        //TestSolution's Fitness is the inverse of the Score (F = 1/S)
        assert_eq!(population.individual(0, GAPopulationSortBasis::Raw).score(), f);
        assert_eq!(population.individual(1, GAPopulationSortBasis::Raw).score(), f_m);
        assert_eq!(population.individual(0, GAPopulationSortBasis::Scaled).fitness(), i_f_m);
        assert_eq!(population.individual(1, GAPopulationSortBasis::Scaled).fitness(), i_f);
    }

    #[test]
    #[allow(unused_variables)]
    fn test_rank_selector()
    {
        let f = VAL;
        let f_m = VAL - 1.0;
        let i_f = 1.0 / f;
        let i_f_m = 1.0 / f_m;

        let mut population
          = GAPopulation::new(vec![TestSolution { fitness: f },
                                   TestSolution { fitness: f_m }],
                              GAPopulationSortOrder::HighIsBest);

        {
            let raw_score_selection = GARawScoreBasedSelection;
            let mut raw_rank_selector = GARankSelector::new(&mut population, &raw_score_selection);

            // Best Raw score is that of 1st solution.
            assert_eq!(raw_rank_selector.select().score(), f);
        }

        {
            let scaled_score_selection = GAScaledScoreBasedSelection;
            let mut scaled_rank_selector = GARankSelector::new(&mut population, &scaled_score_selection);

            // Best Scaled score is that of 2nd solution (because fitness is inverse of score). Weird. In this case, LowIsBest.
            assert_eq!(scaled_rank_selector.select().fitness(), i_f_m);
        }
    }

    #[test]
    fn test_uniform_selector()
    {
        let f = VAL;
        let f_m = VAL - 1.0;

        let mut population
          = GAPopulation::new(vec![TestSolution { fitness: f },
                                   TestSolution { fitness: f_m }],
                              GAPopulationSortOrder::HighIsBest);

        let mut uniform_selector = GAUniformSelector::new(&mut population);

        let selected_individual = uniform_selector.select();
        assert!(selected_individual.score() == f || selected_individual.score() == f_m);
    }
}
<|endoftext|>


<commit_before>extern crate csv;
extern crate vikos;

use std::default::Default;
use vikos::{cost, model, teacher};

#[test]
fn estimate_median() {
    use vikos::Teacher;

    let features = ();
    let history = [1.0, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //median is seven

    let cost = cost::LeastAbsoluteDeviation {};
    let mut model = 0.0;

    let teacher = teacher::GradientDescentAl { l0: 0.9, t: 9.0 };
    let mut training = teacher.new_training(&model);

    for &truth in history.iter().cycle().take(150) {
        teacher.teach_event(&mut training, &mut model, &cost, &features, truth);
        println!("model: {:?}", model);
    }

    assert!(model < 7.1);
    assert!(model > 6.9);
}

#[test]
fn estimate_mean() {
    use vikos::Teacher;

    let features = ();
    let history = [1f64, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //mean is 9

    let cost = cost::LeastSquares {};
    let mut model = 0.0;

    let teacher = teacher::GradientDescentAl { l0: 0.3, t: 4.0 };
    let mut training = teacher.new_training(&model);

    for &truth in history.iter().cycle().take(100) {
        teacher.teach_event(&mut training, &mut model, &cost, &features, truth);
        println!("model: {:?}", model);
    }

    assert!(model < 9.1);
    assert!(model > 8.9);
}

#[test]
fn linear_stochastic_gradient_descent() {
    use vikos::learn_history;

    let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

    let mut model = model::Linear { m: 0.0, c: 0.0 };

    let teacher = teacher::GradientDescent { learning_rate: 0.2 };

    let cost = cost::LeastSquares {};
    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    assert!(model.m < 1.1);
    assert!(model.m > 0.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn linear_stochastic_gradient_descent_iter() {
    use vikos::Teacher;

    let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

    let cost = cost::LeastSquares {};
    let mut model = model::Linear { m: 0.0, c: 0.0 };

    let teacher = teacher::GradientDescent { learning_rate: 0.2 };
    let mut training = teacher.new_training(&model);

    for &(features, truth) in history.iter().cycle().take(20) {
        teacher.teach_event(&mut training, &mut model, &cost, &features, truth);
        println!("model: {:?}", model);
    }

    assert!(model.m < 1.1);
    assert!(model.m > 0.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn linear_sgd_2d() {
    use vikos::learn_history;

    let history = [([0.0, 7.0], 17.0), ([1.0, 2.0], 8.0), ([2.0, -2.0], 1.0)];
    let mut model = model::Linear {
        m: [0.0, 0.0],
        c: 0.0,
    };
    let cost = cost::LeastSquares {};
    let teacher = teacher::Momentum {
        l0: 0.009,
        t: 1000.0,
        inertia: 0.995,
    };

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(1500).cloned(),
    );

    println!("{:?}", model);

    assert!(model.m[0] < 1.1);
    assert!(model.m[0] > 0.9);
    assert!(model.m[1] < 2.1);
    assert!(model.m[1] > 1.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn linear_nesterov_2d() {
    use vikos::learn_history;

    let history = [([0.0, 7.0], 17.0), ([1.0, 2.0], 8.0), ([2.0, -2.0], 1.0)];
    let mut model = model::Linear {
        m: [0.0, 0.0],
        c: 0.0,
    };
    let cost = cost::LeastSquares {};
    let teacher = teacher::Nesterov {
        l0: 0.009,
        t: 1000.0,
        inertia: 0.995,
    };

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(1500).cloned(),
    );

    println!("{:?}", model);

    assert!(model.m[0] < 1.1);
    assert!(model.m[0] > 0.9);
    assert!(model.m[1] < 2.1);
    assert!(model.m[1] > 1.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn logistic_sgd_2d_least_squares() {
    use vikos::{learn_history, Model};

    let history = [
        ([2.7, 2.5], 0.0),
        ([1.4, 2.3], 0.0),
        ([3.3, 4.4], 0.0),
        ([1.3, 1.8], 0.0),
        ([3.0, 3.0], 0.0),
        ([7.6, 2.7], 1.0),
        ([5.3, 2.0], 1.0),
        ([6.9, 1.7], 1.0),
        ([8.6, -0.2], 1.0),
        ([7.6, 3.5], 1.0),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::LeastSquares {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(40).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).round() == truth)
        .fold(
            0,
            |errors, correct| if correct { errors } else { errors + 1 },
        );

    assert_eq!(0, classification_errors);
}

#[test]
fn logistic_sgd_2d_max_likelihood() {
    use vikos::{learn_history, Model};

    let history = [
        ([2.7, 2.5], 0.0),
        ([1.4, 2.3], 0.0),
        ([3.3, 4.4], 0.0),
        ([1.3, 1.8], 0.0),
        ([3.0, 3.0], 0.0),
        ([7.6, 2.7], 1.0),
        ([5.3, 2.0], 1.0),
        ([6.9, 1.7], 1.0),
        ([8.6, -0.2], 1.0),
        ([7.6, 3.5], 1.0),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).round() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn logistic_sgd_2d_max_likelihood_bool() {
    use vikos::{learn_history, Crisp, Model};

    let history = [
        ([2.7, 2.5], false),
        ([1.4, 2.3], false),
        ([3.3, 4.4], false),
        ([1.3, 1.8], false),
        ([3.0, 3.0], false),
        ([7.6, 2.7], true),
        ([5.3, 2.0], true),
        ([6.9, 1.7], true),
        ([8.6, -0.2], true),
        ([7.6, 3.5], true),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn logistic_adagard_2d_max_likelihood_bool() {
    use vikos::{learn_history, Crisp, Model};

    let history = [
        ([2.7, 2.5], false),
        ([1.4, 2.3], false),
        ([3.3, 4.4], false),
        ([1.3, 1.8], false),
        ([3.0, 3.0], false),
        ([7.6, 2.7], true),
        ([5.3, 2.0], true),
        ([6.9, 1.7], true),
        ([8.6, -0.2], true),
        ([7.6, 3.5], true),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::Adagard {
        learning_rate: 3.0,
        epsilon: 10000.0,
    };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(60).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn generalized_linear_model_as_logistic_regression() {
    use vikos::{learn_history, Crisp, Model};

    let history = [
        ([2.7, 2.5], false),
        ([1.4, 2.3], false),
        ([3.3, 4.4], false),
        ([1.3, 1.8], false),
        ([3.0, 3.0], false),
        ([7.6, 2.7], true),
        ([5.3, 2.0], true),
        ([6.9, 1.7], true),
        ([8.6, -0.2], true),
        ([7.6, 3.5], true),
    ];

    let mut model = model::GeneralizedLinearModel::new(
        |x| 1.0 / (1.0 + x.exp()),
        |x| -x.exp() / (1.0 + x.exp()).powi(2),
    );
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn iris() {
    use csv;
    use vikos::{learn_history, Crisp, Model};

    let mut model = model::OneVsRest::<[model::Logistic<[f64; 4]>; 3]>::default();
    let teacher = vikos::teacher::Nesterov {
        l0: 0.0001,
        t: 1000.0,
        inertia: 0.99,
    };
    let cost = cost::MaxLikelihood {};

    let history: Vec<_> = csv::Reader::from_path("examples/data/iris.csv")
        .expect("File is ok")
        .deserialize()
        .map(|row| {
            let (t, f): (String, _) = row.unwrap();
            (t, f)
        })
        .map(|(truth, features)| {
            (
                features,
                match truth.as_ref() {
                    "setosa" => 0,
                    "versicolor" => 1,
                    "virginica" => 2,
                    _ => panic!("unknow class"),
                },
            )
        })
        .collect();

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(3000).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(3, classification_errors);
}
<commit_msg>fix edition idioms<commit_after>use csv;
use std::default::Default;
use vikos::{cost, model, teacher};

#[test]
fn estimate_median() {
    use vikos::Teacher;

    let features = ();
    let history = [1.0, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //median is seven

    let cost = cost::LeastAbsoluteDeviation {};
    let mut model = 0.0;

    let teacher = teacher::GradientDescentAl { l0: 0.9, t: 9.0 };
    let mut training = teacher.new_training(&model);

    for &truth in history.iter().cycle().take(150) {
        teacher.teach_event(&mut training, &mut model, &cost, &features, truth);
        println!("model: {:?}", model);
    }

    assert!(model < 7.1);
    assert!(model > 6.9);
}

#[test]
fn estimate_mean() {
    use vikos::Teacher;

    let features = ();
    let history = [1f64, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //mean is 9

    let cost = cost::LeastSquares {};
    let mut model = 0.0;

    let teacher = teacher::GradientDescentAl { l0: 0.3, t: 4.0 };
    let mut training = teacher.new_training(&model);

    for &truth in history.iter().cycle().take(100) {
        teacher.teach_event(&mut training, &mut model, &cost, &features, truth);
        println!("model: {:?}", model);
    }

    assert!(model < 9.1);
    assert!(model > 8.9);
}

#[test]
fn linear_stochastic_gradient_descent() {
    use vikos::learn_history;

    let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

    let mut model = model::Linear { m: 0.0, c: 0.0 };

    let teacher = teacher::GradientDescent { learning_rate: 0.2 };

    let cost = cost::LeastSquares {};
    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    assert!(model.m < 1.1);
    assert!(model.m > 0.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn linear_stochastic_gradient_descent_iter() {
    use vikos::Teacher;

    let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

    let cost = cost::LeastSquares {};
    let mut model = model::Linear { m: 0.0, c: 0.0 };

    let teacher = teacher::GradientDescent { learning_rate: 0.2 };
    let mut training = teacher.new_training(&model);

    for &(features, truth) in history.iter().cycle().take(20) {
        teacher.teach_event(&mut training, &mut model, &cost, &features, truth);
        println!("model: {:?}", model);
    }

    assert!(model.m < 1.1);
    assert!(model.m > 0.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn linear_sgd_2d() {
    use vikos::learn_history;

    let history = [([0.0, 7.0], 17.0), ([1.0, 2.0], 8.0), ([2.0, -2.0], 1.0)];
    let mut model = model::Linear {
        m: [0.0, 0.0],
        c: 0.0,
    };
    let cost = cost::LeastSquares {};
    let teacher = teacher::Momentum {
        l0: 0.009,
        t: 1000.0,
        inertia: 0.995,
    };

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(1500).cloned(),
    );

    println!("{:?}", model);

    assert!(model.m[0] < 1.1);
    assert!(model.m[0] > 0.9);
    assert!(model.m[1] < 2.1);
    assert!(model.m[1] > 1.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn linear_nesterov_2d() {
    use vikos::learn_history;

    let history = [([0.0, 7.0], 17.0), ([1.0, 2.0], 8.0), ([2.0, -2.0], 1.0)];
    let mut model = model::Linear {
        m: [0.0, 0.0],
        c: 0.0,
    };
    let cost = cost::LeastSquares {};
    let teacher = teacher::Nesterov {
        l0: 0.009,
        t: 1000.0,
        inertia: 0.995,
    };

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(1500).cloned(),
    );

    println!("{:?}", model);

    assert!(model.m[0] < 1.1);
    assert!(model.m[0] > 0.9);
    assert!(model.m[1] < 2.1);
    assert!(model.m[1] > 1.9);
    assert!(model.c < 3.1);
    assert!(model.c > 2.9);
}

#[test]
fn logistic_sgd_2d_least_squares() {
    use vikos::{learn_history, Model};

    let history = [
        ([2.7, 2.5], 0.0),
        ([1.4, 2.3], 0.0),
        ([3.3, 4.4], 0.0),
        ([1.3, 1.8], 0.0),
        ([3.0, 3.0], 0.0),
        ([7.6, 2.7], 1.0),
        ([5.3, 2.0], 1.0),
        ([6.9, 1.7], 1.0),
        ([8.6, -0.2], 1.0),
        ([7.6, 3.5], 1.0),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::LeastSquares {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(40).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).round() == truth)
        .fold(
            0,
            |errors, correct| if correct { errors } else { errors + 1 },
        );

    assert_eq!(0, classification_errors);
}

#[test]
fn logistic_sgd_2d_max_likelihood() {
    use vikos::{learn_history, Model};

    let history = [
        ([2.7, 2.5], 0.0),
        ([1.4, 2.3], 0.0),
        ([3.3, 4.4], 0.0),
        ([1.3, 1.8], 0.0),
        ([3.0, 3.0], 0.0),
        ([7.6, 2.7], 1.0),
        ([5.3, 2.0], 1.0),
        ([6.9, 1.7], 1.0),
        ([8.6, -0.2], 1.0),
        ([7.6, 3.5], 1.0),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).round() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn logistic_sgd_2d_max_likelihood_bool() {
    use vikos::{learn_history, Crisp, Model};

    let history = [
        ([2.7, 2.5], false),
        ([1.4, 2.3], false),
        ([3.3, 4.4], false),
        ([1.3, 1.8], false),
        ([3.0, 3.0], false),
        ([7.6, 2.7], true),
        ([5.3, 2.0], true),
        ([6.9, 1.7], true),
        ([8.6, -0.2], true),
        ([7.6, 3.5], true),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn logistic_adagard_2d_max_likelihood_bool() {
    use vikos::{learn_history, Crisp, Model};

    let history = [
        ([2.7, 2.5], false),
        ([1.4, 2.3], false),
        ([3.3, 4.4], false),
        ([1.3, 1.8], false),
        ([3.0, 3.0], false),
        ([7.6, 2.7], true),
        ([5.3, 2.0], true),
        ([6.9, 1.7], true),
        ([8.6, -0.2], true),
        ([7.6, 3.5], true),
    ];

    let mut model = model::Logistic::default();
    let teacher = teacher::Adagard {
        learning_rate: 3.0,
        epsilon: 10000.0,
    };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(60).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn generalized_linear_model_as_logistic_regression() {
    use vikos::{learn_history, Crisp, Model};

    let history = [
        ([2.7, 2.5], false),
        ([1.4, 2.3], false),
        ([3.3, 4.4], false),
        ([1.3, 1.8], false),
        ([3.0, 3.0], false),
        ([7.6, 2.7], true),
        ([5.3, 2.0], true),
        ([6.9, 1.7], true),
        ([8.6, -0.2], true),
        ([7.6, 3.5], true),
    ];

    let mut model = model::GeneralizedLinearModel::new(
        |x| 1.0 / (1.0 + x.exp()),
        |x| -x.exp() / (1.0 + x.exp()).powi(2),
    );
    let teacher = teacher::GradientDescent { learning_rate: 0.3 };
    let cost = cost::MaxLikelihood {};

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(20).cloned(),
    );

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(0, classification_errors);
}

#[test]
fn iris() {
    use csv;
    use vikos::{learn_history, Crisp, Model};

    let mut model = model::OneVsRest::<[model::Logistic<[f64; 4]>; 3]>::default();
    let teacher = vikos::teacher::Nesterov {
        l0: 0.0001,
        t: 1000.0,
        inertia: 0.99,
    };
    let cost = cost::MaxLikelihood {};

    let history: Vec<_> = csv::Reader::from_path("examples/data/iris.csv")
        .expect("File is ok")
        .deserialize()
        .map(|row| {
            let (t, f): (String, _) = row.unwrap();
            (t, f)
        })
        .map(|(truth, features)| {
            (
                features,
                match truth.as_ref() {
                    "setosa" => 0,
                    "versicolor" => 1,
                    "virginica" => 2,
                    _ => panic!("unknow class"),
                },
            )
        })
        .collect();

    learn_history(
        &teacher,
        &cost,
        &mut model,
        history.iter().cycle().take(3000).cloned(),
    );

    println!("{:?}", model);

    let classification_errors = history
        .iter()
        .map(|&(input, truth)| model.predict(&input).crisp() == truth)
        .map(|correct| if correct { 0 } else { 1 })
        .sum();

    assert_eq!(3, classification_errors);
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::collections::HashMap;

use clap::{App, AppSettings, Arg};

struct Help {
    short: &'static str,
    long: &'static str,
}

macro_rules! doc {
    ($map:expr, $name:expr, $short:expr) => {
        doc!($map, $name, $short, $short)
    };
    ($map:expr, $name:expr, $short:expr, $long:expr) => {
        $map.insert($name, Help {
            short: $short,
            long: concat!($long, "\n ")
        });
    };
}

pub fn build_app() -> App<'static, 'static> {
    let helps = usage();
    let arg = |name| {
        Arg::with_name(name).help(helps[name].short).long_help(
            helps[name].long,
        )
    };

    App::new("fd")
        .version(crate_version!())
        .usage("fd [FLAGS/OPTIONS] [<pattern>] [<path>]")
        .setting(AppSettings::ColoredHelp)
        .setting(AppSettings::DeriveDisplayOrder)
        .arg(arg("hidden").long("hidden").short("H"))
        .arg(arg("no-ignore").long("no-ignore").short("I"))
        .arg(
            arg("rg-alias-hidden-ignore")
                .short("u")
                .multiple(true)
                .hidden(true),
        )
        .arg(
            arg("case-sensitive")
                .long("case-sensitive")
                .short("s")
                .overrides_with("ignore-case"),
        )
        .arg(
            arg("ignore-case")
                .long("ignore-case")
                .short("i")
                .overrides_with("case-sensitive"),
        )
        .arg(arg("absolute-path").long("absolute-path").short("a"))
        .arg(arg("follow").long("follow").short("L").alias("dereference"))
        .arg(arg("full-path").long("full-path").short("p"))
        .arg(arg("null_separator").long("print0").short("0"))
        .arg(arg("depth").long("max-depth").short("d").takes_value(true))
        .arg(
            arg("file-type")
                .long("type")
                .short("t")
                .takes_value(true)
                .value_name("filetype")
                .possible_values(&["f", "file", "d", "directory", "l", "symlink"])
                .hide_possible_values(true),
        )
        .arg(
            arg("extension")
                .long("extension")
                .short("e")
                .takes_value(true)
                .value_name("ext"),
        )
        .arg(
            arg("exec")
                .long("exec")
                .short("x")
                .multiple(true)
                .min_values(1)
                .allow_hyphen_values(true)
                .value_terminator(";")
                .value_name("cmd"),
        )
        .arg(
            arg("exclude")
                .long("exclude")
                .short("E")
                .takes_value(true)
                .value_name("pattern")
                .number_of_values(1)
                .multiple(true),
        )
        .arg(
            arg("color")
                .long("color")
                .short("c")
                .takes_value(true)
                .value_name("when")
                .possible_values(&["never", "auto", "always"])
                .hide_possible_values(true),
        )
        .arg(
            arg("threads")
                .long("threads")
                .short("j")
                .takes_value(true)
                .value_name("num"),
        )
        .arg(
            arg("max-buffer-time")
                .long("max-buffer-time")
                .takes_value(true)
                .hidden(true),
        )
        .arg(arg("pattern"))
        .arg(arg("path"))
}

#[cfg_attr(rustfmt, rustfmt_skip)]
fn usage() -> HashMap<&'static str, Help> {
    let mut h = HashMap::new();
    doc!(h, "hidden"
        , "Search hidden files and directories"
        , "Include hidden directories and files in the search results (default: hidden files \
           and directories are skipped).");
    doc!(h, "no-ignore"
        , "Do not respect .(git)ignore files"
        , "Show search results from files and directories that would otherwise be ignored by \
            '.*ignore' files.");
    doc!(h, "case-sensitive"
        , "Case-sensitive search (default: smart case)"
        , "Perform a case-sensitive search. By default, fd uses case-insensitive searches, \
           unless the pattern contains an uppercase character (smart case).");
    doc!(h, "ignore-case"
        , "Case-insensitive search (default: smart case)"
        , "Perform a case-insensitive search. By default, fd uses case-insensitive searches, \
           unless the pattern contains an uppercase character (smart case).");
    doc!(h, "absolute-path"
        , "Show absolute instead of relative paths"
        , "Shows the full path starting from the root as opposed to relative paths.");
    doc!(h, "follow"
        , "Follow symbolic links"
        , "By default, fd does not descend into symlinked directories. Using this flag, symbolic \
           links are also traversed.");
    doc!(h, "full-path"
        , "Search full path (default: file-/dirname only)"
        , "By default, the search pattern is only matched against the filename (or directory \
           name). Using this flag, the pattern is matched against the full path.");
    doc!(h, "null_separator"
        , "Separate results by the null character"
        , "Separate search results by the null character (instead of newlines). Useful for \
           piping results to 'xargs'.");
    doc!(h, "depth"
        , "Set maximum search depth (default: none)"
        , "Limit the directory traversal to a given depth. By default, there is no limit \
           on the search depth.");
    doc!(h, "file-type"
        , "Filter by type: f(ile), d(irectory), (sym)l(ink)"
        , "Filter the search by type:\n  \
             'f' or 'file':         regular files\n  \
             'd' or 'directory':    directories\n  \
             'l' or 'symlink':      symbolic links");
    doc!(h, "extension"
        , "Filter by file extension"
        , "(Additionally) filter search results by their file extension.");
    doc!(h, "exec"
        , "Execute a command for each search result"
        , "Execute a command for each search result.\n\
           All arguments following -exec are are taken to be arguments to the command until the \
           argument ';' is encountered.\n\
           Each occurrence of the following placeholders is substituted by a path derived from the \
           current search result before the command is executed:\n  \
             '{}':   path\n  \
             '{/}':  basename\n  \
             '{//}': parent directory\n  \
             '{.}':  path without file extension\n  \
             '{/.}': basename without file extension");
    doc!(h, "exclude"
        , "Exclude entries that match the given glob pattern."
        , "Exclude files/directories that match the given glob pattern. This overrides any \
           other ignore logic. Multiple exclude patterns can be specified.");
    doc!(h, "color"
        , "When to use colors: never, *auto*, always"
        , "Declare when to use color for the pattern match output:\n  \
             'auto':      show colors if the output goes to an interactive console (default)\n  \
             'never':     do not use colorized output\n  \
             'always':    always use colorized output");
    doc!(h, "threads"
        , "Set number of threads to use for searching & executing"
        , "Set number of threads to use for searching & executing (default: number of available \
           CPU cores)");
    doc!(h, "max-buffer-time"
        , "the time (in ms) to buffer, before streaming to the console"
        , "Amount of time in milliseconds to buffer, before streaming the search results to\
           the console.");
    doc!(h, "pattern"
        , "the search pattern, a regular expression (optional)");
    doc!(h, "path"
        , "the root directory for the filesystem search (optional)"
        , "The directory where the filesystem search is rooted (optional). \
           If omitted, search the current working directory.");
    doc!(h, "rg-alias-hidden-ignore"
        , "Alias for no-ignore and/or hidden"
        , "Alias for no-ignore ('u') and no-ignore and hidden ('uu')");

    h
}
<commit_msg>Fix typos<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::collections::HashMap;

use clap::{App, AppSettings, Arg};

struct Help {
    short: &'static str,
    long: &'static str,
}

macro_rules! doc {
    ($map:expr, $name:expr, $short:expr) => {
        doc!($map, $name, $short, $short)
    };
    ($map:expr, $name:expr, $short:expr, $long:expr) => {
        $map.insert($name, Help {
            short: $short,
            long: concat!($long, "\n ")
        });
    };
}

pub fn build_app() -> App<'static, 'static> {
    let helps = usage();
    let arg = |name| {
        Arg::with_name(name).help(helps[name].short).long_help(
            helps[name].long,
        )
    };

    App::new("fd")
        .version(crate_version!())
        .usage("fd [FLAGS/OPTIONS] [<pattern>] [<path>]")
        .setting(AppSettings::ColoredHelp)
        .setting(AppSettings::DeriveDisplayOrder)
        .arg(arg("hidden").long("hidden").short("H"))
        .arg(arg("no-ignore").long("no-ignore").short("I"))
        .arg(
            arg("rg-alias-hidden-ignore")
                .short("u")
                .multiple(true)
                .hidden(true),
        )
        .arg(
            arg("case-sensitive")
                .long("case-sensitive")
                .short("s")
                .overrides_with("ignore-case"),
        )
        .arg(
            arg("ignore-case")
                .long("ignore-case")
                .short("i")
                .overrides_with("case-sensitive"),
        )
        .arg(arg("absolute-path").long("absolute-path").short("a"))
        .arg(arg("follow").long("follow").short("L").alias("dereference"))
        .arg(arg("full-path").long("full-path").short("p"))
        .arg(arg("null_separator").long("print0").short("0"))
        .arg(arg("depth").long("max-depth").short("d").takes_value(true))
        .arg(
            arg("file-type")
                .long("type")
                .short("t")
                .takes_value(true)
                .value_name("filetype")
                .possible_values(&["f", "file", "d", "directory", "l", "symlink"])
                .hide_possible_values(true),
        )
        .arg(
            arg("extension")
                .long("extension")
                .short("e")
                .takes_value(true)
                .value_name("ext"),
        )
        .arg(
            arg("exec")
                .long("exec")
                .short("x")
                .multiple(true)
                .min_values(1)
                .allow_hyphen_values(true)
                .value_terminator(";")
                .value_name("cmd"),
        )
        .arg(
            arg("exclude")
                .long("exclude")
                .short("E")
                .takes_value(true)
                .value_name("pattern")
                .number_of_values(1)
                .multiple(true),
        )
        .arg(
            arg("color")
                .long("color")
                .short("c")
                .takes_value(true)
                .value_name("when")
                .possible_values(&["never", "auto", "always"])
                .hide_possible_values(true),
        )
        .arg(
            arg("threads")
                .long("threads")
                .short("j")
                .takes_value(true)
                .value_name("num"),
        )
        .arg(
            arg("max-buffer-time")
                .long("max-buffer-time")
                .takes_value(true)
                .hidden(true),
        )
        .arg(arg("pattern"))
        .arg(arg("path"))
}

#[cfg_attr(rustfmt, rustfmt_skip)]
fn usage() -> HashMap<&'static str, Help> {
    let mut h = HashMap::new();
    doc!(h, "hidden"
        , "Search hidden files and directories"
        , "Include hidden directories and files in the search results (default: hidden files \
           and directories are skipped).");
    doc!(h, "no-ignore"
        , "Do not respect .(git)ignore files"
        , "Show search results from files and directories that would otherwise be ignored by \
            '.*ignore' files.");
    doc!(h, "case-sensitive"
        , "Case-sensitive search (default: smart case)"
        , "Perform a case-sensitive search. By default, fd uses case-insensitive searches, \
           unless the pattern contains an uppercase character (smart case).");
    doc!(h, "ignore-case"
        , "Case-insensitive search (default: smart case)"
        , "Perform a case-insensitive search. By default, fd uses case-insensitive searches, \
           unless the pattern contains an uppercase character (smart case).");
    doc!(h, "absolute-path"
        , "Show absolute instead of relative paths"
        , "Shows the full path starting from the root as opposed to relative paths.");
    doc!(h, "follow"
        , "Follow symbolic links"
        , "By default, fd does not descend into symlinked directories. Using this flag, symbolic \
           links are also traversed.");
    doc!(h, "full-path"
        , "Search full path (default: file-/dirname only)"
        , "By default, the search pattern is only matched against the filename (or directory \
           name). Using this flag, the pattern is matched against the full path.");
    doc!(h, "null_separator"
        , "Separate results by the null character"
        , "Separate search results by the null character (instead of newlines). Useful for \
           piping results to 'xargs'.");
    doc!(h, "depth"
        , "Set maximum search depth (default: none)"
        , "Limit the directory traversal to a given depth. By default, there is no limit \
           on the search depth.");
    doc!(h, "file-type"
        , "Filter by type: f(ile), d(irectory), (sym)l(ink)"
        , "Filter the search by type:\n  \
             'f' or 'file':         regular files\n  \
             'd' or 'directory':    directories\n  \
             'l' or 'symlink':      symbolic links");
    doc!(h, "extension"
        , "Filter by file extension"
        , "(Additionally) filter search results by their file extension.");
    doc!(h, "exec"
        , "Execute a command for each search result"
        , "Execute a command for each search result.\n\
           All arguments following --exec are taken to be arguments to the command until the \
           argument ';' is encountered.\n\
           Each occurrence of the following placeholders is substituted by a path derived from the \
           current search result before the command is executed:\n  \
             '{}':   path\n  \
             '{/}':  basename\n  \
             '{//}': parent directory\n  \
             '{.}':  path without file extension\n  \
             '{/.}': basename without file extension");
    doc!(h, "exclude"
        , "Exclude entries that match the given glob pattern."
        , "Exclude files/directories that match the given glob pattern. This overrides any \
           other ignore logic. Multiple exclude patterns can be specified.");
    doc!(h, "color"
        , "When to use colors: never, *auto*, always"
        , "Declare when to use color for the pattern match output:\n  \
             'auto':      show colors if the output goes to an interactive console (default)\n  \
             'never':     do not use colorized output\n  \
             'always':    always use colorized output");
    doc!(h, "threads"
        , "Set number of threads to use for searching & executing"
        , "Set number of threads to use for searching & executing (default: number of available \
           CPU cores)");
    doc!(h, "max-buffer-time"
        , "the time (in ms) to buffer, before streaming to the console"
        , "Amount of time in milliseconds to buffer, before streaming the search results to\
           the console.");
    doc!(h, "pattern"
        , "the search pattern, a regular expression (optional)");
    doc!(h, "path"
        , "the root directory for the filesystem search (optional)"
        , "The directory where the filesystem search is rooted (optional). \
           If omitted, search the current working directory.");
    doc!(h, "rg-alias-hidden-ignore"
        , "Alias for no-ignore and/or hidden"
        , "Alias for no-ignore ('u') and no-ignore and hidden ('uu')");

    h
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::fmt;
use std::slice;
use std::vec;

#[deriving(PartialEq)]
pub struct NumericValue {
    pub representation: String,
    pub value: f64,
    pub int_value: Option<i64>,
}

#[deriving(PartialEq)]
pub struct SourceLocation {
    pub line: uint,  // First line is 1
    pub column: uint,  // First character of a line is at column 1
}

pub type Node = (ComponentValue, SourceLocation);  // TODO this is not a good name

#[deriving(PartialEq)]
pub enum ComponentValue {
    // Preserved tokens.
    Ident(String),
    AtKeyword(String),
    Hash(String),
    IDHash(String),  // Hash that is a valid ID selector.
    QuotedString(String),
    URL(String),
    Delim(char),
    Number(NumericValue),
    Percentage(NumericValue),
    Dimension(NumericValue, String),
    UnicodeRange(u32, u32),  // (start, end) of range
    WhiteSpace,
    Colon,  // :
    Semicolon,  // ;
    Comma,  // ,
    IncludeMatch, // ~=
    DashMatch, // |=
    PrefixMatch, // ^=
    SuffixMatch, // $=
    SubstringMatch, // *=
    Column, // ||
    CDO,  // <!--
    CDC,  // -->

    // Function
    Function(String, Vec<ComponentValue>),  // name, arguments

    // Simple block
    ParenthesisBlock(Vec<ComponentValue>),  // (…)
    SquareBracketBlock(Vec<ComponentValue>),  // […]
    CurlyBracketBlock(Vec<Node>),  // {…}

    // These are always invalid
    BadURL,
    BadString,
    CloseParenthesis, // )
    CloseSquareBracket, // ]
    CloseCurlyBracket, // }
}

impl ComponentValue {
    pub fn to_css(&mut self) -> String {
        let mut css = String::new();
        self.to_css_push(&mut css);
        css
    }

    pub fn to_css_push(&self, css: &mut String) {
        ::serializer::to_css_push(self, css)
    }
}

#[deriving(PartialEq)]
pub struct Declaration {
    pub location: SourceLocation,
    pub name: String,
    pub value: Vec<ComponentValue>,
    pub important: bool,
}

#[deriving(PartialEq)]
pub struct QualifiedRule {
    pub location: SourceLocation,
    pub prelude: Vec<ComponentValue>,
    pub block: Vec<Node>,
}

#[deriving(PartialEq)]
pub struct AtRule {
    pub location: SourceLocation,
    pub name: String,
    pub prelude: Vec<ComponentValue>,
    pub block: Option<Vec<Node>>,
}

#[deriving(PartialEq)]
pub enum DeclarationListItem {
    Declaration_(Declaration),
    // A better idea for a name that means "at-rule" but is not "AtRule"?
    DeclAtRule(AtRule),
}

#[deriving(PartialEq)]
pub enum Rule {
    QualifiedRule_(QualifiedRule),
    AtRule_(AtRule),
}

#[deriving(PartialEq)]
pub struct SyntaxError {
    pub location: SourceLocation,
    pub reason: ErrorReason,
}

#[deriving(PartialEq, Show)]
pub enum ErrorReason {
    ErrEmptyInput,  // Parsing a single "thing", found only whitespace.
    ErrExtraInput,  // Found more non-whitespace after parsing a single "thing".
    ErrMissingQualifiedRuleBlock,  // EOF in a qualified rule prelude, before '{'
    ErrInvalidDeclarationSyntax,
    ErrInvalidBangImportantSyntax,
    // This is meant to be extended
}

impl fmt::Show for SyntaxError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:u}:{:u} {}", self.location.line, self.location.column, self.reason)
    }
}

pub trait SkipWhitespaceIterable<'a> {
    fn skip_whitespace(self) -> SkipWhitespaceIterator<'a>;
}

impl<'a> SkipWhitespaceIterable<'a> for &'a [ComponentValue] {
    fn skip_whitespace(self) -> SkipWhitespaceIterator<'a> {
        SkipWhitespaceIterator{ iter_with_whitespace: self.iter() }
    }
}

#[deriving(Clone)]
pub struct SkipWhitespaceIterator<'a> {
    pub iter_with_whitespace: slice::Items<'a, ComponentValue>,
}

impl<'a> Iterator<&'a ComponentValue> for SkipWhitespaceIterator<'a> {
    fn next(&mut self) -> Option<&'a ComponentValue> {
        for component_value in self.iter_with_whitespace {
            if component_value != &WhiteSpace { return Some(component_value) }
        }
        None
    }
}

pub trait MoveSkipWhitespaceIterable {
    fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator;
}

impl MoveSkipWhitespaceIterable for Vec<ComponentValue> {
    fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator {
        MoveSkipWhitespaceIterator{ iter_with_whitespace: self.into_iter() }
    }
}

pub struct MoveSkipWhitespaceIterator {
    iter_with_whitespace: vec::MoveItems<ComponentValue>,
}

impl Iterator<ComponentValue> for MoveSkipWhitespaceIterator {
    fn next(&mut self) -> Option<ComponentValue> {
        for component_value in self.iter_with_whitespace {
            if component_value != WhiteSpace { return Some(component_value) }
        }
        None
    }
}
<commit_msg>add show instances, and remove useless mut<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::fmt;
use std::slice;
use std::vec;

#[deriving(PartialEq, Show)]
pub struct NumericValue {
    pub representation: String,
    pub value: f64,
    pub int_value: Option<i64>,
}

#[deriving(PartialEq, Show)]
pub struct SourceLocation {
    pub line: uint,  // First line is 1
    pub column: uint,  // First character of a line is at column 1
}

pub type Node = (ComponentValue, SourceLocation);  // TODO this is not a good name

#[deriving(PartialEq, Show)]
pub enum ComponentValue {
    // Preserved tokens.
    Ident(String),
    AtKeyword(String),
    Hash(String),
    IDHash(String),  // Hash that is a valid ID selector.
    QuotedString(String),
    URL(String),
    Delim(char),
    Number(NumericValue),
    Percentage(NumericValue),
    Dimension(NumericValue, String),
    UnicodeRange(u32, u32),  // (start, end) of range
    WhiteSpace,
    Colon,  // :
    Semicolon,  // ;
    Comma,  // ,
    IncludeMatch, // ~=
    DashMatch, // |=
    PrefixMatch, // ^=
    SuffixMatch, // $=
    SubstringMatch, // *=
    Column, // ||
    CDO,  // <!--
    CDC,  // -->

    // Function
    Function(String, Vec<ComponentValue>),  // name, arguments

    // Simple block
    ParenthesisBlock(Vec<ComponentValue>),  // (…)
    SquareBracketBlock(Vec<ComponentValue>),  // […]
    CurlyBracketBlock(Vec<Node>),  // {…}

    // These are always invalid
    BadURL,
    BadString,
    CloseParenthesis, // )
    CloseSquareBracket, // ]
    CloseCurlyBracket, // }
}

impl ComponentValue {
    pub fn to_css(&self) -> String {
        let mut css = String::new();
        self.to_css_push(&mut css);
        css
    }

    pub fn to_css_push(&self, css: &mut String) {
        ::serializer::to_css_push(self, css)
    }
}

#[deriving(PartialEq)]
pub struct Declaration {
    pub location: SourceLocation,
    pub name: String,
    pub value: Vec<ComponentValue>,
    pub important: bool,
}

#[deriving(PartialEq)]
pub struct QualifiedRule {
    pub location: SourceLocation,
    pub prelude: Vec<ComponentValue>,
    pub block: Vec<Node>,
}

#[deriving(PartialEq)]
pub struct AtRule {
    pub location: SourceLocation,
    pub name: String,
    pub prelude: Vec<ComponentValue>,
    pub block: Option<Vec<Node>>,
}

#[deriving(PartialEq)]
pub enum DeclarationListItem {
    Declaration_(Declaration),
    // A better idea for a name that means "at-rule" but is not "AtRule"?
    DeclAtRule(AtRule),
}

#[deriving(PartialEq)]
pub enum Rule {
    QualifiedRule_(QualifiedRule),
    AtRule_(AtRule),
}

#[deriving(PartialEq)]
pub struct SyntaxError {
    pub location: SourceLocation,
    pub reason: ErrorReason,
}

#[deriving(PartialEq, Show)]
pub enum ErrorReason {
    ErrEmptyInput,  // Parsing a single "thing", found only whitespace.
    ErrExtraInput,  // Found more non-whitespace after parsing a single "thing".
    ErrMissingQualifiedRuleBlock,  // EOF in a qualified rule prelude, before '{'
    ErrInvalidDeclarationSyntax,
    ErrInvalidBangImportantSyntax,
    // This is meant to be extended
}

impl fmt::Show for SyntaxError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:u}:{:u} {}", self.location.line, self.location.column, self.reason)
    }
}

pub trait SkipWhitespaceIterable<'a> {
    fn skip_whitespace(self) -> SkipWhitespaceIterator<'a>;
}

impl<'a> SkipWhitespaceIterable<'a> for &'a [ComponentValue] {
    fn skip_whitespace(self) -> SkipWhitespaceIterator<'a> {
        SkipWhitespaceIterator{ iter_with_whitespace: self.iter() }
    }
}

#[deriving(Clone)]
pub struct SkipWhitespaceIterator<'a> {
    pub iter_with_whitespace: slice::Items<'a, ComponentValue>,
}

impl<'a> Iterator<&'a ComponentValue> for SkipWhitespaceIterator<'a> {
    fn next(&mut self) -> Option<&'a ComponentValue> {
        for component_value in self.iter_with_whitespace {
            if component_value != &WhiteSpace { return Some(component_value) }
        }
        None
    }
}

pub trait MoveSkipWhitespaceIterable {
    fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator;
}

impl MoveSkipWhitespaceIterable for Vec<ComponentValue> {
    fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator {
        MoveSkipWhitespaceIterator{ iter_with_whitespace: self.into_iter() }
    }
}

pub struct MoveSkipWhitespaceIterator {
    iter_with_whitespace: vec::MoveItems<ComponentValue>,
}

impl Iterator<ComponentValue> for MoveSkipWhitespaceIterator {
    fn next(&mut self) -> Option<ComponentValue> {
        for component_value in self.iter_with_whitespace {
            if component_value != WhiteSpace { return Some(component_value) }
        }
        None
    }
}
<|endoftext|>