<commit_before>// Copyright (c) 2017 Gilad Naaman
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

/// Calculates the offset of the specified field from the start of the struct.
/// This macro supports arbitrary amount of subscripts and recursive member-accesses.
///
/// *Note*: This macro may not make much sense when used on structs that are not `#[repr(C, packed)]`
///
/// ## Examples
/// ```
/// #[macro_use]
/// extern crate memoffset;
///
/// #[repr(C, packed)]
/// struct Foo {
///     a: u32,
///     b: u64,
///     c: [u8; 5]
/// }
///
/// fn main() {
///     assert_eq!(offset_of!(Foo, a), 0);
///     assert_eq!(offset_of!(Foo, b), 4);
/// }
/// ```
#[macro_export]
#[cfg(memoffset_maybe_uninit)]
macro_rules! offset_of {
    ($parent:tt, $field:tt) => {{
        // Make sure the field actually exists. This line ensures that a
        // compile-time error is generated if $field is accessed through a
        // Deref impl.
        let $parent { $field: _, .. };

        // Create an instance of the container and calculate the offset to its field.
        // Here we're using an uninitialized instance of $parent.
        // Since we're not using its field, there's no UB caused by reading uninitialized memory.
        // There *IS*, though, UB caused by creating references to uninitialized data,
        // which is illegal since the compiler is allowed to assume that a reference
        // points to valid data.
        // However, currently we just cannot avoid UB completely.
        let val = $crate::mem::MaybeUninit::<$parent>::uninit();
        let base_ptr = val.as_ptr();
        #[allow(unused_unsafe)] // for when the macro is used in an unsafe block
        let field_ptr = unsafe { &(*base_ptr).$field as *const _ };
        let offset = (field_ptr as usize) - (base_ptr as usize);
        offset
    }};
}

#[macro_export]
#[cfg(not(memoffset_maybe_uninit))]
macro_rules! offset_of {
    ($parent:tt, $field:tt) => {{
        // Make sure the field actually exists. This line ensures that a
        // compile-time error is generated if $field is accessed through a
        // Deref impl.
        let $parent { $field: _, .. };

        // This is UB since we're dealing with dangling references.
        // We're never dereferencing it, but it's UB nonetheless.
        // See above for a better version that only works with newer Rust.
        let non_null = $crate::ptr::NonNull::<$parent>::dangling();
        #[allow(unused_unsafe)]
        let base_ptr = unsafe { non_null.as_ref() as *const _ };
        #[allow(unused_unsafe)]
        let field_ptr = unsafe { &(*base_ptr).$field as *const _ };
        let offset = (field_ptr as usize) - (base_ptr as usize);
        offset
    }};
}

#[cfg(test)]
mod tests {
    #[repr(C, packed)]
    struct Foo {
        a: u32,
        b: [u8; 4],
        c: i64,
    }

    #[test]
    fn offset_simple() {
        assert_eq!(offset_of!(Foo, a), 0);
        assert_eq!(offset_of!(Foo, b), 4);
        assert_eq!(offset_of!(Foo, c), 8);
    }

    #[test]
    fn tuple_struct() {
        #[repr(C, packed)]
        struct Tup(i32, i32);

        assert_eq!(offset_of!(Tup, 0), 0);
    }
}
<commit_msg>try to fix old macro<commit_after>// Copyright (c) 2017 Gilad Naaman
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

/// Calculates the offset of the specified field from the start of the struct.
/// This macro supports arbitrary amount of subscripts and recursive member-accesses.
///
/// *Note*: This macro may not make much sense when used on structs that are not `#[repr(C, packed)]`
///
/// ## Examples
/// ```
/// #[macro_use]
/// extern crate memoffset;
///
/// #[repr(C, packed)]
/// struct Foo {
///     a: u32,
///     b: u64,
///     c: [u8; 5]
/// }
///
/// fn main() {
///     assert_eq!(offset_of!(Foo, a), 0);
///     assert_eq!(offset_of!(Foo, b), 4);
/// }
/// ```
#[macro_export]
#[cfg(memoffset_maybe_uninit)]
macro_rules! offset_of {
    ($parent:tt, $field:tt) => {{
        // Make sure the field actually exists. This line ensures that a
        // compile-time error is generated if $field is accessed through a
        // Deref impl.
        let $parent { $field: _, .. };

        // Create an instance of the container and calculate the offset to its field.
        // Here we're using an uninitialized instance of $parent.
        // Since we're not using its field, there's no UB caused by reading uninitialized memory.
        // There *IS*, though, UB caused by creating references to uninitialized data,
        // which is illegal since the compiler is allowed to assume that a reference
        // points to valid data.
        // However, currently we just cannot avoid UB completely.
        let val = $crate::mem::MaybeUninit::<$parent>::uninit();
        let base_ptr = val.as_ptr();
        #[allow(unused_unsafe)] // for when the macro is used in an unsafe block
        let field_ptr = unsafe { &(*base_ptr).$field as *const _ };
        let offset = (field_ptr as usize) - (base_ptr as usize);
        offset
    }};
}

#[macro_export]
#[cfg(not(memoffset_maybe_uninit))]
macro_rules! offset_of {
    ($parent:tt, $field:tt) => {{
        // Make sure the field actually exists. This line ensures that a
        // compile-time error is generated if $field is accessed through a
        // Deref impl.
        let $parent { $field: _, .. };

        // This is UB since we're dealing with dangling references.
        // We're never dereferencing it, but it's UB nonetheless.
        // See above for a better version that only works with newer Rust.
        let non_null = $crate::ptr::NonNull::<$parent>::dangling();
        #[allow(unused_unsafe)]
        let base_ptr = unsafe { non_null.as_ref() as *const $parent };
        #[allow(unused_unsafe)]
        let field_ptr = unsafe { &(*base_ptr).$field as *const _ };
        let offset = (field_ptr as usize) - (base_ptr as usize);
        offset
    }};
}

#[cfg(test)]
mod tests {
    #[repr(C, packed)]
    struct Foo {
        a: u32,
        b: [u8; 4],
        c: i64,
    }

    #[test]
    fn offset_simple() {
        assert_eq!(offset_of!(Foo, a), 0);
        assert_eq!(offset_of!(Foo, b), 4);
        assert_eq!(offset_of!(Foo, c), 8);
    }

    #[test]
    fn tuple_struct() {
        #[repr(C, packed)]
        struct Tup(i32, i32);

        assert_eq!(offset_of!(Tup, 0), 0);
    }
}
<|endoftext|>


<commit_before>use std::collections;

use num::traits::ToPrimitive;
use num::rational;
use itertools::Itertools;
use bio::stats::logprobs;

use model;

pub type CV = f64;

#[allow(unused_parens)]
pub fn pmf(pmfs: &[model::expressionset::PMF]) -> model::diffexp::PMF {
    let mut pmf = collections::HashMap::new();
    let combinations = iproduct!(pmfs);

    for groups in combinations {
        let prob = groups.iter().map(|group| group.prob).fold(0.0, |s, p| s + p);
        if prob >= model::MIN_PROB {
            // mean
            let mean = {
                let summands = groups.iter().map(|group| {
                    group.prob +
                    (*group.value.numer() as f64 / *group.value.denom() as f64).ln()
                }).collect_vec();
                logprobs::sum(&summands).exp()
            };
            // standard deviation
            let std = {
                let summands = groups.iter().map(|group| {
                    group.prob +
                    (*group.value.numer() as f64 / *group.value.denom() as f64 - mean).ln().powi(2)
                }).collect_vec();
                logprobs::sum(&summands).exp().sqrt()
            };
            let cv = rational::Ratio::from_float(std / mean).unwrap();

            let mut p = pmf.entry(cv).or_insert(0.0);
            *p = logprobs::add(*p, prob);
        }
    }

    let pmf = pmf.iter().map(|(cv, p)| {
        model::pmf::Entry {
            value: cv.numer().to_f64().unwrap().log2() - cv.denom().to_f64().unwrap().log2(),
            prob: *p
        }
    }).collect_vec();

    model::diffexp::PMF::new(pmf)
}
<commit_msg>Use sample mean and STD for CV.<commit_after>use std::collections;

use num::traits::ToPrimitive;
use num::rational;
use itertools::Itertools;
use bio::stats::logprobs;

use model;

pub type CV = f64;

#[allow(unused_parens)]
pub fn pmf(pmfs: &[model::expressionset::PMF]) -> model::diffexp::PMF {
    let mut pmf = collections::HashMap::new();
    let combinations = iproduct!(pmfs);
    let n = pmfs.len() as f64;

    for groups in combinations {
        let prob = groups.iter().map(|group| group.prob).fold(0.0, |s, p| s + p);
        if prob >= model::MIN_PROB {
            // sample mean
            let mean = groups.iter().map(|group| {
                *group.value.numer() as f64 / *group.value.denom() as f64
            }).fold(0.0, |s, e| s + e) / n;
            // sample standard deviation
            let std = (groups.iter().map(|group| {
                    (*group.value.numer() as f64 / *group.value.denom() as f64 - mean).powi(2)
            }).fold(0.0, |s, e| s + e) / n).sqrt();

            let cv = rational::Ratio::from_float(std / mean).unwrap();

            let mut p = pmf.entry(cv).or_insert(0.0);
            *p = logprobs::add(*p, prob);
        }
    }

    let pmf = pmf.iter().map(|(cv, p)| {
        model::pmf::Entry {
            value: cv.numer().to_f64().unwrap().log2() - cv.denom().to_f64().unwrap().log2(),
            prob: *p
        }
    }).collect_vec();

    model::diffexp::PMF::new(pmf)
}
<|endoftext|>


<commit_before>#![crate_id = "main"]
#![crate_type = "staticlib"]
#![no_std]
#![feature(globs, asm, lang_items)]

extern crate core;

#[path = "arch/x86/"]
pub mod platform {
	pub mod vga;
	pub mod cpu;
	pub mod mmu;
	mod io;
	pub mod keyboard;
}

pub mod kernel {
	pub mod main;
	pub mod interrupts;
	mod stdio;
	mod keyboard;
}

#[lang = "stack_exhausted"] extern fn stack_exhausted() {}
#[lang = "eh_personality"] extern fn eh_personality() {}
<commit_msg>crate_id is changed to crate_name<commit_after>#![crate_name = "main"]
#![crate_type = "staticlib"]
#![no_std]
#![feature(globs, asm, lang_items)]

extern crate core;

#[path = "arch/x86/"]
pub mod platform {
	pub mod vga;
	pub mod cpu;
	pub mod mmu;
	mod io;
	pub mod keyboard;
}

pub mod kernel {
	pub mod main;
	pub mod interrupts;
	mod stdio;
	mod keyboard;
}

#[lang = "stack_exhausted"] extern fn stack_exhausted() {}
#[lang = "eh_personality"] extern fn eh_personality() {}
<|endoftext|>


<commit_before>use gettextrs::gettext;

use gstreamer as gst;
use gstreamer::prelude::*;
use gstreamer::PadExt;

use glib;
use glib::ObjectExt;

use std::error::Error;
use std::path::Path;
use std::sync::mpsc::Sender;

use super::ContextMessage;

pub struct TocSetterContext {
    pipeline: gst::Pipeline,
    muxer: Option<gst::Element>,
    position_query: gst::query::Position<gst::Query>,
}

impl TocSetterContext {
    pub fn check_requirements() -> Result<(), String> {
        // Exporting to Mastroska containers is only
        // available from gst-plugins-good 1.13.1
        let (major, minor, micro, _nano) = gst::version();
        if major >= 1 && minor >= 13 && micro >= 1 {
            gst::ElementFactory::make("matroskamux", None).map_or(
                Err(gettext(
                    "Missing `matroskamux`\ncheck your gst-plugins-good install",
                )),
                |_| Ok(()),
            )
        } else {
            Err(gettext(
                "Matroska export requires\ngst-plugins-good >= 1.14",
            ))
        }
    }

    pub fn new(
        input_path: &Path,
        output_path: &Path,
        ctx_tx: Sender<ContextMessage>,
    ) -> Result<TocSetterContext, String> {
        info!(
            "{}",
            gettext("Exporting to {}...").replacen("{}", output_path.to_str().unwrap(), 1)
        );

        let mut this = TocSetterContext {
            pipeline: gst::Pipeline::new("pipeline"),
            muxer: None,
            position_query: gst::Query::new_position(gst::Format::Time),
        };

        this.build_pipeline(input_path, output_path);
        this.register_bus_inspector(ctx_tx);

        match this.pipeline.set_state(gst::State::Paused) {
            gst::StateChangeReturn::Failure => Err("Could not set media in Paused state".into()),
            _ => Ok(this),
        }
    }

    pub fn get_muxer(&self) -> Option<&gst::Element> {
        self.muxer.as_ref()
    }

    pub fn export(&mut self) -> Result<(), String> {
        match self.pipeline.set_state(gst::State::Playing) {
            gst::StateChangeReturn::Failure => Err("Could not set media in palying state".into()),
            _ => Ok(()),
        }
    }

    pub fn get_position(&mut self) -> u64 {
        self.pipeline.query(&mut self.position_query);
        self.position_query.get_result().get_value() as u64
    }

    fn build_pipeline(&mut self, input_path: &Path, output_path: &Path) {
        // Input
        let filesrc = gst::ElementFactory::make("filesrc", None).unwrap();
        filesrc
            .set_property("location", &gst::Value::from(input_path.to_str().unwrap()))
            .unwrap();

        let parsebin = gst::ElementFactory::make("parsebin", None).unwrap();

        {
            self.pipeline.add_many(&[&filesrc, &parsebin]).unwrap();
            filesrc.link(&parsebin).unwrap();
            parsebin.sync_state_with_parent().unwrap();
        }

        // Muxer and output sink
        let muxer = gst::ElementFactory::make("matroskamux", None).unwrap();
        muxer
            .set_property("writing-app", &gst::Value::from("media-toc"))
            .unwrap();

        let filesink = gst::ElementFactory::make("filesink", "filesink").unwrap();
        filesink
            .set_property("location", &gst::Value::from(output_path.to_str().unwrap()))
            .unwrap();

        {
            self.pipeline.add_many(&[&muxer, &filesink]).unwrap();
            muxer.link(&filesink).unwrap();
            filesink.sync_state_with_parent().unwrap();
        }

        self.muxer = Some(muxer.clone());

        let pipeline_cb = self.pipeline.clone();
        parsebin.connect_pad_added(move |_element, pad| {
            let queue = gst::ElementFactory::make("queue", None).unwrap();
            pipeline_cb.add(&queue).unwrap();
            let queue_sink_pad = queue.get_static_pad("sink").unwrap();
            assert_eq!(pad.link(&queue_sink_pad), gst::PadLinkReturn::Ok);

            let queue_src_pad = queue.get_static_pad("src").unwrap();
            let muxer_sink_pad = muxer.get_compatible_pad(&queue_src_pad, None).unwrap();
            assert_eq!(queue_src_pad.link(&muxer_sink_pad), gst::PadLinkReturn::Ok);

            // Listen to incoming events and drop Upstream TOCs
            muxer_sink_pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, |_pad, probe_info| {
                if let Some(ref data) = probe_info.data {
                    if let gst::PadProbeData::Event(ref event) = *data {
                        if let gst::EventView::Toc(ref _toc) = event.view() {
                            return gst::PadProbeReturn::Drop;
                        }
                    }
                }
                gst::PadProbeReturn::Ok
            });

            for element in &[&queue, &muxer] {
                element.sync_state_with_parent().unwrap();
            }
        });
    }

    // Uses ctx_tx to notify the UI controllers about the inspection process
    fn register_bus_inspector(&self, ctx_tx: Sender<ContextMessage>) {
        let mut init_done = false;
        self.pipeline.get_bus().unwrap().add_watch(move |_, msg| {
            match msg.view() {
                gst::MessageView::Eos(..) => {
                    ctx_tx.send(ContextMessage::Eos).unwrap();
                    return glib::Continue(false);
                }
                gst::MessageView::Error(err) => {
                    ctx_tx
                        .send(ContextMessage::FailedToExport(
                            err.get_error().description().to_owned(),
                        ))
                        .unwrap();
                    return glib::Continue(false);
                }
                gst::MessageView::AsyncDone(_) => {
                    if !init_done {
                        init_done = true;
                        ctx_tx.send(ContextMessage::InitDone).unwrap();
                    } else {
                        ctx_tx.send(ContextMessage::AsyncDone).unwrap();
                    }
                }
                _ => (),
            }

            glib::Continue(true)
        });
    }
}
<commit_msg>Export: fix GStreamer version check<commit_after>use gettextrs::gettext;

use gstreamer as gst;
use gstreamer::prelude::*;
use gstreamer::PadExt;

use glib;
use glib::ObjectExt;

use std::error::Error;
use std::path::Path;
use std::sync::mpsc::Sender;

use super::ContextMessage;

pub struct TocSetterContext {
    pipeline: gst::Pipeline,
    muxer: Option<gst::Element>,
    position_query: gst::query::Position<gst::Query>,
}

impl TocSetterContext {
    pub fn check_requirements() -> Result<(), String> {
        // Exporting to Mastroska containers is only
        // available from gst-plugins-good 1.13.1
        let (major, minor, _micro, _nano) = gst::version();
        if major >= 1 && minor >= 14 {
            gst::ElementFactory::make("matroskamux", None).map_or(
                Err(gettext(
                    "Missing `matroskamux`\ncheck your gst-plugins-good install",
                )),
                |_| Ok(()),
            )
        } else {
            Err(gettext(
                "Matroska export requires\ngst-plugins-good >= 1.14",
            ))
        }
    }

    pub fn new(
        input_path: &Path,
        output_path: &Path,
        ctx_tx: Sender<ContextMessage>,
    ) -> Result<TocSetterContext, String> {
        info!(
            "{}",
            gettext("Exporting to {}...").replacen("{}", output_path.to_str().unwrap(), 1)
        );

        let mut this = TocSetterContext {
            pipeline: gst::Pipeline::new("pipeline"),
            muxer: None,
            position_query: gst::Query::new_position(gst::Format::Time),
        };

        this.build_pipeline(input_path, output_path);
        this.register_bus_inspector(ctx_tx);

        match this.pipeline.set_state(gst::State::Paused) {
            gst::StateChangeReturn::Failure => Err("Could not set media in Paused state".into()),
            _ => Ok(this),
        }
    }

    pub fn get_muxer(&self) -> Option<&gst::Element> {
        self.muxer.as_ref()
    }

    pub fn export(&mut self) -> Result<(), String> {
        match self.pipeline.set_state(gst::State::Playing) {
            gst::StateChangeReturn::Failure => Err("Could not set media in palying state".into()),
            _ => Ok(()),
        }
    }

    pub fn get_position(&mut self) -> u64 {
        self.pipeline.query(&mut self.position_query);
        self.position_query.get_result().get_value() as u64
    }

    fn build_pipeline(&mut self, input_path: &Path, output_path: &Path) {
        // Input
        let filesrc = gst::ElementFactory::make("filesrc", None).unwrap();
        filesrc
            .set_property("location", &gst::Value::from(input_path.to_str().unwrap()))
            .unwrap();

        let parsebin = gst::ElementFactory::make("parsebin", None).unwrap();

        {
            self.pipeline.add_many(&[&filesrc, &parsebin]).unwrap();
            filesrc.link(&parsebin).unwrap();
            parsebin.sync_state_with_parent().unwrap();
        }

        // Muxer and output sink
        let muxer = gst::ElementFactory::make("matroskamux", None).unwrap();
        muxer
            .set_property("writing-app", &gst::Value::from("media-toc"))
            .unwrap();

        let filesink = gst::ElementFactory::make("filesink", "filesink").unwrap();
        filesink
            .set_property("location", &gst::Value::from(output_path.to_str().unwrap()))
            .unwrap();

        {
            self.pipeline.add_many(&[&muxer, &filesink]).unwrap();
            muxer.link(&filesink).unwrap();
            filesink.sync_state_with_parent().unwrap();
        }

        self.muxer = Some(muxer.clone());

        let pipeline_cb = self.pipeline.clone();
        parsebin.connect_pad_added(move |_element, pad| {
            let queue = gst::ElementFactory::make("queue", None).unwrap();
            pipeline_cb.add(&queue).unwrap();
            let queue_sink_pad = queue.get_static_pad("sink").unwrap();
            assert_eq!(pad.link(&queue_sink_pad), gst::PadLinkReturn::Ok);

            let queue_src_pad = queue.get_static_pad("src").unwrap();
            let muxer_sink_pad = muxer.get_compatible_pad(&queue_src_pad, None).unwrap();
            assert_eq!(queue_src_pad.link(&muxer_sink_pad), gst::PadLinkReturn::Ok);

            // Listen to incoming events and drop Upstream TOCs
            muxer_sink_pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, |_pad, probe_info| {
                if let Some(ref data) = probe_info.data {
                    if let gst::PadProbeData::Event(ref event) = *data {
                        if let gst::EventView::Toc(ref _toc) = event.view() {
                            return gst::PadProbeReturn::Drop;
                        }
                    }
                }
                gst::PadProbeReturn::Ok
            });

            for element in &[&queue, &muxer] {
                element.sync_state_with_parent().unwrap();
            }
        });
    }

    // Uses ctx_tx to notify the UI controllers about the inspection process
    fn register_bus_inspector(&self, ctx_tx: Sender<ContextMessage>) {
        let mut init_done = false;
        self.pipeline.get_bus().unwrap().add_watch(move |_, msg| {
            match msg.view() {
                gst::MessageView::Eos(..) => {
                    ctx_tx.send(ContextMessage::Eos).unwrap();
                    return glib::Continue(false);
                }
                gst::MessageView::Error(err) => {
                    ctx_tx
                        .send(ContextMessage::FailedToExport(
                            err.get_error().description().to_owned(),
                        ))
                        .unwrap();
                    return glib::Continue(false);
                }
                gst::MessageView::AsyncDone(_) => {
                    if !init_done {
                        init_done = true;
                        ctx_tx.send(ContextMessage::InitDone).unwrap();
                    } else {
                        ctx_tx.send(ContextMessage::AsyncDone).unwrap();
                    }
                }
                _ => (),
            }

            glib::Continue(true)
        });
    }
}
<|endoftext|>


<commit_before>use libc;
use std::thread;
use std::rc::Rc;
use std::sync::{Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
use std::collections::BTreeSet;
use parking_lot::{Mutex, MutexGuard, RwLock};
use concurrent_hashmap::ConcHashMap;
use ram::schema::Schemas;
use ram::segs::{Segment, SEGMENT_SIZE};
use ram::cell::{Cell, ReadError, WriteError};
use server::ServerMeta;

pub struct Chunk {
    pub id: usize,
    pub addr: usize,
    pub index: ConcHashMap<u64, Mutex<usize>>,
    pub segs: Vec<Segment>,
    pub seg_round: AtomicUsize,
    pub meta: Rc<ServerMeta>,
    pub backup_storage: Option<String>,
}

pub struct Chunks {
    pub list: Vec<Chunk>,
}

impl Chunk {
    fn new (id: usize, size: usize, meta: Rc<ServerMeta>, back_storage: Option<String>) -> Chunk {
        let mem_ptr = unsafe {libc::malloc(size)} as usize;
        let seg_count = size / SEGMENT_SIZE;
        let mut segments = Vec::<Segment>::new();
        for seg_idx in 0..seg_count {
            let seg_addr = mem_ptr + seg_idx * SEGMENT_SIZE;
            segments.push(Segment {
                addr: seg_addr,
                id: seg_idx,
                bound: seg_addr + SEGMENT_SIZE,
                last: AtomicUsize::new(seg_addr),
                lock: RwLock::new(()),
                tombstones: Mutex::new(BTreeSet::new()),
            });
        }
        debug!("creating chunk at {}, segments {}", mem_ptr, seg_count + 1);
        Chunk {
            id: id,
            addr: mem_ptr,
            index: ConcHashMap::<u64, Mutex<usize>>::new(),
            meta: meta,
            segs: segments,
            seg_round: AtomicUsize::new(0),
            backup_storage: back_storage
        }
    }
    pub fn try_acquire(&self, size: usize) -> Option<usize> {
        let mut retried = 0;
        loop {
            let n = self.seg_round.load(Ordering::Relaxed);
            let seg_id = n % self.segs.len();
            let seg_acquire = self.segs[seg_id].try_acquire(size);
            match seg_acquire {
                None => {
                    if retried > self.segs.len() * 2 {return None;}
                    self.seg_round.fetch_add(1, Ordering::Relaxed);
                    retried += 1;
                },
                _ => {return seg_acquire;}
            }
        }
    }
    fn locate_segment(&self, location: usize) -> &Segment {
        let offset = location - self.addr;
        let seg_id = offset / SEGMENT_SIZE;
        return &self.segs[seg_id];
    }
    fn location_of(&self, hash: u64) -> Option<MutexGuard<usize>> {
        match self.index.find(&hash) {
            Some(index) => {
                let index = index.get();
                let index_lock = index.lock();
                if *index_lock == 0 {
                    return None
                }
                return Some(index_lock);
            },
            None => None
        }
    }
    fn put_tombstone(&self, location: usize) {
        let seg = self.locate_segment(location);
        seg.put_cell_tombstone(location);
    }
    fn read_cell(&self, hash: u64) -> Result<Cell, ReadError> {
        match self.location_of(hash) {
            Some(loc) => {
                Cell::from_chunk_raw(*loc, self)
            },
            None => Err(ReadError::CellDoesNotExisted)
        }
    }
    fn write_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if self.location_of(hash).is_some() {
            return Err(WriteError::CellAlreadyExisted);
        } else {
            let written = cell.write_to_chunk(self);
            let need_rollback = AtomicBool::new(false);
            if let Ok(loc) = written {
                self.index.upsert(
                    hash,
                    Mutex::new(loc),
                    &|m| {
                        let mut inserted_loc = m.lock();
                        if *inserted_loc == 0 {
                            *inserted_loc = loc
                        } else {
                            need_rollback.store(true, Ordering::Relaxed);
                        }
                    }
                );
                if need_rollback.load(Ordering::Relaxed) {
                    self.put_tombstone(loc);
                    return Err(WriteError::CellAlreadyExisted)
                }
            }
            return written
        }
    }
    fn update_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if let Some(mut cell_location) = self.location_of(hash) {
            let written = cell.write_to_chunk(self);
            if let Ok(new_location) = written {
                *cell_location = new_location;
            }
            return written;
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn update_cell_by<U>(&self, hash: u64, update: U) -> Result<usize, WriteError>
        where U: Fn(Cell) -> Option<Cell> {
        if let Some(mut cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    let mut new_cell = update(cell);
                    if let Some(mut new_cell) = new_cell {
                        let written = new_cell.write_to_chunk(self);
                        if let Ok(new_location) = written {
                            *cell_location = new_location;
                        }
                        return written;
                    } else {
                        return Err(WriteError::UserCanceledUpdate);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn dispose (&mut self) {
        debug!("disposing chunk at {}", self.addr);
        unsafe {
            libc::free(self.addr as *mut libc::c_void)
        }
    }
}

impl Drop for Chunk {
    fn drop(&mut self) {
        self.dispose();
    }
}

impl Chunks {
    pub fn new (count: usize, size: usize, meta: Rc<ServerMeta>, backup_storage: Option<String>) -> Chunks {
        let chunk_size = size / count;
        let mut chunks = Vec::new();
        debug!("Creating {} chunks, total {} bytes", count, size);
        for i in 0..count {
            let backup_storage = match backup_storage {
                Some(ref dir) => Some(format!("{}/data-{}.bak", dir, i)),
                None => None
            };
            chunks.push(Chunk::new(i, chunk_size, meta.clone(), backup_storage));
        }
        Chunks {
            list: chunks
        }
    }

    pub fn new_dummy(count: usize, size: usize) -> Chunks {
        Chunks::new(count, size, Rc::<ServerMeta>::new(ServerMeta {
            schemas: Schemas::new(None)
        }), None)
    }
}<commit_msg>remove cell and put tombstone<commit_after>use libc;
use std::thread;
use std::rc::Rc;
use std::sync::{Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
use std::collections::BTreeSet;
use parking_lot::{Mutex, MutexGuard, RwLock};
use concurrent_hashmap::ConcHashMap;
use ram::schema::Schemas;
use ram::segs::{Segment, SEGMENT_SIZE};
use ram::cell::{Cell, ReadError, WriteError};
use server::ServerMeta;

pub struct Chunk {
    pub id: usize,
    pub addr: usize,
    pub index: ConcHashMap<u64, Mutex<usize>>,
    pub segs: Vec<Segment>,
    pub seg_round: AtomicUsize,
    pub meta: Rc<ServerMeta>,
    pub backup_storage: Option<String>,
}

pub struct Chunks {
    pub list: Vec<Chunk>,
}

impl Chunk {
    fn new (id: usize, size: usize, meta: Rc<ServerMeta>, back_storage: Option<String>) -> Chunk {
        let mem_ptr = unsafe {libc::malloc(size)} as usize;
        let seg_count = size / SEGMENT_SIZE;
        let mut segments = Vec::<Segment>::new();
        for seg_idx in 0..seg_count {
            let seg_addr = mem_ptr + seg_idx * SEGMENT_SIZE;
            segments.push(Segment {
                addr: seg_addr,
                id: seg_idx,
                bound: seg_addr + SEGMENT_SIZE,
                last: AtomicUsize::new(seg_addr),
                lock: RwLock::new(()),
                tombstones: Mutex::new(BTreeSet::new()),
            });
        }
        debug!("creating chunk at {}, segments {}", mem_ptr, seg_count + 1);
        Chunk {
            id: id,
            addr: mem_ptr,
            index: ConcHashMap::<u64, Mutex<usize>>::new(),
            meta: meta,
            segs: segments,
            seg_round: AtomicUsize::new(0),
            backup_storage: back_storage
        }
    }
    pub fn try_acquire(&self, size: usize) -> Option<usize> {
        let mut retried = 0;
        loop {
            let n = self.seg_round.load(Ordering::Relaxed);
            let seg_id = n % self.segs.len();
            let seg_acquire = self.segs[seg_id].try_acquire(size);
            match seg_acquire {
                None => {
                    if retried > self.segs.len() * 2 {return None;}
                    self.seg_round.fetch_add(1, Ordering::Relaxed);
                    retried += 1;
                },
                _ => {return seg_acquire;}
            }
        }
    }
    fn locate_segment(&self, location: usize) -> &Segment {
        let offset = location - self.addr;
        let seg_id = offset / SEGMENT_SIZE;
        return &self.segs[seg_id];
    }
    fn location_of(&self, hash: u64) -> Option<MutexGuard<usize>> {
        match self.index.find(&hash) {
            Some(index) => {
                let index = index.get();
                let index_lock = index.lock();
                if *index_lock == 0 {
                    return None
                }
                return Some(index_lock);
            },
            None => None
        }
    }
    fn put_tombstone(&self, location: usize) {
        let seg = self.locate_segment(location);
        seg.put_cell_tombstone(location);
    }
    fn read_cell(&self, hash: u64) -> Result<Cell, ReadError> {
        match self.location_of(hash) {
            Some(loc) => {
                Cell::from_chunk_raw(*loc, self)
            },
            None => Err(ReadError::CellDoesNotExisted)
        }
    }
    fn write_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if self.location_of(hash).is_some() {
            return Err(WriteError::CellAlreadyExisted);
        } else {
            let written = cell.write_to_chunk(self);
            let need_rollback = AtomicBool::new(false);
            if let Ok(loc) = written {
                self.index.upsert(
                    hash,
                    Mutex::new(loc),
                    &|m| {
                        let mut inserted_loc = m.lock();
                        if *inserted_loc == 0 {
                            *inserted_loc = loc
                        } else {
                            need_rollback.store(true, Ordering::Relaxed);
                        }
                    }
                );
                if need_rollback.load(Ordering::Relaxed) {
                    self.put_tombstone(loc);
                    return Err(WriteError::CellAlreadyExisted)
                }
            }
            return written
        }
    }
    fn update_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if let Some(mut cell_location) = self.location_of(hash) {
            let written = cell.write_to_chunk(self);
            if let Ok(new_location) = written {
                let old_location = *cell_location;
                *cell_location = new_location;
                self.put_tombstone(old_location);
            }
            return written;
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn update_cell_by<U>(&self, hash: u64, update: U) -> Result<usize, WriteError>
        where U: Fn(Cell) -> Option<Cell> {
        if let Some(mut cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    let mut new_cell = update(cell);
                    if let Some(mut new_cell) = new_cell {
                        let written = new_cell.write_to_chunk(self);
                        if let Ok(new_location) = written {
                            let old_location = *cell_location;
                            *cell_location = new_location;
                            self.put_tombstone(old_location);
                        }
                        return written;
                    } else {
                        return Err(WriteError::UserCanceledUpdate);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn remove_cell(&self, hash: u64) -> Result<(), WriteError> {
        if let Some(cell_location) = self.location_of(hash) {
            self.index.remove(&hash);
            self.put_tombstone(*cell_location);
            return Ok(());
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn dispose (&mut self) {
        debug!("disposing chunk at {}", self.addr);
        unsafe {
            libc::free(self.addr as *mut libc::c_void)
        }
    }
}

impl Drop for Chunk {
    fn drop(&mut self) {
        self.dispose();
    }
}

impl Chunks {
    pub fn new (count: usize, size: usize, meta: Rc<ServerMeta>, backup_storage: Option<String>) -> Chunks {
        let chunk_size = size / count;
        let mut chunks = Vec::new();
        debug!("Creating {} chunks, total {} bytes", count, size);
        for i in 0..count {
            let backup_storage = match backup_storage {
                Some(ref dir) => Some(format!("{}/data-{}.bak", dir, i)),
                None => None
            };
            chunks.push(Chunk::new(i, chunk_size, meta.clone(), backup_storage));
        }
        Chunks {
            list: chunks
        }
    }

    pub fn new_dummy(count: usize, size: usize) -> Chunks {
        Chunks::new(count, size, Rc::<ServerMeta>::new(ServerMeta {
            schemas: Schemas::new(None)
        }), None)
    }
}<|endoftext|>


<commit_before>#[cfg(test)]
mod spec_tests;

const ADDRESSABLE_MEMORY: usize = 65536;

pub struct Memory {
  addr:[u8; ADDRESSABLE_MEMORY]
}

impl Memory {
  pub fn new() -> Memory {
    Memory {
      addr: [0; ADDRESSABLE_MEMORY]
    }
  }

  pub fn store_many(&mut self, addr: u16, data: &[u8]) {
    for i in 0..data.len() - 1 {
      self.store(addr + i as u16, data[i]);
    }
  }

  pub fn get_bytes(&self) -> &[u8] {
    &self.addr
  }

  pub fn store(&mut self, addr: u16, data: u8) {
    let addr = addr as usize;
    self.addr[addr] = data;
  }

  pub fn store16(&mut self, addr: u16, data: u16) {
    let lowb = (data & 0xff) as u8;
    let highb = ((data >> 8) & 0xff) as u8;
    self.store(addr, lowb);
    self.store(addr + 1, highb);
  }

  pub fn load(&mut self, addr: u16) -> u8 {
    let addr = addr as usize;
    self.addr[addr]
  }

  pub fn load16(&mut self, addr: u16) -> u16 {
    let addr = addr as usize;
    self.addr[addr] as u16 | (self.addr[addr + 1] as u16) << 8
  }

  pub fn load_zp_indexed(&mut self, addr: u8, index: u8) -> u8 {
    let wrapped_addr = (addr as u16 + index as u16) as u8;
    self.load(wrapped_addr as u16)
 }

  pub fn load16_zp_indexed(&mut self, addr: u8, index: u8) -> u16 {
    let wrapped_addr = (addr as u16 + index as u16) as u8;
    self.load16(wrapped_addr as u16)
 }

  // probably a premature optimization, but we implement inc and dec here so
  // that we can alter the values in place.
  pub fn inc(&mut self, addr: u16) -> u8 {
    let addr = addr as usize;
    self.addr[addr] = (self.addr[addr] as u16 + 1) as u8;
    self.addr[addr]
  }

  pub fn dec(&mut self, addr: u16) -> u8 {
    let addr = addr as usize;
    self.addr[addr] = (self.addr[addr] as i16 - 1) as u8;
    self.addr[addr]
  }
}

<commit_msg>Fix another off-by-one error<commit_after>#[cfg(test)]
mod spec_tests;

const ADDRESSABLE_MEMORY: usize = 65536;

pub struct Memory {
  addr:[u8; ADDRESSABLE_MEMORY]
}

impl Memory {
  pub fn new() -> Memory {
    Memory {
      addr: [0; ADDRESSABLE_MEMORY]
    }
  }

  pub fn store_many(&mut self, addr: u16, data: &[u8]) {
    for i in 0..data.len() {
      self.store(addr + i as u16, data[i]);
    }
  }

  pub fn get_bytes(&self) -> &[u8] {
    &self.addr
  }

  pub fn store(&mut self, addr: u16, data: u8) {
    let addr = addr as usize;
    self.addr[addr] = data;
  }

  pub fn store16(&mut self, addr: u16, data: u16) {
    let lowb = (data & 0xff) as u8;
    let highb = ((data >> 8) & 0xff) as u8;
    self.store(addr, lowb);
    self.store(addr + 1, highb);
  }

  pub fn load(&mut self, addr: u16) -> u8 {
    let addr = addr as usize;
    self.addr[addr]
  }

  pub fn load16(&mut self, addr: u16) -> u16 {
    let addr = addr as usize;
    self.addr[addr] as u16 | (self.addr[addr + 1] as u16) << 8
  }

  pub fn load_zp_indexed(&mut self, addr: u8, index: u8) -> u8 {
    let wrapped_addr = (addr as u16 + index as u16) as u8;
    self.load(wrapped_addr as u16)
 }

  pub fn load16_zp_indexed(&mut self, addr: u8, index: u8) -> u16 {
    let wrapped_addr = (addr as u16 + index as u16) as u8;
    self.load16(wrapped_addr as u16)
 }

  // probably a premature optimization, but we implement inc and dec here so
  // that we can alter the values in place.
  pub fn inc(&mut self, addr: u16) -> u8 {
    let addr = addr as usize;
    self.addr[addr] = (self.addr[addr] as u16 + 1) as u8;
    self.addr[addr]
  }

  pub fn dec(&mut self, addr: u16) -> u8 {
    let addr = addr as usize;
    self.addr[addr] = (self.addr[addr] as i16 - 1) as u8;
    self.addr[addr]
  }
}

<|endoftext|>


<commit_before>//! Reminisce is a lightweight library intended to be used for detecting and
//! reading from joysticks.

#![feature(core, std_misc, libc, os)]
extern crate libc;

#[cfg(target_os = "linux")]
mod linux;

#[cfg(target_os = "linux")]
pub use linux::*;

#[cfg(target_os = "windows")]
mod windows;

#[cfg(target_os = "windows")]
pub use windows::*;

use std::ops::Deref;

#[derive(Copy, Debug)]
/// An event from a joystick
pub enum Event {
	/// Fires when a button is pressed with the button's index
	ButtonPressed(u8),
	/// Fires when a button is released with the button's index
	ButtonReleased(u8),
	/// Fires when a joystick / axis is moved with the axis index and its value,
	/// which is between MIN_JOYSTICK_VALUE and MAX_JOYSTICK_VALUE
	JoystickMoved(u8, i16)
}
/// Convert a raw event into a Reminisce event
pub trait IntoEvent {
	/// Convert this into a Reminisce event
	fn into_event(self) -> Event;
}

#[derive(Debug)]
/// A joystick that tracks its state
pub struct SmartJoystick {
	js: Joystick,
	/// The axes on the joystick, normalised to floats between -1.0 and 1.0
	///
	/// Typically the first two of these are the primary analog stick's x and y
	/// co-ordinates
	pub axes: Vec<f32>,
	/// The buttons on the joystick as booleans indicating if they are pressed
	///
	/// The order of these usually indicates the button priority with the first
	/// two being accept and back buttons
	pub buttons: Vec<bool>
}
impl SmartJoystick {
	/// Create a new joystick from its index
	pub fn new(index: u8) -> Result<SmartJoystick, &'static str> {
		let js = try!(Joystick::new(index));
		Ok(SmartJoystick {
			axes: vec![0.0f32; js.get_num_axes() as usize],
			buttons: vec![false; js.get_num_buttons() as usize],
			js: js
		})
	}
	/// Poll this joystick for events
	pub fn poll(&mut self) -> Option<Event> {
		let event = self.js.poll();
		match event {
			Some(Event::JoystickMoved(i, v)) => self.axes[i as usize] = v as f32 / MAX_JOYSTICK_VALUE as f32,
			Some(Event::ButtonPressed(i)) => self.buttons[i as usize] = true,
			Some(Event::ButtonReleased(i)) => self.buttons[i as usize] = false,
			_ => ()
		}
		event
	}
	/// Update the joystick's state
	pub fn update(&mut self) {
		while let Some(_) = self.poll() {}
	}
}
impl Deref for SmartJoystick {
	type Target = Joystick;
	fn deref(&self) -> &Joystick {
		&self.js
	}
}
<commit_msg>Improve API of `SmartJoystick`<commit_after>//! Reminisce is a lightweight library intended to be used for detecting and
//! reading from joysticks.

#![feature(core, std_misc, libc, os)]
extern crate libc;

#[cfg(target_os = "linux")]
mod linux;

#[cfg(target_os = "linux")]
pub use linux::*;

#[cfg(target_os = "windows")]
mod windows;

#[cfg(target_os = "windows")]
pub use windows::*;

use std::ops::Deref;

#[derive(Copy, Debug)]
/// An event from a joystick
pub enum Event {
	/// Fires when a button is pressed with the button's index
	ButtonPressed(u8),
	/// Fires when a button is released with the button's index
	ButtonReleased(u8),
	/// Fires when a joystick / axis is moved with the axis index and its value,
	/// which is between MIN_JOYSTICK_VALUE and MAX_JOYSTICK_VALUE
	JoystickMoved(u8, i16)
}
/// Convert a raw event into a Reminisce event
pub trait IntoEvent {
	/// Convert this into a Reminisce event
	fn into_event(self) -> Event;
}

#[derive(Debug)]
/// A joystick that tracks its state
pub struct SmartJoystick {
	js: Joystick,
	axes: Vec<i16>,
	buttons: Vec<bool>
}
impl SmartJoystick {
	/// Create a new joystick from its index
	pub fn new(index: u8) -> Result<SmartJoystick, &'static str> {
		let js = try!(Joystick::new(index));
		Ok(SmartJoystick {
			axes: vec![0; js.get_num_axes() as usize],
			buttons: vec![false; js.get_num_buttons() as usize],
			js: js
		})
	}
	/// Poll this joystick for events
	pub fn poll(&mut self) -> Option<Event> {
		let event = self.js.poll();
		match event {
			Some(Event::JoystickMoved(i, v)) => self.axes[i as usize] = v,
			Some(Event::ButtonPressed(i)) => self.buttons[i as usize] = true,
			Some(Event::ButtonReleased(i)) => self.buttons[i as usize] = false,
			_ => ()
		}
		event
	}
	/// Get the value of a specific axis from its index
	///
	/// Typically the first two of these are the primary analog stick's x and y
	/// co-ordinates
	pub fn get_axis(&self, index: usize) -> i16 {
		self.axes[index]
	}
	/// Get the value of a specific axis normalised to between -1.0 and 1.0
	pub fn get_normalised_axis(&self, index: usize) -> f32 {
		self.axes[index] as f32 / MAX_JOYSTICK_VALUE as f32
	}
	/// Get the value (if it is pressed or not) of a specific button
	///
	/// The first two buttons are usually the accept and back buttons
	pub fn get_button(&self, index: usize) -> bool {
		self.buttons[index]
	}
	/// Update the joystick's state
	pub fn update(&mut self) {
		while let Some(_) = self.poll() {}
	}
}
impl Deref for SmartJoystick {
	type Target = Joystick;
	fn deref(&self) -> &Joystick {
		&self.js
	}
}
<|endoftext|>


<commit_before>use std::cell::Cell;
use std::sync::atomic::{AtomicUsize, Ordering};
use super::internal::*;
use super::*;
use super::len::*;

// The consumer for find_first/find_last has fake indexes representing the lower
// and upper bounds of the "range" of data it consumes. This range does not
// correspond to indexes from the consumed iterator but rather indicate the
// consumer's position relative to other consumers. The purpose is to allow a
// consumer to know it should stop consuming items when another consumer finds a
// better match.

// An indexed consumer could specialize to use the real indexes instead, but we
// don't implement that for now. The only downside of the current approach is
// that in some cases, iterators very close to each other will have the same
// range and therefore not be able to stop processing if one of them finds a
// better match than the others.

#[derive(Copy, Clone)]
enum MatchPosition {
    Leftmost,
    Rightmost,
}

pub fn find_first<PAR_ITER, FIND_OP>(pi: PAR_ITER, find_op: FIND_OP) -> Option<PAR_ITER::Item>
    where PAR_ITER: ParallelIterator,
          FIND_OP: Fn(&PAR_ITER::Item) -> bool + Sync
{
    let best_found = AtomicUsize::new(usize::max_value());
    let consumer = FindConsumer::new(&find_op, MatchPosition::Leftmost, &best_found);
    pi.drive_unindexed(consumer)
}

pub fn find_last<PAR_ITER, FIND_OP>(pi: PAR_ITER, find_op: FIND_OP) -> Option<PAR_ITER::Item>
    where PAR_ITER: ParallelIterator,
          FIND_OP: Fn(&PAR_ITER::Item) -> bool + Sync
{
    let best_found = AtomicUsize::new(0);
    let consumer = FindConsumer::new(&find_op, MatchPosition::Rightmost, &best_found);
    pi.drive_unindexed(consumer)
}

struct FindConsumer<'f, FIND_OP: 'f> {
    find_op: &'f FIND_OP,
    lower_bound: Cell<usize>,
    upper_bound: usize,
    match_position: MatchPosition,
    best_found: &'f AtomicUsize,
}

impl<'f, FIND_OP> FindConsumer<'f, FIND_OP> {
    fn new(find_op: &'f FIND_OP,
           match_position: MatchPosition,
           best_found: &'f AtomicUsize) -> Self {
        FindConsumer {
            find_op: find_op,
            lower_bound: Cell::new(0),
            upper_bound: usize::max_value(),
            match_position: match_position,
            best_found: best_found,
        }
    }
}

impl<'f, ITEM, FIND_OP> Consumer<ITEM> for FindConsumer<'f, FIND_OP>
    where ITEM: Send,
          FIND_OP: Fn(&ITEM) -> bool + Sync
{
    type Folder = FindFolder<'f, ITEM, FIND_OP>;
    type Reducer = FindReducer;
    type Result = Option<ITEM>;

    fn cost(&mut self, cost: f64) -> f64 {
        cost * FUNC_ADJUSTMENT
    }

    fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) {
        let dir = self.match_position;
        (self.split_off(),
         self,
         FindReducer { match_position: dir })
    }

    fn into_folder(self) -> Self::Folder {
        FindFolder {
            find_op: self.find_op,
            boundary: match self.match_position {
                MatchPosition::Leftmost => self.lower_bound.get(),
                MatchPosition::Rightmost => self.upper_bound
            },
            match_position: self.match_position,
            best_found: self.best_found,
            item: None,
        }
    }

    fn full(&self) -> bool {
        let best = self.best_found.load(Ordering::Relaxed);
        match self.match_position {
            // can stop consuming if the best found index so far is *strictly*
            // better than anything this consumer will find
            MatchPosition::Leftmost => best < self.lower_bound.get(),
            MatchPosition::Rightmost => best > self.upper_bound
        }
    }
}

impl<'f, ITEM, FIND_OP> UnindexedConsumer<ITEM> for FindConsumer<'f, FIND_OP>
    where ITEM: Send,
          FIND_OP: Fn(&ITEM) -> bool + Sync
{
    fn split_off(&self) -> Self {
        // Upper bound for one consumer will be lower bound for the other. This
        // overlap is okay, because only one of the bounds will be used for
        // comparing against best_found; the other is kept only to be able to
        // divide the range in half
        let old_lower_bound = self.lower_bound.get();
        let median = old_lower_bound + ((self.upper_bound - old_lower_bound) / 2);
        self.lower_bound.set(median);

        FindConsumer {
            find_op: self.find_op,
            lower_bound: Cell::new(old_lower_bound),
            upper_bound: median,
            match_position: self.match_position,
            best_found: self.best_found,
        }
    }

    fn to_reducer(&self) -> Self::Reducer {
        FindReducer { match_position: self.match_position }
    }
}

struct FindFolder<'f, ITEM, FIND_OP: 'f> {
    find_op: &'f FIND_OP,
    boundary: usize,
    match_position: MatchPosition,
    best_found: &'f AtomicUsize,
    item: Option<ITEM>,
}

impl<'f, FIND_OP: 'f + Fn(&ITEM) -> bool, ITEM> Folder<ITEM> for FindFolder<'f, ITEM, FIND_OP> {
    type Result = Option<ITEM>;

    fn consume(mut self, item: ITEM) -> Self {
        if (self.find_op)(&item) {
            // This may sometimes set best_found to a worse index than it was
            // before, depending on timing. This means more consumers will
            // continue to run than necessary, but the reducer will still ensure
            // the correct value is returned.
            self.best_found.swap(self.boundary, Ordering::Relaxed);
            self.item = Some(item);
        }
        self
    }

    fn complete(self) -> Self::Result {
        self.item
    }

    fn full(&self) -> bool {
        let best_found = self.best_found.load(Ordering::Relaxed);
        match self.match_position {
            MatchPosition::Leftmost => best_found < self.boundary,
            MatchPosition::Rightmost => best_found > self.boundary,
        }
    }
}

struct FindReducer {
    match_position: MatchPosition
}

impl<ITEM> Reducer<Option<ITEM>> for FindReducer {
    fn reduce(self, left: Option<ITEM>, right: Option<ITEM>) -> Option<ITEM> {
        match self.match_position {
            MatchPosition::Leftmost => left.or(right),
            MatchPosition::Rightmost => right.or(left)
        }
    }
}
<commit_msg>Refactor full calculations for find_first/last<commit_after>use std::cell::Cell;
use std::sync::atomic::{AtomicUsize, Ordering};
use super::internal::*;
use super::*;
use super::len::*;

// The consumer for find_first/find_last has fake indexes representing the lower
// and upper bounds of the "range" of data it consumes. This range does not
// correspond to indexes from the consumed iterator but rather indicate the
// consumer's position relative to other consumers. The purpose is to allow a
// consumer to know it should stop consuming items when another consumer finds a
// better match.

// An indexed consumer could specialize to use the real indexes instead, but we
// don't implement that for now. The only downside of the current approach is
// that in some cases, iterators very close to each other will have the same
// range and therefore not be able to stop processing if one of them finds a
// better match than the others.

#[derive(Copy, Clone)]
enum MatchPosition {
    Leftmost,
    Rightmost,
}

// Returns true if pos1 is a better match than pos2 according to MatchPosition
fn better_position(pos1: usize, pos2: usize, mp: MatchPosition) -> bool {
    match mp {
        MatchPosition::Leftmost => pos1 < pos2,
        MatchPosition::Rightmost => pos1 > pos2,
    }
}

pub fn find_first<PAR_ITER, FIND_OP>(pi: PAR_ITER, find_op: FIND_OP) -> Option<PAR_ITER::Item>
    where PAR_ITER: ParallelIterator,
          FIND_OP: Fn(&PAR_ITER::Item) -> bool + Sync
{
    let best_found = AtomicUsize::new(usize::max_value());
    let consumer = FindConsumer::new(&find_op, MatchPosition::Leftmost, &best_found);
    pi.drive_unindexed(consumer)
}

pub fn find_last<PAR_ITER, FIND_OP>(pi: PAR_ITER, find_op: FIND_OP) -> Option<PAR_ITER::Item>
    where PAR_ITER: ParallelIterator,
          FIND_OP: Fn(&PAR_ITER::Item) -> bool + Sync
{
    let best_found = AtomicUsize::new(0);
    let consumer = FindConsumer::new(&find_op, MatchPosition::Rightmost, &best_found);
    pi.drive_unindexed(consumer)
}

struct FindConsumer<'f, FIND_OP: 'f> {
    find_op: &'f FIND_OP,
    lower_bound: Cell<usize>,
    upper_bound: usize,
    match_position: MatchPosition,
    best_found: &'f AtomicUsize,
}

impl<'f, FIND_OP> FindConsumer<'f, FIND_OP> {
    fn new(find_op: &'f FIND_OP,
           match_position: MatchPosition,
           best_found: &'f AtomicUsize) -> Self {
        FindConsumer {
            find_op: find_op,
            lower_bound: Cell::new(0),
            upper_bound: usize::max_value(),
            match_position: match_position,
            best_found: best_found,
        }
    }

    fn current_index(&self) -> usize {
        match self.match_position {
            MatchPosition::Leftmost => self.lower_bound.get(),
            MatchPosition::Rightmost => self.upper_bound
        }
    }
}

impl<'f, ITEM, FIND_OP> Consumer<ITEM> for FindConsumer<'f, FIND_OP>
    where ITEM: Send,
          FIND_OP: Fn(&ITEM) -> bool + Sync
{
    type Folder = FindFolder<'f, ITEM, FIND_OP>;
    type Reducer = FindReducer;
    type Result = Option<ITEM>;

    fn cost(&mut self, cost: f64) -> f64 {
        cost * FUNC_ADJUSTMENT
    }

    fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) {
        let dir = self.match_position;
        (self.split_off(),
         self,
         FindReducer { match_position: dir })
    }

    fn into_folder(self) -> Self::Folder {
        FindFolder {
            find_op: self.find_op,
            boundary: self.current_index(),
            match_position: self.match_position,
            best_found: self.best_found,
            item: None,
        }
    }

    fn full(&self) -> bool {
        // can stop consuming if the best found index so far is *strictly*
        // better than anything this consumer will find
        better_position(self.best_found.load(Ordering::Relaxed),
                        self.current_index(),
                        self.match_position)
    }
}

impl<'f, ITEM, FIND_OP> UnindexedConsumer<ITEM> for FindConsumer<'f, FIND_OP>
    where ITEM: Send,
          FIND_OP: Fn(&ITEM) -> bool + Sync
{
    fn split_off(&self) -> Self {
        // Upper bound for one consumer will be lower bound for the other. This
        // overlap is okay, because only one of the bounds will be used for
        // comparing against best_found; the other is kept only to be able to
        // divide the range in half
        let old_lower_bound = self.lower_bound.get();
        let median = old_lower_bound + ((self.upper_bound - old_lower_bound) / 2);
        self.lower_bound.set(median);

        FindConsumer {
            find_op: self.find_op,
            lower_bound: Cell::new(old_lower_bound),
            upper_bound: median,
            match_position: self.match_position,
            best_found: self.best_found,
        }
    }

    fn to_reducer(&self) -> Self::Reducer {
        FindReducer { match_position: self.match_position }
    }
}

struct FindFolder<'f, ITEM, FIND_OP: 'f> {
    find_op: &'f FIND_OP,
    boundary: usize,
    match_position: MatchPosition,
    best_found: &'f AtomicUsize,
    item: Option<ITEM>,
}

impl<'f, FIND_OP: 'f + Fn(&ITEM) -> bool, ITEM> Folder<ITEM> for FindFolder<'f, ITEM, FIND_OP> {
    type Result = Option<ITEM>;

    fn consume(mut self, item: ITEM) -> Self {
        if (self.find_op)(&item) {
            // This may sometimes set best_found to a worse index than it was
            // before, depending on timing. This means more consumers will
            // continue to run than necessary, but the reducer will still ensure
            // the correct value is returned.
            self.best_found.swap(self.boundary, Ordering::Relaxed);
            self.item = Some(item);
        }
        self
    }

    fn complete(self) -> Self::Result {
        self.item
    }

    fn full(&self) -> bool {
        better_position(self.best_found.load(Ordering::Relaxed),
                        self.boundary,
                        self.match_position)
    }
}

struct FindReducer {
    match_position: MatchPosition
}

impl<ITEM> Reducer<Option<ITEM>> for FindReducer {
    fn reduce(self, left: Option<ITEM>, right: Option<ITEM>) -> Option<ITEM> {
        match self.match_position {
            MatchPosition::Leftmost => left.or(right),
            MatchPosition::Rightmost => right.or(left)
        }
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::bindings::codegen::EventBinding;
use dom::bindings::codegen::EventBinding::EventConstants;
use dom::bindings::js::JS;
use dom::bindings::utils::{Reflectable, Reflector, reflect_dom_object};
use dom::bindings::error::Fallible;
use dom::eventtarget::EventTarget;
use dom::window::Window;
use servo_util::str::DOMString;

use geom::point::Point2D;

pub enum Event_ {
    ResizeEvent(uint, uint),
    ReflowEvent,
    ClickEvent(uint, Point2D<f32>),
    MouseDownEvent(uint, Point2D<f32>),
    MouseUpEvent(uint, Point2D<f32>),
    MouseMoveEvent(Point2D<f32>)
}

#[deriving(Encodable)]
pub enum EventPhase {
    PhaseNone      = EventConstants::NONE,
    PhaseCapturing = EventConstants::CAPTURING_PHASE,
    PhaseAtTarget  = EventConstants::AT_TARGET,
    PhaseBubbling  = EventConstants::BUBBLING_PHASE,
}

#[deriving(Eq, Encodable)]
pub enum EventTypeId {
    HTMLEventTypeId,
    UIEventTypeId,
    MouseEventTypeId,
    KeyEventTypeId
}

#[deriving(Encodable)]
pub struct Event {
    type_id: EventTypeId,
    reflector_: Reflector,
    current_target: Option<JS<EventTarget>>,
    target: Option<JS<EventTarget>>,
    type_: DOMString,
    phase: EventPhase,
    canceled: bool,
    stop_propagation: bool,
    stop_immediate: bool,
    cancelable: bool,
    bubbles: bool,
    trusted: bool,
    dispatching: bool,
    initialized: bool,
}

impl Event {
    pub fn new_inherited(type_id: EventTypeId) -> Event {
        Event {
            type_id: type_id,
            reflector_: Reflector::new(),
            current_target: None,
            target: None,
            phase: PhaseNone,
            type_: ~"",
            canceled: false,
            cancelable: true,
            bubbles: true,
            trusted: false,
            dispatching: false,
            stop_propagation: false,
            stop_immediate: false,
            initialized: false,
        }
    }

    pub fn new(window: &JS<Window>) -> JS<Event> {
        reflect_dom_object(~Event::new_inherited(HTMLEventTypeId),
                           window,
                           EventBinding::Wrap)
    }

    pub fn EventPhase(&self) -> u16 {
        self.phase as u16
    }

    pub fn Type(&self) -> DOMString {
        self.type_.clone()
    }

    pub fn GetTarget(&self) -> Option<JS<EventTarget>> {
        self.target.clone()
    }

    pub fn GetCurrentTarget(&self) -> Option<JS<EventTarget>> {
        self.current_target.clone()
    }

    pub fn DefaultPrevented(&self) -> bool {
        self.canceled
    }

    pub fn PreventDefault(&mut self) {
        if self.cancelable {
            self.canceled = true
        }
    }

    pub fn StopPropagation(&mut self) {
        self.stop_propagation = true;
    }

    pub fn StopImmediatePropagation(&mut self) {
        self.stop_immediate = true;
        self.stop_propagation = true;
    }

    pub fn Bubbles(&self) -> bool {
        self.bubbles
    }

    pub fn Cancelable(&self) -> bool {
        self.cancelable
    }

    pub fn TimeStamp(&self) -> u64 {
        0
    }

    pub fn InitEvent(&mut self,
                     type_: DOMString,
                     bubbles: bool,
                     cancelable: bool) {
        self.type_ = type_;
        self.cancelable = cancelable;
        self.bubbles = bubbles;
        self.initialized = true;
        self.canceled = false;
    }

    pub fn IsTrusted(&self) -> bool {
        self.trusted
    }

    pub fn Constructor(global: &JS<Window>,
                       type_: DOMString,
                       init: &EventBinding::EventInit) -> Fallible<JS<Event>> {
        let mut ev = Event::new(global);
        ev.get_mut().InitEvent(type_, init.bubbles, init.cancelable);
        Ok(ev)
    }
}

impl Reflectable for Event {
    fn reflector<'a>(&'a self) -> &'a Reflector {
        &self.reflector_
    }

    fn mut_reflector<'a>(&'a mut self) -> &'a mut Reflector {
        &mut self.reflector_
    }
}
<commit_msg>auto merge of #2196 : hazz/servo/initevent, r=Ms2ger<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::bindings::codegen::EventBinding;
use dom::bindings::codegen::EventBinding::EventConstants;
use dom::bindings::js::JS;
use dom::bindings::utils::{Reflectable, Reflector, reflect_dom_object};
use dom::bindings::error::Fallible;
use dom::eventtarget::EventTarget;
use dom::window::Window;
use servo_util::str::DOMString;

use geom::point::Point2D;

pub enum Event_ {
    ResizeEvent(uint, uint),
    ReflowEvent,
    ClickEvent(uint, Point2D<f32>),
    MouseDownEvent(uint, Point2D<f32>),
    MouseUpEvent(uint, Point2D<f32>),
    MouseMoveEvent(Point2D<f32>)
}

#[deriving(Encodable)]
pub enum EventPhase {
    PhaseNone      = EventConstants::NONE,
    PhaseCapturing = EventConstants::CAPTURING_PHASE,
    PhaseAtTarget  = EventConstants::AT_TARGET,
    PhaseBubbling  = EventConstants::BUBBLING_PHASE,
}

#[deriving(Eq, Encodable)]
pub enum EventTypeId {
    HTMLEventTypeId,
    UIEventTypeId,
    MouseEventTypeId,
    KeyEventTypeId
}

#[deriving(Encodable)]
pub struct Event {
    type_id: EventTypeId,
    reflector_: Reflector,
    current_target: Option<JS<EventTarget>>,
    target: Option<JS<EventTarget>>,
    type_: DOMString,
    phase: EventPhase,
    canceled: bool,
    stop_propagation: bool,
    stop_immediate: bool,
    cancelable: bool,
    bubbles: bool,
    trusted: bool,
    dispatching: bool,
    initialized: bool,
}

impl Event {
    pub fn new_inherited(type_id: EventTypeId) -> Event {
        Event {
            type_id: type_id,
            reflector_: Reflector::new(),
            current_target: None,
            target: None,
            phase: PhaseNone,
            type_: ~"",
            canceled: false,
            cancelable: true,
            bubbles: true,
            trusted: false,
            dispatching: false,
            stop_propagation: false,
            stop_immediate: false,
            initialized: false,
        }
    }

    pub fn new(window: &JS<Window>) -> JS<Event> {
        reflect_dom_object(~Event::new_inherited(HTMLEventTypeId),
                           window,
                           EventBinding::Wrap)
    }

    pub fn EventPhase(&self) -> u16 {
        self.phase as u16
    }

    pub fn Type(&self) -> DOMString {
        self.type_.clone()
    }

    pub fn GetTarget(&self) -> Option<JS<EventTarget>> {
        self.target.clone()
    }

    pub fn GetCurrentTarget(&self) -> Option<JS<EventTarget>> {
        self.current_target.clone()
    }

    pub fn DefaultPrevented(&self) -> bool {
        self.canceled
    }

    pub fn PreventDefault(&mut self) {
        if self.cancelable {
            self.canceled = true
        }
    }

    pub fn StopPropagation(&mut self) {
        self.stop_propagation = true;
    }

    pub fn StopImmediatePropagation(&mut self) {
        self.stop_immediate = true;
        self.stop_propagation = true;
    }

    pub fn Bubbles(&self) -> bool {
        self.bubbles
    }

    pub fn Cancelable(&self) -> bool {
        self.cancelable
    }

    pub fn TimeStamp(&self) -> u64 {
        0
    }

    pub fn InitEvent(&mut self,
                     type_: DOMString,
                     bubbles: bool,
                     cancelable: bool) {
        self.initialized = true;
        if self.dispatching {
            return;
        }
        self.stop_propagation = false;
        self.stop_immediate = false;
        self.canceled = false;
        self.trusted = false;
        self.target = None;
        self.type_ = type_;
        self.bubbles = bubbles;
        self.cancelable = cancelable;
    }

    pub fn IsTrusted(&self) -> bool {
        self.trusted
    }

    pub fn Constructor(global: &JS<Window>,
                       type_: DOMString,
                       init: &EventBinding::EventInit) -> Fallible<JS<Event>> {
        let mut ev = Event::new(global);
        ev.get_mut().InitEvent(type_, init.bubbles, init.cancelable);
        Ok(ev)
    }
}

impl Reflectable for Event {
    fn reflector<'a>(&'a self) -> &'a Reflector {
        &self.reflector_
    }

    fn mut_reflector<'a>(&'a mut self) -> &'a mut Reflector {
        &mut self.reflector_
    }
}
<|endoftext|>


<commit_before>use Endpoint;
use PageableEndpoint;
use response::list as response;
use Pager;
use Client;

#[derive(Serialize, Debug, Clone)]
pub struct GetEntryList {
	pub kat: Option<String>,
	pub medium: Option<String>,
	pub is_h: Option<String>,
	pub start: Option<String>,
	pub sort: Option<String>,
	pub sort_type: Option<String>,
	pub p: Option<usize>,
	pub limit: Option<usize>,
}

impl Endpoint for GetEntryList {
	type ResponseType = Vec<response::EntryList>;
	#[doc(hidden)]
	const URL: &'static str = "list/entrylist";
}

impl PageableEndpoint for GetEntryList {
	/// Default pager:
	///
	/// ```
	/// p     = 0
	/// limit = 3500
	///```

	fn pager(self, client: Client) -> Pager<Self>
	{
		debug!("new pager with data: {:?}", self);
		Pager::new(client, self, Some(0), Some(3500))
	}

	fn page_mut(&mut self) -> &mut Option<usize>
	{
		&mut self.p
	}

	fn limit_mut(&mut self) -> &mut Option<usize>
	{
		&mut self.limit
	}
}

impl GetEntryList {
	/// Creates an instance with everything `None`
	pub fn with_default() -> Self {
		Self {
			is_h: None,
			kat: None,
			medium: None,
			limit: None,
			p: None,
			sort_type: None,
			sort: None,
			start: None,
		}
	}
}
<commit_msg>ignore code in codeblock<commit_after>use Endpoint;
use PageableEndpoint;
use response::list as response;
use Pager;
use Client;

#[derive(Serialize, Debug, Clone)]
pub struct GetEntryList {
	pub kat: Option<String>,
	pub medium: Option<String>,
	pub is_h: Option<String>,
	pub start: Option<String>,
	pub sort: Option<String>,
	pub sort_type: Option<String>,
	pub p: Option<usize>,
	pub limit: Option<usize>,
}

impl Endpoint for GetEntryList {
	type ResponseType = Vec<response::EntryList>;
	#[doc(hidden)]
	const URL: &'static str = "list/entrylist";
}

impl PageableEndpoint for GetEntryList {
	/// Default pager:
	///
	/// ```ignore
	/// p     = 0
	/// limit = 3500
	/// ```

	fn pager(self, client: Client) -> Pager<Self>
	{
		debug!("new pager with data: {:?}", self);
		Pager::new(client, self, Some(0), Some(3500))
	}

	fn page_mut(&mut self) -> &mut Option<usize>
	{
		&mut self.p
	}

	fn limit_mut(&mut self) -> &mut Option<usize>
	{
		&mut self.limit
	}
}

impl GetEntryList {
	/// Creates an instance with everything `None`
	pub fn with_default() -> Self {
		Self {
			is_h: None,
			kat: None,
			medium: None,
			limit: None,
			p: None,
			sort_type: None,
			sort: None,
			start: None,
		}
	}
}
<|endoftext|>


<commit_before>//! Experiments with parallel processing
//!
//! The provided functions focus on the possibility of
//! returning results while the parser proceeds. Sequences are processesd in
//! batches (`RecordSet`) because sending across channels has a performance
//! impact. FASTA/FASTQ records can be accessed in both the 'worker' function and
//! (after processing) a function running in the main thread.
//!
//! # Search first occurrence of a sequence pattern
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use seq_io::parallel::read_parallel;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//!
//! read_parallel(reader, 4, 2, |record_set| {
//!     // this function does the heavy work
//!     for (i, record) in record_set.into_iter().enumerate() {
//!         // this is not very efficient code, just for demonstration
//!         if let Some(pos) = record.seq().windows(3).position(|s| s == b"AAA") {
//!             return Some((i, pos));
//!         }
//!     }
//!     None
//! }, |record_sets| {
//!     // This function runs in the main thread. It provides a streaming iterator over
//!     // record sets and the corresponding return values from the worker function
//!     // (not necessarily in the same order as in the file)
//!     while let Some(result) = record_sets.next() {
//!         let (record_set, found) = result.unwrap();
//!         if let Some((i, pos)) = found {
//!             let record = record_set.into_iter().nth(i).unwrap();
//!             println!("Found AAA in record {} at position {}", record.id().unwrap(), pos);
//!              // this will also stop the worker threads, although with some delay
//!             return;
//!         }
//!     }
//! });
//! ```
//!
//! # Per-record processsing
//! The `parallel_fasta` / `parallel_fastq` functions are designed to efficiently pass
//! results for **each record** to the main thread without having to care about record sets.
//! This example filters sequences by the occurrence of a pattern:
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use std::fs::File;
//! use std::io::BufWriter;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//! let mut writer = BufWriter::new(File::create("filtered.fastq").unwrap());
//!
//! seq_io::parallel::parallel_fastq(reader, 4, 2,
//!     |record, found| { // runs in worker
//!         *found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
//!     },
//!     |record, found| { // runs in main thread
//!         if *found {
//!             record.write(&mut writer).unwrap();
//!         }
//!         // `false` will stop the reader
//!         true
//! }).unwrap();
//! ```

use std::io;

extern crate crossbeam;
extern crate scoped_threadpool;

use std::sync::mpsc;
use std::marker::PhantomData;

pub trait Reader: Send {
    type DataSet: Default + Send;
    type Err: Send;
    fn fill_data(&mut self, record: &mut Self::DataSet) -> Option<Result<(), Self::Err>>;
}

pub fn read_parallel<P, O, W, F, Out>(mut reader: P, n_threads: u32, queue_len: usize, work: W, func: F) -> Out
    where P: Reader,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(&mut P::DataSet) -> O,
          F: FnOnce(&mut ParallelRecordsets<P::DataSet, P::Err, O>) -> Out,
{

    let (done_send, done_recv) = mpsc::sync_channel(queue_len);
    let (empty_send, empty_recv): (mpsc::SyncSender<Option<P::DataSet>>, _) = mpsc::sync_channel(queue_len);

    crossbeam::scope(|scope| {

        scope.spawn(move || {

            let mut pool = scoped_threadpool::Pool::new(n_threads);

            pool.scoped(|pool_scope| {

                let work = &work;

                loop {
                    // recycle an old DataSet sent back after use by the streaming iterator
                    let mut data =
                        if let Ok(Some(r)) = empty_recv.recv() {
                            r
                        } else {
                            // 'ParallelRecordsets::stop()' called
                            return;
                        };

                    // each time, we need a new reference
                    let done_send = done_send.clone();

                    if let Some(res) = reader.fill_data(&mut data) {
                        match res {
                            Ok(_) => {
                                // expensive work carried out by func()
                                pool_scope.execute(move || {

                                    let out = work(&mut data);

                                    done_send.send(Some(Ok((data, out)))).ok();
                                });
                            }
                            Err(e) => {
                                done_send.send(Some(Err(e))).ok();
                                break;
                            }
                        }
                    } else {
                        break;
                    }
                }

                // make sure that the 'done' signal is only sent after everything else is done
                pool_scope.join_all();

                done_send.send(None).ok();
            });
        });

        for _ in 0..queue_len  {
            empty_send.send(Some(P::DataSet::default())).unwrap();
        }

        let mut records = ParallelRecordsets {
            empty_send: empty_send,
            done_recv: done_recv,
            current_recordset: P::DataSet::default(),
        };

        let out = func(&mut records);

        records.stop();

        out
    })
}

pub struct ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
{
    empty_send: mpsc::SyncSender<Option<R>>,
    done_recv: mpsc::Receiver<Option<Result<(R, O), E>>>,
    current_recordset: R,
}

impl<R, E, O> ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
 {

    pub fn next<'b>(&'b mut self) -> Option<Result<(&'b R, O), E>> {
        if let Some(result) = self.done_recv.recv().unwrap() {
            match result {
                Ok((r, o)) => {
                     let prev_rset = ::std::mem::replace(&mut self.current_recordset, r);
                     self.empty_send.send(Some(prev_rset)).ok(); // error: channel closed is not a problem, happens after calling stop()
                    return Some(Ok((&self.current_recordset, o)));
                }
                Err(e) => return Some(Err(e))
            }
        } else {
            // 'done' signal received
            return None;
        }
    }

    // has to be called before object goes out of scope
    // currently, the signal is delayed (has to wait until it is popped from the
    // 'empty' queue)
    // TODO: should an AtomicBool be used?
    fn stop(self) {
        self.empty_send.send(None).ok();
    }
}

#[macro_export]
macro_rules! parallel_record_impl {
    ($name:ident, $io_r:tt, $rdr:ty, $record:ty, $err:ty) => {

        pub fn $name<$io_r, O, W, F>(parser: $rdr, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<(), $err>
            where $io_r: io::Read + Send,
                  O: Default + Send,
                  W: Send + Sync,
                  W: Fn($record, &mut O),
                  F: FnMut($record, &O) -> bool,
        {
            let reader = $crate::parallel::ReusableReader::new(parser);

            $crate::parallel::read_parallel(reader, n_threads, queue_len, |d| {
                let mut iter = d.0.into_iter();
                let out: &mut Vec<O> = &mut d.1;
                for mut x in out.iter_mut().zip(&mut iter) {
                    work(x.1, &mut x.0);
                }
                for i in iter {
                    out.push(O::default());
                    work(i, out.last_mut().unwrap())
                }

            }, |records| {
                while let Some(result) = records.next() {
                    let (r, _) = result?;
                    for x in r.0.into_iter().zip(&r.1) {
                        if ! func(x.0, x.1) {
                            break;
                        }
                    }
                }
                Ok(())
            })
        }
     };
}

parallel_record_impl!(parallel_fasta, R, fasta::Reader<R>, fasta::RefRecord, fasta::ParseError);

parallel_record_impl!(parallel_fastq, R, fastq::Reader<R>, fastq::RefRecord, fastq::ParseError);

/// Wrapper for `parallel::Reader` instances allowing
/// the output to be reused in order to save allocations.
/// Used by `parallel_fasta`/`parallel_fastq`
///
/// ```no_run
/// use seq_io::fastq::{Reader,Record,RecordSet};
/// use seq_io::parallel::{read_parallel,ReusableReader};
///
/// let inner = Reader::from_path("seqs.fastq").unwrap();
/// let reader = ReusableReader::new(inner);
///
/// read_parallel(reader, 4, 2, |&mut (ref record_set, ref mut out): &mut (RecordSet, Vec<bool>)| {
///     out.clear();
///     for record in record_set {
///         let found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
///         out.push(found);
///     }
/// }, |record_sets| {
///     while let Some(result) = record_sets.next() {
///         let &(ref record_set, ref out) = result.unwrap().0;
///         for (record, found) in record_set.into_iter().zip(out) {
///             // ...
///         }
///     }
/// });
/// ```
pub struct ReusableReader<P, O>(P, PhantomData<O>);

impl<P, O> ReusableReader<P, O> {
    pub fn new(p: P) -> ReusableReader<P, O> {
        ReusableReader(p, PhantomData)
    }
}

impl<P, O> Reader for ReusableReader<P, O>
    where P: Reader,
          O: Default + Send {

    type DataSet = (P::DataSet, O);
    type Err = P::Err;
    fn fill_data(&mut self, data: &mut Self::DataSet) -> Option<Result<(), P::Err>> {
        self.0.fill_data(&mut data.0)
    }
}

// trait impls

use super::fasta;

impl<R, S> Reader for fasta::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fasta::RecordSet;
    type Err = fasta::ParseError;
    fn fill_data(&mut self, rset: &mut fasta::RecordSet) -> Option<Result<(), fasta::ParseError>> {
        self.read_record_set(rset)
    }
}

use super::fastq;

impl<R, S> Reader for fastq::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fastq::RecordSet;
    type Err = fastq::ParseError;
    fn fill_data(&mut self, rset: &mut fastq::RecordSet) -> Option<Result<(), fastq::ParseError>> {
        self.read_record_set(rset)
    }
}
<commit_msg>Add 'parallel_records' function even if currently not usable.<commit_after>//! Experiments with parallel processing
//!
//! The provided functions focus on the possibility of
//! returning results while the parser proceeds. Sequences are processesd in
//! batches (`RecordSet`) because sending across channels has a performance
//! impact. FASTA/FASTQ records can be accessed in both the 'worker' function and
//! (after processing) a function running in the main thread.
//!
//! # Search first occurrence of a sequence pattern
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use seq_io::parallel::read_parallel;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//!
//! read_parallel(reader, 4, 2, |record_set| {
//!     // this function does the heavy work
//!     for (i, record) in record_set.into_iter().enumerate() {
//!         // this is not very efficient code, just for demonstration
//!         if let Some(pos) = record.seq().windows(3).position(|s| s == b"AAA") {
//!             return Some((i, pos));
//!         }
//!     }
//!     None
//! }, |record_sets| {
//!     // This function runs in the main thread. It provides a streaming iterator over
//!     // record sets and the corresponding return values from the worker function
//!     // (not necessarily in the same order as in the file)
//!     while let Some(result) = record_sets.next() {
//!         let (record_set, found) = result.unwrap();
//!         if let Some((i, pos)) = found {
//!             let record = record_set.into_iter().nth(i).unwrap();
//!             println!("Found AAA in record {} at position {}", record.id().unwrap(), pos);
//!              // this will also stop the worker threads, although with some delay
//!             return;
//!         }
//!     }
//! });
//! ```
//!
//! # Per-record processsing
//! The `parallel_fasta` / `parallel_fastq` functions are designed to efficiently pass
//! results for **each record** to the main thread without having to care about record sets.
//! This example filters sequences by the occurrence of a pattern:
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use std::fs::File;
//! use std::io::BufWriter;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//! let mut writer = BufWriter::new(File::create("filtered.fastq").unwrap());
//!
//! seq_io::parallel::parallel_fastq(reader, 4, 2,
//!     |record, found| { // runs in worker
//!         *found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
//!     },
//!     |record, found| { // runs in main thread
//!         if *found {
//!             record.write(&mut writer).unwrap();
//!         }
//!         // `false` will stop the reader
//!         true
//! }).unwrap();
//! ```

use std::io;

extern crate crossbeam;
extern crate scoped_threadpool;

use std::sync::mpsc;
use std::marker::PhantomData;

pub trait Reader: Send {
    type DataSet: Default + Send;
    type Err: Send;
    fn fill_data(&mut self, record: &mut Self::DataSet) -> Option<Result<(), Self::Err>>;
}

pub fn read_parallel<P, O, W, F, Out>(mut reader: P, n_threads: u32, queue_len: usize, work: W, func: F) -> Out
    where P: Reader,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(&mut P::DataSet) -> O,
          F: FnOnce(&mut ParallelRecordsets<P::DataSet, P::Err, O>) -> Out,
{

    let (done_send, done_recv) = mpsc::sync_channel(queue_len);
    let (empty_send, empty_recv): (mpsc::SyncSender<Option<P::DataSet>>, _) = mpsc::sync_channel(queue_len);

    crossbeam::scope(|scope| {

        scope.spawn(move || {

            let mut pool = scoped_threadpool::Pool::new(n_threads);

            pool.scoped(|pool_scope| {

                let work = &work;

                loop {
                    // recycle an old DataSet sent back after use by the streaming iterator
                    let mut data =
                        if let Ok(Some(r)) = empty_recv.recv() {
                            r
                        } else {
                            // 'ParallelRecordsets::stop()' called
                            return;
                        };

                    // each time, we need a new reference
                    let done_send = done_send.clone();

                    if let Some(res) = reader.fill_data(&mut data) {
                        match res {
                            Ok(_) => {
                                // expensive work carried out by func()
                                pool_scope.execute(move || {

                                    let out = work(&mut data);

                                    done_send.send(Some(Ok((data, out)))).ok();
                                });
                            }
                            Err(e) => {
                                done_send.send(Some(Err(e))).ok();
                                break;
                            }
                        }
                    } else {
                        break;
                    }
                }

                // make sure that the 'done' signal is only sent after everything else is done
                pool_scope.join_all();

                done_send.send(None).ok();
            });
        });

        for _ in 0..queue_len  {
            empty_send.send(Some(P::DataSet::default())).unwrap();
        }

        let mut records = ParallelRecordsets {
            empty_send: empty_send,
            done_recv: done_recv,
            current_recordset: P::DataSet::default(),
        };

        let out = func(&mut records);

        records.stop();

        out
    })
}

pub struct ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
{
    empty_send: mpsc::SyncSender<Option<R>>,
    done_recv: mpsc::Receiver<Option<Result<(R, O), E>>>,
    current_recordset: R,
}

impl<R, E, O> ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
 {

    pub fn next<'b>(&'b mut self) -> Option<Result<(&'b R, O), E>> {
        if let Some(result) = self.done_recv.recv().unwrap() {
            match result {
                Ok((r, o)) => {
                     let prev_rset = ::std::mem::replace(&mut self.current_recordset, r);
                     self.empty_send.send(Some(prev_rset)).ok(); // error: channel closed is not a problem, happens after calling stop()
                    return Some(Ok((&self.current_recordset, o)));
                }
                Err(e) => return Some(Err(e))
            }
        } else {
            // 'done' signal received
            return None;
        }
    }

    // has to be called before object goes out of scope
    // currently, the signal is delayed (has to wait until it is popped from the
    // 'empty' queue)
    // TODO: should an AtomicBool be used?
    fn stop(self) {
        self.empty_send.send(None).ok();
    }
}

#[macro_export]
macro_rules! parallel_record_impl {
    ($name:ident, $io_r:tt, $rdr:ty, $record:ty, $err:ty) => {

        pub fn $name<$io_r, O, W, F>(parser: $rdr, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<(), $err>
            where $io_r: io::Read + Send,
                  O: Default + Send,
                  W: Send + Sync,
                  W: Fn($record, &mut O),
                  F: FnMut($record, &O) -> bool,
        {
            let reader = $crate::parallel::ReusableReader::new(parser);

            $crate::parallel::read_parallel(reader, n_threads, queue_len, |d| {
                let mut iter = d.0.into_iter();
                let out: &mut Vec<O> = &mut d.1;
                for mut x in out.iter_mut().zip(&mut iter) {
                    work(x.1, &mut x.0);
                }
                for i in iter {
                    out.push(O::default());
                    work(i, out.last_mut().unwrap())
                }

            }, |records| {
                while let Some(result) = records.next() {
                    let (r, _) = result?;
                    for x in r.0.into_iter().zip(&r.1) {
                        if ! func(x.0, x.1) {
                            break;
                        }
                    }
                }
                Ok(())
            })
        }
     };
}

parallel_record_impl!(parallel_fasta, R, fasta::Reader<R>, fasta::RefRecord, fasta::ParseError);

parallel_record_impl!(parallel_fastq, R, fastq::Reader<R>, fastq::RefRecord, fastq::ParseError);

/// Wrapper for `parallel::Reader` instances allowing
/// the output to be reused in order to save allocations.
/// Used by `parallel_fasta`/`parallel_fastq`
///
/// ```no_run
/// use seq_io::fastq::{Reader,Record,RecordSet};
/// use seq_io::parallel::{read_parallel,ReusableReader};
///
/// let inner = Reader::from_path("seqs.fastq").unwrap();
/// let reader = ReusableReader::new(inner);
///
/// read_parallel(reader, 4, 2, |&mut (ref record_set, ref mut out): &mut (RecordSet, Vec<bool>)| {
///     out.clear();
///     for record in record_set {
///         let found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
///         out.push(found);
///     }
/// }, |record_sets| {
///     while let Some(result) = record_sets.next() {
///         let &(ref record_set, ref out) = result.unwrap().0;
///         for (record, found) in record_set.into_iter().zip(out) {
///             // ...
///         }
///     }
/// });
/// ```
pub struct ReusableReader<P, O>(P, PhantomData<O>);

impl<P, O> ReusableReader<P, O> {
    pub fn new(p: P) -> ReusableReader<P, O> {
        ReusableReader(p, PhantomData)
    }
}

impl<P, O> Reader for ReusableReader<P, O>
    where P: Reader,
          O: Default + Send {

    type DataSet = (P::DataSet, O);
    type Err = P::Err;
    fn fill_data(&mut self, data: &mut Self::DataSet) -> Option<Result<(), P::Err>> {
        self.0.fill_data(&mut data.0)
    }
}

/// Using this function currently does not work due to a
/// [compiler bug](https://github.com/rust-lang/rust/issues/42950).
///
/// `parallel_fasta`/`parallel_fastq` provide the same functionality for now
/// (implemented using `parallel_record_impl` macro)
pub fn parallel_records<P, O, W, F, Out>(parser: P, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<(), P::Err>
    where P: Reader,
          for<'a> &'a P::DataSet: IntoIterator,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(<&P::DataSet as IntoIterator>::Item, &mut O),
          F: FnMut(<&P::DataSet as IntoIterator>::Item, &O) -> bool,
{
    let reader = ReusableReader(parser, PhantomData);

    read_parallel(reader, n_threads, queue_len, |d| {
        let mut iter = d.0.into_iter();
        let out: &mut Vec<O> = &mut d.1;
        for mut x in out.iter_mut().zip(&mut iter) {
            work(x.1, &mut x.0);
        }
        for i in iter {
            out.push(O::default());
            work(i, out.last_mut().unwrap())
        }

    }, |records| {
        while let Some(result) = records.next() {
            let (r, _) = result?;
            for x in r.0.into_iter().zip(&r.1) {
                if ! func(x.0, x.1) {
                    break;
                }
            }
        }
        Ok(())
    })
}

// trait impls

use super::fasta;

impl<R, S> Reader for fasta::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fasta::RecordSet;
    type Err = fasta::ParseError;
    fn fill_data(&mut self, rset: &mut fasta::RecordSet) -> Option<Result<(), fasta::ParseError>> {
        self.read_record_set(rset)
    }
}

use super::fastq;

impl<R, S> Reader for fastq::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fastq::RecordSet;
    type Err = fastq::ParseError;
    fn fill_data(&mut self, rset: &mut fastq::RecordSet) -> Option<Result<(), fastq::ParseError>> {
        self.read_record_set(rset)
    }
}
<|endoftext|>