<commit_before>use native::task;
use std::comm::channel;

struct WorkUnit<ARG, RV> {
    arg: ARG,
    rv: Sender<RV>,
}

enum MessageToWorker<ARG, RV> {
    Work(WorkUnit<ARG, RV>),
    Halt,
}

enum MessageToDispatcher<ARG, RV> {
    Dispatch(WorkUnit<ARG, RV>),
    HaltAll,
    RegisterWorker(Sender<Sender<Sender<MessageToWorker<ARG, RV>>>>),
}

struct WorkQueue<ARG, RV> {
    dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

impl<ARG:Send, RV:Send> WorkQueue<ARG, RV> {
    pub fn create() -> WorkQueue<ARG, RV> {
        let (dispatcher, dispatcher_inbox) = channel::<MessageToDispatcher<ARG, RV>>();

        // dispatcher
        task::spawn(proc() {
            let (want_work, idle_worker) = channel::<Sender<MessageToWorker<ARG, RV>>>();
            let mut worker_count = 0;
            let inbox = dispatcher_inbox;
            let idle_worker = idle_worker;
            loop {
                match inbox.recv() {
                    Dispatch(work_item) => {
                        idle_worker.recv().send(Work(work_item));
                    },
                    RegisterWorker(want_idle_sender) => {
                        worker_count += 1;
                        want_idle_sender.send(want_work.clone());
                    }
                    HaltAll => {
                        while worker_count > 0 {
                            idle_worker.recv().send(Halt);
                            worker_count -= 1;
                        }
                        return;
                    },
                };
            }
        });
        return WorkQueue{dispatcher: dispatcher};
    }

    pub fn register_worker(&self) -> Sender<Sender<MessageToWorker<ARG, RV>>> {
        let (reg_s, reg_r) = channel::<Sender<Sender<MessageToWorker<ARG, RV>>>>();
        self.dispatcher.send(RegisterWorker(reg_s));
        return reg_r.recv();
    }

    pub fn execute(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch(WorkUnit{arg: arg, rv: rv}));
        return wait_for_rv;
    }
}

// rustc complais "cannot implement a destructor on a structure with
// type parameters", but our destruction is safe, we only send
// a simple message on a channel.
#[unsafe_destructor]
impl<ARG:Send, RV:Send> Drop for WorkQueue<ARG, RV> {
    fn drop(&mut self) {
        self.dispatcher.send(HaltAll);
    }
}

fn spawn_test_worker(queue: &WorkQueue<int, int>) {
    let want_work = queue.register_worker();
    task::spawn(proc() {
        let want_work = want_work;
        loop {
            let (idle, get_work_unit) = channel::<MessageToWorker<int, int>>();
            want_work.send(idle);
            let work_unit = match get_work_unit.recv() {
                Work(wu) => wu,
                Halt     => return
            };
            let rv = work_unit.arg * 2;
            work_unit.rv.send(rv);
        }
    });
}

#[test]
fn test_queue() {
    let queue = WorkQueue::<int, int>::create();
    for _ in range(0, 3) {
        spawn_test_worker(&queue);
    }
    let mut promise_list: ~[Receiver<int>] = ~[];
    for c in range(0, 10) {
        let rv = queue.execute(c);
        promise_list.push(rv);
    }
    let return_list = promise_list.map(|promise| promise.recv());
    assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
}
<commit_msg>create queue proxy so we can enqueue from tasks<commit_after>use native::task;
use std::comm::channel;

struct WorkUnit<ARG, RV> {
    arg: ARG,
    rv: Sender<RV>,
}

enum MessageToWorker<ARG, RV> {
    Work(WorkUnit<ARG, RV>),
    Halt,
}

enum MessageToDispatcher<ARG, RV> {
    Dispatch(WorkUnit<ARG, RV>),
    HaltAll,
    RegisterWorker(Sender<Sender<Sender<MessageToWorker<ARG, RV>>>>),
}

struct WorkQueue<ARG, RV> {
    dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

struct WorkQueueProxy<ARG, RV> {
    dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

impl<ARG:Send, RV:Send> WorkQueue<ARG, RV> {
    pub fn create() -> WorkQueue<ARG, RV> {
        let (dispatcher, dispatcher_inbox) = channel::<MessageToDispatcher<ARG, RV>>();

        // dispatcher
        task::spawn(proc() {
            let (want_work, idle_worker) = channel::<Sender<MessageToWorker<ARG, RV>>>();
            let mut worker_count = 0;
            let inbox = dispatcher_inbox;
            let idle_worker = idle_worker;
            loop {
                match inbox.recv() {
                    Dispatch(work_item) => {
                        idle_worker.recv().send(Work(work_item));
                    },
                    RegisterWorker(want_idle_sender) => {
                        worker_count += 1;
                        want_idle_sender.send(want_work.clone());
                    }
                    HaltAll => {
                        while worker_count > 0 {
                            idle_worker.recv().send(Halt);
                            worker_count -= 1;
                        }
                        return;
                    },
                };
            }
        });
        return WorkQueue{dispatcher: dispatcher};
    }

    pub fn proxy(&self) -> WorkQueueProxy<ARG, RV> {
        return WorkQueueProxy{dispatcher: self.dispatcher.clone()};
    }

    pub fn register_worker(&self) -> Sender<Sender<MessageToWorker<ARG, RV>>> {
        let (reg_s, reg_r) = channel::<Sender<Sender<MessageToWorker<ARG, RV>>>>();
        self.dispatcher.send(RegisterWorker(reg_s));
        return reg_r.recv();
    }

    pub fn execute(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch(WorkUnit{arg: arg, rv: rv}));
        return wait_for_rv;
    }
}

// rustc complais "cannot implement a destructor on a structure with
// type parameters", but our destruction is safe, we only send
// a simple message on a channel.
#[unsafe_destructor]
impl<ARG:Send, RV:Send> Drop for WorkQueue<ARG, RV> {
    fn drop(&mut self) {
        self.dispatcher.send(HaltAll);
    }
}

impl<ARG:Send, RV:Send> WorkQueueProxy<ARG, RV> {
    pub fn execute(&self, arg: ARG) -> Receiver<RV> {
        let (rv, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch(WorkUnit{arg: arg, rv: rv}));
        return wait_for_rv;
    }
}

fn spawn_test_worker(queue: &WorkQueue<int, int>) {
    let want_work = queue.register_worker();
    task::spawn(proc() {
        let want_work = want_work;
        loop {
            let (idle, get_work_unit) = channel::<MessageToWorker<int, int>>();
            want_work.send(idle);
            let work_unit = match get_work_unit.recv() {
                Work(wu) => wu,
                Halt     => return
            };
            let rv = work_unit.arg * 2;
            work_unit.rv.send(rv);
        }
    });
}

#[test]
fn test_queue() {
    let queue = WorkQueue::<int, int>::create();
    for _ in range(0, 3) {
        spawn_test_worker(&queue);
    }
    let mut promise_list: ~[Receiver<int>] = ~[];
    for c in range(0, 10) {
        let rv = queue.execute(c);
        promise_list.push(rv);
    }
    let return_list = promise_list.map(|promise| promise.recv());
    assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
}

#[test]
fn test_enqueue_from_tasks() {
    let queue = WorkQueue::<int, int>::create();
    for _ in range(0, 3) {
        spawn_test_worker(&queue);
    }
    let mut promise_list: ~[Receiver<int>] = ~[];
    for c in range(0, 10) {
        let queue_proxy = queue.proxy();
        let (done, promise) = channel::<int>();
        promise_list.push(promise);
        task::spawn(proc() {
            let done = done;
            let queue = queue_proxy;
            let rv = queue.execute(c);
            done.send(rv.recv());
        });
    }
    let return_list = promise_list.map(|promise| promise.recv());
    assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
}
<|endoftext|>


<commit_before>extern crate cgmath;
pub use self::cgmath::*;
pub use self::cgmath::num_traits::{Float, NumCast, One};
use std::ops::Add;

/// Shorthand constructor for a `Point1`.
pub fn pt1<S>(x: S) -> Point1<S> {
    Point1 { x }
}

/// Shorthand constructor for a `Point2`.
pub fn pt2<S>(x: S, y: S) -> Point2<S> {
    Point2 { x, y }
}

/// Shorthand constructor for a `Point3`.
pub fn pt3<S>(x: S, y: S, z: S) -> Point3<S> {
    Point3 { x, y, z }
}

/// Map a value from a given range to a new given range.
pub fn map_range<X, Y>(val: X, in_min: X, in_max: X, out_min: Y, out_max: Y) -> Y
where
    X: NumCast,
    Y: NumCast,
{
    let val_f: f64 = NumCast::from(val).unwrap();
    let in_min_f: f64 = NumCast::from(in_min).unwrap();
    let in_max_f: f64 = NumCast::from(in_max).unwrap();
    let out_min_f: f64 = NumCast::from(out_min).unwrap();
    let out_max_f: f64 = NumCast::from(out_max).unwrap();
    NumCast::from(
        (val_f - in_min_f) / (in_max_f - in_min_f) * (out_max_f - out_min_f) + out_min_f
    ).unwrap()
}

/// The max between two partially ordered values.
pub fn partial_max<T>(a: T, b: T) -> T
where
    T: PartialOrd,
{
    if a >= b { a } else { b }
}

/// The min between two partially ordered values.
pub fn partial_min<T>(a: T, b: T) -> T
where
    T: PartialOrd,
{
    if a <= b { a } else { b }
}

/// Clamp a value between some range.
pub fn clamp<T>(n: T, start: T, end: T) -> T
where
    T: PartialOrd,
{
    if start <= end {
        if n < start { start } else if n > end { end } else { n }
    } else {
        if n < end { end } else if n > start { start } else { n }
    }
}

pub fn two<S>() -> S
where
    S: Add<Output=S> + One,
{
    S::one() + S::one()
}

/// Models the C++ fmod function.
#[inline]
pub fn fmod<F>(numer: F, denom: F) -> F
where
    F: Float,
{
    let rquot: F = (numer / denom).floor();
    numer - rquot * denom
}
<commit_msg>Add simple angle conversion functions to math module<commit_after>extern crate cgmath;
pub use self::cgmath::*;
pub use self::cgmath::num_traits::{Float, NumCast, One};
use std::ops::Add;

/// Shorthand constructor for a `Point1`.
pub fn pt1<S>(x: S) -> Point1<S> {
    Point1 { x }
}

/// Shorthand constructor for a `Point2`.
pub fn pt2<S>(x: S, y: S) -> Point2<S> {
    Point2 { x, y }
}

/// Shorthand constructor for a `Point3`.
pub fn pt3<S>(x: S, y: S, z: S) -> Point3<S> {
    Point3 { x, y, z }
}

/// Map a value from a given range to a new given range.
pub fn map_range<X, Y>(val: X, in_min: X, in_max: X, out_min: Y, out_max: Y) -> Y
where
    X: NumCast,
    Y: NumCast,
{
    let val_f: f64 = NumCast::from(val).unwrap();
    let in_min_f: f64 = NumCast::from(in_min).unwrap();
    let in_max_f: f64 = NumCast::from(in_max).unwrap();
    let out_min_f: f64 = NumCast::from(out_min).unwrap();
    let out_max_f: f64 = NumCast::from(out_max).unwrap();
    NumCast::from(
        (val_f - in_min_f) / (in_max_f - in_min_f) * (out_max_f - out_min_f) + out_min_f
    ).unwrap()
}

/// The max between two partially ordered values.
pub fn partial_max<T>(a: T, b: T) -> T
where
    T: PartialOrd,
{
    if a >= b { a } else { b }
}

/// The min between two partially ordered values.
pub fn partial_min<T>(a: T, b: T) -> T
where
    T: PartialOrd,
{
    if a <= b { a } else { b }
}

/// Clamp a value between some range.
pub fn clamp<T>(n: T, start: T, end: T) -> T
where
    T: PartialOrd,
{
    if start <= end {
        if n < start { start } else if n > end { end } else { n }
    } else {
        if n < end { end } else if n > start { start } else { n }
    }
}

pub fn two<S>() -> S
where
    S: Add<Output=S> + One,
{
    S::one() + S::one()
}

/// Models the C++ fmod function.
#[inline]
pub fn fmod<F>(numer: F, denom: F) -> F
where
    F: Float,
{
    let rquot: F = (numer / denom).floor();
    numer - rquot * denom
}

/// Convert the given angle in degrees to the same angle in radians.
pub fn deg_to_rad<S>(deg: S) -> S
where
    S: BaseFloat,
{
    Rad::from(Deg(deg)).0
}

/// Convert the given angle in radians to the same angle in degrees.
pub fn rad_to_deg<S>(rad: S) -> S
where
    S: BaseFloat,
{
    Deg::from(Rad(rad)).0
}

/// Convert the given value as a number of "turns" into the equivalent angle in radians.
pub fn turns_to_rad<S>(turns: S) -> S
where
    S: BaseFloat,
{
    turns * NumCast::from(2.0 * ::std::f64::consts::PI).unwrap()
}

/// Convert the given value in radians to the equivalent value as a number of turns.
pub fn rad_to_turns<S>(rad: S) -> S
where
    S: BaseFloat,
{
    rad / NumCast::from(2.0 * ::std::f64::consts::PI).unwrap()
}
<|endoftext|>


<commit_before>use std::collections::BTreeMap;

use std::io;
use std::fs;

use rustc_serialize::json::ToJson;
use rustc_serialize::json::Json;

use hoedown::Html;
use hoedown::renderer::html::Flags;
use hoedown::Markdown;
use hoedown::Render;

static MARKDOWN_PATH: &'static str = "markdown/";

// TODO: add a data caching system for rendered markdown.

pub fn make_data_from_markdown(page_name: &str) -> io::Result<BTreeMap<String, Json>> {
    let mut page_path = MARKDOWN_PATH.to_owned();
    page_path += page_name;
    page_path += ".md";

    let mut data = BTreeMap::new();
    match fs::File::open(page_path) {
        Ok(file) => {
            let markdown = Markdown::read_from(file);
            let mut html = Html::new(Flags::empty(), 0);
            data.insert("content".to_owned(), html.render(&markdown).to_str().unwrap()
                        .replace("
", "
")
                        .replace("
", "
").to_json());
            // I'm not happy with the way hoedown converts markdown code part into html,
            // so I manually replace some things. It should probably be handled
            // in a better way though.
            Ok(data)
        }
        Err(e) => {
            Err(e)
        }
    }
}

<commit_msg>Update hoedown render part.<commit_after>use std::collections::BTreeMap;

use std::io;
use std::fs;

use rustc_serialize::json::ToJson;
use rustc_serialize::json::Json;

use hoedown::Html;
use hoedown::renderer::html::Flags;
use hoedown::Markdown;
use hoedown::Render;

static MARKDOWN_PATH: &'static str = "markdown/";

// TODO: add a data caching system for rendered markdown.

pub fn make_data_from_markdown(page_name: &str) -> io::Result<BTreeMap<String, Json>> {
    let mut page_path = MARKDOWN_PATH.to_owned();
    page_path += page_name;
    page_path += ".md";

    let mut data = BTreeMap::new();
    match fs::File::open(page_path) {
        Ok(file) => {
            let markdown = Markdown::read_from(file);
            let mut html = Html::new(Flags::empty(), 0);
            data.insert("content".to_owned(), html.render(&markdown).to_str().unwrap().to_json());
            Ok(data)
        }
        Err(e) => {
            Err(e)
        }
    }
}

<|endoftext|>


<commit_before>use std::fmt;

use crate::dictionary::CONF_LINE_WORD_MODIFIER__VISIBLE;
const LINE_WIDTH: usize = 20;

#[derive(Debug)]
struct HangmanChar {
    char_: char,
    visible: bool,
}

impl fmt::Display for HangmanChar {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        if self.visible {
            write!(f, "{}", self.char_)
        } else {
            write!(f, "_")
        }
    }
}

pub enum State {
    Ongoing,
    Victory,
    Defeat,
}

#[derive(Debug)]
pub struct Game {
    word: Vec<HangmanChar>,
    pub lives: u8,
    pub last_guess: char,
}

impl Game {
    pub fn get_state(&self) -> State {
        if self.lives == 0 {
            State::Defeat
        } else if self.word.iter().all(|c| c.visible) {
            State::Victory
        } else {
            State::Ongoing
        }
    }

    pub fn new(wordstr: &str, l: u8) -> Self {
        // parse wordsstr, filp 'visible' every CONF_LINE_WORD_MODIFIER__VISIBLE
        let w = wordstr
            .chars()
            // for every * found flip v_acc
            .scan(true, |v_acc, c| {
                *v_acc ^= c == CONF_LINE_WORD_MODIFIER__VISIBLE;
                if c == CONF_LINE_WORD_MODIFIER__VISIBLE {
                    Some(None)
                } else {
                    Some(Some(HangmanChar{char_: c, visible: *v_acc}))
                }
            })
            // ommit None and unwrap
            .filter_map(|s|s)
            //.inspect(|ref x| println!("after scan:\t{:?}", x))
            .collect();

        println!("{:?}",w);

        Self {
            word: w,
            lives: l,
            last_guess: ' ',
        }
    }

    pub fn guess(&mut self, char_: char) {
        if char_ == '\n' {
            return;
        };
        self.last_guess = char_;
        let mut found = false;
        for h_char in self.word.iter_mut() {
            if h_char.char_.eq_ignore_ascii_case(&char_) {
                h_char.visible = true;
                found = true;
            }
        }

        if !found {
            self.lives -= 1;
        }

        if self.lives == 0 {
            for hc in self.word.iter_mut() {
                hc.visible = true;
            }
        }
    }

    pub fn visible_chars(&self) -> usize {
        self.word.iter().filter(|hc| !hc.visible).count()
    }
}

impl fmt::Display for Game {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        writeln!(
            f,
            "\x1b[KLives:\t{}\tLast guess: {}\n",
            self.lives, self.last_guess
        )?;

        let mut linebreak = false;
        for (n, c) in self.word.iter().enumerate() {
            if n % LINE_WIDTH == 0 {
                linebreak = true
            };
            if n == 0 {
                linebreak = false
            };
            if linebreak && (c.char_ == ' ') {
                linebreak = false;
                writeln!(f, " {}", c)?
            } else {
                write!(f, " {}", c)?
            }
        }
        writeln!(f)
    }
}
<commit_msg>loop over references to containers<commit_after>use std::fmt;

use crate::dictionary::CONF_LINE_WORD_MODIFIER__VISIBLE;
const LINE_WIDTH: usize = 20;

#[derive(Debug)]
struct HangmanChar {
    char_: char,
    visible: bool,
}

impl fmt::Display for HangmanChar {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        if self.visible {
            write!(f, "{}", self.char_)
        } else {
            write!(f, "_")
        }
    }
}

pub enum State {
    Ongoing,
    Victory,
    Defeat,
}

#[derive(Debug)]
pub struct Game {
    word: Vec<HangmanChar>,
    pub lives: u8,
    pub last_guess: char,
}

impl Game {
    pub fn get_state(&self) -> State {
        if self.lives == 0 {
            State::Defeat
        } else if self.word.iter().all(|c| c.visible) {
            State::Victory
        } else {
            State::Ongoing
        }
    }

    pub fn new(wordstr: &str, l: u8) -> Self {
        // parse wordsstr, filp 'visible' every CONF_LINE_WORD_MODIFIER__VISIBLE
        let w = wordstr
            .chars()
            // for every * found flip v_acc
            .scan(true, |v_acc, c| {
                *v_acc ^= c == CONF_LINE_WORD_MODIFIER__VISIBLE;
                if c == CONF_LINE_WORD_MODIFIER__VISIBLE {
                    Some(None)
                } else {
                    Some(Some(HangmanChar{char_: c, visible: *v_acc}))
                }
            })
            // ommit None and unwrap
            .filter_map(|s|s)
            //.inspect(|ref x| println!("after scan:\t{:?}", x))
            .collect();

        println!("{:?}",w);

        Self {
            word: w,
            lives: l,
            last_guess: ' ',
        }
    }

    pub fn guess(&mut self, char_: char) {
        if char_ == '\n' {
            return;
        };
        self.last_guess = char_;
        let mut found = false;
        for h_char in  &mut self.word {
            if h_char.char_.eq_ignore_ascii_case(&char_) {
                h_char.visible = true;
                found = true;
            }
        }

        if !found {
            self.lives -= 1;
        }

        if self.lives == 0 {
            for hc in &mut self.word {
                hc.visible = true;
            }
        }
    }

    pub fn visible_chars(&self) -> usize {
        self.word.iter().filter(|hc| !hc.visible).count()
    }
}

impl fmt::Display for Game {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        writeln!(
            f,
            "\x1b[KLives:\t{}\tLast guess: {}\n",
            self.lives, self.last_guess
        )?;

        let mut linebreak = false;
        for (n, c) in self.word.iter().enumerate() {
            if n % LINE_WIDTH == 0 {
                linebreak = true
            };
            if n == 0 {
                linebreak = false
            };
            if linebreak && (c.char_ == ' ') {
                linebreak = false;
                writeln!(f, " {}", c)?
            } else {
                write!(f, " {}", c)?
            }
        }
        writeln!(f)
    }
}
<|endoftext|>


<commit_before>//! "Diff"ing iterators for caching elements to sequential collections without requiring the new
//! elements' iterator to be `Clone`.
//!
//! - [**Diff**](./enum.Diff.html) (produced by the [**diff**](./fn.diff.html) function) describes
//! the difference between two non-`Clone` iterators `a` and `b` after breaking ASAP from a
//! comparison with enough data to update `a`'s collection.
//! - [**copy_on_diff**](./fn.copy_on_diff.html) is an application of [**diff**] that compares two
//! iterators `a` and `b`, borrowing the source of `a` if they are the same or creating a new owned
//! collection with `b`'s elements if they are different.

use adaptors::PutBack;
use std::borrow::{Cow, ToOwned};
use std::iter::FromIterator;

/// A type returned by the [`diff`](./fn.diff.html) function.
///
/// `Diff` represents the way in which the elements (of type `E`) yielded by the iterator `I`
/// differ to some other iterator yielding borrowed elements of the same type.
///
/// `I` is some `Iterator` yielding elements of type `E`.
pub enum Diff<I>
    where I: Iterator,
{
    /// The index of the first non-matching element along with the iterator's remaining elements
    /// starting with the first mis-matched element.
    FirstMismatch(usize, PutBack<I>),
    /// The remaining elements of the iterator.
    Longer(PutBack<I>),
    /// The total number of elements that were in the iterator.
    Shorter(usize),
}

/// Compares every element yielded by both elems and new_elems in lock-step and returns a `Diff`
/// which describes how `j` differs from `i`.
///
/// This function is useful for caching some iterator `j` in some sequential collection without
/// requiring `j` to be `Clone` in order to compare it to the collection before determining if the
/// collection needs to be updated. The returned function returns as soon as a difference is found,
/// producing a `Diff` that provides the data necessary to update the collection without ever
/// requiring `J` to be `Clone`. This allows for efficiently caching iterators like `Map` or
/// `Filter` that do not implement `Clone`.
///
/// If the number of elements yielded by `j` is less than the number of elements yielded by `i`,
/// the number of `j` elements yielded will be returned as `Diff::Shorter`.
///
/// If the two elements of a step differ, the index of those elements along with the remaining
/// elements of `j` are returned as `Diff::FirstMismatch`.
///
/// If `i` becomes exhausted before `j` becomes exhausted, the remaining `j` elements will be
/// returned as `Diff::Longer`.
///
/// See [`copy_on_diff`](./fn.copy_on_diff.html) for an application of `diff`.
pub fn diff<'a, I, J>(i: I, j: J) -> Option<Diff<J::IntoIter>>
    where I: IntoIterator<Item=&'a J::Item>,
          J: IntoIterator,
          J::Item: PartialEq + 'a,
{
    let mut j = j.into_iter();
    for (idx, i_elem) in i.into_iter().enumerate() {
        match j.next() {
            None => return Some(Diff::Shorter(idx)),
            Some(j_elem) => if *i_elem != j_elem {
                return Some(Diff::FirstMismatch(idx, PutBack::value(j_elem, j)));
            },
        }
    }
    j.next().map(|elem| Diff::Longer(PutBack::value(elem, j)))
}

/// Returns `Cow::Borrowed` `collection` if `collection` contains the same elements as yielded by
/// `update`'s iterator.
///
/// Collects into a new `C::Owned` and returns `Cow::Owned` if either the number of elements or the
/// elements themselves differ.
///
/// # Examples
///
/// ```
/// use itertools::copy_on_diff;
/// use std::borrow::Cow;
///
/// let a = vec![0, 1, 2];
/// let b = vec![0.0, 1.0, 2.0];
/// let b_map = b.into_iter().map(|f| f as i32);
/// let cow = copy_on_diff(&a, b_map);
///
/// assert!(match cow {
///     Cow::Borrowed(slice) => slice == &a,
///     _ => false,
/// });
/// ```
///
/// ```
/// use itertools::copy_on_diff;
/// use std::borrow::Cow;
///
/// let a = vec![0, 1, 2, 3];
/// let b = vec![0.0, 1.0, 2.0];
/// let b_map = b.into_iter().map(|f| f as i32);
/// let cow = copy_on_diff(&a, b_map);
///
/// assert!(match cow {
///     Cow::Owned(vec) => vec == vec![0, 1, 2],
///     _ => false,
/// });
/// ```
pub fn copy_on_diff<'a, C, U, T: 'a>(collection: &'a C, update: U) -> Cow<'a, C>
    where &'a C: IntoIterator<Item=&'a T>,
          <&'a C as IntoIterator>::IntoIter: Clone,
          C: ToOwned,
          <C as ToOwned>::Owned: FromIterator<T>,
          U: IntoIterator<Item=T>,
          T: Clone + PartialEq,
{
    let c_iter = collection.into_iter();
    match diff(c_iter.clone(), update.into_iter()) {
        Some(diff) => match diff {
            Diff::FirstMismatch(idx, mismatch) =>
                Cow::Owned(c_iter.take(idx).cloned().chain(mismatch).collect()),
            Diff::Longer(remaining) =>
                Cow::Owned(c_iter.cloned().chain(remaining).collect()),
            Diff::Shorter(num_update) =>
                Cow::Owned(c_iter.cloned().take(num_update).collect()),
        },
        None => Cow::Borrowed(collection),
    }
}
<commit_msg>Make Diff symmetric and adjust diff function accordingly. Add diff_by_ref function that retains original behaviour friendlier for iterator caching.<commit_after>//! "Diff"ing iterators for caching elements to sequential collections without requiring the new
//! elements' iterator to be `Clone`.
//!
//! - [**Diff**](./enum.Diff.html) (produced by the [**diff**](./fn.diff.html) function) describes
//! the difference between two non-`Clone` iterators `a` and `b` after breaking ASAP from a
//! comparison with enough data to update `a`'s collection.
//! - [**copy_on_diff**](./fn.copy_on_diff.html) is an application of [**diff**] that compares two
//! iterators `a` and `b`, borrowing the source of `a` if they are the same or creating a new owned
//! collection with `b`'s elements if they are different.

use adaptors::PutBack;
use std::borrow::{Cow, ToOwned};
use std::iter::FromIterator;

/// A type returned by the [`diff`](./fn.diff.html) function.
///
/// `Diff` represents the way in which the elements yielded by the iterator `I` differ to some
/// iterator `J`.
pub enum Diff<I, J>
    where I: Iterator,
          J: Iterator,
{
    /// The index of the first non-matching element along with the iterator's remaining elements
    /// starting with the first mis-matched element.
    FirstMismatch(usize, PutBack<I>, PutBack<J>),
    /// The total number of elements that were in the iterator.
    Shorter(usize, PutBack<I>),
    /// The remaining elements of the iterator.
    Longer(usize, PutBack<J>),
}

/// Compares every element yielded by both `i` and `j` in lock-step and returns a `Diff` which
/// describes how `j` differs from `i`.
///
/// If the number of elements yielded by `j` is less than the number of elements yielded by `i`,
/// the number of `j` elements yielded will be returned along with `i`'s remaining elements as
/// `Diff::Shorter`.
///
/// If the two elements of a step differ, the index of those elements along with the remaining
/// elements of both `i` and `j` are returned as `Diff::FirstMismatch`.
///
/// If `i` becomes exhausted before `j` becomes exhausted, the number of elements in `i` along with
/// the remaining `j` elements will be returned as `Diff::Longer`.
pub fn diff<I, J>(i: I, j: J) -> Option<Diff<I::IntoIter, J::IntoIter>>
    where I: IntoIterator,
          J: IntoIterator,
          I::Item: PartialEq<J::Item>,
{
    diff_internal(i, j, |ie, je| ie != je)
}

/// Similar to [`diff`](./fn.diff), however expects `i` to yield references to its elements.
///
/// This function is useful for caching some iterator `j` in some sequential collection without
/// requiring `j` to be `Clone` in order to compare it to the collection before determining if the
/// collection needs to be updated. The function returns as soon as a difference is found,
/// producing a `Diff` that provides the data necessary to update the collection without ever
/// requiring `J` to be `Clone`. This allows for efficiently caching iterators like `Map` or
/// `Filter` that do not implement `Clone`.
///
/// See [`copy_on_diff`](./fn.copy_on_diff.html) for an application of `diff`.
pub fn diff_by_ref<'a, I, J>(i: I, j: J) -> Option<Diff<I::IntoIter, J::IntoIter>>
    where I: IntoIterator<Item=&'a J::Item>,
          J: IntoIterator,
          J::Item: PartialEq + 'a,
{
    diff_internal(i, j, |ie, je| *ie != je)
}

// Compares every element yielded by both `i` and `j`  with the given `is_diff` function in
// lock-step.
//
// Returns a `Diff` which describes how `j` differs from `i`.
fn diff_internal<I, J, F>(i: I, j: J, is_diff: F) -> Option<Diff<I::IntoIter, J::IntoIter>>
    where I: IntoIterator,
          J: IntoIterator,
          F: Fn(&I::Item, &J::Item) -> bool,
{
    let mut i = i.into_iter();
    let mut j = j.into_iter();
    let mut idx = 0;
    while let Some(i_elem) = i.next() {
        match j.next() {
            None => return Some(Diff::Shorter(idx, PutBack::value(i_elem, i))),
            Some(j_elem) => if is_diff(&i_elem, &j_elem) {
                let remaining_i = PutBack::value(i_elem, i);
                let remaining_j = PutBack::value(j_elem, j);
                return Some(Diff::FirstMismatch(idx, remaining_i, remaining_j));
            },
        }
        idx += 1;
    }
    j.next().map(|j_elem| Diff::Longer(idx, PutBack::value(j_elem, j)))
}

/// Returns `Cow::Borrowed` `collection` if `collection` contains the same elements as yielded by
/// `update`'s iterator.
///
/// Collects into a new `C::Owned` and returns `Cow::Owned` if either the number of elements or the
/// elements themselves differ.
///
/// # Examples
///
/// ```
/// use itertools::copy_on_diff;
/// use std::borrow::Cow;
///
/// let a = vec![0, 1, 2];
/// let b = vec![0.0, 1.0, 2.0];
/// let b_map = b.into_iter().map(|f| f as i32);
/// let cow = copy_on_diff(&a, b_map);
///
/// assert!(match cow {
///     Cow::Borrowed(slice) => slice == &a,
///     _ => false,
/// });
/// ```
///
/// ```
/// use itertools::copy_on_diff;
/// use std::borrow::Cow;
///
/// let a = vec![0, 1, 2, 3];
/// let b = vec![0.0, 1.0, 2.0];
/// let b_map = b.into_iter().map(|f| f as i32);
/// let cow = copy_on_diff(&a, b_map);
///
/// assert!(match cow {
///     Cow::Owned(vec) => vec == vec![0, 1, 2],
///     _ => false,
/// });
/// ```
pub fn copy_on_diff<'a, C, U, T: 'a>(collection: &'a C, update: U) -> Cow<'a, C>
    where &'a C: IntoIterator<Item=&'a T>,
          <&'a C as IntoIterator>::IntoIter: Clone,
          C: ToOwned,
          <C as ToOwned>::Owned: FromIterator<T>,
          U: IntoIterator<Item=T>,
          T: Clone + PartialEq,
{
    let c_iter = collection.into_iter();
    match diff_by_ref(c_iter.clone(), update.into_iter()) {
        Some(diff) => match diff {
            Diff::FirstMismatch(idx, _, mismatch) =>
                Cow::Owned(c_iter.take(idx).cloned().chain(mismatch).collect()),
            Diff::Longer(_, remaining) =>
                Cow::Owned(c_iter.cloned().chain(remaining).collect()),
            Diff::Shorter(num_update, _) =>
                Cow::Owned(c_iter.cloned().take(num_update).collect()),
        },
        None => Cow::Borrowed(collection),
    }
}
<|endoftext|>


<commit_before>use std::sync::Arc;
use std::collections::HashMap;
use std::{fs, fmt, path};
use std::io::{self, Seek, SeekFrom, Write, Read};
use std::path::PathBuf;
use torrent::Info;
use util::hash_to_id;
use ring::digest;
use amy;
use {handle, CONFIG};

const POLL_INT_MS: usize = 1000;

pub struct Disk {
    poll: amy::Poller,
    ch: handle::Handle<Request, Response>,
    files: FileCache,
}

struct FileCache {
    files: HashMap<path::PathBuf, fs::File>,
}

pub enum Request {
    Write {
        tid: usize,
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        path: Option<String>,
    },
    Read {
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        context: Ctx,
        path: Option<String>,
    },
    Serialize {
        tid: usize,
        data: Vec<u8>,
        hash: [u8; 20],
    },
    Delete {
        tid: usize,
        hash: [u8; 20],
        files: Vec<PathBuf>,
        path: Option<String>,
    },

    Validate {
        tid: usize,
        info: Arc<Info>,
        path: Option<String>,
    },
    Shutdown,
}

pub struct Ctx {
    pub pid: usize,
    pub tid: usize,
    pub idx: u32,
    pub begin: u32,
    pub length: u32,
}

impl Ctx {
    pub fn new(pid: usize, tid: usize, idx: u32, begin: u32, length: u32) -> Ctx {
        Ctx {
            pid,
            tid,
            idx,
            begin,
            length,
        }
    }
}

impl FileCache {
    pub fn new() -> FileCache {
        FileCache { files: HashMap::new() }
    }

    pub fn get_file<F: FnOnce(&mut fs::File) -> io::Result<()>>(
        &mut self,
        path: &path::Path,
        f: F,
    ) -> io::Result<()> {
        if self.files.contains_key(path) {
            f(self.files.get_mut(path).unwrap())?;
        } else {
            // TODO: LRU maybe?
            if self.files.len() >= CONFIG.net.max_open_files {
                let removal = self.files.iter().map(|(id, _)| id.clone()).next().unwrap();
                self.files.remove(&removal);
            }
            fs::create_dir_all(path.parent().unwrap())?;
            let mut file = fs::OpenOptions::new()
                .write(true)
                .create(true)
                .read(true)
                .open(path)?;
            f(&mut file)?;
            self.files.insert(path.to_path_buf(), file);
        }
        Ok(())
    }

    pub fn remove_file(&mut self, path: &path::Path) {
        self.files.remove(path);
    }
}

impl Request {
    pub fn write(
        tid: usize,
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        path: Option<String>,
    ) -> Request {
        Request::Write {
            tid,
            data,
            locations,
            path,
        }
    }

    pub fn read(
        context: Ctx,
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        path: Option<String>,
    ) -> Request {
        Request::Read {
            context,
            data,
            locations,
            path,
        }
    }

    pub fn serialize(tid: usize, data: Vec<u8>, hash: [u8; 20]) -> Request {
        Request::Serialize { tid, data, hash }
    }

    pub fn validate(tid: usize, info: Arc<Info>, path: Option<String>) -> Request {
        Request::Validate { tid, info, path }
    }

    pub fn delete(
        tid: usize,
        hash: [u8; 20],
        files: Vec<PathBuf>,
        path: Option<String>,
    ) -> Request {
        Request::Delete {
            tid,
            hash,
            files,
            path,
        }
    }

    pub fn shutdown() -> Request {
        Request::Shutdown
    }

    fn execute(self, fc: &mut FileCache) -> io::Result<Option<Response>> {
        let sd = &CONFIG.disk.session;
        let dd = &CONFIG.disk.directory;
        match self {
            Request::Write {
                data,
                locations,
                path,
                ..
            } => {
                for loc in locations {
                    let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                    pb.push(&loc.file);
                    fc.get_file(&pb, |f| {
                        f.seek(SeekFrom::Start(loc.offset))?;
                        f.write_all(&data[loc.start..loc.end])?;
                        Ok(())
                    })?;
                }
            }
            Request::Read {
                context,
                mut data,
                locations,
                path,
                ..
            } => {
                for loc in locations {
                    let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                    pb.push(&loc.file);
                    fc.get_file(&pb, |f| {
                        f.seek(SeekFrom::Start(loc.offset))?;
                        f.read_exact(&mut data[loc.start..loc.end])?;
                        Ok(())
                    })?;
                }
                let data = Arc::new(data);
                return Ok(Some(Response::read(context, data)));
            }
            Request::Serialize { data, hash, .. } => {
                let mut pb = path::PathBuf::from(sd);
                pb.push(hash_to_id(&hash));
                let mut f = fs::OpenOptions::new().write(true).create(true).open(&pb)?;
                f.write_all(&data)?;
            }
            Request::Delete { hash, files, path, .. } => {
                let mut spb = path::PathBuf::from(sd);
                spb.push(hash_to_id(&hash));
                fs::remove_file(spb)?;

                for file in files {
                    let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                    pb.push(&file);
                    fc.remove_file(&pb);
                }
            }
            Request::Validate { tid, info, path } => {
                let mut invalid = Vec::new();
                let mut buf = vec![0u8; info.piece_len as usize];
                let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                let mut cf = pb.clone();

                let mut f = fs::OpenOptions::new().read(true).open(&pb);

                for i in 0..info.pieces() {
                    let mut valid = true;
                    let mut ctx = digest::Context::new(&digest::SHA1);
                    let locs = info.piece_disk_locs(i);
                    let mut pos = 0;
                    for loc in locs {
                        if loc.file != cf {
                            pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                            pb.push(&loc.file);
                            f = fs::OpenOptions::new().read(true).open(&pb);
                            cf = loc.file;
                        }
                        if let Ok(Ok(amnt)) = f.as_mut().map(|file| file.read(&mut buf[pos..])) {
                            ctx.update(&buf[pos..pos + amnt]);
                            pos += amnt;
                        } else {
                            valid = false;
                        }
                    }
                    let digest = ctx.finish();
                    if !valid || digest.as_ref() != &info.hashes[i as usize][..] {
                        invalid.push(i);
                    }
                }
                return Ok(Some(Response::validation_complete(tid, invalid)));
            }
            Request::Shutdown => unreachable!(),
        }
        Ok(None)
    }

    pub fn tid(&self) -> usize {
        match *self {
            Request::Serialize { tid, .. } |
            Request::Validate { tid, .. } |
            Request::Delete { tid, .. } |
            Request::Write { tid, .. } => tid,
            Request::Read { ref context, .. } => context.tid,
            Request::Shutdown => unreachable!(),
        }
    }
}

impl fmt::Debug for Request {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "disk::Request")
    }
}

pub struct Location {
    pub file: PathBuf,
    pub offset: u64,
    pub start: usize,
    pub end: usize,
}

impl Location {
    pub fn new(file: PathBuf, offset: u64, start: u64, end: u64) -> Location {
        Location {
            file,
            offset,
            start: start as usize,
            end: end as usize,
        }
    }
}

pub enum Response {
    Read {
        context: Ctx,
        data: Arc<Box<[u8; 16_384]>>,
    },
    ValidationComplete { tid: usize, invalid: Vec<u32> },
    Error { tid: usize, err: io::Error },
}

impl Response {
    pub fn read(context: Ctx, data: Arc<Box<[u8; 16_384]>>) -> Response {
        Response::Read { context, data }
    }

    pub fn error(tid: usize, err: io::Error) -> Response {
        Response::Error { tid, err }
    }

    pub fn validation_complete(tid: usize, invalid: Vec<u32>) -> Response {
        Response::ValidationComplete { tid, invalid }
    }

    pub fn tid(&self) -> usize {
        match *self {
            Response::Read { ref context, .. } => context.tid,
            Response::ValidationComplete { tid, .. } |
            Response::Error { tid, .. } => tid,
        }
    }
}

impl fmt::Debug for Response {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "disk::Response")
    }
}

impl Disk {
    pub fn new(poll: amy::Poller, ch: handle::Handle<Request, Response>) -> Disk {
        Disk {
            poll,
            ch,
            files: FileCache::new(),
        }
    }

    pub fn run(&mut self) {
        let sd = &CONFIG.disk.session;
        fs::create_dir_all(sd).unwrap();

        loop {
            match self.poll.wait(POLL_INT_MS) {
                Ok(_) => {
                    if self.handle_events() {
                        break;
                    }
                }
                Err(e) => {
                    error!("Failed to poll for events: {:?}", e);
                }
            }
        }

    }

    pub fn handle_events(&mut self) -> bool {
        loop {
            match self.ch.recv() {
                Ok(Request::Shutdown) => {
                    return true;
                }
                Ok(r) => {
                    trace!("Handling disk job!");
                    let tid = r.tid();
                    match r.execute(&mut self.files) {
                        Ok(Some(r)) => {
                            self.ch.send(r).ok();
                        }
                        Ok(None) => {}
                        Err(e) => {
                            self.ch.send(Response::error(tid, e)).ok();
                        }
                    }
                }
                _ => break,
            }
        }
        false
    }
}

pub fn start(creg: &mut amy::Registrar) -> io::Result<handle::Handle<Response, Request>> {
    let poll = amy::Poller::new()?;
    let mut reg = poll.get_registrar()?;
    let (ch, dh) = handle::Handle::new(creg, &mut reg)?;
    dh.run("disk", move |h| Disk::new(poll, h).run());
    Ok(ch)
}
<commit_msg>Make disk hash jobs interruptable<commit_after>use std::sync::Arc;
use std::collections::{HashMap, VecDeque};
use std::{fs, fmt, path};
use std::io::{self, Seek, SeekFrom, Write, Read};
use std::path::PathBuf;
use torrent::Info;
use util::hash_to_id;
use ring::digest;
use amy;
use {handle, CONFIG};

const POLL_INT_MS: usize = 1000;

pub struct Disk {
    poll: amy::Poller,
    ch: handle::Handle<Request, Response>,
    files: FileCache,
    active: VecDeque<Request>,
}

struct FileCache {
    files: HashMap<path::PathBuf, fs::File>,
}

pub enum Request {
    Write {
        tid: usize,
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        path: Option<String>,
    },
    Read {
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        context: Ctx,
        path: Option<String>,
    },
    Serialize {
        tid: usize,
        data: Vec<u8>,
        hash: [u8; 20],
    },
    Delete {
        tid: usize,
        hash: [u8; 20],
        files: Vec<PathBuf>,
        path: Option<String>,
    },

    Validate {
        tid: usize,
        info: Arc<Info>,
        path: Option<String>,
        idx: u32,
        invalid: Vec<u32>,
    },
    Shutdown,
}

pub struct Ctx {
    pub pid: usize,
    pub tid: usize,
    pub idx: u32,
    pub begin: u32,
    pub length: u32,
}

impl Ctx {
    pub fn new(pid: usize, tid: usize, idx: u32, begin: u32, length: u32) -> Ctx {
        Ctx {
            pid,
            tid,
            idx,
            begin,
            length,
        }
    }
}

impl FileCache {
    pub fn new() -> FileCache {
        FileCache { files: HashMap::new() }
    }

    pub fn get_file<F: FnOnce(&mut fs::File) -> io::Result<()>>(
        &mut self,
        path: &path::Path,
        f: F,
    ) -> io::Result<()> {
        if self.files.contains_key(path) {
            f(self.files.get_mut(path).unwrap())?;
        } else {
            // TODO: LRU maybe?
            if self.files.len() >= CONFIG.net.max_open_files {
                let removal = self.files.iter().map(|(id, _)| id.clone()).next().unwrap();
                self.files.remove(&removal);
            }
            fs::create_dir_all(path.parent().unwrap())?;
            let mut file = fs::OpenOptions::new()
                .write(true)
                .create(true)
                .read(true)
                .open(path)?;
            f(&mut file)?;
            self.files.insert(path.to_path_buf(), file);
        }
        Ok(())
    }

    pub fn remove_file(&mut self, path: &path::Path) {
        self.files.remove(path);
    }
}

impl Request {
    pub fn write(
        tid: usize,
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        path: Option<String>,
    ) -> Request {
        Request::Write {
            tid,
            data,
            locations,
            path,
        }
    }

    pub fn read(
        context: Ctx,
        data: Box<[u8; 16_384]>,
        locations: Vec<Location>,
        path: Option<String>,
    ) -> Request {
        Request::Read {
            context,
            data,
            locations,
            path,
        }
    }

    pub fn serialize(tid: usize, data: Vec<u8>, hash: [u8; 20]) -> Request {
        Request::Serialize { tid, data, hash }
    }

    pub fn validate(tid: usize, info: Arc<Info>, path: Option<String>) -> Request {
        Request::Validate {
            tid,
            info,
            path,
            idx: 0,
            invalid: Vec::new(),
        }
    }

    pub fn delete(
        tid: usize,
        hash: [u8; 20],
        files: Vec<PathBuf>,
        path: Option<String>,
    ) -> Request {
        Request::Delete {
            tid,
            hash,
            files,
            path,
        }
    }

    pub fn shutdown() -> Request {
        Request::Shutdown
    }

    fn execute(self, fc: &mut FileCache) -> io::Result<Result<Response, Option<Self>>> {
        let sd = &CONFIG.disk.session;
        let dd = &CONFIG.disk.directory;
        match self {
            Request::Write {
                data,
                locations,
                path,
                ..
            } => {
                for loc in locations {
                    let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                    pb.push(&loc.file);
                    fc.get_file(&pb, |f| {
                        f.seek(SeekFrom::Start(loc.offset))?;
                        f.write_all(&data[loc.start..loc.end])?;
                        Ok(())
                    })?;
                }
            }
            Request::Read {
                context,
                mut data,
                locations,
                path,
                ..
            } => {
                for loc in locations {
                    let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                    pb.push(&loc.file);
                    fc.get_file(&pb, |f| {
                        f.seek(SeekFrom::Start(loc.offset))?;
                        f.read_exact(&mut data[loc.start..loc.end])?;
                        Ok(())
                    })?;
                }
                let data = Arc::new(data);
                return Ok(Ok(Response::read(context, data)));
            }
            Request::Serialize { data, hash, .. } => {
                let mut pb = path::PathBuf::from(sd);
                pb.push(hash_to_id(&hash));
                let mut f = fs::OpenOptions::new().write(true).create(true).open(&pb)?;
                f.write_all(&data)?;
            }
            Request::Delete { hash, files, path, .. } => {
                let mut spb = path::PathBuf::from(sd);
                spb.push(hash_to_id(&hash));
                fs::remove_file(spb)?;

                for file in files {
                    let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                    pb.push(&file);
                    fc.remove_file(&pb);
                }
            }
            Request::Validate {
                tid,
                info,
                path,
                mut idx,
                mut invalid,
            } => {
                let mut buf = vec![0u8; info.piece_len as usize];
                let mut pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                let mut cf = pb.clone();

                let mut f = fs::OpenOptions::new().read(true).open(&pb);

                let mut max = 0;

                while idx < info.pieces() && max < 10 {
                    let mut valid = true;
                    let mut ctx = digest::Context::new(&digest::SHA1);
                    let locs = info.piece_disk_locs(idx);
                    let mut pos = 0;
                    for loc in locs {
                        if loc.file != cf {
                            pb = path::PathBuf::from(path.as_ref().unwrap_or(dd));
                            pb.push(&loc.file);
                            f = fs::OpenOptions::new().read(true).open(&pb);
                            cf = loc.file.clone();
                        }
                        // Because this is pausable/resumable, we need to seek to the proper
                        // file position.
                        f.as_mut()
                            .map(|file| file.seek(SeekFrom::Start(loc.offset)))
                            .ok();
                        if let Ok(Ok(amnt)) = f.as_mut().map(|file| file.read(&mut buf[pos..])) {
                            ctx.update(&buf[pos..pos + amnt]);
                            pos += amnt;
                        } else {
                            valid = false;
                        }
                    }
                    let digest = ctx.finish();
                    if !valid || digest.as_ref() != &info.hashes[idx as usize][..] {
                        invalid.push(idx);
                    }

                    max += 1;
                    idx += 1;
                }
                if idx == info.pieces() {
                    return Ok(Ok(Response::validation_complete(tid, invalid)));
                } else {
                    return Ok(Err(Some(Request::Validate {
                        tid,
                        info,
                        path,
                        idx,
                        invalid,
                    })));
                }
            }
            Request::Shutdown => unreachable!(),
        }
        Ok(Err(None))
    }

    pub fn tid(&self) -> usize {
        match *self {
            Request::Serialize { tid, .. } |
            Request::Validate { tid, .. } |
            Request::Delete { tid, .. } |
            Request::Write { tid, .. } => tid,
            Request::Read { ref context, .. } => context.tid,
            Request::Shutdown => unreachable!(),
        }
    }
}

impl fmt::Debug for Request {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "disk::Request")
    }
}

pub struct Location {
    pub file: PathBuf,
    pub offset: u64,
    pub start: usize,
    pub end: usize,
}

impl Location {
    pub fn new(file: PathBuf, offset: u64, start: u64, end: u64) -> Location {
        Location {
            file,
            offset,
            start: start as usize,
            end: end as usize,
        }
    }
}

pub enum Response {
    Read {
        context: Ctx,
        data: Arc<Box<[u8; 16_384]>>,
    },
    ValidationComplete { tid: usize, invalid: Vec<u32> },
    Error { tid: usize, err: io::Error },
}

impl Response {
    pub fn read(context: Ctx, data: Arc<Box<[u8; 16_384]>>) -> Response {
        Response::Read { context, data }
    }

    pub fn error(tid: usize, err: io::Error) -> Response {
        Response::Error { tid, err }
    }

    pub fn validation_complete(tid: usize, invalid: Vec<u32>) -> Response {
        Response::ValidationComplete { tid, invalid }
    }

    pub fn tid(&self) -> usize {
        match *self {
            Response::Read { ref context, .. } => context.tid,
            Response::ValidationComplete { tid, .. } |
            Response::Error { tid, .. } => tid,
        }
    }
}

impl fmt::Debug for Response {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "disk::Response")
    }
}

impl Disk {
    pub fn new(poll: amy::Poller, ch: handle::Handle<Request, Response>) -> Disk {
        Disk {
            poll,
            ch,
            files: FileCache::new(),
            active: VecDeque::new(),
        }
    }

    pub fn run(&mut self) {
        let sd = &CONFIG.disk.session;
        fs::create_dir_all(sd).unwrap();

        loop {
            match self.poll.wait(POLL_INT_MS) {
                Ok(_) => {
                    if self.handle_events() {
                        break;
                    }
                }
                Err(e) => {
                    error!("Failed to poll for events: {:?}", e);
                }
            }
            if !self.active.is_empty() {
                if self.handle_active() {
                    break;
                }
            }
        }
    }

    fn handle_active(&mut self) -> bool {
        while let Some(j) = self.active.pop_front() {
            let tid = j.tid();
            match j.execute(&mut self.files) {
                Ok(Ok(r)) => {
                    self.ch.send(r).ok();
                }
                Ok(Err(Some(s))) => {
                    self.active.push_front(s);
                }
                Ok(Err(None)) => {}
                Err(e) => {
                    self.ch.send(Response::error(tid, e)).ok();
                }
            }
            match self.poll.wait(10) {
                Ok(_) => {
                    if self.handle_events() {
                        return true;
                    }
                }
                Err(e) => {
                    error!("Failed to poll for events: {:?}", e);
                }
            }
        }
        false
    }

    pub fn handle_events(&mut self) -> bool {
        loop {
            match self.ch.recv() {
                Ok(Request::Shutdown) => {
                    return true;
                }
                Ok(r) => {
                    trace!("Handling disk job!");
                    let tid = r.tid();
                    match r.execute(&mut self.files) {
                        Ok(Ok(r)) => {
                            self.ch.send(r).ok();
                        }
                        Ok(Err(Some(s))) => {
                            self.active.push_back(s);
                        }
                        Ok(Err(None)) => {}
                        Err(e) => {
                            self.ch.send(Response::error(tid, e)).ok();
                        }
                    }
                }
                _ => break,
            }
        }
        false
    }
}

pub fn start(creg: &mut amy::Registrar) -> io::Result<handle::Handle<Response, Request>> {
    let poll = amy::Poller::new()?;
    let mut reg = poll.get_registrar()?;
    let (ch, dh) = handle::Handle::new(creg, &mut reg)?;
    dh.run("disk", move |h| Disk::new(poll, h).run());
    Ok(ch)
}
<|endoftext|>


<commit_before>extern crate tempfile;
extern crate zmq;

use std::collections::HashMap;
use std::io::Read;
use std::sync::Arc;
use std::sync::Mutex;
use time;

use backend;
use dispatcher::server;
use helpers;
use helpers::{NewTaskMessage, TaskProgress, TaskReport, TaskStatus};
use models::Service;
use std::error::Error;
use zmq::SNDMORE;

/// Specifies the binding and operation parameters for a ZMQ ventilator component
pub struct Ventilator {
  /// port to listen on
  pub port: usize,
  /// the size of the dispatch queue
  /// (also the batch size for Task store queue requests)
  pub queue_size: usize,
  /// size of an individual message chunk sent via zeromq
  /// (keep this small to avoid large RAM use, increase to reduce network bandwidth)
  pub message_size: usize,
  /// address for the Task store postgres endpoint
  pub backend_address: String,
}

impl Ventilator {
  /// Starts a new dispatch `Server` (ZMQ Ventilator), to serve tasks to processing workers.
  /// The ventilator shares state with other manager threads via queues for tasks in progress,
  /// as well as a queue for completed tasks pending persisting to disk.
  /// A job limit can be provided as a termination condition for the sink server.
  pub fn start(
    &self,
    services_arc: Arc<Mutex<HashMap<String, Option<Service>>>>,
    progress_queue_arc: Arc<Mutex<HashMap<i64, TaskProgress>>>,
    done_queue_arc: Arc<Mutex<Vec<TaskReport>>>,
    job_limit: Option<usize>,
  ) -> Result<(), Box<Error>>
  {
    // We have a Ventilator-exclusive "queues" stack for tasks to be dispatched
    let mut queues: HashMap<String, Vec<TaskProgress>> = HashMap::new();
    // Assuming this is the only And tidy up the postgres tasks:
    let backend = backend::from_address(&self.backend_address);
    backend.clear_limbo_tasks()?;
    // Ok, let's bind to a port and start broadcasting
    let context = zmq::Context::new();
    let ventilator = context.socket(zmq::ROUTER)?;
    let port_str = self.port.to_string();
    let address = "tcp://*:".to_string() + &port_str;
    assert!(ventilator.bind(&address).is_ok());
    let mut source_job_count: usize = 0;

    loop {
      let mut msg = zmq::Message::new()?;
      let mut identity = zmq::Message::new()?;
      ventilator.recv(&mut identity, 0)?;
      ventilator.recv(&mut msg, 0)?;
      let service_name = msg.as_str().unwrap_or_default().to_string();
      // println!("Task requested for service: {}", service_name.clone());
      let request_time = time::get_time();
      source_job_count += 1;

      let mut dispatched_task_opt: Option<TaskProgress> = None;
      // Requests for unknown service names will be silently ignored.
      if let Some(service) = server::get_sync_service(&service_name, &services_arc, &backend) {
        if !queues.contains_key(&service_name) {
          queues.insert(service_name.clone(), Vec::new());
        }
        let mut task_queue: &mut Vec<TaskProgress> = queues
          .get_mut(&service_name)
          .unwrap_or_else(|| panic!("Could not obtain queue mutex lock in main ventilator loop"));
        if task_queue.is_empty() {
          println!(
            "No tasks in task queue for service {:?}, fetching up to {:?} more from backend...",
            service_name, self.queue_size
          );
          // Refetch a new batch of tasks
          let now = time::get_time().sec;
          let fetched_tasks = backend
            .fetch_tasks(&service, self.queue_size)
            .unwrap_or(Vec::new());
          task_queue.extend(fetched_tasks.into_iter().map(|task| TaskProgress {
            task,
            created_at: now,
            retries: 0,
          }));

          // This is a good time to also take care that none of the old tasks are dead in the
          // progress queue since the re-fetch happens infrequently, and directly
          // implies the progress queue will grow
          let expired_tasks = server::timeout_progress_tasks(&progress_queue_arc);
          for expired_t in expired_tasks {
            if expired_t.retries > 1 {
              // Too many retries, mark as fatal failure
              server::push_done_queue(
                &done_queue_arc,
                TaskReport {
                  task: expired_t.task.clone(),
                  status: TaskStatus::Fatal,
                  messages: vec![NewTaskMessage::new(
                    expired_t.task.id,
                    "fatal",
                    "cortex".to_string(),
                    "never_completed_with_retries".to_string(),
                    String::new(),
                  )],
                },
              );
            } else {
              // We can still retry, re-add to the dispatch queue
              task_queue.push(TaskProgress {
                task: expired_t.task,
                created_at: expired_t.created_at,
                retries: expired_t.retries + 1,
              });
            }
          }
        }
        ventilator.send_msg(identity, SNDMORE)?;
        if let Some(current_task_progress) = task_queue.pop() {
          dispatched_task_opt = Some(current_task_progress.clone());

          let current_task = current_task_progress.task;
          let taskid = current_task.id;
          let serviceid = current_task.service_id;

          ventilator.send_str(&taskid.to_string(), SNDMORE)?;
          if serviceid == 1 {
            // No payload needed for init
            ventilator.send(&[], 0)?;
          } else {
            // Regular services fetch the task payload and transfer it to the worker
            let file_opt = helpers::prepare_input_stream(&current_task);
            if file_opt.is_ok() {
              let mut file = file_opt?;
              let mut total_outgoing: usize = 0;
              loop {
                // Stream input data via zmq
                let mut data = vec![0; self.message_size];
                let size = file.read(&mut data)?;
                total_outgoing += size;
                data.truncate(size);

                if size < self.message_size {
                  // If exhausted, send the last frame
                  ventilator.send(&data, 0)?;
                  // And terminate
                  break;
                } else {
                  // If more to go, send the frame and indicate there's more to come
                  ventilator.send(&data, SNDMORE)?;
                }
              }
              let responded_time = time::get_time();
              let request_duration = (responded_time - request_time).num_milliseconds();
              println!(
                "Source job {}, message size: {}, took {}ms.",
                source_job_count, total_outgoing, request_duration
              );
            } else {
              ventilator.send(&[], 0)?;
            }
          }
        } else {
          println!("No jobs in queue, sent mock task reply.");
          ventilator.send_str("0", SNDMORE)?;
          ventilator.send(&[], 0)?;
        }
      }
      // Record that a task has been dispatched in the progress queue
      if let Some(dispatched_task) = dispatched_task_opt {
        server::push_progress_task(&progress_queue_arc, dispatched_task);
      }
      if let Some(limit_number) = job_limit {
        if source_job_count >= limit_number {
          println!(
            "Manager job limit of {:?} reached, terminating Ventilator thread...",
            limit_number
          );
          break;
        }
      }
    }
    Ok(())
  }
}
<commit_msg>more verbose logging in ventilator logic<commit_after>extern crate tempfile;
extern crate zmq;

use std::collections::HashMap;
use std::io::Read;
use std::sync::Arc;
use std::sync::Mutex;
use time;

use backend;
use dispatcher::server;
use helpers;
use helpers::{NewTaskMessage, TaskProgress, TaskReport, TaskStatus};
use models::Service;
use std::error::Error;
use zmq::SNDMORE;

/// Specifies the binding and operation parameters for a ZMQ ventilator component
pub struct Ventilator {
  /// port to listen on
  pub port: usize,
  /// the size of the dispatch queue
  /// (also the batch size for Task store queue requests)
  pub queue_size: usize,
  /// size of an individual message chunk sent via zeromq
  /// (keep this small to avoid large RAM use, increase to reduce network bandwidth)
  pub message_size: usize,
  /// address for the Task store postgres endpoint
  pub backend_address: String,
}

impl Ventilator {
  /// Starts a new dispatch `Server` (ZMQ Ventilator), to serve tasks to processing workers.
  /// The ventilator shares state with other manager threads via queues for tasks in progress,
  /// as well as a queue for completed tasks pending persisting to disk.
  /// A job limit can be provided as a termination condition for the sink server.
  pub fn start(
    &self,
    services_arc: Arc<Mutex<HashMap<String, Option<Service>>>>,
    progress_queue_arc: Arc<Mutex<HashMap<i64, TaskProgress>>>,
    done_queue_arc: Arc<Mutex<Vec<TaskReport>>>,
    job_limit: Option<usize>,
  ) -> Result<(), Box<Error>>
  {
    // We have a Ventilator-exclusive "queues" stack for tasks to be dispatched
    let mut queues: HashMap<String, Vec<TaskProgress>> = HashMap::new();
    // Assuming this is the only And tidy up the postgres tasks:
    let backend = backend::from_address(&self.backend_address);
    backend.clear_limbo_tasks()?;
    // Ok, let's bind to a port and start broadcasting
    let context = zmq::Context::new();
    let ventilator = context.socket(zmq::ROUTER)?;
    let port_str = self.port.to_string();
    let address = "tcp://*:".to_string() + &port_str;
    assert!(ventilator.bind(&address).is_ok());
    let mut source_job_count: usize = 0;

    loop {
      let mut msg = zmq::Message::new()?;
      let mut identity = zmq::Message::new()?;
      ventilator.recv(&mut identity, 0)?;
      ventilator.recv(&mut msg, 0)?;
      let service_name = msg.as_str().unwrap_or_default().to_string();
      // println!("Task requested for service: {}", service_name.clone());
      let request_time = time::get_time();
      source_job_count += 1;

      let mut dispatched_task_opt: Option<TaskProgress> = None;
      // Requests for unknown service names will be silently ignored.
      if let Some(service) = server::get_sync_service(&service_name, &services_arc, &backend) {
        if !queues.contains_key(&service_name) {
          queues.insert(service_name.clone(), Vec::new());
        }
        let mut task_queue: &mut Vec<TaskProgress> = queues
          .get_mut(&service_name)
          .unwrap_or_else(|| panic!("Could not obtain queue mutex lock in main ventilator loop"));
        if task_queue.is_empty() {
          println!(
            "-- No tasks in task queue for service {:?}, fetching up to {:?} more from backend...",
            service_name, self.queue_size
          );
          // Refetch a new batch of tasks
          let now = time::get_time().sec;
          let fetched_tasks = backend
            .fetch_tasks(&service, self.queue_size)
            .unwrap_or(Vec::new());
          task_queue.extend(fetched_tasks.into_iter().map(|task| TaskProgress {
            task,
            created_at: now,
            retries: 0,
          }));

          // This is a good time to also take care that none of the old tasks are dead in the
          // progress queue since the re-fetch happens infrequently, and directly
          // implies the progress queue will grow
          let expired_tasks = server::timeout_progress_tasks(&progress_queue_arc);
          for expired_t in expired_tasks {
            if expired_t.retries > 1 {
              // Too many retries, mark as fatal failure
              server::push_done_queue(
                &done_queue_arc,
                TaskReport {
                  task: expired_t.task.clone(),
                  status: TaskStatus::Fatal,
                  messages: vec![NewTaskMessage::new(
                    expired_t.task.id,
                    "fatal",
                    "cortex".to_string(),
                    "never_completed_with_retries".to_string(),
                    String::new(),
                  )],
                },
              );
            } else {
              // We can still retry, re-add to the dispatch queue
              task_queue.push(TaskProgress {
                task: expired_t.task,
                created_at: expired_t.created_at,
                retries: expired_t.retries + 1,
              });
            }
          }
        }
        ventilator.send_msg(identity, SNDMORE)?;
        if let Some(current_task_progress) = task_queue.pop() {
          dispatched_task_opt = Some(current_task_progress.clone());

          let current_task = current_task_progress.task;
          let taskid = current_task.id;
          let serviceid = current_task.service_id;
          println!(
            "Replying for source job {:?} with task id {:?}",
            source_job_count, taskid,
          );
          ventilator.send_str(&taskid.to_string(), SNDMORE)?;
          if serviceid == 1 {
            // No payload needed for init
            ventilator.send(&[], 0)?;
          } else {
            // Regular services fetch the task payload and transfer it to the worker
            let file_opt = helpers::prepare_input_stream(&current_task);
            if file_opt.is_ok() {
              let mut file = file_opt?;
              let mut total_outgoing: usize = 0;
              loop {
                // Stream input data via zmq
                let mut data = vec![0; self.message_size];
                let size = file.read(&mut data)?;
                total_outgoing += size;
                data.truncate(size);

                if size < self.message_size {
                  // If exhausted, send the last frame
                  ventilator.send(&data, 0)?;
                  // And terminate
                  break;
                } else {
                  // If more to go, send the frame and indicate there's more to come
                  ventilator.send(&data, SNDMORE)?;
                }
              }
              let responded_time = time::get_time();
              let request_duration = (responded_time - request_time).num_milliseconds();
              println!(
                "Source job {}, message size: {}, took {}ms.",
                source_job_count, total_outgoing, request_duration
              );
            } else {
              println!("-- Failed to prepare input stream for taskid {:?}", taskid);
              ventilator.send(&[], 0)?;
            }
          }
        } else {
          println!("-- No jobs in queue, sent mock task reply.");
          ventilator.send_str("0", SNDMORE)?;
          ventilator.send(&[], 0)?;
        }
      } else {
        println!(
          "-- No such service in ventilator request: {:?}",
          service_name
        );
      }
      // Record that a task has been dispatched in the progress queue
      if let Some(dispatched_task) = dispatched_task_opt {
        server::push_progress_task(&progress_queue_arc, dispatched_task);
      }
      if let Some(limit_number) = job_limit {
        if source_job_count >= limit_number {
          println!(
            "Manager job limit of {:?} reached, terminating Ventilator thread...",
            limit_number
          );
          break;
        }
      }
    }
    Ok(())
  }
}
<|endoftext|>


<commit_before>use crate::value::AMQPValue;

use std::{
    collections::{BTreeMap, btree_map},
    borrow, fmt, str,
};

use serde::{Deserialize, Serialize};

/// Enumeration referencing all the available AMQP types
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub enum AMQPType {
    /// A bool
    Boolean,
    /// An i8
    ShortShortInt,
    /// A u8
    ShortShortUInt,
    /// An i16
    ShortInt,
    /// A u16
    ShortUInt,
    /// An i32
    LongInt,
    /// A u32
    LongUInt,
    /// An i64
    LongLongInt,
    /// A u64
    LongLongUInt,
    /// An f32
    Float,
    /// An f64
    Double,
    /// A decimal value represented by a scale and a value
    DecimalValue,
    /// Deprecated, a String
    ShortString,
    /// A String
    LongString,
    /// An array of AMQPValue
    FieldArray,
    /// A timestamp (u32)
    Timestamp,
    /// A Map<String, AMQPValue>
    FieldTable,
    /// An array of bytes, RabbitMQ specific
    ByteArray, /* ByteArray is specific to RabbitMQ */
    /// No value
    Void,
}

impl AMQPType {
    /// Get the AMQPType corresponding to the given id.
    /// We don't strictly follow the spec here but rather the RabbitMQ implementation
    /// 's' means ShortInt (like 'U') instead of ShortString
    /// 'l' and 'L' both mean LongLongInt (no LongLongUInt)
    pub fn from_id(id: char) -> Option<AMQPType> {
        match id {
            't' => Some(AMQPType::Boolean),
            'b' => Some(AMQPType::ShortShortInt),
            'B' => Some(AMQPType::ShortShortUInt),
            /* Specs says 'U', RabbitMQ says 's' (which means ShortString in specs) */
            's' |
            'U' => Some(AMQPType::ShortInt),
            'u' => Some(AMQPType::ShortUInt),
            'I' => Some(AMQPType::LongInt),
            'i' => Some(AMQPType::LongUInt),
            /* RabbitMQ treats both 'l' and 'L' as LongLongInt and ignores LongLongUInt */
            'L' |
            'l' => Some(AMQPType::LongLongInt),
            'f' => Some(AMQPType::Float),
            'd' => Some(AMQPType::Double),
            'D' => Some(AMQPType::DecimalValue),
            'S' => Some(AMQPType::LongString),
            'A' => Some(AMQPType::FieldArray),
            'T' => Some(AMQPType::Timestamp),
            'F' => Some(AMQPType::FieldTable),
            'x' => Some(AMQPType::ByteArray),
            'V' => Some(AMQPType::Void),
            _   => None,
        }
    }

    /// Get the id from an AMQPType
    /// We don't strictly follow the spec here but rather the RabbitMQ implementation
    /// ShortString doesn't have an id, we return '_' instead
    /// ShortInt is supposed to be 'U' but we use 's'
    /// LongLongUInt is supposed to be 'L' but we return 'l' as LongLongInt
    pub fn get_id(self) -> char {
        match self {
            AMQPType::Boolean        => 't',
            AMQPType::ShortShortInt  => 'b',
            AMQPType::ShortShortUInt => 'B',
            /* Specs says 'U', RabbitMQ says 's' (which means ShortString in specs) */
            AMQPType::ShortInt       => 's',
            AMQPType::ShortUInt      => 'u',
            AMQPType::LongInt        => 'I',
            AMQPType::LongUInt       => 'i',
            /* RabbitMQ treats both 'l' and 'L' as LongLongInt and ignores LongLongUInt */
            AMQPType::LongLongInt    |
            AMQPType::LongLongUInt   => 'l',
            AMQPType::Float          => 'f',
            AMQPType::Double         => 'd',
            AMQPType::DecimalValue   => 'D',
            /* ShortString only exists for internal usage, we shouldn't ever have to use this */
            AMQPType::ShortString    => '_',
            AMQPType::LongString     => 'S',
            AMQPType::FieldArray     => 'A',
            AMQPType::Timestamp      => 'T',
            AMQPType::FieldTable     => 'F',
            AMQPType::ByteArray      => 'x',
            AMQPType::Void           => 'V',
        }
    }
}

impl fmt::Display for AMQPType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}

/// A bool
pub type Boolean        = bool;
/// An i8
pub type ShortShortInt  = i8;
/// A u8
pub type ShortShortUInt = u8;
/// An i16
pub type ShortInt       = i16;
/// A u16
pub type ShortUInt      = u16;
/// An i32
pub type LongInt        = i32;
/// A u32
pub type LongUInt       = u32;
/// An i64
pub type LongLongInt    = i64;
/// A u64
pub type LongLongUInt   = u64;
/// A f32
pub type Float          = f32;
/// A f64
pub type Double         = f64;
/// A timestamp (u32)
pub type Timestamp      = LongLongUInt;
/// No value
pub type Void           = ();

/// A String (deprecated)
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub struct ShortString(String);
/// A String
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub struct LongString(String);
/// An array of AMQPValue
#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
pub struct FieldArray(Vec<AMQPValue>);
/// A Map<String, AMQPValue>
#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
pub struct FieldTable(BTreeMap<ShortString, AMQPValue>);
/// An array of bytes (RabbitMQ specific)
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub struct ByteArray(Vec<u8>);

/// A Decimal value composed of a scale and a value
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub struct DecimalValue {
    /// The scale of the value
    pub scale: ShortShortUInt,
    /// The actual value
    pub value: LongUInt,
}

/// A Reference pointing to a ShortString
pub struct ShortStringRef<'a>(pub &'a str);
/// A Reference pointing to a LongString
pub struct LongStringRef<'a>(pub &'a str);

impl<'a> ShortString {
    /// Get a reference to a ShortString
    pub fn as_ref(&'a self) -> ShortStringRef<'a> {
        ShortStringRef(&self.0)
    }

    /// Get a reference to a LongString as &str
    pub fn as_str(&'a self) -> &'a str {
        self.0.as_str()
    }

    /// Splits a string slice by whitespace.
    pub fn split_whitespace(&'a self) -> str::SplitWhitespace<'a> {
        self.0.split_whitespace()
    }
}

impl From<String> for ShortString {
    fn from(s: String) -> Self {
        Self(s)
    }
}

impl From<&str> for ShortString {
    fn from(s: &str) -> Self {
        s.to_owned().into()
    }
}

impl borrow::Borrow<str> for ShortString {
    fn borrow(&self) -> &str {
        self.0.borrow()
    }
}

impl fmt::Display for ShortString {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl<'a> ShortStringRef<'a> {
    pub(crate) fn as_bytes(&self) -> &[u8] {
        self.0.as_bytes()
    }

    pub(crate) fn len(&self) -> usize {
        self.0.len()
    }
}

impl Default for ShortStringRef<'static> {
    fn default() -> Self {
        Self("")
    }
}

impl<'a> From<&'a str> for ShortStringRef<'a> {
    fn from(s: &'a str) -> Self {
        Self(s)
    }
}

impl<'a> LongString {
    /// Get a reference to a LongString
    pub fn as_ref(&'a self) -> LongStringRef<'a> {
        LongStringRef(&self.0)
    }

    /// Get a reference to a LongString as &str
    pub fn as_str(&'a self) -> &'a str {
        self.0.as_str()
    }

    /// Splits a string slice by whitespace.
    pub fn split_whitespace(&'a self) -> str::SplitWhitespace<'a> {
        self.0.split_whitespace()
    }
}

impl From<String> for LongString {
    fn from(s: String) -> Self {
        Self(s)
    }
}

impl From<&str> for LongString {
    fn from(s: &str) -> Self {
        s.to_owned().into()
    }
}

impl borrow::Borrow<str> for LongString {
    fn borrow(&self) -> &str {
        self.0.borrow()
    }
}

impl fmt::Display for LongString {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl<'a> LongStringRef<'a> {
    pub(crate) fn as_bytes(&self) -> &[u8] {
        self.0.as_bytes()
    }

    pub(crate) fn len(&self) -> usize {
        self.0.len()
    }
}

impl Default for LongStringRef<'static> {
    fn default() -> Self {
        Self("")
    }
}

impl<'a> From<&'a str> for LongStringRef<'a> {
    fn from(s: &'a str) -> Self {
        Self(s)
    }
}

impl FieldArray {
    pub(crate) fn as_slice(&self) -> &[AMQPValue] {
        self.0.as_slice()
    }

    pub(crate) fn push(&mut self, v: AMQPValue) {
        self.0.push(v);
    }
}

impl From<Vec<AMQPValue>> for FieldArray {
    fn from(v: Vec<AMQPValue>) -> Self {
        Self(v)
    }
}

impl FieldTable {
    /// Insert a new entry in the table
    pub fn insert(&mut self, k: ShortString, v: AMQPValue) {
        self.0.insert(k, v);
    }

    /// Check whether the table contains the given key
    pub fn contains_key(&self, k: &str) -> bool {
        self.0.contains_key(k)
    }
}

impl<'a> IntoIterator for &'a FieldTable {
    type Item = (&'a ShortString, &'a AMQPValue);
    type IntoIter = btree_map::Iter<'a, ShortString, AMQPValue>;

    fn into_iter(self) -> Self::IntoIter {
        self.0.iter()
    }
}

impl From<BTreeMap<ShortString, AMQPValue>> for FieldTable {
    fn from(m: BTreeMap<ShortString, AMQPValue>) -> Self {
        Self(m)
    }
}

impl ByteArray {
    pub(crate) fn as_slice(&self) -> &[u8] {
        self.0.as_slice()
    }

    pub(crate) fn len(&self) -> usize {
        self.0.len()
    }
}

impl From<Vec<u8>> for ByteArray {
    fn from(v: Vec<u8>) -> Self {
        Self(v)
    }
}

impl From<&[u8]> for ByteArray {
    fn from(v: &[u8]) -> Self {
        Self(v.to_vec())
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_type_from_id() {
        assert_eq!(AMQPType::from_id('T'), Some(AMQPType::Timestamp));
        assert_eq!(AMQPType::from_id('S'), Some(AMQPType::LongString));
        assert_eq!(AMQPType::from_id('s'), Some(AMQPType::ShortInt));
        assert_eq!(AMQPType::from_id('U'), Some(AMQPType::ShortInt));
        assert_eq!(AMQPType::from_id('l'), Some(AMQPType::LongLongInt));
        assert_eq!(AMQPType::from_id('z'), None);
    }

    #[test]
    fn test_type_get_id() {
        assert_eq!(AMQPType::LongLongInt.get_id(),  'l');
        assert_eq!(AMQPType::LongLongUInt.get_id(), 'l');
        assert_eq!(AMQPType::ShortString.get_id(),  '_');
    }

    #[test]
    fn test_type_to_string() {
        assert_eq!(AMQPType::Boolean.to_string(), "Boolean");
        assert_eq!(AMQPType::Void.to_string(),    "Void");
    }
}
<commit_msg>types: add FieldTable::inner<commit_after>use crate::value::AMQPValue;

use std::{
    collections::{BTreeMap, btree_map},
    borrow, fmt, str,
};

use serde::{Deserialize, Serialize};

/// Enumeration referencing all the available AMQP types
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub enum AMQPType {
    /// A bool
    Boolean,
    /// An i8
    ShortShortInt,
    /// A u8
    ShortShortUInt,
    /// An i16
    ShortInt,
    /// A u16
    ShortUInt,
    /// An i32
    LongInt,
    /// A u32
    LongUInt,
    /// An i64
    LongLongInt,
    /// A u64
    LongLongUInt,
    /// An f32
    Float,
    /// An f64
    Double,
    /// A decimal value represented by a scale and a value
    DecimalValue,
    /// Deprecated, a String
    ShortString,
    /// A String
    LongString,
    /// An array of AMQPValue
    FieldArray,
    /// A timestamp (u32)
    Timestamp,
    /// A Map<String, AMQPValue>
    FieldTable,
    /// An array of bytes, RabbitMQ specific
    ByteArray, /* ByteArray is specific to RabbitMQ */
    /// No value
    Void,
}

impl AMQPType {
    /// Get the AMQPType corresponding to the given id.
    /// We don't strictly follow the spec here but rather the RabbitMQ implementation
    /// 's' means ShortInt (like 'U') instead of ShortString
    /// 'l' and 'L' both mean LongLongInt (no LongLongUInt)
    pub fn from_id(id: char) -> Option<AMQPType> {
        match id {
            't' => Some(AMQPType::Boolean),
            'b' => Some(AMQPType::ShortShortInt),
            'B' => Some(AMQPType::ShortShortUInt),
            /* Specs says 'U', RabbitMQ says 's' (which means ShortString in specs) */
            's' |
            'U' => Some(AMQPType::ShortInt),
            'u' => Some(AMQPType::ShortUInt),
            'I' => Some(AMQPType::LongInt),
            'i' => Some(AMQPType::LongUInt),
            /* RabbitMQ treats both 'l' and 'L' as LongLongInt and ignores LongLongUInt */
            'L' |
            'l' => Some(AMQPType::LongLongInt),
            'f' => Some(AMQPType::Float),
            'd' => Some(AMQPType::Double),
            'D' => Some(AMQPType::DecimalValue),
            'S' => Some(AMQPType::LongString),
            'A' => Some(AMQPType::FieldArray),
            'T' => Some(AMQPType::Timestamp),
            'F' => Some(AMQPType::FieldTable),
            'x' => Some(AMQPType::ByteArray),
            'V' => Some(AMQPType::Void),
            _   => None,
        }
    }

    /// Get the id from an AMQPType
    /// We don't strictly follow the spec here but rather the RabbitMQ implementation
    /// ShortString doesn't have an id, we return '_' instead
    /// ShortInt is supposed to be 'U' but we use 's'
    /// LongLongUInt is supposed to be 'L' but we return 'l' as LongLongInt
    pub fn get_id(self) -> char {
        match self {
            AMQPType::Boolean        => 't',
            AMQPType::ShortShortInt  => 'b',
            AMQPType::ShortShortUInt => 'B',
            /* Specs says 'U', RabbitMQ says 's' (which means ShortString in specs) */
            AMQPType::ShortInt       => 's',
            AMQPType::ShortUInt      => 'u',
            AMQPType::LongInt        => 'I',
            AMQPType::LongUInt       => 'i',
            /* RabbitMQ treats both 'l' and 'L' as LongLongInt and ignores LongLongUInt */
            AMQPType::LongLongInt    |
            AMQPType::LongLongUInt   => 'l',
            AMQPType::Float          => 'f',
            AMQPType::Double         => 'd',
            AMQPType::DecimalValue   => 'D',
            /* ShortString only exists for internal usage, we shouldn't ever have to use this */
            AMQPType::ShortString    => '_',
            AMQPType::LongString     => 'S',
            AMQPType::FieldArray     => 'A',
            AMQPType::Timestamp      => 'T',
            AMQPType::FieldTable     => 'F',
            AMQPType::ByteArray      => 'x',
            AMQPType::Void           => 'V',
        }
    }
}

impl fmt::Display for AMQPType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}

/// A bool
pub type Boolean        = bool;
/// An i8
pub type ShortShortInt  = i8;
/// A u8
pub type ShortShortUInt = u8;
/// An i16
pub type ShortInt       = i16;
/// A u16
pub type ShortUInt      = u16;
/// An i32
pub type LongInt        = i32;
/// A u32
pub type LongUInt       = u32;
/// An i64
pub type LongLongInt    = i64;
/// A u64
pub type LongLongUInt   = u64;
/// A f32
pub type Float          = f32;
/// A f64
pub type Double         = f64;
/// A timestamp (u32)
pub type Timestamp      = LongLongUInt;
/// No value
pub type Void           = ();

/// A String (deprecated)
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub struct ShortString(String);
/// A String
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub struct LongString(String);
/// An array of AMQPValue
#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
pub struct FieldArray(Vec<AMQPValue>);
/// A Map<String, AMQPValue>
#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
pub struct FieldTable(BTreeMap<ShortString, AMQPValue>);
/// An array of bytes (RabbitMQ specific)
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub struct ByteArray(Vec<u8>);

/// A Decimal value composed of a scale and a value
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub struct DecimalValue {
    /// The scale of the value
    pub scale: ShortShortUInt,
    /// The actual value
    pub value: LongUInt,
}

/// A Reference pointing to a ShortString
pub struct ShortStringRef<'a>(pub &'a str);
/// A Reference pointing to a LongString
pub struct LongStringRef<'a>(pub &'a str);

impl<'a> ShortString {
    /// Get a reference to a ShortString
    pub fn as_ref(&'a self) -> ShortStringRef<'a> {
        ShortStringRef(&self.0)
    }

    /// Get a reference to a LongString as &str
    pub fn as_str(&'a self) -> &'a str {
        self.0.as_str()
    }

    /// Splits a string slice by whitespace.
    pub fn split_whitespace(&'a self) -> str::SplitWhitespace<'a> {
        self.0.split_whitespace()
    }
}

impl From<String> for ShortString {
    fn from(s: String) -> Self {
        Self(s)
    }
}

impl From<&str> for ShortString {
    fn from(s: &str) -> Self {
        s.to_owned().into()
    }
}

impl borrow::Borrow<str> for ShortString {
    fn borrow(&self) -> &str {
        self.0.borrow()
    }
}

impl fmt::Display for ShortString {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl<'a> ShortStringRef<'a> {
    pub(crate) fn as_bytes(&self) -> &[u8] {
        self.0.as_bytes()
    }

    pub(crate) fn len(&self) -> usize {
        self.0.len()
    }
}

impl Default for ShortStringRef<'static> {
    fn default() -> Self {
        Self("")
    }
}

impl<'a> From<&'a str> for ShortStringRef<'a> {
    fn from(s: &'a str) -> Self {
        Self(s)
    }
}

impl<'a> LongString {
    /// Get a reference to a LongString
    pub fn as_ref(&'a self) -> LongStringRef<'a> {
        LongStringRef(&self.0)
    }

    /// Get a reference to a LongString as &str
    pub fn as_str(&'a self) -> &'a str {
        self.0.as_str()
    }

    /// Splits a string slice by whitespace.
    pub fn split_whitespace(&'a self) -> str::SplitWhitespace<'a> {
        self.0.split_whitespace()
    }
}

impl From<String> for LongString {
    fn from(s: String) -> Self {
        Self(s)
    }
}

impl From<&str> for LongString {
    fn from(s: &str) -> Self {
        s.to_owned().into()
    }
}

impl borrow::Borrow<str> for LongString {
    fn borrow(&self) -> &str {
        self.0.borrow()
    }
}

impl fmt::Display for LongString {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl<'a> LongStringRef<'a> {
    pub(crate) fn as_bytes(&self) -> &[u8] {
        self.0.as_bytes()
    }

    pub(crate) fn len(&self) -> usize {
        self.0.len()
    }
}

impl Default for LongStringRef<'static> {
    fn default() -> Self {
        Self("")
    }
}

impl<'a> From<&'a str> for LongStringRef<'a> {
    fn from(s: &'a str) -> Self {
        Self(s)
    }
}

impl FieldArray {
    pub(crate) fn as_slice(&self) -> &[AMQPValue] {
        self.0.as_slice()
    }

    pub(crate) fn push(&mut self, v: AMQPValue) {
        self.0.push(v);
    }
}

impl From<Vec<AMQPValue>> for FieldArray {
    fn from(v: Vec<AMQPValue>) -> Self {
        Self(v)
    }
}

impl FieldTable {
    /// Insert a new entry in the table
    pub fn insert(&mut self, k: ShortString, v: AMQPValue) {
        self.0.insert(k, v);
    }

    /// Check whether the table contains the given key
    pub fn contains_key(&self, k: &str) -> bool {
        self.0.contains_key(k)
    }

    /// Access the inner BTreeMap to perform lookups
    pub fn inner(&self) -> &BTreeMap<ShortString, AMQPValue> {
        &self.0
    }
}

impl<'a> IntoIterator for &'a FieldTable {
    type Item = (&'a ShortString, &'a AMQPValue);
    type IntoIter = btree_map::Iter<'a, ShortString, AMQPValue>;

    fn into_iter(self) -> Self::IntoIter {
        self.0.iter()
    }
}

impl From<BTreeMap<ShortString, AMQPValue>> for FieldTable {
    fn from(m: BTreeMap<ShortString, AMQPValue>) -> Self {
        Self(m)
    }
}

impl ByteArray {
    pub(crate) fn as_slice(&self) -> &[u8] {
        self.0.as_slice()
    }

    pub(crate) fn len(&self) -> usize {
        self.0.len()
    }
}

impl From<Vec<u8>> for ByteArray {
    fn from(v: Vec<u8>) -> Self {
        Self(v)
    }
}

impl From<&[u8]> for ByteArray {
    fn from(v: &[u8]) -> Self {
        Self(v.to_vec())
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_type_from_id() {
        assert_eq!(AMQPType::from_id('T'), Some(AMQPType::Timestamp));
        assert_eq!(AMQPType::from_id('S'), Some(AMQPType::LongString));
        assert_eq!(AMQPType::from_id('s'), Some(AMQPType::ShortInt));
        assert_eq!(AMQPType::from_id('U'), Some(AMQPType::ShortInt));
        assert_eq!(AMQPType::from_id('l'), Some(AMQPType::LongLongInt));
        assert_eq!(AMQPType::from_id('z'), None);
    }

    #[test]
    fn test_type_get_id() {
        assert_eq!(AMQPType::LongLongInt.get_id(),  'l');
        assert_eq!(AMQPType::LongLongUInt.get_id(), 'l');
        assert_eq!(AMQPType::ShortString.get_id(),  '_');
    }

    #[test]
    fn test_type_to_string() {
        assert_eq!(AMQPType::Boolean.to_string(), "Boolean");
        assert_eq!(AMQPType::Void.to_string(),    "Void");
    }
}
<|endoftext|>