<commit_before>//! A naive implementation of left-leaning red-black trees
//! that are isomorphic to 2-3 trees. This implementation
//! borrows heavily from https://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf
//! and https://github.com/rcatolino/rbtree-rust.
//! Debug assertions borrow heavily from the implementations
//! at http://algs4.cs.princeton.edu/33balanced.

// I have spent a lot of hours trying all possible ways to
// implement this data structure. Most of the attemps
// led to confusing compilation errors. The heart of the
// issue is how to represent the Option<Box<Node>>.

// I tried using the newtype pattern
// https://doc.rust-lang.org/book/structs.html#tuple-structs.
// I (and others) had difficulty writing the
// destructuring lets correctly. See
// https://www.reddit.com/r/rust/comments/2cmjfn/how_to_do_typedefsnewtypes

// I tried creating a struct with a single field
// but had trouble accessing the field correctly
// ("let", "let ref", "let mut", "let ref mut").
// In addition the dummy field obfuscated the implementation.

// I tried using a type alias but then it's not
// possible to implement Display and Debug.

// Eventually I settled on using no abstraction for
// Option<Box<Node>>. The implementation became
// much easier to write.

// Also the implementation became much easier after I
// saw from rcatolino that some methods should be defined
// on a trait that wraps Box<Node> and other method
// should be defined on a trait that wraps Option<Box<Node>>.

use std::cmp::Ordering;
use std::fmt;
use std::i32;
use std::mem;

#[derive(PartialEq, Eq, Copy, Clone)]
enum Color {
    Red,
    Black,
}

// This instance of the newtype pattern
// https://doc.rust-lang.org/book/structs.html#tuple-structs
// is used only for fmt::Display and fmt::Debug.
struct Link<'a>(&'a Option<Box<Node>>);

pub struct LLRBTree {
    root: Option<Box<Node>>,
}

struct Node {
    key: i32,
    val: i32,
    color: Color,
    left: Option<Box<Node>>,
    right: Option<Box<Node>>,
}

impl LLRBTree {
    pub fn new() -> Self {
        LLRBTree { root: None }
    }

    pub fn get(&self, key: i32) -> Option<i32> {
        self.root.get(key)
    }

    pub fn insert(&mut self, key: i32, value: i32) -> Option<i32> {
        let prev = self.root.insert(key, value);
        self.root.mutate().color = Color::Black;
        debug_assert!(self.check());
        prev
    }

    fn is_bst(&self) -> bool {
        self.root.is_bst(i32::MIN, i32::MAX)
    }

    fn is_23(&self) -> bool {
        self.root.is_23(true)
    }

    fn is_balanced(&self) -> bool {
        let mut black = 0;
        let mut next = &self.root;
        loop {
            match next {
                &None => break,
                &Some(ref node) => {
                    if !node.is_red() {
                        black += 1;
                    }
                    next = &node.left;
                }
            }
        }
        self.root.is_balanced(black)
    }

    fn check(&self) -> bool {
        debug_assert!(self.is_bst(), "Not a binary search tree");
        debug_assert!(self.is_23(), "Not a 2-3 tree");
        debug_assert!(self.is_balanced(), "Not balanced");
        true
    }
}

impl fmt::Display for LLRBTree {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", Link(&self.root))
    }
}

impl fmt::Debug for LLRBTree {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}", Link(&self.root))
    }
}

impl Node {
    fn new(key: i32, val: i32) -> Self {
        Node {
            key: key,
            val: val,
            color: Color::Red,
            left: None,
            right: None,
        }
    }
}

trait BoxNode {
    fn is_red(&self) -> bool;
    fn left_rotate(&mut self);
    fn right_rotate(&mut self);
    fn flip_colors(&mut self);
}

impl BoxNode for Box<Node> {
    fn is_red(&self) -> bool {
        self.color == Color::Red
    }

    fn left_rotate(&mut self) {
        debug_assert!(self.right.is_red());
        let mut child = self.right.take().unwrap();
        mem::swap(self, &mut child);
        mem::swap(&mut child.right, &mut self.left);
        self.color = child.color;
        child.color = Color::Red;
        self.left = Some(child);
    }

    fn right_rotate(&mut self) {
        debug_assert!(self.left.is_red());
        let mut child = self.left.take().unwrap();
        mem::swap(self, &mut child);
        mem::swap(&mut child.left, &mut self.right);
        self.color = child.color;
        child.color = Color::Red;
        self.right = Some(child);
    }

    fn flip_colors(&mut self) {
        debug_assert!(!self.is_red());
        debug_assert!(self.left.is_red());
        debug_assert!(self.right.is_red());
        self.color = Color::Red;
        self.left.mutate().color = Color::Black;
        self.right.mutate().color = Color::Black;
    }
}

trait OptionBoxNode {
    fn get(&self, key: i32) -> Option<i32>;
    fn is_red(&self) -> bool;
    fn insert(&mut self, key: i32, val: i32) -> Option<i32>;
    fn reference(&mut self) -> &Box<Node>;
    fn mutate(&mut self) -> &mut Box<Node>;
    fn is_bst(&self, min: i32, max: i32) -> bool;
    fn is_23(&self, root: bool) -> bool;
    fn is_balanced(&self, black: i32) -> bool;
}

impl OptionBoxNode for Option<Box<Node>> {
    fn get(&self, key: i32) -> Option<i32> {
        match *self {
            None => None,
            Some(ref node) => {
                match key.cmp(&node.key) {
                    Ordering::Equal => Some(node.val),
                    Ordering::Less => node.left.get(key),
                    Ordering::Greater => node.right.get(key),
                }
            }
        }
    }

    fn is_red(&self) -> bool {
        match *self {
            None => false,
            Some(ref node) => node.is_red(),
        }
    }

    fn insert(&mut self, key: i32, value: i32) -> Option<i32> {
        let prev = match *self {
            None => {
                *self = new_leaf(key, value);
                None
            }
            Some(ref mut node) => {
                match key.cmp(&node.key) {
                    Ordering::Equal => {
                        let prev = node.val;
                        node.val = value;
                        Some(prev)
                    }
                    Ordering::Less => node.left.insert(key, value),
                    Ordering::Greater => node.right.insert(key, value),
                }
            }
        };
        let node = self.mutate();
        if !node.left.is_red() && node.right.is_red() {
            node.left_rotate();
        }
        if node.left.is_red() && node.left.reference().left.is_red() {
            node.right_rotate();
        }
        if node.left.is_red() && node.right.is_red() {
            node.flip_colors();
        }
        prev
    }

    fn reference(&mut self) -> &Box<Node> {
        self.as_ref().unwrap()
    }

    fn mutate(&mut self) -> &mut Box<Node> {
        self.as_mut().unwrap()
    }

    fn is_bst(&self, min: i32, max: i32) -> bool {
        match *self {
            None => true,
            Some(ref node) => {
                if node.key.cmp(&min) != Ordering::Greater {
                    return false;
                }
                if node.key.cmp(&max) != Ordering::Less {
                    return false;
                }
                node.left.is_bst(min, node.key) && node.right.is_bst(node.key, max)
            }
        }
    }

    fn is_23(&self, root: bool) -> bool {
        match *self {
            None => true,
            Some(ref node) => {
                if node.right.is_red() {
                    return false;
                }
                if !root && node.is_red() && node.left.is_red() {
                    return false;
                }
                node.left.is_23(false) && node.right.is_23(false)
            }
        }
    }

    fn is_balanced(&self, mut black: i32) -> bool {
        match *self {
            None => black == 0,
            Some(ref node) => {
                if !node.is_red() {
                    black -= 1;
                }
                node.left.is_balanced(black) && node.right.is_balanced(black)
            }
        }
    }
}

impl<'a> fmt::Display for Link<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let Link(ref link) = *self;
        match *link {
            &None => write!(f, "nil"),
            &Some(ref node) => {
                write!(f,
                       "({} {} {})",
                       node.key,
                       Link(&node.left),
                       Link(&node.right))
            }
        }
    }
}

impl<'a> fmt::Debug for Link<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let Link(ref link) = *self;
        match *link {
            &None => write!(f, "nil"),
            &Some(ref node) => {
                write!(f,
                       "({},{},{} {:?} {:?})",
                       node.key,
                       node.val,
                       node.color,
                       Link(&node.left),
                       Link(&node.right))
            }
        }
    }
}

impl fmt::Display for Color {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Color::Red => write!(f, "red"),
            Color::Black => write!(f, "black"),
        }
    }
}

fn new_leaf(key: i32, val: i32) -> Option<Box<Node>> {
    Some(Box::new(Node::new(key, val)))
}

#[test]
fn basic_construction() {
    let mut tree = LLRBTree::new();
    assert_eq!(tree.get(0), None);
    assert_eq!(tree.insert(0, 1), None);
    assert_eq!(tree.insert(0, 2), Some(1));
    assert_eq!(tree.get(0), Some(2));
}

#[test]
fn insert_sequence() {
    let mut tree = LLRBTree::new();
    for i in 0..256 {
        assert_eq!(tree.insert(i, i), None);
    }
    for i in 0..256 {
        assert_eq!(tree.get(i), Some(i));
    }
}
<commit_msg>Generalize flip_colors() method.<commit_after>//! A naive implementation of left-leaning red-black trees
//! that are isomorphic to 2-3 trees. This implementation
//! borrows heavily from https://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf
//! and https://github.com/rcatolino/rbtree-rust.
//! Debug assertions borrow heavily from the implementations
//! at http://algs4.cs.princeton.edu/33balanced.

// I have spent a lot of hours trying all possible ways to
// implement this data structure. Most of the attemps
// led to confusing compilation errors. The heart of the
// issue is how to represent the Option<Box<Node>>.

// I tried using the newtype pattern
// https://doc.rust-lang.org/book/structs.html#tuple-structs.
// I (and others) had difficulty writing the
// destructuring lets correctly. See
// https://www.reddit.com/r/rust/comments/2cmjfn/how_to_do_typedefsnewtypes

// I tried creating a struct with a single field
// but had trouble accessing the field correctly
// ("let", "let ref", "let mut", "let ref mut").
// In addition the dummy field obfuscated the implementation.

// I tried using a type alias but then it's not
// possible to implement Display and Debug.

// Eventually I settled on using no abstraction for
// Option<Box<Node>>. The implementation became
// much easier to write.

// Also the implementation became much easier after I
// saw from rcatolino that some methods should be defined
// on a trait that wraps Box<Node> and other method
// should be defined on a trait that wraps Option<Box<Node>>.

use std::cmp::Ordering;
use std::fmt;
use std::i32;
use std::mem;

#[derive(PartialEq, Eq, Copy, Clone)]
enum Color {
    Red,
    Black,
}

// This instance of the newtype pattern
// https://doc.rust-lang.org/book/structs.html#tuple-structs
// is used only for fmt::Display and fmt::Debug.
struct Link<'a>(&'a Option<Box<Node>>);

pub struct LLRBTree {
    root: Option<Box<Node>>,
}

struct Node {
    key: i32,
    val: i32,
    color: Color,
    left: Option<Box<Node>>,
    right: Option<Box<Node>>,
}

impl LLRBTree {
    pub fn new() -> Self {
        LLRBTree { root: None }
    }

    pub fn get(&self, key: i32) -> Option<i32> {
        self.root.get(key)
    }

    pub fn insert(&mut self, key: i32, value: i32) -> Option<i32> {
        let prev = self.root.insert(key, value);
        self.root.mutate().color = Color::Black;
        debug_assert!(self.check());
        prev
    }

    fn is_bst(&self) -> bool {
        self.root.is_bst(i32::MIN, i32::MAX)
    }

    fn is_23(&self) -> bool {
        self.root.is_23(true)
    }

    fn is_balanced(&self) -> bool {
        let mut black = 0;
        let mut next = &self.root;
        loop {
            match next {
                &None => break,
                &Some(ref node) => {
                    if !node.is_red() {
                        black += 1;
                    }
                    next = &node.left;
                }
            }
        }
        self.root.is_balanced(black)
    }

    fn check(&self) -> bool {
        debug_assert!(self.is_bst(), "Not a binary search tree");
        debug_assert!(self.is_23(), "Not a 2-3 tree");
        debug_assert!(self.is_balanced(), "Not balanced");
        true
    }
}

impl fmt::Display for LLRBTree {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", Link(&self.root))
    }
}

impl fmt::Debug for LLRBTree {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}", Link(&self.root))
    }
}

impl Node {
    fn new(key: i32, val: i32) -> Self {
        Node {
            key: key,
            val: val,
            color: Color::Red,
            left: None,
            right: None,
        }
    }
}

trait BoxNode {
    fn is_red(&self) -> bool;
    fn left_rotate(&mut self);
    fn right_rotate(&mut self);
    fn flip_colors(&mut self);
}

impl BoxNode for Box<Node> {
    fn is_red(&self) -> bool {
        self.color == Color::Red
    }

    fn left_rotate(&mut self) {
        debug_assert!(self.right.is_red());
        let mut child = self.right.take().unwrap();
        mem::swap(self, &mut child);
        mem::swap(&mut child.right, &mut self.left);
        self.color = child.color;
        child.color = Color::Red;
        self.left = Some(child);
    }

    fn right_rotate(&mut self) {
        debug_assert!(self.left.is_red());
        let mut child = self.left.take().unwrap();
        mem::swap(self, &mut child);
        mem::swap(&mut child.left, &mut self.right);
        self.color = child.color;
        child.color = Color::Red;
        self.right = Some(child);
    }

    fn flip_colors(&mut self) {
        debug_assert!(self.left.is_some());
        debug_assert!(self.right.is_some());
        debug_assert!(self.color != self.left.color());
        debug_assert!(self.color != self.right.color());
        self.color.flip();
        self.left.mutate().color.flip();
        self.right.mutate().color.flip();
    }
}

trait OptionBoxNode {
    fn get(&self, key: i32) -> Option<i32>;
    fn is_red(&self) -> bool;
    fn color(&self) -> Color;
    fn insert(&mut self, key: i32, val: i32) -> Option<i32>;
    fn reference(&mut self) -> &Box<Node>;
    fn mutate(&mut self) -> &mut Box<Node>;
    fn is_bst(&self, min: i32, max: i32) -> bool;
    fn is_23(&self, root: bool) -> bool;
    fn is_balanced(&self, black: i32) -> bool;
}

impl OptionBoxNode for Option<Box<Node>> {
    fn get(&self, key: i32) -> Option<i32> {
        match *self {
            None => None,
            Some(ref node) => {
                match key.cmp(&node.key) {
                    Ordering::Equal => Some(node.val),
                    Ordering::Less => node.left.get(key),
                    Ordering::Greater => node.right.get(key),
                }
            }
        }
    }

    fn is_red(&self) -> bool {
        match *self {
            None => false,
            Some(ref node) => node.is_red(),
        }
    }

    fn color(&self) -> Color {
        self.as_ref().unwrap().color
    }

    fn insert(&mut self, key: i32, value: i32) -> Option<i32> {
        let prev = match *self {
            None => {
                *self = new_leaf(key, value);
                None
            }
            Some(ref mut node) => {
                match key.cmp(&node.key) {
                    Ordering::Equal => {
                        let prev = node.val;
                        node.val = value;
                        Some(prev)
                    }
                    Ordering::Less => node.left.insert(key, value),
                    Ordering::Greater => node.right.insert(key, value),
                }
            }
        };
        let node = self.mutate();
        if !node.left.is_red() && node.right.is_red() {
            node.left_rotate();
        }
        if node.left.is_red() && node.left.reference().left.is_red() {
            node.right_rotate();
        }
        if node.left.is_red() && node.right.is_red() {
            node.flip_colors();
        }
        prev
    }

    fn reference(&mut self) -> &Box<Node> {
        self.as_ref().unwrap()
    }

    fn mutate(&mut self) -> &mut Box<Node> {
        self.as_mut().unwrap()
    }

    fn is_bst(&self, min: i32, max: i32) -> bool {
        match *self {
            None => true,
            Some(ref node) => {
                if node.key.cmp(&min) != Ordering::Greater {
                    return false;
                }
                if node.key.cmp(&max) != Ordering::Less {
                    return false;
                }
                node.left.is_bst(min, node.key) && node.right.is_bst(node.key, max)
            }
        }
    }

    fn is_23(&self, root: bool) -> bool {
        match *self {
            None => true,
            Some(ref node) => {
                if node.right.is_red() {
                    return false;
                }
                if !root && node.is_red() && node.left.is_red() {
                    return false;
                }
                node.left.is_23(false) && node.right.is_23(false)
            }
        }
    }

    fn is_balanced(&self, mut black: i32) -> bool {
        match *self {
            None => black == 0,
            Some(ref node) => {
                if !node.is_red() {
                    black -= 1;
                }
                node.left.is_balanced(black) && node.right.is_balanced(black)
            }
        }
    }
}

impl<'a> fmt::Display for Link<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let Link(ref link) = *self;
        match *link {
            &None => write!(f, "nil"),
            &Some(ref node) => {
                write!(f,
                       "({} {} {})",
                       node.key,
                       Link(&node.left),
                       Link(&node.right))
            }
        }
    }
}

impl<'a> fmt::Debug for Link<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let Link(ref link) = *self;
        match *link {
            &None => write!(f, "nil"),
            &Some(ref node) => {
                write!(f,
                       "({},{},{} {:?} {:?})",
                       node.key,
                       node.val,
                       node.color,
                       Link(&node.left),
                       Link(&node.right))
            }
        }
    }
}

impl Color {
    fn flip(&mut self) {
        match *self {
            Color::Red => *self = Color::Black,
            Color::Black => *self = Color::Red,
        }
    }
}

impl fmt::Display for Color {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Color::Red => write!(f, "red"),
            Color::Black => write!(f, "black"),
        }
    }
}

fn new_leaf(key: i32, val: i32) -> Option<Box<Node>> {
    Some(Box::new(Node::new(key, val)))
}

#[test]
fn basic_construction() {
    let mut tree = LLRBTree::new();
    assert_eq!(tree.get(0), None);
    assert_eq!(tree.insert(0, 1), None);
    assert_eq!(tree.insert(0, 2), Some(1));
    assert_eq!(tree.get(0), Some(2));
}

#[test]
fn insert_sequence() {
    let mut tree = LLRBTree::new();
    for i in 0..256 {
        assert_eq!(tree.insert(i, i), None);
    }
    for i in 0..256 {
        assert_eq!(tree.get(i), Some(i));
    }
}
<|endoftext|>


<commit_before>// Needed for reading a Json File
// extern crate rustc_serialize;
// use rustc_serialize::json::Json;
// use std::fs::File;
// use std::io::Read;

use std::ops::Deref;
use toml::Value;
use uuid::Uuid;

use libimagstore::store::Store;
use libimagstore::storeid::IntoStoreId;
use module_path::ModuleEntryPath;

/// With the uuid we get the storeid and than we can delete the entry
fn deleteFunc(uuid: Uuid, store : &Store) {
	// With this we can read from a .json File
	// let mut file = File::open("text.json").unwrap();
	// let mut data = String::new();
	// file.rad_to_string(&mut data).unwrap();
	//
	// let jeson = Json::from_str(&data).unwrap();
	// println!("{}", json.find_path(&["uuid"]).unwrap());

	// With the uuid we get the storeid
	let store_id = ModuleEntryPath::new(format!("taskwarrior/{}", uuid)).into_storeid();
	// It deletes an entry
	store.delete(store_id);
}

<commit_msg>Fixed the store.delete(...) Return Problem<commit_after>// Needed for reading a Json File
// extern crate rustc_serialize;
// use rustc_serialize::json::Json;
// use std::fs::File;
// use std::io::Read;

use uuid::Uuid;

use libimagstore::store::Store;
use libimagstore::storeid::IntoStoreId;
use module_path::ModuleEntryPath;

use error::{TodoError, TodoErrorKind};

/// With the uuid we get the storeid and than we can delete the entry
pub fn deleteFunc(uuid: Uuid, store : &Store) {
	// With this we can read from a .json File
	// let mut file = File::open("text.json").unwrap();
	// let mut data = String::new();
	// file.rad_to_string(&mut data).unwrap();
	//
	// let jeson = Json::from_str(&data).unwrap();
	// println!("{}", json.find_path(&["uuid"]).unwrap());

	// With the uuid we get the storeid
	let store_id = ModuleEntryPath::new(format!("taskwarrior/{}", uuid)).into_storeid();
	// It deletes an entry
	if let Err(e) = store.delete(store_id) {
		return Err(TodoError::new(TodoErrorKind::StoreError, Some(Box::new(e)))).unwrap();
	}

}

<|endoftext|>


<commit_before>//! OS-specific functionality.

#![stable(feature = "os", since = "1.0.0")]
#![allow(missing_docs, nonstandard_style, missing_debug_implementations)]

// When documenting libstd we want to show unix/windows/linux modules as these are the "main
// modules" that are used across platforms, so all modules are enabled when `cfg(doc)` is set.
// This should help show platform-specific functionality in a hopefully cross-platform way in the
// documentation.
// Note that we deliberately avoid `cfg_if!` here to work around a rust-analyzer bug that would make
// `std::os` submodules unusable: https://github.com/rust-analyzer/rust-analyzer/issues/6038

#[cfg(doc)]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::unix_ext as unix;

#[cfg(doc)]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::windows_ext as windows;

#[cfg(doc)]
#[doc(cfg(target_os = "linux"))]
pub mod linux;

#[cfg(doc)]
pub use crate::sys::wasi_ext as wasi;

// If we're not documenting libstd then we just expose the main modules as we otherwise would.

#[cfg(not(doc))]
#[cfg(any(target_os = "redox", unix, target_os = "vxworks", target_os = "hermit"))]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::ext as unix;

#[cfg(not(doc))]
#[cfg(windows)]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::ext as windows;

#[cfg(not(doc))]
#[cfg(any(target_os = "linux", target_os = "l4re"))]
pub mod linux;

#[cfg(target_os = "android")]
pub mod android;
#[cfg(target_os = "dragonfly")]
pub mod dragonfly;
#[cfg(target_os = "emscripten")]
pub mod emscripten;
#[cfg(all(target_vendor = "fortanix", target_env = "sgx"))]
pub mod fortanix_sgx;
#[cfg(target_os = "freebsd")]
pub mod freebsd;
#[cfg(target_os = "fuchsia")]
pub mod fuchsia;
#[cfg(target_os = "haiku")]
pub mod haiku;
#[cfg(target_os = "illumos")]
pub mod illumos;
#[cfg(target_os = "ios")]
pub mod ios;
#[cfg(target_os = "macos")]
pub mod macos;
#[cfg(target_os = "netbsd")]
pub mod netbsd;
#[cfg(target_os = "openbsd")]
pub mod openbsd;
#[cfg(target_os = "redox")]
pub mod redox;
#[cfg(target_os = "solaris")]
pub mod solaris;
#[cfg(target_os = "vxworks")]
pub mod vxworks;
#[cfg(target_os = "wasi")]
pub mod wasi;

pub mod raw;
<commit_msg>Make the main `wasi` module `cfg(not(doc))`.<commit_after>//! OS-specific functionality.

#![stable(feature = "os", since = "1.0.0")]
#![allow(missing_docs, nonstandard_style, missing_debug_implementations)]

// When documenting libstd we want to show unix/windows/linux modules as these are the "main
// modules" that are used across platforms, so all modules are enabled when `cfg(doc)` is set.
// This should help show platform-specific functionality in a hopefully cross-platform way in the
// documentation.
// Note that we deliberately avoid `cfg_if!` here to work around a rust-analyzer bug that would make
// `std::os` submodules unusable: https://github.com/rust-analyzer/rust-analyzer/issues/6038

#[cfg(doc)]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::unix_ext as unix;

#[cfg(doc)]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::windows_ext as windows;

#[cfg(doc)]
#[doc(cfg(target_os = "linux"))]
pub mod linux;

#[cfg(doc)]
pub use crate::sys::wasi_ext as wasi;

// If we're not documenting libstd then we just expose the main modules as we otherwise would.

#[cfg(not(doc))]
#[cfg(any(target_os = "redox", unix, target_os = "vxworks", target_os = "hermit"))]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::ext as unix;

#[cfg(not(doc))]
#[cfg(windows)]
#[stable(feature = "rust1", since = "1.0.0")]
pub use crate::sys::ext as windows;

#[cfg(not(doc))]
#[cfg(any(target_os = "linux", target_os = "l4re"))]
pub mod linux;

#[cfg(not(doc))]
#[cfg(target_os = "wasi")]
pub mod wasi;

#[cfg(target_os = "android")]
pub mod android;
#[cfg(target_os = "dragonfly")]
pub mod dragonfly;
#[cfg(target_os = "emscripten")]
pub mod emscripten;
#[cfg(all(target_vendor = "fortanix", target_env = "sgx"))]
pub mod fortanix_sgx;
#[cfg(target_os = "freebsd")]
pub mod freebsd;
#[cfg(target_os = "fuchsia")]
pub mod fuchsia;
#[cfg(target_os = "haiku")]
pub mod haiku;
#[cfg(target_os = "illumos")]
pub mod illumos;
#[cfg(target_os = "ios")]
pub mod ios;
#[cfg(target_os = "macos")]
pub mod macos;
#[cfg(target_os = "netbsd")]
pub mod netbsd;
#[cfg(target_os = "openbsd")]
pub mod openbsd;
#[cfg(target_os = "redox")]
pub mod redox;
#[cfg(target_os = "solaris")]
pub mod solaris;
#[cfg(target_os = "vxworks")]
pub mod vxworks;

pub mod raw;
<|endoftext|>


<commit_before>use rustc_serialize::json::{self, Json, ToJson};

use super::chrono::UTC;

use super::params::*;

use super::rs_es::Client;
use super::rs_es::query::{Filter, Query};
use super::rs_es::units::JsonVal;
use super::rs_es::operations::search::{Sort, SortField, Order};
use super::rs_es::operations::index::IndexResult;
use super::rs_es::operations::mapping::*;
use super::rs_es::error::EsError;

use searchspot::terms::VectorOfTerms;
use searchspot::resource::*;

#[derive(RustcEncodable, RustcDecodable, Debug)]
pub struct Talent {
  pub id:                 u32,
  pub accepted:           bool,
  pub work_roles:         Vec<String>,
  pub work_experience:    String,
  pub work_locations:     Vec<String>,
  pub work_authorization: String,
  pub company_ids:        Vec<u32>,
  pub batch_starts_at:    String,
  pub batch_ends_at:      String,
  pub added_to_batch_at:  String,
  pub weight:             i32,
  pub blocked_companies:  Vec<u32>
}

impl ToJson for Talent {
  fn to_json(&self) -> Json {
    json::encode(&self).unwrap()
                       .to_json()
  }
}

impl Resource for Talent {
  /// Populate the ElasticSearch index with `self`.
  // I'm having problems with bulk actions. Let's wait for the next iteration.
  fn index(&self, mut es: &mut Client, index: &str) -> Result<IndexResult, EsError> {
    es.index(index, "talent")
      .with_doc(&self)
      .with_id(&*self.id.to_string())
      .send()
  }

  /// Query ElasticSearch on given `indexes` and `params` and return the IDs of
  /// the found talents.
  fn search(mut es: &mut Client, default_index: &str, params: &Map) -> Vec<u32> {
    let now   = UTC::now().to_rfc3339();
    let epoch = match params.find(&["epoch"]) {
      Some(epoch) => String::from_value(&epoch).unwrap_or(now),
      _           => now
    };

    let index: Vec<&str> = match params.find(&["index"]) {
      Some(&Value::String(ref index)) => vec![&index[..]],
      _ => vec![default_index]
    };

    let result = es.search_query()
                   .with_indexes(&*index)
                   .with_query(&Talent::search_filters(params, &*epoch))
                   .with_sort(&Talent::sorting_criteria())
                   .with_size(1000) // TODO
                   .send();

    match result {
      Ok(result) => {
        let mut results = result.hits.hits.into_iter()
                                          .map(|hit| hit.source.unwrap()["id"]
                                                               .as_u64()
                                                               .unwrap() as u32)
                                          .collect::<Vec<u32>>();
        results.dedup();
        results
      },
      Err(err) => {
        println!("{:?}", err);
        vec![]
      }
    }
  }

  /// Reset the given index. All the data will be destroyed and then the index
  /// will be created again. The map that will be used is hardcoded.
  #[allow(unused_must_use)]
  fn reset_index(mut es: &mut Client, index: &str) -> Result<MappingResult, EsError> {
    let mapping = hashmap! {
      "talent" => hashmap! {
        "id" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        },

        "work_roles" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "work_experience" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "work_locations" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "work_authorization" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "company_ids" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        },

        "accepted" => hashmap! {
          "type" => "boolean",
          "index" => "not_analyzed"
        },

        "batch_starts_at" => hashmap! {
          "type" => "date",
          "format" => "dateOptionalTime",
          "index" => "not_analyzed"
        },

        "batch_ends_at" => hashmap! {
          "type" => "date",
          "format" => "dateOptionalTime",
          "index" => "not_analyzed"
        },

        "added_to_batch_at" => hashmap! {
          "type" => "date",
          "format" => "dateOptionalTime",
          "index" => "not_analyzed"
        },

        "weight" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        },

        "blocked_companies" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        }
      }
    };

    es.delete_index(index);

    MappingOperation::new(&mut es, index, &mapping).send()
  }

  /// Return a `Vec<Filter>` with visibility criteria for the talents.
  /// The `epoch` must be given as `I64` (UNIX time in seconds) and is
  /// the range in which batches are searched.
  /// If `presented_talents` is provided, talents who match the IDs
  /// contained there skip the standard visibility criteria.
  ///
  /// Basically, the talents must be accepted into the platform and must be
  /// inside a living batch to match the visibility criteria.
  fn visibility_filters(epoch: &str, presented_talents: Vec<i32>) -> Vec<Filter> {
    let visibility_rules = Filter::build_bool()
                                  .with_must(
                                    vec![
                                      Filter::build_term("accepted", true)
                                             .build(),
                                      Filter::build_range("batch_starts_at")
                                             .with_lte(JsonVal::from(epoch))
                                             .with_format("dateOptionalTime")
                                             .build(),
                                      Filter::build_range("batch_ends_at")
                                             .with_gte(JsonVal::from(epoch))
                                             .with_format("dateOptionalTime")
                                             .build()
                                    ])
                                  .build();

    if presented_talents.len() > 0 { // preferred over !_.is_empty()
      let presented_talents_filters = Filter::build_bool()
                                             .with_must(
                                               vec![
                                                 <Filter as VectorOfTerms<i32>>::build_terms(
                                                   "ids", &presented_talents)
                                               ].into_iter()
                                                .flat_map(|x| x)
                                                .collect::<Vec<Filter>>())
                                             .build();
      vec![
        Filter::build_bool()
               .with_should(vec![visibility_rules, presented_talents_filters])
               .build()
      ]
    }
    else {
      vec![visibility_rules]
    }
  }

  /// Given parameters inside the query string mapped inside a `Map`,
  /// and the `epoch` (defined as UNIX time in seconds) for batches,
  /// return a `Query` for ElasticSearch.
  ///
  /// Considering a single row, the terms inside there are ORred,
  /// while through the rows there is an AND.
  /// I.e.: given ["Fullstack", "DevOps"] as `work_roles`, found talents
  /// will present at least one of these roles), but both `work_roles`
  /// and `work_location`, if provided, must be matched successfully.
  fn search_filters(params: &Map, epoch: &str) -> Query {
    let company_id = i32_vec_from_params!(params, "company_id");

    Query::build_filtered(Filter::build_bool()
                                 .with_must(
                                   vec![
                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_roles", &vec_from_params!(params, "work_roles")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_experience", &vec_from_params!(params, "work_experience")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                      "work_authorization", &vec_from_params!(params, "work_authorization")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_locations", &vec_from_params!(params, "work_locations")),

                                     Talent::visibility_filters(epoch,
                                       i32_vec_from_params!(params, "presented_talents"))
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .with_must_not(
                                   vec![
                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "company_ids", &company_id),

                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "blocked_companies", &company_id)
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .build())
          .build()
  }

  /// Return a `Sort` that makes values be sorted for given fields, descendently.
  fn sorting_criteria() -> Sort {
    Sort::new(
      vec![
        SortField::new("batch_starts_at",   Some(Order::Desc)).build(),
        SortField::new("weight",            Some(Order::Desc)).build(),
        SortField::new("added_to_batch_at", Some(Order::Desc)).build()
      ])
  }
}

#[cfg(test)]
#[allow(non_upper_case_globals)]
mod tests {
  extern crate chrono;
  use self::chrono::*;

  extern crate rs_es;
  use self::rs_es::Client;

  extern crate params;
  use self::params::*;

  extern crate searchspot;
  use searchspot::config::*;
  use searchspot::resource::*;

  use resources::user::Talent;

  const CONFIG_FILE: &'static str = "examples/tests.toml";

  lazy_static! {
    static ref config: Config = Config::from_file(CONFIG_FILE.to_owned());
  }

  pub fn make_client() -> Client {
    Client::new(&*config.es.host, config.es.port)
  }

  macro_rules! epoch_from_year {
    ($year:expr) => {
      UTC.datetime_from_str(&format!("{}-01-01 12:00:00", $year),
        "%Y-%m-%d %H:%M:%S").unwrap().to_rfc3339()
    }
  }

  pub fn populate_index(mut client: &mut Client) -> bool {
    vec![
      Talent {
        id:                 1,
        accepted:           true,
        work_roles:         vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             -5,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 2,
        accepted:           true,
        work_roles:         vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 3,
        accepted:           false,
        work_roles:         vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2007"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 4,
        accepted:           true,
        work_roles:         vec!["Fullstack".to_owned(), "DevOps".to_owned()],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization:  "yes".to_owned(),
        company_ids:        vec![6],
        batch_starts_at:    epoch_from_year!("2008"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             0,
        blocked_companies:  vec![]
      }
    ].iter()
     .map(|talent| talent.index(&mut client, &config.es.index)
                         .is_ok())
     .collect::<Vec<bool>>()
     .into_iter()
     .all(|result| result)
  }

  fn refresh_index(mut client: &mut Client) {
    client.refresh()
          .with_indexes(&[&config.es.index])
          .send()
          .unwrap();
  }

  #[test]
  fn test_search() {
    let mut client = make_client();
    assert!(Talent::reset_index(&mut client, &*config.es.index).is_ok());
    refresh_index(&mut client);
    assert!(populate_index(&mut client));
    refresh_index(&mut client);

    // no parameters are given
    {
      let results = Talent::search(&mut client, &*config.es.index, &Map::new());
      assert_eq!(vec![4, 2, 1], results);
    }

    // a non existing index is given
    {
      let mut map = Map::new();
      map.assign("index", Value::String("lololol".to_owned())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    // a date that doesn't match given indexes is given
    {
      let mut map = Map::new();
      map.assign("epoch", Value::String(epoch_from_year!("2040"))).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    {
      let mut map = Map::new();
      map.assign("work_roles[]", Value::String("Fullstack".to_owned())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert_eq!(vec![4], results);
    }

    // filtering for given company_id
    {
      let mut map = Map::new();
      map.assign("company_id", Value::String("6".into())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert_eq!(vec![2, 1], results);
    }
  }
}
<commit_msg>resources/user: test decoding of talents from json<commit_after>use rustc_serialize::json::{self, Json, ToJson};

use super::chrono::UTC;

use super::params::*;

use super::rs_es::Client;
use super::rs_es::query::{Filter, Query};
use super::rs_es::units::JsonVal;
use super::rs_es::operations::search::{Sort, SortField, Order};
use super::rs_es::operations::index::IndexResult;
use super::rs_es::operations::mapping::*;
use super::rs_es::error::EsError;

use searchspot::terms::VectorOfTerms;
use searchspot::resource::*;

#[derive(RustcEncodable, RustcDecodable, Debug)]
pub struct Talent {
  pub id:                 u32,
  pub accepted:           bool,
  pub work_roles:         Vec<String>,
  pub work_experience:    String,
  pub work_locations:     Vec<String>,
  pub work_authorization: String,
  pub company_ids:        Vec<u32>,
  pub batch_starts_at:    String,
  pub batch_ends_at:      String,
  pub added_to_batch_at:  String,
  pub weight:             i32,
  pub blocked_companies:  Vec<u32>
}

impl ToJson for Talent {
  fn to_json(&self) -> Json {
    json::encode(&self).unwrap()
                       .to_json()
  }
}

impl Resource for Talent {
  /// Populate the ElasticSearch index with `self`.
  // I'm having problems with bulk actions. Let's wait for the next iteration.
  fn index(&self, mut es: &mut Client, index: &str) -> Result<IndexResult, EsError> {
    es.index(index, "talent")
      .with_doc(&self)
      .with_id(&*self.id.to_string())
      .send()
  }

  /// Query ElasticSearch on given `indexes` and `params` and return the IDs of
  /// the found talents.
  fn search(mut es: &mut Client, default_index: &str, params: &Map) -> Vec<u32> {
    let now   = UTC::now().to_rfc3339();
    let epoch = match params.find(&["epoch"]) {
      Some(epoch) => String::from_value(&epoch).unwrap_or(now),
      _           => now
    };

    let index: Vec<&str> = match params.find(&["index"]) {
      Some(&Value::String(ref index)) => vec![&index[..]],
      _ => vec![default_index]
    };

    let result = es.search_query()
                   .with_indexes(&*index)
                   .with_query(&Talent::search_filters(params, &*epoch))
                   .with_sort(&Talent::sorting_criteria())
                   .with_size(1000) // TODO
                   .send();

    match result {
      Ok(result) => {
        let mut results = result.hits.hits.into_iter()
                                          .map(|hit| hit.source.unwrap()["id"]
                                                               .as_u64()
                                                               .unwrap() as u32)
                                          .collect::<Vec<u32>>();
        results.dedup();
        results
      },
      Err(err) => {
        println!("{:?}", err);
        vec![]
      }
    }
  }

  /// Reset the given index. All the data will be destroyed and then the index
  /// will be created again. The map that will be used is hardcoded.
  #[allow(unused_must_use)]
  fn reset_index(mut es: &mut Client, index: &str) -> Result<MappingResult, EsError> {
    let mapping = hashmap! {
      "talent" => hashmap! {
        "id" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        },

        "work_roles" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "work_experience" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "work_locations" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "work_authorization" => hashmap! {
          "type" => "string",
          "index" => "not_analyzed"
        },

        "company_ids" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        },

        "accepted" => hashmap! {
          "type" => "boolean",
          "index" => "not_analyzed"
        },

        "batch_starts_at" => hashmap! {
          "type" => "date",
          "format" => "dateOptionalTime",
          "index" => "not_analyzed"
        },

        "batch_ends_at" => hashmap! {
          "type" => "date",
          "format" => "dateOptionalTime",
          "index" => "not_analyzed"
        },

        "added_to_batch_at" => hashmap! {
          "type" => "date",
          "format" => "dateOptionalTime",
          "index" => "not_analyzed"
        },

        "weight" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        },

        "blocked_companies" => hashmap! {
          "type" => "integer",
          "index" => "not_analyzed"
        }
      }
    };

    es.delete_index(index);

    MappingOperation::new(&mut es, index, &mapping).send()
  }

  /// Return a `Vec<Filter>` with visibility criteria for the talents.
  /// The `epoch` must be given as `I64` (UNIX time in seconds) and is
  /// the range in which batches are searched.
  /// If `presented_talents` is provided, talents who match the IDs
  /// contained there skip the standard visibility criteria.
  ///
  /// Basically, the talents must be accepted into the platform and must be
  /// inside a living batch to match the visibility criteria.
  fn visibility_filters(epoch: &str, presented_talents: Vec<i32>) -> Vec<Filter> {
    let visibility_rules = Filter::build_bool()
                                  .with_must(
                                    vec![
                                      Filter::build_term("accepted", true)
                                             .build(),
                                      Filter::build_range("batch_starts_at")
                                             .with_lte(JsonVal::from(epoch))
                                             .with_format("dateOptionalTime")
                                             .build(),
                                      Filter::build_range("batch_ends_at")
                                             .with_gte(JsonVal::from(epoch))
                                             .with_format("dateOptionalTime")
                                             .build()
                                    ])
                                  .build();

    if presented_talents.len() > 0 { // preferred over !_.is_empty()
      let presented_talents_filters = Filter::build_bool()
                                             .with_must(
                                               vec![
                                                 <Filter as VectorOfTerms<i32>>::build_terms(
                                                   "ids", &presented_talents)
                                               ].into_iter()
                                                .flat_map(|x| x)
                                                .collect::<Vec<Filter>>())
                                             .build();
      vec![
        Filter::build_bool()
               .with_should(vec![visibility_rules, presented_talents_filters])
               .build()
      ]
    }
    else {
      vec![visibility_rules]
    }
  }

  /// Given parameters inside the query string mapped inside a `Map`,
  /// and the `epoch` (defined as UNIX time in seconds) for batches,
  /// return a `Query` for ElasticSearch.
  ///
  /// Considering a single row, the terms inside there are ORred,
  /// while through the rows there is an AND.
  /// I.e.: given ["Fullstack", "DevOps"] as `work_roles`, found talents
  /// will present at least one of these roles), but both `work_roles`
  /// and `work_location`, if provided, must be matched successfully.
  fn search_filters(params: &Map, epoch: &str) -> Query {
    let company_id = i32_vec_from_params!(params, "company_id");

    Query::build_filtered(Filter::build_bool()
                                 .with_must(
                                   vec![
                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_roles", &vec_from_params!(params, "work_roles")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_experience", &vec_from_params!(params, "work_experience")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                      "work_authorization", &vec_from_params!(params, "work_authorization")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_locations", &vec_from_params!(params, "work_locations")),

                                     Talent::visibility_filters(epoch,
                                       i32_vec_from_params!(params, "presented_talents"))
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .with_must_not(
                                   vec![
                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "company_ids", &company_id),

                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "blocked_companies", &company_id)
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .build())
          .build()
  }

  /// Return a `Sort` that makes values be sorted for given fields, descendently.
  fn sorting_criteria() -> Sort {
    Sort::new(
      vec![
        SortField::new("batch_starts_at",   Some(Order::Desc)).build(),
        SortField::new("weight",            Some(Order::Desc)).build(),
        SortField::new("added_to_batch_at", Some(Order::Desc)).build()
      ])
  }
}

#[cfg(test)]
#[allow(non_upper_case_globals)]
mod tests {
  use rustc_serialize::json;

  extern crate chrono;
  use self::chrono::*;

  extern crate rs_es;
  use self::rs_es::Client;

  extern crate params;
  use self::params::*;

  extern crate searchspot;
  use searchspot::config::*;
  use searchspot::resource::*;

  use resources::user::Talent;

  const CONFIG_FILE: &'static str = "examples/tests.toml";

  lazy_static! {
    static ref config: Config = Config::from_file(CONFIG_FILE.to_owned());
  }

  pub fn make_client() -> Client {
    Client::new(&*config.es.host, config.es.port)
  }

  macro_rules! epoch_from_year {
    ($year:expr) => {
      UTC.datetime_from_str(&format!("{}-01-01 12:00:00", $year),
        "%Y-%m-%d %H:%M:%S").unwrap().to_rfc3339()
    }
  }

  pub fn populate_index(mut client: &mut Client) -> bool {
    vec![
      Talent {
        id:                 1,
        accepted:           true,
        work_roles:         vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             -5,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 2,
        accepted:           true,
        work_roles:         vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 3,
        accepted:           false,
        work_roles:         vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2007"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 4,
        accepted:           true,
        work_roles:         vec!["Fullstack".to_owned(), "DevOps".to_owned()],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization:  "yes".to_owned(),
        company_ids:        vec![6],
        batch_starts_at:    epoch_from_year!("2008"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             0,
        blocked_companies:  vec![]
      }
    ].iter()
     .map(|talent| talent.index(&mut client, &config.es.index)
                         .is_ok())
     .collect::<Vec<bool>>()
     .into_iter()
     .all(|result| result)
  }

  fn refresh_index(mut client: &mut Client) {
    client.refresh()
          .with_indexes(&[&config.es.index])
          .send()
          .unwrap();
  }

  #[test]
  fn test_search() {
    let mut client = make_client();
    assert!(Talent::reset_index(&mut client, &*config.es.index).is_ok());
    refresh_index(&mut client);
    assert!(populate_index(&mut client));
    refresh_index(&mut client);

    // no parameters are given
    {
      let results = Talent::search(&mut client, &*config.es.index, &Map::new());
      assert_eq!(vec![4, 2, 1], results);
    }

    // a non existing index is given
    {
      let mut map = Map::new();
      map.assign("index", Value::String("lololol".to_owned())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    // a date that doesn't match given indexes is given
    {
      let mut map = Map::new();
      map.assign("epoch", Value::String(epoch_from_year!("2040"))).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    {
      let mut map = Map::new();
      map.assign("work_roles[]", Value::String("Fullstack".to_owned())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert_eq!(vec![4], results);
    }

    // filtering for given company_id
    {
      let mut map = Map::new();
      map.assign("company_id", Value::String("6".into())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert_eq!(vec![2, 1], results);
    }
  }

  #[test]
  fn test_json_decode() {
    let payload = "{
      \"id\":13,
      \"work_roles\":[\"C/C++ Engineer\"],
      \"work_languages\":[],
      \"work_experience\":\"8+\",
      \"work_locations\":[\"Berlin\"],
      \"work_authorization\":\"yes\",
      \"company_ids\":[],
      \"accepted\":true,
      \"batch_starts_at\":\"2016-03-04T12:24:00+01:00\",
      \"batch_ends_at\":\"2016-04-11T12:24:00+02:00\",
      \"added_to_batch_at\":\"2016-03-11T12:24:37+01:00\",
      \"weight\":0,
      \"blocked_companies\":[]
    }".to_owned();

    let resource: Result<Talent, _> = json::decode(&payload);
    assert!(resource.is_ok());
    assert_eq!(resource.unwrap().work_roles, vec!["C/C++ Engineer"]);
  }
}
<|endoftext|>


<commit_before>extern crate serde_json;
extern crate zmq;

use std::cell::RefCell;
use std::thread;

use errors::common::CommonError;
use errors::pool::PoolError;
use utils::json::{JsonDecodable, JsonEncodable};
use utils::sequence::SequenceUtils;

struct RemoteAgent {
    socket: zmq::Socket,
    addr: String,
    public_key: String,
    secret_key: String,
    server_key: String,
}

struct AgentWorker {
    cmd_socket: zmq::Socket,
    agent_connections: Vec<RemoteAgent>,
}

struct Agent {
    cmd_socket: zmq::Socket,
    worker: Option<thread::JoinHandle<()>>,
}

impl Drop for Agent {
    fn drop(&mut self) {
        trace!("agent drop >>");
        self.cmd_socket.send_str(AgentWorkerCommand::Exit.to_json().unwrap().as_str(), zmq::DONTWAIT).unwrap(); //TODO
        self.worker.take().unwrap().join().unwrap();
        trace!("agent drop <<");
    }
}

pub struct AgentService {
    agent: RefCell<Option<Agent>>,
}

impl Agent {
    pub fn new() -> Agent {
        let ctx = zmq::Context::new();
        let (send_soc, recv_soc) = _create_zmq_socket_pair("agent", true).unwrap();
        let mut worker = AgentWorker {
            cmd_socket: recv_soc,
            agent_connections: Vec::new(),
        };
        Agent {
            cmd_socket: send_soc,
            worker: Some(thread::spawn(move || { worker.run() }))
        }
    }
}

impl AgentService {
    pub fn new() -> AgentService {
        AgentService { agent: RefCell::new((None)) }
    }

    pub fn connect(&self, sender_did: &str, my_sk: &str, my_pk: &str, endpoint: &str) -> Result<(), CommonError> {
        let mut agent = self.agent.borrow_mut();
        if agent.is_none() {
            *agent = Some(Agent::new());
        }
        let conn_handle = SequenceUtils::get_next_id();
        let connect_cmd: AgentWorkerCommand = AgentWorkerCommand::Connect(ConnectCmd {
            my_did: sender_did.to_string(),
            my_sk: my_sk.to_string(),
            my_pk: my_pk.to_string(),
            endpoint: endpoint.to_string(),
        });
        agent.as_ref().unwrap().cmd_socket.send_str(connect_cmd.to_json().unwrap().as_str(), zmq::DONTWAIT).unwrap();
        Ok(())
    }
}

impl AgentWorker {
    pub fn run(&mut self) {
        loop {
            trace!("agent worker poll loop >>");
            self.cmd_socket.poll(zmq::POLLIN, -1).unwrap();
            let s = self.cmd_socket.recv_string(zmq::DONTWAIT).unwrap().unwrap();
            let cmd = AgentWorkerCommand::from_json(s.as_str()).unwrap();
            match cmd {
                AgentWorkerCommand::Connect(cmd) => unimplemented!(),
                AgentWorkerCommand::Exit => break,
            }
            info!("received cmd {}", s);
            trace!("agent worker poll loop <<");
        }
        trace!("agent poll finished");
    }
}

impl RemoteAgent {
    fn new(pub_key: &str, sec_key: &str, ver_key: &str, addr: &str) -> Result<RemoteAgent, zmq::Error> {
        Ok(RemoteAgent {
            socket: zmq::Context::new().socket(zmq::SocketType::DEALER)?,
            public_key: pub_key.to_string(),
            secret_key: sec_key.to_string(),
            server_key: ver_key.to_string(),
            addr: addr.to_string(),
        })
    }

    fn connect(&self) -> Result<(), PoolError> {
        impl From<zmq::EncodeError> for PoolError {
            fn from(err: zmq::EncodeError) -> PoolError {
                PoolError::InvalidState(format!("Invalid data stored RemoteAgent detected while connect {:?}", err))
            }
        }
        self.socket.set_identity(self.public_key.as_bytes())?;
        self.socket.set_curve_secretkey(self.secret_key.as_str())?;
        self.socket.set_curve_publickey(self.public_key.as_str())?;
        self.socket.set_curve_serverkey(self.server_key.as_str())?;
        self.socket.set_linger(0)?; //TODO set correct timeout
        self.socket.connect(self.addr.as_str())?;
        self.socket.send_str("DID", zmq::DONTWAIT)?;
        Ok(())
    }
}

#[serde(tag = "cmd")]
#[derive(Serialize, Deserialize, Debug)]
enum AgentWorkerCommand {
    Connect(ConnectCmd),
    Exit,
}

impl JsonEncodable for AgentWorkerCommand {}

impl<'a> JsonDecodable<'a> for AgentWorkerCommand {}

#[derive(Serialize, Deserialize, Debug)]
struct ConnectCmd {
    endpoint: String,
    my_did: String,
    my_sk: String,
    my_pk: String,
}

fn _create_zmq_socket_pair(address: &str, connect_and_bind: bool) -> Result<(zmq::Socket, zmq::Socket), zmq::Error> {
    let ctx = zmq::Context::new();
    let recv_soc = ctx.socket(zmq::SocketType::PAIR)?;
    let send_soc = ctx.socket(zmq::SocketType::PAIR)?;
    if connect_and_bind {
        let address = format!("inproc://{}", address);
        recv_soc.bind(&address)?;
        send_soc.connect(&address)?;
    }
    Ok((send_soc, recv_soc))
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::sync::mpsc::channel;

    use utils::timeout::TimeoutUtils;

    #[test]
    fn agent_can_be_dropped() {
        let (sender, receiver) = channel();
        thread::spawn(move || {
            {
                let agent = Agent::new();
            }
            sender.send(true).unwrap();
        });
        receiver.recv_timeout(TimeoutUtils::short_timeout()).expect("drop not finished");
    }

    #[test]
    fn agent_service_connect_works() {
        let (sender, receiver) = channel();
        let (send_soc, recv_soc) = _create_zmq_socket_pair("test_connect", true).unwrap();
        let agent = Agent {
            cmd_socket: send_soc,
            worker: Some(thread::spawn(move || {
                sender.send(recv_soc.recv_string(0).unwrap().unwrap()).unwrap()
            }))
        };
        let agent_service = AgentService {
            agent: RefCell::new(Some(agent)),
        };
        agent_service.connect("sd", "sk", "pk", "ep").unwrap();
        let str = receiver.recv_timeout(TimeoutUtils::short_timeout()).unwrap();
        assert_eq!(str, r#"{"cmd":"Connect","endpoint":"ep","my_did":"sd","my_sk":"sk","my_pk":"pk"}"#);
    }

    #[test]
    fn remote_agent_connect_works() {
        let dest = "test_agent_connect";
        let addr: String = format!("inproc://{}", dest);
        let (send_soc, recv_soc) = _create_zmq_socket_pair(dest, false).unwrap();
        recv_soc.bind(addr.as_str()).unwrap(); //TODO enable CurveCP
        let send_key_pair = zmq::CurveKeyPair::new().unwrap();
        let recv_key_pair = zmq::CurveKeyPair::new().unwrap();
        let agent = RemoteAgent {
            socket: send_soc,
            addr: addr,
            server_key: recv_key_pair.public_key,
            secret_key: send_key_pair.secret_key,
            public_key: send_key_pair.public_key,
        };
        agent.connect().unwrap();
        assert_eq!(recv_soc.recv_string(zmq::DONTWAIT).unwrap().unwrap(), "DID");
    }

    #[test]
    fn agent_service_static_create_zmq_socket_pair_works() {
        let msg = "msg";
        let sockets = _create_zmq_socket_pair("test_pair", true).unwrap();
        sockets.0.send_str(msg, zmq::DONTWAIT).unwrap();
        assert_eq!(sockets.1.recv_string(zmq::DONTWAIT).unwrap().unwrap(), msg);
    }
}<commit_msg>Implement AgentWorker::connect.<commit_after>extern crate serde_json;
extern crate zmq;

use std::cell::RefCell;
use std::thread;

use errors::common::CommonError;
use errors::pool::PoolError;
use utils::json::{JsonDecodable, JsonEncodable};
use utils::sequence::SequenceUtils;

struct RemoteAgent {
    socket: zmq::Socket,
    addr: String,
    public_key: String,
    secret_key: String,
    server_key: String,
}

struct AgentWorker {
    cmd_socket: zmq::Socket,
    agent_connections: Vec<RemoteAgent>,
}

struct Agent {
    cmd_socket: zmq::Socket,
    worker: Option<thread::JoinHandle<()>>,
}

impl Drop for Agent {
    fn drop(&mut self) {
        trace!("agent drop >>");
        self.cmd_socket.send_str(AgentWorkerCommand::Exit.to_json().unwrap().as_str(), zmq::DONTWAIT).unwrap(); //TODO
        self.worker.take().unwrap().join().unwrap();
        trace!("agent drop <<");
    }
}

pub struct AgentService {
    agent: RefCell<Option<Agent>>,
}

impl Agent {
    pub fn new() -> Agent {
        let ctx = zmq::Context::new();
        let (send_soc, recv_soc) = _create_zmq_socket_pair("agent", true).unwrap();
        let mut worker = AgentWorker {
            cmd_socket: recv_soc,
            agent_connections: Vec::new(),
        };
        Agent {
            cmd_socket: send_soc,
            worker: Some(thread::spawn(move || { worker.run() }))
        }
    }
}

impl AgentService {
    pub fn new() -> AgentService {
        AgentService { agent: RefCell::new((None)) }
    }

    pub fn connect(&self, sender_did: &str, my_sk: &str, my_pk: &str, endpoint: &str) -> Result<(), CommonError> {
        let mut agent = self.agent.borrow_mut();
        if agent.is_none() {
            *agent = Some(Agent::new());
        }
        let conn_handle = SequenceUtils::get_next_id();
        let connect_cmd: AgentWorkerCommand = AgentWorkerCommand::Connect(ConnectCmd {
            did: sender_did.to_string(),
            secret_key: my_sk.to_string(),
            public_key: my_pk.to_string(),
            endpoint: endpoint.to_string(),
            server_key: "FIXME".to_string(), //FIXME
        });
        agent.as_ref().unwrap().cmd_socket.send_str(connect_cmd.to_json().unwrap().as_str(), zmq::DONTWAIT).unwrap();
        Ok(())
    }
}

impl AgentWorker {
    fn run(&mut self) {
        loop {
            trace!("agent worker poll loop >>");
            self.cmd_socket.poll(zmq::POLLIN, -1).unwrap();
            let s = self.cmd_socket.recv_string(zmq::DONTWAIT).unwrap().unwrap();
            let cmd = AgentWorkerCommand::from_json(s.as_str()).unwrap();
            match cmd {
                AgentWorkerCommand::Connect(cmd) => self.connect(&cmd).unwrap(),
                AgentWorkerCommand::Exit => break,
            }
            info!("received cmd {}", s);
            trace!("agent worker poll loop <<");
        }
        trace!("agent poll finished");
    }

    fn connect(&mut self, cmd: &ConnectCmd) -> Result<(), PoolError> {
        let ra = RemoteAgent::new(cmd.public_key.as_str(), cmd.secret_key.as_str(),
                                  cmd.server_key.as_str(), cmd.endpoint.as_str())
            .map_err(map_err_trace!("RemoteAgent::new failed"))?;
        ra.connect().map_err(map_err_trace!("RemoteAgent::connect failed"))?;
        self.agent_connections.push(ra);
        Ok(())
    }
}

impl RemoteAgent {
    fn new(pub_key: &str, sec_key: &str, ver_key: &str, addr: &str) -> Result<RemoteAgent, zmq::Error> {
        Ok(RemoteAgent {
            socket: zmq::Context::new().socket(zmq::SocketType::DEALER)?,
            public_key: pub_key.to_string(),
            secret_key: sec_key.to_string(),
            server_key: ver_key.to_string(),
            addr: addr.to_string(),
        })
    }

    fn connect(&self) -> Result<(), PoolError> {
        impl From<zmq::EncodeError> for PoolError {
            fn from(err: zmq::EncodeError) -> PoolError {
                PoolError::InvalidState(format!("Invalid data stored RemoteAgent detected while connect {:?}", err))
            }
        }
        self.socket.set_identity(self.public_key.as_bytes()).map_err(map_err_trace!())?;
        self.socket.set_curve_secretkey(self.secret_key.as_str()).map_err(map_err_trace!())?;
        self.socket.set_curve_publickey(self.public_key.as_str()).map_err(map_err_trace!())?;
        self.socket.set_curve_serverkey(self.server_key.as_str()).map_err(map_err_trace!())?;
        self.socket.set_linger(0).map_err(map_err_trace!())?; //TODO set correct timeout
        self.socket.connect(self.addr.as_str()).map_err(map_err_trace!())?;
        self.socket.send_str("DID", zmq::DONTWAIT).map_err(map_err_trace!())?;
        Ok(())
    }
}

#[serde(tag = "cmd")]
#[derive(Serialize, Deserialize, Debug)]
enum AgentWorkerCommand {
    Connect(ConnectCmd),
    Exit,
}

impl JsonEncodable for AgentWorkerCommand {}

impl<'a> JsonDecodable<'a> for AgentWorkerCommand {}

#[derive(Serialize, Deserialize, Debug)]
struct ConnectCmd {
    endpoint: String,
    did: String,
    secret_key: String,
    public_key: String,
    server_key: String,
}

fn _create_zmq_socket_pair(address: &str, connect_and_bind: bool) -> Result<(zmq::Socket, zmq::Socket), zmq::Error> {
    let ctx = zmq::Context::new();
    let recv_soc = ctx.socket(zmq::SocketType::PAIR)?;
    let send_soc = ctx.socket(zmq::SocketType::PAIR)?;
    if connect_and_bind {
        let address = format!("inproc://{}", address);
        recv_soc.bind(&address)?;
        send_soc.connect(&address)?;
    }
    Ok((send_soc, recv_soc))
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::sync::mpsc::channel;

    use utils::timeout::TimeoutUtils;

    #[test]
    fn agent_can_be_dropped() {
        let (sender, receiver) = channel();
        thread::spawn(move || {
            {
                let agent = Agent::new();
            }
            sender.send(true).unwrap();
        });
        receiver.recv_timeout(TimeoutUtils::short_timeout()).expect("drop not finished");
    }

    #[test]
    fn agent_service_connect_works() {
        let (sender, receiver) = channel();
        let (send_soc, recv_soc) = _create_zmq_socket_pair("test_connect", true).unwrap();
        let agent = Agent {
            cmd_socket: send_soc,
            worker: Some(thread::spawn(move || {
                sender.send(recv_soc.recv_string(0).unwrap().unwrap()).unwrap()
            }))
        };
        let agent_service = AgentService {
            agent: RefCell::new(Some(agent)),
        };
        agent_service.connect("sd", "sk", "pk", "ep").unwrap();
        let expected_cmd = ConnectCmd {
            server_key: "FIXME".to_string(),
            public_key: "pk".to_string(),
            secret_key: "sk".to_string(),
            endpoint: "ep".to_string(),
            did: "sd".to_string(),
        };
        let str = receiver.recv_timeout(TimeoutUtils::short_timeout()).unwrap();
        assert_eq!(str, AgentWorkerCommand::Connect(expected_cmd).to_json().unwrap());
    }

    #[test]
    fn agent_worker_connect_works() {
        ::utils::logger::LoggerUtils::init();
        let send_key_pair = zmq::CurveKeyPair::new().unwrap();
        let recv_key_pair = zmq::CurveKeyPair::new().unwrap();
        let ctx = zmq::Context::new();
        let recv_soc = ctx.socket(zmq::SocketType::ROUTER).unwrap();
        recv_soc.set_curve_publickey(recv_key_pair.public_key.as_str()).unwrap();
        recv_soc.set_curve_secretkey(recv_key_pair.secret_key.as_str()).unwrap();
        recv_soc.set_curve_server(true).unwrap();
        recv_soc.bind("tcp://127.0.0.1:*").unwrap();
        let addr = recv_soc.get_last_endpoint().unwrap().unwrap();
        info!("addr {}", addr);

        let mut agent_worker = AgentWorker {
            agent_connections: Vec::new(),
            cmd_socket: zmq::Context::new().socket(zmq::SocketType::PAIR).unwrap(),
        };
        let cmd = ConnectCmd {
            endpoint: addr,
            public_key: send_key_pair.public_key.to_string(),
            secret_key: send_key_pair.secret_key.to_string(),
            did: "".to_string(),
            server_key: recv_key_pair.public_key.to_string(),
        };

        agent_worker.connect(&cmd).unwrap();

        assert_eq!(agent_worker.agent_connections.len(), 1);
        recv_soc.recv_string(0).unwrap().unwrap(); //ignore identity
        assert_eq!(recv_soc.recv_string(zmq::DONTWAIT).unwrap().unwrap(), "DID");
    }

    #[test]
    fn remote_agent_connect_works() {
        let dest = "test_agent_connect";
        let addr: String = format!("inproc://{}", dest);
        let (send_soc, recv_soc) = _create_zmq_socket_pair(dest, false).unwrap();
        recv_soc.bind(addr.as_str()).unwrap(); //TODO enable CurveCP
        let send_key_pair = zmq::CurveKeyPair::new().unwrap();
        let recv_key_pair = zmq::CurveKeyPair::new().unwrap();
        let agent = RemoteAgent {
            socket: send_soc,
            addr: addr,
            server_key: recv_key_pair.public_key,
            secret_key: send_key_pair.secret_key,
            public_key: send_key_pair.public_key,
        };
        agent.connect().unwrap();
        assert_eq!(recv_soc.recv_string(zmq::DONTWAIT).unwrap().unwrap(), "DID");
    }

    #[test]
    fn agent_service_static_create_zmq_socket_pair_works() {
        let msg = "msg";
        let sockets = _create_zmq_socket_pair("test_pair", true).unwrap();
        sockets.0.send_str(msg, zmq::DONTWAIT).unwrap();
        assert_eq!(sockets.1.recv_string(zmq::DONTWAIT).unwrap().unwrap(), msg);
    }
}<|endoftext|>


<commit_before>extern crate ndarray;

use std::f32;
use std::mem;
use self::ndarray::Zip;
use tensor::Tensor;
use ndarray_ext::NdArray;
use ndarray_ext;
use ops;

pub struct ArgMax { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMin { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMax { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceSum { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMean { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMinGrad { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMaxGrad { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceSumGrad { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMeanGrad { pub axis: isize, pub keep_dim: bool, }

impl ops::Op for ArgMax {
    fn name(&self) -> &str {
        "ArgMax"
    }

    // cf. https://github.com/tensorflow/compiler/tf2xla/kernels/index_ops.cc
    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = xs[0];
        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };
        let x_shape = x.shape();

        // 1. Make binary mask tensor (maximum is 1)
        let mut mask = {
            let max_fn = f32::max;
            let maxed = x.fold_axis(ndarray::Axis(axis), f32::MIN, move |&a, &b| max_fn(a, b));
            let maxed = ndarray_ext::expand_dims(maxed, axis);
            let mut mask = NdArray::zeros(x.shape());
            Zip::from(&mut mask).and(x).and_broadcast(&maxed).apply(
                |r, a, b| *r = ((a == b) as i32) as f32
            );
            mask
        };

        // 2. Reshape the mask to 2-ranked. e.g. (2, 3, 4) -> (8, 3) (let `axis` be 1)
        let mask = {
            // move axis to first, and remaining is put together in the 2nd axis
            let reduction_len = x_shape[axis];
            ndarray_ext::roll_axis(&mut mask, ndarray::Axis(0), ndarray::Axis(axis));
            let shape2d = (reduction_len, mask.len() / reduction_len);
            // unwrap is safe
            let mut mask = mask.into_shape(shape2d).unwrap();
            mask.swap_axes(0, 1);
            mask
        };

        // 3. Make indices (vertical vector)
        let indices = {
            let cols = mask.shape()[1];
            // unwrap is safe
            ndarray::Array::range(0., cols as f32, 1.)
                .into_shape((cols, 1))
                .unwrap()
        };

        // 4. Dot product between mask and index-tensor
        let mat = mask.dot(&indices);

        // 5. reshape it
        let result = {
            let mut final_shape = x_shape.to_vec();
            if self.keep_dim {
                final_shape[axis] = 1;
            } else {
                final_shape.remove(axis);
            }
            // unwrap is safe maybe (95% confidence...)
            mat.into_dyn()
               .into_shape(ndarray::IxDyn(final_shape.as_slice()))
               .unwrap()
        };

        result
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None]
    }
}

impl ops::Op for ReduceMin {
    fn name(&self) -> &str {
        "ReduceMin"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0];

        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };

        let min_fn = f32::min;
        let mut min = x.fold_axis(ndarray::Axis(axis), f32::MAX, move |&a, &b| min_fn(a, b));

        if self.keep_dim {
            ndarray_ext::expand_dims(min, axis)
        } else {
            min
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceMinGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], output, gy]))]
    }
}

impl ops::Op for ReduceMinGrad {
    fn name(&self) -> &str {
        "ReduceMinGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0].view();
        let y = xs[1].view();
        let gy = xs[2].view();
        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };

        let (y, gy) = if self.keep_dim {
            (y, gy)
        } else {
            (
                ndarray_ext::expand_dims_view(y, axis),
                ndarray_ext::expand_dims_view(gy, axis),
            )
        };

        // compare x and y
        let mut mask = NdArray::zeros(x.shape());
        Zip::from(&mut mask).and(x).and_broadcast(&y).apply(
            |r, a, b| {
                *r = ((a == b) as i32) as f32
            },
        );

        mask *= &gy;
        mask
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None, None]
    }
}

impl ops::Op for ReduceMax {
    fn name(&self) -> &str {
        "ReduceMax"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0];

        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };

        let max_fn = f32::max;
        let mut maxed = x.fold_axis(ndarray::Axis(axis), f32::MIN, move |&a, &b| max_fn(a, b));

        if self.keep_dim {
            ndarray_ext::expand_dims(maxed, axis)
        } else {
            maxed
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceMaxGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], output, gy]))]
    }
}

impl ops::Op for ReduceMaxGrad {
    fn name(&self) -> &str {
        "ReduceMaxGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0].view();
        let y = xs[1].view();
        let gy = xs[2].view();
        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };

        let (y, gy) = if self.keep_dim {
            (y, gy)
        } else {
            (
                ndarray_ext::expand_dims_view(y, axis),
                ndarray_ext::expand_dims_view(gy, axis),
            )
        };

        // compare x and y
        let mut mask = NdArray::zeros(x.shape());
        Zip::from(&mut mask).and(x).and_broadcast(&y).apply(
            |r, a, b| {
                *r = ((a == b) as i32) as f32
            },
        );

        mask *= &gy;
        mask
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None, None]
    }
}

impl ops::Op for ReduceMean {
    fn name(&self) -> &str {
        "ReduceMean"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x: &NdArray = xs[0];

        // TODO
        if 1 == x.ndim() {
            panic!(
                "ReduceMean: input of row vector is not supported.\
                Consider converting it to vertical vector."
            )
        }

        let axis = if -1 == self.axis {
            x.ndim()
        } else {
            self.axis as usize
        };

        if self.keep_dim {
            let ret = x.mean(ndarray::Axis(axis));
            ndarray_ext::expand_dims(ret, axis)
        } else {
            x.mean(ndarray::Axis(axis))
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceMeanGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], gy]))]
    }
}

impl ops::Op for ReduceMeanGrad {
    fn name(&self) -> &str {
        "ReduceMeanGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = xs[0];
        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };

        // add dim for broadcast (the result is "view")
        let gy = if self.keep_dim {
            xs[1].view()
        } else {
            ndarray_ext::expand_dims_view(xs[1].view(), axis)
        };

        // do broadcast and division
        if let Some(gx) = gy.broadcast(x.shape()) {
            let mut gx = gx.to_owned();
            let reduction_len = x.shape()[axis] as f32;
            gx /= reduction_len;
            gx
        } else {
            panic!("Lop implementation of immediate successor of ReduceMean is wrong")
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None]
    }
}

impl ops::Op for ReduceSum {
    fn name(&self) -> &str {
        "ReduceSum"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = &xs[0];

        let axis = if -1 == self.axis {
            x.ndim()
        } else {
            self.axis as usize
        };

        if self.keep_dim {
            let ret = x.sum(ndarray::Axis(axis));
            ndarray_ext::expand_dims(ret, axis)
        } else {
            x.sum(ndarray::Axis(axis))
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceSumGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], gy]))]
    }
}

impl ops::Op for ReduceSumGrad {
    fn name(&self) -> &str {
        "ReduceSumGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = xs[0];
        let axis = if -1 == self.axis {
            x.ndim() - 1
        } else {
            self.axis as usize
        };

        // add dim for broadcast (the result is "view")
        let gy = if self.keep_dim {
            xs[1].view()
        } else {
            ndarray_ext::expand_dims_view(xs[1].view(), axis)
        };

        // do broadcast
        if let Some(gx) = gy.broadcast(x.shape()) {
            gx.to_owned()
        } else {
            panic!("Lop implementation of immediate successor of ReduceSum is wrong")
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None]
    }
}
<commit_msg>style fix<commit_after>extern crate ndarray;

use std::f32;
use std::mem;
use self::ndarray::Zip;
use tensor::Tensor;
use ndarray_ext::NdArray;
use ndarray_ext;
use ops;

pub struct ArgMax { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMin { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMax { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceSum { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMean { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMinGrad { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMaxGrad { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceSumGrad { pub axis: isize, pub keep_dim: bool, }
pub struct ReduceMeanGrad { pub axis: isize, pub keep_dim: bool, }

impl ops::Op for ArgMax {
    fn name(&self) -> &str {
        "ArgMax"
    }

    // cf. https://github.com/tensorflow/compiler/tf2xla/kernels/index_ops.cc
    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = xs[0];
        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };
        let x_shape = x.shape();

        // 1. Make binary mask tensor (maximum is 1)
        let mut mask = {
            let max_fn = f32::max;
            let maxed = x.fold_axis(ndarray::Axis(axis), f32::MIN, move |&a, &b| max_fn(a, b));
            let maxed = ndarray_ext::expand_dims(maxed, axis);
            let mut mask = NdArray::zeros(x.shape());
            Zip::from(&mut mask).and(x).and_broadcast(&maxed).apply(
                |r, a, b| *r = ((a == b) as i32) as f32
            );
            mask
        };

        // 2. Reshape the mask to 2-ranked. e.g. (2, 3, 4) -> (8, 3) (let `axis` be 1)
        let mask = {
            // move axis to first, and remaining is put together in the 2nd axis
            let reduction_len = x_shape[axis];
            ndarray_ext::roll_axis(&mut mask, ndarray::Axis(0), ndarray::Axis(axis));
            let shape2d = (reduction_len, mask.len() / reduction_len);
            // unwrap is safe
            let mut mask = mask.into_shape(shape2d).unwrap();
            mask.swap_axes(0, 1);
            mask
        };

        // 3. Make indices (vertical vector)
        let indices = {
            let cols = mask.shape()[1];
            // unwrap is safe
            ndarray::Array::range(0., cols as f32, 1.)
                .into_shape((cols, 1))
                .unwrap()
        };

        // 4. Dot product between mask and index-tensor
        let mat = mask.dot(&indices);

        // 5. reshape it
        let result = {
            let mut final_shape = x_shape.to_vec();
            if self.keep_dim {
                final_shape[axis] = 1;
            } else {
                final_shape.remove(axis);
            }
            // unwrap is safe (95% confidence...)
            mat.into_dyn()
               .into_shape(ndarray::IxDyn(final_shape.as_slice()))
               .unwrap()
        };

        result
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None]
    }
}

impl ops::Op for ReduceMin {
    fn name(&self) -> &str {
        "ReduceMin"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0];

        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };

        let min_fn = f32::min;
        let mut min = x.fold_axis(ndarray::Axis(axis), f32::MAX, move |&a, &b| min_fn(a, b));

        if self.keep_dim {
            ndarray_ext::expand_dims(min, axis)
        } else {
            min
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceMinGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], output, gy]))]
    }
}

impl ops::Op for ReduceMinGrad {
    fn name(&self) -> &str {
        "ReduceMinGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0].view();
        let y = xs[1].view();
        let gy = xs[2].view();
        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };

        let (y, gy) = if self.keep_dim {
            (y, gy)
        } else {
            (
                ndarray_ext::expand_dims_view(y, axis),
                ndarray_ext::expand_dims_view(gy, axis),
            )
        };

        // compare x and y
        let mut mask = NdArray::zeros(x.shape());
        Zip::from(&mut mask).and(x).and_broadcast(&y).apply(
            |r, a, b| {
                *r = ((a == b) as i32) as f32
            },
        );

        mask *= &gy;
        mask
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None, None]
    }
}

impl ops::Op for ReduceMax {
    fn name(&self) -> &str {
        "ReduceMax"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0];

        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };

        let max_fn = f32::max;
        let mut maxed = x.fold_axis(ndarray::Axis(axis), f32::MIN, move |&a, &b| max_fn(a, b));

        if self.keep_dim {
            ndarray_ext::expand_dims(maxed, axis)
        } else {
            maxed
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceMaxGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], output, gy]))]
    }
}

impl ops::Op for ReduceMaxGrad {
    fn name(&self) -> &str {
        "ReduceMaxGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], train: bool) -> NdArray {
        let x = xs[0].view();
        let y = xs[1].view();
        let gy = xs[2].view();
        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };

        let (y, gy) = if self.keep_dim {
            (y, gy)
        } else {
            (
                ndarray_ext::expand_dims_view(y, axis),
                ndarray_ext::expand_dims_view(gy, axis),
            )
        };

        // compare x and y
        let mut mask = NdArray::zeros(x.shape());
        Zip::from(&mut mask).and(x).and_broadcast(&y).apply(
            |r, a, b| {
                *r = ((a == b) as i32) as f32
            },
        );

        mask *= &gy;
        mask
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None, None]
    }
}

impl ops::Op for ReduceMean {
    fn name(&self) -> &str {
        "ReduceMean"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x: &NdArray = xs[0];

        // TODO
        if 1 == x.ndim() {
            panic!(
                "ReduceMean: input of row vector is not supported.\
                Consider converting it to vertical vector."
            )
        }

        let axis = if -1 == self.axis { x.ndim() } else { self.axis as usize };

        if self.keep_dim {
            let ret = x.mean(ndarray::Axis(axis));
            ndarray_ext::expand_dims(ret, axis)
        } else {
            x.mean(ndarray::Axis(axis))
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceMeanGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], gy]))]
    }
}

impl ops::Op for ReduceMeanGrad {
    fn name(&self) -> &str {
        "ReduceMeanGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = xs[0];
        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };

        // add dim for broadcast (the result is "view")
        let gy = if self.keep_dim {
            xs[1].view()
        } else {
            ndarray_ext::expand_dims_view(xs[1].view(), axis)
        };

        // do broadcast and division
        if let Some(gx) = gy.broadcast(x.shape()) {
            let mut gx = gx.to_owned();
            let reduction_len = x.shape()[axis] as f32;
            gx /= reduction_len;
            gx
        } else {
            panic!("Lop implementation of immediate successor of ReduceMean is wrong")
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None]
    }
}

impl ops::Op for ReduceSum {
    fn name(&self) -> &str {
        "ReduceSum"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = &xs[0];

        let axis = if -1 == self.axis { x.ndim() } else { self.axis as usize };

        if self.keep_dim {
            let ret = x.sum(ndarray::Axis(axis));
            ndarray_ext::expand_dims(ret, axis)
        } else {
            x.sum(ndarray::Axis(axis))
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        let grad_op = ReduceSumGrad {
            axis: self.axis,
            keep_dim: self.keep_dim,
        };
        vec![Some(ops::apply_op(grad_op, &[inputs[0], gy]))]
    }
}

impl ops::Op for ReduceSumGrad {
    fn name(&self) -> &str {
        "ReduceSumGrad"
    }

    fn compute(&mut self, xs: &[&NdArray], _: bool) -> NdArray {
        let x = xs[0];
        let axis = if -1 == self.axis { x.ndim() - 1 } else { self.axis as usize };

        // add dim for broadcast (the result is "view")
        let gy = if self.keep_dim {
            xs[1].view()
        } else {
            ndarray_ext::expand_dims_view(xs[1].view(), axis)
        };

        // do broadcast
        if let Some(gx) = gy.broadcast(x.shape()) {
            gx.to_owned()
        } else {
            panic!("Lop implementation of immediate successor of ReduceSum is wrong")
        }
    }

    fn lop(&self, gy: &Tensor, inputs: &[&Tensor], output: &Tensor) -> Vec<Option<Tensor>> {
        vec![None, None]
    }
}
<|endoftext|>