<commit_before>extern crate chrono;
extern crate walkdir;

use self::chrono::{Duration, UTC};
use self::walkdir::{DirEntry as WalkDirEntry, WalkDir, WalkDirIterator};

use std::borrow::Borrow;
use std::collections::{BTreeSet, VecDeque};
use std::ffi::OsString;
use std::fs::{self, DirEntry};
use std::io::Error as IoError;
use std::os::unix::fs::MetadataExt;
use std::path::{MAIN_SEPARATOR, Path, PathBuf};
use std::result::Result;

macro_rules! path_buf {
    ($($x: expr),+) => ({
        let mut path = PathBuf::new();
        $(path.push($x);)+
        path
    });
}

pub fn trim_current_dir_prefix<S: AsRef<str>>(path_name: S) -> String {
    let pattern = format!(".{}", MAIN_SEPARATOR);

    path_name
        .as_ref()
        .trim_left_matches(&pattern)
        .to_string()
}

pub fn ls<P: AsRef<Path>>(path: P) -> Result<Vec<String>, IoError> {
    let dirs = try!(fs::read_dir(path));
    let dirs = dirs
        .filter_map(Result::ok)
        .map(|d| d.file_name())
        .map(OsString::into_string)
        .filter_map(Result::ok)
        .collect::<Vec<String>>();
    Ok(dirs)
}

pub fn is_empty_dir<P: AsRef<Path>>(path: P) -> bool {
    match fs::read_dir(path) {
        Ok(rd) => rd.count() == 0,
        Err(_) => false,
    }
}

pub fn is_recently_accessed<P: AsRef<Path>, D: Borrow<Duration>>(p: P, moratorium: D) -> bool {
    let threshold = UTC::now() - *moratorium.borrow();

    let accessed_time = match p.as_ref().metadata() {
        Ok(m)  => m.atime(),
        Err(_) => unreachable!("Wrong to use this function!!"),
    };

    accessed_time > threshold.timestamp()
}

fn is_hidden_name(file_name: &str) -> bool {
    file_name.starts_with(".") && file_name.len() > 1 && file_name != ".."
}

fn is_hidden(entry: &WalkDirEntry) -> bool {
    entry
        .file_name()
        .to_str()
        .map_or(false, is_hidden_name)
}

pub fn walk_dir<P: AsRef<Path>>(root: P) -> BTreeSet<String> {
    let walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_map(Result::ok)
        .filter(|e| !e.file_type().is_dir())
        .collect::<Vec<WalkDirEntry>>();

    walker
        .into_iter()
        .filter_map(|e| e.path().to_str().map(trim_current_dir_prefix))
        .collect::<BTreeSet<String>>()
}

pub fn potential_empty_dirs<P: AsRef<Path>>(root: P) -> Result<BTreeSet<PathBuf>, IoError> {
    fn potential_empty_dirs(mut result: BTreeSet<PathBuf>, mut target_dirs: VecDeque<PathBuf>) -> Result<BTreeSet<PathBuf>, IoError> {
        match target_dirs.pop_front() {
            None => Ok(result),
            Some(mut target_dir) => {
                let entries = try!(fs::read_dir(&target_dir))
                    .filter_map(Result::ok)
                    .collect::<Vec<DirEntry>>();

                let include_file = entries
                    .iter()
                    .any(|entry| entry
                        .file_type()
                        .map(|t| t.is_file())
                        .unwrap_or(true)
                    );
                if include_file {
                    loop {
                        result.remove(&target_dir);
                        if !target_dir.pop(){
                            break;
                        }
                    }
                }

                let dirs = entries
                    .into_iter()
                    .filter_map(|entry| {
                        let is_dir = entry
                            .file_type()
                            .map(|t| t.is_dir())
                            .unwrap_or(false);
                        if is_dir {
                            Some(entry.path())
                        } else {
                            None
                        }
                    })
                    .collect::<BTreeSet<PathBuf>>();
                for dir in dirs.clone().into_iter() {
                    result.insert(dir);
                }

                let mut dirs = dirs.into_iter().collect::<VecDeque<PathBuf>>();
                target_dirs.append(&mut dirs);

                potential_empty_dirs(result, target_dirs)
            },
        }
    }

    let mut result     : BTreeSet<PathBuf> = BTreeSet::new();
    let mut target_dirs: VecDeque<PathBuf> = VecDeque::new();

    let root = root.as_ref().to_path_buf();
    result.insert(root.clone());
    target_dirs.push_back(root);

    potential_empty_dirs(result, target_dirs)
}

pub fn dirs_ordered_by_descending_depth<P: AsRef<Path>>(root: P) -> Vec<PathBuf> {
    let mut walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_entry(|e| e.file_type().is_dir())
        .filter_map(Result::ok)
        .collect::<Vec<WalkDirEntry>>();

    walker.sort_by(|l, r| {
        let l_depth = &l.depth();
        let r_depth = &r.depth();
        r_depth.cmp(l_depth)
    });

    walker
        .into_iter()
        .map(|e| e.path().to_path_buf())
        .collect::<Vec<PathBuf>>()
}

#[test]
fn is_hidden_name_should_return_true() {
    let file_names = [
        ".a",
        ".ab",
        ".abc",
        "..a",
        "...",
    ];
    for file_name in &file_names {
        assert!(is_hidden_name(file_name));
    }
}
#[test]
fn is_hidden_name_should_return_false() {
    let file_names = [
        "",

        ".",
        "..",
        "a",
        "ab",
        "abc",
    ];
    for file_name in &file_names {
        assert!(!is_hidden_name(file_name));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::collections::BTreeSet;
    use std::fs::{self, File};
    use std::io::Write;
    use std::path::{Path, PathBuf};

    struct Helper {
        d1: String,
        d2: String,
        d3: String,
        f1: String,
        f2: String,
    }
    impl Helper {
        fn new(suffix: &str) -> Self {
            Helper {
                d1: format!("directory1_{}", suffix),
                d2: format!("directory2_{}", suffix),
                d3: format!("directory3_{}", suffix),
                f1: format!("file1_{}"     , suffix),
                f2: format!("file2_{}"     , suffix),
            }
        }

        fn path_to_d1(&self) -> PathBuf {
            PathBuf::new().join(&self.d1)
        }
        fn path_to_d2(&self) -> PathBuf {
            self.path_to_d1().join(&self.d2)
        }
        fn path_to_d3(&self) -> PathBuf {
            self.path_to_d2().join(&self.d3)
        }

        fn path_to_f1(&self) -> PathBuf {
            self.path_to_d1().join(&self.f1)
        }
        fn path_to_f2(&self) -> PathBuf {
            self.path_to_d2().join(&self.f2)
        }

        fn remove_dirs_and_files(&self) {
            fs::remove_dir_all(self.path_to_d1()).ok();
        }

        fn create_dirs_and_files(&self) {
            self.remove_dirs_and_files();

            fs::create_dir_all(self.path_to_d3()).ok();

            let mut f = File::create(self.path_to_f1()).unwrap();
            f.write("\n".as_ref()).ok();
            fs::copy(self.path_to_f1(), self.path_to_f2()).ok();
        }

        fn to_string_forcely(path: PathBuf) -> String {
            path.to_str().unwrap().to_string()
        }
    }

    #[test]
    fn path_buf_macro_should_create_path_buf() {
        let paths = [
            (Path::new("path").to_path_buf()          , path_buf!["path"              ]),
            (Path::new("path").join("to")             , path_buf!["path", "to"        ]),
            (Path::new("path").join("to").join("file"), path_buf!["path", "to", "file"]),
        ];
        for &(ref correct, ref calculated) in &paths {
            assert_eq!(correct, calculated);
        }
    }

    #[test]
    fn trim_current_dir_prefix_should_return_trimmed() {
        let path_names = [
            (""   , "./"    ),
            (""   , "././"  ),
            (""   , "./././"),
            ("a"  , "./a"   ),
            ("a/" , "./a/"  ),
            ("a/b", "./a/b" ),
        ];
        for &(ref correct, ref input) in &path_names {
            assert_eq!(correct.to_string(), trim_current_dir_prefix(input));
        }
    }
    #[test]
    fn trim_current_dir_prefix_should_return_as_it_is() {
        let path_names = [
            "",
            "/",
            "/a",
            "/a/b",
            "a",
            "a/",
            "a/b",
        ];
        for path_name in &path_names {
            assert_eq!(path_name.to_string(), trim_current_dir_prefix(path_name));
        }
    }

    #[test]
    fn ls_should_return_ok() {
        let helper = Helper::new("ls_Ok");
        helper.create_dirs_and_files();

        let empty_vec: Vec<String> = Vec::new();

        assert_eq!(vec![helper.d2.clone(), helper.f1.clone()], ls(helper.path_to_d1()).unwrap());
        assert_eq!(vec![helper.d3.clone(), helper.f2.clone()], ls(helper.path_to_d2()).unwrap());
        assert_eq!(empty_vec                                 , ls(helper.path_to_d3()).unwrap());

        helper.remove_dirs_and_files();
    }

    #[test]
    fn is_empty_dir_should_return_true() {
        let helper = Helper::new("is_empty_dir_true");
        helper.create_dirs_and_files();

        assert!(is_empty_dir(helper.path_to_d3()));

        helper.remove_dirs_and_files();
    }
    #[test]
    fn is_empty_dir_should_return_false() {
        let helper = Helper::new("is_empty_dir_false");
        helper.create_dirs_and_files();

        assert!(!is_empty_dir(helper.path_to_d1()));
        assert!(!is_empty_dir(helper.path_to_d2()));

        helper.remove_dirs_and_files();
    }

    #[test]
    fn walk_dir_should_return_b_tree_set() {
        let helper = Helper::new("walk_dir_BTreeSet");
        helper.create_dirs_and_files();

        let mut correct = BTreeSet::new();
        correct.insert(Helper::to_string_forcely(helper.path_to_f1()));
        correct.insert(Helper::to_string_forcely(helper.path_to_f2()));

        assert_eq!(correct, walk_dir(helper.path_to_d1()));

        helper.remove_dirs_and_files();
    }

    #[test]
    fn dirs_ordered_by_descending_depth_should_return_vec() {
        let helper = Helper::new("dirs_ordered_by_descending_depth_Vec");
        helper.create_dirs_and_files();

        let correct = vec![helper.path_to_d3(), helper.path_to_d2(), helper.path_to_d1()];

        assert_eq!(correct, dirs_ordered_by_descending_depth(helper.path_to_d1()));

        helper.remove_dirs_and_files();
    }
}
<commit_msg>[Refactoring] Break if target_dir doesn't exist in BTreeSet<commit_after>extern crate chrono;
extern crate walkdir;

use self::chrono::{Duration, UTC};
use self::walkdir::{DirEntry as WalkDirEntry, WalkDir, WalkDirIterator};

use std::borrow::Borrow;
use std::collections::{BTreeSet, VecDeque};
use std::ffi::OsString;
use std::fs::{self, DirEntry};
use std::io::Error as IoError;
use std::os::unix::fs::MetadataExt;
use std::path::{MAIN_SEPARATOR, Path, PathBuf};
use std::result::Result;

macro_rules! path_buf {
    ($($x: expr),+) => ({
        let mut path = PathBuf::new();
        $(path.push($x);)+
        path
    });
}

pub fn trim_current_dir_prefix<S: AsRef<str>>(path_name: S) -> String {
    let pattern = format!(".{}", MAIN_SEPARATOR);

    path_name
        .as_ref()
        .trim_left_matches(&pattern)
        .to_string()
}

pub fn ls<P: AsRef<Path>>(path: P) -> Result<Vec<String>, IoError> {
    let dirs = try!(fs::read_dir(path));
    let dirs = dirs
        .filter_map(Result::ok)
        .map(|d| d.file_name())
        .map(OsString::into_string)
        .filter_map(Result::ok)
        .collect::<Vec<String>>();
    Ok(dirs)
}

pub fn is_empty_dir<P: AsRef<Path>>(path: P) -> bool {
    match fs::read_dir(path) {
        Ok(rd) => rd.count() == 0,
        Err(_) => false,
    }
}

pub fn is_recently_accessed<P: AsRef<Path>, D: Borrow<Duration>>(p: P, moratorium: D) -> bool {
    let threshold = UTC::now() - *moratorium.borrow();

    let accessed_time = match p.as_ref().metadata() {
        Ok(m)  => m.atime(),
        Err(_) => unreachable!("Wrong to use this function!!"),
    };

    accessed_time > threshold.timestamp()
}

fn is_hidden_name(file_name: &str) -> bool {
    file_name.starts_with(".") && file_name.len() > 1 && file_name != ".."
}

fn is_hidden(entry: &WalkDirEntry) -> bool {
    entry
        .file_name()
        .to_str()
        .map_or(false, is_hidden_name)
}

pub fn walk_dir<P: AsRef<Path>>(root: P) -> BTreeSet<String> {
    let walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_map(Result::ok)
        .filter(|e| !e.file_type().is_dir())
        .collect::<Vec<WalkDirEntry>>();

    walker
        .into_iter()
        .filter_map(|e| e.path().to_str().map(trim_current_dir_prefix))
        .collect::<BTreeSet<String>>()
}

pub fn potential_empty_dirs<P: AsRef<Path>>(root: P) -> Result<BTreeSet<PathBuf>, IoError> {
    fn potential_empty_dirs(mut result: BTreeSet<PathBuf>, mut target_dirs: VecDeque<PathBuf>) -> Result<BTreeSet<PathBuf>, IoError> {
        match target_dirs.pop_front() {
            None => Ok(result),
            Some(mut target_dir) => {
                let entries = try!(fs::read_dir(&target_dir))
                    .filter_map(Result::ok)
                    .collect::<Vec<DirEntry>>();

                let include_file = entries
                    .iter()
                    .any(|entry| entry
                        .file_type()
                        .map(|t| t.is_file())
                        .unwrap_or(true)
                    );
                if include_file {
                    loop {
                        if !(result.remove(&target_dir) && target_dir.pop()) {
                            break;
                        }
                    }
                }

                let dirs = entries
                    .into_iter()
                    .filter_map(|entry| {
                        let is_dir = entry
                            .file_type()
                            .map(|t| t.is_dir())
                            .unwrap_or(false);
                        if is_dir {
                            Some(entry.path())
                        } else {
                            None
                        }
                    })
                    .collect::<BTreeSet<PathBuf>>();
                for dir in dirs.clone().into_iter() {
                    result.insert(dir);
                }

                let mut dirs = dirs.into_iter().collect::<VecDeque<PathBuf>>();
                target_dirs.append(&mut dirs);

                potential_empty_dirs(result, target_dirs)
            },
        }
    }

    let mut result     : BTreeSet<PathBuf> = BTreeSet::new();
    let mut target_dirs: VecDeque<PathBuf> = VecDeque::new();

    let root = root.as_ref().to_path_buf();
    result.insert(root.clone());
    target_dirs.push_back(root);

    potential_empty_dirs(result, target_dirs)
}

pub fn dirs_ordered_by_descending_depth<P: AsRef<Path>>(root: P) -> Vec<PathBuf> {
    let mut walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_entry(|e| e.file_type().is_dir())
        .filter_map(Result::ok)
        .collect::<Vec<WalkDirEntry>>();

    walker.sort_by(|l, r| {
        let l_depth = &l.depth();
        let r_depth = &r.depth();
        r_depth.cmp(l_depth)
    });

    walker
        .into_iter()
        .map(|e| e.path().to_path_buf())
        .collect::<Vec<PathBuf>>()
}

#[test]
fn is_hidden_name_should_return_true() {
    let file_names = [
        ".a",
        ".ab",
        ".abc",
        "..a",
        "...",
    ];
    for file_name in &file_names {
        assert!(is_hidden_name(file_name));
    }
}
#[test]
fn is_hidden_name_should_return_false() {
    let file_names = [
        "",

        ".",
        "..",
        "a",
        "ab",
        "abc",
    ];
    for file_name in &file_names {
        assert!(!is_hidden_name(file_name));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::collections::BTreeSet;
    use std::fs::{self, File};
    use std::io::Write;
    use std::path::{Path, PathBuf};

    struct Helper {
        d1: String,
        d2: String,
        d3: String,
        f1: String,
        f2: String,
    }
    impl Helper {
        fn new(suffix: &str) -> Self {
            Helper {
                d1: format!("directory1_{}", suffix),
                d2: format!("directory2_{}", suffix),
                d3: format!("directory3_{}", suffix),
                f1: format!("file1_{}"     , suffix),
                f2: format!("file2_{}"     , suffix),
            }
        }

        fn path_to_d1(&self) -> PathBuf {
            PathBuf::new().join(&self.d1)
        }
        fn path_to_d2(&self) -> PathBuf {
            self.path_to_d1().join(&self.d2)
        }
        fn path_to_d3(&self) -> PathBuf {
            self.path_to_d2().join(&self.d3)
        }

        fn path_to_f1(&self) -> PathBuf {
            self.path_to_d1().join(&self.f1)
        }
        fn path_to_f2(&self) -> PathBuf {
            self.path_to_d2().join(&self.f2)
        }

        fn remove_dirs_and_files(&self) {
            fs::remove_dir_all(self.path_to_d1()).ok();
        }

        fn create_dirs_and_files(&self) {
            self.remove_dirs_and_files();

            fs::create_dir_all(self.path_to_d3()).ok();

            let mut f = File::create(self.path_to_f1()).unwrap();
            f.write("\n".as_ref()).ok();
            fs::copy(self.path_to_f1(), self.path_to_f2()).ok();
        }

        fn to_string_forcely(path: PathBuf) -> String {
            path.to_str().unwrap().to_string()
        }
    }

    #[test]
    fn path_buf_macro_should_create_path_buf() {
        let paths = [
            (Path::new("path").to_path_buf()          , path_buf!["path"              ]),
            (Path::new("path").join("to")             , path_buf!["path", "to"        ]),
            (Path::new("path").join("to").join("file"), path_buf!["path", "to", "file"]),
        ];
        for &(ref correct, ref calculated) in &paths {
            assert_eq!(correct, calculated);
        }
    }

    #[test]
    fn trim_current_dir_prefix_should_return_trimmed() {
        let path_names = [
            (""   , "./"    ),
            (""   , "././"  ),
            (""   , "./././"),
            ("a"  , "./a"   ),
            ("a/" , "./a/"  ),
            ("a/b", "./a/b" ),
        ];
        for &(ref correct, ref input) in &path_names {
            assert_eq!(correct.to_string(), trim_current_dir_prefix(input));
        }
    }
    #[test]
    fn trim_current_dir_prefix_should_return_as_it_is() {
        let path_names = [
            "",
            "/",
            "/a",
            "/a/b",
            "a",
            "a/",
            "a/b",
        ];
        for path_name in &path_names {
            assert_eq!(path_name.to_string(), trim_current_dir_prefix(path_name));
        }
    }

    #[test]
    fn ls_should_return_ok() {
        let helper = Helper::new("ls_Ok");
        helper.create_dirs_and_files();

        let empty_vec: Vec<String> = Vec::new();

        assert_eq!(vec![helper.d2.clone(), helper.f1.clone()], ls(helper.path_to_d1()).unwrap());
        assert_eq!(vec![helper.d3.clone(), helper.f2.clone()], ls(helper.path_to_d2()).unwrap());
        assert_eq!(empty_vec                                 , ls(helper.path_to_d3()).unwrap());

        helper.remove_dirs_and_files();
    }

    #[test]
    fn is_empty_dir_should_return_true() {
        let helper = Helper::new("is_empty_dir_true");
        helper.create_dirs_and_files();

        assert!(is_empty_dir(helper.path_to_d3()));

        helper.remove_dirs_and_files();
    }
    #[test]
    fn is_empty_dir_should_return_false() {
        let helper = Helper::new("is_empty_dir_false");
        helper.create_dirs_and_files();

        assert!(!is_empty_dir(helper.path_to_d1()));
        assert!(!is_empty_dir(helper.path_to_d2()));

        helper.remove_dirs_and_files();
    }

    #[test]
    fn walk_dir_should_return_b_tree_set() {
        let helper = Helper::new("walk_dir_BTreeSet");
        helper.create_dirs_and_files();

        let mut correct = BTreeSet::new();
        correct.insert(Helper::to_string_forcely(helper.path_to_f1()));
        correct.insert(Helper::to_string_forcely(helper.path_to_f2()));

        assert_eq!(correct, walk_dir(helper.path_to_d1()));

        helper.remove_dirs_and_files();
    }

    #[test]
    fn dirs_ordered_by_descending_depth_should_return_vec() {
        let helper = Helper::new("dirs_ordered_by_descending_depth_Vec");
        helper.create_dirs_and_files();

        let correct = vec![helper.path_to_d3(), helper.path_to_d2(), helper.path_to_d1()];

        assert_eq!(correct, dirs_ordered_by_descending_depth(helper.path_to_d1()));

        helper.remove_dirs_and_files();
    }
}
<|endoftext|>


<commit_before>macro_rules! get(
    ($e:expr) => (match $e { Some(e) => e, None => return None })
);

/// `doc!` is a shortcut that helps building `Document`
/// objects.
///
/// Assuming that `field1` and `field2` are `Field` instances.
/// You can create a document with a value of `value1` for `field1`
/// `value2` for `field2`, as follows :
///
/// ```c
/// doc!(
///     field1 => value1,
///     field2 => value2,
/// )
/// ```
///
/// The value can be a `u64`, a `&str`, a `i64`, or a `String`.
///
/// # Warning
///
/// The document hence created, is not yet validated against a schema.
/// Nothing prevents its user from creating an invalid document missing a
/// field, or associating a `String` to a `u64` field for instance.
///
/// # Example
///
/// ```
/// #[macro_use]
/// extern crate tantivy;
///
/// use tantivy::schema::{SchemaBuilder, TEXT, FAST};
///
/// //...
///
/// # fn main() {
/// let mut schema_builder = SchemaBuilder::new();
/// let title = schema_builder.add_text_field("title", TEXT);
/// let author = schema_builder.add_text_field("text", TEXT);
/// let likes = schema_builder.add_u64_field("num_u64", FAST);
/// let schema = schema_builder.build();
/// let doc = doc!(
/// 	title => "Life Aquatic",
/// 	author => "Wes Anderson",
/// 	likes => 4u64
/// );
/// # }
/// ```
#[macro_export]
macro_rules! doc(
    () => {
        {
            ($crate::Document::default())
        }
    }; // avoids a warning due to the useless `mut`.
    ($($field:expr => $value:expr),*) => {
        {
            let mut document = $crate::Document::default();
            $(
                document.add($crate::schema::FieldValue::new($field, $value.into()));
            )*
            document
        }
    };
);
<commit_msg>Support trailing commas using ',+ ,' trick from Blandy 2017. (#250)<commit_after>macro_rules! get(
    ($e:expr) => (match $e { Some(e) => e, None => return None })
);

/// `doc!` is a shortcut that helps building `Document`
/// objects.
///
/// Assuming that `field1` and `field2` are `Field` instances.
/// You can create a document with a value of `value1` for `field1`
/// `value2` for `field2`, as follows :
///
/// ```c
/// doc!(
///     field1 => value1,
///     field2 => value2,
/// )
/// ```
///
/// The value can be a `u64`, a `&str`, a `i64`, or a `String`.
///
/// # Warning
///
/// The document hence created, is not yet validated against a schema.
/// Nothing prevents its user from creating an invalid document missing a
/// field, or associating a `String` to a `u64` field for instance.
///
/// # Example
///
/// ```
/// #[macro_use]
/// extern crate tantivy;
///
/// use tantivy::schema::{SchemaBuilder, TEXT, FAST};
///
/// //...
///
/// # fn main() {
/// let mut schema_builder = SchemaBuilder::new();
/// let title = schema_builder.add_text_field("title", TEXT);
/// let author = schema_builder.add_text_field("text", TEXT);
/// let likes = schema_builder.add_u64_field("num_u64", FAST);
/// let schema = schema_builder.build();
/// let doc = doc!(
/// 	title => "Life Aquatic",
/// 	author => "Wes Anderson",
/// 	likes => 4u64
/// );
/// # }
/// ```
#[macro_export]
macro_rules! doc(
    () => {
        {
            ($crate::Document::default())
        }
    }; // avoids a warning due to the useless `mut`.
    ($($field:expr => $value:expr),*) => {
        {
            let mut document = $crate::Document::default();
            $(
                document.add($crate::schema::FieldValue::new($field, $value.into()));
            )*
            document
        }
    };
    // if there is a trailing comma retry with the trailing comma stripped.
    ($($field:expr => $value:expr),+ ,) => {
        doc!( $( $field => $value ), *);
    };
);

#[cfg(test)]
mod test {
    use schema::{SchemaBuilder, TEXT, FAST};

    #[test]
    fn test_doc_basic() {
        let mut schema_builder = SchemaBuilder::new();
        let title = schema_builder.add_text_field("title", TEXT);
        let author = schema_builder.add_text_field("text", TEXT);
        let likes = schema_builder.add_u64_field("num_u64", FAST);
        let _schema = schema_builder.build();
        let _doc = doc!(
            title => "Life Aquatic",
            author => "Wes Anderson",
            likes => 4u64
            );
    }

    #[test]
    fn test_doc_trailing_comma() {
        let mut schema_builder = SchemaBuilder::new();
        let title = schema_builder.add_text_field("title", TEXT);
        let author = schema_builder.add_text_field("text", TEXT);
        let likes = schema_builder.add_u64_field("num_u64", FAST);
        let _schema = schema_builder.build();
        let _doc = doc!(
            title => "Life Aquatic",
            author => "Wes Anderson",
            likes => 4u64,
            );
    }
}
<|endoftext|>


<commit_before>/// Return early with an error.
///
/// This macro is equivalent to `return Err(`[`anyhow!($args...)`][anyhow!]`)`.
///
/// The surrounding function's or closure's return value is required to be
/// `Result<_,`[`anyhow::Error`][crate::Error]`>`.
///
/// # Example
///
/// ```
/// # use anyhow::{bail, Result};
/// #
/// # fn has_permission(user: usize, resource: usize) -> bool {
/// #     true
/// # }
/// #
/// # fn main() -> Result<()> {
/// #     let user = 0;
/// #     let resource = 0;
/// #
/// if !has_permission(user, resource) {
///     bail!("permission denied for accessing {}", resource);
/// }
/// #     Ok(())
/// # }
/// ```
///
/// ```
/// # use anyhow::{bail, Result};
/// # use thiserror::Error;
/// #
/// # const MAX_DEPTH: usize = 1;
/// #
/// #[derive(Error, Debug)]
/// enum ScienceError {
///     #[error("recursion limit exceeded")]
///     RecursionLimitExceeded,
///     # #[error("...")]
///     # More = (stringify! {
///     ...
///     # }, 1).1,
/// }
///
/// # fn main() -> Result<()> {
/// #     let depth = 0;
/// #
/// if depth > MAX_DEPTH {
///     bail!(ScienceError::RecursionLimitExceeded);
/// }
/// #     Ok(())
/// # }
/// ```
#[macro_export]
macro_rules! bail {
    ($msg:literal $(,)?) => {
        return $crate::private::Err($crate::anyhow!($msg))
    };
    ($err:expr $(,)?) => {
        return $crate::private::Err($crate::anyhow!($err))
    };
    ($fmt:expr, $($arg:tt)*) => {
        return $crate::private::Err($crate::anyhow!($fmt, $($arg)*))
    };
}

/// Return early with an error if a condition is not satisfied.
///
/// This macro is equivalent to `if !$cond { return
/// Err(`[`anyhow!($args...)`][anyhow!]`); }`.
///
/// The surrounding function's or closure's return value is required to be
/// `Result<_,`[`anyhow::Error`][crate::Error]`>`.
///
/// Analogously to `assert!`, `ensure!` takes a condition and exits the function
/// if the condition fails. Unlike `assert!`, `ensure!` returns an `Error`
/// rather than panicking.
///
/// # Example
///
/// ```
/// # use anyhow::{ensure, Result};
/// #
/// # fn main() -> Result<()> {
/// #     let user = 0;
/// #
/// ensure!(user == 0, "only user 0 is allowed");
/// #     Ok(())
/// # }
/// ```
///
/// ```
/// # use anyhow::{ensure, Result};
/// # use thiserror::Error;
/// #
/// # const MAX_DEPTH: usize = 1;
/// #
/// #[derive(Error, Debug)]
/// enum ScienceError {
///     #[error("recursion limit exceeded")]
///     RecursionLimitExceeded,
///     # #[error("...")]
///     # More = (stringify! {
///     ...
///     # }, 1).1,
/// }
///
/// # fn main() -> Result<()> {
/// #     let depth = 0;
/// #
/// ensure!(depth <= MAX_DEPTH, ScienceError::RecursionLimitExceeded);
/// #     Ok(())
/// # }
/// ```
#[macro_export]
macro_rules! ensure {
    ($cond:expr $(,)?) => {
        if !$cond {
            return $crate::private::Err($crate::Error::msg(
                $crate::private::concat!("Condition failed: `", $crate::private::stringify!($cond), "`")
            ));
        }
    };
    ($cond:expr, $msg:literal $(,)?) => {
        if !$cond {
            return $crate::private::Err($crate::anyhow!($msg));
        }
    };
    ($cond:expr, $err:expr $(,)?) => {
        if !$cond {
            return $crate::private::Err($crate::anyhow!($err));
        }
    };
    ($cond:expr, $fmt:expr, $($arg:tt)*) => {
        if !$cond {
            return $crate::private::Err($crate::anyhow!($fmt, $($arg)*));
        }
    };
}

/// Construct an ad-hoc error from a string or existing non-`anyhow` error
/// value.
///
/// This evaluates to an [`Error`][crate::Error]. It can take either just a
/// string, or a format string with arguments. It also can take any custom type
/// which implements `Debug` and `Display`.
///
/// If called with a single argument whose type implements `std::error::Error`
/// (in addition to `Debug` and `Display`, which are always required), then that
/// Error impl's `source` is preserved as the `source` of the resulting
/// `anyhow::Error`.
///
/// # Example
///
/// ```
/// # type V = ();
/// #
/// use anyhow::{anyhow, Result};
///
/// fn lookup(key: &str) -> Result<V> {
///     if key.len() != 16 {
///         return Err(anyhow!("key length must be 16 characters, got {:?}", key));
///     }
///
///     // ...
///     # Ok(())
/// }
/// ```
#[macro_export]
macro_rules! anyhow {
    ($msg:literal $(,)?) => ({
        let error = $crate::private::format_err($crate::private::format_args!($msg));
        error
    });
    ($err:expr $(,)?) => ({
        use $crate::private::kind::*;
        let error = match $err {
            error => (&error).anyhow_kind().new(error),
        };
        error
    });
    ($fmt:expr, $($arg:tt)*) => {
        $crate::Error::msg($crate::private::format!($fmt, $($arg)*))
    };
}
<commit_msg>Add workaround for the bail msg from log4rs crate<commit_after>/// Return early with an error.
///
/// This macro is equivalent to `return Err(`[`anyhow!($args...)`][anyhow!]`)`.
///
/// The surrounding function's or closure's return value is required to be
/// `Result<_,`[`anyhow::Error`][crate::Error]`>`.
///
/// # Example
///
/// ```
/// # use anyhow::{bail, Result};
/// #
/// # fn has_permission(user: usize, resource: usize) -> bool {
/// #     true
/// # }
/// #
/// # fn main() -> Result<()> {
/// #     let user = 0;
/// #     let resource = 0;
/// #
/// if !has_permission(user, resource) {
///     bail!("permission denied for accessing {}", resource);
/// }
/// #     Ok(())
/// # }
/// ```
///
/// ```
/// # use anyhow::{bail, Result};
/// # use thiserror::Error;
/// #
/// # const MAX_DEPTH: usize = 1;
/// #
/// #[derive(Error, Debug)]
/// enum ScienceError {
///     #[error("recursion limit exceeded")]
///     RecursionLimitExceeded,
///     # #[error("...")]
///     # More = (stringify! {
///     ...
///     # }, 1).1,
/// }
///
/// # fn main() -> Result<()> {
/// #     let depth = 0;
/// #
/// if depth > MAX_DEPTH {
///     bail!(ScienceError::RecursionLimitExceeded);
/// }
/// #     Ok(())
/// # }
/// ```
#[cfg(doc)]
#[macro_export]
macro_rules! bail {
    ($msg:literal $(,)?) => {
        return $crate::private::Err($crate::anyhow!($msg))
    };
    ($err:expr $(,)?) => {
        return $crate::private::Err($crate::anyhow!($err))
    };
    ($fmt:expr, $($arg:tt)*) => {
        return $crate::private::Err($crate::anyhow!($fmt, $($arg)*))
    };
}

// Workaround for crates that intentionally contained `{}` in an error message
// prior to https://github.com/dtolnay/anyhow/issues/55 catching the missing
// format args.
#[cfg(not(doc))]
#[macro_export]
macro_rules! bail {
    // https://github.com/estk/log4rs/blob/afa0351af56b3bfd1780389700051d7e4d8bbdc9/src/append/rolling_file/policy/compound/roll/fixed_window.rs#L261
    ("pattern does not contain `{}`") => {
        return $crate::private::Err($crate::Error::msg("pattern does not contain `{}`"))
    };
    ($msg:literal $(,)?) => {
        return $crate::private::Err($crate::anyhow!($msg))
    };
    ($err:expr $(,)?) => {
        return $crate::private::Err($crate::anyhow!($err))
    };
    ($fmt:expr, $($arg:tt)*) => {
        return $crate::private::Err($crate::anyhow!($fmt, $($arg)*))
    };
}

/// Return early with an error if a condition is not satisfied.
///
/// This macro is equivalent to `if !$cond { return
/// Err(`[`anyhow!($args...)`][anyhow!]`); }`.
///
/// The surrounding function's or closure's return value is required to be
/// `Result<_,`[`anyhow::Error`][crate::Error]`>`.
///
/// Analogously to `assert!`, `ensure!` takes a condition and exits the function
/// if the condition fails. Unlike `assert!`, `ensure!` returns an `Error`
/// rather than panicking.
///
/// # Example
///
/// ```
/// # use anyhow::{ensure, Result};
/// #
/// # fn main() -> Result<()> {
/// #     let user = 0;
/// #
/// ensure!(user == 0, "only user 0 is allowed");
/// #     Ok(())
/// # }
/// ```
///
/// ```
/// # use anyhow::{ensure, Result};
/// # use thiserror::Error;
/// #
/// # const MAX_DEPTH: usize = 1;
/// #
/// #[derive(Error, Debug)]
/// enum ScienceError {
///     #[error("recursion limit exceeded")]
///     RecursionLimitExceeded,
///     # #[error("...")]
///     # More = (stringify! {
///     ...
///     # }, 1).1,
/// }
///
/// # fn main() -> Result<()> {
/// #     let depth = 0;
/// #
/// ensure!(depth <= MAX_DEPTH, ScienceError::RecursionLimitExceeded);
/// #     Ok(())
/// # }
/// ```
#[macro_export]
macro_rules! ensure {
    ($cond:expr $(,)?) => {
        if !$cond {
            return $crate::private::Err($crate::Error::msg(
                $crate::private::concat!("Condition failed: `", $crate::private::stringify!($cond), "`")
            ));
        }
    };
    ($cond:expr, $msg:literal $(,)?) => {
        if !$cond {
            return $crate::private::Err($crate::anyhow!($msg));
        }
    };
    ($cond:expr, $err:expr $(,)?) => {
        if !$cond {
            return $crate::private::Err($crate::anyhow!($err));
        }
    };
    ($cond:expr, $fmt:expr, $($arg:tt)*) => {
        if !$cond {
            return $crate::private::Err($crate::anyhow!($fmt, $($arg)*));
        }
    };
}

/// Construct an ad-hoc error from a string or existing non-`anyhow` error
/// value.
///
/// This evaluates to an [`Error`][crate::Error]. It can take either just a
/// string, or a format string with arguments. It also can take any custom type
/// which implements `Debug` and `Display`.
///
/// If called with a single argument whose type implements `std::error::Error`
/// (in addition to `Debug` and `Display`, which are always required), then that
/// Error impl's `source` is preserved as the `source` of the resulting
/// `anyhow::Error`.
///
/// # Example
///
/// ```
/// # type V = ();
/// #
/// use anyhow::{anyhow, Result};
///
/// fn lookup(key: &str) -> Result<V> {
///     if key.len() != 16 {
///         return Err(anyhow!("key length must be 16 characters, got {:?}", key));
///     }
///
///     // ...
///     # Ok(())
/// }
/// ```
#[macro_export]
macro_rules! anyhow {
    ($msg:literal $(,)?) => ({
        let error = $crate::private::format_err($crate::private::format_args!($msg));
        error
    });
    ($err:expr $(,)?) => ({
        use $crate::private::kind::*;
        let error = match $err {
            error => (&error).anyhow_kind().new(error),
        };
        error
    });
    ($fmt:expr, $($arg:tt)*) => {
        $crate::Error::msg($crate::private::format!($fmt, $($arg)*))
    };
}
<|endoftext|>


<commit_before>

error_chain! {
    links { }

    foreign_links {
        Io(::std::io::Error) #[cfg(unix)];
    }

    errors {
        HttpStatus(e: u32) {
            description("http request returned an unsuccessful status code")
            display("http request returned an unsuccessful status code: {}", e)
        }
        FileNotFound {
            description("file not found")
        }
        BackendUnavailable(be: &'static str) {
            description("download backend unavailable")
            display("download backend '{}' unavailable", be)
        }
    }
}
<commit_msg>Remove unix-only flag on std::io::Error error_chain foreign link<commit_after>

error_chain! {
    links { }

    foreign_links {
        Io(::std::io::Error);
    }

    errors {
        HttpStatus(e: u32) {
            description("http request returned an unsuccessful status code")
            display("http request returned an unsuccessful status code: {}", e)
        }
        FileNotFound {
            description("file not found")
        }
        BackendUnavailable(be: &'static str) {
            description("download backend unavailable")
            display("download backend '{}' unavailable", be)
        }
    }
}
<|endoftext|>


<commit_before>use std::ffi::{CStr, CString};
use std::marker::PhantomData;

use tensorflow_sys as tf;

use crate::eager::Context;
use crate::{AnyTensor, DataType, Result, Status, Tensor, TensorType};

/// A handle to a tensor on a device.
///
/// Constructing a TensorHandle requires a reference to an execute context so that the
/// generated handle will not out live the context.
/// ```
/// # use tensorflow::Tensor;
/// use tensorflow::eager::{ContextOptions, Context, TensorHandle};
///
/// let opts = ContextOptions::new();
/// let ctx = Context::new(opts).unwrap();
///
/// let t = Tensor::from(&[3i32]);
/// let h = TensorHandle::new(&ctx, &t).unwrap();
/// let v: Tensor<i32> = h.resolve().unwrap();
/// assert_eq!(&v[..], &[3i32]);
/// ```
///
/// Since TensorHandle cannot be alive beyond the lifetime of the context, the following
/// code will not compile.
/// ```compile_fail
/// # use tensorflow::Tensor;
/// use tensorflow::eager::{ContextOptions, Context, TensorHandle};
///
/// let h = {
///     let opts = ContextOptions::new();
///     let ctx = Context::new(opts).unwrap();
///
///     let t = Tensor::from(&[3i32]);
///     let h = TensorHandle::new(&ctx, &t).unwrap();
///     h
/// };
/// ```
#[derive(Debug)]
pub struct TensorHandle<'a> {
    inner: *mut tf::TFE_TensorHandle,
    // TensorHandle should not live longer than a given context.
    ctx: PhantomData<&'a Context>,
}

impl<'a> Drop for TensorHandle<'a> {
    fn drop(&mut self) {
        unsafe {
            tf::TFE_DeleteTensorHandle(self.inner);
        }
    }
}

impl<'a> TensorHandle<'a> {
    /// Crate a TensorHandle from the input Tensor
    pub fn new<T: TensorType>(_ctx: &'a Context, t: &Tensor<T>) -> Result<TensorHandle<'a>> {
        let status = Status::new();
        let inner = unsafe { tf::TFE_NewTensorHandle(t.inner()?, status.inner) };

        if inner.is_null() {
            Err(status)
        } else {
            Ok(TensorHandle {
                inner,
                ctx: PhantomData,
            })
        }
    }

    /// Return the DataType that corresponds to this type.
    pub fn data_type(&self) -> DataType {
        unsafe { DataType::from_c(tf::TFE_TensorHandleDataType(self.inner)) }
    }

    /// Return the number of dimensions.
    ///
    /// This function will block till the operation that produces the TensorHandle has completed.
    pub fn num_dims(&self) -> Result<usize> {
        let status = Status::new();
        let num_dims = unsafe { tf::TFE_TensorHandleNumDims(self.inner, status.inner) };
        if status.is_ok() {
            // num_dims >= 0 when the status is ok, so we can safely cast it to u64.
            Ok(num_dims as usize)
        } else {
            Err(status)
        }
    }

    /// Return the number of elements
    pub fn num_elements(&self) -> Result<u64> {
        let status = Status::new();
        let num_elements = unsafe { tf::TFE_TensorHandleNumElements(self.inner, status.inner) };
        if status.is_ok() {
            // num_elements >= 0 when the status is ok, so we can safely cast it to u64.
            Ok(num_elements as u64)
        } else {
            Err(status)
        }
    }

    /// Return the number of elements for a given dim_index.
    ///
    /// This function will block till the operation that produces the TensorHandle has completed.
    pub fn dim(&self, dim_index: i32) -> Result<u64> {
        let status = Status::new();
        let dim = unsafe { tf::TFE_TensorHandleDim(self.inner, dim_index, status.inner) };
        if status.is_ok() {
            // dim >= 0 when the status is ok, so we can safely cast it to u64.
            Ok(dim as u64)
        } else {
            Err(status)
        }
    }

    /// Return the device of the operation that produced the current TensorHandle.
    ///
    /// If the TensorHandle was produced by a copy, returns the destination device of the copy.
    /// Note that the returned device name is not always the device holding the tensor handle's memory.
    /// If you want the latter, use backing_device_name.
    ///
    /// This function will block till the operation that produces the current TensorHandle has completed.
    pub fn device_name(&self) -> Result<String> {
        let status = Status::new();
        unsafe {
            let device_name = tf::TFE_TensorHandleDeviceName(self.inner, status.inner);
            if status.is_ok() {
                Ok(CStr::from_ptr(device_name).to_str()?.to_string())
            } else {
                Err(status)
            }
        }
    }

    /// Returns the name of the device in whose memory underlying the current TensorHandle resides.
    ///
    /// This function will block till the operation that produces the current TensorHandle has completed.
    pub fn backing_device_name(&self) -> Result<String> {
        let status = Status::new();
        unsafe {
            let device_name = tf::TFE_TensorHandleBackingDeviceName(self.inner, status.inner);
            if status.is_ok() {
                Ok(CStr::from_ptr(device_name).to_str()?.to_string())
            } else {
                Err(status)
            }
        }
    }

    /// Return a new TensorHandle that shares the underlying tensor with the current TensorHandle.
    pub fn copy_sharing_tensor(&self) -> Result<Self> {
        let status = Status::new();
        let inner = unsafe { tf::TFE_TensorHandleCopySharingTensor(self.inner, status.inner) };
        if status.is_ok() {
            Ok(Self {
                inner,
                ctx: self.ctx,
            })
        } else {
            Err(status)
        }
    }

    /// This function will block till the operation that produces the current TensorHandle has completed.
    /// The memory returned might alias the internal memory used by TensorFlow.
    /// Hence, callers should not mutate this memory.
    pub fn resolve<T: TensorType>(&self) -> Result<Tensor<T>> {
        let mut status = Status::new();
        let tf_tensor = unsafe { tf::TFE_TensorHandleResolve(self.inner, status.inner) };
        if !status.is_ok() {
            return Err(status);
        }
        if self.data_type() != T::data_type() {
            let msg = format!(
                "The expected data type ({}) and underlying data type ({}) did not match.",
                T::data_type(),
                self.data_type()
            );

            status.set_lossy(crate::Code::InvalidArgument, &msg);
            return Err(status);
        }

        // Safely unwrap since data_type was checked beforehand.
        unsafe { Ok(Tensor::from_tf_tensor(tf_tensor).unwrap()) }
    }

    /// Create a new TensorHandle with the same contents as the current TensorHandle but placed
    /// in the memory of the device name 'device_name'.
    /// If source and destination are the same device, then this creates a new handle
    /// that shares the underlying buffer. Otherwise, it currently requires at least
    /// one of the source or destination devices to be CPU (i.e., for the source or
    /// destination tensor to be placed in host memory).
    /// If async execution is enabled, the copy may be enqueued and the call will
    /// return "non-ready" TensorHandle. Else, this function returns after the copy has
    /// been done.
    pub fn copy_to_device(h: &TensorHandle, ctx: &'a Context, device_name: &str) -> Result<Self> {
        let status = Status::new();

        let device_name = CString::new(device_name)?;
        unsafe {
            let inner = tf::TFE_TensorHandleCopyToDevice(
                h.inner,
                ctx.inner,
                device_name.as_ptr(),
                status.inner,
            );

            if status.is_ok() {
                Ok(Self {
                    inner,
                    ctx: PhantomData,
                })
            } else {
                Err(status)
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::eager::ContextOptions;

    #[test]
    fn test_tensor_handle() {
        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();

        let t = Tensor::new(&[2, 3])
            .with_values(&[0_i32, 1, 2, 3, 4, 5])
            .unwrap();
        let h = TensorHandle::new(&ctx, &t).unwrap();

        assert_eq!(h.data_type(), DataType::Int32);
        assert_eq!(h.num_elements().unwrap(), 6);
        assert_eq!(h.num_dims().unwrap(), 2);
        assert_eq!(h.dim(0).unwrap(), 2);
        assert_eq!(h.dim(1).unwrap(), 3);
    }

    #[test]
    fn test_copy_sharing_tensor() {
        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();

        let t = Tensor::new(&[2, 3])
            .with_values(&[0_i32, 1, 2, 3, 4, 5])
            .unwrap();
        let h = TensorHandle::new(&ctx, &t).unwrap();
        let h_copy = h.copy_sharing_tensor().unwrap();
        let t2 = h_copy.resolve::<i32>().unwrap();

        // t and t2 may share the same memory, but it's difficuly to check
        // since the `resolve` does not guarantee that.
        assert_eq!(&t[..], &t2[..]);
    }

    #[cfg(feature = "tensorflow_gpu")]
    #[test]
    fn test_copy_to_device() {
        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();
        let devices = ctx.device_list().unwrap();
        if !(devices.iter().any(|d| d.device_type == "GPU")) {
            eprintln!("Skipping test_copy_to_device because no GPU device is found");
            return;
        }

        let t = Tensor::new(&[2, 2]).with_values(&[0_i32, 1, 2, 3]).unwrap();
        let h = TensorHandle::new(&ctx, &t).unwrap();
        let target_device = "/job:localhost/replica:0/task:0/device:GPU:0";
        let h_gpu = TensorHandle::copy_to_device(&h, &ctx, target_device).unwrap();
        assert_eq!(h_gpu.device_name().unwrap(), target_device);
        let t2 = h_gpu.resolve::<i32>().unwrap();

        assert_eq!(&t[..], &t2[..]);
    }

    #[cfg(feature = "tensorflow_gpu")]
    #[test]
    fn test_copy_to_device_lifetime() {
        let values = [0_i32, 1, 2, 3];
        let target_device = "/job:localhost/replica:0/task:0/device:GPU:0";

        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();
        let devices = ctx.device_list().unwrap();
        if !(devices.iter().any(|d| d.device_type == "GPU")) {
            eprintln!("Skipping test_copy_to_device because no GPU device is found");
            return;
        }

        let h_gpu = {
            // Create a temporal Context
            let opts = ContextOptions::new();
            let ctx2 = Context::new(opts).unwrap();
            let t = Tensor::new(&[2, 2]).with_values(&values).unwrap();

            // Create a TensorHandle managed by the context `ctx2`.
            let h = TensorHandle::new(&ctx2, &t).unwrap();

            // Copy to GPU. This creates a new handle managed by the context `ctx`.
            TensorHandle::copy_to_device(&h, &ctx, target_device).unwrap()
        };
        assert_eq!(h_gpu.device_name().unwrap(), target_device);
        let t2 = h_gpu.resolve::<i32>().unwrap();

        assert_eq!(&values[..], &t2[..]);
    }
}
<commit_msg>update the doc of copy_to_device<commit_after>use std::ffi::{CStr, CString};
use std::marker::PhantomData;

use tensorflow_sys as tf;

use crate::eager::Context;
use crate::{AnyTensor, DataType, Result, Status, Tensor, TensorType};

/// A handle to a tensor on a device.
///
/// Constructing a TensorHandle requires a reference to an execute context so that the
/// generated handle will not out live the context.
/// ```
/// # use tensorflow::Tensor;
/// use tensorflow::eager::{ContextOptions, Context, TensorHandle};
///
/// let opts = ContextOptions::new();
/// let ctx = Context::new(opts).unwrap();
///
/// let t = Tensor::from(&[3i32]);
/// let h = TensorHandle::new(&ctx, &t).unwrap();
/// let v: Tensor<i32> = h.resolve().unwrap();
/// assert_eq!(&v[..], &[3i32]);
/// ```
///
/// Since TensorHandle cannot be alive beyond the lifetime of the context, the following
/// code will not compile.
/// ```compile_fail
/// # use tensorflow::Tensor;
/// use tensorflow::eager::{ContextOptions, Context, TensorHandle};
///
/// let h = {
///     let opts = ContextOptions::new();
///     let ctx = Context::new(opts).unwrap();
///
///     let t = Tensor::from(&[3i32]);
///     let h = TensorHandle::new(&ctx, &t).unwrap();
///     h
/// };
/// ```
#[derive(Debug)]
pub struct TensorHandle<'a> {
    inner: *mut tf::TFE_TensorHandle,
    // TensorHandle should not live longer than a given context.
    ctx: PhantomData<&'a Context>,
}

impl<'a> Drop for TensorHandle<'a> {
    fn drop(&mut self) {
        unsafe {
            tf::TFE_DeleteTensorHandle(self.inner);
        }
    }
}

impl<'a> TensorHandle<'a> {
    /// Crate a TensorHandle from the input Tensor
    pub fn new<T: TensorType>(_ctx: &'a Context, t: &Tensor<T>) -> Result<TensorHandle<'a>> {
        let status = Status::new();
        let inner = unsafe { tf::TFE_NewTensorHandle(t.inner()?, status.inner) };

        if inner.is_null() {
            Err(status)
        } else {
            Ok(TensorHandle {
                inner,
                ctx: PhantomData,
            })
        }
    }

    /// Return the DataType that corresponds to this type.
    pub fn data_type(&self) -> DataType {
        unsafe { DataType::from_c(tf::TFE_TensorHandleDataType(self.inner)) }
    }

    /// Return the number of dimensions.
    ///
    /// This function will block till the operation that produces the TensorHandle has completed.
    pub fn num_dims(&self) -> Result<usize> {
        let status = Status::new();
        let num_dims = unsafe { tf::TFE_TensorHandleNumDims(self.inner, status.inner) };
        if status.is_ok() {
            // num_dims >= 0 when the status is ok, so we can safely cast it to u64.
            Ok(num_dims as usize)
        } else {
            Err(status)
        }
    }

    /// Return the number of elements
    pub fn num_elements(&self) -> Result<u64> {
        let status = Status::new();
        let num_elements = unsafe { tf::TFE_TensorHandleNumElements(self.inner, status.inner) };
        if status.is_ok() {
            // num_elements >= 0 when the status is ok, so we can safely cast it to u64.
            Ok(num_elements as u64)
        } else {
            Err(status)
        }
    }

    /// Return the number of elements for a given dim_index.
    ///
    /// This function will block till the operation that produces the TensorHandle has completed.
    pub fn dim(&self, dim_index: i32) -> Result<u64> {
        let status = Status::new();
        let dim = unsafe { tf::TFE_TensorHandleDim(self.inner, dim_index, status.inner) };
        if status.is_ok() {
            // dim >= 0 when the status is ok, so we can safely cast it to u64.
            Ok(dim as u64)
        } else {
            Err(status)
        }
    }

    /// Return the device of the operation that produced the current TensorHandle.
    ///
    /// If the TensorHandle was produced by a copy, returns the destination device of the copy.
    /// Note that the returned device name is not always the device holding the tensor handle's memory.
    /// If you want the latter, use backing_device_name.
    ///
    /// This function will block till the operation that produces the current TensorHandle has completed.
    pub fn device_name(&self) -> Result<String> {
        let status = Status::new();
        unsafe {
            let device_name = tf::TFE_TensorHandleDeviceName(self.inner, status.inner);
            if status.is_ok() {
                Ok(CStr::from_ptr(device_name).to_str()?.to_string())
            } else {
                Err(status)
            }
        }
    }

    /// Returns the name of the device in whose memory underlying the current TensorHandle resides.
    ///
    /// This function will block till the operation that produces the current TensorHandle has completed.
    pub fn backing_device_name(&self) -> Result<String> {
        let status = Status::new();
        unsafe {
            let device_name = tf::TFE_TensorHandleBackingDeviceName(self.inner, status.inner);
            if status.is_ok() {
                Ok(CStr::from_ptr(device_name).to_str()?.to_string())
            } else {
                Err(status)
            }
        }
    }

    /// Return a new TensorHandle that shares the underlying tensor with the current TensorHandle.
    pub fn copy_sharing_tensor(&self) -> Result<Self> {
        let status = Status::new();
        let inner = unsafe { tf::TFE_TensorHandleCopySharingTensor(self.inner, status.inner) };
        if status.is_ok() {
            Ok(Self {
                inner,
                ctx: self.ctx,
            })
        } else {
            Err(status)
        }
    }

    /// This function will block till the operation that produces the current TensorHandle has completed.
    /// The memory returned might alias the internal memory used by TensorFlow.
    /// Hence, callers should not mutate this memory.
    pub fn resolve<T: TensorType>(&self) -> Result<Tensor<T>> {
        let mut status = Status::new();
        let tf_tensor = unsafe { tf::TFE_TensorHandleResolve(self.inner, status.inner) };
        if !status.is_ok() {
            return Err(status);
        }
        if self.data_type() != T::data_type() {
            let msg = format!(
                "The expected data type ({}) and underlying data type ({}) did not match.",
                T::data_type(),
                self.data_type()
            );

            status.set_lossy(crate::Code::InvalidArgument, &msg);
            return Err(status);
        }

        // Safely unwrap since data_type was checked beforehand.
        unsafe { Ok(Tensor::from_tf_tensor(tf_tensor).unwrap()) }
    }

    /// Create a new TensorHandle with the same contents as `h` but placed
    /// in the memory of the device name 'device_name'.
    /// If source and destination are the same device, then this creates a new handle
    /// that shares the underlying buffer. Otherwise, it currently requires at least
    /// one of the source or destination devices to be CPU (i.e., for the source or
    /// destination tensor to be placed in host memory).
    /// If async execution is enabled, the copy may be enqueued and the call will
    /// return "non-ready" TensorHandle. Else, this function returns after the copy has
    /// been done.
    pub fn copy_to_device(h: &TensorHandle, ctx: &'a Context, device_name: &str) -> Result<Self> {
        let status = Status::new();

        let device_name = CString::new(device_name)?;
        unsafe {
            let inner = tf::TFE_TensorHandleCopyToDevice(
                h.inner,
                ctx.inner,
                device_name.as_ptr(),
                status.inner,
            );

            if status.is_ok() {
                Ok(Self {
                    inner,
                    ctx: PhantomData,
                })
            } else {
                Err(status)
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::eager::ContextOptions;

    #[test]
    fn test_tensor_handle() {
        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();

        let t = Tensor::new(&[2, 3])
            .with_values(&[0_i32, 1, 2, 3, 4, 5])
            .unwrap();
        let h = TensorHandle::new(&ctx, &t).unwrap();

        assert_eq!(h.data_type(), DataType::Int32);
        assert_eq!(h.num_elements().unwrap(), 6);
        assert_eq!(h.num_dims().unwrap(), 2);
        assert_eq!(h.dim(0).unwrap(), 2);
        assert_eq!(h.dim(1).unwrap(), 3);
    }

    #[test]
    fn test_copy_sharing_tensor() {
        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();

        let t = Tensor::new(&[2, 3])
            .with_values(&[0_i32, 1, 2, 3, 4, 5])
            .unwrap();
        let h = TensorHandle::new(&ctx, &t).unwrap();
        let h_copy = h.copy_sharing_tensor().unwrap();
        let t2 = h_copy.resolve::<i32>().unwrap();

        // t and t2 may share the same memory, but it's difficuly to check
        // since the `resolve` does not guarantee that.
        assert_eq!(&t[..], &t2[..]);
    }

    #[cfg(feature = "tensorflow_gpu")]
    #[test]
    fn test_copy_to_device() {
        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();
        let devices = ctx.device_list().unwrap();
        if !(devices.iter().any(|d| d.device_type == "GPU")) {
            eprintln!("Skipping test_copy_to_device because no GPU device is found");
            return;
        }

        let t = Tensor::new(&[2, 2]).with_values(&[0_i32, 1, 2, 3]).unwrap();
        let h = TensorHandle::new(&ctx, &t).unwrap();
        let target_device = "/job:localhost/replica:0/task:0/device:GPU:0";
        let h_gpu = TensorHandle::copy_to_device(&h, &ctx, target_device).unwrap();
        assert_eq!(h_gpu.device_name().unwrap(), target_device);
        let t2 = h_gpu.resolve::<i32>().unwrap();

        assert_eq!(&t[..], &t2[..]);
    }

    #[cfg(feature = "tensorflow_gpu")]
    #[test]
    fn test_copy_to_device_lifetime() {
        let values = [0_i32, 1, 2, 3];
        let target_device = "/job:localhost/replica:0/task:0/device:GPU:0";

        let opts = ContextOptions::new();
        let ctx = Context::new(opts).unwrap();
        let devices = ctx.device_list().unwrap();
        if !(devices.iter().any(|d| d.device_type == "GPU")) {
            eprintln!("Skipping test_copy_to_device because no GPU device is found");
            return;
        }

        let h_gpu = {
            // Create a temporal Context
            let opts = ContextOptions::new();
            let ctx2 = Context::new(opts).unwrap();
            let t = Tensor::new(&[2, 2]).with_values(&values).unwrap();

            // Create a TensorHandle managed by the context `ctx2`.
            let h = TensorHandle::new(&ctx2, &t).unwrap();

            // Copy to GPU. This creates a new handle managed by the context `ctx`.
            TensorHandle::copy_to_device(&h, &ctx, target_device).unwrap()
        };
        assert_eq!(h_gpu.device_name().unwrap(), target_device);
        let t2 = h_gpu.resolve::<i32>().unwrap();

        assert_eq!(&values[..], &t2[..]);
    }
}
<|endoftext|>


<commit_before>use std::rc::Rc;

use super::expr::*;

pub trait EhtVisitor {
    fn do_visit_expr(&mut self, expr: &mut Expr) {
        match &mut expr.value {
            ExprKind::Const(..) => (),
            ExprKind::HierName(_) => (),
            ExprKind::EmptyQueue => (),
            ExprKind::Concat(list) => for item in list { self.visit_expr(item) },
            ExprKind::MultConcat(_, subexpr) => self.visit_expr(subexpr),
            ExprKind::AssignPattern(_, pattern) => {
                match pattern {
                    AssignPattern::Simple(list) => {
                        for item in list {
                            self.visit_expr(item);
                        }
                    }
                    _ => unimplemented!(),
                }
            }
            ExprKind::Select(parent, dim) => {
                self.visit_expr(parent);
                match &mut dim.value {
                    DimKind::Range(..) => (),
                    DimKind::Value(expr) |
                    DimKind::PlusRange(expr, _) |
                    DimKind::MinusRange(expr, _) => self.visit_expr(expr),
                }
            }
            ExprKind::Member(parent, _) => self.visit_expr(parent),
            ExprKind::SysTfCall(_, args) => {
                for arg in args {
                    if let Some(arg) = arg { self.visit_expr(arg) }
                }
            }
            ExprKind::FuncCall { .. } => (),
            ExprKind::ConstCast(..) => unimplemented!(),
            ExprKind::TypeCast(_, rhs) |
            ExprKind::SignCast(_, rhs) |
            ExprKind::WidthCast(_, rhs) => self.visit_expr(rhs),
            ExprKind::Unary(_, expr) => self.visit_expr(expr),
            ExprKind::Binary(lhs, _, rhs) => {
                self.visit_expr(lhs);
                self.visit_expr(rhs);
            }
            ExprKind::PrefixIncDec(..) => unimplemented!(),
            ExprKind::PostfixIncDec(lhs, _) => self.visit_expr(lhs),
            ExprKind::Assign(lhs, rhs) |
            ExprKind::NonblockAssign(lhs, rhs) |
            ExprKind::BinaryAssign(lhs, _, rhs) => {
                self.visit_expr(lhs);
                self.visit_expr(rhs);
            }
            ExprKind::Paren(expr) => self.visit_expr(expr),
            ExprKind::MinTypMax(..) => unimplemented!(),
            ExprKind::Cond(cond, t, f) => {
                self.visit_expr(cond);
                self.visit_expr(t);
                self.visit_expr(f);
            }
        }
    }

    fn visit_expr(&mut self, expr: &mut Expr) {
        self.do_visit_expr(expr);
    }

    fn do_visit_stmt(&mut self, stmt: &mut Stmt) {
        match &mut stmt.value {
            StmtKind::Empty => (),
            StmtKind::TimingCtrl(_ctrl, stmt) => {
                self.visit_stmt(stmt);
            },
            StmtKind::If { cond, success, failure, .. } => {
                self.visit_expr(cond);
                self.visit_stmt(success);
                if let Some(f) = failure { self.visit_stmt(f); }
            },
            StmtKind::Case { expr, items, .. } => {
                self.visit_expr(expr);
                for (conds, stmt) in items {
                    for cond in conds { self.visit_expr(cond); }
                    self.visit_stmt(stmt);
                }
            },
            StmtKind::For { init, cond, update, body, .. } => {
                for expr in init { self.visit_expr(expr); }
                if let Some(expr) = cond { self.visit_expr(expr); }
                for expr in update { self.visit_expr(expr); }
                self.visit_stmt(body);
            },
            StmtKind::Assert { expr, success, failure, .. } => {
                self.visit_expr(expr);
                if let Some(stmt) = success { self.visit_stmt(stmt); }
                if let Some(stmt) = failure { self.visit_stmt(stmt); }
            },
            StmtKind::SeqBlock(list) => {
                for stmt in list {
                    self.visit_stmt(stmt);
                }
            }
            StmtKind::Expr(expr) => {
                self.visit_expr(expr);
            }
            StmtKind::DataDecl(decl) => {
                let decl = Rc::get_mut(decl).unwrap();
                if let Some(expr) = &mut decl.init { self.visit_expr(expr); }
            }
        }
    }

    fn visit_stmt(&mut self, stmt: &mut Stmt) {
        self.do_visit_stmt(stmt);
    }
}
<commit_msg>Extend eht_visit to function calls<commit_after>use std::rc::Rc;

use super::expr::*;

pub trait EhtVisitor {
    fn do_visit_expr(&mut self, expr: &mut Expr) {
        match &mut expr.value {
            ExprKind::Const(..) => (),
            ExprKind::HierName(_) => (),
            ExprKind::EmptyQueue => (),
            ExprKind::Concat(list) => for item in list { self.visit_expr(item) },
            ExprKind::MultConcat(_, subexpr) => self.visit_expr(subexpr),
            ExprKind::AssignPattern(_, pattern) => {
                match pattern {
                    AssignPattern::Simple(list) => {
                        for item in list {
                            self.visit_expr(item);
                        }
                    }
                    _ => unimplemented!(),
                }
            }
            ExprKind::Select(parent, dim) => {
                self.visit_expr(parent);
                match &mut dim.value {
                    DimKind::Range(..) => (),
                    DimKind::Value(expr) |
                    DimKind::PlusRange(expr, _) |
                    DimKind::MinusRange(expr, _) => self.visit_expr(expr),
                }
            }
            ExprKind::Member(parent, _) => self.visit_expr(parent),
            ExprKind::SysTfCall(_, args) => {
                for arg in args {
                    if let Some(arg) = arg { self.visit_expr(arg) }
                }
            }
            ExprKind::FuncCall { expr, args } => {
                self.visit_expr(expr);
                for arg in args {
                    if let Some(arg) = arg { self.visit_expr(arg) }
                }
            }
            ExprKind::ConstCast(..) => unimplemented!(),
            ExprKind::TypeCast(_, rhs) |
            ExprKind::SignCast(_, rhs) |
            ExprKind::WidthCast(_, rhs) => self.visit_expr(rhs),
            ExprKind::Unary(_, expr) => self.visit_expr(expr),
            ExprKind::Binary(lhs, _, rhs) => {
                self.visit_expr(lhs);
                self.visit_expr(rhs);
            }
            ExprKind::PrefixIncDec(..) => unimplemented!(),
            ExprKind::PostfixIncDec(lhs, _) => self.visit_expr(lhs),
            ExprKind::Assign(lhs, rhs) |
            ExprKind::NonblockAssign(lhs, rhs) |
            ExprKind::BinaryAssign(lhs, _, rhs) => {
                self.visit_expr(lhs);
                self.visit_expr(rhs);
            }
            ExprKind::Paren(expr) => self.visit_expr(expr),
            ExprKind::MinTypMax(..) => unimplemented!(),
            ExprKind::Cond(cond, t, f) => {
                self.visit_expr(cond);
                self.visit_expr(t);
                self.visit_expr(f);
            }
        }
    }

    fn visit_expr(&mut self, expr: &mut Expr) {
        self.do_visit_expr(expr);
    }

    fn do_visit_stmt(&mut self, stmt: &mut Stmt) {
        match &mut stmt.value {
            StmtKind::Empty => (),
            StmtKind::TimingCtrl(_ctrl, stmt) => {
                self.visit_stmt(stmt);
            },
            StmtKind::If { cond, success, failure, .. } => {
                self.visit_expr(cond);
                self.visit_stmt(success);
                if let Some(f) = failure { self.visit_stmt(f); }
            },
            StmtKind::Case { expr, items, .. } => {
                self.visit_expr(expr);
                for (conds, stmt) in items {
                    for cond in conds { self.visit_expr(cond); }
                    self.visit_stmt(stmt);
                }
            },
            StmtKind::For { init, cond, update, body, .. } => {
                for expr in init { self.visit_expr(expr); }
                if let Some(expr) = cond { self.visit_expr(expr); }
                for expr in update { self.visit_expr(expr); }
                self.visit_stmt(body);
            },
            StmtKind::Assert { expr, success, failure, .. } => {
                self.visit_expr(expr);
                if let Some(stmt) = success { self.visit_stmt(stmt); }
                if let Some(stmt) = failure { self.visit_stmt(stmt); }
            },
            StmtKind::SeqBlock(list) => {
                for stmt in list {
                    self.visit_stmt(stmt);
                }
            }
            StmtKind::Expr(expr) => {
                self.visit_expr(expr);
            }
            StmtKind::DataDecl(decl) => {
                let decl = Rc::get_mut(decl).unwrap();
                if let Some(expr) = &mut decl.init { self.visit_expr(expr); }
            }
        }
    }

    fn visit_stmt(&mut self, stmt: &mut Stmt) {
        self.do_visit_stmt(stmt);
    }
}
<|endoftext|>


<commit_before>//! Provides a representation for memory across different frameworks.
//!
//! Memory is allocated by a device in a way that it is accessible for its computations.
//!
//! Normally you will want to use [SharedMemory][shared_mem] which handles synchronization
//! of the latest memory copy to the required device.
//!
//! [shared_mem]: ../shared_mem/index.html

use frameworks::native::flatbox::FlatBox;
use frameworks::opencl::memory::Memory;

/// Specifies Memory behavior accross frameworks.
pub trait IMemory { }

#[derive(Debug)]
/// Container for all known IMemory implementations
pub enum MemoryType {
    /// A OpenCL Context
    Native(FlatBox),
    /// A OpenCL Context
    OpenCL(Memory),
}
<commit_msg>feat/memory: implement MemoryType unwrappers<commit_after>//! Provides a representation for memory across different frameworks.
//!
//! Memory is allocated by a device in a way that it is accessible for its computations.
//!
//! Normally you will want to use [SharedMemory][shared_mem] which handles synchronization
//! of the latest memory copy to the required device.
//!
//! [shared_mem]: ../shared_mem/index.html

use frameworks::native::flatbox::FlatBox;
use frameworks::opencl::memory::Memory;

/// Specifies Memory behavior accross frameworks.
pub trait IMemory { }

#[derive(Debug)]
/// Container for all known IMemory implementations
pub enum MemoryType {
    /// A OpenCL Context
    Native(FlatBox),
    /// A OpenCL Context
    OpenCL(Memory),
}

impl MemoryType {
    fn as_native(&self) -> Option<&FlatBox> {
        match *self {
            MemoryType::Native(ref ret) => Some(ret),
            _ => None,
        }
    }

    fn as_native_mut(&mut self) -> Option<&mut FlatBox> {
        match *self {
            MemoryType::Native(ref mut ret) => Some(ret),
            _ => None,
        }
    }

    fn as_opencl(&self) -> Option<&Memory> {
        match *self {
            MemoryType::OpenCL(ref ret) => Some(ret),
            _ => None,
        }
    }

    fn as_opencl_mut(&mut self) -> Option<&mut Memory> {
        match *self {
            MemoryType::OpenCL(ref mut ret) => Some(ret),
            _ => None,
        }
    }
}
<|endoftext|>


<commit_before>//! Provides an animated color value, so you can have colors change over time

use std::cmp::{Eq, Ord, PartialOrd, PartialEq, Ordering};

use linalg;
use film::Colorf;

/// ColorKeyframe is a color associated with a specific time
#[derive(Debug, Copy, Clone)]
pub struct ColorKeyframe {
    pub color: Colorf,
    pub time: f32,
}

impl ColorKeyframe {
    pub fn new(color: &Colorf, time: f32) -> ColorKeyframe {
        ColorKeyframe { color: *color, time: time }
    }
}
impl Ord for ColorKeyframe {
    fn cmp(&self, other: &ColorKeyframe) -> Ordering {
        self.partial_cmp(other).unwrap()
    }
}
impl PartialOrd for ColorKeyframe {
    fn partial_cmp(&self, other: &ColorKeyframe) -> Option<Ordering> {
        self.time.partial_cmp(&other.time)
    }
}
impl Eq for ColorKeyframe {}
impl PartialEq for ColorKeyframe {
    fn eq(&self, other: &ColorKeyframe) -> bool {
        self.time == other.time
    }
}

/// AnimatedColor is a list of colors associated with time points in the scene
/// that will compute the color at the desired time by blending the two nearest ones
#[derive(Debug, Clone)]
pub struct AnimatedColor {
    /// List of color keyframes in time order
    keyframes: Vec<ColorKeyframe>,
}

impl AnimatedColor {
    /// Create an animated transform that will blend between the passed keyframes
    pub fn with_keyframes(mut keyframes: Vec<ColorKeyframe>) -> AnimatedColor {
        keyframes.sort();
        AnimatedColor { keyframes: keyframes }
    }
    /// Compute the color at the desired time
    pub fn color(&self, time: f32) -> Colorf {
        if self.keyframes.is_empty() {
            Colorf::black()
        } else if self.keyframes.len() == 1 {
            self.keyframes[0].color
        } else {
            // TODO: Binary search here somehow? Or does the BTreeSet have some faster impl
            // of take/skip while?
            let first = self.keyframes.iter().take_while(|k| k.time < time).last();
            let second = self.keyframes.iter().skip_while(|k| k.time < time).next();
            if first.is_none() {
                self.keyframes.first().unwrap().color
            } else if second.is_none() {
                self.keyframes.last().unwrap().color
            } else {
                let mut color = Colorf::black();
                let f = first.unwrap().color;
                let s = second.unwrap().color;
                color.r = linalg::lerp(time, &f.r, &s.r);
                color.g = linalg::lerp(time, &f.g, &s.g);
                color.b = linalg::lerp(time, &f.b, &s.b);
                color.a = linalg::lerp(time, &f.a, &s.a);
                color
            }
        }
    }
}

<commit_msg>Use a normalized t value that we send to lerp!<commit_after>//! Provides an animated color value, so you can have colors change over time

use std::cmp::{Eq, Ord, PartialOrd, PartialEq, Ordering};

use linalg;
use film::Colorf;

/// ColorKeyframe is a color associated with a specific time
#[derive(Debug, Copy, Clone)]
pub struct ColorKeyframe {
    pub color: Colorf,
    pub time: f32,
}

impl ColorKeyframe {
    pub fn new(color: &Colorf, time: f32) -> ColorKeyframe {
        ColorKeyframe { color: *color, time: time }
    }
}
impl Ord for ColorKeyframe {
    fn cmp(&self, other: &ColorKeyframe) -> Ordering {
        self.partial_cmp(other).unwrap()
    }
}
impl PartialOrd for ColorKeyframe {
    fn partial_cmp(&self, other: &ColorKeyframe) -> Option<Ordering> {
        self.time.partial_cmp(&other.time)
    }
}
impl Eq for ColorKeyframe {}
impl PartialEq for ColorKeyframe {
    fn eq(&self, other: &ColorKeyframe) -> bool {
        self.time == other.time
    }
}

/// AnimatedColor is a list of colors associated with time points in the scene
/// that will compute the color at the desired time by blending the two nearest ones
#[derive(Debug, Clone)]
pub struct AnimatedColor {
    /// List of color keyframes in time order
    keyframes: Vec<ColorKeyframe>,
}

impl AnimatedColor {
    /// Create an animated transform that will blend between the passed keyframes
    pub fn with_keyframes(mut keyframes: Vec<ColorKeyframe>) -> AnimatedColor {
        keyframes.sort();
        AnimatedColor { keyframes: keyframes }
    }
    /// Compute the color at the desired time
    pub fn color(&self, time: f32) -> Colorf {
        if self.keyframes.is_empty() {
            Colorf::black()
        } else if self.keyframes.len() == 1 {
            self.keyframes[0].color
        } else {
            // TODO: Binary search here?
            let first = self.keyframes.iter().take_while(|k| k.time < time).last();
            let second = self.keyframes.iter().skip_while(|k| k.time < time).next();
            if first.is_none() {
                self.keyframes.first().unwrap().color
            } else if second.is_none() {
                self.keyframes.last().unwrap().color
            } else {
                let mut color = Colorf::black();
                let fk = first.unwrap();
                let sk = second.unwrap();
                let t = (time - fk.time) / (sk.time - fk.time);
                color.r = linalg::lerp(t, &fk.color.r, &sk.color.r);
                color.g = linalg::lerp(t, &fk.color.g, &sk.color.g);
                color.b = linalg::lerp(t, &fk.color.b, &sk.color.b);
                color.a = linalg::lerp(t, &fk.color.a, &sk.color.a);
                color
            }
        }
    }
}

<|endoftext|>


<commit_before>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;

struct WordVecBuilder {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

impl WordVecBuilder {
    fn new(word: String) -> WordVecBuilder {
        WordVecBuilder {
            word: word,
            count: 0,
            vec: Box::new([0f32; 10_000]),
        }
    }

    fn normalize(mut self, word_freqs: &Vec<f32>) -> WordVec {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count /  word_freqs[i];
        }

        WordVec {
            word: self.word,
            count: self.count,
            vec: self.vec,
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

}

impl<'a> Add for &'a WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] + other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub for &'a WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] - other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVecBuilder>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<String>,
    focus: usize,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    fn new(words: Vec<String>) -> LanguageModelBuilder {
        assert_eq!(words.len(), 10_000);

        let word_vecs = words.iter()
                             .map(|s| WordVecBuilder::new(s.clone()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            words: words,
            word_vecs: word_vecs,
        }
    }

    fn build(self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| v.count as f32 / total_words as f32)
                                       .collect::<Vec<_>>();
        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs.into_iter()
                                     .map(|b| b.normalize(&word_freqs))
                                     .collect(),
        }
    }

    fn newFile(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            focus: 0,
            builder: self,
        }
    }
}

impl<'a> WordAcceptor<'a> {
    fn word(&mut self, word: String) {
        if let Some(idx) = self.builder.words.get(&word) {
            self.window.push_back(word);
            if (self.window.len() > 21) {
                self.window.pop_front();
            }
        }
    }
}<commit_msg>Finish basic word acceptor<commit_after>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;

struct WordVecBuilder {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

impl WordVecBuilder {
    fn new(word: String) -> WordVecBuilder {
        WordVecBuilder {
            word: word,
            count: 0,
            vec: Box::new([0f32; 10_000]),
        }
    }

    fn normalize(mut self, word_freqs: &Vec<f32>) -> WordVec {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count /  word_freqs[i];
        }

        WordVec {
            word: self.word,
            count: self.count,
            vec: self.vec,
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

}

impl<'a> Add for &'a WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] + other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub for &'a WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] - other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: u32,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVecBuilder>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<String>,
    focus: usize,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    fn new(words: Vec<String>) -> LanguageModelBuilder {
        assert_eq!(words.len(), 10_000);

        let word_vecs = words.iter()
                             .map(|s| WordVecBuilder::new(s.clone()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 21,
            words: words,
            word_vecs: word_vecs,
        }
    }

    fn build(self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| v.count as f32 / total_words as f32)
                                       .collect::<Vec<_>>();
        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs.into_iter()
                                     .map(|b| b.normalize(&word_freqs))
                                     .collect(),
        }
    }

    fn new_file(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            focus: 0,
            builder: self,
        }
    }

    fn add_word(&mut self, word: &str) {
        let idx = self.words[word];
        let word_vec = &mut self.word_vecs[idx];
        word_vec.inc(idx);
    }
}

impl<'a> WordAcceptor<'a> {
    fn word(&mut self, word: String) {
        let idx = self.builder.words.get(&word).map(|i| *i);

        if let Some(idx) = idx {
            self.window.push_back(word);
            if (self.window.len() > 21) {
                self.window.pop_front();
            }
            if (self.window.len() == 21) {
                for i in (0..21).filter(|a| *a != 10) {
                    self.builder.add_word(&self.window[i]);
                }
            }
        }
    }
}<|endoftext|>


<commit_before>use std::default::Default;
use std::hash::Hash;
use std::option;

#[cfg(feature = "with-serde")]
use serde;

use crate::Message;

/// Wrapper around `Option<Box<T>>`, convenient newtype.
///
/// # Examples
///
/// ```no_run
/// # use protobuf::MessageField;
/// # use std::ops::Add;
/// # struct Address {
/// # }
/// # struct Customer {
/// #     address: MessageField<Address>,
/// # }
/// # impl Customer {
/// #     fn new() -> Customer { unimplemented!() }
/// # }
/// #
/// #
/// # fn make_address() -> Address { unimplemented!() }
/// let mut customer = Customer::new();
///
/// // field of type `SingularPtrField` can be initialized like this
/// customer.address = MessageField::some(make_address());
/// // or using `Option` and `Into`
/// customer.address = Some(make_address()).into();
/// ```
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub struct MessageField<T>(pub Option<Box<T>>);

impl<T> MessageField<T> {
    /// Construct `SingularPtrField` from given object.
    #[inline]
    pub fn some(value: T) -> MessageField<T> {
        MessageField(Some(Box::new(value)))
    }

    /// Construct an empty `SingularPtrField`.
    #[inline]
    pub const fn none() -> MessageField<T> {
        MessageField(None)
    }

    /// Construct `SingularPtrField` from optional.
    #[inline]
    pub fn from_option(option: Option<T>) -> MessageField<T> {
        match option {
            Some(x) => MessageField::some(x),
            None => MessageField::none(),
        }
    }

    /// True iff this object contains data.
    #[inline]
    pub fn is_some(&self) -> bool {
        self.0.is_some()
    }

    /// True iff this object contains no data.
    #[inline]
    pub fn is_none(&self) -> bool {
        self.0.is_none()
    }

    /// Convert into `Option<T>`.
    #[inline]
    pub fn into_option(self) -> Option<T> {
        self.0.map(|v| *v)
    }

    /// View data as reference option.
    #[inline]
    pub fn as_ref(&self) -> Option<&T> {
        self.0.as_ref().map(|v| &**v)
    }

    /// View data as mutable reference option.
    #[inline]
    pub fn as_mut(&mut self) -> Option<&mut T> {
        self.0.as_mut().map(|v| &mut **v)
    }

    /// Get data as reference.
    /// Panics if empty.
    #[inline]
    pub fn get_ref(&self) -> &T {
        self.as_ref().unwrap()
    }

    /// Get data as mutable reference.
    /// Panics if empty.
    #[inline]
    pub fn get_mut_ref(&mut self) -> &mut T {
        self.as_mut().unwrap()
    }

    /// Take the data.
    /// Panics if empty
    #[inline]
    pub fn unwrap(self) -> T {
        *self.0.unwrap()
    }

    /// Take the data or return supplied default element if empty.
    #[inline]
    pub fn unwrap_or(self, def: T) -> T {
        self.0.map(|v| *v).unwrap_or(def)
    }

    /// Take the data or return supplied default element if empty.
    #[inline]
    pub fn unwrap_or_else<F>(self, f: F) -> T
    where
        F: FnOnce() -> T,
    {
        self.0.map(|v| *v).unwrap_or_else(f)
    }

    /// Apply given function to contained data to construct another `SingularPtrField`.
    /// Returns empty `SingularPtrField` if this object is empty.
    #[inline]
    pub fn map<U, F>(self, f: F) -> MessageField<U>
    where
        F: FnOnce(T) -> U,
    {
        MessageField::from_option(self.into_option().map(f))
    }

    /// View data as iterator.
    #[inline]
    pub fn iter(&self) -> option::IntoIter<&T> {
        self.as_ref().into_iter()
    }

    /// View data as mutable iterator.
    #[inline]
    pub fn mut_iter(&mut self) -> option::IntoIter<&mut T> {
        self.as_mut().into_iter()
    }

    /// Take data as option, leaving this object empty.
    #[inline]
    pub fn take(&mut self) -> Option<T> {
        self.0.take().map(|v| *v)
    }

    /// Clear this object, but do not call destructor of underlying data.
    #[inline]
    pub fn clear(&mut self) {
        self.0 = None;
    }
}

impl<T: Default> MessageField<T> {
    /// Get contained data, consume self. Return default value for type if this is empty.
    #[inline]
    pub fn unwrap_or_default(self) -> T {
        *self.0.unwrap_or_default()
    }

    /// Set object to `Some(T::default())`.
    // TODO: inline
    #[inline]
    pub fn set_default(&mut self) -> &mut T {
        *self = MessageField::some(Default::default());
        self.as_mut().unwrap()
    }
}

impl<M: Message + Default> MessageField<M> {
    /// Get a reference to contained value or a default instance.
    pub fn get_or_default(&self) -> &M {
        self.as_ref().unwrap_or_else(|| M::default_instance())
    }

    /// Get a mutable reference to contained value, initialize if not initialized yet.
    pub fn mut_or_default(&mut self) -> &mut M {
        if self.is_none() {
            *self = MessageField::some(Default::default());
        }
        self.get_mut_ref()
    }
}

impl<T> Default for MessageField<T> {
    #[inline]
    fn default() -> MessageField<T> {
        MessageField::none()
    }
}

impl<T> From<Option<T>> for MessageField<T> {
    fn from(o: Option<T>) -> Self {
        MessageField::from_option(o)
    }
}

impl<'a, T> IntoIterator for &'a MessageField<T> {
    type Item = &'a T;
    type IntoIter = option::IntoIter<&'a T>;

    fn into_iter(self) -> option::IntoIter<&'a T> {
        self.iter()
    }
}

#[cfg(feature = "with-serde")]
impl<T: serde::Serialize> serde::Serialize for MessageField<T> {
    fn serialize<S>(
        &self,
        serializer: S,
    ) -> Result<<S as serde::Serializer>::Ok, <S as serde::Serializer>::Error>
    where
        S: serde::Serializer,
    {
        self.as_ref().serialize(serializer)
    }
}

#[cfg(feature = "with-serde")]
impl<'de, T: serde::Deserialize<'de>> serde::Deserialize<'de> for MessageField<T> {
    fn deserialize<D>(deserializer: D) -> Result<Self, <D as serde::Deserializer<'de>>::Error>
    where
        D: serde::Deserializer<'de>,
    {
        Option::deserialize(deserializer).map(MessageField::from)
    }
}
<commit_msg>impl Deref and DerefMut for MessageField<commit_after>use std::default::Default;
use std::hash::Hash;
use std::ops::Deref;
use std::ops::DerefMut;
use std::option;

#[cfg(feature = "with-serde")]
use serde;

use crate::Message;

/// Wrapper around `Option<Box<T>>`, convenient newtype.
///
/// # Examples
///
/// ```no_run
/// # use protobuf::MessageField;
/// # use std::ops::Add;
/// # struct Address {
/// # }
/// # struct Customer {
/// #     address: MessageField<Address>,
/// # }
/// # impl Customer {
/// #     fn new() -> Customer { unimplemented!() }
/// # }
/// #
/// #
/// # fn make_address() -> Address { unimplemented!() }
/// let mut customer = Customer::new();
///
/// // field of type `SingularPtrField` can be initialized like this
/// customer.address = MessageField::some(make_address());
/// // or using `Option` and `Into`
/// customer.address = Some(make_address()).into();
/// ```
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub struct MessageField<T>(pub Option<Box<T>>);

impl<T> MessageField<T> {
    /// Construct `SingularPtrField` from given object.
    #[inline]
    pub fn some(value: T) -> MessageField<T> {
        MessageField(Some(Box::new(value)))
    }

    /// Construct an empty `SingularPtrField`.
    #[inline]
    pub const fn none() -> MessageField<T> {
        MessageField(None)
    }

    /// Construct `SingularPtrField` from optional.
    #[inline]
    pub fn from_option(option: Option<T>) -> MessageField<T> {
        match option {
            Some(x) => MessageField::some(x),
            None => MessageField::none(),
        }
    }

    /// True iff this object contains data.
    #[inline]
    pub fn is_some(&self) -> bool {
        self.0.is_some()
    }

    /// True iff this object contains no data.
    #[inline]
    pub fn is_none(&self) -> bool {
        self.0.is_none()
    }

    /// Convert into `Option<T>`.
    #[inline]
    pub fn into_option(self) -> Option<T> {
        self.0.map(|v| *v)
    }

    /// View data as reference option.
    #[inline]
    pub fn as_ref(&self) -> Option<&T> {
        self.0.as_ref().map(|v| &**v)
    }

    /// View data as mutable reference option.
    #[inline]
    pub fn as_mut(&mut self) -> Option<&mut T> {
        self.0.as_mut().map(|v| &mut **v)
    }

    /// Get data as reference.
    /// Panics if empty.
    #[inline]
    pub fn get_ref(&self) -> &T {
        self.as_ref().unwrap()
    }

    /// Get data as mutable reference.
    /// Panics if empty.
    #[inline]
    pub fn get_mut_ref(&mut self) -> &mut T {
        self.as_mut().unwrap()
    }

    /// Take the data.
    /// Panics if empty
    #[inline]
    pub fn unwrap(self) -> T {
        *self.0.unwrap()
    }

    /// Take the data or return supplied default element if empty.
    #[inline]
    pub fn unwrap_or(self, def: T) -> T {
        self.0.map(|v| *v).unwrap_or(def)
    }

    /// Take the data or return supplied default element if empty.
    #[inline]
    pub fn unwrap_or_else<F>(self, f: F) -> T
    where
        F: FnOnce() -> T,
    {
        self.0.map(|v| *v).unwrap_or_else(f)
    }

    /// Apply given function to contained data to construct another `SingularPtrField`.
    /// Returns empty `SingularPtrField` if this object is empty.
    #[inline]
    pub fn map<U, F>(self, f: F) -> MessageField<U>
    where
        F: FnOnce(T) -> U,
    {
        MessageField::from_option(self.into_option().map(f))
    }

    /// View data as iterator.
    #[inline]
    pub fn iter(&self) -> option::IntoIter<&T> {
        self.as_ref().into_iter()
    }

    /// View data as mutable iterator.
    #[inline]
    pub fn mut_iter(&mut self) -> option::IntoIter<&mut T> {
        self.as_mut().into_iter()
    }

    /// Take data as option, leaving this object empty.
    #[inline]
    pub fn take(&mut self) -> Option<T> {
        self.0.take().map(|v| *v)
    }

    /// Clear this object, but do not call destructor of underlying data.
    #[inline]
    pub fn clear(&mut self) {
        self.0 = None;
    }
}

impl<T: Default> MessageField<T> {
    /// Get contained data, consume self. Return default value for type if this is empty.
    #[inline]
    pub fn unwrap_or_default(self) -> T {
        *self.0.unwrap_or_default()
    }

    /// Set object to `Some(T::default())`.
    // TODO: inline
    #[inline]
    pub fn set_default(&mut self) -> &mut T {
        *self = MessageField::some(Default::default());
        self.as_mut().unwrap()
    }
}

impl<M: Message + Default> MessageField<M> {
    /// Get a reference to contained value or a default instance.
    pub fn get_or_default(&self) -> &M {
        self.as_ref().unwrap_or_else(|| M::default_instance())
    }

    /// Get a mutable reference to contained value, initialize if not initialized yet.
    pub fn mut_or_default(&mut self) -> &mut M {
        if self.is_none() {
            *self = MessageField::some(Default::default());
        }
        self.get_mut_ref()
    }
}

/// Get a reference to contained value or a default instance if the field is not initialized.
impl<M: Message + Default> Deref for MessageField<M> {
    type Target = M;

    fn deref(&self) -> &Self::Target {
        self.get_or_default()
    }
}

/// Get a mutable reference to the message **and** initialize the message if not initialized yet.
///
/// Note that part about initializing is not conventional.
/// Generally `DerefMut` is not supposed to modify the state.
impl<M: Message + Default> DerefMut for MessageField<M> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        self.mut_or_default()
    }
}

impl<T> Default for MessageField<T> {
    #[inline]
    fn default() -> MessageField<T> {
        MessageField::none()
    }
}

impl<T> From<Option<T>> for MessageField<T> {
    fn from(o: Option<T>) -> Self {
        MessageField::from_option(o)
    }
}

impl<'a, T> IntoIterator for &'a MessageField<T> {
    type Item = &'a T;
    type IntoIter = option::IntoIter<&'a T>;

    fn into_iter(self) -> option::IntoIter<&'a T> {
        self.iter()
    }
}

#[cfg(feature = "with-serde")]
impl<T: serde::Serialize> serde::Serialize for MessageField<T> {
    fn serialize<S>(
        &self,
        serializer: S,
    ) -> Result<<S as serde::Serializer>::Ok, <S as serde::Serializer>::Error>
    where
        S: serde::Serializer,
    {
        self.as_ref().serialize(serializer)
    }
}

#[cfg(feature = "with-serde")]
impl<'de, T: serde::Deserialize<'de>> serde::Deserialize<'de> for MessageField<T> {
    fn deserialize<D>(deserializer: D) -> Result<Self, <D as serde::Deserializer<'de>>::Error>
    where
        D: serde::Deserializer<'de>,
    {
        Option::deserialize(deserializer).map(MessageField::from)
    }
}
<|endoftext|>