<commit_before>extern crate walkdir;

use self::walkdir::{DirEntry as WalkDirEntry, WalkDir, WalkDirIterator};

use std::collections::BTreeSet;
use std::ffi::OsString;
use std::fs;
use std::io::Error as IoError;
use std::path::{Path, PathBuf};
use std::result::Result;

macro_rules! path_buf {
    ($($x: expr),+) => {
        PathBuf::new()
            $(.join($x))+
    };
}

pub fn ls<P: AsRef<Path>>(path: P) -> Result<Vec<String>, IoError> {
    let dirs = try!(fs::read_dir(path));
    let dirs = dirs
        .filter_map(Result::ok)
        .map(|d| d.file_name())
        .map(OsString::into_string)
        .filter_map(Result::ok)
        .collect::<Vec<String>>();
    Ok(dirs)
}

pub fn is_empty_dir<P: AsRef<Path>>(path: P) -> bool {
    match fs::read_dir(path) {
        Ok(rd) => rd.count() == 0,
        Err(_) => false,
    }
}

fn is_hidden_name(file_name: &str) -> bool {
    file_name.starts_with(".") && file_name.len() > 1 && file_name != ".."
}
#[test]
fn is_hidden_name_should_return_true() {
    let file_names = [
        ".a",
        ".ab",
        ".abc",
        "..a",
        "...",
    ];
    for file_name in &file_names {
        assert!(is_hidden_name(file_name));
    }
}
#[test]
fn is_hidden_name_should_return_false() {
    let file_names = [
        "",

        ".",
        "..",
        "a",
        "ab",
        "abc",
    ];
    for file_name in &file_names {
        assert!(!is_hidden_name(file_name));
    }
}

fn is_hidden(entry: &WalkDirEntry) -> bool {
    entry
        .file_name()
        .to_str()
        .map_or(false, is_hidden_name)
}

pub fn walk_dir<P: AsRef<Path>>(root: P) -> BTreeSet<String> {
    let walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_map(Result::ok)
        .filter(|e| !e.file_type().is_dir())
        .collect::<Vec<WalkDirEntry>>();

    walker
        .into_iter()
        .filter_map(|e| e.path().to_str().map(|s| s.to_string()))
        .collect::<BTreeSet<String>>()
}

pub fn dirs_ordered_by_descending_depth<P: AsRef<Path>>(root: P) -> Vec<PathBuf> {
    let mut walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_entry(|e| e.file_type().is_dir())
        .filter_map(Result::ok)
        .collect::<Vec<WalkDirEntry>>();

    walker.sort_by(|l, r| {
        let l_depth = &l.depth();
        let r_depth = &r.depth();
        r_depth.cmp(l_depth)
    });

    walker
        .into_iter()
        .map(|e| e.path().to_path_buf())
        .collect::<Vec<PathBuf>>()
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::{self, File};
    use std::io::Write;
    use std::path::{Path, PathBuf};

    struct Helper;
    impl Helper {
        fn d1() -> String { "directory1".to_string() }
        fn d2() -> String { "directory2".to_string() }
        fn d3() -> String { "directory3".to_string() }

        fn f1() -> String { "file1".to_string() }
        fn f2() -> String { "file2".to_string() }

        fn path_to_d1() -> PathBuf {
            PathBuf::new().join(Self::d1())
        }
        fn path_to_d2() -> PathBuf {
            Self::path_to_d1().join(Self::d2())
        }
        fn path_to_d3() -> PathBuf {
            Self::path_to_d2().join(Self::d3())
        }

        fn path_to_f1() -> PathBuf {
            Self::path_to_d1().join(Self::f1())
        }
        fn path_to_f2() -> PathBuf {
            Self::path_to_d2().join(Self::f2())
        }

        fn remove_dirs_and_files() {
            fs::remove_dir_all(Self::path_to_d1()).ok();
        }

        fn create_dirs_and_files() {
            Self::remove_dirs_and_files();

            fs::create_dir_all(Self::path_to_d3()).ok();

            let mut f = File::create(Self::path_to_f1()).unwrap();
            f.write("\n".as_ref()).ok();
            fs::copy(Self::path_to_f1(), Self::path_to_f2()).ok();
        }
    }

    #[test]
    fn path_buf_macro_should_create_path_buf() {
        let paths = [
            (Path::new("path").to_path_buf()          , path_buf!["path"              ]),
            (Path::new("path").join("to")             , path_buf!["path", "to"        ]),
            (Path::new("path").join("to").join("file"), path_buf!["path", "to", "file"]),
        ];
        for &(ref correct, ref calculated) in &paths {
            assert_eq!(correct, calculated);
        }
    }

    #[test]
    fn ls_should_return_ok() {
        Helper::create_dirs_and_files();

        let empty_vec: Vec<String> = Vec::new();

        assert_eq!(vec![Helper::d2(), Helper::f1()], ls(Helper::path_to_d1()).unwrap());
        assert_eq!(vec![Helper::d3(), Helper::f2()], ls(Helper::path_to_d2()).unwrap());
        assert_eq!(empty_vec                       , ls(Helper::path_to_d3()).unwrap());

        Helper::remove_dirs_and_files();
    }

    #[test]
    fn is_empty_dir_should_return_true() {
        Helper::create_dirs_and_files();

        assert!(is_empty_dir(Helper::path_to_d3()));

        Helper::remove_dirs_and_files();
    }
    #[test]
    fn is_empty_dir_should_return_false() {
        Helper::create_dirs_and_files();

        assert!(!is_empty_dir(Helper::path_to_d1()));
        assert!(!is_empty_dir(Helper::path_to_d2()));

        Helper::remove_dirs_and_files();
    }

    #[test]
    fn walk_dir_should_return_b_tree_set() {
    }

    #[test]
    fn dirs_ordered_by_descending_depth_should_return_vec() {
    }
}
<commit_msg>[Test] Add test for walk_dir()<commit_after>extern crate walkdir;

use self::walkdir::{DirEntry as WalkDirEntry, WalkDir, WalkDirIterator};

use std::collections::BTreeSet;
use std::ffi::OsString;
use std::fs;
use std::io::Error as IoError;
use std::path::{Path, PathBuf};
use std::result::Result;

macro_rules! path_buf {
    ($($x: expr),+) => {
        PathBuf::new()
            $(.join($x))+
    };
}

pub fn ls<P: AsRef<Path>>(path: P) -> Result<Vec<String>, IoError> {
    let dirs = try!(fs::read_dir(path));
    let dirs = dirs
        .filter_map(Result::ok)
        .map(|d| d.file_name())
        .map(OsString::into_string)
        .filter_map(Result::ok)
        .collect::<Vec<String>>();
    Ok(dirs)
}

pub fn is_empty_dir<P: AsRef<Path>>(path: P) -> bool {
    match fs::read_dir(path) {
        Ok(rd) => rd.count() == 0,
        Err(_) => false,
    }
}

fn is_hidden_name(file_name: &str) -> bool {
    file_name.starts_with(".") && file_name.len() > 1 && file_name != ".."
}
#[test]
fn is_hidden_name_should_return_true() {
    let file_names = [
        ".a",
        ".ab",
        ".abc",
        "..a",
        "...",
    ];
    for file_name in &file_names {
        assert!(is_hidden_name(file_name));
    }
}
#[test]
fn is_hidden_name_should_return_false() {
    let file_names = [
        "",

        ".",
        "..",
        "a",
        "ab",
        "abc",
    ];
    for file_name in &file_names {
        assert!(!is_hidden_name(file_name));
    }
}

fn is_hidden(entry: &WalkDirEntry) -> bool {
    entry
        .file_name()
        .to_str()
        .map_or(false, is_hidden_name)
}

pub fn walk_dir<P: AsRef<Path>>(root: P) -> BTreeSet<String> {
    let walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_map(Result::ok)
        .filter(|e| !e.file_type().is_dir())
        .collect::<Vec<WalkDirEntry>>();

    walker
        .into_iter()
        .filter_map(|e| e.path().to_str().map(|s| s.to_string()))
        .collect::<BTreeSet<String>>()
}

pub fn dirs_ordered_by_descending_depth<P: AsRef<Path>>(root: P) -> Vec<PathBuf> {
    let mut walker = WalkDir::new(root)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter_entry(|e| e.file_type().is_dir())
        .filter_map(Result::ok)
        .collect::<Vec<WalkDirEntry>>();

    walker.sort_by(|l, r| {
        let l_depth = &l.depth();
        let r_depth = &r.depth();
        r_depth.cmp(l_depth)
    });

    walker
        .into_iter()
        .map(|e| e.path().to_path_buf())
        .collect::<Vec<PathBuf>>()
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::collections::BTreeSet;
    use std::fs::{self, File};
    use std::io::Write;
    use std::path::{Path, PathBuf};

    struct Helper;
    impl Helper {
        fn d1() -> String { "directory1".to_string() }
        fn d2() -> String { "directory2".to_string() }
        fn d3() -> String { "directory3".to_string() }

        fn f1() -> String { "file1".to_string() }
        fn f2() -> String { "file2".to_string() }

        fn path_to_d1() -> PathBuf {
            PathBuf::new().join(Self::d1())
        }
        fn path_to_d2() -> PathBuf {
            Self::path_to_d1().join(Self::d2())
        }
        fn path_to_d3() -> PathBuf {
            Self::path_to_d2().join(Self::d3())
        }

        fn path_to_f1() -> PathBuf {
            Self::path_to_d1().join(Self::f1())
        }
        fn path_to_f2() -> PathBuf {
            Self::path_to_d2().join(Self::f2())
        }

        fn remove_dirs_and_files() {
            fs::remove_dir_all(Self::path_to_d1()).ok();
        }

        fn create_dirs_and_files() {
            Self::remove_dirs_and_files();

            fs::create_dir_all(Self::path_to_d3()).ok();

            let mut f = File::create(Self::path_to_f1()).unwrap();
            f.write("\n".as_ref()).ok();
            fs::copy(Self::path_to_f1(), Self::path_to_f2()).ok();
        }

        fn to_string_forcely(path: PathBuf) -> String {
            path.to_str().unwrap().to_string()
        }
    }

    #[test]
    fn path_buf_macro_should_create_path_buf() {
        let paths = [
            (Path::new("path").to_path_buf()          , path_buf!["path"              ]),
            (Path::new("path").join("to")             , path_buf!["path", "to"        ]),
            (Path::new("path").join("to").join("file"), path_buf!["path", "to", "file"]),
        ];
        for &(ref correct, ref calculated) in &paths {
            assert_eq!(correct, calculated);
        }
    }

    #[test]
    fn ls_should_return_ok() {
        Helper::create_dirs_and_files();

        let empty_vec: Vec<String> = Vec::new();

        assert_eq!(vec![Helper::d2(), Helper::f1()], ls(Helper::path_to_d1()).unwrap());
        assert_eq!(vec![Helper::d3(), Helper::f2()], ls(Helper::path_to_d2()).unwrap());
        assert_eq!(empty_vec                       , ls(Helper::path_to_d3()).unwrap());

        Helper::remove_dirs_and_files();
    }

    #[test]
    fn is_empty_dir_should_return_true() {
        Helper::create_dirs_and_files();

        assert!(is_empty_dir(Helper::path_to_d3()));

        Helper::remove_dirs_and_files();
    }
    #[test]
    fn is_empty_dir_should_return_false() {
        Helper::create_dirs_and_files();

        assert!(!is_empty_dir(Helper::path_to_d1()));
        assert!(!is_empty_dir(Helper::path_to_d2()));

        Helper::remove_dirs_and_files();
    }

    #[test]
    fn walk_dir_should_return_b_tree_set() {
        Helper::create_dirs_and_files();

        let mut correct = BTreeSet::new();
        correct.insert(Helper::to_string_forcely(Helper::path_to_f1()));
        correct.insert(Helper::to_string_forcely(Helper::path_to_f2()));

        assert_eq!(correct, walk_dir(Helper::path_to_d1()));

        Helper::remove_dirs_and_files();
    }

    #[test]
    fn dirs_ordered_by_descending_depth_should_return_vec() {
    }
}
<|endoftext|>


<commit_before>use base::{AbstractCFType, CFAllocatorRef, CFIndex, CFRelease, CFTypeRef, kCFAllocatorDefault};
use libc::{c_int, c_void};
use unsafe::reinterpret_cast;

struct __CFNumber { private: () }
pub type CFNumberRef = *__CFNumber;

const kCFNumberSInt32Type: c_int = 3;

struct CFNumber {
    obj: CFNumberRef,

    drop {
        unsafe {
            CFRelease(reinterpret_cast(&self.obj));
        }
    }
}

mod CFNumber {
    fn wrap(obj: CFNumberRef) -> CFNumber {
        CFNumber { obj: obj }
    }

    fn new_number<T:copy ConvertibleToCFNumber>(n: T) -> CFNumber {
        unsafe {
            CFNumber {
                obj: CFNumberCreate(kCFAllocatorDefault, n.cf_number_type(), reinterpret_cast(&&n))
            }
        }
    }
}

impl CFNumber : AbstractCFType {
    pure fn as_type_ref(&self) -> CFTypeRef {
        unsafe {
            reinterpret_cast(&self.obj)
        }
    }
}

trait ConvertibleToCFNumber {
    // FIXME: Should be static, but that breaks.
    pure fn cf_number_type(self) -> CFNumberType;
}

impl i32 : ConvertibleToCFNumber {
    pure fn cf_number_type(self) -> CFNumberType { kCFNumberSInt32Type as CFNumberType }
}

type CFNumberType = CFIndex;

#[link_args="-framework CoreFoundation"]
#[nolink]
extern {
    fn CFNumberCreate(allocator: CFAllocatorRef, theType: CFNumberType, valuePtr: *c_void)
                   -> CFNumberRef;
}

<commit_msg>More syntax fixing.<commit_after>use base::{AbstractCFType, CFAllocatorRef, CFIndex, CFRelease, CFTypeRef, kCFAllocatorDefault};
use libc::{c_int, c_void};
use unsafe::reinterpret_cast;

struct __CFNumber { private: () }
pub type CFNumberRef = *__CFNumber;

const kCFNumberSInt32Type: c_int = 3;

struct CFNumber {
    obj: CFNumberRef,

    drop {
        unsafe {
            CFRelease(reinterpret_cast(&self.obj));
        }
    }
}

mod CFNumber {
    fn wrap(obj: CFNumberRef) -> CFNumber {
        CFNumber { obj: obj }
    }

    fn new_number<T:copy ConvertibleToCFNumber>(n: T) -> CFNumber {
        unsafe {
            CFNumber {
                obj: CFNumberCreate(kCFAllocatorDefault, n.cf_number_type(), reinterpret_cast(& &n))
            }
        }
    }
}

impl CFNumber : AbstractCFType {
    pure fn as_type_ref(&self) -> CFTypeRef {
        unsafe {
            reinterpret_cast(&self.obj)
        }
    }
}

trait ConvertibleToCFNumber {
    // FIXME: Should be static, but that breaks.
    pure fn cf_number_type(self) -> CFNumberType;
}

impl i32 : ConvertibleToCFNumber {
    pure fn cf_number_type(self) -> CFNumberType { kCFNumberSInt32Type as CFNumberType }
}

type CFNumberType = CFIndex;

#[link_args="-framework CoreFoundation"]
#[nolink]
extern {
    fn CFNumberCreate(allocator: CFAllocatorRef, theType: CFNumberType, valuePtr: *c_void)
                   -> CFNumberRef;
}

<|endoftext|>


<commit_before>use utils::Hash;
use installation::PackageState;

struct PackageCache(Vec<(Hash, PackageState)>);<commit_msg>Rename PackageCache to Cache<commit_after>use utils::Hash;
use installation::PackageState;

pub struct Cache(Vec<(Hash, PackageState)>);

impl Cache {

}<|endoftext|>


<commit_before>use crate::{
    ast::{FromInputValue, InputValue, Selection, ToInputValue},
    executor::ExecutionResult,
    schema::meta::MetaType,
    value::{ScalarValue, Value},
};

use crate::{
    executor::{Executor, Registry},
    types::base::GraphQLType,
};

impl<S, T, CtxT> GraphQLType<S> for Option<T>
where
    S: ScalarValue,
    T: GraphQLType<S, Context = CtxT>,
{
    type Context = CtxT;
    type TypeInfo = T::TypeInfo;

    fn name(_: &T::TypeInfo) -> Option<&str> {
        None
    }

    fn meta<'r>(info: &T::TypeInfo, registry: &mut Registry<'r, S>) -> MetaType<'r, S>
    where
        S: 'r,
    {
        registry.build_nullable_type::<T>(info).into_meta()
    }

    fn resolve(
        &self,
        info: &T::TypeInfo,
        _: Option<&[Selection<S>]>,
        executor: &Executor<CtxT, S>,
    ) -> ExecutionResult<S> {
        match *self {
            Some(ref obj) => executor.resolve(info, obj),
            None => Ok(Value::null()),
        }
    }
}

impl<S, T> FromInputValue<S> for Option<T>
where
    T: FromInputValue<S>,
    S: ScalarValue,
{
    fn from_input_value<'a>(v: &'a InputValue<S>) -> Option<Option<T>> {
        match v {
            &InputValue::Null => Some(None),
            v => match v.convert() {
                Some(x) => Some(Some(x)),
                None => None,
            },
        }
    }
}

impl<S, T> ToInputValue<S> for Option<T>
where
    T: ToInputValue<S>,
    S: ScalarValue,
{
    fn to_input_value(&self) -> InputValue<S> {
        match *self {
            Some(ref v) => v.to_input_value(),
            None => InputValue::null(),
        }
    }
}

impl<S, T, CtxT> GraphQLType<S> for Vec<T>
where
    T: GraphQLType<S, Context = CtxT>,
    S: ScalarValue,
{
    type Context = CtxT;
    type TypeInfo = T::TypeInfo;

    fn name(_: &T::TypeInfo) -> Option<&str> {
        None
    }

    fn meta<'r>(info: &T::TypeInfo, registry: &mut Registry<'r, S>) -> MetaType<'r, S>
    where
        S: 'r,
    {
        registry.build_list_type::<T>(info).into_meta()
    }

    fn resolve(
        &self,
        info: &T::TypeInfo,
        _: Option<&[Selection<S>]>,
        executor: &Executor<CtxT, S>,
    ) -> ExecutionResult<S> {
        resolve_into_list(executor, info, self.iter())
    }
}

impl<T, S> FromInputValue<S> for Vec<T>
where
    T: FromInputValue<S>,
    S: ScalarValue,
{
    fn from_input_value<'a>(v: &'a InputValue<S>) -> Option<Vec<T>>
where {
        match *v {
            InputValue::List(ref ls) => {
                let v: Vec<_> = ls.iter().filter_map(|i| i.item.convert()).collect();

                if v.len() == ls.len() {
                    Some(v)
                } else {
                    None
                }
            }
            ref other => {
                if let Some(e) = other.convert() {
                    Some(vec![e])
                } else {
                    None
                }
            }
        }
    }
}

impl<T, S> ToInputValue<S> for Vec<T>
where
    T: ToInputValue<S>,
    S: ScalarValue,
{
    fn to_input_value(&self) -> InputValue<S> {
        InputValue::list(self.iter().map(|v| v.to_input_value()).collect())
    }
}

impl<'a, S, T, CtxT> GraphQLType<S> for &'a [T]
where
    S: ScalarValue,
    T: GraphQLType<S, Context = CtxT>,
{
    type Context = CtxT;
    type TypeInfo = T::TypeInfo;

    fn name(_: &T::TypeInfo) -> Option<&str> {
        None
    }

    fn meta<'r>(info: &T::TypeInfo, registry: &mut Registry<'r, S>) -> MetaType<'r, S>
    where
        S: 'r,
    {
        registry.build_list_type::<T>(info).into_meta()
    }

    fn resolve(
        &self,
        info: &T::TypeInfo,
        _: Option<&[Selection<S>]>,
        executor: &Executor<CtxT, S>,
    ) -> ExecutionResult<S> {
        resolve_into_list(executor, info, self.iter())
    }
}

impl<'a, T, S> ToInputValue<S> for &'a [T]
where
    T: ToInputValue<S>,
    S: ScalarValue,
{
    fn to_input_value(&self) -> InputValue<S> {
        InputValue::list(self.iter().map(|v| v.to_input_value()).collect())
    }
}

fn resolve_into_list<S, T, I>(
    executor: &Executor<T::Context, S>,
    info: &T::TypeInfo,
    iter: I,
) -> ExecutionResult<S>
where
    S: ScalarValue,
    I: Iterator<Item = T> + ExactSizeIterator,
    T: GraphQLType<S>,
{
    let stop_on_null = executor
        .current_type()
        .list_contents()
        .expect("Current type is not a list type")
        .is_non_null();
    let mut result = Vec::with_capacity(iter.len());

    for o in iter {
        match executor.resolve(info, &o) {
            Ok(value) if stop_on_null && value.is_null() => return Ok(value),
            Ok(value) => result.push(value),
            Err(e) => return Err(e),
        }
    }

    Ok(Value::list(result))
}

#[cfg(feature = "async")]
async fn resolve_into_list_async<'a, S, T, I>(
    executor: &'a Executor<'a, T::Context, S>,
    info: &'a T::TypeInfo,
    items: I,
) -> ExecutionResult<S>
where
    S: ScalarValue + Send + Sync,
    I: Iterator<Item = T> + ExactSizeIterator,
    T: crate::GraphQLTypeAsync<S>,
    T::TypeInfo: Send + Sync,
    T::Context: Send + Sync,
{
    use futures::stream::{FuturesOrdered, StreamExt};
    use std::iter::FromIterator;

    let stop_on_null = executor
        .current_type()
        .list_contents()
        .expect("Current type is not a list type")
        .is_non_null();

    let iter =
        items.map(|item| async move { executor.resolve_into_value_async(info, &item).await });
    let mut futures = FuturesOrdered::from_iter(iter);

    let mut values = Vec::with_capacity(futures.len());
    while let Some(value) = futures.next().await {
        if stop_on_null && value.is_null() {
            return Ok(value);
        }
        values.push(value);
    }

    Ok(Value::list(values))
}

#[cfg(feature = "async")]
impl<S, T, CtxT> crate::GraphQLTypeAsync<S> for Vec<T>
where
    T: crate::GraphQLTypeAsync<S, Context = CtxT>,
    T::TypeInfo: Send + Sync,
    S: ScalarValue + Send + Sync,
    CtxT: Send + Sync,
{
    fn resolve_async<'a>(
        &'a self,
        info: &'a Self::TypeInfo,
        selection_set: Option<&'a [Selection<S>]>,
        executor: &'a Executor<Self::Context, S>,
    ) -> crate::BoxFuture<'a, ExecutionResult<S>> {
        let f = resolve_into_list_async(executor, info, self.iter());
        Box::pin(f)
    }
}

#[cfg(feature = "async")]
impl<S, T, CtxT> crate::GraphQLTypeAsync<S> for &[T]
where
    T: crate::GraphQLTypeAsync<S, Context = CtxT>,
    T::TypeInfo: Send + Sync,
    S: ScalarValue + Send + Sync,
    CtxT: Send + Sync,
{
    fn resolve_async<'a>(
        &'a self,
        info: &'a Self::TypeInfo,
        selection_set: Option<&'a [Selection<S>]>,
        executor: &'a Executor<Self::Context, S>,
    ) -> crate::BoxFuture<'a, ExecutionResult<S>> {
        let f = resolve_into_list_async(executor, info, self.iter());
        Box::pin(f)
    }
}

#[cfg(feature = "async")]
impl<S, T, CtxT> crate::GraphQLTypeAsync<S> for Option<T>
where
    T: crate::GraphQLTypeAsync<S, Context = CtxT>,
    T::TypeInfo: Send + Sync,
    S: ScalarValue + Send + Sync,
    CtxT: Send + Sync,
{
    fn resolve_async<'a>(
        &'a self,
        info: &'a Self::TypeInfo,
        selection_set: Option<&'a [Selection<S>]>,
        executor: &'a Executor<Self::Context, S>,
    ) -> crate::BoxFuture<'a, ExecutionResult<S>> {
        let f = async move {
            let value = match *self {
                Some(ref obj) => executor.resolve_into_value_async(info, obj).await,
                None => Value::null(),
            };
            Ok(value)
        };
        Box::pin(f)
    }
}
<commit_msg>Fix pattern move error on stable<commit_after>use crate::{
    ast::{FromInputValue, InputValue, Selection, ToInputValue},
    executor::ExecutionResult,
    schema::meta::MetaType,
    value::{ScalarValue, Value},
};

use crate::{
    executor::{Executor, Registry},
    types::base::GraphQLType,
};

impl<S, T, CtxT> GraphQLType<S> for Option<T>
where
    S: ScalarValue,
    T: GraphQLType<S, Context = CtxT>,
{
    type Context = CtxT;
    type TypeInfo = T::TypeInfo;

    fn name(_: &T::TypeInfo) -> Option<&str> {
        None
    }

    fn meta<'r>(info: &T::TypeInfo, registry: &mut Registry<'r, S>) -> MetaType<'r, S>
    where
        S: 'r,
    {
        registry.build_nullable_type::<T>(info).into_meta()
    }

    fn resolve(
        &self,
        info: &T::TypeInfo,
        _: Option<&[Selection<S>]>,
        executor: &Executor<CtxT, S>,
    ) -> ExecutionResult<S> {
        match *self {
            Some(ref obj) => executor.resolve(info, obj),
            None => Ok(Value::null()),
        }
    }
}

impl<S, T> FromInputValue<S> for Option<T>
where
    T: FromInputValue<S>,
    S: ScalarValue,
{
    fn from_input_value<'a>(v: &'a InputValue<S>) -> Option<Option<T>> {
        match v {
            &InputValue::Null => Some(None),
            v => match v.convert() {
                Some(x) => Some(Some(x)),
                None => None,
            },
        }
    }
}

impl<S, T> ToInputValue<S> for Option<T>
where
    T: ToInputValue<S>,
    S: ScalarValue,
{
    fn to_input_value(&self) -> InputValue<S> {
        match *self {
            Some(ref v) => v.to_input_value(),
            None => InputValue::null(),
        }
    }
}

impl<S, T, CtxT> GraphQLType<S> for Vec<T>
where
    T: GraphQLType<S, Context = CtxT>,
    S: ScalarValue,
{
    type Context = CtxT;
    type TypeInfo = T::TypeInfo;

    fn name(_: &T::TypeInfo) -> Option<&str> {
        None
    }

    fn meta<'r>(info: &T::TypeInfo, registry: &mut Registry<'r, S>) -> MetaType<'r, S>
    where
        S: 'r,
    {
        registry.build_list_type::<T>(info).into_meta()
    }

    fn resolve(
        &self,
        info: &T::TypeInfo,
        _: Option<&[Selection<S>]>,
        executor: &Executor<CtxT, S>,
    ) -> ExecutionResult<S> {
        resolve_into_list(executor, info, self.iter())
    }
}

impl<T, S> FromInputValue<S> for Vec<T>
where
    T: FromInputValue<S>,
    S: ScalarValue,
{
    fn from_input_value<'a>(v: &'a InputValue<S>) -> Option<Vec<T>>
where {
        match *v {
            InputValue::List(ref ls) => {
                let v: Vec<_> = ls.iter().filter_map(|i| i.item.convert()).collect();

                if v.len() == ls.len() {
                    Some(v)
                } else {
                    None
                }
            }
            ref other => {
                if let Some(e) = other.convert() {
                    Some(vec![e])
                } else {
                    None
                }
            }
        }
    }
}

impl<T, S> ToInputValue<S> for Vec<T>
where
    T: ToInputValue<S>,
    S: ScalarValue,
{
    fn to_input_value(&self) -> InputValue<S> {
        InputValue::list(self.iter().map(|v| v.to_input_value()).collect())
    }
}

impl<'a, S, T, CtxT> GraphQLType<S> for &'a [T]
where
    S: ScalarValue,
    T: GraphQLType<S, Context = CtxT>,
{
    type Context = CtxT;
    type TypeInfo = T::TypeInfo;

    fn name(_: &T::TypeInfo) -> Option<&str> {
        None
    }

    fn meta<'r>(info: &T::TypeInfo, registry: &mut Registry<'r, S>) -> MetaType<'r, S>
    where
        S: 'r,
    {
        registry.build_list_type::<T>(info).into_meta()
    }

    fn resolve(
        &self,
        info: &T::TypeInfo,
        _: Option<&[Selection<S>]>,
        executor: &Executor<CtxT, S>,
    ) -> ExecutionResult<S> {
        resolve_into_list(executor, info, self.iter())
    }
}

impl<'a, T, S> ToInputValue<S> for &'a [T]
where
    T: ToInputValue<S>,
    S: ScalarValue,
{
    fn to_input_value(&self) -> InputValue<S> {
        InputValue::list(self.iter().map(|v| v.to_input_value()).collect())
    }
}

fn resolve_into_list<S, T, I>(
    executor: &Executor<T::Context, S>,
    info: &T::TypeInfo,
    iter: I,
) -> ExecutionResult<S>
where
    S: ScalarValue,
    I: Iterator<Item = T> + ExactSizeIterator,
    T: GraphQLType<S>,
{
    let stop_on_null = executor
        .current_type()
        .list_contents()
        .expect("Current type is not a list type")
        .is_non_null();
    let mut result = Vec::with_capacity(iter.len());

    for o in iter {
        match executor.resolve(info, &o) {
            Ok(value) => {
                if stop_on_null && value.is_null() {
                    return Ok(value);
                } else {
                    result.push(value)
                }
            }
            Err(e) => return Err(e),
        }
    }

    Ok(Value::list(result))
}

#[cfg(feature = "async")]
async fn resolve_into_list_async<'a, S, T, I>(
    executor: &'a Executor<'a, T::Context, S>,
    info: &'a T::TypeInfo,
    items: I,
) -> ExecutionResult<S>
where
    S: ScalarValue + Send + Sync,
    I: Iterator<Item = T> + ExactSizeIterator,
    T: crate::GraphQLTypeAsync<S>,
    T::TypeInfo: Send + Sync,
    T::Context: Send + Sync,
{
    use futures::stream::{FuturesOrdered, StreamExt};
    use std::iter::FromIterator;

    let stop_on_null = executor
        .current_type()
        .list_contents()
        .expect("Current type is not a list type")
        .is_non_null();

    let iter =
        items.map(|item| async move { executor.resolve_into_value_async(info, &item).await });
    let mut futures = FuturesOrdered::from_iter(iter);

    let mut values = Vec::with_capacity(futures.len());
    while let Some(value) = futures.next().await {
        if stop_on_null && value.is_null() {
            return Ok(value);
        }
        values.push(value);
    }

    Ok(Value::list(values))
}

#[cfg(feature = "async")]
impl<S, T, CtxT> crate::GraphQLTypeAsync<S> for Vec<T>
where
    T: crate::GraphQLTypeAsync<S, Context = CtxT>,
    T::TypeInfo: Send + Sync,
    S: ScalarValue + Send + Sync,
    CtxT: Send + Sync,
{
    fn resolve_async<'a>(
        &'a self,
        info: &'a Self::TypeInfo,
        selection_set: Option<&'a [Selection<S>]>,
        executor: &'a Executor<Self::Context, S>,
    ) -> crate::BoxFuture<'a, ExecutionResult<S>> {
        let f = resolve_into_list_async(executor, info, self.iter());
        Box::pin(f)
    }
}

#[cfg(feature = "async")]
impl<S, T, CtxT> crate::GraphQLTypeAsync<S> for &[T]
where
    T: crate::GraphQLTypeAsync<S, Context = CtxT>,
    T::TypeInfo: Send + Sync,
    S: ScalarValue + Send + Sync,
    CtxT: Send + Sync,
{
    fn resolve_async<'a>(
        &'a self,
        info: &'a Self::TypeInfo,
        selection_set: Option<&'a [Selection<S>]>,
        executor: &'a Executor<Self::Context, S>,
    ) -> crate::BoxFuture<'a, ExecutionResult<S>> {
        let f = resolve_into_list_async(executor, info, self.iter());
        Box::pin(f)
    }
}

#[cfg(feature = "async")]
impl<S, T, CtxT> crate::GraphQLTypeAsync<S> for Option<T>
where
    T: crate::GraphQLTypeAsync<S, Context = CtxT>,
    T::TypeInfo: Send + Sync,
    S: ScalarValue + Send + Sync,
    CtxT: Send + Sync,
{
    fn resolve_async<'a>(
        &'a self,
        info: &'a Self::TypeInfo,
        selection_set: Option<&'a [Selection<S>]>,
        executor: &'a Executor<Self::Context, S>,
    ) -> crate::BoxFuture<'a, ExecutionResult<S>> {
        let f = async move {
            let value = match *self {
                Some(ref obj) => executor.resolve_into_value_async(info, obj).await,
                None => Value::null(),
            };
            Ok(value)
        };
        Box::pin(f)
    }
}
<|endoftext|>


<commit_before>// Demosaic methods adapted from dcraw 9.27
use decoders::Image;
use imageops::fcol;

pub fn ppg(img: &Image, inb: &[f32]) -> Vec<f32> {
  let mut out: Vec<f32> = vec![0.0; (img.width*img.height*3) as usize];

  let mut ipos = 0;
  let mut opos = 0;
  for row in 0..img.width {
    for col in 0..img.height {
      let color = fcol(img, row, col);

      out[opos+0] = inb[ipos];
      out[opos+1] = inb[ipos];
      out[opos+2] = inb[ipos];

      ipos += 1;
      opos += 3;
    }
  }

  out
}
<commit_msg>demosaic: implement the simplest demosaic possible<commit_after>// Demosaic methods adapted from dcraw 9.27
use decoders::Image;
use imageops::fcol;
use std::cmp;

pub fn ppg(img: &Image, inb: &[f32]) -> Vec<f32> {
  let mut out: Vec<f32> = vec![0.0; (img.width*img.height*3) as usize];

  // First we set the colors we already have
  let mut ipos = 0;
  let mut opos = 0;
  for row in 0..img.width {
    for col in 0..img.height {
      let color = fcol(img, row, col);
      out[opos+color] = inb[ipos];
      ipos += 1;
      opos += 3;
    }
  }

  // Now we go around the image setting the unset colors to the average of the
  // surrounding pixels
  for row in 0..img.height {
    for col in 0..img.width {
      let mut sums: [f32; 4] = [0.0;4];
      let mut counts: [f32; 4] = [0.0;4];

      for y in (cmp::max(0,(row as isize)-1) as usize) .. cmp::min(img.height, row+1) {
        for x in (cmp::max(0,(col as isize)-1) as usize) .. cmp::min(img.width, col+1) {
          let color = fcol(img, x, y);
          //println!("Fetching color {} at pos {}x{}", color, x, y);
          sums[color] += inb[y*img.width+x];
          counts[color] += 1.0;
        }
      }

      let color = fcol(img, row, col);
      for c in 0..3 {
        if c != color {
          out[(row*img.width+col)*3+c] = sums[c] / counts[c];
        }
      }
    }
  }

  out
}
<|endoftext|>


<commit_before>use bifrost::utils::fut_exec::wait;
use client::AsyncClient;
use dovahkiin::types::custom_types::bytes::SmallBytes;
use dovahkiin::types::custom_types::id::Id;
use dovahkiin::types::custom_types::map::Map;
use dovahkiin::types::type_id_of;
use dovahkiin::types::value::ToValue;
use index::btree::Ordering;
use index::EntryKey;
use index::Slice;
use index::{id_from_key, key_with_id};
use itertools::Itertools;
use parking_lot::Mutex;
use parking_lot::RwLock;
use ram::cell::Cell;
use ram::schema::Field;
use ram::schema::Schema;
use ram::types::*;
use std::cell::RefCell;
use std::collections::btree_map::Range;
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, BTreeSet, LinkedList};
use std::fmt::Debug;
use std::marker::PhantomData;
use std::mem;
use std::sync::Arc;
use utils::lru_cache::LRUCache;

// LevelTree items cannot been added or removed individually
// Items must been merged from higher level in bulk
// Deletion will be performed when merging to or from higher level tree
// Because tree update will not to be performed in parallel. Unlike memtable, a single r/w lock
// should be sufficient. Thus concurrency control will be simple and efficient.

type Tombstones = BTreeSet<EntryKey>;
type PageCache<S> = Arc<Mutex<LRUCache<Id, Arc<SSPage<S>>>>>;

const PAGE_SCHEMA: &'static str = "NEB_SSTABLE_PAGE";

lazy_static! {
    static ref PAGE_SCHEMA_ID: u32 = key_hash(PAGE_SCHEMA) as u32;
}

pub struct LevelTree<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    neb_client: Arc<AsyncClient>,
    index: BTreeMap<EntryKey, Id>,
    tombstones: Tombstones,
    pages: PageCache<S>,
    marker: PhantomData<S>,
}
struct SSPage<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    id: Id,
    slice: S,
}

impl<S> SSPage<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn from_cell(cell: Cell) -> Self {
        let mut slice = S::init();
        let slice_len = slice.as_slice().len();
        let cell_id = cell.id();
        let keys = cell.data;
        let keys_len = keys.len().unwrap();
        debug_assert!(slice_len == keys_len);
        let keys_array = if let Value::PrimArray(PrimitiveArray::SmallBytes(ref array)) = keys {
            array
        } else {
            panic!()
        };
        for (i, val) in keys_array.iter().enumerate() {
            slice.as_slice()[i] = EntryKey::from(val.as_slice());
        }
        Self { id: cell_id, slice }
    }
}

pub fn page_slice_to_cell<S>(slice: S, id: &Id) -> Cell
where
    S: Slice<Item = EntryKey>,
{
    let value = slice
        .as_slice_immute()
        .iter()
        .map(|key| SmallBytes::from_vec(key.as_slice().to_vec()))
        .collect_vec()
        .value();
    Cell::new_with_id(*PAGE_SCHEMA_ID, id, value)
}

impl<S> LevelTree<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn new(neb_client: &Arc<AsyncClient>) -> Self {
        let client = neb_client.clone();
        Self {
            neb_client: neb_client.clone(),
            index: BTreeMap::new(),
            tombstones: Tombstones::new(),
            // TODO: carefully set the capacity
            pages: Arc::new(Mutex::new(LRUCache::new(
                100,
                move |id| {
                    wait(client.read_cell(*id))
                        .unwrap()
                        .map(|cell| Arc::new(SSPage::from_cell(cell)))
                        .ok()
                },
                |_, _| {
                    // will do nothing on evict for pages will be write to back storage once updated
                },
            ))),
            marker: PhantomData {},
        }
    }

    pub fn seek(&self, key: &EntryKey, ordering: Ordering) -> RTCursor<S> {
        let mut range = match ordering {
            Ordering::Forward => self.index.range::<EntryKey, _>(key..),
            Ordering::Backward => self.index.range::<EntryKey, _>(..=key),
        };
        let first = match ordering {
            Ordering::Forward => range.next(),
            Ordering::Backward => range.next_back(),
        }
        .map(|(_, page_id)| self.pages.lock().get_or_fetch(page_id).unwrap().clone());

        let (pos, page_len) = if let Some(ref page) = first {
            (
                page.slice
                    .as_slice_immute()
                    .binary_search(key)
                    .unwrap_or_else(|i| i),
                page.slice.as_slice_immute().len(),
            )
        } else {
            (0, 0)
        };
        let mut cursor = RTCursor {
            ordering,
            range,
            page_len,
            current: first,
            index: pos,
            cache: self.pages.clone(),
        };

        if cursor.current.is_some() {
            match ordering {
                Ordering::Forward => {
                    if pos >= page_len {
                        cursor.next();
                    }
                }
                Ordering::Backward => {}
            }
        }

        cursor
    }

    pub fn merge(&mut self, mut source: Vec<EntryKey>, tombstones: &mut Tombstones) {
        let mut pages_cache = self.pages.lock();
        let (keys_to_removed, mut merged, mut ids_to_reuse) = {
            let (target_keys, target_pages): (Vec<_>, Vec<_>) = {
                let first = source.first().unwrap();
                let last = source.last().unwrap();
                self.index
                    .range::<EntryKey, _>(first..=last)
                    .map(|(k, id)| (k, pages_cache.get_or_fetch(id).unwrap().clone()))
                    .unzip()
            };
            let page_size = target_pages.first().map(|p| p.slice.len()).unwrap_or(0);
            let target_page_ids = target_pages.iter().map(|p| p.id).collect::<LinkedList<_>>();
            let mut target = target_pages
                .iter()
                .map(|page| page.slice.as_slice_immute())
                .flat_map(|x| x)
                .map(|x| (*x).clone())
                .collect_vec();

            let mut merged: Vec<S> = vec![S::init()];
            let src_len = source.len();
            let tgt_len = target.len();
            let mut srci = 0;
            let mut tgti = 0;

            let mut merging_slice_i = 0;

            while srci < src_len || tgti < tgt_len {
                let mut from_source = false;
                let mut lowest: &mut EntryKey = if srci >= src_len {
                    // source overflowed
                    tgti += 1;
                    &mut target[tgti - 1]
                } else if tgti >= tgt_len {
                    // target overflowed
                    srci += 1;
                    from_source = true;
                    &mut source[srci - 1]
                } else {
                    // normal case, compare
                    let t = &mut target[tgti];
                    let s = &mut source[srci];
                    if t <= s {
                        tgti += 1;
                        t
                    } else {
                        srci += 1;
                        from_source = true;
                        s
                    }
                };
                if from_source {
                    if tombstones.remove(lowest) {
                        continue;
                    }
                } else {
                    if self.tombstones.remove(lowest) {
                        continue;
                    }
                }
                mem::swap(
                    &mut merged.last_mut().unwrap().as_slice()[merging_slice_i],
                    lowest,
                );
                merging_slice_i += 1;
                if merging_slice_i >= page_size {
                    merging_slice_i = 0;
                    merged.push(S::init());
                }
            }
            if merging_slice_i == 0 {
                let merged_len = merged.len();
                merged.remove(merged_len - 1);
            }
            (
                target_keys.into_iter().map(|x| x.clone()).collect_vec(),
                merged,
                target_page_ids,
            )
        };

        for k in keys_to_removed {
            self.index.remove(&k).unwrap();
        }

        for id in &ids_to_reuse {
            pages_cache.remove(id);
        }

        for mut page in merged {
            let index_key = page.as_slice()[0].clone();
            let reuse_id = ids_to_reuse.pop_back();
            let id = reuse_id.unwrap_or_else(|| Id::rand());
            let cell = page_slice_to_cell(page, &id);
            if reuse_id.is_some() {
                wait(self.neb_client.update_cell(cell)).unwrap();
            } else {
                wait(self.neb_client.write_cell(cell)).unwrap();
            }
            self.index.insert(index_key, id);
        }
    }
}

pub trait SortableEntrySlice: Sized + Slice<Item = EntryKey> {}

pub struct RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    index: usize,
    ordering: Ordering,
    page_len: usize,
    range: Range<'a, EntryKey, Id>,
    current: Option<Arc<SSPage<S>>>,
    cache: PageCache<S>,
}

impl<'a, S> RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn next(&mut self) -> bool {
        if self.current.is_some() {
            match self.ordering {
                Ordering::Forward => {
                    if self.index + 1 >= self.page_len {
                        if let Some((_, page_id)) = self.range.next() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = 0;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index += 1;
                    }
                }
                Ordering::Backward => {
                    if self.index == 0 {
                        if let Some((_, page_id)) = self.range.next_back() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = self.page_len - 1;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index -= 1;
                    }
                }
            }
            true
        } else {
            false
        }
    }

    pub fn current(&self) -> Option<&EntryKey> {
        if self.index <= 0 || self.index >= self.page_len {
            return None;
        }
        if let Some(ref page) = self.current {
            Some(&page.slice.as_slice_immute()[self.index])
        } else {
            None
        }
    }

    fn get_page(&self, id: &Id) -> Arc<SSPage<S>> {
        self.cache.lock().get_or_fetch(id).unwrap().clone()
    }
}

pub fn get_schema() -> Schema {
    Schema {
        id: *PAGE_SCHEMA_ID,
        name: String::from(PAGE_SCHEMA),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new("*", type_id_of(Type::SmallBytes), false, true, None),
    }
}

macro_rules! impl_sspage_slice {
    ($t: ty, $et: ty, $n: expr) => {
        impl_slice_ops!($t, $et, $n);
        impl SortableEntrySlice for $t {}
    };
}

#[cfg(test)]
mod test {
    use super::*;
    use client;
    use index::sstable::LevelTree;
    use server::NebServer;
    use server::ServerOptions;
    use std::ptr;
    use std::sync::Arc;

    impl_sspage_slice!([EntryKey; 1000], EntryKey, 1000);

    pub fn init() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5200");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 16 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(super::get_schema());

        let mut tree: LevelTree<[EntryKey; 1000]> = LevelTree::new(&client);
        let id = Id::unit_id();
        let key = smallvec![1, 2, 3, 4, 5, 6];
        let mut tombstones = BTreeSet::new();
        info!("test insertion");

        let mut key_id = key.clone();
        key_with_id(&mut key_id, &id);

        tree.merge(vec![key_id], &mut tombstones);
        let mut cursor = tree.seek(&key, Ordering::Forward);
        assert_eq!(id_from_key(cursor.current().unwrap()), id);
    }
}
<commit_msg>fix merge, log seek<commit_after>use bifrost::utils::fut_exec::wait;
use client::AsyncClient;
use dovahkiin::types::custom_types::bytes::SmallBytes;
use dovahkiin::types::custom_types::id::Id;
use dovahkiin::types::custom_types::map::Map;
use dovahkiin::types::type_id_of;
use dovahkiin::types::value::ToValue;
use index::btree::Ordering;
use index::EntryKey;
use index::Slice;
use index::{id_from_key, key_with_id};
use itertools::Itertools;
use parking_lot::Mutex;
use parking_lot::RwLock;
use ram::cell::Cell;
use ram::schema::Field;
use ram::schema::Schema;
use ram::types::*;
use std::cell::RefCell;
use std::collections::btree_map::Range;
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, BTreeSet, LinkedList};
use std::fmt::Debug;
use std::marker::PhantomData;
use std::mem;
use std::sync::Arc;
use utils::lru_cache::LRUCache;

// LevelTree items cannot been added or removed individually
// Items must been merged from higher level in bulk
// Deletion will be performed when merging to or from higher level tree
// Because tree update will not to be performed in parallel. Unlike memtable, a single r/w lock
// should be sufficient. Thus concurrency control will be simple and efficient.

type Tombstones = BTreeSet<EntryKey>;
type PageCache<S> = Arc<Mutex<LRUCache<Id, Arc<SSPage<S>>>>>;

const PAGE_SCHEMA: &'static str = "NEB_SSTABLE_PAGE";

lazy_static! {
    static ref PAGE_SCHEMA_ID: u32 = key_hash(PAGE_SCHEMA) as u32;
}

pub struct LevelTree<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    neb_client: Arc<AsyncClient>,
    index: BTreeMap<EntryKey, Id>,
    tombstones: Tombstones,
    pages: PageCache<S>,
    marker: PhantomData<S>,
}
struct SSPage<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    id: Id,
    slice: S,
    len: usize,
}

impl<S> SSPage<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn from_cell(cell: Cell) -> Self {
        let mut slice = S::init();
        let slice_len = slice.as_slice().len();
        let cell_id = cell.id();
        let keys = cell.data;
        let keys_len = keys.len().unwrap();
        debug_assert!(slice_len == keys_len);
        let keys_array = if let Value::PrimArray(PrimitiveArray::SmallBytes(ref array)) = keys {
            array
        } else {
            panic!()
        };
        for (i, val) in keys_array.iter().enumerate() {
            slice.as_slice()[i] = EntryKey::from(val.as_slice());
        }
        Self {
            id: cell_id,
            slice,
            len: keys_array.len(),
        }
    }

    pub fn to_cell(&self) -> Cell
    {
        let value = self
            .slice
            .as_slice_immute()
            .iter()
            .map(|key| SmallBytes::from_vec(key.as_slice().to_vec()))
            .take(self.len)
            .collect_vec()
            .value();
        Cell::new_with_id(*PAGE_SCHEMA_ID, &self.id, value)
    }

    pub fn new() -> Self {
        SSPage {
            slice: S::init(),
            id: Id::unit_id(),
            len: 0,
        }
    }
}

impl<S> LevelTree<S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn new(neb_client: &Arc<AsyncClient>) -> Self {
        let client = neb_client.clone();
        let mut index = BTreeMap::new();
        Self {
            neb_client: neb_client.clone(),
            index,
            tombstones: Tombstones::new(),
            // TODO: carefully set the capacity
            pages: Arc::new(Mutex::new(LRUCache::new(
                100,
                move |id| {
                    wait(client.read_cell(*id))
                        .unwrap()
                        .map(|cell| Arc::new(SSPage::from_cell(cell)))
                        .ok()
                },
                |_, _| {
                    // will do nothing on evict for pages will be write to back storage once updated
                },
            ))),
            marker: PhantomData {},
        }
    }

    pub fn seek(&self, key: &EntryKey, ordering: Ordering) -> RTCursor<S> {
        debug!("Seeking for {:?}, in index len {}", key, self.index.len());
        let mut range = match ordering {
            Ordering::Forward => self.index.range::<EntryKey, _>(key..),
            Ordering::Backward => self.index.range::<EntryKey, _>(..=key),
        };
        let first = match ordering {
            Ordering::Forward => range.next(),
            Ordering::Backward => range.next_back(),
        }
        .map(|(_, page_id)| {
            debug!("First page id is {:?}", page_id);
            self.pages.lock().get_or_fetch(page_id).unwrap().clone()
        });

        let (pos, page_len) = if let Some(ref page) = first {
            (
                page.slice
                    .as_slice_immute()
                    .binary_search(key)
                    .unwrap_or_else(|i| i),
                page.slice.as_slice_immute().len(),
            )
        } else {
            (0, 0)
        };
        let mut cursor = RTCursor {
            ordering,
            range,
            page_len,
            current: first,
            index: pos,
            cache: self.pages.clone(),
        };

        if cursor.current.is_some() {
            match ordering {
                Ordering::Forward => {
                    if pos >= page_len {
                        cursor.next();
                    }
                }
                Ordering::Backward => {}
            }
        }

        cursor
    }

    pub fn merge(&mut self, mut source: Vec<EntryKey>, tombstones: &mut Tombstones) {
        let mut pages_cache = self.pages.lock();
        let (keys_to_removed, mut merged, mut ids_to_reuse) = {
            let (target_keys, target_pages): (Vec<_>, Vec<_>) = {
                let first = source.first().unwrap();
                let last = source.last().unwrap();
                self.index
                    .range::<EntryKey, _>(first..=last)
                    .map(|(k, id)| (k, pages_cache.get_or_fetch(id).unwrap().clone()))
                    .unzip()
            };
            let mut merged: Vec<SSPage<S>> = vec![SSPage::new()];
            let page_size = merged[0].slice.len();
            let target_page_ids = target_pages.iter().map(|p| p.id).collect::<LinkedList<_>>();
            let mut target = target_pages
                .iter()
                .map(|page| &page.slice.as_slice_immute()[..page.len])
                .flat_map(|x| x)
                .map(|x| (*x).clone())
                .collect_vec();

            let src_len = source.len();
            let tgt_len = target.len();

            debug!("Src len {}, target len {}", src_len, tgt_len);

            let mut srci = 0;
            let mut tgti = 0;

            while srci < src_len || tgti < tgt_len {
                let mut from_source = false;
                let mut lowest: &mut EntryKey = if srci >= src_len {
                    // source overflowed
                    tgti += 1;
                    &mut target[tgti - 1]
                } else if tgti >= tgt_len {
                    // target overflowed
                    srci += 1;
                    from_source = true;
                    &mut source[srci - 1]
                } else {
                    // normal case, compare
                    let t = &mut target[tgti];
                    let s = &mut source[srci];
                    if t <= s {
                        tgti += 1;
                        t
                    } else {
                        srci += 1;
                        from_source = true;
                        s
                    }
                };

                if from_source {
                    if tombstones.remove(lowest) {
                        continue;
                    }
                } else {
                    if self.tombstones.remove(lowest) {
                        continue;
                    }
                }

                let merging_page_len = {
                    let merging_page = merged.last_mut().unwrap();
                    let page_len = merging_page.len;
                    debug!("i is {}, picking {:?}", page_len, lowest);
                    mem::swap(&mut merging_page.slice.as_slice()[page_len], lowest);
                    merging_page.len += 1;
                    merging_page.len
                };

                if merging_page_len >= page_size {
                    merged.push(SSPage::new());
                }
            }
            if merged.last().unwrap().len == 0 {
                let merged_len = merged.len();
                merged.remove(merged_len - 1);
            }
            (
                target_keys.into_iter().map(|x| x.clone()).collect_vec(),
                merged,
                target_page_ids,
            )
        };

        debug!("Merged {} pages", merged.len());

        for k in keys_to_removed {
            debug!("Remove stale page index {:?}", k);
            self.index.remove(&k).unwrap();
        }

        for id in &ids_to_reuse {
            debug!("Remove stale page cache {:?}", id);
            pages_cache.remove(id);
        }

        for mut page in merged {
            let reuse_id = ids_to_reuse.pop_back();
            let id = reuse_id.unwrap_or_else(|| Id::rand());
            let index_key = page.slice.as_slice_immute()[0].clone();
            page.id = id;
            let cell = page.to_cell();
            if reuse_id.is_some() {
                debug!("Reuse page id by updating {:?}", cell.id());
                wait(self.neb_client.update_cell(cell)).unwrap();
            } else {
                debug!("Inserting page with id {:?}", cell.id());
                wait(self.neb_client.write_cell(cell)).unwrap();
            }

            debug!("Insert page index key {:?}, id {:?}", index_key, id);
            self.index.insert(index_key, id);
        }
    }
}

pub trait SortableEntrySlice: Sized + Slice<Item = EntryKey> {}

pub struct RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    index: usize,
    ordering: Ordering,
    page_len: usize,
    range: Range<'a, EntryKey, Id>,
    current: Option<Arc<SSPage<S>>>,
    cache: PageCache<S>,
}

impl<'a, S> RTCursor<'a, S>
where
    S: Slice<Item = EntryKey> + SortableEntrySlice,
{
    pub fn next(&mut self) -> bool {
        if self.current.is_some() {
            match self.ordering {
                Ordering::Forward => {
                    if self.index + 1 >= self.page_len {
                        if let Some((_, page_id)) = self.range.next() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = 0;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index += 1;
                    }
                }
                Ordering::Backward => {
                    if self.index == 0 {
                        if let Some((_, page_id)) = self.range.next_back() {
                            let next_page = self.get_page(page_id);
                            self.current = Some(next_page);
                            self.index = self.page_len - 1;
                        } else {
                            self.current = None;
                            return false;
                        }
                    } else {
                        self.index -= 1;
                    }
                }
            }
            true
        } else {
            false
        }
    }

    pub fn current(&self) -> Option<&EntryKey> {
        if self.index <= 0 || self.index >= self.page_len {
            return None;
        }
        if let Some(ref page) = self.current {
            Some(&page.slice.as_slice_immute()[self.index])
        } else {
            None
        }
    }

    fn get_page(&self, id: &Id) -> Arc<SSPage<S>> {
        self.cache.lock().get_or_fetch(id).unwrap().clone()
    }
}

pub fn get_schema() -> Schema {
    Schema {
        id: *PAGE_SCHEMA_ID,
        name: String::from(PAGE_SCHEMA),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new("*", type_id_of(Type::SmallBytes), false, true, None),
    }
}

macro_rules! impl_sspage_slice {
    ($t: ty, $et: ty, $n: expr) => {
        impl_slice_ops!($t, $et, $n);
        impl SortableEntrySlice for $t {}
    };
}

#[cfg(test)]
mod test {
    use super::*;
    use client;
    use index::sstable::LevelTree;
    use server::NebServer;
    use server::ServerOptions;
    use std::ptr;
    use std::sync::Arc;

    type SmallPage = [EntryKey; 5];

    impl_sspage_slice!(SmallPage, EntryKey, 5);

    pub fn init() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5200");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 16 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(super::get_schema());

        let mut tree: LevelTree<SmallPage> = LevelTree::new(&client);
        let id = Id::unit_id();
        let key = smallvec![1, 2, 3, 4, 5, 6];
        let mut tombstones = BTreeSet::new();
        info!("test insertion");

        let mut key_id = key.clone();
        key_with_id(&mut key_id, &id);

        tree.merge(vec![key_id], &mut tombstones);
        let mut cursor = tree.seek(&key, Ordering::Forward);
        assert_eq!(id_from_key(cursor.current().unwrap()), id);
    }
}
<|endoftext|>


<commit_before>use super::Result;

#[derive(Debug,PartialEq)]
pub struct Memory {
    total: u64,
    free: u64,
    used: u64,
    buffers: u64,
    cached: u64,
    swap_total: u64,
    swap_free: u64,
    swap_used: u64,
}

const PROC_MEMORY_NUMBER_OF_FIELDS: usize = 6;
const SYS_MEMORY_NUMBER_OF_FIELDS: usize = 1;

impl Memory {
    /// Total amount of physical memory in Kb.
    pub fn total(&self) -> u64 {
        self.total
    }

    pub fn free(&self) -> u64 {
        self.free
    }

    /// Total amount of used physical memory in Kb.
    pub fn used(&self) -> u64 {
        self.used
    }

    /// Total amount of swap space in Kb.
    pub fn swap_total(&self) -> u64 {
        self.swap_total
    }

    /// Total amount of free swap space in Kb.
    pub fn swap_free(&self) -> u64 {
        self.swap_free
    }

    /// Total amount of used swap space in Kb.
    pub fn swap_used(&self) -> u64 {
        self.swap_used
    }
}

/// Read the current memory status of the system.
#[cfg(target_os = "linux")]
pub fn read() -> Result<Memory> {
    os::read()
}

/// Read the current memory status of the container.
#[cfg(target_os = "linux")]
pub fn read_from_container() -> Result<Memory> {
    os::read_from_container()
}

#[cfg(target_os = "linux")]
mod os {
    use std::io::BufRead;
    use std::path::Path;

    use super::{Memory,PROC_MEMORY_NUMBER_OF_FIELDS,SYS_MEMORY_NUMBER_OF_FIELDS};
    use super::super::{ProbeError,Result};
    use super::super::{file_to_buf_reader,parse_u64,dir_exists};

    #[inline]
    pub fn read() -> Result<Memory> {
        read_and_parse_proc_memory(&Path::new("/proc/memory.stat"))
    }

    pub fn read_from_container() -> Result<Memory> {
        let sys_fs_dir = Path::new("/sys/fs/cgroup/memory/");
        if dir_exists(sys_fs_dir) {
            read_and_parse_sys_memory(&sys_fs_dir)
        } else {
            let message = format!("Directory `{}` not found", sys_fs_dir.to_str().unwrap_or("unknown path"));
            Err(ProbeError::UnexpectedContent(message))
        }
    }

    #[inline]
    pub fn read_and_parse_proc_memory(path: &Path) -> Result<Memory> {
        let mut memory = Memory {
            total: 0,
            free: 0,
            used: 0,
            buffers: 0,
            cached: 0,
            swap_total: 0,
            swap_free: 0,
            swap_used: 0,
        };
        let mut free = 0;

        let reader = try!(file_to_buf_reader(path));

        let mut fields_encountered = 0;
        for line in reader.lines() {
            let line = try!(line);
            let segments: Vec<&str> = line.split_whitespace().collect();
            let value: u64 = try!(parse_u64(segments[1]));

            // If this is a field we recognize set it's value and increment the
            // number of fields we encountered.
            fields_encountered += match segments[0] {
                "MemTotal:" => {
                    memory.total = value;
                    1
                },
                "MemFree:" => {
                    free = value;
                    1
                },
                "Buffers:" => {
                    memory.buffers = value;
                    1
                },
                "Cached:" => {
                    memory.cached = value;
                    1
                },
                "SwapTotal:" => {
                    memory.swap_total = value;
                    1
                },
                "SwapFree:" => {
                    memory.swap_free = value;
                    1
                },
                _ => 0
            };

            if fields_encountered == PROC_MEMORY_NUMBER_OF_FIELDS {
                break
            }
        }

        if fields_encountered != PROC_MEMORY_NUMBER_OF_FIELDS {
            return Err(ProbeError::UnexpectedContent("Did not encounter all expected fields".to_owned()))
        }

        // Total amount of free physical memory in Kb.
        // Includes buffers and caches, these will be freed
        // up by the OS when the memory is needed.
        memory.free = free + memory.buffers + memory.cached;
        memory.used = memory.total - memory.free;
        memory.swap_used = memory.swap_total - memory.swap_free;

        Ok(memory)
    }

    pub fn read_and_parse_sys_memory(path: &Path) -> Result<Memory> {
        let mut memory = Memory {
            total: 0,
            free: 0,
            used: 0,
            buffers: 0,
            cached: 0,
            swap_total: 0,
            swap_free: 0,
            swap_used: 0,
        };

        memory.total = bytes_to_kilo_bytes(try!(read_file_value_as_u64(&path.join("memory.limit_in_bytes"))));
        let used_memory = bytes_to_kilo_bytes(try!(read_file_value_as_u64(&path.join("memory.usage_in_bytes"))));

        let mut fields_encountered = 0;
        let reader = try!(file_to_buf_reader(&path.join("memory.stat")));
        for line in reader.lines() {
            let line = try!(line);
            let segments: Vec<&str> = line.split_whitespace().collect();
            let value = try!(parse_u64(&segments[1]));

            fields_encountered += match segments[0] {
                "cache" => {
                    memory.cached = bytes_to_kilo_bytes(value);
                    1
                },
                _ => 0
            };

            if fields_encountered == SYS_MEMORY_NUMBER_OF_FIELDS {
                break
            }
        }
        memory.used = used_memory - memory.cached;
        memory.free = memory.total - memory.used;

        // If swap is not configured for the container, read 0 as value
        memory.swap_total = match read_file_value_as_u64(&path.join("memory.memsw.limit_in_bytes")) {
            Ok(value) => bytes_to_kilo_bytes(value).checked_sub(memory.total).unwrap_or(0),
            Err(_) => 0
        };
        // If swap is not configured for the container, read 0 as value
        memory.swap_used = match read_file_value_as_u64(&path.join("memory.memsw.usage_in_bytes")) {
            Ok(value) => bytes_to_kilo_bytes(value).checked_sub(used_memory).unwrap_or(0),
            Err(_) => 0
        };
        memory.swap_free = memory.swap_total.checked_sub(memory.swap_used).unwrap_or(0);

        Ok(memory)
    }

    fn read_file_value_as_u64(path: &Path) -> Result<u64> {
        let mut reader = try!(file_to_buf_reader(path));
        let mut line = String::new();
        try!(reader.read_line(&mut line));
        parse_u64(&line.trim())
    }

    fn bytes_to_kilo_bytes(bytes: u64) -> u64 {
        bytes.checked_div(1024).unwrap_or(0)
    }
}

#[cfg(test)]
mod tests {
    use std::path::Path;
    use super::super::ProbeError;
    use super::Memory;

    #[test]
    fn test_read_memory() {
        assert!(super::read().is_ok());
    }

    #[test]
    fn test_read_from_container() {
        assert!(super::read_from_container().is_ok());
    }

    #[test]
    fn test_read_and_parse_proc_memory() {
        let path = Path::new("fixtures/linux/memory/proc_meminfo");
        let memory = super::os::read_and_parse_proc_memory(&path).unwrap();

        let expected = Memory {
            total: 376072,
            free: 324248,
            used: 51824,
            buffers: 22820,
            cached: 176324,
            swap_total: 1101816,
            swap_free: 1100644,
            swap_used: 1172,
        };
        assert_eq!(expected, memory);
        assert_eq!(memory.total, memory.used + memory.free);
        assert_eq!(memory.swap_total, memory.swap_used + memory.swap_free);
    }

    #[test]
    fn test_read_and_parse_memory_wrong_path() {
        let path = Path::new("/nonsense");
        match super::os::read_and_parse_proc_memory(&path) {
            Err(ProbeError::IO(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_memory_incomplete() {
        let path = Path::new("fixtures/linux/memory/proc_meminfo_incomplete");
        match super::os::read_and_parse_proc_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_memory_garbage() {
        let path = Path::new("fixtures/linux/memory/proc_meminfo_garbage");
        match super::os::read_and_parse_proc_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory/");
        let memory = super::os::read_and_parse_sys_memory(&path).unwrap();

        let expected = Memory {
            total: 512000, // 500mb
            free: 503400, // total - used
            used: 8600,
            buffers: 0,
            cached: 58928,
            swap_total: 1_488_000, // reported swap total - reported memory total
            swap_free: 1_055_528,
            swap_used: 432_472, // reported swap used - (reported memory used, including cache)
        };
        assert_eq!(expected, memory);
        assert_eq!(memory.total, memory.used + memory.free);
        assert_eq!(memory.swap_total, memory.swap_used + memory.swap_free);
    }

    #[test]
    fn test_read_and_parse_sys_memory_wrong_path() {
        let path = Path::new("/nonsense");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::IO(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_incomplete() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_incomplete/");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_missing_files() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_missing_files/");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::IO(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_garbage() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_garbage/");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_no_swap() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_without_swap/");
        let memory = super::os::read_and_parse_sys_memory(&path).unwrap();

        let expected = Memory {
            total: 512000, // 500mb
            free: 503400, // total - used
            used: 8600,
            buffers: 0,
            cached: 58928,
            swap_total: 0, // Reads 0 swap
            swap_free: 0,  // Reads 0 swap
            swap_used: 0,
        };
        assert_eq!(expected, memory);
        assert_eq!(memory.total, memory.used + memory.free);
        assert_eq!(memory.swap_total, memory.swap_used + memory.swap_free);
    }
}
<commit_msg>Fix proc memory file name<commit_after>use super::Result;

#[derive(Debug,PartialEq)]
pub struct Memory {
    total: u64,
    free: u64,
    used: u64,
    buffers: u64,
    cached: u64,
    swap_total: u64,
    swap_free: u64,
    swap_used: u64,
}

const PROC_MEMORY_NUMBER_OF_FIELDS: usize = 6;
const SYS_MEMORY_NUMBER_OF_FIELDS: usize = 1;

impl Memory {
    /// Total amount of physical memory in Kb.
    pub fn total(&self) -> u64 {
        self.total
    }

    pub fn free(&self) -> u64 {
        self.free
    }

    /// Total amount of used physical memory in Kb.
    pub fn used(&self) -> u64 {
        self.used
    }

    /// Total amount of swap space in Kb.
    pub fn swap_total(&self) -> u64 {
        self.swap_total
    }

    /// Total amount of free swap space in Kb.
    pub fn swap_free(&self) -> u64 {
        self.swap_free
    }

    /// Total amount of used swap space in Kb.
    pub fn swap_used(&self) -> u64 {
        self.swap_used
    }
}

/// Read the current memory status of the system.
#[cfg(target_os = "linux")]
pub fn read() -> Result<Memory> {
    os::read()
}

/// Read the current memory status of the container.
#[cfg(target_os = "linux")]
pub fn read_from_container() -> Result<Memory> {
    os::read_from_container()
}

#[cfg(target_os = "linux")]
mod os {
    use std::io::BufRead;
    use std::path::Path;

    use super::{Memory,PROC_MEMORY_NUMBER_OF_FIELDS,SYS_MEMORY_NUMBER_OF_FIELDS};
    use super::super::{ProbeError,Result};
    use super::super::{file_to_buf_reader,parse_u64,dir_exists};

    #[inline]
    pub fn read() -> Result<Memory> {
        read_and_parse_proc_memory(&Path::new("/proc/meminfo"))
    }

    pub fn read_from_container() -> Result<Memory> {
        let sys_fs_dir = Path::new("/sys/fs/cgroup/memory/");
        if dir_exists(sys_fs_dir) {
            read_and_parse_sys_memory(&sys_fs_dir)
        } else {
            let message = format!("Directory `{}` not found", sys_fs_dir.to_str().unwrap_or("unknown path"));
            Err(ProbeError::UnexpectedContent(message))
        }
    }

    #[inline]
    pub fn read_and_parse_proc_memory(path: &Path) -> Result<Memory> {
        let mut memory = Memory {
            total: 0,
            free: 0,
            used: 0,
            buffers: 0,
            cached: 0,
            swap_total: 0,
            swap_free: 0,
            swap_used: 0,
        };
        let mut free = 0;

        let reader = try!(file_to_buf_reader(path));

        let mut fields_encountered = 0;
        for line in reader.lines() {
            let line = try!(line);
            let segments: Vec<&str> = line.split_whitespace().collect();
            let value: u64 = try!(parse_u64(segments[1]));

            // If this is a field we recognize set it's value and increment the
            // number of fields we encountered.
            fields_encountered += match segments[0] {
                "MemTotal:" => {
                    memory.total = value;
                    1
                },
                "MemFree:" => {
                    free = value;
                    1
                },
                "Buffers:" => {
                    memory.buffers = value;
                    1
                },
                "Cached:" => {
                    memory.cached = value;
                    1
                },
                "SwapTotal:" => {
                    memory.swap_total = value;
                    1
                },
                "SwapFree:" => {
                    memory.swap_free = value;
                    1
                },
                _ => 0
            };

            if fields_encountered == PROC_MEMORY_NUMBER_OF_FIELDS {
                break
            }
        }

        if fields_encountered != PROC_MEMORY_NUMBER_OF_FIELDS {
            return Err(ProbeError::UnexpectedContent("Did not encounter all expected fields".to_owned()))
        }

        // Total amount of free physical memory in Kb.
        // Includes buffers and caches, these will be freed
        // up by the OS when the memory is needed.
        memory.free = free + memory.buffers + memory.cached;
        memory.used = memory.total - memory.free;
        memory.swap_used = memory.swap_total - memory.swap_free;

        Ok(memory)
    }

    pub fn read_and_parse_sys_memory(path: &Path) -> Result<Memory> {
        let mut memory = Memory {
            total: 0,
            free: 0,
            used: 0,
            buffers: 0,
            cached: 0,
            swap_total: 0,
            swap_free: 0,
            swap_used: 0,
        };

        memory.total = bytes_to_kilo_bytes(try!(read_file_value_as_u64(&path.join("memory.limit_in_bytes"))));
        let used_memory = bytes_to_kilo_bytes(try!(read_file_value_as_u64(&path.join("memory.usage_in_bytes"))));

        let mut fields_encountered = 0;
        let reader = try!(file_to_buf_reader(&path.join("memory.stat")));
        for line in reader.lines() {
            let line = try!(line);
            let segments: Vec<&str> = line.split_whitespace().collect();
            let value = try!(parse_u64(&segments[1]));

            fields_encountered += match segments[0] {
                "cache" => {
                    memory.cached = bytes_to_kilo_bytes(value);
                    1
                },
                _ => 0
            };

            if fields_encountered == SYS_MEMORY_NUMBER_OF_FIELDS {
                break
            }
        }
        memory.used = used_memory - memory.cached;
        memory.free = memory.total - memory.used;

        // If swap is not configured for the container, read 0 as value
        memory.swap_total = match read_file_value_as_u64(&path.join("memory.memsw.limit_in_bytes")) {
            Ok(value) => bytes_to_kilo_bytes(value).checked_sub(memory.total).unwrap_or(0),
            Err(_) => 0
        };
        // If swap is not configured for the container, read 0 as value
        memory.swap_used = match read_file_value_as_u64(&path.join("memory.memsw.usage_in_bytes")) {
            Ok(value) => bytes_to_kilo_bytes(value).checked_sub(used_memory).unwrap_or(0),
            Err(_) => 0
        };
        memory.swap_free = memory.swap_total.checked_sub(memory.swap_used).unwrap_or(0);

        Ok(memory)
    }

    fn read_file_value_as_u64(path: &Path) -> Result<u64> {
        let mut reader = try!(file_to_buf_reader(path));
        let mut line = String::new();
        try!(reader.read_line(&mut line));
        parse_u64(&line.trim())
    }

    fn bytes_to_kilo_bytes(bytes: u64) -> u64 {
        bytes.checked_div(1024).unwrap_or(0)
    }
}

#[cfg(test)]
mod tests {
    use std::path::Path;
    use super::super::ProbeError;
    use super::Memory;

    #[test]
    fn test_read_memory() {
        assert!(super::read().is_ok());
    }

    #[test]
    fn test_read_from_container() {
        assert!(super::read_from_container().is_ok());
    }

    #[test]
    fn test_read_and_parse_proc_memory() {
        let path = Path::new("fixtures/linux/memory/proc_meminfo");
        let memory = super::os::read_and_parse_proc_memory(&path).unwrap();

        let expected = Memory {
            total: 376072,
            free: 324248,
            used: 51824,
            buffers: 22820,
            cached: 176324,
            swap_total: 1101816,
            swap_free: 1100644,
            swap_used: 1172,
        };
        assert_eq!(expected, memory);
        assert_eq!(memory.total, memory.used + memory.free);
        assert_eq!(memory.swap_total, memory.swap_used + memory.swap_free);
    }

    #[test]
    fn test_read_and_parse_memory_wrong_path() {
        let path = Path::new("/nonsense");
        match super::os::read_and_parse_proc_memory(&path) {
            Err(ProbeError::IO(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_memory_incomplete() {
        let path = Path::new("fixtures/linux/memory/proc_meminfo_incomplete");
        match super::os::read_and_parse_proc_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_memory_garbage() {
        let path = Path::new("fixtures/linux/memory/proc_meminfo_garbage");
        match super::os::read_and_parse_proc_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory/");
        let memory = super::os::read_and_parse_sys_memory(&path).unwrap();

        let expected = Memory {
            total: 512000, // 500mb
            free: 503400, // total - used
            used: 8600,
            buffers: 0,
            cached: 58928,
            swap_total: 1_488_000, // reported swap total - reported memory total
            swap_free: 1_055_528,
            swap_used: 432_472, // reported swap used - (reported memory used, including cache)
        };
        assert_eq!(expected, memory);
        assert_eq!(memory.total, memory.used + memory.free);
        assert_eq!(memory.swap_total, memory.swap_used + memory.swap_free);
    }

    #[test]
    fn test_read_and_parse_sys_memory_wrong_path() {
        let path = Path::new("/nonsense");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::IO(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_incomplete() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_incomplete/");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_missing_files() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_missing_files/");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::IO(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_garbage() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_garbage/");
        match super::os::read_and_parse_sys_memory(&path) {
            Err(ProbeError::UnexpectedContent(_)) => (),
            r => panic!("Unexpected result: {:?}", r)
        }
    }

    #[test]
    fn test_read_and_parse_sys_memory_no_swap() {
        let path = Path::new("fixtures/linux/sys/fs/cgroup/memory_without_swap/");
        let memory = super::os::read_and_parse_sys_memory(&path).unwrap();

        let expected = Memory {
            total: 512000, // 500mb
            free: 503400, // total - used
            used: 8600,
            buffers: 0,
            cached: 58928,
            swap_total: 0, // Reads 0 swap
            swap_free: 0,  // Reads 0 swap
            swap_used: 0,
        };
        assert_eq!(expected, memory);
        assert_eq!(memory.total, memory.used + memory.free);
        assert_eq!(memory.swap_total, memory.swap_used + memory.swap_free);
    }
}
<|endoftext|>


<commit_before>extern crate byteorder;
use std::io::Cursor;
use std::io::Write;
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};

#[derive(Debug)]
pub enum MemoryError {
  PPUAccessViolation(String),
  APUAccessViolation(String),
}

pub struct Memory {
    pub mem: Vec<u8>,
}

impl Memory {
    pub fn new() -> Memory {
        let mut m =  Memory {
            mem: Vec::with_capacity(::std::u16::MAX as usize)
        };
        unsafe { &m.mem.set_len(::std::u16::MAX as usize); }
        return m;
    }

    // meant for a 'raw' write interface, not meant to be used by the 6502 processor itself, more
    // tests and other tools to be able to read blocks of memory quickly and easily
    //
    pub fn write(self:&mut Memory, index:u16, inp: &[u8]) {
        let idx = index as usize;
        if inp.len() + idx > self.mem.len() {
            panic!("memory vec length is {}, input array goes from {} to {}",self.mem.len(),index,inp.len()+idx)
        }
        &self.mem[idx..(idx+inp.len())].clone_from_slice(inp);
    }
    pub fn read8(self:&Memory,addr: u16) -> Result<u8,MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) => Ok(self.mem[raddr]),
            Err(err) => Err(err)
        }
    }
    pub fn read16(self:&Memory,addr: u16)  -> Result<u16,MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) =>{
                // splices are exclusive on the upper range (half open)
                let mut rdr = Cursor::new(&self.mem[raddr..(raddr+2)]);
                Ok(rdr.read_u16::<LittleEndian>().unwrap())
            },
            Err(err) => Err(err)
        }
    }
    pub fn write8(self:&mut Memory,addr: u16, val:u8) -> Option<MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) => {
                self.mem[raddr] = val;
                Option::None
            },
            Err(err) => Option::Some(err)
        }
    }
    pub fn write16(self:&mut Memory,addr: u16, val:u16) -> Option<MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) => {
                // splices are exclusive on the upper range (half open)
                (&mut self.mem[raddr..(raddr+2)]).write_u16::<LittleEndian>(val).unwrap();
                Option::None
            },
            Err(err) => Option::Some(err)
        }
    }

    // https://en.wikibooks.org/wiki/NES_Programming/Memory_Map
    //
    fn resolve_address(self:&Memory,addr: u16) -> Result<usize,MemoryError> {
        if addr <= 0x1FFF { return Ok( (addr & 0x7FF) as usize); }
        if addr <= 0x2007 { return Err(MemoryError::PPUAccessViolation("PPU Memory access is currently not implemented".to_string())); }
        if addr <= 0x3FFF { return Ok( (addr & 0x7FF & 0x08) as usize); }
        if addr <= 0x401F { return Err(MemoryError::APUAccessViolation("Memory access is currently not implemented".to_string())); }
        Ok(addr as usize)
    }
}
<commit_msg>memory write8/16 return Result<(),Error> (better semantics)<commit_after>extern crate byteorder;
use std::io::Cursor;
use std::io::Write;
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};

#[derive(Debug)]
pub enum MemoryError {
  PPUAccessViolation(String),
  APUAccessViolation(String),
}

pub struct Memory {
    pub mem: Vec<u8>,
}

impl Memory {
    pub fn new() -> Memory {
        let mut m =  Memory {
            mem: Vec::with_capacity(::std::u16::MAX as usize)
        };
        unsafe { &m.mem.set_len(::std::u16::MAX as usize); }
        return m;
    }

    // meant for a 'raw' write interface, not meant to be used by the 6502 processor itself, more
    // tests and other tools to be able to read blocks of memory quickly and easily
    //
    pub fn write(self:&mut Memory, index:u16, inp: &[u8]) {
        let idx = index as usize;
        if inp.len() + idx > self.mem.len() {
            panic!("memory vec length is {}, input array goes from {} to {}",self.mem.len(),index,inp.len()+idx)
        }
        &self.mem[idx..(idx+inp.len())].clone_from_slice(inp);
    }
    pub fn read8(self:&Memory,addr: u16) -> Result<u8,MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) => Ok(self.mem[raddr]),
            Err(err) => Err(err)
        }
    }
    pub fn read16(self:&Memory,addr: u16)  -> Result<u16,MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) =>{
                // splices are exclusive on the upper range (half open)
                let mut rdr = Cursor::new(&self.mem[raddr..(raddr+2)]);
                Ok(rdr.read_u16::<LittleEndian>().unwrap())
            },
            Err(err) => Err(err)
        }
    }
    pub fn write8(self:&mut Memory,addr: u16, val:u8) -> Result<(),MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) => {
                self.mem[raddr] = val;
                Ok(())
            },
            Err(err) => Err(err)
        }
    }
    pub fn write16(self:&mut Memory,addr: u16, val:u16) -> Result<(),MemoryError> {
        match self.resolve_address(addr) {
            Ok(raddr) => {
                // splices are exclusive on the upper range (half open)
                (&mut self.mem[raddr..(raddr+2)]).write_u16::<LittleEndian>(val).unwrap();
                Ok(())
            },
            Err(err) => Err(err)
        }
    }

    // https://en.wikibooks.org/wiki/NES_Programming/Memory_Map
    //
    fn resolve_address(self:&Memory,addr: u16) -> Result<usize,MemoryError> {
        if addr <= 0x1FFF { return Ok( (addr & 0x7FF) as usize); }
        if addr <= 0x2007 { return Err(MemoryError::PPUAccessViolation("PPU Memory access is currently not implemented".to_string())); }
        if addr <= 0x3FFF { return Ok( (addr & 0x7FF & 0x08) as usize); }
        if addr <= 0x401F { return Err(MemoryError::APUAccessViolation("Memory access is currently not implemented".to_string())); }
        Ok(addr as usize)
    }
}
<|endoftext|>


<commit_before>use super::*;

use crate::ffi::OsStr;
use crate::mem;
use crate::ptr;
use crate::sys::cvt;

macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(t) => t,
            Err(e) => panic!("received error for `{}`: {}", stringify!($e), e),
        }
    };
}

// See #14232 for more information, but it appears that signal delivery to a
// newly spawned process may just be raced in the macOS, so to prevent this
// test from being flaky we ignore it on macOS.
#[test]
#[cfg_attr(target_os = "macos", ignore)]
// When run under our current QEMU emulation test suite this test fails,
// although the reason isn't very clear as to why. For now this test is
// ignored there.
#[cfg_attr(target_arch = "arm", ignore)]
#[cfg_attr(target_arch = "aarch64", ignore)]
#[cfg_attr(target_arch = "riscv64", ignore)]
fn test_process_mask() {
    unsafe {
        // Test to make sure that a signal mask does not get inherited.
        let mut cmd = Command::new(OsStr::new("cat"));

        let mut set = mem::MaybeUninit::<libc::sigset_t>::uninit();
        let mut old_set = mem::MaybeUninit::<libc::sigset_t>::uninit();
        t!(cvt(sigemptyset(set.as_mut_ptr())));
        t!(cvt(sigaddset(set.as_mut_ptr(), libc::SIGINT)));
        t!(cvt(libc::pthread_sigmask(libc::SIG_SETMASK, set.as_ptr(), old_set.as_mut_ptr())));

        cmd.stdin(Stdio::MakePipe);
        cmd.stdout(Stdio::MakePipe);

        let (mut cat, mut pipes) = t!(cmd.spawn(Stdio::Null, true));
        let stdin_write = pipes.stdin.take().unwrap();
        let stdout_read = pipes.stdout.take().unwrap();

        t!(cvt(libc::pthread_sigmask(libc::SIG_SETMASK, old_set.as_ptr(), ptr::null_mut())));

        t!(cvt(libc::kill(cat.id() as libc::pid_t, libc::SIGINT)));
        // We need to wait until SIGINT is definitely delivered. The
        // easiest way is to write something to cat, and try to read it
        // back: if SIGINT is unmasked, it'll get delivered when cat is
        // next scheduled.
        let _ = stdin_write.write(b"Hello");
        drop(stdin_write);

        // Either EOF or failure (EPIPE) is okay.
        let mut buf = [0; 5];
        if let Ok(ret) = stdout_read.read(&mut buf) {
            assert_eq!(ret, 0);
        }

        t!(cat.wait());
    }
}
<commit_msg>Avoid overlapping cfg attributes when both macOS and aarch64<commit_after>use super::*;

use crate::ffi::OsStr;
use crate::mem;
use crate::ptr;
use crate::sys::cvt;

macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(t) => t,
            Err(e) => panic!("received error for `{}`: {}", stringify!($e), e),
        }
    };
}

#[test]
#[cfg_attr(
    any(
        // See #14232 for more information, but it appears that signal delivery to a
        // newly spawned process may just be raced in the macOS, so to prevent this
        // test from being flaky we ignore it on macOS.
        target_os = "macos",
        // When run under our current QEMU emulation test suite this test fails,
        // although the reason isn't very clear as to why. For now this test is
        // ignored there.
        target_arch = "arm",
        target_arch = "aarch64",
        target_arch = "riscv64",
    ),
    ignore
)]
fn test_process_mask() {
    unsafe {
        // Test to make sure that a signal mask does not get inherited.
        let mut cmd = Command::new(OsStr::new("cat"));

        let mut set = mem::MaybeUninit::<libc::sigset_t>::uninit();
        let mut old_set = mem::MaybeUninit::<libc::sigset_t>::uninit();
        t!(cvt(sigemptyset(set.as_mut_ptr())));
        t!(cvt(sigaddset(set.as_mut_ptr(), libc::SIGINT)));
        t!(cvt(libc::pthread_sigmask(libc::SIG_SETMASK, set.as_ptr(), old_set.as_mut_ptr())));

        cmd.stdin(Stdio::MakePipe);
        cmd.stdout(Stdio::MakePipe);

        let (mut cat, mut pipes) = t!(cmd.spawn(Stdio::Null, true));
        let stdin_write = pipes.stdin.take().unwrap();
        let stdout_read = pipes.stdout.take().unwrap();

        t!(cvt(libc::pthread_sigmask(libc::SIG_SETMASK, old_set.as_ptr(), ptr::null_mut())));

        t!(cvt(libc::kill(cat.id() as libc::pid_t, libc::SIGINT)));
        // We need to wait until SIGINT is definitely delivered. The
        // easiest way is to write something to cat, and try to read it
        // back: if SIGINT is unmasked, it'll get delivered when cat is
        // next scheduled.
        let _ = stdin_write.write(b"Hello");
        drop(stdin_write);

        // Either EOF or failure (EPIPE) is okay.
        let mut buf = [0; 5];
        if let Ok(ret) = stdout_read.read(&mut buf) {
            assert_eq!(ret, 0);
        }

        t!(cat.wait());
    }
}
<|endoftext|>