<commit_before>// Copyright 2017 Serde Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! This crate provides a convenient concise way to write unit tests for
//! implementations of [`Serialize`] and [`Deserialize`].
//!
//! [`Serialize`]: https://docs.serde.rs/serde/ser/trait.Serialize.html
//! [`Deserialize`]: https://docs.serde.rs/serde/de/trait.Deserialize.html
//!
//! The `Serialize` impl for a value can be characterized by the sequence of
//! [`Serializer`] calls that are made in the course of serializing the value,
//! so `serde_test` provides a [`Token`] abstraction which corresponds roughly
//! to `Serializer` method calls. There is an [`assert_ser_tokens`] function to
//! test that a value serializes to a particular sequence of method calls, an
//! [`assert_de_tokens`] function to test that a value can be deserialized from
//! a particular sequence of method calls, and an [`assert_tokens`] function to
//! test both directions. There are also functions to test expected failure
//! conditions.
//!
//! [`Serializer`]: https://docs.serde.rs/serde/ser/trait.Serializer.html
//! [`Token`]: https://docs.serde.rs/serde_test/enum.Token.html
//! [`assert_ser_tokens`]: https://docs.serde.rs/serde_test/fn.assert_ser_tokens.html
//! [`assert_de_tokens`]: https://docs.serde.rs/serde_test/fn.assert_de_tokens.html
//! [`assert_tokens`]: https://docs.serde.rs/serde_test/fn.assert_tokens.html
//!
//! Here is an example from the [`linked-hash-map`] crate.
//!
//! [`linked-hash-map`]: https://github.com/contain-rs/linked-hash-map
//!
//! ```rust
//! # extern crate serde;
//! #
//! # macro_rules! ignore {
//! #     ($($tt:tt)+) => {}
//! # }
//! #
//! # ignore! {
//! extern crate linked_hash_map;
//! use linked_hash_map::LinkedHashMap;
//! # }
//!
//! extern crate serde_test;
//! use serde_test::{Token, assert_tokens};
//!
//! # use std::fmt;
//! # use std::marker::PhantomData;
//! #
//! # use serde::ser::{Serialize, Serializer, SerializeMap};
//! # use serde::de::{Deserialize, Deserializer, Visitor, MapAccess};
//! #
//! # // Dumb immitation of LinkedHashMap.
//! # #[derive(PartialEq, Debug)]
//! # struct LinkedHashMap<K, V>(Vec<(K, V)>);
//! #
//! # impl<K, V> LinkedHashMap<K, V> {
//! #     fn new() -> Self {
//! #         LinkedHashMap(Vec::new())
//! #     }
//! #
//! #     fn insert(&mut self, k: K, v: V) {
//! #         self.0.push((k, v));
//! #     }
//! # }
//! #
//! # impl<K, V> Serialize for LinkedHashMap<K, V>
//! #     where K: Serialize,
//! #           V: Serialize
//! # {
//! #     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
//! #         where S: Serializer
//! #     {
//! #         let mut map = serializer.serialize_map(Some(self.0.len()))?;
//! #         for &(ref k, ref v) in &self.0 {
//! #             map.serialize_entry(k, v)?;
//! #         }
//! #         map.end()
//! #     }
//! # }
//! #
//! # struct LinkedHashMapVisitor<K, V>(PhantomData<(K, V)>);
//! #
//! # impl<'de, K, V> Visitor<'de> for LinkedHashMapVisitor<K, V>
//! #     where K: Deserialize<'de>,
//! #           V: Deserialize<'de>
//! # {
//! #     type Value = LinkedHashMap<K, V>;
//! #
//! #     fn expecting(&self, _: &mut fmt::Formatter) -> fmt::Result {
//! #         unimplemented!()
//! #     }
//! #
//! #     fn visit_map<M>(self, mut access: M) -> Result<Self::Value, M::Error>
//! #         where M: MapAccess<'de>
//! #     {
//! #         let mut map = LinkedHashMap::new();
//! #         while let Some((key, value)) = access.next_entry()? {
//! #             map.insert(key, value);
//! #         }
//! #         Ok(map)
//! #     }
//! # }
//! #
//! # impl<'de, K, V> Deserialize<'de> for LinkedHashMap<K, V>
//! #     where K: Deserialize<'de>,
//! #           V: Deserialize<'de>
//! # {
//! #     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
//! #         where D: Deserializer<'de>
//! #     {
//! #         deserializer.deserialize_map(LinkedHashMapVisitor(PhantomData))
//! #     }
//! # }
//! #
//! #[test]
//! # fn not_a_test_ser_de_empty() {}
//! fn test_ser_de_empty() {
//!     let map = LinkedHashMap::<char, u32>::new();
//!
//!     assert_tokens(&map, &[
//!         Token::Map { len: Some(0) },
//!         Token::MapEnd,
//!     ]);
//! }
//!
//! #[test]
//! # fn not_a_test_ser_de() {}
//! fn test_ser_de() {
//!     let mut map = LinkedHashMap::new();
//!     map.insert('b', 20);
//!     map.insert('a', 10);
//!     map.insert('c', 30);
//!
//!     assert_tokens(&map, &[
//!         Token::Map { len: Some(3) },
//!         Token::Char('b'),
//!         Token::I32(20),
//!
//!         Token::Char('a'),
//!         Token::I32(10),
//!
//!         Token::Char('c'),
//!         Token::I32(30),
//!         Token::MapEnd,
//!     ]);
//! }
//! #
//! # fn main() {
//! #     test_ser_de_empty();
//! #     test_ser_de();
//! # }
//! ```

#![doc(html_root_url = "https://docs.rs/serde_test/1.0.17")]

#[macro_use]
extern crate serde;

mod ser;
mod de;
mod error;

mod configure;
mod token;
mod assert;

pub use token::Token;
pub use assert::{assert_tokens, assert_ser_tokens, assert_ser_tokens_error,
                 assert_de_tokens, assert_de_tokens_error};

// Not public API.
#[doc(hidden)]
pub use assert::{assert_tokens_readable, assert_de_tokens_readable, assert_ser_tokens_readable};

// Not public API.
#[doc(hidden)]
pub use de::Deserializer;
<commit_msg>Export configure api<commit_after>// Copyright 2017 Serde Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! This crate provides a convenient concise way to write unit tests for
//! implementations of [`Serialize`] and [`Deserialize`].
//!
//! [`Serialize`]: https://docs.serde.rs/serde/ser/trait.Serialize.html
//! [`Deserialize`]: https://docs.serde.rs/serde/de/trait.Deserialize.html
//!
//! The `Serialize` impl for a value can be characterized by the sequence of
//! [`Serializer`] calls that are made in the course of serializing the value,
//! so `serde_test` provides a [`Token`] abstraction which corresponds roughly
//! to `Serializer` method calls. There is an [`assert_ser_tokens`] function to
//! test that a value serializes to a particular sequence of method calls, an
//! [`assert_de_tokens`] function to test that a value can be deserialized from
//! a particular sequence of method calls, and an [`assert_tokens`] function to
//! test both directions. There are also functions to test expected failure
//! conditions.
//!
//! [`Serializer`]: https://docs.serde.rs/serde/ser/trait.Serializer.html
//! [`Token`]: https://docs.serde.rs/serde_test/enum.Token.html
//! [`assert_ser_tokens`]: https://docs.serde.rs/serde_test/fn.assert_ser_tokens.html
//! [`assert_de_tokens`]: https://docs.serde.rs/serde_test/fn.assert_de_tokens.html
//! [`assert_tokens`]: https://docs.serde.rs/serde_test/fn.assert_tokens.html
//!
//! Here is an example from the [`linked-hash-map`] crate.
//!
//! [`linked-hash-map`]: https://github.com/contain-rs/linked-hash-map
//!
//! ```rust
//! # extern crate serde;
//! #
//! # macro_rules! ignore {
//! #     ($($tt:tt)+) => {}
//! # }
//! #
//! # ignore! {
//! extern crate linked_hash_map;
//! use linked_hash_map::LinkedHashMap;
//! # }
//!
//! extern crate serde_test;
//! use serde_test::{Token, assert_tokens};
//!
//! # use std::fmt;
//! # use std::marker::PhantomData;
//! #
//! # use serde::ser::{Serialize, Serializer, SerializeMap};
//! # use serde::de::{Deserialize, Deserializer, Visitor, MapAccess};
//! #
//! # // Dumb immitation of LinkedHashMap.
//! # #[derive(PartialEq, Debug)]
//! # struct LinkedHashMap<K, V>(Vec<(K, V)>);
//! #
//! # impl<K, V> LinkedHashMap<K, V> {
//! #     fn new() -> Self {
//! #         LinkedHashMap(Vec::new())
//! #     }
//! #
//! #     fn insert(&mut self, k: K, v: V) {
//! #         self.0.push((k, v));
//! #     }
//! # }
//! #
//! # impl<K, V> Serialize for LinkedHashMap<K, V>
//! #     where K: Serialize,
//! #           V: Serialize
//! # {
//! #     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
//! #         where S: Serializer
//! #     {
//! #         let mut map = serializer.serialize_map(Some(self.0.len()))?;
//! #         for &(ref k, ref v) in &self.0 {
//! #             map.serialize_entry(k, v)?;
//! #         }
//! #         map.end()
//! #     }
//! # }
//! #
//! # struct LinkedHashMapVisitor<K, V>(PhantomData<(K, V)>);
//! #
//! # impl<'de, K, V> Visitor<'de> for LinkedHashMapVisitor<K, V>
//! #     where K: Deserialize<'de>,
//! #           V: Deserialize<'de>
//! # {
//! #     type Value = LinkedHashMap<K, V>;
//! #
//! #     fn expecting(&self, _: &mut fmt::Formatter) -> fmt::Result {
//! #         unimplemented!()
//! #     }
//! #
//! #     fn visit_map<M>(self, mut access: M) -> Result<Self::Value, M::Error>
//! #         where M: MapAccess<'de>
//! #     {
//! #         let mut map = LinkedHashMap::new();
//! #         while let Some((key, value)) = access.next_entry()? {
//! #             map.insert(key, value);
//! #         }
//! #         Ok(map)
//! #     }
//! # }
//! #
//! # impl<'de, K, V> Deserialize<'de> for LinkedHashMap<K, V>
//! #     where K: Deserialize<'de>,
//! #           V: Deserialize<'de>
//! # {
//! #     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
//! #         where D: Deserializer<'de>
//! #     {
//! #         deserializer.deserialize_map(LinkedHashMapVisitor(PhantomData))
//! #     }
//! # }
//! #
//! #[test]
//! # fn not_a_test_ser_de_empty() {}
//! fn test_ser_de_empty() {
//!     let map = LinkedHashMap::<char, u32>::new();
//!
//!     assert_tokens(&map, &[
//!         Token::Map { len: Some(0) },
//!         Token::MapEnd,
//!     ]);
//! }
//!
//! #[test]
//! # fn not_a_test_ser_de() {}
//! fn test_ser_de() {
//!     let mut map = LinkedHashMap::new();
//!     map.insert('b', 20);
//!     map.insert('a', 10);
//!     map.insert('c', 30);
//!
//!     assert_tokens(&map, &[
//!         Token::Map { len: Some(3) },
//!         Token::Char('b'),
//!         Token::I32(20),
//!
//!         Token::Char('a'),
//!         Token::I32(10),
//!
//!         Token::Char('c'),
//!         Token::I32(30),
//!         Token::MapEnd,
//!     ]);
//! }
//! #
//! # fn main() {
//! #     test_ser_de_empty();
//! #     test_ser_de();
//! # }
//! ```

#![doc(html_root_url = "https://docs.rs/serde_test/1.0.17")]

#[macro_use]
extern crate serde;

mod ser;
mod de;
mod error;

mod configure;
mod token;
mod assert;

pub use token::Token;
pub use assert::{assert_tokens, assert_ser_tokens, assert_ser_tokens_error,
                 assert_de_tokens, assert_de_tokens_error};

pub use configure::{Configure, Compact, Readable};

// Not public API.
#[doc(hidden)]
pub use assert::{assert_tokens_readable, assert_de_tokens_readable, assert_ser_tokens_readable};

// Not public API.
#[doc(hidden)]
pub use de::Deserializer;
<|endoftext|>


<commit_before>//! This file contains the string defaults values as well as environment variable strings
//! for use in the clap layer of the application. This is not the final form for defaults.
//! Eventually this will be composed of fully typed default values. But as a first step we
//! need a spot to consolidate those values and help simplify some of the logic around them.

use clap::ArgMatches;

use habitat_core::{env as henv,
                   fs::{cache_key_path,
                        CACHE_KEY_PATH},
                   os::process::{ShutdownSignal,
                                 ShutdownTimeout},
                   package::PackageIdent};
use std::{path::PathBuf,
          str::FromStr};

pub const RING_ENVVAR: &str = "HAB_RING";
pub const RING_KEY_ENVVAR: &str = "HAB_RING_KEY";

pub const LISTEN_HTTP_DEFAULT_PORT: u16 = 9631;
pub const LISTEN_HTTP_DEFAULT_IP: &str = "0.0.0.0";
lazy_static! {
    pub static ref LISTEN_HTTP_DEFAULT_ADDR: String =
        { format!("{}:{}", LISTEN_HTTP_DEFAULT_IP, LISTEN_HTTP_DEFAULT_PORT) };
}

pub const PACKAGE_TARGET_ENVVAR: &str = "HAB_PACKAGE_TARGET";
lazy_static! {
    pub static ref SHUTDOWN_TIMEOUT_DEFAULT: String = ShutdownTimeout::default().to_string();
}

// We allow this on Windows as well as Unix, even though we don't use
// ShutdownSignal on Windows, because we want to allow Windows CLI
// users to interact with Unix Supervisors.
lazy_static! {
    pub static ref SHUTDOWN_SIGNAL_DEFAULT: String = ShutdownSignal::default().to_string();
}

const SYSTEMDRIVE_ENVVAR: &str = "SYSTEMDRIVE";

lazy_static! {
    /// The default filesystem root path to base all commands from. This is lazily generated on
    /// first call and reflects on the presence and value of the environment variable keyed as
    /// `FS_ROOT_ENVVAR`.
    pub static ref FS_ROOT: PathBuf = {
        use crate::hcore::fs::FS_ROOT_ENVVAR;

        if cfg!(target_os = "windows") {
            match (henv::var(FS_ROOT_ENVVAR), henv::var(SYSTEMDRIVE_ENVVAR)) {
                (Ok(path), _) => PathBuf::from(path),
                (Err(_), Ok(system_drive)) => PathBuf::from(format!("{}{}", system_drive, "\\")),
                (Err(_), Err(_)) => unreachable!(
                    "Windows should always have a SYSTEMDRIVE \
                    environment variable."
                ),
            }
        } else if let Ok(root) = henv::var(FS_ROOT_ENVVAR) {
            PathBuf::from(root)
        } else {
            PathBuf::from("/")
        }
    };
}

pub const BINLINK_DIR_ENVVAR: &str = "HAB_BINLINK_DIR";

/// Default Binlink Dir
#[cfg(target_os = "windows")]
pub const DEFAULT_BINLINK_DIR: &str = "/hab/bin";
#[cfg(target_os = "linux")]
pub const DEFAULT_BINLINK_DIR: &str = "/bin";
#[cfg(target_os = "macos")]
pub const DEFAULT_BINLINK_DIR: &str = "/usr/local/bin";

/// We require the value at the clap layer (see cli::arg_cache_key_path),
/// so we can safely unwrap, but we need some additional logic to calculate
// the dynamic "default" value if the argument has the default signifier value:
/// CACHE_KEY_PATH. An empty value can't stand for default since it is invalid.
pub fn cache_key_path_from_matches(matches: &ArgMatches<'_>) -> PathBuf {
    match matches.value_of("CACHE_KEY_PATH")
                 .expect("CACHE_KEY_PATH required")
    {
        CACHE_KEY_PATH => cache_key_path(Some(&*FS_ROOT)),
        val => PathBuf::from(val),
    }
}

pub fn file_into_idents(path: &str) -> Result<Vec<PackageIdent>, habitat_core::error::Error> {
    let s = std::fs::read_to_string(&path).map_err(|_| {
                habitat_core::error::Error::FileNotFound(format!("Could not open file {}", path))
            })?;

    s.lines()
     .map(|line| PackageIdent::from_str(line.trim()))
     .collect()
}
<commit_msg>hab pkg download file can have comments, whitespace<commit_after>//! This file contains the string defaults values as well as environment variable strings
//! for use in the clap layer of the application. This is not the final form for defaults.
//! Eventually this will be composed of fully typed default values. But as a first step we
//! need a spot to consolidate those values and help simplify some of the logic around them.

use clap::ArgMatches;

use habitat_core::{env as henv,
                   fs::{cache_key_path,
                        CACHE_KEY_PATH},
                   os::process::{ShutdownSignal,
                                 ShutdownTimeout},
                   package::PackageIdent};
use std::{path::PathBuf,
          str::FromStr};

pub const RING_ENVVAR: &str = "HAB_RING";
pub const RING_KEY_ENVVAR: &str = "HAB_RING_KEY";

pub const LISTEN_HTTP_DEFAULT_PORT: u16 = 9631;
pub const LISTEN_HTTP_DEFAULT_IP: &str = "0.0.0.0";
lazy_static! {
    pub static ref LISTEN_HTTP_DEFAULT_ADDR: String =
        { format!("{}:{}", LISTEN_HTTP_DEFAULT_IP, LISTEN_HTTP_DEFAULT_PORT) };
}

pub const PACKAGE_TARGET_ENVVAR: &str = "HAB_PACKAGE_TARGET";
lazy_static! {
    pub static ref SHUTDOWN_TIMEOUT_DEFAULT: String = ShutdownTimeout::default().to_string();
}

// We allow this on Windows as well as Unix, even though we don't use
// ShutdownSignal on Windows, because we want to allow Windows CLI
// users to interact with Unix Supervisors.
lazy_static! {
    pub static ref SHUTDOWN_SIGNAL_DEFAULT: String = ShutdownSignal::default().to_string();
}

const SYSTEMDRIVE_ENVVAR: &str = "SYSTEMDRIVE";

lazy_static! {
    /// The default filesystem root path to base all commands from. This is lazily generated on
    /// first call and reflects on the presence and value of the environment variable keyed as
    /// `FS_ROOT_ENVVAR`.
    pub static ref FS_ROOT: PathBuf = {
        use crate::hcore::fs::FS_ROOT_ENVVAR;

        if cfg!(target_os = "windows") {
            match (henv::var(FS_ROOT_ENVVAR), henv::var(SYSTEMDRIVE_ENVVAR)) {
                (Ok(path), _) => PathBuf::from(path),
                (Err(_), Ok(system_drive)) => PathBuf::from(format!("{}{}", system_drive, "\\")),
                (Err(_), Err(_)) => unreachable!(
                    "Windows should always have a SYSTEMDRIVE \
                    environment variable."
                ),
            }
        } else if let Ok(root) = henv::var(FS_ROOT_ENVVAR) {
            PathBuf::from(root)
        } else {
            PathBuf::from("/")
        }
    };
}

pub const BINLINK_DIR_ENVVAR: &str = "HAB_BINLINK_DIR";

/// Default Binlink Dir
#[cfg(target_os = "windows")]
pub const DEFAULT_BINLINK_DIR: &str = "/hab/bin";
#[cfg(target_os = "linux")]
pub const DEFAULT_BINLINK_DIR: &str = "/bin";
#[cfg(target_os = "macos")]
pub const DEFAULT_BINLINK_DIR: &str = "/usr/local/bin";

/// We require the value at the clap layer (see cli::arg_cache_key_path),
/// so we can safely unwrap, but we need some additional logic to calculate
// the dynamic "default" value if the argument has the default signifier value:
/// CACHE_KEY_PATH. An empty value can't stand for default since it is invalid.
pub fn cache_key_path_from_matches(matches: &ArgMatches<'_>) -> PathBuf {
    match matches.value_of("CACHE_KEY_PATH")
                 .expect("CACHE_KEY_PATH required")
    {
        CACHE_KEY_PATH => cache_key_path(Some(&*FS_ROOT)),
        val => PathBuf::from(val),
    }
}

pub fn file_into_idents(path: &str) -> Result<Vec<PackageIdent>, habitat_core::error::Error> {
    let s = std::fs::read_to_string(&path).map_err(|_| {
                habitat_core::error::Error::FileNotFound(format!("Could not open file {}", path))
            })?;

    s.lines().filter_map(line_to_ident).collect()
}

fn line_to_ident(line: &str) -> Option<Result<PackageIdent, habitat_core::error::Error>> {
    let trimmed = line.split('#').nth(0).unwrap_or("").trim();
    match trimmed.len() {
        0 => None,
        _ => Some(PackageIdent::from_str(trimmed)),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    // These tests aren't super pretty because we can't do PartialEq for the Option<Error<X>> stuff,
    // so lots of unwrap
    fn test_line_to_ident() {
        assert!(line_to_ident("").is_none());
        assert!(line_to_ident("# foo").is_none());
        assert!(line_to_ident("   # foo").is_none());
        assert!(line_to_ident("\n\r# foo").is_none());

        assert_eq!(line_to_ident("core/gzip").unwrap().unwrap(),
                   PackageIdent::from_str("core/gzip").unwrap());
        assert_eq!(line_to_ident("core/gzip #foo").unwrap().unwrap(),
                   PackageIdent::from_str("core/gzip").unwrap());

        assert!(line_to_ident("core").unwrap().is_err());

        assert!(line_to_ident("core # not").unwrap().is_err());
    }
}
<|endoftext|>


<commit_before>use alloc::arc::Arc;
use alloc::boxed::Box;

use core::{cmp, mem, ptr};
use core::sync::atomic::{AtomicBool, Ordering};

use drivers::disk::*;
use drivers::pio::*;
use drivers::pciconfig::PCIConfig;

use common::debug;
use common::queue::Queue;
use common::memory::{self, Memory};
use common::resource::{NoneResource, Resource, ResourceSeek, ResourceType, URL, VecResource};
use common::string::{String, ToString};
use common::vec::Vec;

use programs::session::SessionItem;

use syscall::call::sys_yield;

/// The header of the fs
#[repr(packed)]
pub struct Header {
    pub signature: [u8; 8],
    pub version: u32,
    pub name: [u8; 244],
    pub extents: [Extent; 16],
}

/// Data for a node
#[repr(packed)]
pub struct NodeData {
    pub name: [u8; 256],
    pub extents: [Extent; 16],
}

/// A file node
pub struct Node {
    pub address: u64,
    pub name: String,
    pub extents: [Extent; 16],
}

impl Node {
    /// Create a new file node from an address and some data
    pub fn new(address: u64, data: &NodeData) -> Self {
        let mut utf8: Vec<u8> = Vec::new();
        for i in 0..data.name.len() {
            let c = data.name[i];
            if c == 0 {
                break;
            } else {
                utf8.push(c);
            }
        }

        Node {
            address: address,
            name: String::from_utf8(&utf8),
            extents: data.extents,
        }
    }
}

impl Clone for Node {
    fn clone(&self) -> Self {
        Node {
            address: self.address,
            name: self.name.clone(),
            extents: self.extents,
        }
    }
}

/// A file system
pub struct FileSystem {
    pub disk: Disk,
    pub header: Header,
    pub nodes: Vec<Node>,
}

impl FileSystem {
    /// Create a file system from a disk
    pub fn from_disk(mut disk: Disk) -> Option<Self> {
        unsafe {
            if disk.identify() {
                debug::d(" Disk Found");

                let header_ptr = Memory::<Header>::new(1).unwrap();
                disk.read(1, 1, header_ptr.address());
                let header = header_ptr.read(0);
                drop(header_ptr);

                if header.signature[0] == 'R' as u8 && header.signature[1] == 'E' as u8 &&
                   header.signature[2] == 'D' as u8 &&
                   header.signature[3] == 'O' as u8 &&
                   header.signature[4] == 'X' as u8 &&
                   header.signature[5] == 'F' as u8 &&
                   header.signature[6] == 'S' as u8 &&
                   header.signature[7] == '\0' as u8 &&
                   header.version == 0xFFFFFFFF {

                    debug::d(" Redox Filesystem\n");

                    let mut nodes = Vec::new();
                    for extent in &header.extents {
                        if extent.block > 0 && extent.length > 0 {
                            if let Some(mut data) = Memory::<NodeData>::new(extent.length as usize /
                                                           mem::size_of::<NodeData>()) {
                                let sectors = (extent.length as usize + 511) / 512;
                                let mut sector: usize = 0;
                                while sectors - sector >= 65536 {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: 65536 * 512,
                                        },
                                        mem: data.address() + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    unsafe {
                                        disk.read(extent.block + sector as u64,
                                                  0,
                                                  data.address() + sector * 512);
                                    }

                                        /*
                                        disk.request(request.clone());

                                        while request.complete.load(Ordering::SeqCst) == false {
                                            disk.on_poll();
                                        }
                                        */

                                    sector += 65535;
                                }
                                if sector < sectors {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: (sectors - sector) as u64 * 512,
                                        },
                                        mem: data.address() + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    unsafe {
                                        disk.read(extent.block + sector as u64,
                                                  (sectors - sector) as u16,
                                                  data.address() + sector * 512);
                                    }
                                        /*
                                        disk.request(request.clone());

                                        while request.complete.load(Ordering::SeqCst) == false {
                                            disk.on_poll();
                                        }
                                        */
                                }

                                for i in 0..extent.length as usize / mem::size_of::<NodeData>() {
                                    nodes.push(Node::new(extent.block + i as u64, &data[i]));
                                }
                            }
                        }
                    }

                    return Some(FileSystem {
                        disk: disk,
                        header: header,
                        nodes: nodes,
                    });
                } else {
                    debug::d(" Unknown Filesystem\n");
                }
            } else {
                debug::d(" Disk Not Found\n");
            }
        }

        Option::None
    }

    /// Get node with a given filename
    pub fn node(&self, filename: &String) -> Option<Node> {
        for node in self.nodes.iter() {
            if node.name == *filename {
                return Option::Some(node.clone());
            }
        }

        Option::None
    }

    /// List nodes in a given directory
    pub fn list(&self, directory: &String) -> Vec<String> {
        let mut ret = Vec::<String>::new();

        for node in self.nodes.iter() {
            if node.name.starts_with(directory.clone()) {
                ret.push(node.name.substr(directory.len(), node.name.len() - directory.len()));
            }
        }

        ret
    }
}

/// A file resource
pub struct FileResource {
    pub scheme: *mut FileScheme,
    pub node: Node,
    pub vec: Vec<u8>,
    pub seek: usize,
    pub dirty: bool,
}

impl Resource for FileResource {
    fn url(&self) -> URL {
        URL::from_string(&("file:///".to_string() + &self.node.name))
    }

    fn stat(&self) -> ResourceType {
        ResourceType::File
    }

    fn read(&mut self, buf: &mut [u8]) -> Option<usize> {
        let mut i = 0;
        while i < buf.len() && self.seek < self.vec.len() {
            match self.vec.get(self.seek) {
                Option::Some(b) => buf[i] = *b,
                Option::None => (),
            }
            self.seek += 1;
            i += 1;
        }
        Option::Some(i)
    }

    fn write(&mut self, buf: &[u8]) -> Option<usize> {
        let mut i = 0;
        while i < buf.len() && self.seek < self.vec.len() {
            self.vec.set(self.seek, buf[i]);
            self.seek += 1;
            i += 1;
        }
        while i < buf.len() {
            self.vec.push(buf[i]);
            self.seek += 1;
            i += 1;
        }
        if i > 0 {
            self.dirty = true;
        }
        Option::Some(i)
    }

    fn seek(&mut self, pos: ResourceSeek) -> Option<usize> {
        match pos {
            ResourceSeek::Start(offset) => self.seek = offset,
            ResourceSeek::Current(offset) =>
                self.seek = cmp::max(0, self.seek as isize + offset) as usize,
            ResourceSeek::End(offset) =>
                self.seek = cmp::max(0, self.vec.len() as isize + offset) as usize,
        }
        while self.vec.len() < self.seek {
            self.vec.push(0);
        }
        Option::Some(self.seek)
    }

    // TODO: Rename to sync
    // TODO: Check to make sure proper amount of bytes written. See Disk::write
    // TODO: Allow reallocation
    fn sync(&mut self) -> bool {
        if self.dirty {
            let block_size: usize = 512;

            let mut node_dirty = false;
            let mut pos: isize = 0;
            let mut remaining = self.vec.len() as isize;
            for ref mut extent in &mut self.node.extents {
                //Make sure it is a valid extent
                if extent.block > 0 && extent.length > 0 {
                    let current_sectors = (extent.length as usize + block_size - 1) / block_size;
                    let max_size = current_sectors * 512;

                    let size = cmp::min(remaining as usize, max_size);
                    let sectors = (size + block_size - 1) / block_size;

                    if size as u64 != extent.length {
                        extent.length = size as u64;
                        node_dirty = true;
                    }

                    unsafe {
                        let data = self.vec.as_ptr().offset(pos) as usize;
                        //TODO: Make sure data is copied safely into an zeroed area of the right size!

                        let sectors = (extent.length as usize + 511) / 512;
                        let mut sector: usize = 0;
                        while sectors - sector >= 65536 {
                            let request = Request {
                                extent: Extent {
                                    block: extent.block + sector as u64,
                                    length: 65536 * 512,
                                },
                                mem: data + sector * 512,
                                read: false,
                                complete: Arc::new(AtomicBool::new(false)),
                            };

                            (*self.scheme).fs.disk.request(request.clone());

                            while request.complete.load(Ordering::SeqCst) == false {
                                sys_yield();
                            }

                            sector += 65535;
                        }
                        if sector < sectors {
                            let request = Request {
                                extent: Extent {
                                    block: extent.block + sector as u64,
                                    length: (sectors - sector) as u64 * 512,
                                },
                                mem: data + sector * 512,
                                read: false,
                                complete: Arc::new(AtomicBool::new(false)),
                            };

                            (*self.scheme).fs.disk.request(request.clone());

                            while request.complete.load(Ordering::SeqCst) == false {
                                sys_yield();
                            }
                        }
                    }

                    pos += size as isize;
                    remaining -= size as isize;
                }
            }

            if node_dirty {
                debug::d("Node dirty, should rewrite\n");
            }

            self.dirty = false;

            if remaining > 0 {
                debug::d("Need to reallocate file, extra: ");
                debug::ds(remaining);
                debug::dl();
                return false;
            }
        }
        true
    }
}

impl Drop for FileResource {
    fn drop(&mut self) {
        self.sync();
    }
}

/// A file scheme (pci + fs)
pub struct FileScheme {
    pci: PCIConfig,
    fs: FileSystem,
}

impl FileScheme {
    ///TODO Allow busmaster for secondary
    /// Create a new file scheme from a PCI configuration
    pub fn new(mut pci: PCIConfig) -> Option<Box<Self>> {
        unsafe { pci.flag(4, 4, true) }; // Bus mastering

        let base = unsafe { pci.read(0x20) } as u16 & 0xFFF0;

        debug::d("IDE on ");
        debug::dh(base as usize);
        debug::dl();

        debug::d("Primary Master:");
        if let Some(fs) = FileSystem::from_disk(Disk::primary_master(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        debug::d("Primary Slave:");
        if let Some(fs) = FileSystem::from_disk(Disk::primary_slave(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        debug::d("Secondary Master:");
        if let Some(fs) = FileSystem::from_disk(Disk::secondary_master(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        debug::d("Secondary Slave:");
        if let Some(fs) = FileSystem::from_disk(Disk::secondary_slave(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        None
    }
}

impl SessionItem for FileScheme {
    fn on_irq(&mut self, irq: u8) {
        if irq == self.fs.disk.irq {
            self.on_poll();
        }
    }

    fn on_poll(&mut self) {
        unsafe {
            self.fs.disk.on_poll();
        }
    }

    fn scheme(&self) -> String {
        return "file".to_string();
    }

    fn open(&mut self, url: &URL) -> Box<Resource> {
        let path = url.path();
        if path.len() == 0 || path.ends_with("/".to_string()) {
            let mut list = String::new();
            let mut dirs: Vec<String> = Vec::new();

            for file in self.fs.list(&path).iter() {
                let line;
                match file.find("/".to_string()) {
                    Option::Some(index) => {
                        let dirname = file.substr(0, index + 1);
                        let mut found = false;
                        for dir in dirs.iter() {
                            if dirname == *dir {
                                found = true;
                                break;
                            }
                        }
                        if found {
                            line = String::new();
                        } else {
                            line = dirname.clone();
                            dirs.push(dirname);
                        }
                    }
                    Option::None => line = file.clone(),
                }
                if line.len() > 0 {
                    if list.len() > 0 {
                        list = list + '\n' + line;
                    } else {
                        list = line;
                    }
                }
            }

            return box VecResource::new(url.clone(), ResourceType::Dir, list.to_utf8());
        } else {
            match self.fs.node(&path) {
                Option::Some(node) => {
                    let mut vec: Vec<u8> = Vec::new();
                    //TODO: Handle more extents
                    for extent in &node.extents {
                        if extent.block > 0 && extent.length > 0 {
                            if let Some(mut data) = Memory::<u8>::new(extent.length as usize) {
                                let sectors = (extent.length as usize + 511) / 512;
                                let mut sector: usize = 0;
                                while sectors - sector >= 65536 {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: 65536 * 512,
                                        },
                                        mem: unsafe { data.address() } + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    unsafe {
                                        self.fs.disk.request(request.clone());
                                    }

                                    while request.complete.load(Ordering::SeqCst) == false {
                                        sys_yield();
                                    }

                                    sector += 65535;
                                }
                                if sector < sectors {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: (sectors - sector) as u64 * 512,
                                        },
                                        mem: unsafe { data.address() } + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    unsafe {
                                        self.fs.disk.request(request.clone());
                                    }

                                    while request.complete.load(Ordering::SeqCst) == false {
                                        sys_yield();
                                    }
                                }

                                vec.push_all(&Vec {
                                    data: unsafe { data.into_raw() },
                                    length: extent.length as usize,
                                });
                            }
                        }
                    }

                    return box FileResource {
                        scheme: self,
                        node: node,
                        vec: vec,
                        seek: 0,
                        dirty: false,
                    };
                }
                Option::None => {
                    return box NoneResource;
                }
            }
        }
    }
}
<commit_msg>Remove unnessesary unsafe blocks<commit_after>use alloc::arc::Arc;
use alloc::boxed::Box;

use core::{cmp, mem, ptr};
use core::sync::atomic::{AtomicBool, Ordering};

use drivers::disk::*;
use drivers::pio::*;
use drivers::pciconfig::PCIConfig;

use common::debug;
use common::queue::Queue;
use common::memory::{self, Memory};
use common::resource::{NoneResource, Resource, ResourceSeek, ResourceType, URL, VecResource};
use common::string::{String, ToString};
use common::vec::Vec;

use programs::session::SessionItem;

use syscall::call::sys_yield;

/// The header of the fs
#[repr(packed)]
pub struct Header {
    pub signature: [u8; 8],
    pub version: u32,
    pub name: [u8; 244],
    pub extents: [Extent; 16],
}

/// Data for a node
#[repr(packed)]
pub struct NodeData {
    pub name: [u8; 256],
    pub extents: [Extent; 16],
}

/// A file node
pub struct Node {
    pub address: u64,
    pub name: String,
    pub extents: [Extent; 16],
}

impl Node {
    /// Create a new file node from an address and some data
    pub fn new(address: u64, data: &NodeData) -> Self {
        let mut utf8: Vec<u8> = Vec::new();
        for i in 0..data.name.len() {
            let c = data.name[i];
            if c == 0 {
                break;
            } else {
                utf8.push(c);
            }
        }

        Node {
            address: address,
            name: String::from_utf8(&utf8),
            extents: data.extents,
        }
    }
}

impl Clone for Node {
    fn clone(&self) -> Self {
        Node {
            address: self.address,
            name: self.name.clone(),
            extents: self.extents,
        }
    }
}

/// A file system
pub struct FileSystem {
    pub disk: Disk,
    pub header: Header,
    pub nodes: Vec<Node>,
}

impl FileSystem {
    /// Create a file system from a disk
    pub fn from_disk(mut disk: Disk) -> Option<Self> {
        unsafe {
            if disk.identify() {
                debug::d(" Disk Found");

                let header_ptr = Memory::<Header>::new(1).unwrap();
                disk.read(1, 1, header_ptr.address());
                let header = header_ptr.read(0);
                drop(header_ptr);

                if header.signature[0] == 'R' as u8 && header.signature[1] == 'E' as u8 &&
                   header.signature[2] == 'D' as u8 &&
                   header.signature[3] == 'O' as u8 &&
                   header.signature[4] == 'X' as u8 &&
                   header.signature[5] == 'F' as u8 &&
                   header.signature[6] == 'S' as u8 &&
                   header.signature[7] == '\0' as u8 &&
                   header.version == 0xFFFFFFFF {

                    debug::d(" Redox Filesystem\n");

                    let mut nodes = Vec::new();
                    for extent in &header.extents {
                        if extent.block > 0 && extent.length > 0 {
                            if let Some(mut data) = Memory::<NodeData>::new(extent.length as usize /
                                                           mem::size_of::<NodeData>()) {
                                let sectors = (extent.length as usize + 511) / 512;
                                let mut sector: usize = 0;
                                while sectors - sector >= 65536 {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: 65536 * 512,
                                        },
                                        mem: data.address() + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    disk.read(extent.block + sector as u64,
                                              0,
                                              data.address() + sector * 512);

                                        /*
                                        disk.request(request.clone());

                                        while request.complete.load(Ordering::SeqCst) == false {
                                            disk.on_poll();
                                        }
                                        */

                                    sector += 65535;
                                }
                                if sector < sectors {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: (sectors - sector) as u64 * 512,
                                        },
                                        mem: data.address() + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    unsafe {
                                        disk.read(extent.block + sector as u64,
                                                  (sectors - sector) as u16,
                                                  data.address() + sector * 512);
                                    }
                                        /*
                                        disk.request(request.clone());

                                        while request.complete.load(Ordering::SeqCst) == false {
                                            disk.on_poll();
                                        }
                                        */
                                }

                                for i in 0..extent.length as usize / mem::size_of::<NodeData>() {
                                    nodes.push(Node::new(extent.block + i as u64, &data[i]));
                                }
                            }
                        }
                    }

                    return Some(FileSystem {
                        disk: disk,
                        header: header,
                        nodes: nodes,
                    });
                } else {
                    debug::d(" Unknown Filesystem\n");
                }
            } else {
                debug::d(" Disk Not Found\n");
            }
        }

        Option::None
    }

    /// Get node with a given filename
    pub fn node(&self, filename: &String) -> Option<Node> {
        for node in self.nodes.iter() {
            if node.name == *filename {
                return Option::Some(node.clone());
            }
        }

        Option::None
    }

    /// List nodes in a given directory
    pub fn list(&self, directory: &String) -> Vec<String> {
        let mut ret = Vec::<String>::new();

        for node in self.nodes.iter() {
            if node.name.starts_with(directory.clone()) {
                ret.push(node.name.substr(directory.len(), node.name.len() - directory.len()));
            }
        }

        ret
    }
}

/// A file resource
pub struct FileResource {
    pub scheme: *mut FileScheme,
    pub node: Node,
    pub vec: Vec<u8>,
    pub seek: usize,
    pub dirty: bool,
}

impl Resource for FileResource {
    fn url(&self) -> URL {
        URL::from_string(&("file:///".to_string() + &self.node.name))
    }

    fn stat(&self) -> ResourceType {
        ResourceType::File
    }

    fn read(&mut self, buf: &mut [u8]) -> Option<usize> {
        let mut i = 0;
        while i < buf.len() && self.seek < self.vec.len() {
            match self.vec.get(self.seek) {
                Option::Some(b) => buf[i] = *b,
                Option::None => (),
            }
            self.seek += 1;
            i += 1;
        }
        Option::Some(i)
    }

    fn write(&mut self, buf: &[u8]) -> Option<usize> {
        let mut i = 0;
        while i < buf.len() && self.seek < self.vec.len() {
            self.vec.set(self.seek, buf[i]);
            self.seek += 1;
            i += 1;
        }
        while i < buf.len() {
            self.vec.push(buf[i]);
            self.seek += 1;
            i += 1;
        }
        if i > 0 {
            self.dirty = true;
        }
        Option::Some(i)
    }

    fn seek(&mut self, pos: ResourceSeek) -> Option<usize> {
        match pos {
            ResourceSeek::Start(offset) => self.seek = offset,
            ResourceSeek::Current(offset) =>
                self.seek = cmp::max(0, self.seek as isize + offset) as usize,
            ResourceSeek::End(offset) =>
                self.seek = cmp::max(0, self.vec.len() as isize + offset) as usize,
        }
        while self.vec.len() < self.seek {
            self.vec.push(0);
        }
        Option::Some(self.seek)
    }

    // TODO: Rename to sync
    // TODO: Check to make sure proper amount of bytes written. See Disk::write
    // TODO: Allow reallocation
    fn sync(&mut self) -> bool {
        if self.dirty {
            let block_size: usize = 512;

            let mut node_dirty = false;
            let mut pos: isize = 0;
            let mut remaining = self.vec.len() as isize;
            for ref mut extent in &mut self.node.extents {
                //Make sure it is a valid extent
                if extent.block > 0 && extent.length > 0 {
                    let current_sectors = (extent.length as usize + block_size - 1) / block_size;
                    let max_size = current_sectors * 512;

                    let size = cmp::min(remaining as usize, max_size);
                    let sectors = (size + block_size - 1) / block_size;

                    if size as u64 != extent.length {
                        extent.length = size as u64;
                        node_dirty = true;
                    }

                    unsafe {
                        let data = self.vec.as_ptr().offset(pos) as usize;
                        //TODO: Make sure data is copied safely into an zeroed area of the right size!

                        let sectors = (extent.length as usize + 511) / 512;
                        let mut sector: usize = 0;
                        while sectors - sector >= 65536 {
                            let request = Request {
                                extent: Extent {
                                    block: extent.block + sector as u64,
                                    length: 65536 * 512,
                                },
                                mem: data + sector * 512,
                                read: false,
                                complete: Arc::new(AtomicBool::new(false)),
                            };

                            (*self.scheme).fs.disk.request(request.clone());

                            while request.complete.load(Ordering::SeqCst) == false {
                                sys_yield();
                            }

                            sector += 65535;
                        }
                        if sector < sectors {
                            let request = Request {
                                extent: Extent {
                                    block: extent.block + sector as u64,
                                    length: (sectors - sector) as u64 * 512,
                                },
                                mem: data + sector * 512,
                                read: false,
                                complete: Arc::new(AtomicBool::new(false)),
                            };

                            (*self.scheme).fs.disk.request(request.clone());

                            while request.complete.load(Ordering::SeqCst) == false {
                                sys_yield();
                            }
                        }
                    }

                    pos += size as isize;
                    remaining -= size as isize;
                }
            }

            if node_dirty {
                debug::d("Node dirty, should rewrite\n");
            }

            self.dirty = false;

            if remaining > 0 {
                debug::d("Need to reallocate file, extra: ");
                debug::ds(remaining);
                debug::dl();
                return false;
            }
        }
        true
    }
}

impl Drop for FileResource {
    fn drop(&mut self) {
        self.sync();
    }
}

/// A file scheme (pci + fs)
pub struct FileScheme {
    pci: PCIConfig,
    fs: FileSystem,
}

impl FileScheme {
    ///TODO Allow busmaster for secondary
    /// Create a new file scheme from a PCI configuration
    pub fn new(mut pci: PCIConfig) -> Option<Box<Self>> {
        unsafe { pci.flag(4, 4, true) }; // Bus mastering

        let base = unsafe { pci.read(0x20) } as u16 & 0xFFF0;

        debug::d("IDE on ");
        debug::dh(base as usize);
        debug::dl();

        debug::d("Primary Master:");
        if let Some(fs) = FileSystem::from_disk(Disk::primary_master(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        debug::d("Primary Slave:");
        if let Some(fs) = FileSystem::from_disk(Disk::primary_slave(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        debug::d("Secondary Master:");
        if let Some(fs) = FileSystem::from_disk(Disk::secondary_master(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        debug::d("Secondary Slave:");
        if let Some(fs) = FileSystem::from_disk(Disk::secondary_slave(base)) {
            return Some(box FileScheme {
                pci: pci,
                fs: fs,
            });
        }

        None
    }
}

impl SessionItem for FileScheme {
    fn on_irq(&mut self, irq: u8) {
        if irq == self.fs.disk.irq {
            self.on_poll();
        }
    }

    fn on_poll(&mut self) {
        unsafe {
            self.fs.disk.on_poll();
        }
    }

    fn scheme(&self) -> String {
        return "file".to_string();
    }

    fn open(&mut self, url: &URL) -> Box<Resource> {
        let path = url.path();
        if path.len() == 0 || path.ends_with("/".to_string()) {
            let mut list = String::new();
            let mut dirs: Vec<String> = Vec::new();

            for file in self.fs.list(&path).iter() {
                let line;
                match file.find("/".to_string()) {
                    Option::Some(index) => {
                        let dirname = file.substr(0, index + 1);
                        let mut found = false;
                        for dir in dirs.iter() {
                            if dirname == *dir {
                                found = true;
                                break;
                            }
                        }
                        if found {
                            line = String::new();
                        } else {
                            line = dirname.clone();
                            dirs.push(dirname);
                        }
                    }
                    Option::None => line = file.clone(),
                }
                if line.len() > 0 {
                    if list.len() > 0 {
                        list = list + '\n' + line;
                    } else {
                        list = line;
                    }
                }
            }

            return box VecResource::new(url.clone(), ResourceType::Dir, list.to_utf8());
        } else {
            match self.fs.node(&path) {
                Option::Some(node) => {
                    let mut vec: Vec<u8> = Vec::new();
                    //TODO: Handle more extents
                    for extent in &node.extents {
                        if extent.block > 0 && extent.length > 0 {
                            if let Some(mut data) = Memory::<u8>::new(extent.length as usize) {
                                let sectors = (extent.length as usize + 511) / 512;
                                let mut sector: usize = 0;
                                while sectors - sector >= 65536 {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: 65536 * 512,
                                        },
                                        mem: unsafe { data.address() } + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    self.fs.disk.request(request.clone());

                                    while request.complete.load(Ordering::SeqCst) == false {
                                        sys_yield();
                                    }

                                    sector += 65535;
                                }
                                if sector < sectors {
                                    let request = Request {
                                        extent: Extent {
                                            block: extent.block + sector as u64,
                                            length: (sectors - sector) as u64 * 512,
                                        },
                                        mem: unsafe { data.address() } + sector * 512,
                                        read: true,
                                        complete: Arc::new(AtomicBool::new(false)),
                                    };

                                    self.fs.disk.request(request.clone());

                                    while request.complete.load(Ordering::SeqCst) == false {
                                        sys_yield();
                                    }
                                }

                                vec.push_all(&Vec {
                                    data: unsafe { data.into_raw() },
                                    length: extent.length as usize,
                                });
                            }
                        }
                    }

                    return box FileResource {
                        scheme: self,
                        node: node,
                        vec: vec,
                        seek: 0,
                        dirty: false,
                    };
                }
                Option::None => {
                    return box NoneResource;
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate cgmath;
extern crate winit;

#[cfg(windows)]
use winit::os::windows::WindowExt;

#[macro_use]
extern crate vulkano;

#[path = "support/teapot.rs"]
mod teapot;

use std::ffi::OsStr;
use std::os::windows::ffi::OsStrExt;
use std::mem;
use std::ptr;
use std::sync::Arc;

mod vs { include!{concat!(env!("OUT_DIR"), "/shaders/examples/teapot_vs.glsl")} }
mod fs { include!{concat!(env!("OUT_DIR"), "/shaders/examples/teapot_fs.glsl")} }

fn main() {
    // The start of this example is exactly the same as `triangle`. You should read the
    // `triangle` example if you haven't done so yet.

    // TODO: for the moment the AMD driver crashes if you don't pass an ApplicationInfo, but in theory it's optional
    let app = vulkano::instance::ApplicationInfo { application_name: "test", application_version: 1, engine_name: "test", engine_version: 1 };
    let extensions = vulkano::instance::InstanceExtensions {
        khr_surface: true,
        khr_win32_surface: true,
        .. vulkano::instance::InstanceExtensions::none()
    };
    let instance = vulkano::instance::Instance::new(Some(&app), &extensions, None).expect("failed to create instance");

    let physical = vulkano::instance::PhysicalDevice::enumerate(&instance)
                            .next().expect("no device available");
    println!("Using device: {} (type: {:?})", physical.name(), physical.ty());

    let window = winit::WindowBuilder::new().build().unwrap();
    let surface = unsafe { vulkano::swapchain::Surface::from_hwnd(&instance, ptr::null() as *const () /* FIXME */, window.get_hwnd()).unwrap() };

    let queue = physical.queue_families().find(|q| q.supports_graphics() &&
                                                   surface.is_supported(q).unwrap_or(false))
                                                .expect("couldn't find a graphical queue family");

    let device_ext = vulkano::device::DeviceExtensions {
        khr_swapchain: true,
    };

    let (device, queues) = vulkano::device::Device::new(&physical, physical.supported_features(),
                                                        &device_ext, None, [(queue, 0.5)].iter().cloned())
                                                                .expect("failed to create device");
    let queue = queues.into_iter().next().unwrap();

    let (swapchain, images) = {
        let caps = surface.get_capabilities(&physical).expect("failed to get surface capabilities");

        let dimensions = caps.current_extent.unwrap_or([1280, 1024]);
        let present = caps.present_modes[0];
        let usage = caps.supported_usage_flags;

        vulkano::swapchain::Swapchain::new(&device, &surface, 3,
                                           vulkano::format::B8G8R8A8Srgb, dimensions, 1,
                                           &usage, &queue, vulkano::swapchain::SurfaceTransform::Identity,
                                           vulkano::swapchain::CompositeAlpha::Opaque,
                                           present, true).expect("failed to create swapchain")
    };

    let cb_pool = vulkano::command_buffer::CommandBufferPool::new(&device, &queue.family())
                                                  .expect("failed to create command buffer pool");

    let depth_buffer = vulkano::image::attachment::AttachmentImage::new(&device, images[0].dimensions().width_height(), vulkano::format::D16Unorm).unwrap();

    let vertex_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer
                               ::array(&device, teapot::VERTICES.len(),
                                       &vulkano::buffer::Usage::all(), Some(queue.family()))
                                       .expect("failed to create buffer");

    {
        let mut mapping = vertex_buffer.try_write().unwrap();
        for (o, i) in mapping.iter_mut().zip(teapot::VERTICES.iter()) {
            *o = *i;
        }
    }

    let normals_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer
                                ::array(&device, teapot::NORMALS.len(),
                                        &vulkano::buffer::Usage::all(), Some(queue.family()))
                                        .expect("failed to create buffer");

    {
        let mut mapping = normals_buffer.try_write().unwrap();
        for (o, i) in mapping.iter_mut().zip(teapot::NORMALS.iter()) {
            *o = *i;
        }
    }

    let index_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer
                              ::array(&device, teapot::INDICES.len(),
                                      &vulkano::buffer::Usage::all(), Some(queue.family()))
                                      .expect("failed to create buffer");

    {
        let mut mapping = index_buffer.try_write().unwrap();
        for (o, i) in mapping.iter_mut().zip(teapot::INDICES.iter()) {
            *o = *i;
        }
    }

    // note: this teapot was meant for OpenGL where the origin is at the lower left
    //       instead the origin is at the upper left in vulkan, so we reverse the Y axis
    let proj = cgmath::perspective(cgmath::rad(3.141592 / 2.0), { let d = images[0].dimensions(); d.width() as f32 / d.height() as f32 }, 0.01, 100.0);
    let view = cgmath::Matrix4::look_at(cgmath::Point3::new(0.3, 0.3, 1.0), cgmath::Point3::new(0.0, 0.0, 0.0), cgmath::Vector3::new(0.0, -1.0, 0.0));
    let scale = cgmath::Matrix4::from_scale(0.01);

    let uniform_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer::<vs::ty::Data>
                               ::new(&device, &vulkano::buffer::Usage::all(), Some(queue.family()))
                               .expect("failed to create buffer");
    {
        let mut mapping = uniform_buffer.try_write().unwrap();
        mapping.worldview = (view * scale).into();
        mapping.proj = proj.into();
    }

    let vs = vs::Shader::load(&device).expect("failed to create shader module");
    let fs = fs::Shader::load(&device).expect("failed to create shader module");

    mod renderpass {
        single_pass_renderpass!{
            attachments: {
                color: {
                    load: Clear,
                    store: Store,
                    format: B8G8R8A8Srgb,
                },
                depth: {
                    load: Clear,
                    store: DontCare,
                    format: D16Unorm,
                }
            },
            pass: {
                color: [color],
                depth_stencil: {depth}
            }
        }
    }

    let renderpass = renderpass::CustomRenderPass::new(&device).unwrap();

    let descriptor_pool = vulkano::descriptor_set::DescriptorPool::new(&device).unwrap();
    let descriptor_set_layout = vulkano::descriptor_set::DescriptorSetLayout::new(&device, vs::Set0).unwrap();

    mod pipeline_layout { pipeline_from_sets!(::vs::Set0); }
    let pipeline_layout = vulkano::descriptor_set::PipelineLayout::new(&device, pipeline_layout::Layout, (descriptor_set_layout.clone(),)).unwrap();
    let set = vulkano::descriptor_set::DescriptorSet::new(&descriptor_pool, &descriptor_set_layout,
                                                          uniform_buffer.clone() as std::sync::Arc<_>).unwrap();

    let pipeline = {
        let ia = vulkano::pipeline::input_assembly::InputAssembly::triangle_list();
        let raster = Default::default();
        let ms = vulkano::pipeline::multisample::Multisample::disabled();
        let blend = vulkano::pipeline::blend::Blend {
            logic_op: None,
            blend_constants: Some([0.0; 4]),
        };

        let viewports = vulkano::pipeline::viewport::ViewportsState::Fixed {
            data: vec![(
                vulkano::pipeline::viewport::Viewport {
                    origin: [0.0, 0.0],
                    dimensions: [images[0].dimensions().width() as f32, images[0].dimensions().height() as f32],
                    depth_range: 0.0 .. 1.0
                },
                vulkano::pipeline::viewport::Scissor::irrelevant()
            )],
        };

        vulkano::pipeline::GraphicsPipeline::new(&device, vulkano::pipeline::vertex::TwoBuffersDefinition::new(), &vs.main_entry_point(), &ia, &viewports,
                                                 &raster, &ms, &blend, &fs.main_entry_point(),
                                                 &pipeline_layout, vulkano::framebuffer::Subpass::from(&renderpass, 0).unwrap())
                                                 .unwrap()
    };

    let framebuffers = images.iter().map(|image| {
        let attachments = renderpass::AList {
            color: &image,
            depth: &depth_buffer,
        };

        vulkano::framebuffer::Framebuffer::new(&renderpass, (1244, 699, 1), attachments).unwrap()
    }).collect::<Vec<_>>();

    let command_buffers = framebuffers.iter().map(|framebuffer| {
        vulkano::command_buffer::PrimaryCommandBufferBuilder::new(&cb_pool).unwrap()
            .draw_inline(&renderpass, &framebuffer, renderpass::ClearValues {
                 color: [0.0, 0.0, 1.0, 1.0],
                 depth: 1.0,
             })
            .draw_indexed(&pipeline, (&vertex_buffer, &normals_buffer), &index_buffer, &vulkano::command_buffer::DynamicState::none(), set.clone())
            .draw_end()
            .build().unwrap()
    }).collect::<Vec<_>>();

    let mut submissions: Vec<Arc<vulkano::command_buffer::Submission>> = Vec::new();

    loop {
        submissions.retain(|s| !s.destroying_would_block());

        let image_num = swapchain.acquire_next_image(1000000).unwrap();
        submissions.push(vulkano::command_buffer::submit(&command_buffers[image_num], &queue).unwrap());
        swapchain.present(&queue, image_num).unwrap();

        for ev in window.poll_events() {
            match ev {
                winit::Event::Closed => break,
                _ => ()
            }
        }
    }
}
<commit_msg>Fix viewport in teapot example<commit_after>extern crate cgmath;
extern crate winit;

#[cfg(windows)]
use winit::os::windows::WindowExt;

#[macro_use]
extern crate vulkano;

#[path = "support/teapot.rs"]
mod teapot;

use std::ffi::OsStr;
use std::os::windows::ffi::OsStrExt;
use std::mem;
use std::ptr;
use std::sync::Arc;

mod vs { include!{concat!(env!("OUT_DIR"), "/shaders/examples/teapot_vs.glsl")} }
mod fs { include!{concat!(env!("OUT_DIR"), "/shaders/examples/teapot_fs.glsl")} }

fn main() {
    // The start of this example is exactly the same as `triangle`. You should read the
    // `triangle` example if you haven't done so yet.

    // TODO: for the moment the AMD driver crashes if you don't pass an ApplicationInfo, but in theory it's optional
    let app = vulkano::instance::ApplicationInfo { application_name: "test", application_version: 1, engine_name: "test", engine_version: 1 };
    let extensions = vulkano::instance::InstanceExtensions {
        khr_surface: true,
        khr_win32_surface: true,
        .. vulkano::instance::InstanceExtensions::none()
    };
    let instance = vulkano::instance::Instance::new(Some(&app), &extensions, None).expect("failed to create instance");

    let physical = vulkano::instance::PhysicalDevice::enumerate(&instance)
                            .next().expect("no device available");
    println!("Using device: {} (type: {:?})", physical.name(), physical.ty());

    let window = winit::WindowBuilder::new().build().unwrap();
    let surface = unsafe { vulkano::swapchain::Surface::from_hwnd(&instance, ptr::null() as *const () /* FIXME */, window.get_hwnd()).unwrap() };

    let queue = physical.queue_families().find(|q| q.supports_graphics() &&
                                                   surface.is_supported(q).unwrap_or(false))
                                                .expect("couldn't find a graphical queue family");

    let device_ext = vulkano::device::DeviceExtensions {
        khr_swapchain: true,
    };

    let (device, queues) = vulkano::device::Device::new(&physical, physical.supported_features(),
                                                        &device_ext, None, [(queue, 0.5)].iter().cloned())
                                                                .expect("failed to create device");
    let queue = queues.into_iter().next().unwrap();

    let (swapchain, images) = {
        let caps = surface.get_capabilities(&physical).expect("failed to get surface capabilities");

        let dimensions = caps.current_extent.unwrap_or([1280, 1024]);
        let present = caps.present_modes[0];
        let usage = caps.supported_usage_flags;

        vulkano::swapchain::Swapchain::new(&device, &surface, 3,
                                           vulkano::format::B8G8R8A8Srgb, dimensions, 1,
                                           &usage, &queue, vulkano::swapchain::SurfaceTransform::Identity,
                                           vulkano::swapchain::CompositeAlpha::Opaque,
                                           present, true).expect("failed to create swapchain")
    };

    let cb_pool = vulkano::command_buffer::CommandBufferPool::new(&device, &queue.family())
                                                  .expect("failed to create command buffer pool");

    let depth_buffer = vulkano::image::attachment::AttachmentImage::new(&device, images[0].dimensions().width_height(), vulkano::format::D16Unorm).unwrap();

    let vertex_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer
                               ::array(&device, teapot::VERTICES.len(),
                                       &vulkano::buffer::Usage::all(), Some(queue.family()))
                                       .expect("failed to create buffer");

    {
        let mut mapping = vertex_buffer.try_write().unwrap();
        for (o, i) in mapping.iter_mut().zip(teapot::VERTICES.iter()) {
            *o = *i;
        }
    }

    let normals_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer
                                ::array(&device, teapot::NORMALS.len(),
                                        &vulkano::buffer::Usage::all(), Some(queue.family()))
                                        .expect("failed to create buffer");

    {
        let mut mapping = normals_buffer.try_write().unwrap();
        for (o, i) in mapping.iter_mut().zip(teapot::NORMALS.iter()) {
            *o = *i;
        }
    }

    let index_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer
                              ::array(&device, teapot::INDICES.len(),
                                      &vulkano::buffer::Usage::all(), Some(queue.family()))
                                      .expect("failed to create buffer");

    {
        let mut mapping = index_buffer.try_write().unwrap();
        for (o, i) in mapping.iter_mut().zip(teapot::INDICES.iter()) {
            *o = *i;
        }
    }

    // note: this teapot was meant for OpenGL where the origin is at the lower left
    //       instead the origin is at the upper left in vulkan, so we reverse the Y axis
    let proj = cgmath::perspective(cgmath::rad(3.141592 / 2.0), { let d = images[0].dimensions(); d.width() as f32 / d.height() as f32 }, 0.01, 100.0);
    let view = cgmath::Matrix4::look_at(cgmath::Point3::new(0.3, 0.3, 1.0), cgmath::Point3::new(0.0, 0.0, 0.0), cgmath::Vector3::new(0.0, -1.0, 0.0));
    let scale = cgmath::Matrix4::from_scale(0.01);

    let uniform_buffer = vulkano::buffer::cpu_access::CpuAccessibleBuffer::<vs::ty::Data>
                               ::new(&device, &vulkano::buffer::Usage::all(), Some(queue.family()))
                               .expect("failed to create buffer");
    {
        let mut mapping = uniform_buffer.try_write().unwrap();
        mapping.worldview = (view * scale).into();
        mapping.proj = proj.into();
    }

    let vs = vs::Shader::load(&device).expect("failed to create shader module");
    let fs = fs::Shader::load(&device).expect("failed to create shader module");

    mod renderpass {
        single_pass_renderpass!{
            attachments: {
                color: {
                    load: Clear,
                    store: Store,
                    format: B8G8R8A8Srgb,
                },
                depth: {
                    load: Clear,
                    store: DontCare,
                    format: D16Unorm,
                }
            },
            pass: {
                color: [color],
                depth_stencil: {depth}
            }
        }
    }

    let renderpass = renderpass::CustomRenderPass::new(&device).unwrap();

    let descriptor_pool = vulkano::descriptor_set::DescriptorPool::new(&device).unwrap();
    let descriptor_set_layout = vulkano::descriptor_set::DescriptorSetLayout::new(&device, vs::Set0).unwrap();

    mod pipeline_layout { pipeline_from_sets!(::vs::Set0); }
    let pipeline_layout = vulkano::descriptor_set::PipelineLayout::new(&device, pipeline_layout::Layout, (descriptor_set_layout.clone(),)).unwrap();
    let set = vulkano::descriptor_set::DescriptorSet::new(&descriptor_pool, &descriptor_set_layout,
                                                          uniform_buffer.clone() as std::sync::Arc<_>).unwrap();

    let pipeline = {
        let ia = vulkano::pipeline::input_assembly::InputAssembly::triangle_list();
        let raster = Default::default();
        let ms = vulkano::pipeline::multisample::Multisample::disabled();
        let blend = vulkano::pipeline::blend::Blend {
            logic_op: None,
            blend_constants: Some([0.0; 4]),
        };

        let viewports = vulkano::pipeline::viewport::ViewportsState::Fixed {
            data: vec![(
                vulkano::pipeline::viewport::Viewport {
                    origin: [0.0, 0.0],
                    dimensions: [images[0].dimensions().width() as f32, images[0].dimensions().height() as f32],
                    depth_range: 0.0 .. 1.0
                },
                vulkano::pipeline::viewport::Scissor::irrelevant()
            )],
        };

        vulkano::pipeline::GraphicsPipeline::new(&device, vulkano::pipeline::vertex::TwoBuffersDefinition::new(), &vs.main_entry_point(), &ia, &viewports,
                                                 &raster, &ms, &blend, &fs.main_entry_point(),
                                                 &pipeline_layout, vulkano::framebuffer::Subpass::from(&renderpass, 0).unwrap())
                                                 .unwrap()
    };

    let framebuffers = images.iter().map(|image| {
        let attachments = renderpass::AList {
            color: &image,
            depth: &depth_buffer,
        };

        vulkano::framebuffer::Framebuffer::new(&renderpass, (image.dimensions().width(), image.dimensions().height(), 1), attachments).unwrap()
    }).collect::<Vec<_>>();

    let command_buffers = framebuffers.iter().map(|framebuffer| {
        vulkano::command_buffer::PrimaryCommandBufferBuilder::new(&cb_pool).unwrap()
            .draw_inline(&renderpass, &framebuffer, renderpass::ClearValues {
                 color: [0.0, 0.0, 1.0, 1.0],
                 depth: 1.0,
             })
            .draw_indexed(&pipeline, (&vertex_buffer, &normals_buffer), &index_buffer, &vulkano::command_buffer::DynamicState::none(), set.clone())
            .draw_end()
            .build().unwrap()
    }).collect::<Vec<_>>();

    let mut submissions: Vec<Arc<vulkano::command_buffer::Submission>> = Vec::new();

    loop {
        submissions.retain(|s| !s.destroying_would_block());

        let image_num = swapchain.acquire_next_image(1000000).unwrap();
        submissions.push(vulkano::command_buffer::submit(&command_buffers[image_num], &queue).unwrap());
        swapchain.present(&queue, image_num).unwrap();

        for ev in window.poll_events() {
            match ev {
                winit::Event::Closed => break,
                _ => ()
            }
        }
    }
}
<|endoftext|>


<commit_before>/// From <https://github.com/graphql/graphql-js/blob/8c96dc8276f2de27b8af9ffbd71a4597d483523f/src/utilities/introspectionQuery.js#L21>s
pub(crate) const INTROSPECTION_QUERY: &str = include_str!("./query.graphql");
pub(crate) const INTROSPECTION_QUERY_WITHOUT_DESCRIPTIONS: &str =
    include_str!("./query_without_descriptions.graphql");

/// The desired GraphQL introspection format for the canonical query
/// (<https://github.com/graphql/graphql-js/blob/8c96dc8276f2de27b8af9ffbd71a4597d483523f/src/utilities/introspectionQuery.js#L21>)
pub enum IntrospectionFormat {
    /// The canonical GraphQL introspection query.
    All,
    /// The canonical GraphQL introspection query without descriptions.
    WithoutDescriptions,
}

impl Default for IntrospectionFormat {
    fn default() -> Self {
        IntrospectionFormat::All
    }
}
<commit_msg>Remove typo in comment<commit_after>/// From <https://github.com/graphql/graphql-js/blob/8c96dc8276f2de27b8af9ffbd71a4597d483523f/src/utilities/introspectionQuery.js#L21>
pub(crate) const INTROSPECTION_QUERY: &str = include_str!("./query.graphql");
pub(crate) const INTROSPECTION_QUERY_WITHOUT_DESCRIPTIONS: &str =
    include_str!("./query_without_descriptions.graphql");

/// The desired GraphQL introspection format for the canonical query
/// (<https://github.com/graphql/graphql-js/blob/8c96dc8276f2de27b8af9ffbd71a4597d483523f/src/utilities/introspectionQuery.js#L21>)
pub enum IntrospectionFormat {
    /// The canonical GraphQL introspection query.
    All,
    /// The canonical GraphQL introspection query without descriptions.
    WithoutDescriptions,
}

impl Default for IntrospectionFormat {
    fn default() -> Self {
        IntrospectionFormat::All
    }
}
<|endoftext|>


<commit_before>#[allow(ctypes)];
#[no_std];
#[no_core];

mod zero;

enum Color {
    Black       = 0,
    Blue        = 1,
    Green       = 2,
    Cyan        = 3,
    Red         = 4,
    Pink        = 5,
    Brown       = 6,
    LightGray   = 7,
    DarkGray    = 8,
    LightBlue   = 9,
    LightGreen  = 10,
    LightCyan   = 11,
    LightRed    = 12,
    LightPink   = 13,
    Yellow      = 14,
    White       = 15,
}

fn range(lo: uint, hi: uint, it: &fn(uint)) {
    let mut iter = lo;
    while iter < hi {
        it(iter);
        iter += 1;
    }
}

unsafe fn clear_screen(background: Color) {
    range(0, 80*25, |i| {
        *((0xb8000 + i * 2) as *mut u16) = (background as u16) << 12;
    });
}

#[no_mangle]
pub unsafe fn main() {
    clear_screen(LightRed);
}
<commit_msg>Write a lot of 'c's to the screen<commit_after>#[allow(ctypes)];
#[no_std];
#[no_core];

static VGA_WIDTH  : u16 = 80;
static VGA_HEIGHT : u16 = 24;

mod zero;

enum Color {
    Black       = 0,
    Blue        = 1,
    Green       = 2,
    Cyan        = 3,
    Red         = 4,
    Pink        = 5,
    Brown       = 6,
    LightGray   = 7,
    DarkGray    = 8,
    LightBlue   = 9,
    LightGreen  = 10,
    LightCyan   = 11,
    LightRed    = 12,
    LightPink   = 13,
    Yellow      = 14,
    White       = 15,
}

fn range(lo: uint, hi: uint, it: &fn(uint)) {
    let mut iter = lo;
    while iter < hi {
        it(iter);
        iter += 1;
    }
}

unsafe fn clear_screen(background: Color) {
    range(0, 80*25, |i| {
        *((0xb8000 + i * 2) as *mut u16) = make_vgaentry(99, Black, Red);

    });
}

unsafe fn putchar(x: u16, y: u16, c: u8) {
    let idx : u16 = y * VGA_WIDTH + x;
    *((0xb8000 + idx) as *mut u16) = make_vgaentry(c, Black, Yellow);
}

fn make_vgaentry(c: u8, fg: Color, bg: Color) -> u16 {
    let color = fg as u16 | (bg as u16 << 4);
    return c as u16 | (color << 8);
}

#[no_mangle]
pub unsafe fn main() {
    clear_screen(White);
    putchar(15, 15, 99);
}
<|endoftext|>


<commit_before>pub mod doc_id_set;
pub mod statistics;
pub mod planner;

use kite::query::Query;
use kite::collectors::{Collector, DocumentMatch};
use byteorder::{ByteOrder, BigEndian};

use key_builder::KeyBuilder;
use document_index::DocRef;
use super::RocksDBIndexReader;
use search::doc_id_set::DocIdSet;
use search::statistics::StatisticsReader;
use search::planner::{SearchPlan, plan_query};
use search::planner::boolean_query::BooleanQueryOp;
use search::planner::score_function::{CombinatorScorer, ScoreFunctionOp};

impl<'a> RocksDBIndexReader<'a> {
    fn search_chunk_boolean_phase(&self, plan: &SearchPlan, chunk: u32) -> DocIdSet {
        // Execute boolean query
        let mut stack = Vec::new();
        for op in plan.boolean_query.iter() {
            match *op {
                BooleanQueryOp::PushEmpty => {
                    stack.push(DocIdSet::new_filled(0));
                }
                BooleanQueryOp::PushFull => {
                    stack.push(DocIdSet::new_filled(65536));
                }
                BooleanQueryOp::PushTermDirectory(field_ref, term_ref) => {
                    let kb = KeyBuilder::chunk_dir_list(chunk, field_ref.ord(), term_ref.ord());
                    match self.snapshot.get(&kb.key()) {
                        Ok(Some(docid_set)) => {
                            let data = DocIdSet::FromRDB(docid_set);
                            stack.push(data);
                        }
                        Ok(None) => stack.push(DocIdSet::new_filled(0)),
                        Err(e) => {},  // FIXME
                    }
                }
                BooleanQueryOp::PushDeletionList => {
                    let kb = KeyBuilder::chunk_del_list(chunk);
                    match self.snapshot.get(&kb.key()) {
                        Ok(Some(deletion_list)) => {
                            let data = DocIdSet::FromRDB(deletion_list);
                            stack.push(data);
                        }
                        Ok(None) => stack.push(DocIdSet::new_filled(0)),
                        Err(e) => {},  // FIXME
                    }
                }
                BooleanQueryOp::And => {
                    let b = stack.pop().expect("stack underflow");
                    let a = stack.pop().expect("stack underflow");
                    stack.push(a.intersection(&b));
                }
                BooleanQueryOp::Or => {
                    let b = stack.pop().expect("stack underflow");
                    let a = stack.pop().expect("stack underflow");
                    stack.push(a.union(&b));
                }
                BooleanQueryOp::AndNot => {
                    let b = stack.pop().expect("stack underflow");
                    let a = stack.pop().expect("stack underflow");
                    stack.push(a.exclusion(&b));
                }
            }
        }

        if !stack.len() == 1 {
            // TODO: Error
        }
        let mut matches = stack.pop().unwrap();

        // Invert the list if the query is negated
        if plan.boolean_query_is_negated {
            let kb = KeyBuilder::chunk_stat(chunk, b"total_docs");
            let total_docs = match self.snapshot.get(&kb.key()) {
                Ok(Some(total_docs)) => BigEndian::read_i64(&total_docs) as u32,
                Ok(None) => 0,
                Err(e) => 0,  // FIXME
            };

            let all_docs = DocIdSet::new_filled(total_docs);
            matches = all_docs.exclusion(&matches);
        }

        matches
    }

    fn score_doc(&self, doc_id: u16, plan: &SearchPlan, chunk: u32, mut stats: &mut StatisticsReader) -> f64 {
        // Execute score function
        let mut stack = Vec::new();
        for op in plan.score_function.iter() {
            match *op {
                ScoreFunctionOp::Literal(val) => stack.push(val),
                ScoreFunctionOp::TermScorer(field_ref, term_ref, ref scorer) => {
                    let kb = KeyBuilder::chunk_dir_list(chunk, field_ref.ord(), term_ref.ord());
                    match self.snapshot.get(&kb.key()) {
                        Ok(Some(data)) => {
                            let docid_set = DocIdSet::FromRDB(data);

                            if docid_set.contains_doc(doc_id) {
                                // Read field length
                                // TODO: we only need this for BM25
                                let kb = KeyBuilder::stored_field_value(chunk, doc_id, field_ref.ord(), b"len");
                                let field_length = match self.snapshot.get(&kb.key()) {
                                    Ok(Some(value)) => {
                                        let length_sqrt = (value[0] as f64) / 3.0 + 1.0;
                                        length_sqrt * length_sqrt
                                    }
                                    Ok(None) => 1.0,
                                    Err(e) => 1.0,  // TODO Error
                                };

                                // Read term frequency
                                let mut value_type = vec![b't', b'f'];
                                value_type.extend(term_ref.ord().to_string().as_bytes());
                                let kb = KeyBuilder::stored_field_value(chunk, doc_id, field_ref.ord(), &value_type);
                                let term_frequency = match self.snapshot.get(&kb.key()) {
                                    Ok(Some(value)) => BigEndian::read_i64(&value),
                                    Ok(None) => 1,
                                    Err(e) => 1,  // TODO Error
                                };

                                let score = scorer.similarity_model.score(1, field_length, stats.total_tokens(field_ref) as u64, stats.total_docs(field_ref) as u64, stats.term_document_frequency(field_ref, term_ref) as u64);
                                stack.push(score * scorer.boost);
                            } else {
                                stack.push(0.0f64);
                            }
                        }
                        Ok(None) => stack.push(0.0f64),
                        Err(e) => {},  // FIXME
                    }
                }
                ScoreFunctionOp::CombinatorScorer(num_vals, ref scorer) => {
                    let score = match *scorer {
                        CombinatorScorer::Avg => {
                            let mut total_score = 0.0f64;

                            for _ in 0..num_vals {
                                total_score += stack.pop().expect("stack underflow");
                            }

                            total_score / num_vals as f64
                        }
                        CombinatorScorer::Max => {
                            let mut max_score = 0.0f64;

                            for _ in 0..num_vals {
                                let score = stack.pop().expect("stack underflow");
                                if score > max_score {
                                    max_score = score
                                }
                            }

                            max_score
                        }
                    };

                    stack.push(score);
                }
            }
        }

        stack.pop().expect("stack underflow")
    }

    fn search_chunk<C: Collector>(&self, collector: &mut C, plan: &SearchPlan, chunk: u32, mut stats: &mut StatisticsReader) {
        let matches = self.search_chunk_boolean_phase(plan, chunk);

        // Score documents and pass to collector
        for doc in matches.iter() {
            let score = self.score_doc(doc, plan, chunk, &mut stats);

            let doc_ref = DocRef::from_chunk_ord(chunk, doc);
            let doc_match = DocumentMatch::new_scored(doc_ref.as_u64(), score);
            collector.collect(doc_match);
        }
    }

    pub fn search<C: Collector>(&self, collector: &mut C, query: &Query) {
        // Plan query
        let plan = plan_query(&self, query, collector.needs_score());

        // Initialise statistics reader
        let mut stats = StatisticsReader::new(&self);

        // Run query on each chunk
        for chunk in self.store.chunks.iter_active(&self.snapshot) {
            self.search_chunk(collector, &plan, chunk, &mut stats);
        }
    }
}
<commit_msg>Only pass in what's needed to query executors<commit_after>pub mod doc_id_set;
pub mod statistics;
pub mod planner;

use kite::query::Query;
use kite::collectors::{Collector, DocumentMatch};
use byteorder::{ByteOrder, BigEndian};

use key_builder::KeyBuilder;
use document_index::DocRef;
use super::RocksDBIndexReader;
use search::doc_id_set::DocIdSet;
use search::statistics::StatisticsReader;
use search::planner::{SearchPlan, plan_query};
use search::planner::boolean_query::BooleanQueryOp;
use search::planner::score_function::{CombinatorScorer, ScoreFunctionOp};

impl<'a> RocksDBIndexReader<'a> {
    fn search_chunk_boolean_phase(&self, boolean_query: &Vec<BooleanQueryOp>, is_negated: bool, chunk: u32) -> DocIdSet {
        // Execute boolean query
        let mut stack = Vec::new();
        for op in boolean_query.iter() {
            match *op {
                BooleanQueryOp::PushEmpty => {
                    stack.push(DocIdSet::new_filled(0));
                }
                BooleanQueryOp::PushFull => {
                    stack.push(DocIdSet::new_filled(65536));
                }
                BooleanQueryOp::PushTermDirectory(field_ref, term_ref) => {
                    let kb = KeyBuilder::chunk_dir_list(chunk, field_ref.ord(), term_ref.ord());
                    match self.snapshot.get(&kb.key()) {
                        Ok(Some(docid_set)) => {
                            let data = DocIdSet::FromRDB(docid_set);
                            stack.push(data);
                        }
                        Ok(None) => stack.push(DocIdSet::new_filled(0)),
                        Err(e) => {},  // FIXME
                    }
                }
                BooleanQueryOp::PushDeletionList => {
                    let kb = KeyBuilder::chunk_del_list(chunk);
                    match self.snapshot.get(&kb.key()) {
                        Ok(Some(deletion_list)) => {
                            let data = DocIdSet::FromRDB(deletion_list);
                            stack.push(data);
                        }
                        Ok(None) => stack.push(DocIdSet::new_filled(0)),
                        Err(e) => {},  // FIXME
                    }
                }
                BooleanQueryOp::And => {
                    let b = stack.pop().expect("stack underflow");
                    let a = stack.pop().expect("stack underflow");
                    stack.push(a.intersection(&b));
                }
                BooleanQueryOp::Or => {
                    let b = stack.pop().expect("stack underflow");
                    let a = stack.pop().expect("stack underflow");
                    stack.push(a.union(&b));
                }
                BooleanQueryOp::AndNot => {
                    let b = stack.pop().expect("stack underflow");
                    let a = stack.pop().expect("stack underflow");
                    stack.push(a.exclusion(&b));
                }
            }
        }

        if !stack.len() == 1 {
            // TODO: Error
        }
        let mut matches = stack.pop().unwrap();

        // Invert the list if the query is negated
        if is_negated {
            let kb = KeyBuilder::chunk_stat(chunk, b"total_docs");
            let total_docs = match self.snapshot.get(&kb.key()) {
                Ok(Some(total_docs)) => BigEndian::read_i64(&total_docs) as u32,
                Ok(None) => 0,
                Err(e) => 0,  // FIXME
            };

            let all_docs = DocIdSet::new_filled(total_docs);
            matches = all_docs.exclusion(&matches);
        }

        matches
    }

    fn score_doc(&self, doc_id: u16, score_function: &Vec<ScoreFunctionOp>, chunk: u32, mut stats: &mut StatisticsReader) -> f64 {
        // Execute score function
        let mut stack = Vec::new();
        for op in score_function.iter() {
            match *op {
                ScoreFunctionOp::Literal(val) => stack.push(val),
                ScoreFunctionOp::TermScorer(field_ref, term_ref, ref scorer) => {
                    let kb = KeyBuilder::chunk_dir_list(chunk, field_ref.ord(), term_ref.ord());
                    match self.snapshot.get(&kb.key()) {
                        Ok(Some(data)) => {
                            let docid_set = DocIdSet::FromRDB(data);

                            if docid_set.contains_doc(doc_id) {
                                // Read field length
                                // TODO: we only need this for BM25
                                let kb = KeyBuilder::stored_field_value(chunk, doc_id, field_ref.ord(), b"len");
                                let field_length = match self.snapshot.get(&kb.key()) {
                                    Ok(Some(value)) => {
                                        let length_sqrt = (value[0] as f64) / 3.0 + 1.0;
                                        length_sqrt * length_sqrt
                                    }
                                    Ok(None) => 1.0,
                                    Err(e) => 1.0,  // TODO Error
                                };

                                // Read term frequency
                                let mut value_type = vec![b't', b'f'];
                                value_type.extend(term_ref.ord().to_string().as_bytes());
                                let kb = KeyBuilder::stored_field_value(chunk, doc_id, field_ref.ord(), &value_type);
                                let term_frequency = match self.snapshot.get(&kb.key()) {
                                    Ok(Some(value)) => BigEndian::read_i64(&value),
                                    Ok(None) => 1,
                                    Err(e) => 1,  // TODO Error
                                };

                                let score = scorer.similarity_model.score(1, field_length, stats.total_tokens(field_ref) as u64, stats.total_docs(field_ref) as u64, stats.term_document_frequency(field_ref, term_ref) as u64);
                                stack.push(score * scorer.boost);
                            } else {
                                stack.push(0.0f64);
                            }
                        }
                        Ok(None) => stack.push(0.0f64),
                        Err(e) => {},  // FIXME
                    }
                }
                ScoreFunctionOp::CombinatorScorer(num_vals, ref scorer) => {
                    let score = match *scorer {
                        CombinatorScorer::Avg => {
                            let mut total_score = 0.0f64;

                            for _ in 0..num_vals {
                                total_score += stack.pop().expect("stack underflow");
                            }

                            total_score / num_vals as f64
                        }
                        CombinatorScorer::Max => {
                            let mut max_score = 0.0f64;

                            for _ in 0..num_vals {
                                let score = stack.pop().expect("stack underflow");
                                if score > max_score {
                                    max_score = score
                                }
                            }

                            max_score
                        }
                    };

                    stack.push(score);
                }
            }
        }

        stack.pop().expect("stack underflow")
    }

    fn search_chunk<C: Collector>(&self, collector: &mut C, plan: &SearchPlan, chunk: u32, mut stats: &mut StatisticsReader) {
        let matches = self.search_chunk_boolean_phase(&plan.boolean_query, plan.boolean_query_is_negated, chunk);

        // Score documents and pass to collector
        for doc in matches.iter() {
            let score = self.score_doc(doc, &plan.score_function, chunk, &mut stats);

            let doc_ref = DocRef::from_chunk_ord(chunk, doc);
            let doc_match = DocumentMatch::new_scored(doc_ref.as_u64(), score);
            collector.collect(doc_match);
        }
    }

    pub fn search<C: Collector>(&self, collector: &mut C, query: &Query) {
        // Plan query
        let plan = plan_query(&self, query, collector.needs_score());

        // Initialise statistics reader
        let mut stats = StatisticsReader::new(&self);

        // Run query on each chunk
        for chunk in self.store.chunks.iter_active(&self.snapshot) {
            self.search_chunk(collector, &plan, chunk, &mut stats);
        }
    }
}
<|endoftext|>