<commit_before>//! A not-quite-trivial HTTP server which responds to requests by showing the request and response
//! headers and any other information it has.

extern mod extra;
extern mod http;

use std::rt::io::net::ip::{SocketAddr, Ipv4Addr};
use std::rt::io::Writer;
use extra::time;

use http::server::{Config, Server, ServerUtil, Request, ResponseWriter};
use http::headers::HeaderEnum;

#[deriving(Clone)]
struct InfoServer;

impl Server for InfoServer {
    fn get_config(&self) -> Config {
        Config { bind_address: SocketAddr { ip: Ipv4Addr(127, 0, 0, 1), port: 8001 } }
    }

    fn handle_request(&self, r: &Request, w: &mut ResponseWriter) {
        w.headers.date = Some(time::now_utc());
        w.headers.content_type = Some(~"text/plain; charset=UTF-8");
        w.headers.server = Some(~"Rust Thingummy/0.0-pre");
        w.write(bytes!("<!DOCTYPE html><title>Rust HTTP server</title>"));

        w.write(bytes!("
Request
"));
        let s = fmt!("

Method

%s

Host

%?

Request URI

%?

HTTP version

%?

Close connection

%?

",
            r.method.to_str(),
            r.headers.host,
            r.request_uri,
            r.version,
            r.close_connection);
        w.write(s.as_bytes());
        w.write(bytes!("
Extension headers
"));
        w.write(bytes!("

Name	Value

"));
        for header in r.headers.iter() {
            let line = fmt!("
%s	%s
",
                            header.header_name(),
                            header.header_value());
            w.write(line.as_bytes());
        }
        w.write(bytes!("

"));
        w.write(bytes!("
Body

"));
        w.write(r.body.as_bytes());
        w.write(bytes!("
"));

        w.write(bytes!("
Response
"));
        let s = fmt!("

Status

%s

", w.status.to_str());
        w.write(s.as_bytes());
        w.write(bytes!("
Headers
"));
        w.write(bytes!("

Name	Value

"));
        {
            let h = w.headers.clone();
            for header in h.iter() {
                let line = fmt!("
%s	%s
",
                                header.header_name(),
                                header.header_value());
                w.write(line.as_bytes());
            }
        }
        w.write(bytes!("

"));
        // Server doesn't yet support the chunked transfer coding, nor was Content-Length specified,
        // so we'd better close the connection to ensure the page finishes loading.
        // DON'T COPY THIS WAY OF DOING THINGS. EVER. OR ELSE.
        unsafe { ::std::cast::transmute::<&Request, &mut Request>(r).close_connection = true; }
    }
}

fn main() {
    InfoServer.serve_forever();
}
<commit_msg>Serve info example as text/html.<commit_after>//! A not-quite-trivial HTTP server which responds to requests by showing the request and response
//! headers and any other information it has.

extern mod extra;
extern mod http;

use std::rt::io::net::ip::{SocketAddr, Ipv4Addr};
use std::rt::io::Writer;
use extra::time;

use http::server::{Config, Server, ServerUtil, Request, ResponseWriter};
use http::headers::HeaderEnum;

#[deriving(Clone)]
struct InfoServer;

impl Server for InfoServer {
    fn get_config(&self) -> Config {
        Config { bind_address: SocketAddr { ip: Ipv4Addr(127, 0, 0, 1), port: 8001 } }
    }

    fn handle_request(&self, r: &Request, w: &mut ResponseWriter) {
        w.headers.date = Some(time::now_utc());
        w.headers.content_type = Some(~"text/html; charset=UTF-8");
        w.headers.server = Some(~"Rust Thingummy/0.0-pre");
        w.write(bytes!("<!DOCTYPE html><title>Rust HTTP server</title>"));

        w.write(bytes!("
Request
"));
        let s = fmt!("

Method

%s

Host

%?

Request URI

%?

HTTP version

%?

Close connection

%?

",
            r.method.to_str(),
            r.headers.host,
            r.request_uri,
            r.version,
            r.close_connection);
        w.write(s.as_bytes());
        w.write(bytes!("
Extension headers
"));
        w.write(bytes!("

Name	Value

"));
        for header in r.headers.iter() {
            let line = fmt!("
%s	%s
",
                            header.header_name(),
                            header.header_value());
            w.write(line.as_bytes());
        }
        w.write(bytes!("

"));
        w.write(bytes!("
Body

"));
        w.write(r.body.as_bytes());
        w.write(bytes!("
"));

        w.write(bytes!("
Response
"));
        let s = fmt!("

Status

%s

", w.status.to_str());
        w.write(s.as_bytes());
        w.write(bytes!("
Headers
"));
        w.write(bytes!("

Name	Value

"));
        {
            let h = w.headers.clone();
            for header in h.iter() {
                let line = fmt!("
%s	%s
",
                                header.header_name(),
                                header.header_value());
                w.write(line.as_bytes());
            }
        }
        w.write(bytes!("

"));
        // Server doesn't yet support the chunked transfer coding, nor was Content-Length specified,
        // so we'd better close the connection to ensure the page finishes loading.
        // DON'T COPY THIS WAY OF DOING THINGS. EVER. OR ELSE.
        unsafe { ::std::cast::transmute::<&Request, &mut Request>(r).close_connection = true; }
    }
}

fn main() {
    InfoServer.serve_forever();
}
<|endoftext|>


<commit_before>use crate::convert::*;
use crate::folded_multiply::*;
use core::hash::Hasher;

///This constant come from Kunth's prng (Empirically it works better than those from splitmix32).
const MULTIPLE: u64 = crate::random_state::MULTIPLE;
const INCREMENT: u64 = 1442695040888963407;
const ROT: u32 = 23; //17

/// A `Hasher` for hashing an arbitrary stream of bytes.
///
/// Instances of [`AHasher`] represent state that is updated while hashing data.
///
/// Each method updates the internal state based on the new data provided. Once
/// all of the data has been provided, the resulting hash can be obtained by calling
/// `finish()`
///
/// [Clone] is also provided in case you wish to calculate hashes for two different items that
/// start with the same data.
///
#[derive(Debug, Clone)]
pub struct AHasher {
    buffer: u64,
    pad: u64,
}

impl AHasher {
    /// Creates a new hasher keyed to the provided key.
    #[inline]
    pub fn new_with_keys(key1: u64, key2: u64) -> AHasher {
        AHasher {
            buffer: key1,
            pad: key2,
        }
    }

    #[cfg(test)]
    pub(crate) fn test_with_keys(key1: u64, key2: u64) -> AHasher {
        use crate::random_state::scramble_keys;
        let (k1, k2) = scramble_keys(key1, key2);
        AHasher { buffer: k1, pad: k2 }
    }

    /// This update function has the goal of updating the buffer with a single multiply
    /// FxHash does this but is vulnerable to attack. To avoid this input needs to be masked to with an
    /// unpredictable value. Other hashes such as murmurhash have taken this approach but were found vulnerable
    /// to attack. The attack was based on the idea of reversing the pre-mixing (Which is necessarily
    /// reversible otherwise bits would be lost) then placing a difference in the highest bit before the
    /// multiply used to mix the data. Because a multiply can never affect the bits to the right of it, a
    /// subsequent update that also differed in this bit could result in a predictable collision.
    ///
    /// This version avoids this vulnerability while still only using a single multiply. It takes advantage
    /// of the fact that when a 64 bit multiply is performed the upper 64 bits are usually computed and thrown
    /// away. Instead it creates two 128 bit values where the upper 64 bits are zeros and multiplies them.
    /// (The compiler is smart enough to turn this into a 64 bit multiplication in the assembly)
    /// Then the upper bits are added to the lower bits to produce a single 64 bit result.
    ///
    /// To understand why this is a good scrambling function it helps to understand multiply-with-carry PRNGs:
    /// https://en.wikipedia.org/wiki/Multiply-with-carry_pseudorandom_number_generator
    /// If the multiple is chosen well, this creates a long period, decent quality PRNG.
    /// Notice that this function is equivalent to this except the `buffer`/`state` is being xored with each
    /// new block of data. In the event that data is all zeros, it is exactly equivalent to a MWC PRNG.
    ///
    /// This is impervious to attack because every bit buffer at the end is dependent on every bit in
    /// `new_data ^ buffer`. For example suppose two inputs differed in only the 5th bit. Then when the
    /// multiplication is performed the `result` will differ in bits 5-69. More specifically it will differ by
    /// 2^5 * MULTIPLE. However in the next step bits 65-128 are turned into a separate 64 bit value. So the
    /// differing bits will be in the lower 6 bits of this value. The two intermediate values that differ in
    /// bits 5-63 and in bits 0-5 respectively get added together. Producing an output that differs in every
    /// bit. The addition carries in the multiplication and at the end additionally mean that the even if an
    /// attacker somehow knew part of (but not all) the contents of the buffer before hand,
    /// they would not be able to predict any of the bits in the buffer at the end.
    #[inline(always)]
    fn update(&mut self, new_data: u64) {
        self.buffer = (new_data ^ self.buffer).folded_multiply(MULTIPLE);
    }

    /// This update function updates the buffer with the new information in a way that can't be canceled
    /// with a subsequent update without knowledge of the content of the buffer prior to the update.
    ///
    /// To achieve this the input needs to be modified in an unpredictable (to an attacker) way before it is
    /// combined with the value in the buffer. This is done by xoring it with `key`.
    ///
    /// Other hashes such as murmurhash have taken that approach but were found vulnerable to attack.
    /// The attack was based on the idea of reversing any pre-mixing (Which is necessarily reversible otherwise
    /// bits would be lost) then placing a difference in the highest bit before the multiply. Because a multiply
    /// can never affect the bits to the right of it, a subsequent update that also only differed in the high
    /// order bit could cancel out the change to `buffer` from the first update. This allowed murmurhash to be
    /// attacked. In this update function aHash avoids this vulnerability by rotating and performing a second
    /// multiply.
    ///
    /// This makes it impossible for an attacker to place a single bit difference between
    /// two blocks so as to cancel each other. (While the transform is still reversible if you know the key)
    ///
    /// This is similar to the above update function but is designed to run in a loop
    /// that will be unrolled and vectorized. So instead of using the buffer, it uses a 'key' that it updates
    /// and returns. The buffer is only xored at the end. This structure is so that when the method is inlined,
    /// the compiler will unroll any loop this gets placed in and the loop can be automatically vectorized
    /// and the rotates, xors, and multiplies can be paralleled.
    ///
    /// The key needs to be incremented between consecutive calls to prevent (a,b) from hashing the same as (b,a).
    /// The adding of the increment is moved to the bottom rather than the top. This allows one less add to be
    /// performed overall, but more importantly, it follows the multiply, which is expensive. So the CPU can
    /// run another operation afterwords if does not depend on the output of the multiply operation.
    #[inline(always)]
    fn ordered_update(&mut self, new_data: u64, key: u64) -> u64 {
        self.buffer ^= (new_data ^ key)
            .wrapping_mul(MULTIPLE)
            .rotate_left(ROT)
            .wrapping_mul(MULTIPLE);
        key.wrapping_add(INCREMENT)
    }
}

/// Provides methods to hash all of the primitive types.
impl Hasher for AHasher {
    #[inline]
    fn write_u8(&mut self, i: u8) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u16(&mut self, i: u16) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u32(&mut self, i: u32) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u64(&mut self, i: u64) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u128(&mut self, i: u128) {
        let data: [u64; 2] = i.convert();
        self.update(data[0]);
        self.update(data[1]);
    }

    #[inline]
    fn write_usize(&mut self, i: usize) {
        self.write_u64(i as u64);
    }

    #[inline]
    fn write(&mut self, input: &[u8]) {
        let mut data = input;
        let length = data.len() as u64;
        //Needs to be an add rather than an xor because otherwise it could be canceled with carefully formed input.
        self.buffer = self.buffer.wrapping_add(length.wrapping_mul(MULTIPLE));
        //A 'binary search' on sizes reduces the number of comparisons.
        if data.len() > 8 {
            if data.len() > 16 {
                let tail = data.read_last_u64();
                let mut key: u64 = self.buffer;
                while data.len() > 8 {
                    let (val, rest) = data.read_u64();
                    key = self.ordered_update(val, key);
                    data = rest;
                }
                self.update(tail);
            } else {
                self.update(data.read_u64().0);
                self.update(data.read_last_u64());
            }
        } else {
            if data.len() >= 2 {
                if data.len() >= 4 {
                    let block: [u32; 2] = [data.read_u32().0, data.read_last_u32()];
                    self.update(block.convert());
                } else {
                    let block: [u16; 2] = [data.read_u16().0, data.read_last_u16()];
                    let val: u32 = block.convert();
                    self.update(val as u64);
                }
            } else {
                let value = if data.len() > 0 {
                    data[0] //len 1
                } else {
                    0
                };
                self.update(value as u64);
            }
        }
    }
    #[inline]
    fn finish(&self) -> u64 {
        self.buffer ^ self.pad
    }
}

#[cfg(test)]
mod tests {
    use crate::convert::Convert;
    use crate::fallback_hash::*;

    #[test]
    fn test_hash() {
        let mut hasher = AHasher::new_with_keys(0, 0);
        let value: u64 = 1 << 32;
        hasher.update(value);
        let result = hasher.buffer;
        let mut hasher = AHasher::new_with_keys(0, 0);
        let value2: u64 = 1;
        hasher.update(value2);
        let result2 = hasher.buffer;
        let result: [u8; 8] = result.convert();
        let result2: [u8; 8] = result2.convert();
        assert_ne!(hex::encode(result), hex::encode(result2));
    }

    #[test]
    fn test_conversion() {
        let input: &[u8] = "dddddddd".as_bytes();
        let bytes: u64 = as_array!(input, 8).convert();
        assert_eq!(bytes, 0x6464646464646464);
    }
}
<commit_msg>Strengthen fallback<commit_after>use crate::convert::*;
use crate::folded_multiply::*;
use core::hash::Hasher;

///This constant come from Kunth's prng (Empirically it works better than those from splitmix32).
const MULTIPLE: u64 = crate::random_state::MULTIPLE;
const INCREMENT: u64 = 1442695040888963407;
const ROT: u32 = 23; //17

/// A `Hasher` for hashing an arbitrary stream of bytes.
///
/// Instances of [`AHasher`] represent state that is updated while hashing data.
///
/// Each method updates the internal state based on the new data provided. Once
/// all of the data has been provided, the resulting hash can be obtained by calling
/// `finish()`
///
/// [Clone] is also provided in case you wish to calculate hashes for two different items that
/// start with the same data.
///
#[derive(Debug, Clone)]
pub struct AHasher {
    buffer: u64,
    pad: u64,
}

impl AHasher {
    /// Creates a new hasher keyed to the provided key.
    #[inline]
    pub fn new_with_keys(key1: u64, key2: u64) -> AHasher {
        AHasher {
            buffer: key1,
            pad: key2,
        }
    }

    #[cfg(test)]
    pub(crate) fn test_with_keys(key1: u64, key2: u64) -> AHasher {
        use crate::random_state::scramble_keys;
        let (k1, k2) = scramble_keys(key1, key2);
        AHasher { buffer: k1, pad: k2 }
    }

    /// This update function has the goal of updating the buffer with a single multiply
    /// FxHash does this but is vulnerable to attack. To avoid this input needs to be masked to with an
    /// unpredictable value. Other hashes such as murmurhash have taken this approach but were found vulnerable
    /// to attack. The attack was based on the idea of reversing the pre-mixing (Which is necessarily
    /// reversible otherwise bits would be lost) then placing a difference in the highest bit before the
    /// multiply used to mix the data. Because a multiply can never affect the bits to the right of it, a
    /// subsequent update that also differed in this bit could result in a predictable collision.
    ///
    /// This version avoids this vulnerability while still only using a single multiply. It takes advantage
    /// of the fact that when a 64 bit multiply is performed the upper 64 bits are usually computed and thrown
    /// away. Instead it creates two 128 bit values where the upper 64 bits are zeros and multiplies them.
    /// (The compiler is smart enough to turn this into a 64 bit multiplication in the assembly)
    /// Then the upper bits are added to the lower bits to produce a single 64 bit result.
    ///
    /// To understand why this is a good scrambling function it helps to understand multiply-with-carry PRNGs:
    /// https://en.wikipedia.org/wiki/Multiply-with-carry_pseudorandom_number_generator
    /// If the multiple is chosen well, this creates a long period, decent quality PRNG.
    /// Notice that this function is equivalent to this except the `buffer`/`state` is being xored with each
    /// new block of data. In the event that data is all zeros, it is exactly equivalent to a MWC PRNG.
    ///
    /// This is impervious to attack because every bit buffer at the end is dependent on every bit in
    /// `new_data ^ buffer`. For example suppose two inputs differed in only the 5th bit. Then when the
    /// multiplication is performed the `result` will differ in bits 5-69. More specifically it will differ by
    /// 2^5 * MULTIPLE. However in the next step bits 65-128 are turned into a separate 64 bit value. So the
    /// differing bits will be in the lower 6 bits of this value. The two intermediate values that differ in
    /// bits 5-63 and in bits 0-5 respectively get added together. Producing an output that differs in every
    /// bit. The addition carries in the multiplication and at the end additionally mean that the even if an
    /// attacker somehow knew part of (but not all) the contents of the buffer before hand,
    /// they would not be able to predict any of the bits in the buffer at the end.
    #[inline(always)]
    fn update(&mut self, new_data: u64) {
        self.buffer = (new_data ^ self.buffer).folded_multiply(MULTIPLE);
    }

    /// This update function updates the buffer with the new information in a way that can't be canceled
    /// with a subsequent update without knowledge of the content of the buffer prior to the update.
    ///
    /// To achieve this the input needs to be modified in an unpredictable (to an attacker) way before it is
    /// combined with the value in the buffer. This is done by xoring it with `key`.
    ///
    /// Other hashes such as murmurhash have taken that approach but were found vulnerable to attack.
    /// The attack was based on the idea of reversing any pre-mixing (Which is necessarily reversible otherwise
    /// bits would be lost) then placing a difference in the highest bit before the multiply. Because a multiply
    /// can never affect the bits to the right of it, a subsequent update that also only differed in the high
    /// order bit could cancel out the change to `buffer` from the first update. This allowed murmurhash to be
    /// attacked. In this update function aHash avoids this vulnerability by rotating and performing a second
    /// multiply.
    ///
    /// This makes it impossible for an attacker to place a single bit difference between
    /// two blocks so as to cancel each other. (While the transform is still reversible if you know the key)
    ///
    /// This is similar to the above update function but is designed to run in a loop
    /// that will be unrolled and vectorized. So instead of using the buffer, it uses a 'key' that it updates
    /// and returns. The buffer is only xored at the end. This structure is so that when the method is inlined,
    /// the compiler will unroll any loop this gets placed in and the loop can be automatically vectorized
    /// and the rotates, xors, and multiplies can be paralleled.
    ///
    /// The key needs to be incremented between consecutive calls to prevent (a,b) from hashing the same as (b,a).
    /// The adding of the increment is moved to the bottom rather than the top. This allows one less add to be
    /// performed overall, but more importantly, it follows the multiply, which is expensive. So the CPU can
    /// run another operation afterwords if does not depend on the output of the multiply operation.
    #[inline(always)]
    fn ordered_update(&mut self, new_data: u64, key: u64) -> u64 {
        self.buffer ^= (new_data ^ key)
            .wrapping_mul(MULTIPLE)
            .rotate_left(ROT)
            .wrapping_mul(MULTIPLE);
        key.wrapping_add(INCREMENT)
    }
}

/// Provides methods to hash all of the primitive types.
impl Hasher for AHasher {
    #[inline]
    fn write_u8(&mut self, i: u8) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u16(&mut self, i: u16) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u32(&mut self, i: u32) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u64(&mut self, i: u64) {
        self.update(i as u64);
    }

    #[inline]
    fn write_u128(&mut self, i: u128) {
        let data: [u64; 2] = i.convert();
        self.update(data[0]);
        self.update(data[1]);
    }

    #[inline]
    fn write_usize(&mut self, i: usize) {
        self.write_u64(i as u64);
    }

    #[inline]
    fn write(&mut self, input: &[u8]) {
        let mut data = input;
        let length = data.len() as u64;
        //Needs to be an add rather than an xor because otherwise it could be canceled with carefully formed input.
        self.buffer = self.buffer.wrapping_add(length.wrapping_mul(MULTIPLE));
        //A 'binary search' on sizes reduces the number of comparisons.
        if data.len() > 8 {
            if data.len() > 16 {
                let tail = data.read_last_u64();
                let mut key: u64 = self.buffer;
                while data.len() > 8 {
                    let (val, rest) = data.read_u64();
                    key = self.ordered_update(val, key);
                    data = rest;
                }
                self.update(tail);
            } else {
                self.update(data.read_u64().0);
                self.update(data.read_last_u64());
            }
        } else {
            if data.len() >= 2 {
                if data.len() >= 4 {
                    let block: [u32; 2] = [data.read_u32().0, data.read_last_u32()];
                    self.update(block.convert());
                } else {
                    let value = [data.read_u16().0 as u32, data[data.len() - 1] as u32];
                    self.update(value.convert());
                }
            } else {
                let value = if data.len() > 0 {
                    data[0] //len 1
                } else {
                    0
                };
                self.update(value as u64);
            }
        }
    }
    #[inline]
    fn finish(&self) -> u64 {
        let rot = (self.pad & 63) as u32;
        (self.buffer ^ self.pad).rotate_left(rot)
    }
}

#[cfg(test)]
mod tests {
    use crate::convert::Convert;
    use crate::fallback_hash::*;

    #[test]
    fn test_hash() {
        let mut hasher = AHasher::new_with_keys(0, 0);
        let value: u64 = 1 << 32;
        hasher.update(value);
        let result = hasher.buffer;
        let mut hasher = AHasher::new_with_keys(0, 0);
        let value2: u64 = 1;
        hasher.update(value2);
        let result2 = hasher.buffer;
        let result: [u8; 8] = result.convert();
        let result2: [u8; 8] = result2.convert();
        assert_ne!(hex::encode(result), hex::encode(result2));
    }

    #[test]
    fn test_conversion() {
        let input: &[u8] = "dddddddd".as_bytes();
        let bytes: u64 = as_array!(input, 8).convert();
        assert_eq!(bytes, 0x6464646464646464);
    }
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::time::Instant;

fn main() {
    let uptime = Instant::now().inner().as_secs();

    let mut width = 0;
    let mut height = 0;
    if let Ok(display) = File::open("display:") {
        let path = display.path().map(|path| path.into_os_string().into_string().unwrap_or(String::new())).unwrap_or(String::new());
        let res = path.split(":").nth(1).unwrap_or("");
        width = res.split("/").nth(0).unwrap_or("").parse::<i32>().unwrap_or(0);
        height = res.split("/").nth(1).unwrap_or("").parse::<i32>().unwrap_or(0);
    }

    println!("\x1B[38;5;75m                `.-/+NMN+-.`                   \x1B[0m\x1B[38;5;75mroot\x1B[0m@\x1B[38;5;75mhostname\x1B[0m");
    println!("\x1B[38;5;75m           `:+oo+/-.-yds--/+oo+:`              \x1B[0m\x1B[38;5;75mOS:\x1B[0m redox-os");
    println!("\x1B[38;5;75m        `/ss/++::/+o++++o+/:```:ss/`           \x1B[0m\x1B[38;5;75mKernel:\x1B[0m redox");
    println!("\x1B[38;5;75m        `/ss/++::/+o++++o+/:```:ss/`           \x1B[0m\x1B[38;5;75mUptime:\x1B[0m {}s", uptime);
    println!("\x1B[38;5;75m      `+h+``oMMN+.````````.:+syyy:/h+`         \x1B[0m\x1B[38;5;75mShell:\x1B[0m ion");
    println!("\x1B[38;5;75m     /h/+mmm/://:+oo+//+oo+:. hNNh.`/h/        \x1B[0m\x1B[38;5;75mResolution:\x1B[0m {}x{}", width, height);
    println!("\x1B[38;5;75m    oy` ydds`/s+:`        `:+s/-.+Ndd-so       \x1B[0m\x1B[38;5;75mDE:\x1B[0m orbital");
    println!("\x1B[38;5;75m   os `yo  /y:                :y/.dmM- so      \x1B[0m\x1B[38;5;75mWM:\x1B[0m orbital");
    println!("\x1B[38;5;75m  :h  s+  os`    \x1B[0msmhhhyyy/\x1B[38;5;75m     `so  +s  h:     \x1B[0m\x1B[38;5;75mFont:\x1B[0m unifont");
    println!("\x1B[38;5;75m  m. -h  /h      \x1B[0myM    .oM+\x1B[38;5;75m      h/  h- .m     \x1B[0m");
    println!("\x1B[38;5;75m  N  s+  d.      \x1B[0myM     -Ms\x1B[38;5;75m      .d  +s  m     \x1B[0m");
    println!("\x1B[38;5;75m  h  y/  M       \x1B[0myM :+sydy`\x1B[38;5;75m       M  /y  h     \x1B[0m");
    println!("\x1B[38;5;75m  M  oo  y/      \x1B[0myM .yNy.  \x1B[38;5;75m      /y  oo  M     \x1B[0m");
    println!("\x1B[38;5;75m  y/ `m` .d.     \x1B[0myM   :md- \x1B[38;5;75m     .d.:hNy /y     \x1B[0m");
    println!("\x1B[38;5;75m  .d` :h:--h:    \x1B[0m+s    `ss`\x1B[38;5;75m    :h- oMNh`d.     \x1B[0m");
    println!("\x1B[38;5;75m   :d.-MMN:.oo:              :oo.+sd+..d:      \x1B[0m");
    println!("\x1B[38;5;75m    -d//oyy////so/:oyo..ydhos/. +MMM::d-       \x1B[0m");
    println!("\x1B[38;5;75m     `sy- yMMN. `./MMMo+dNm/ ./ss-./ys`        \x1B[0m");
    println!("\x1B[38;5;75m       .ss/++:+oo+//:-..:+ooo+-``:ss.          \x1B[0m");
    println!("\x1B[38;5;75m         `:ss/-` `.--::--.` `-/ss:`            \x1B[0m");
    println!("\x1B[38;5;75m             ./oooooooooooooo/.                \x1B[0m");
}
<commit_msg>Formatting<commit_after>use std::fs::File;
use std::time::Instant;

fn main() {
    let uptime = Instant::now().inner().as_secs();

    let mut width = 0;
    let mut height = 0;
    if let Ok(display) = File::open("display:") {
        let path = display.path().map(|path| path.into_os_string().into_string().unwrap_or(String::new())).unwrap_or(String::new());
        let res = path.split(":").nth(1).unwrap_or("");
        width = res.split("/").nth(0).unwrap_or("").parse::<i32>().unwrap_or(0);
        height = res.split("/").nth(1).unwrap_or("").parse::<i32>().unwrap_or(0);
    }

    println!("\x1B[38;5;75m                `.-/+NMN+-.`                   \x1B[0m\x1B[38;5;75mroot\x1B[0m@\x1B[38;5;75mhostname\x1B[0m");
    println!("\x1B[38;5;75m           `:+oo+/-.-yds--/+oo+:`              \x1B[0m\x1B[38;5;75mOS:\x1B[0m redox-os");
    println!("\x1B[38;5;75m        `/ss/++::/+o++++o+/:```:ss/`           \x1B[0m\x1B[38;5;75mKernel:\x1B[0m redox");
    println!("\x1B[38;5;75m        `/ss/++::/+o++++o+/:```:ss/`           \x1B[0m\x1B[38;5;75mUptime:\x1B[0m {}s", uptime);
    println!("\x1B[38;5;75m      `+h+``oMMN+.````````.:+syyy:/h+`         \x1B[0m\x1B[38;5;75mShell:\x1B[0m ion");
    println!("\x1B[38;5;75m     /h/+mmm/://:+oo+//+oo+:. hNNh.`/h/        \x1B[0m\x1B[38;5;75mResolution:\x1B[0m {}x{}", width, height);
    println!("\x1B[38;5;75m    oy` ydds`/s+:`        `:+s/-.+Ndd-so       \x1B[0m\x1B[38;5;75mDE:\x1B[0m orbital");
    println!("\x1B[38;5;75m   os `yo  /y:                :y/.dmM- so      \x1B[0m\x1B[38;5;75mWM:\x1B[0m orbital");
    println!("\x1B[38;5;75m  :h  s+  os`   \x1B[0m smhhhyyy/  \x1B[38;5;75m   `so  +s  h:     \x1B[0m\x1B[38;5;75mFont:\x1B[0m unifont");
    println!("\x1B[38;5;75m  m. -h  /h     \x1B[0m yM    .oM+ \x1B[38;5;75m     h/  h- .m     \x1B[0m");
    println!("\x1B[38;5;75m  N  s+  d.     \x1B[0m yM     -Ms \x1B[38;5;75m     .d  +s  m     \x1B[0m");
    println!("\x1B[38;5;75m  h  y/  M      \x1B[0m yM :+sydy` \x1B[38;5;75m      M  /y  h     \x1B[0m");
    println!("\x1B[38;5;75m  M  oo  y/     \x1B[0m yM .yNy.   \x1B[38;5;75m     /y  oo  M     \x1B[0m");
    println!("\x1B[38;5;75m  y/ `m` .d.    \x1B[0m yM   :md-  \x1B[38;5;75m    .d.:hNy /y     \x1B[0m");
    println!("\x1B[38;5;75m  .d` :h:--h:   \x1B[0m +s    `ss` \x1B[38;5;75m   :h- oMNh`d.     \x1B[0m");
    println!("\x1B[38;5;75m   :d.-MMN:.oo:              :oo.+sd+..d:      \x1B[0m");
    println!("\x1B[38;5;75m    -d//oyy////so/:oyo..ydhos/. +MMM::d-       \x1B[0m");
    println!("\x1B[38;5;75m     `sy- yMMN. `./MMMo+dNm/ ./ss-./ys`        \x1B[0m");
    println!("\x1B[38;5;75m       .ss/++:+oo+//:-..:+ooo+-``:ss.          \x1B[0m");
    println!("\x1B[38;5;75m         `:ss/-` `.--::--.` `-/ss:`            \x1B[0m");
    println!("\x1B[38;5;75m             ./oooooooooooooo/.                \x1B[0m");
}
<|endoftext|>


<commit_before>use std::sync::Arc;
use std::thread;
use std::time::Duration;

use parking_lot::{Condvar, Mutex};

use super::*;

#[derive(Debug, Clone, Copy)]
pub(crate) enum ShutdownState {
    Running,
    ShuttingDown,
    ShutDown,
}

impl ShutdownState {
    fn is_running(self) -> bool {
        if let ShutdownState::Running = self {
            true
        } else {
            false
        }
    }

    fn is_shutdown(self) -> bool {
        if let ShutdownState::ShutDown = self {
            true
        } else {
            false
        }
    }
}

#[derive(Debug)]
pub(crate) struct Flusher {
    shutdown: Arc<Mutex<ShutdownState>>,
    sc: Arc<Condvar>,
    join_handle: Mutex<Option<std::thread::JoinHandle<()>>>,
}

impl Flusher {
    /// Spawns a thread that periodically calls `callback` until dropped.
    pub(crate) fn new(
        name: String,
        pagecache: Arc<PageCache<Frag>>,
        flush_every_ms: u64,
    ) -> Self {
        #[allow(clippy::mutex_atomic)] // mutex used in CondVar below
        let shutdown = Arc::new(Mutex::new(ShutdownState::Running));
        let sc = Arc::new(Condvar::new());

        let join_handle = thread::Builder::new()
            .name(name)
            .spawn({
                let shutdown = shutdown.clone();
                let sc = sc.clone();
                move || run(shutdown, sc, pagecache, flush_every_ms)
            })
            .unwrap();

        Self {
            shutdown,
            sc,
            join_handle: Mutex::new(Some(join_handle)),
        }
    }
}

fn run(
    shutdown: Arc<Mutex<ShutdownState>>,
    sc: Arc<Condvar>,
    pagecache: Arc<PageCache<Frag>>,
    flush_every_ms: u64,
) {
    let flush_every = Duration::from_millis(flush_every_ms);
    let mut shutdown = shutdown.lock();
    let mut wrote_data = false;
    while shutdown.is_running() || wrote_data {
        let before = std::time::Instant::now();
        match pagecache.flush() {
            Ok(0) => {
                wrote_data = false;
                if !shutdown.is_running() {
                    break;
                }
                // we had no dirty data to flush,
                // so we can spend a little effort
                // cleaning up the file. try not to
                // spend more than half of our sleep
                // time rewriting pages though.
                while before.elapsed() < flush_every / 2 {
                    match pagecache.attempt_gc() {
                        Err(e) => {
                            error!(
                                "failed to clean file from async flush thread: {}",
                                e
                            );

                            #[cfg(feature = "failpoints")]
                            pagecache.set_failpoint(e);

                            *shutdown = ShutdownState::ShutDown;
                            sc.notify_all();
                            return;
                        }
                        Ok(false) => break,
                        Ok(true) => {}
                    }
                }
            }
            Ok(_) => {
                wrote_data = true;
                // at some point, we may want to
                // put adaptive logic here to tune
                // sleeps based on how much work
                // we accomplished
            }
            Err(e) => {
                error!("failed to flush from periodic flush thread: {}", e);

                #[cfg(feature = "failpoints")]
                pagecache.set_failpoint(e);

                *shutdown = ShutdownState::ShutDown;
                sc.notify_all();
                return;
            }
        }

        let sleep_duration = flush_every
            .checked_sub(before.elapsed())
            .unwrap_or(Duration::from_millis(1));

        sc.wait_for(&mut shutdown, sleep_duration);
    }
    *shutdown = ShutdownState::ShutDown;
    sc.notify_all();
}

impl Drop for Flusher {
    fn drop(&mut self) {
        let mut shutdown = self.shutdown.lock();
        if shutdown.is_running() {
            *shutdown = ShutdownState::ShuttingDown;
            self.sc.notify_all();
        }

        while !shutdown.is_shutdown() {
            self.sc.wait_for(&mut shutdown, Duration::from_millis(100));
        }

        let mut join_handle_opt = self.join_handle.lock();
        if let Some(join_handle) = join_handle_opt.take() {
            if let Err(e) = join_handle.join() {
                error!("error joining Periodic thread: {:?}", e);
            }
        }
    }
}
<commit_msg>More aggressively gc segments<commit_after>use std::sync::Arc;
use std::thread;
use std::time::Duration;

use parking_lot::{Condvar, Mutex};

use super::*;

#[derive(Debug, Clone, Copy)]
pub(crate) enum ShutdownState {
    Running,
    ShuttingDown,
    ShutDown,
}

impl ShutdownState {
    fn is_running(self) -> bool {
        if let ShutdownState::Running = self {
            true
        } else {
            false
        }
    }

    fn is_shutdown(self) -> bool {
        if let ShutdownState::ShutDown = self {
            true
        } else {
            false
        }
    }
}

#[derive(Debug)]
pub(crate) struct Flusher {
    shutdown: Arc<Mutex<ShutdownState>>,
    sc: Arc<Condvar>,
    join_handle: Mutex<Option<std::thread::JoinHandle<()>>>,
}

impl Flusher {
    /// Spawns a thread that periodically calls `callback` until dropped.
    pub(crate) fn new(
        name: String,
        pagecache: Arc<PageCache<Frag>>,
        flush_every_ms: u64,
    ) -> Self {
        #[allow(clippy::mutex_atomic)] // mutex used in CondVar below
        let shutdown = Arc::new(Mutex::new(ShutdownState::Running));
        let sc = Arc::new(Condvar::new());

        let join_handle = thread::Builder::new()
            .name(name)
            .spawn({
                let shutdown = shutdown.clone();
                let sc = sc.clone();
                move || run(shutdown, sc, pagecache, flush_every_ms)
            })
            .unwrap();

        Self {
            shutdown,
            sc,
            join_handle: Mutex::new(Some(join_handle)),
        }
    }
}

fn run(
    shutdown: Arc<Mutex<ShutdownState>>,
    sc: Arc<Condvar>,
    pagecache: Arc<PageCache<Frag>>,
    flush_every_ms: u64,
) {
    let flush_every = Duration::from_millis(flush_every_ms);
    let mut shutdown = shutdown.lock();
    let mut wrote_data = false;
    while shutdown.is_running() || wrote_data {
        let before = std::time::Instant::now();
        match pagecache.flush() {
            Ok(0) => {
                wrote_data = false;
                if !shutdown.is_running() {
                    break;
                }
            }
            Ok(_) => {
                wrote_data = true;
                // at some point, we may want to
                // put adaptive logic here to tune
                // sleeps based on how much work
                // we accomplished
            }
            Err(e) => {
                error!("failed to flush from periodic flush thread: {}", e);

                #[cfg(feature = "failpoints")]
                pagecache.set_failpoint(e);

                *shutdown = ShutdownState::ShutDown;
                sc.notify_all();
                return;
            }
        }

        // so we can spend a little effort
        // cleaning up the segments. try not to
        // spend more than half of our sleep
        // time rewriting pages though.
        while shutdown.is_running() && before.elapsed() < flush_every / 2 {
            match pagecache.attempt_gc() {
                Err(e) => {
                    error!(
                        "failed to clean file from periodic flush thread: {}",
                        e
                    );

                    #[cfg(feature = "failpoints")]
                    pagecache.set_failpoint(e);

                    *shutdown = ShutdownState::ShutDown;
                    sc.notify_all();
                    return;
                }
                Ok(false) => break,
                Ok(true) => {}
            }
        }

        let sleep_duration = flush_every
            .checked_sub(before.elapsed())
            .unwrap_or(Duration::from_millis(1));

        sc.wait_for(&mut shutdown, sleep_duration);
    }
    *shutdown = ShutdownState::ShutDown;
    sc.notify_all();
}

impl Drop for Flusher {
    fn drop(&mut self) {
        let mut shutdown = self.shutdown.lock();
        if shutdown.is_running() {
            *shutdown = ShutdownState::ShuttingDown;
            self.sc.notify_all();
        }

        while !shutdown.is_shutdown() {
            self.sc.wait_for(&mut shutdown, Duration::from_millis(100));
        }

        let mut join_handle_opt = self.join_handle.lock();
        if let Some(join_handle) = join_handle_opt.take() {
            if let Err(e) = join_handle.join() {
                error!("error joining Periodic thread: {:?}", e);
            }
        }
    }
}
<|endoftext|>


<commit_before>/// The meaning of GPS time in the point records.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum GpsTimeType {
    /// GPS Week Time (the same as previous versions of LAS).
    Week,
    /// Standard GPS Time minus 1e9.
    Standard,
}

impl GpsTimeType {
    /// Returns true if this time type is gps standard time.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::GpsTimeType;
    /// assert!(!GpsTimeType::Week.is_standard());
    /// assert!(GpsTimeType::Standard.is_standard());
    /// ```
    pub fn is_standard(&self) -> bool {
        match *self {
            GpsTimeType::Week => false,
            GpsTimeType::Standard => true,
        }
    }
}

impl From<GpsTimeType> for u16 {
    fn from(gps_time_type: GpsTimeType) -> u16 {
        match gps_time_type {
            GpsTimeType::Week => 0,
            GpsTimeType::Standard => 1,
        }
    }
}

impl From<u16> for GpsTimeType {
    fn from(n: u16) -> GpsTimeType {
        match n & 1 {
            0 => GpsTimeType::Week,
            1 => GpsTimeType::Standard,
            _ => unreachable!(),
        }
    }
}
<commit_msg>Add default impl for GpsTimeType<commit_after>/// The meaning of GPS time in the point records.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum GpsTimeType {
    /// GPS Week Time (the same as previous versions of LAS).
    Week,
    /// Standard GPS Time minus 1e9.
    Standard,
}

impl GpsTimeType {
    /// Returns true if this time type is gps standard time.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::GpsTimeType;
    /// assert!(!GpsTimeType::Week.is_standard());
    /// assert!(GpsTimeType::Standard.is_standard());
    /// ```
    pub fn is_standard(&self) -> bool {
        match *self {
            GpsTimeType::Week => false,
            GpsTimeType::Standard => true,
        }
    }
}

impl From<GpsTimeType> for u16 {
    fn from(gps_time_type: GpsTimeType) -> u16 {
        match gps_time_type {
            GpsTimeType::Week => 0,
            GpsTimeType::Standard => 1,
        }
    }
}

impl From<u16> for GpsTimeType {
    fn from(n: u16) -> GpsTimeType {
        match n & 1 {
            0 => GpsTimeType::Week,
            1 => GpsTimeType::Standard,
            _ => unreachable!(),
        }
    }
}

impl Default for GpsTimeType {
    fn default() -> GpsTimeType {
        GpsTimeType::Week
    }
}
<|endoftext|>


<commit_before>mod statistics;
mod planner;

use kite::query::Query;
use kite::collectors::{Collector, DocumentMatch};
use byteorder::{ByteOrder, BigEndian};

use super::RocksDBIndexReader;
use doc_id_set::DocIdSet;
use segment::{Segment, SegmentReadError};
use search::statistics::{StatisticsReader, RocksDBStatisticsReader};
use search::planner::{SearchPlan, plan_query};
use search::planner::boolean_query::BooleanQueryOp;
use search::planner::score_function::{CombinatorScorer, ScoreFunctionOp};

impl<'a> RocksDBIndexReader<'a> {
    fn search_segment_boolean_phase<S: Segment>(&self, boolean_query: &Vec<BooleanQueryOp>, is_negated: bool, segment: &S) -> Result<DocIdSet, SegmentReadError> {
        // Execute boolean query
        let mut stack = Vec::new();
        for op in boolean_query.iter() {
            match *op {
                BooleanQueryOp::PushEmpty => {
                    stack.push(DocIdSet::new_filled(0));
                }
                BooleanQueryOp::PushFull => {
                    stack.push(DocIdSet::new_filled(65536));
                }
                BooleanQueryOp::PushTermDirectory(field_ref, term_ref) => {
                    match try!(segment.load_term_directory(field_ref, term_ref)) {
                        Some(doc_id_set) => stack.push(doc_id_set),
                        None => stack.push(DocIdSet::new_filled(0)),
                    }
                }
                BooleanQueryOp::PushDeletionList => {
                     match try!(segment.load_deletion_list()) {
                        Some(doc_id_set) => stack.push(doc_id_set),
                        None => stack.push(DocIdSet::new_filled(0)),
                    }
                }
                BooleanQueryOp::And => {
                    let b = stack.pop().expect("boolean query executor: stack underflow");
                    let a = stack.pop().expect("boolean query executor: stack underflow");
                    stack.push(a.intersection(&b));
                }
                BooleanQueryOp::Or => {
                    let b = stack.pop().expect("boolean query executor: stack underflow");
                    let a = stack.pop().expect("boolean query executor: stack underflow");
                    stack.push(a.union(&b));
                }
                BooleanQueryOp::AndNot => {
                    let b = stack.pop().expect("boolean query executor: stack underflow");
                    let a = stack.pop().expect("boolean query executor: stack underflow");
                    stack.push(a.exclusion(&b));
                }
            }
        }

        if !stack.len() == 1 {
            // This shouldn't be possible unless there's a bug in the planner
            panic!("boolean query executor: stack size too big ({})", stack.len());
        }
        let mut matches = stack.pop().unwrap();

        // Invert the list if the query is negated
        if is_negated {
            let total_docs = try!(segment.load_statistic(b"total_docs")).unwrap_or(0);
            let all_docs = DocIdSet::new_filled(total_docs as u32);
            matches = all_docs.exclusion(&matches);
        }

        Ok(matches)
    }

    fn score_doc<S: Segment, R: StatisticsReader>(&self, doc_id: u16, score_function: &Vec<ScoreFunctionOp>, segment: &S, mut stats: &mut R) -> Result<f64, SegmentReadError> {
        // Execute score function
        let mut stack = Vec::new();
        for op in score_function.iter() {
            match *op {
                ScoreFunctionOp::Literal(val) => stack.push(val),
                ScoreFunctionOp::TermScorer(field_ref, term_ref, ref scorer) => {
                    // TODO: Check this isn't really slow
                    match try!(segment.load_term_directory(field_ref, term_ref)) {
                        Some(doc_id_set) => {
                            if doc_id_set.contains_doc(doc_id) {
                                // Read field length
                                // TODO: we only need this for BM25
                                let field_length_raw = try!(segment.load_stored_field_value_raw(doc_id, field_ref, b"len"));
                                let field_length = match field_length_raw {
                                    Some(value) => {
                                        let length_sqrt = (value[0] as f64) / 3.0 + 1.0;
                                        length_sqrt * length_sqrt
                                    }
                                    None => 1.0
                                };

                                // Read term frequency
                                let mut value_type = vec![b't', b'f'];
                                value_type.extend(term_ref.ord().to_string().as_bytes());
                                let term_frequency_raw = try!(segment.load_stored_field_value_raw(doc_id, field_ref, &value_type));
                                let term_frequency = match term_frequency_raw {
                                    Some(value) => BigEndian::read_i64(&value),
                                    None => 1,
                                };

                                let score = scorer.similarity_model.score(term_frequency as u32, field_length, try!(stats.total_tokens(field_ref)) as u64, try!(stats.total_docs(field_ref)) as u64, try!(stats.term_document_frequency(field_ref, term_ref)) as u64);
                                stack.push(score * scorer.boost);
                            } else {
                                stack.push(0.0f64);
                            }
                        }
                        None => stack.push(0.0f64),
                    }
                }
                ScoreFunctionOp::CombinatorScorer(num_vals, ref scorer) => {
                    let score = match *scorer {
                        CombinatorScorer::Avg => {
                            let mut total_score = 0.0f64;

                            for _ in 0..num_vals {
                                total_score += stack.pop().expect("document scorer: stack underflow");
                            }

                            total_score / num_vals as f64
                        }
                        CombinatorScorer::Max => {
                            let mut max_score = 0.0f64;

                            for _ in 0..num_vals {
                                let score = stack.pop().expect("document scorer: stack underflow");
                                if score > max_score {
                                    max_score = score
                                }
                            }

                            max_score
                        }
                    };

                    stack.push(score);
                }
            }
        }

        if !stack.len() == 1 {
            // This shouldn't be possible unless there's a bug in the planner
            panic!("document scorer: stack size too big ({})", stack.len());
        }

        Ok(stack.pop().expect("document scorer: stack underflow"))
    }

    fn search_segment<C: Collector, S: Segment, R: StatisticsReader>(&self, collector: &mut C, plan: &SearchPlan, segment: &S, mut stats: &mut R) -> Result<(), SegmentReadError> {
        let matches = try!(self.search_segment_boolean_phase(&plan.boolean_query, plan.boolean_query_is_negated, segment));

        // Score documents and pass to collector
        for doc in matches.iter() {
            let score = try!(self.score_doc(doc, &plan.score_function, segment, stats));

            let doc_ref = segment.doc_ref(doc);
            let doc_match = DocumentMatch::new_scored(doc_ref.as_u64(), score);
            collector.collect(doc_match);
        }

        Ok(())
    }

    pub fn search<C: Collector>(&self, collector: &mut C, query: &Query) -> Result<(), SegmentReadError> {
        // Plan query
        let plan = plan_query(&self, query, collector.needs_score());

        // Initialise statistics reader
        let mut stats = RocksDBStatisticsReader::new(&self);

        // Run query on each segment
        for segment in self.store.segments.iter_active(&self) {
            try!(self.search_segment(collector, &plan, &segment, &mut stats));
        }

        Ok(())
    }
}
<commit_msg>Moved generic search code out of RocksDBIndexReader<commit_after>mod statistics;
mod planner;

use kite::query::Query;
use kite::collectors::{Collector, DocumentMatch};
use byteorder::{ByteOrder, BigEndian};

use super::RocksDBIndexReader;
use doc_id_set::DocIdSet;
use segment::{Segment, SegmentReadError};
use search::statistics::{StatisticsReader, RocksDBStatisticsReader};
use search::planner::{SearchPlan, plan_query};
use search::planner::boolean_query::BooleanQueryOp;
use search::planner::score_function::{CombinatorScorer, ScoreFunctionOp};

fn search_segment_boolean_phase<S: Segment>(boolean_query: &Vec<BooleanQueryOp>, is_negated: bool, segment: &S) -> Result<DocIdSet, SegmentReadError> {
    // Execute boolean query
    let mut stack = Vec::new();
    for op in boolean_query.iter() {
        match *op {
            BooleanQueryOp::PushEmpty => {
                stack.push(DocIdSet::new_filled(0));
            }
            BooleanQueryOp::PushFull => {
                stack.push(DocIdSet::new_filled(65536));
            }
            BooleanQueryOp::PushTermDirectory(field_ref, term_ref) => {
                match try!(segment.load_term_directory(field_ref, term_ref)) {
                    Some(doc_id_set) => stack.push(doc_id_set),
                    None => stack.push(DocIdSet::new_filled(0)),
                }
            }
            BooleanQueryOp::PushDeletionList => {
                    match try!(segment.load_deletion_list()) {
                    Some(doc_id_set) => stack.push(doc_id_set),
                    None => stack.push(DocIdSet::new_filled(0)),
                }
            }
            BooleanQueryOp::And => {
                let b = stack.pop().expect("boolean query executor: stack underflow");
                let a = stack.pop().expect("boolean query executor: stack underflow");
                stack.push(a.intersection(&b));
            }
            BooleanQueryOp::Or => {
                let b = stack.pop().expect("boolean query executor: stack underflow");
                let a = stack.pop().expect("boolean query executor: stack underflow");
                stack.push(a.union(&b));
            }
            BooleanQueryOp::AndNot => {
                let b = stack.pop().expect("boolean query executor: stack underflow");
                let a = stack.pop().expect("boolean query executor: stack underflow");
                stack.push(a.exclusion(&b));
            }
        }
    }

    if !stack.len() == 1 {
        // This shouldn't be possible unless there's a bug in the planner
        panic!("boolean query executor: stack size too big ({})", stack.len());
    }
    let mut matches = stack.pop().unwrap();

    // Invert the list if the query is negated
    if is_negated {
        let total_docs = try!(segment.load_statistic(b"total_docs")).unwrap_or(0);
        let all_docs = DocIdSet::new_filled(total_docs as u32);
        matches = all_docs.exclusion(&matches);
    }

    Ok(matches)
}

fn score_doc<S: Segment, R: StatisticsReader>(doc_id: u16, score_function: &Vec<ScoreFunctionOp>, segment: &S, mut stats: &mut R) -> Result<f64, SegmentReadError> {
    // Execute score function
    let mut stack = Vec::new();
    for op in score_function.iter() {
        match *op {
            ScoreFunctionOp::Literal(val) => stack.push(val),
            ScoreFunctionOp::TermScorer(field_ref, term_ref, ref scorer) => {
                // TODO: Check this isn't really slow
                match try!(segment.load_term_directory(field_ref, term_ref)) {
                    Some(doc_id_set) => {
                        if doc_id_set.contains_doc(doc_id) {
                            // Read field length
                            // TODO: we only need this for BM25
                            let field_length_raw = try!(segment.load_stored_field_value_raw(doc_id, field_ref, b"len"));
                            let field_length = match field_length_raw {
                                Some(value) => {
                                    let length_sqrt = (value[0] as f64) / 3.0 + 1.0;
                                    length_sqrt * length_sqrt
                                }
                                None => 1.0
                            };

                            // Read term frequency
                            let mut value_type = vec![b't', b'f'];
                            value_type.extend(term_ref.ord().to_string().as_bytes());
                            let term_frequency_raw = try!(segment.load_stored_field_value_raw(doc_id, field_ref, &value_type));
                            let term_frequency = match term_frequency_raw {
                                Some(value) => BigEndian::read_i64(&value),
                                None => 1,
                            };

                            let score = scorer.similarity_model.score(term_frequency as u32, field_length, try!(stats.total_tokens(field_ref)) as u64, try!(stats.total_docs(field_ref)) as u64, try!(stats.term_document_frequency(field_ref, term_ref)) as u64);
                            stack.push(score * scorer.boost);
                        } else {
                            stack.push(0.0f64);
                        }
                    }
                    None => stack.push(0.0f64),
                }
            }
            ScoreFunctionOp::CombinatorScorer(num_vals, ref scorer) => {
                let score = match *scorer {
                    CombinatorScorer::Avg => {
                        let mut total_score = 0.0f64;

                        for _ in 0..num_vals {
                            total_score += stack.pop().expect("document scorer: stack underflow");
                        }

                        total_score / num_vals as f64
                    }
                    CombinatorScorer::Max => {
                        let mut max_score = 0.0f64;

                        for _ in 0..num_vals {
                            let score = stack.pop().expect("document scorer: stack underflow");
                            if score > max_score {
                                max_score = score
                            }
                        }

                        max_score
                    }
                };

                stack.push(score);
            }
        }
    }

    if !stack.len() == 1 {
        // This shouldn't be possible unless there's a bug in the planner
        panic!("document scorer: stack size too big ({})", stack.len());
    }

    Ok(stack.pop().expect("document scorer: stack underflow"))
}

fn search_segment<C: Collector, S: Segment, R: StatisticsReader>(collector: &mut C, plan: &SearchPlan, segment: &S, mut stats: &mut R) -> Result<(), SegmentReadError> {
    let matches = try!(search_segment_boolean_phase(&plan.boolean_query, plan.boolean_query_is_negated, segment));

    // Score documents and pass to collector
    for doc in matches.iter() {
        let score = try!(score_doc(doc, &plan.score_function, segment, stats));

        let doc_ref = segment.doc_ref(doc);
        let doc_match = DocumentMatch::new_scored(doc_ref.as_u64(), score);
        collector.collect(doc_match);
    }

    Ok(())
}

impl<'a> RocksDBIndexReader<'a> {
    pub fn search<C: Collector>(&self, collector: &mut C, query: &Query) -> Result<(), SegmentReadError> {
        // Plan query
        let plan = plan_query(&self, query, collector.needs_score());

        // Initialise statistics reader
        let mut stats = RocksDBStatisticsReader::new(&self);

        // Run query on each segment
        for segment in self.store.segments.iter_active(&self) {
            try!(search_segment(collector, &plan, &segment, &mut stats));
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>#![crate_name = "input"]
#![deny(missing_docs)]
#![deny(missing_copy_implementations)]

//! A flexible structure for user interactions
//! to be used in window frameworks and widgets libraries.

#[macro_use]
extern crate bitflags;
extern crate rustc_serialize;
extern crate viewport;

use std::any::Any;
use std::sync::Arc;

pub use mouse::MouseButton;
pub use keyboard::Key;
pub use controller::{ ControllerAxisArgs, ControllerButton };

pub mod controller;
pub mod keyboard;
pub mod mouse;

pub use after_render::{ AfterRenderArgs, AfterRenderEvent };
pub use controller::{ ControllerAxisEvent };
pub use cursor::CursorEvent;
pub use focus::FocusEvent;
pub use generic_event::GenericEvent;
pub use idle::{ IdleArgs, IdleEvent };
pub use mouse::{ MouseCursorEvent, MouseRelativeEvent, MouseScrollEvent };
pub use press::PressEvent;
pub use release::ReleaseEvent;
pub use resize::ResizeEvent;
pub use render::{ RenderArgs, RenderEvent };
pub use text::TextEvent;
pub use touch::{ Touch, TouchArgs, TouchEvent };
pub use update::{ UpdateArgs, UpdateEvent };

pub mod generic_event;

mod after_render;
mod cursor;
mod focus;
mod idle;
mod press;
mod release;
mod render;
mod resize;
mod text;
mod touch;
mod update;

/// Used to identify events arguments provided by traits.
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct EventId(pub &'static str);

const AFTER_RENDER: EventId = EventId("piston/after_render");
const CONTROLLER_AXIS: EventId = EventId("piston/controller_axis");
const CURSOR: EventId = EventId("piston/cursor");
const FOCUS: EventId = EventId("piston/focus");
const CLOSE: EventId = EventId("piston/close");
const IDLE: EventId = EventId("piston/idle");
const MOUSE_SCROLL: EventId = EventId("piston/mouse_scroll");
const MOUSE_RELATIVE: EventId = EventId("piston/mouse_relative");
const MOUSE_CURSOR: EventId = EventId("piston/mouse_cursor");
const PRESS: EventId = EventId("piston/press");
const RELEASE: EventId = EventId("piston/release");
const RENDER: EventId = EventId("piston/render");
const RESIZE: EventId = EventId("piston/resize");
const TEXT: EventId = EventId("piston/text");
const TOUCH: EventId = EventId("piston/touch");
const UPDATE: EventId = EventId("piston/update");

/// Models different kinds of buttons.
#[derive(Copy, Clone, RustcDecodable, RustcEncodable, PartialEq, Eq, Hash, Debug)]
pub enum Button {
    /// A keyboard button.
    Keyboard(Key),
    /// A mouse button.
    Mouse(MouseButton),
    /// A controller button.
    Controller(ControllerButton),
}

/// Models different kinds of motion.
#[derive(Copy, Clone, RustcDecodable, RustcEncodable, PartialEq, Debug)]
pub enum Motion {
    /// x and y in window coordinates.
    MouseCursor(f64, f64),
    /// x and y in relative coordinates.
    MouseRelative(f64, f64),
    /// x and y in scroll ticks.
    MouseScroll(f64, f64),
    /// controller axis move event.
    ControllerAxis(ControllerAxisArgs),
    /// touch event.
    Touch(TouchArgs),
}

/// Models input events.
#[derive(Clone, Debug)]
pub enum Input {
    /// Pressed a button.
    Press(Button),
    /// Released a button.
    Release(Button),
    /// Moved mouse cursor.
    Move(Motion),
    /// Text (usually from keyboard).
    Text(String),
    /// Window got resized.
    Resize(u32, u32),
    /// Window gained or lost focus.
    Focus(bool),
    /// Window gained or lost cursor.
    Cursor(bool),
    /// Window closed.
    Close,
    /// Render graphics.
    Render(RenderArgs),
    /// After rendering and swapping buffers.
    AfterRender(AfterRenderArgs),
    /// Update the state of the application.
    Update(UpdateArgs),
    /// Do background tasks that can be done incrementally.
    Idle(IdleArgs),
    /// Custom event.
    ///
    /// When comparing two custom events for equality,
    /// they always return `false`.
    Custom(EventId, Arc<Any>),
}

impl PartialEq for Input {
    fn eq(&self, other: &Input) -> bool {
        use Input::*;

        match (self, other) {
            (&Press(ref a), &Press(ref b)) => a == b,
            (&Release(ref a), &Release(ref b)) => a == b,
            (&Move(ref a), &Move(ref b)) => a == b,
            (&Text(ref a), &Text(ref b)) => a == b,
            (&Resize(aw, ah), &Resize(bw, bh)) => aw == bw && ah == bh,
            (&Focus(a), &Focus(b)) => a == b,
            (&Cursor(a), &Cursor(b)) => a == b,
            (&Close, &Close) => true,
            (&Render(ref a), &Render(ref b)) => a == b,
            (&AfterRender(ref a), &AfterRender(ref b)) => a == b,
            (&Update(ref a), &Update(ref b)) => a == b,
            (&Idle(ref a), &Idle(ref b)) => a == b,
            (_, _) => false,
        }
    }
}

impl From<Key> for Button {
    fn from(key: Key) -> Self {
        Button::Keyboard(key)
    }
}

impl From<MouseButton> for Button {
    fn from(btn: MouseButton) -> Self {
        Button::Mouse(btn)
    }
}

impl From<ControllerButton> for Button {
    fn from(btn: ControllerButton) -> Self {
        Button::Controller(btn)
    }
}

impl From<ControllerAxisArgs> for Motion {
    fn from(args: ControllerAxisArgs) -> Self {
        Motion::ControllerAxis(args)
    }
}

impl From<Motion> for Input {
    fn from(motion: Motion) -> Self {
        Input::Move(motion)
    }
}

impl From<RenderArgs> for Input {
    fn from(args: RenderArgs) -> Self {
        Input::Render(args)
    }
}

impl From<AfterRenderArgs> for Input {
    fn from(args: AfterRenderArgs) -> Self {
        Input::AfterRender(args)
    }
}

impl From<UpdateArgs> for Input {
    fn from(args: UpdateArgs) -> Self {
        Input::Update(args)
    }
}

impl From<IdleArgs> for Input {
    fn from(args: IdleArgs) -> Self {
        Input::Idle(args)
    }
}
<commit_msg>Override `Decodable` and `Encodable` for `Input`<commit_after>#![crate_name = "input"]
#![deny(missing_docs)]
#![deny(missing_copy_implementations)]

//! A flexible structure for user interactions
//! to be used in window frameworks and widgets libraries.

#[macro_use]
extern crate bitflags;
extern crate rustc_serialize;
extern crate viewport;

use std::any::Any;
use std::sync::Arc;

use rustc_serialize::{Decoder, Decodable, Encoder, Encodable};

pub use mouse::MouseButton;
pub use keyboard::Key;
pub use controller::{ ControllerAxisArgs, ControllerButton };

pub mod controller;
pub mod keyboard;
pub mod mouse;

pub use after_render::{ AfterRenderArgs, AfterRenderEvent };
pub use controller::{ ControllerAxisEvent };
pub use cursor::CursorEvent;
pub use focus::FocusEvent;
pub use generic_event::GenericEvent;
pub use idle::{ IdleArgs, IdleEvent };
pub use mouse::{ MouseCursorEvent, MouseRelativeEvent, MouseScrollEvent };
pub use press::PressEvent;
pub use release::ReleaseEvent;
pub use resize::ResizeEvent;
pub use render::{ RenderArgs, RenderEvent };
pub use text::TextEvent;
pub use touch::{ Touch, TouchArgs, TouchEvent };
pub use update::{ UpdateArgs, UpdateEvent };

pub mod generic_event;

mod after_render;
mod cursor;
mod focus;
mod idle;
mod press;
mod release;
mod render;
mod resize;
mod text;
mod touch;
mod update;

/// Used to identify events arguments provided by traits.
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct EventId(pub &'static str);

const AFTER_RENDER: EventId = EventId("piston/after_render");
const CONTROLLER_AXIS: EventId = EventId("piston/controller_axis");
const CURSOR: EventId = EventId("piston/cursor");
const FOCUS: EventId = EventId("piston/focus");
const CLOSE: EventId = EventId("piston/close");
const IDLE: EventId = EventId("piston/idle");
const MOUSE_SCROLL: EventId = EventId("piston/mouse_scroll");
const MOUSE_RELATIVE: EventId = EventId("piston/mouse_relative");
const MOUSE_CURSOR: EventId = EventId("piston/mouse_cursor");
const PRESS: EventId = EventId("piston/press");
const RELEASE: EventId = EventId("piston/release");
const RENDER: EventId = EventId("piston/render");
const RESIZE: EventId = EventId("piston/resize");
const TEXT: EventId = EventId("piston/text");
const TOUCH: EventId = EventId("piston/touch");
const UPDATE: EventId = EventId("piston/update");

/// Models different kinds of buttons.
#[derive(Copy, Clone, RustcDecodable, RustcEncodable, PartialEq, Eq, Hash, Debug)]
pub enum Button {
    /// A keyboard button.
    Keyboard(Key),
    /// A mouse button.
    Mouse(MouseButton),
    /// A controller button.
    Controller(ControllerButton),
}

/// Models different kinds of motion.
#[derive(Copy, Clone, RustcDecodable, RustcEncodable, PartialEq, Debug)]
pub enum Motion {
    /// x and y in window coordinates.
    MouseCursor(f64, f64),
    /// x and y in relative coordinates.
    MouseRelative(f64, f64),
    /// x and y in scroll ticks.
    MouseScroll(f64, f64),
    /// controller axis move event.
    ControllerAxis(ControllerAxisArgs),
    /// touch event.
    Touch(TouchArgs),
}

/// Models input events.
#[derive(Clone, Debug)]
pub enum Input {
    /// Pressed a button.
    Press(Button),
    /// Released a button.
    Release(Button),
    /// Moved mouse cursor.
    Move(Motion),
    /// Text (usually from keyboard).
    Text(String),
    /// Window got resized.
    Resize(u32, u32),
    /// Window gained or lost focus.
    Focus(bool),
    /// Window gained or lost cursor.
    Cursor(bool),
    /// Window closed.
    Close,
    /// Render graphics.
    Render(RenderArgs),
    /// After rendering and swapping buffers.
    AfterRender(AfterRenderArgs),
    /// Update the state of the application.
    Update(UpdateArgs),
    /// Do background tasks that can be done incrementally.
    Idle(IdleArgs),
    /// Custom event.
    ///
    /// When comparing two custom events for equality,
    /// they always return `false`.
    Custom(EventId, Arc<Any>),
}

impl PartialEq for Input {
    fn eq(&self, other: &Input) -> bool {
        use Input::*;

        match (self, other) {
            (&Press(ref a), &Press(ref b)) => a == b,
            (&Release(ref a), &Release(ref b)) => a == b,
            (&Move(ref a), &Move(ref b)) => a == b,
            (&Text(ref a), &Text(ref b)) => a == b,
            (&Resize(aw, ah), &Resize(bw, bh)) => aw == bw && ah == bh,
            (&Focus(a), &Focus(b)) => a == b,
            (&Cursor(a), &Cursor(b)) => a == b,
            (&Close, &Close) => true,
            (&Render(ref a), &Render(ref b)) => a == b,
            (&AfterRender(ref a), &AfterRender(ref b)) => a == b,
            (&Update(ref a), &Update(ref b)) => a == b,
            (&Idle(ref a), &Idle(ref b)) => a == b,
            (_, _) => false,
        }
    }
}

impl Decodable for Input {
    fn decode<D: Decoder>(d: &mut D) -> Result<Input, D::Error> {
        d.read_enum("Input", |d| {
            d.read_enum_variant(&[
                "Press", "Release", "Move", "Text",
                "Resize", "Focus", "Cursor", "Close",
                "Render", "AfterRender", "Update", "Idle"
            ], |d, ind| {
                Ok(match ind {
                    0 => Input::Press(try!(d.read_enum_variant_arg(0, |d| Button::decode(d)))),
                    1 => Input::Release(try!(d.read_enum_variant_arg(0, |d| Button::decode(d)))),
                    2 => Input::Move(try!(d.read_enum_variant_arg(0, |d| Motion::decode(d)))),
                    3 => Input::Text(try!(d.read_enum_variant_arg(0, |d| String::decode(d)))),
                    4 => Input::Resize(try!(d.read_enum_variant_arg(0, |d| u32::decode(d))),
                        try!(d.read_enum_variant_arg(1, |d| u32::decode(d)))),
                    5 => Input::Focus(try!(d.read_enum_variant_arg(0, |d| bool::decode(d)))),
                    6 => Input::Cursor(try!(d.read_enum_variant_arg(0, |d| bool::decode(d)))),
                    7 => Input::Close,
                    8 => Input::Render(try!(
                        d.read_enum_variant_arg(0, |d| RenderArgs::decode(d)))),
                    9 => Input::AfterRender(try!(
                        d.read_enum_variant_arg(0, |d| AfterRenderArgs::decode(d)))),
                    10 => Input::Update(try!(
                        d.read_enum_variant_arg(0, |d| UpdateArgs::decode(d)))),
                    11 => Input::Idle(try!(d.read_enum_variant_arg(0, |d| IdleArgs::decode(d)))),
                    _ => unimplemented!(),
                })
            })
        })
    }
}

impl Encodable for Input {
    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
        s.emit_enum("Input", |s| {
            match *self {
                Input::Press(ref button) =>
                    s.emit_enum_variant("Press", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| button.encode(s))),
                Input::Release(ref button) =>
                    s.emit_enum_variant("Release", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| button.encode(s))),
                Input::Move(ref motion) =>
                    s.emit_enum_variant("Move", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| motion.encode(s))),
                Input::Text(ref text) =>
                    s.emit_enum_variant("Text", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| text.encode(s))),
                Input::Resize(w, h) =>
                    s.emit_enum_variant("Resize", 0, 2, |s| {
                        try!(s.emit_enum_variant_arg(0, |s| w.encode(s)));
                        try!(s.emit_enum_variant_arg(1, |s| h.encode(s)));
                        Ok(())
                    }),
                Input::Focus(focus) =>
                    s.emit_enum_variant("Focus", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| focus.encode(s))),
                Input::Cursor(cursor) =>
                    s.emit_enum_variant("Cursor", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| cursor.encode(s))),
                Input::Close =>
                    s.emit_enum_variant("Close", 0, 0, |_s| Ok(())),
                Input::Render(ref render_args) =>
                    s.emit_enum_variant("Render", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| render_args.encode(s))),
                Input::AfterRender(ref after_render_args) =>
                    s.emit_enum_variant("AfterRender", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| after_render_args.encode(s))),
                Input::Update(ref update_args) =>
                    s.emit_enum_variant("Update", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| update_args.encode(s))),
                Input::Idle(ref idle_args) =>
                    s.emit_enum_variant("Idle", 0, 1, |s|
                        s.emit_enum_variant_arg(0, |s| idle_args.encode(s))),
                Input::Custom(_, _) => Ok(()),
            }
        })
    }
}

impl From<Key> for Button {
    fn from(key: Key) -> Self {
        Button::Keyboard(key)
    }
}

impl From<MouseButton> for Button {
    fn from(btn: MouseButton) -> Self {
        Button::Mouse(btn)
    }
}

impl From<ControllerButton> for Button {
    fn from(btn: ControllerButton) -> Self {
        Button::Controller(btn)
    }
}

impl From<ControllerAxisArgs> for Motion {
    fn from(args: ControllerAxisArgs) -> Self {
        Motion::ControllerAxis(args)
    }
}

impl From<Motion> for Input {
    fn from(motion: Motion) -> Self {
        Input::Move(motion)
    }
}

impl From<RenderArgs> for Input {
    fn from(args: RenderArgs) -> Self {
        Input::Render(args)
    }
}

impl From<AfterRenderArgs> for Input {
    fn from(args: AfterRenderArgs) -> Self {
        Input::AfterRender(args)
    }
}

impl From<UpdateArgs> for Input {
    fn from(args: UpdateArgs) -> Self {
        Input::Update(args)
    }
}

impl From<IdleArgs> for Input {
    fn from(args: IdleArgs) -> Self {
        Input::Idle(args)
    }
}

#[cfg(test)]
mod tests {
    use rustc_serialize::json;
    use super::*;

    #[test]
    fn test_encode_decode() {
        let test = |input| {
            let encoded = json::encode(&input).unwrap().to_string();
            let decoded: Input = json::decode(&encoded).unwrap();
            assert_eq!(decoded, input);
        };

        test(Input::Press(Button::Keyboard(Key::A)));
        test(Input::Release(Button::Keyboard(Key::A)));
        test(Input::Move(Motion::MouseCursor(0.0, 0.0)));
        test(Input::Text("hello".into()));
        test(Input::Resize(0, 0));
        test(Input::Focus(true));
        test(Input::Cursor(true));
        test(Input::Close);
        test(Input::Render(RenderArgs {
            width: 0,
            height: 0,
            draw_width: 0,
            draw_height: 0,
            ext_dt: 0.0,
        }));
        test(Input::AfterRender(AfterRenderArgs));
        test(Input::Update(UpdateArgs {dt: 0.0}));
        test(Input::Idle(IdleArgs {dt: 0.0}));
    }
}
<|endoftext|>


<commit_before>
use std::io::prelude::*;
use std::io::BufReader;
use std::fs;
use std::path::PathBuf;
use rustc_serialize::json::Json;
use errors::*;

fn crates_from_file<F>(path: &PathBuf, func: &mut F) -> Result<()>
    where F: FnMut(&str, &str) -> () {

    let reader = try!(fs::File::open(path)
                      .map(|f| BufReader::new(f)));

    let mut name = String::new();
    let mut versions = Vec::new();

    for line in reader.lines() {
        // some crates have invalid UTF-8 (nanny-sys-0.0.7)
        // skip them
        let line = match line {
            Ok(l) => l,
            Err(_) => continue
        };
        let data = match Json::from_str(line.trim()) {
            Ok(d) => d,
            Err(_) => continue
        };

        let obj = try!(data.as_object().ok_or("Not a JSON object"));
        let crate_name = try!(obj.get("name")
                              .and_then(|n| n.as_string())
                              .ok_or("`name` not found in JSON object"));
        let vers = try!(obj.get("vers")
                        .and_then(|n| n.as_string())
                        .ok_or("`vers` not found in JSON object"));

        let yanked = try!(obj.get("yanked")
                          .and_then(|n| n.as_boolean())
                          .ok_or("`vers` not found in JSON object"));

        // Skip yanked crates
        if yanked {
            continue;
        }

        name.clear();
        name.push_str(crate_name);
        versions.push(format!("{}", vers));
    }

    if !name.is_empty() {
        versions.reverse();
        for version in versions {
            func(&name[..], &version[..]);
        }
    }

    Ok(())
}

pub fn crates_from_path<F>(path: &PathBuf, func: &mut F) -> Result<()>
    where F: FnMut(&str, &str) -> () {

    if !path.is_dir() {
        return Err("Not a directory".into());
    }

    for file in try!(path.read_dir()) {
        let file = try!(file);
        let path = file.path();
        // skip files under .git and config.json
        if path.to_str().unwrap().contains(".git") ||
            path.file_name().unwrap() == "config.json" {
                continue;
            }

        if path.is_dir() {
            try!(crates_from_path(&path, func));
        } else {
            try!(crates_from_file(&path, func));
        }
    }

    Ok(())
}
<commit_msg>Stop returning an Err if 'yanked' is not available<commit_after>
use std::io::prelude::*;
use std::io::BufReader;
use std::fs;
use std::path::PathBuf;
use rustc_serialize::json::Json;
use errors::*;

fn crates_from_file<F>(path: &PathBuf, func: &mut F) -> Result<()>
    where F: FnMut(&str, &str) -> () {

    let reader = try!(fs::File::open(path)
                      .map(|f| BufReader::new(f)));

    let mut name = String::new();
    let mut versions = Vec::new();

    for line in reader.lines() {
        // some crates have invalid UTF-8 (nanny-sys-0.0.7)
        // skip them
        let line = match line {
            Ok(l) => l,
            Err(_) => continue
        };
        let data = match Json::from_str(line.trim()) {
            Ok(d) => d,
            Err(_) => continue
        };

        let obj = try!(data.as_object().ok_or("Not a JSON object"));
        let crate_name = try!(obj.get("name")
                              .and_then(|n| n.as_string())
                              .ok_or("`name` not found in JSON object"));
        let vers = try!(obj.get("vers")
                        .and_then(|n| n.as_string())
                        .ok_or("`vers` not found in JSON object"));

        // Skip yanked crates
        if obj.get("yanked").and_then(|n| n.as_boolean()).unwrap_or(false) {
            continue;
        }

        name.clear();
        name.push_str(crate_name);
        versions.push(format!("{}", vers));
    }

    if !name.is_empty() {
        versions.reverse();
        for version in versions {
            func(&name[..], &version[..]);
        }
    }

    Ok(())
}

pub fn crates_from_path<F>(path: &PathBuf, func: &mut F) -> Result<()>
    where F: FnMut(&str, &str) -> () {

    if !path.is_dir() {
        return Err("Not a directory".into());
    }

    for file in try!(path.read_dir()) {
        let file = try!(file);
        let path = file.path();
        // skip files under .git and config.json
        if path.to_str().unwrap().contains(".git") ||
            path.file_name().unwrap() == "config.json" {
                continue;
            }

        if path.is_dir() {
            try!(crates_from_path(&path, func));
        } else {
            try!(crates_from_file(&path, func));
        }
    }

    Ok(())
}
<|endoftext|>