<commit_before>use std::path::PathBuf;

use git2::Repository;
use toml::Value;

use libimagerror::into::IntoError;
use libimagerror::trace::trace_error;
use libimagstore::hook::error::CustomData;
use libimagstore::hook::error::HookErrorKind as HEK;
use libimagstore::hook::result::HookResult;
use libimagutil::debug_result::*;

use vcs::git::action::StoreAction;
use vcs::git::result::Result;
use vcs::git::error::{MapErrInto, GitHookErrorKind as GHEK};

pub struct Runtime {
    repository: Option<Repository>,
    config: Option<Value>,
}

impl Runtime {

    pub fn new(storepath: &PathBuf) -> Runtime {
        Runtime {
            repository: match Repository::open(storepath) {
                Ok(r) => Some(r),
                Err(e) => {
                    trace_error(&e);
                    None
                },
            },

            config: None,
        }
    }

    pub fn set_config(&mut self, cfg: &Value) -> Result<()> {
        self.config = Some(cfg.clone());
        Ok(())
    }

    pub fn has_repository(&self) -> bool {
        self.repository.is_some()
    }

    pub fn has_config(&self) -> bool {
        self.config.is_some()
    }

    pub fn config_value_or_err(&self, action: &StoreAction) -> HookResult<&Value> {
        self.config
            .as_ref()
            .ok_or(GHEK::NoConfigError.into_error())
            .map_err_into(GHEK::ConfigError)
            .map_err(Box::new)
            .map_err(|e| HEK::HookExecutionError.into_error_with_cause(e))
            .map_err(|mut e| e.with_custom_data(CustomData::default().aborting(false)))
            .map_dbg_err(|_| {
                format!("[GIT {} HOOK]: Couldn't get Value object from config", action.uppercase())
            })
    }

    pub fn repository(&self, action: &StoreAction) -> HookResult<&Repository> {
        use vcs::git::error::MapIntoHookError;

        debug!("[GIT {} HOOK]: Getting repository", action.uppercase());
        self.repository
            .as_ref()
            .ok_or(GHEK::MkRepo.into_error())
            .map_err_into(GHEK::RepositoryError)
            .map_into_hook_error()
            .map_dbg_err(|_| format!("[GIT {} HOOK]: Couldn't fetch Repository", action.uppercase()))
            .map_dbg(|_| format!("[GIT {} HOOK]: Repository object fetched", action.uppercase()))
    }

    pub fn ensure_cfg_branch_is_checked_out(&self, action: &StoreAction) -> HookResult<()> {
        use vcs::git::config::ensure_branch;

        debug!("[GIT CREATE HOOK]: Ensuring branch checkout");
        let head = try!(self
                        .repository(action)
                        .and_then(|r| {
                            debug!("Repository fetched, getting head");
                            r.head()
                                .map_dbg_err_str("Couldn't fetch HEAD")
                                .map_dbg_err(|e| format!("\tbecause = {:?}", e))
                                .map_err_into(GHEK::HeadFetchError)
                                .map_err(|e| e.into())
                        }));
        debug!("HEAD fetched");

        // TODO: Fail if not on branch? hmmh... I'm not sure
        if !head.is_branch() {
            debug!("HEAD is not a branch");
            return Err(GHEK::NotOnBranch.into_error().into());
        }
        debug!("HEAD is a branch");

        // Check out appropriate branch ... or fail
        match ensure_branch(self.config.as_ref()) {
            Ok(Some(s)) => {
                debug!("We have to ensure branch: {}", s);
                match head.name().map(|name| {
                    debug!("{} == {}", name, s);
                    name == s
                }) {
                    Some(b) => {
                        if b {
                            debug!("Branch already checked out.");
                            Ok(())
                        } else {
                            debug!("Branch not checked out.");
                            unimplemented!()
                        }
                    },

                    None => Err(GHEK::RepositoryBranchNameFetchingError.into_error())
                        .map_err_into(GHEK::RepositoryBranchError)
                        .map_err_into(GHEK::RepositoryError),
                }
            },
            Ok(None) => {
                debug!("No branch to checkout");
                Ok(())
            },

            Err(e) => Err(e).map_err_into(GHEK::RepositoryError),
        }
        .map_err(Box::new)
        .map_err(|e| HEK::HookExecutionError.into_error_with_cause(e))
        .map_dbg_str("[GIT CREATE HOOK]: Branch checked out")
    }

}

<commit_msg>Add documentation for the Runtime utility<commit_after>use std::path::PathBuf;

use git2::Repository;
use toml::Value;

use libimagerror::into::IntoError;
use libimagerror::trace::trace_error;
use libimagstore::hook::error::CustomData;
use libimagstore::hook::error::HookErrorKind as HEK;
use libimagstore::hook::result::HookResult;
use libimagutil::debug_result::*;

use vcs::git::action::StoreAction;
use vcs::git::result::Result;
use vcs::git::error::{MapErrInto, GitHookErrorKind as GHEK};

/// Runtime object for git hook implementations.
///
/// Contains some utility functionality to hold the repository and the configuration for the hooks.
pub struct Runtime {
    repository: Option<Repository>,
    config: Option<Value>,
}

impl Runtime {

    /// Build a `Runtime` object, pass the store path to build the `Repository` instance the
    /// `Runtime` has to contain.
    ///
    /// If the building of the `Repository` fails, this function `trace_error()`s the error and
    /// returns a `Runtime` object that does _not_ contain a `Repository`.
    pub fn new(storepath: &PathBuf) -> Runtime {
        Runtime {
            repository: match Repository::open(storepath) {
                Ok(r) => Some(r),
                Err(e) => {
                    trace_error(&e);
                    None
                },
            },

            config: None,
        }
    }

    /// Set the configuration for the `Runtime`. Always returns `Ok(())`.
    pub fn set_config(&mut self, cfg: &Value) -> Result<()> {
        self.config = Some(cfg.clone());
        Ok(())
    }

    /// Check whether the `Runtime` has a `Repository`
    pub fn has_repository(&self) -> bool {
        self.repository.is_some()
    }

    /// Check whether the `Runtime` has a configuration
    pub fn has_config(&self) -> bool {
        self.config.is_some()
    }

    /// Get the the config value by reference or get an `Err()` which can be returned to the callee
    /// of the Hook.
    ///
    /// The `action` Argument is required in case of `Err()` so the error message can be build
    /// correctly.
    pub fn config_value_or_err(&self, action: &StoreAction) -> HookResult<&Value> {
        self.config
            .as_ref()
            .ok_or(GHEK::NoConfigError.into_error())
            .map_err_into(GHEK::ConfigError)
            .map_err(Box::new)
            .map_err(|e| HEK::HookExecutionError.into_error_with_cause(e))
            .map_err(|mut e| e.with_custom_data(CustomData::default().aborting(false)))
            .map_dbg_err(|_| {
                format!("[GIT {} HOOK]: Couldn't get Value object from config", action.uppercase())
            })
    }

    /// Get the `Repository` object from the `Runtime` or an `Err()` that can be returned to the
    /// callee of the Hook.
    ///
    /// The `action` Argument is required in case of `Err()` so the error message can be build
    /// correctly.
    pub fn repository(&self, action: &StoreAction) -> HookResult<&Repository> {
        use vcs::git::error::MapIntoHookError;

        debug!("[GIT {} HOOK]: Getting repository", action.uppercase());
        self.repository
            .as_ref()
            .ok_or(GHEK::MkRepo.into_error())
            .map_err_into(GHEK::RepositoryError)
            .map_into_hook_error()
            .map_dbg_err(|_| format!("[GIT {} HOOK]: Couldn't fetch Repository", action.uppercase()))
            .map_dbg(|_| format!("[GIT {} HOOK]: Repository object fetched", action.uppercase()))
    }

    /// Ensure that the branch that is put in the configuration file is checked out, if any.
    pub fn ensure_cfg_branch_is_checked_out(&self, action: &StoreAction) -> HookResult<()> {
        use vcs::git::config::ensure_branch;

        debug!("[GIT CREATE HOOK]: Ensuring branch checkout");
        let head = try!(self
                        .repository(action)
                        .and_then(|r| {
                            debug!("Repository fetched, getting head");
                            r.head()
                                .map_dbg_err_str("Couldn't fetch HEAD")
                                .map_dbg_err(|e| format!("\tbecause = {:?}", e))
                                .map_err_into(GHEK::HeadFetchError)
                                .map_err(|e| e.into())
                        }));
        debug!("HEAD fetched");

        // TODO: Fail if not on branch? hmmh... I'm not sure
        if !head.is_branch() {
            debug!("HEAD is not a branch");
            return Err(GHEK::NotOnBranch.into_error().into());
        }
        debug!("HEAD is a branch");

        // Check out appropriate branch ... or fail
        match ensure_branch(self.config.as_ref()) {
            Ok(Some(s)) => {
                debug!("We have to ensure branch: {}", s);
                match head.name().map(|name| {
                    debug!("{} == {}", name, s);
                    name == s
                }) {
                    Some(b) => {
                        if b {
                            debug!("Branch already checked out.");
                            Ok(())
                        } else {
                            debug!("Branch not checked out.");
                            unimplemented!()
                        }
                    },

                    None => Err(GHEK::RepositoryBranchNameFetchingError.into_error())
                        .map_err_into(GHEK::RepositoryBranchError)
                        .map_err_into(GHEK::RepositoryError),
                }
            },
            Ok(None) => {
                debug!("No branch to checkout");
                Ok(())
            },

            Err(e) => Err(e).map_err_into(GHEK::RepositoryError),
        }
        .map_err(Box::new)
        .map_err(|e| HEK::HookExecutionError.into_error_with_cause(e))
        .map_dbg_str("[GIT CREATE HOOK]: Branch checked out")
    }

}

<|endoftext|>


<commit_before>//
// base64.rs - base64 implementation
//
// The Base64 Alphabet
//
//  Value Encoding  Value Encoding  Value Encoding  Value Encoding
//   0 A            17 R            34 i            51 z
//   1 B            18 S            35 j            52 0
//   2 C            19 T            36 k            53 1
//   3 D            20 U            37 l            54 2
//   4 E            21 V            38 m            55 3
//   5 F            22 W            39 n            56 4
//   6 G            23 X            40 o            57 5
//   7 H            24 Y            41 p            58 6
//   8 I            25 Z            42 q            59 7
//   9 J            26 a            43 r            60 8
//  10 K            27 b            44 s            61 9
//  11 L            28 c            45 t            62 +
//  12 M            29 d            46 u            63 /
//  13 N            30 e            47 v
//  14 O            31 f            48 w         (pad) =
//  15 P            32 g            49 x
//  16 Q            33 h            50 y
//

use std;

import std::map;

export base64, mk_base64;

iface base64 {
    fn encode(src: [u8]) -> [u8];
    fn decode(src: [u8]) -> [u8];
    fn urlsafe_encode(src: [u8]) -> [u8];
    fn urlsafe_decode(src: [u8]) -> [u8];
}

fn mk_base64() -> base64 {
    type _base64 = {padd: u8, table: [u8]};
    impl of base64 for _base64 {
        fn encode(src: [u8]) -> [u8] {
            let srclen = vec::len(src);
            let targ = if srclen % 3u == 0u {
                vec::init_elt_mut((srclen / 3u) * 4u, 0u8)
            } else {
                vec::init_elt_mut((srclen / 3u + 1u) * 4u, 0u8)
            };
            let input = vec::init_elt_mut(3u, 0u8);
            let output = vec::init_elt_mut(4u, 0u8);
            let curr = 0u, src_curr = 0u;
            let table = self.table;

            while srclen > 2u {
                input[0] = src[src_curr];
                input[1] = src[src_curr + 1u];
                input[2] = src[src_curr + 2u];
                srclen -= 3u; src_curr += 3u;

                output[0] = input[0] >> 2u8;
                output[1] = ((input[0] &  3u8) << 4u8) | (input[1] >> 4u8);
                output[2] = ((input[1] & 15u8) << 2u8) | (input[2] >> 6u8);
                output[3] = input[2] & 63u8;

                targ[curr] = table[output[0]]; curr += 1u;
                targ[curr] = table[output[1]]; curr += 1u;
                targ[curr] = table[output[2]]; curr += 1u;
                targ[curr] = table[output[3]]; curr += 1u;
            }

            if srclen != 0u {
                input[0] = 0u8; input[1] = 0u8; input[2] = 0u8;

                alt srclen {
                  1u { input[0] = src[src_curr]; }
                  2u { input[0] = src[src_curr];
                       input[1] = src[src_curr + 1u]; }
                  _  { }
                }

                output[0] = input[0] >> 2u8;
                output[1] = ((input[0] &  3u8) << 4u8) | (input[1] >> 4u8);
                output[2] = ((input[1] & 15u8) << 2u8) | (input[2] >> 6u8);

                targ[curr] = table[output[0]]; curr += 1u;
                targ[curr] = table[output[1]]; curr += 1u;
                targ[curr] = if srclen == 1u {
                    self.padd
                } else {
                    table[output[2]]
                };
                curr += 1u; targ[curr] = self.padd;
            }

            vec::from_mut(targ)
        }
        fn decode(src: [u8]) -> [u8] {
            let srclen = vec::len(src);
            let input  = vec::init_elt_mut(4u, 0u8);
            let output = vec::init_elt_mut(4u, 0u8);
            let targ = if src[srclen - 2u] == self.padd {
                vec::init_elt_mut(srclen / 4u * 3u - 2u, 0u8)
            } else if src[srclen - 1u] == self.padd {
                vec::init_elt_mut(srclen / 4u * 3u - 1u, 0u8)
            } else {
                vec::init_elt_mut(srclen / 4u * 3u, 0u8)
            };
            let curr = 0u, src_curr = 0u;
            let table = self.table;

            while srclen > 4u {
                input[0] = src[src_curr];
                input[1] = src[src_curr + 1u];
                input[2] = src[src_curr + 2u];
                input[3] = src[src_curr + 3u];
                srclen -= 4u; src_curr += 4u;

                output[0] = idx(table, input[0]);
                if output[0] == 64u8 { fail "malformed base64 string"; }

                output[1] = idx(table, input[1]);
                if output[1] == 64u8 { fail "malformed base64 string"; }

                output[2] = idx(table, input[2]);
                if output[2] == 64u8 { fail "malformed base64 string"; }

                output[3] = idx(table, input[3]);
                if output[3] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = (output[0] << 2u8) | (output[1] >> 4u8);
                curr += 1u;

                targ[curr] =
                    ((output[1] & 15u8) << 4u8) | (output[2] >> 2u8);
                curr += 1u;

                targ[curr] = ((output[2] &  3u8) << 6u8) | output[3];
                curr += 1u;
            }

            if srclen == 4u {
                input[0] = src[src_curr];
                input[1] = src[src_curr + 1u];
                input[2] = src[src_curr + 2u];
                input[3] = src[src_curr + 3u];

                output[0] = idx(table, input[0]);
                if output[0] == 64u8 { fail "malformed base64 string"; }

                output[1] = idx(table, input[1]);
                if output[1] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = (output[0] << 2u8) | (output[1] >> 4u8);
                curr += 1u;

                if input[2] == self.padd { ret vec::from_mut(targ); }

                output[2] = idx(table, input[2]);
                if output[2] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = ((output[1] & 15u8) << 4u8) | (output[2] >> 2u8);
                curr += 1u;

                if input[3] == self.padd { ret vec::from_mut(targ); }

                output[3] = idx(table, input[3]);
                if output[3] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = ((output[2] & 3u8) << 6u8) | output[3];
            } else {
                fail "malformed base64 string";
            }

            vec::from_mut(targ)
        }
        fn urlsafe_encode(src: [u8]) -> [u8] {
            let res = self.encode(src);
            vec::map(res) { |i|
                if i == 43u8 { 45u8 }
                else if i == 47u8 { 95u8 }
                else { i }
            }
        }
        fn urlsafe_decode(src: [u8]) -> [u8] {
            let src0 = vec::map(src) { |i|
                if i == 45u8 { 43u8 }
                else if i == 95u8 { 47u8 }
                else { i }
            };
            self.decode(src0)
        }
    }

    let table = vec::init_elt_mut(64u, 0u8), i = 0u8;
    u8::range(65u8, 91u8)  { |j| table[i] = j; i += 1u8; }
    u8::range(97u8, 123u8) { |j| table[i] = j; i += 1u8; }
    u8::range(48u8, 58u8)  { |j| table[i] = j; i += 1u8; }
    table[i] = 43u8; i += 1u8;
    table[i] = 47u8; i += 1u8;

    {padd: 61u8, table: vec::from_mut(table)} as base64
}

fn idx(elems: [const u8], x: u8) -> u8 {
    let i = 0u8;
    while i < 64u8 {
        if elems[i] == x { break; }
        i += 1u8;
    }
    ret i;
}

#[cfg(test)]
mod tests {
    import core::str::{from_bytes, bytes};
    enum mode {
        t_decode,
        t_encode,
        t_urlsafe_encode,
        t_urlsafe_decode
    }
    fn setup(t: mode) -> map::hashmap<str, str> {
        let m = map::new_str_hash::<str>();
        alt t {
          t_decode | t_urlsafe_decode {
            m.insert("cGxlYXN1cmUu", "pleasure.");
            m.insert("bGVhc3VyZS4=",  "leasure.");
            m.insert("ZWFzdXJlLg==",   "easure.");
            m.insert("YXN1cmUu",        "asure.");
            m.insert("c3VyZS4=",         "sure.");
          }
          t_encode | t_urlsafe_encode {
            m.insert("pleasure.", "cGxlYXN1cmUu");
            m.insert("leasure.",  "bGVhc3VyZS4=");
            m.insert("easure.",   "ZWFzdXJlLg==");
            m.insert("asure.",        "YXN1cmUu");
            m.insert("sure.",         "c3VyZS4=");
          }
        }
        m
    }
    fn do_test(t: mode) {
        let b64 = mk_base64();
        let m = setup(t);
        m.keys { |k|
            let expected = m.get(k);
            let actual = alt t {
              t_decode { from_bytes(b64.decode(bytes(k))) }
              t_encode { from_bytes(b64.encode(bytes(k))) }
              t_urlsafe_decode { from_bytes(b64.urlsafe_decode(bytes(k))) }
              t_urlsafe_encode { from_bytes(b64.urlsafe_encode(bytes(k))) }
            };
            #debug("expected: %s", expected);
            #debug("actual:   %s", actual);
            assert expected == actual;
        }
    }
    #[test]
    fn decode() { do_test(t_decode); }
    #[test]
    fn encode() { do_test(t_encode); }
    #[test]
    fn urlsafe_decode() { do_test(t_urlsafe_decode); }
    #[test]
    fn urlsafe_encode() { do_test(t_urlsafe_encode); }
}
<commit_msg>changes<commit_after>//
// base64.rs - base64 implementation
//
// The Base64 Alphabet
//
//  Value Encoding  Value Encoding  Value Encoding  Value Encoding
//   0 A            17 R            34 i            51 z
//   1 B            18 S            35 j            52 0
//   2 C            19 T            36 k            53 1
//   3 D            20 U            37 l            54 2
//   4 E            21 V            38 m            55 3
//   5 F            22 W            39 n            56 4
//   6 G            23 X            40 o            57 5
//   7 H            24 Y            41 p            58 6
//   8 I            25 Z            42 q            59 7
//   9 J            26 a            43 r            60 8
//  10 K            27 b            44 s            61 9
//  11 L            28 c            45 t            62 +
//  12 M            29 d            46 u            63 /
//  13 N            30 e            47 v
//  14 O            31 f            48 w         (pad) =
//  15 P            32 g            49 x
//  16 Q            33 h            50 y
//

use std;

import std::map;

export base64, mk_base64;

iface base64 {
    fn encode(src: [u8]) -> [u8];
    fn decode(src: [u8]) -> [u8];
    fn urlsafe_encode(src: [u8]) -> [u8];
    fn urlsafe_decode(src: [u8]) -> [u8];
}

fn mk_base64() -> base64 {
    type _base64 = {padd: u8, table: [u8]};
    impl of base64 for _base64 {
        fn encode(src: [u8]) -> [u8] {
            let srclen = vec::len(src);
            let targ = if srclen % 3u == 0u {
                vec::init_elt_mut((srclen / 3u) * 4u, 0u8)
            } else {
                vec::init_elt_mut((srclen / 3u + 1u) * 4u, 0u8)
            };
            let input = vec::init_elt_mut(3u, 0u8);
            let output = vec::init_elt_mut(4u, 0u8);
            let curr = 0u, src_curr = 0u;
            let table = self.table;

            while srclen > 2u {
                input[0] = src[src_curr];
                input[1] = src[src_curr + 1u];
                input[2] = src[src_curr + 2u];
                srclen -= 3u; src_curr += 3u;

                output[0] = input[0] >> 2u8;
                output[1] = ((input[0] &  3u8) << 4u8) | (input[1] >> 4u8);
                output[2] = ((input[1] & 15u8) << 2u8) | (input[2] >> 6u8);
                output[3] = input[2] & 63u8;

                targ[curr] = table[output[0]]; curr += 1u;
                targ[curr] = table[output[1]]; curr += 1u;
                targ[curr] = table[output[2]]; curr += 1u;
                targ[curr] = table[output[3]]; curr += 1u;
            }

            if srclen != 0u {
                input[0] = 0u8; input[1] = 0u8; input[2] = 0u8;

                alt srclen {
                  1u { input[0] = src[src_curr]; }
                  2u { input[0] = src[src_curr];
                       input[1] = src[src_curr + 1u]; }
                  _  { }
                }

                output[0] = input[0] >> 2u8;
                output[1] = ((input[0] &  3u8) << 4u8) | (input[1] >> 4u8);
                output[2] = ((input[1] & 15u8) << 2u8) | (input[2] >> 6u8);

                targ[curr] = table[output[0]]; curr += 1u;
                targ[curr] = table[output[1]]; curr += 1u;
                targ[curr] = if srclen == 1u {
                    self.padd
                } else {
                    table[output[2]]
                };
                curr += 1u; targ[curr] = self.padd;
            }

            vec::from_mut(targ)
        }
        fn decode(src: [u8]) -> [u8] {
            let srclen = vec::len(src);

            if srclen % 4u != 0u { fail "malformed base64 string"; }
            if srclen == 0u { ret []; }

            let input  = vec::init_elt_mut(4u, 0u8);
            let output = vec::init_elt_mut(4u, 0u8);
            let targ = if src[srclen - 2u] == self.padd {
                vec::init_elt_mut(srclen / 4u * 3u - 2u, 0u8)
            } else if src[srclen - 1u] == self.padd {
                vec::init_elt_mut(srclen / 4u * 3u - 1u, 0u8)
            } else {
                vec::init_elt_mut(srclen / 4u * 3u, 0u8)
            };
            let curr = 0u, src_curr = 0u;
            let table = self.table;

            while srclen > 4u {
                input[0] = src[src_curr];
                input[1] = src[src_curr + 1u];
                input[2] = src[src_curr + 2u];
                input[3] = src[src_curr + 3u];
                srclen -= 4u; src_curr += 4u;

                output[0] = idx(table, input[0]);
                if output[0] == 64u8 { fail "malformed base64 string"; }

                output[1] = idx(table, input[1]);
                if output[1] == 64u8 { fail "malformed base64 string"; }

                output[2] = idx(table, input[2]);
                if output[2] == 64u8 { fail "malformed base64 string"; }

                output[3] = idx(table, input[3]);
                if output[3] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = (output[0] << 2u8) | (output[1] >> 4u8);
                curr += 1u;

                targ[curr] =
                    ((output[1] & 15u8) << 4u8) | (output[2] >> 2u8);
                curr += 1u;

                targ[curr] = ((output[2] &  3u8) << 6u8) | output[3];
                curr += 1u;
            }

            if srclen == 4u {
                input[0] = src[src_curr];
                input[1] = src[src_curr + 1u];
                input[2] = src[src_curr + 2u];
                input[3] = src[src_curr + 3u];

                output[0] = idx(table, input[0]);
                if output[0] == 64u8 { fail "malformed base64 string"; }

                output[1] = idx(table, input[1]);
                if output[1] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = (output[0] << 2u8) | (output[1] >> 4u8);
                curr += 1u;

                if input[2] == self.padd { ret vec::from_mut(targ); }

                output[2] = idx(table, input[2]);
                if output[2] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = ((output[1] & 15u8) << 4u8) | (output[2] >> 2u8);
                curr += 1u;

                if input[3] == self.padd { ret vec::from_mut(targ); }

                output[3] = idx(table, input[3]);
                if output[3] == 64u8 { fail "malformed base64 string"; }

                targ[curr] = ((output[2] & 3u8) << 6u8) | output[3];
            }

            vec::from_mut(targ)
        }
        fn urlsafe_encode(src: [u8]) -> [u8] {
            let res = self.encode(src);
            vec::map(res) { |i|
                if i == 43u8 { 45u8 }
                else if i == 47u8 { 95u8 }
                else { i }
            }
        }
        fn urlsafe_decode(src: [u8]) -> [u8] {
            let src0 = vec::map(src) { |i|
                if i == 45u8 { 43u8 }
                else if i == 95u8 { 47u8 }
                else { i }
            };
            self.decode(src0)
        }
    }

    let table = vec::init_elt_mut(64u, 0u8), i = 0u8;
    u8::range(65u8, 91u8)  { |j| table[i] = j; i += 1u8; }
    u8::range(97u8, 123u8) { |j| table[i] = j; i += 1u8; }
    u8::range(48u8, 58u8)  { |j| table[i] = j; i += 1u8; }
    table[i] = 43u8; i += 1u8;
    table[i] = 47u8; i += 1u8;

    {padd: 61u8, table: vec::from_mut(table)} as base64
}

fn idx(elems: [const u8], x: u8) -> u8 {
    let i = 0u8;
    while i < 64u8 {
        if elems[i] == x { break; }
        i += 1u8;
    }
    ret i;
}

#[cfg(test)]
mod tests {
    import core::str::{from_bytes, bytes};
    enum mode {
        t_decode,
        t_encode,
        t_urlsafe_encode,
        t_urlsafe_decode
    }
    fn setup(t: mode) -> map::hashmap<str, str> {
        let m = map::new_str_hash::<str>();
        m.insert("", "");
        alt t {
          t_decode | t_urlsafe_decode {
            m.insert("cGxlYXN1cmUu", "pleasure.");
            m.insert("bGVhc3VyZS4=",  "leasure.");
            m.insert("ZWFzdXJlLg==",   "easure.");
            m.insert("YXN1cmUu",        "asure.");
            m.insert("c3VyZS4=",         "sure.");
          }
          t_encode | t_urlsafe_encode {
            m.insert("pleasure.", "cGxlYXN1cmUu");
            m.insert("leasure.",  "bGVhc3VyZS4=");
            m.insert("easure.",   "ZWFzdXJlLg==");
            m.insert("asure.",        "YXN1cmUu");
            m.insert("sure.",         "c3VyZS4=");
          }
        }
        m
    }
    fn do_test(t: mode) {
        let b64 = mk_base64();
        let m = setup(t);
        m.keys { |k|
            let expected = m.get(k);
            let actual = alt t {
              t_decode { from_bytes(b64.decode(bytes(k))) }
              t_encode { from_bytes(b64.encode(bytes(k))) }
              t_urlsafe_decode { from_bytes(b64.urlsafe_decode(bytes(k))) }
              t_urlsafe_encode { from_bytes(b64.urlsafe_encode(bytes(k))) }
            };
            #debug("expected: %s", expected);
            #debug("actual:   %s", actual);
            assert expected == actual;
        }
    }
    #[test]
    fn decode() { do_test(t_decode); }
    #[test]
    fn encode() { do_test(t_encode); }
    #[test]
    fn urlsafe_decode() { do_test(t_urlsafe_decode); }
    #[test]
    fn urlsafe_encode() { do_test(t_urlsafe_encode); }
}
<|endoftext|>


<commit_before>use geometry::bbox::BBox;
use geometry::prim::Prim;
use material::Material;
use raytracer::{Ray, Intersection};
use vec3::Vec3;

#[allow(dead_code)]
pub struct Sphere {
    pub center: Vec3,
    pub radius: f64,
    pub material: Box<Material+Send+Share>
}

impl Prim for Sphere {
    fn intersects<'a>(&'a self, ray: &Ray, t_min: f64, t_max: f64) -> Option<Intersection<'a>> {
        let i = ray.origin - self.center;
        let a = 1.0;
        let b = 2.0 * ray.direction.dot(&i);
        let c = i.dot(&i) - self.radius * self.radius;
        let discriminant = b * b - 4.0 * a * c;

        if discriminant <= 0.0 {
            None
        } else {
            // Up to two intersections
            let disc_sqrt = discriminant.sqrt();
            let t1 = (-b + disc_sqrt) / 2.0 * a;
            let t2 = (-b - disc_sqrt) / 2.0 * a;

            if t1 >= t_min && t1 <= t_max ||
               t2 >= t_min && t2 <= t_max {
                // Valid intersection(s): get nearer intersection
                let t = if t1.abs() < t2.abs() { t1 } else { t2 };
                let intersection_point = ray.origin + ray.direction.scale(t);
                let n = (intersection_point - self.center).unit();

                let u = 0.5 + n.z.atan2(n.x) / ::std::f64::consts::PI_2;
                let v = 0.5 - n.y.asin() / ::std::f64::consts::PI;

                Some(Intersection {
                    n: n,
                    t: t,
                    u: u,
                    v: v,
                    position: intersection_point,
                    material: &self.material
                })
            } else {
                None
            }
        }
    }

    fn bounding(&self) -> Option<BBox> {
        Some(BBox {
            min: Vec3 {
                x: self.center.x - self.radius,
                y: self.center.y - self.radius,
                z: self.center.z - self.radius
            },
            max: Vec3 {
                x: self.center.x + self.radius,
                y: self.center.y + self.radius,
                z: self.center.z + self.radius
            }
        })
    }
}
<commit_msg>Add tests to Sphere<commit_after>use geometry::bbox::BBox;
use geometry::prim::Prim;
use material::Material;
use raytracer::{Ray, Intersection};
use vec3::Vec3;

#[cfg(test)]
use material::materials::FlatMaterial;

#[allow(dead_code)]
pub struct Sphere {
    pub center: Vec3,
    pub radius: f64,
    pub material: Box<Material+Send+Share>
}

impl Prim for Sphere {
    fn intersects<'a>(&'a self, ray: &Ray, t_min: f64, t_max: f64) -> Option<Intersection<'a>> {
        let i = ray.origin - self.center;
        let a = 1.0;
        let b = 2.0 * ray.direction.dot(&i);
        let c = i.dot(&i) - self.radius * self.radius;
        let discriminant = b * b - 4.0 * a * c;

        if discriminant <= 0.0 {
            None
        } else {
            // Up to two intersections
            let disc_sqrt = discriminant.sqrt();
            let t1 = (-b + disc_sqrt) / 2.0 * a;
            let t2 = (-b - disc_sqrt) / 2.0 * a;

            if t1 >= t_min && t1 <= t_max ||
               t2 >= t_min && t2 <= t_max {
                // Valid intersection(s): get nearer intersection
                let t = if t1.abs() < t2.abs() { t1 } else { t2 };
                let intersection_point = ray.origin + ray.direction.scale(t);
                let n = (intersection_point - self.center).unit();

                let u = 0.5 + n.z.atan2(n.x) / ::std::f64::consts::PI_2;
                let v = 0.5 - n.y.asin() / ::std::f64::consts::PI;

                Some(Intersection {
                    n: n,
                    t: t,
                    u: u,
                    v: v,
                    position: intersection_point,
                    material: &self.material
                })
            } else {
                None
            }
        }
    }

    fn bounding(&self) -> Option<BBox> {
        Some(BBox {
            min: Vec3 {
                x: self.center.x - self.radius,
                y: self.center.y - self.radius,
                z: self.center.z - self.radius
            },
            max: Vec3 {
                x: self.center.x + self.radius,
                y: self.center.y + self.radius,
                z: self.center.z + self.radius
            }
        })
    }
}

#[test]
fn it_intersects_and_interpolates() {
    let sphere = Sphere {
        center: Vec3::zero(),
        radius: 1.0,
        material: box FlatMaterial {color: Vec3::one()}
    };

    // Tests actual intersection
    let intersecting_ray = Ray::new(Vec3 {x: 0.0, y: 0.0, z: -2.0}, Vec3 {x: 0.0, y: 0.0, z: 1.0});
    let intersection = sphere.intersects(&intersecting_ray, 0.0, 10.0).unwrap();
    assert_eq!(intersection.position.x, 0.0);
    assert_eq!(intersection.position.y, 0.0);
    assert_eq!(intersection.position.z, -1.0);
    assert_eq!(intersection.n.x, 0.0);
    assert_eq!(intersection.n.y, 0.0);
    assert_eq!(intersection.n.z, -1.0);

    // Ray off to the sides
    let mut non_intersecting_ray = Ray::new(Vec3 {x: 0.0, y: 0.0, z: -2.0}, Vec3 {x: 100.0, y: 100.0, z: 0.1});
    let mut non_intersection = sphere.intersects(&non_intersecting_ray, 0.0, 10.0);
    assert!(non_intersection.is_none());

    non_intersecting_ray = Ray::new(Vec3 {x: 0.0, y: 0.0, z: -2.0}, Vec3 {x: -100.0, y: -100.0, z: 0.1});
    non_intersection = sphere.intersects(&non_intersecting_ray, 0.0, 10.0);
    assert!(non_intersection.is_none());

    // Ray in opposite direction
    non_intersecting_ray = Ray::new(Vec3 {x: 0.0, y: 0.0, z: -2.0}, Vec3 {x: 0.0, y: 0.0, z: -1.0});
    non_intersection = sphere.intersects(&non_intersecting_ray, 0.0, 10.0);
    assert!(non_intersection.is_none());
}

#[test]
fn it_intersects_only_in_tmin_tmax() {
    let sphere = Sphere {
        center: Vec3::zero(),
        radius: 1.0,
        material: box FlatMaterial {color: Vec3::one()}
    };

    // Tests tmin
    let mut intersecting_ray = Ray::new(Vec3 {x: 0.0, y: 0.0, z: -2.0}, Vec3 {x: 0.0, y: 0.0, z: 1.0});
    let mut non_intersection = sphere.intersects(&intersecting_ray, 1000.0, 10000.0);
    assert!(non_intersection.is_none());

    // Tests tmax
    non_intersection = sphere.intersects(&intersecting_ray, 0.0, 0.0001);
    assert!(non_intersection.is_none());
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate crossbeam;
extern crate ipc_channel;
extern crate test;

use ipc_channel::platform;

use std::sync::{mpsc, Mutex};

/// Allows doing multiple inner iterations per bench.iter() run.
///
/// This is mostly to amortise the overhead of spawning a thread in the benchmark
/// when sending larger messages (that might be fragmented).
///
/// Note that you need to compensate the displayed results
/// for the proportionally longer runs yourself,
/// as the benchmark framework doesn't know about the inner iterations...
const ITERATIONS: usize = 1;

fn bench_size(b: &mut test::Bencher, size: usize) {
    let data: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
    let (tx, rx) = platform::channel().unwrap();

    let (wait_tx, wait_rx) = mpsc::channel();
    let wait_rx = Mutex::new(wait_rx);

    if size > platform::OsIpcSender::get_max_fragment_size() {
        b.iter(|| {
            crossbeam::scope(|scope| {
                scope.spawn(|| {
                    let wait_rx = wait_rx.lock().unwrap();
                    for _ in 0..ITERATIONS {
                        tx.send(&data, vec![], vec![]).unwrap();
                        if ITERATIONS > 1 {
                            // Prevent beginning of the next send
                            // from overlapping with receive of last fragment,
                            // as otherwise results of runs with a large tail fragment
                            // are significantly skewed.
                            wait_rx.recv().unwrap();
                        }
                    }
                });
                for _ in 0..ITERATIONS {
                    rx.recv().unwrap();
                    if ITERATIONS > 1 {
                        wait_tx.send(()).unwrap();
                    }
                }
                // For reasons mysterious to me,
                // not returning a value *from every branch*
                // adds some 100 ns or so of overhead to all results --
                // which is quite significant for very short tests...
                0
            })
        });
    } else {
        b.iter(|| {
            for _ in 0..ITERATIONS {
                tx.send(&data, vec![], vec![]).unwrap();
                rx.recv().unwrap();
            }
            0
        });
    }
}

#[bench]
fn size_00_1(b: &mut test::Bencher) {
    bench_size(b, 1);
}
#[bench]
fn size_01_2(b: &mut test::Bencher) {
    bench_size(b, 2);
}
#[bench]
fn size_02_4(b: &mut test::Bencher) {
    bench_size(b, 4);
}
#[bench]
fn size_03_8(b: &mut test::Bencher) {
    bench_size(b, 8);
}
#[bench]
fn size_04_16(b: &mut test::Bencher) {
    bench_size(b, 16);
}
#[bench]
fn size_05_32(b: &mut test::Bencher) {
    bench_size(b, 32);
}
#[bench]
fn size_06_64(b: &mut test::Bencher) {
    bench_size(b, 64);
}
#[bench]
fn size_07_128(b: &mut test::Bencher) {
    bench_size(b, 128);
}
#[bench]
fn size_08_256(b: &mut test::Bencher) {
    bench_size(b, 256);
}
#[bench]
fn size_09_512(b: &mut test::Bencher) {
    bench_size(b, 512);
}
#[bench]
fn size_10_1k(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024);
}
#[bench]
fn size_11_2k(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024);
}
#[bench]
fn size_12_4k(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024);
}
#[bench]
fn size_13_8k(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024);
}
#[bench]
fn size_14_16k(b: &mut test::Bencher) {
    bench_size(b, 16 * 1024);
}
#[bench]
fn size_15_32k(b: &mut test::Bencher) {
    bench_size(b, 32 * 1024);
}
#[bench]
fn size_16_64k(b: &mut test::Bencher) {
    bench_size(b, 64 * 1024);
}
#[bench]
fn size_17_128k(b: &mut test::Bencher) {
    bench_size(b, 128 * 1024);
}
#[bench]
fn size_18_256k(b: &mut test::Bencher) {
    bench_size(b, 256 * 1024);
}
#[bench]
fn size_19_512k(b: &mut test::Bencher) {
    bench_size(b, 512 * 1024);
}
#[bench]
fn size_20_1m(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024 * 1024);
}
#[bench]
fn size_21_2m(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024 * 1024);
}
#[bench]
fn size_22_4m(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024 * 1024);
}
#[bench]
fn size_23_8m(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024 * 1024);
}
<commit_msg>Auto merge of #78 - antrik:bench-create, r=Manishearth<commit_after>#![feature(test)]

extern crate crossbeam;
extern crate ipc_channel;
extern crate test;

use ipc_channel::platform;

use std::sync::{mpsc, Mutex};

/// Allows doing multiple inner iterations per bench.iter() run.
///
/// This is mostly to amortise the overhead of spawning a thread in the benchmark
/// when sending larger messages (that might be fragmented).
///
/// Note that you need to compensate the displayed results
/// for the proportionally longer runs yourself,
/// as the benchmark framework doesn't know about the inner iterations...
const ITERATIONS: usize = 1;

#[bench]
fn create_channel(b: &mut test::Bencher) {
    b.iter(|| {
        for _ in 0..ITERATIONS {
            platform::channel().unwrap();
        }
    });
}

fn bench_size(b: &mut test::Bencher, size: usize) {
    let data: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
    let (tx, rx) = platform::channel().unwrap();

    let (wait_tx, wait_rx) = mpsc::channel();
    let wait_rx = Mutex::new(wait_rx);

    if size > platform::OsIpcSender::get_max_fragment_size() {
        b.iter(|| {
            crossbeam::scope(|scope| {
                scope.spawn(|| {
                    let wait_rx = wait_rx.lock().unwrap();
                    for _ in 0..ITERATIONS {
                        tx.send(&data, vec![], vec![]).unwrap();
                        if ITERATIONS > 1 {
                            // Prevent beginning of the next send
                            // from overlapping with receive of last fragment,
                            // as otherwise results of runs with a large tail fragment
                            // are significantly skewed.
                            wait_rx.recv().unwrap();
                        }
                    }
                });
                for _ in 0..ITERATIONS {
                    rx.recv().unwrap();
                    if ITERATIONS > 1 {
                        wait_tx.send(()).unwrap();
                    }
                }
                // For reasons mysterious to me,
                // not returning a value *from every branch*
                // adds some 100 ns or so of overhead to all results --
                // which is quite significant for very short tests...
                0
            })
        });
    } else {
        b.iter(|| {
            for _ in 0..ITERATIONS {
                tx.send(&data, vec![], vec![]).unwrap();
                rx.recv().unwrap();
            }
            0
        });
    }
}

#[bench]
fn size_00_1(b: &mut test::Bencher) {
    bench_size(b, 1);
}
#[bench]
fn size_01_2(b: &mut test::Bencher) {
    bench_size(b, 2);
}
#[bench]
fn size_02_4(b: &mut test::Bencher) {
    bench_size(b, 4);
}
#[bench]
fn size_03_8(b: &mut test::Bencher) {
    bench_size(b, 8);
}
#[bench]
fn size_04_16(b: &mut test::Bencher) {
    bench_size(b, 16);
}
#[bench]
fn size_05_32(b: &mut test::Bencher) {
    bench_size(b, 32);
}
#[bench]
fn size_06_64(b: &mut test::Bencher) {
    bench_size(b, 64);
}
#[bench]
fn size_07_128(b: &mut test::Bencher) {
    bench_size(b, 128);
}
#[bench]
fn size_08_256(b: &mut test::Bencher) {
    bench_size(b, 256);
}
#[bench]
fn size_09_512(b: &mut test::Bencher) {
    bench_size(b, 512);
}
#[bench]
fn size_10_1k(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024);
}
#[bench]
fn size_11_2k(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024);
}
#[bench]
fn size_12_4k(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024);
}
#[bench]
fn size_13_8k(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024);
}
#[bench]
fn size_14_16k(b: &mut test::Bencher) {
    bench_size(b, 16 * 1024);
}
#[bench]
fn size_15_32k(b: &mut test::Bencher) {
    bench_size(b, 32 * 1024);
}
#[bench]
fn size_16_64k(b: &mut test::Bencher) {
    bench_size(b, 64 * 1024);
}
#[bench]
fn size_17_128k(b: &mut test::Bencher) {
    bench_size(b, 128 * 1024);
}
#[bench]
fn size_18_256k(b: &mut test::Bencher) {
    bench_size(b, 256 * 1024);
}
#[bench]
fn size_19_512k(b: &mut test::Bencher) {
    bench_size(b, 512 * 1024);
}
#[bench]
fn size_20_1m(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024 * 1024);
}
#[bench]
fn size_21_2m(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024 * 1024);
}
#[bench]
fn size_22_4m(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024 * 1024);
}
#[bench]
fn size_23_8m(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024 * 1024);
}
<|endoftext|>


<commit_before>use super::super::messages::{RequestMessage, Message, RequestPropose, RequestTransactions,
                             RequestPrevotes, RequestPrecommits, RequestBlock, Block};
use super::super::blockchain::{Blockchain, View};
use super::super::storage::{Map, List};
use super::super::events::Channel;
use super::{NodeHandler, ExternalMessage, NodeTimeout};

// TODO validate_heights нужно обновлять по любым сообщениям, а не только по status (если они корректно подписаны)
// TODO propose имеет смысл запрашивать только тогда, когда мы знаем, что узел находится на нашей высоте

const REQUEST_ALIVE: i64 = 3_000_000_000; // 3 seconds

impl<B, S> NodeHandler<B, S>
    where B: Blockchain,
          S: Channel<ApplicationEvent = ExternalMessage<B>, Timeout = NodeTimeout> + Clone
{
    pub fn handle_request(&mut self, msg: RequestMessage) {
        // Request are sended to us
        if msg.to() != &self.public_key {
            return;
        }

        // FIXME: we should use some epsilon for checking lifetime < 0
        let lifetime = match (self.channel.get_time() - msg.time()).num_nanoseconds() {
            Some(nanos) => nanos,
            None => {
                // Incorrect time into message
                return;
            }
        };

        // Incorrect time of the request
        if lifetime < 0 || lifetime > REQUEST_ALIVE {
            return;
        }

        if !msg.verify(msg.from()) {
            return;
        }

        match msg {
            RequestMessage::Propose(msg) => self.handle_request_propose(msg),
            RequestMessage::Transactions(msg) => self.handle_request_txs(msg),
            RequestMessage::Prevotes(msg) => self.handle_request_prevotes(msg),
            RequestMessage::Precommits(msg) => self.handle_request_precommits(msg),
            RequestMessage::Peers(msg) => self.handle_request_peers(msg),
            RequestMessage::Block(msg) => self.handle_request_block(msg),
        }
    }

    pub fn handle_request_propose(&mut self, msg: RequestPropose) {
        if msg.height() != self.state.height() {
            return;
        }

        let propose = if msg.height() == self.state.height() {
            self.state.propose(msg.propose_hash()).map(|p| p.message().raw().clone())
        } else {
            return;
        };

        if let Some(propose) = propose {
            self.send_to_peer(*msg.from(), &propose);
        }
    }

    pub fn handle_request_txs(&mut self, msg: RequestTransactions) {
        trace!("HANDLE TRANSACTIONS REQUEST!!!");
        let view = self.blockchain.view();
        for hash in msg.txs() {
            let tx = self.state
                .transactions()
                .get(hash)
                .cloned()
                .or_else(|| view.transactions().get(hash).unwrap());

            if let Some(tx) = tx {
                self.send_to_peer(*msg.from(), tx.raw());
            }
        }
    }

    pub fn handle_request_prevotes(&mut self, msg: RequestPrevotes) {
        if msg.height() != self.state.height() {
            return;
        }

        let has_prevotes = msg.validators();
        let prevotes = self.state
            .prevotes(msg.round(), *msg.propose_hash())
            .iter()
            .filter(|p| !has_prevotes[p.validator() as usize])
            .map(|p| p.raw().clone())
            .collect::<Vec<_>>();

        for prevote in &prevotes {
            self.send_to_peer(*msg.from(), prevote);
        }
    }

    pub fn handle_request_precommits(&mut self, msg: RequestPrecommits) {
        if msg.height() > self.state.height() {
            return;
        }

        let has_precommits = msg.validators();
        let precommits = self.state
            .precommits(msg.round(), *msg.propose_hash())
            .iter()
            .filter(|p| !has_precommits[p.validator() as usize])
            .map(|p| p.raw().clone())
            .collect::<Vec<_>>();

        // FIXME what about msg.height < state.height ?
        // self.blockchain
        //     .view()
        //     .precommits(msg.block_hash())
        //     .values()
        //     .unwrap()
        //     .iter()
        //     .map(|p| p.raw().clone())
        //     .collect()

        for precommit in precommits {
            self.send_to_peer(*msg.from(), &precommit);
        }
    }

    pub fn handle_request_block(&mut self, msg: RequestBlock) {
        trace!("Handle block request with height:{}, our height: {}",
               msg.height(),
               self.state.height());
        if msg.height() >= self.state.height() {
            return;
        }

        let view = self.blockchain.view();
        let height = msg.height();
        let block_hash = view.heights().get(height).unwrap().unwrap();

        let block = view.blocks().get(&block_hash).unwrap().unwrap();
        let precommits = view.precommits(&block_hash)
            .values()
            .unwrap()
            .iter()
            .cloned()
            .collect::<Vec<_>>();
        let transactions = view.block_txs(height)
            .values()
            .unwrap()
            .iter()
            .map(|tx_hash| view.transactions().get(tx_hash).unwrap().unwrap())
            .map(|p| p.raw().clone())
            .collect::<Vec<_>>();

        let block_msg = Block::new(&self.public_key,
                                   msg.from(),
                                   self.channel.get_time(),
                                   block,
                                   precommits,
                                   transactions,
                                   &self.secret_key);
        self.send_to_peer(*msg.from(), block_msg.raw());
    }
}
<commit_msg>fix issue-119<commit_after>use super::super::messages::{RequestMessage, Message, RequestPropose, RequestTransactions,
                             RequestPrevotes, RequestPrecommits, RequestBlock, Block};
use super::super::blockchain::{Blockchain, View};
use super::super::storage::{Map, List};
use super::super::events::Channel;
use super::{NodeHandler, ExternalMessage, NodeTimeout};

// TODO validate_heights нужно обновлять по любым сообщениям, а не только по status (если они корректно подписаны)
// TODO propose имеет смысл запрашивать только тогда, когда мы знаем, что узел находится на нашей высоте

const REQUEST_ALIVE: i64 = 3_000_000_000; // 3 seconds

impl<B, S> NodeHandler<B, S>
    where B: Blockchain,
          S: Channel<ApplicationEvent = ExternalMessage<B>, Timeout = NodeTimeout> + Clone
{
    pub fn handle_request(&mut self, msg: RequestMessage) {
        // Request are sended to us
        if msg.to() != &self.public_key {
            return;
        }

        // FIXME: we should use some epsilon for checking lifetime < 0
        let lifetime = match (self.channel.get_time() - msg.time()).num_nanoseconds() {
            Some(nanos) => nanos,
            None => {
                // Incorrect time into message
                return;
            }
        };

        // Incorrect time of the request
        if lifetime < 0 || lifetime > REQUEST_ALIVE {
            return;
        }

        if !msg.verify(msg.from()) {
            return;
        }

        match msg {
            RequestMessage::Propose(msg) => self.handle_request_propose(msg),
            RequestMessage::Transactions(msg) => self.handle_request_txs(msg),
            RequestMessage::Prevotes(msg) => self.handle_request_prevotes(msg),
            RequestMessage::Precommits(msg) => self.handle_request_precommits(msg),
            RequestMessage::Peers(msg) => self.handle_request_peers(msg),
            RequestMessage::Block(msg) => self.handle_request_block(msg),
        }
    }

    pub fn handle_request_propose(&mut self, msg: RequestPropose) {
        if msg.height() != self.state.height() {
            return;
        }

        let propose = if msg.height() == self.state.height() {
            self.state.propose(msg.propose_hash()).map(|p| p.message().raw().clone())
        } else {
            return;
        };

        if let Some(propose) = propose {
            self.send_to_peer(*msg.from(), &propose);
        }
    }

    pub fn handle_request_txs(&mut self, msg: RequestTransactions) {
        trace!("HANDLE TRANSACTIONS REQUEST!!!");
        let view = self.blockchain.view();
        for hash in msg.txs() {
            let tx = self.state
                .transactions()
                .get(hash)
                .cloned()
                .or_else(|| view.transactions().get(hash).unwrap());

            if let Some(tx) = tx {
                self.send_to_peer(*msg.from(), tx.raw());
            }
        }
    }

    pub fn handle_request_prevotes(&mut self, msg: RequestPrevotes) {
        if msg.height() != self.state.height() {
            return;
        }

        let has_prevotes = msg.validators();
        let prevotes = self.state
            .prevotes(msg.round(), *msg.propose_hash())
            .iter()
            .filter(|p| !has_prevotes[p.validator() as usize])
            .map(|p| p.raw().clone())
            .collect::<Vec<_>>();

        for prevote in &prevotes {
            self.send_to_peer(*msg.from(), prevote);
        }
    }

    pub fn handle_request_precommits(&mut self, msg: RequestPrecommits) {
        if msg.height() > self.state.height() {
            return;
        }

        let has_precommits = msg.validators();
        let precommits = self.state
            .precommits(msg.round(), *msg.block_hash())
            .iter()
            .filter(|p| !has_precommits[p.validator() as usize])
            .map(|p| p.raw().clone())
            .collect::<Vec<_>>();

        // FIXME what about msg.height < state.height ?
        // self.blockchain
        //     .view()
        //     .precommits(msg.block_hash())
        //     .values()
        //     .unwrap()
        //     .iter()
        //     .map(|p| p.raw().clone())
        //     .collect()

        for precommit in precommits {
            self.send_to_peer(*msg.from(), &precommit);
        }
    }

    pub fn handle_request_block(&mut self, msg: RequestBlock) {
        trace!("Handle block request with height:{}, our height: {}",
               msg.height(),
               self.state.height());
        if msg.height() >= self.state.height() {
            return;
        }

        let view = self.blockchain.view();
        let height = msg.height();
        let block_hash = view.heights().get(height).unwrap().unwrap();

        let block = view.blocks().get(&block_hash).unwrap().unwrap();
        let precommits = view.precommits(&block_hash)
            .values()
            .unwrap()
            .iter()
            .cloned()
            .collect::<Vec<_>>();
        let transactions = view.block_txs(height)
            .values()
            .unwrap()
            .iter()
            .map(|tx_hash| view.transactions().get(tx_hash).unwrap().unwrap())
            .map(|p| p.raw().clone())
            .collect::<Vec<_>>();

        let block_msg = Block::new(&self.public_key,
                                   msg.from(),
                                   self.channel.get_time(),
                                   block,
                                   precommits,
                                   transactions,
                                   &self.secret_key);
        self.send_to_peer(*msg.from(), block_msg.raw());
    }
}
<|endoftext|>


<commit_before>//! `sled` is a flash-sympathetic persistent lock-free B+ tree.
//!
//! # Examples
//!
//! ```
//! let t = sled::Db::start_default("my_db").unwrap();
//!
//! t.set(b"yo!", b"v1".to_vec());
//! assert!(t.get(b"yo!").unwrap().unwrap() == &*b"v1".to_vec());
//!
//! t.cas(
//!     b"yo!",                // key
//!     Some(b"v1"),           // old value, None for not present
//!     Some(b"v2".to_vec()),  // new value, None for delete
//! ).unwrap();
//!
//! let mut iter = t.scan(b"a non-present key before yo!");
//! // assert_eq!(iter.next(), Some(Ok((b"yo!".to_vec(), b"v2".to_vec()))));
//! // assert_eq!(iter.next(), None);
//!
//! t.del(b"yo!");
//! assert_eq!(t.get(b"yo!"), Ok(None));
//! ```

#![deny(missing_docs)]
#![cfg_attr(test, deny(clippy::warnings))]
#![cfg_attr(test, deny(clippy::bad_style))]
#![cfg_attr(test, deny(clippy::future_incompatible))]
#![cfg_attr(test, deny(clippy::nonstandard_style))]
#![cfg_attr(test, deny(clippy::rust_2018_compatibility))]
#![cfg_attr(test, deny(clippy::rust_2018_idioms))]

mod binary_search;
mod data;
mod db;
mod frag;
mod iter;
mod ivec;
mod materializer;
mod meta;
mod node;
mod pinned_value;
mod prefix;
mod recovery;
mod subscription;
mod tree;

const META_PID: PageId = 0;
const COUNTER_PID: PageId = 1;
const DEFAULT_TREE_PID: PageId = 3;
const TX_TREE_PID: PageId = 5;

const DEFAULT_TREE_ID: &[u8] = b"__sled__default";
const TX_TREE_ID: &[u8] = b"__sled__transactions";

pub use {
    self::{
        db::Db,
        iter::{Iter, Keys, Values},
        pinned_value::PinnedValue,
        subscription::{Event, Subscriber},
        tree::Tree,
    },
    pagecache::{Config, ConfigBuilder, Error, Meta, Result},
};

use {
    self::{
        binary_search::{
            binary_search_gt, binary_search_lt, binary_search_lub,
            leaf_search,
        },
        data::Data,
        frag::{ChildSplit, Frag, ParentSplit},
        ivec::IVec,
        materializer::BLinkMaterializer,
        node::Node,
        prefix::{
            prefix_cmp, prefix_cmp_encoded, prefix_decode,
            prefix_encode,
        },
        recovery::Recovery,
        subscription::Subscriptions,
    },
    log::{debug, error, trace},
    pagecache::{
        Materializer, Measure, MergeOperator, PageCache, PageGet,
        PageId, M,
    },
    serde::{Deserialize, Serialize},
    sled_sync::{debug_delay, pin, Guard},
};

type Key = Vec<u8>;
type Value = Vec<u8>;

type TreePtr<'g> = pagecache::PagePtr<'g, Frag>;
<commit_msg>Small warning cleanup<commit_after>//! `sled` is a flash-sympathetic persistent lock-free B+ tree.
//!
//! # Examples
//!
//! ```
//! let t = sled::Db::start_default("my_db").unwrap();
//!
//! t.set(b"yo!", b"v1".to_vec());
//! assert!(t.get(b"yo!").unwrap().unwrap() == &*b"v1".to_vec());
//!
//! t.cas(
//!     b"yo!",                // key
//!     Some(b"v1"),           // old value, None for not present
//!     Some(b"v2".to_vec()),  // new value, None for delete
//! ).unwrap();
//!
//! let mut iter = t.scan(b"a non-present key before yo!");
//! // assert_eq!(iter.next(), Some(Ok((b"yo!".to_vec(), b"v2".to_vec()))));
//! // assert_eq!(iter.next(), None);
//!
//! t.del(b"yo!");
//! assert_eq!(t.get(b"yo!"), Ok(None));
//! ```

#![deny(missing_docs)]
#![cfg_attr(test, deny(clippy::warnings))]
#![cfg_attr(test, deny(clippy::bad_style))]
#![cfg_attr(test, deny(clippy::future_incompatible))]
#![cfg_attr(test, deny(clippy::nonstandard_style))]
#![cfg_attr(test, deny(clippy::rust_2018_compatibility))]
#![cfg_attr(test, deny(clippy::rust_2018_idioms))]

mod binary_search;
mod data;
mod db;
mod frag;
mod iter;
mod ivec;
mod materializer;
mod meta;
mod node;
mod pinned_value;
mod prefix;
mod recovery;
mod subscription;
mod tree;

const META_PID: PageId = 0;
const COUNTER_PID: PageId = 1;

const DEFAULT_TREE_ID: &[u8] = b"__sled__default";
const TX_TREE_ID: &[u8] = b"__sled__transactions";

pub use {
    self::{
        db::Db,
        iter::{Iter, Keys, Values},
        pinned_value::PinnedValue,
        subscription::{Event, Subscriber},
        tree::Tree,
    },
    pagecache::{Config, ConfigBuilder, Error, Meta, Result},
};

use {
    self::{
        binary_search::{
            binary_search_gt, binary_search_lt, binary_search_lub,
            leaf_search,
        },
        data::Data,
        frag::{ChildSplit, Frag, ParentSplit},
        ivec::IVec,
        materializer::BLinkMaterializer,
        node::Node,
        prefix::{
            prefix_cmp, prefix_cmp_encoded, prefix_decode,
            prefix_encode,
        },
        recovery::Recovery,
        subscription::Subscriptions,
    },
    log::{debug, error, trace},
    pagecache::{
        Materializer, Measure, MergeOperator, PageCache, PageGet,
        PageId, M,
    },
    serde::{Deserialize, Serialize},
    sled_sync::{debug_delay, pin, Guard},
};

type Key = Vec<u8>;
type Value = Vec<u8>;

type TreePtr<'g> = pagecache::PagePtr<'g, Frag>;
<|endoftext|>


<commit_before>use std::collections::{BTreeMap, HashSet};
use std::hash::Hasher;
use std::iter;
use std::str::CharIndices;
use std::ops;

use itertools::{GroupBy, Itertools};

use Query::*;
use super::*;
use util::*;

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    /// Constructs a new, empty InvertedIndex
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        let analyzed = analyze_doc(doc.content());
        for (ngram, position) in analyzed {
            self.index
                .entry(ngram)
                .or_insert_with(BTreeMap::new)
                .entry(doc.id.clone())
                .or_insert_with(Vec::new)
                .search_coalesce(0, position);
        }
    }

    /// Performs a search to the specification of the given query
    pub fn query(&self, query: &Query) -> Vec<SearchResult> {
        let postings = self.query_rec(query);
        self.compute_results(postings)
    }

    /// A helper method for performing a Match query
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        self.query(&Match(query))
    }

    fn postings(&self, query: &str) -> PostingsMap {
        // TODO(tjk): split this out into `fn analyze(query: &str) -> Iterator<String>`
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        unique_terms.into_iter().flat_map(|word| self.index.get(&word)).merge_postings()
    }

    fn phrase(&self, phrase: &str) -> PostingsMap {
        // TODO(tjk): split this out into `fn analyze(query: &str) -> Iterator<String>`
        let terms: Vec<_> = phrase.split_whitespace().map(str::to_lowercase).collect();
        let postings: Vec<_> = terms.windows(2)
            .map(|adjacent_terms| {
            let term0 = &adjacent_terms[0];
            let term1 = &adjacent_terms[1];
            if let (Some(posting0), Some(posting1)) = (self.index.get(term0), self.index.get(term1)) {
                posting0.intersect_positionally(posting1)
            } else {
                PostingsMap::new()
            }
        }).collect();
        postings.intersect_postings()
    }

    fn query_rec(&self, query: &Query) -> PostingsMap {
        match *query {
            Match(query) => self.postings(query),
            And(queries) => {
                let postings: Vec<_> = queries.iter().map(|q| self.query_rec(q)).collect();
                postings.intersect_postings()
            }
            Or(queries) => queries.into_iter().map(|q| self.query_rec(q)).merge_postings(),
            Phrase(phrase) => self.phrase(phrase)
        }
    }

    fn compute_results(&self, postings: PostingsMap) -> Vec<SearchResult> {
        let mut results: Vec<_> = postings.into_iter()
                                          .map(|(doc_id, positions)| {
                                              SearchResult::new(&self.docs[&doc_id], positions)
                                          })
                                          .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

type Tokens<'a> = iter::FlatMap<
                iter::Enumerate<
                    iter::Filter<
                        GroupBy<bool, CharIndices<'a>, fn(&(usize, char)) -> bool>,
                        fn(&(bool, Vec<(usize, char)>)) -> bool>>,
                iter::Map<ops::Range<usize>, Ngrams>,
                fn((usize, (bool, Vec<(usize, char)>))) -> iter::Map<ops::Range<usize>, Ngrams>>;

fn analyze_doc(doc: &str) -> Tokens {
    doc.char_indices()
       .group_by(is_whitespace as fn(&(usize, char)) -> bool)
       .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
       .enumerate()
       .flat_map(ngrams)
}

fn ngrams((position, (_, chars)): (usize, (bool, Vec<(usize, char)>)))
          -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(position, chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    position: usize,
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(position: usize, chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams {
            position: position,
            chars: chars,
        }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, Position) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, Position::new((start, finish), self.position))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, Position) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, Position);
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, Position) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use Query::{And, Match, Or};
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![Position::new((6, 8), 1),
                                                            Position::new((25, 27), 5)]),
                                        (doc2, vec![Position::new((13, 15), 3)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.positions,
                       &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2",
                                 "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref positions, .. } = search_results.iter().next().unwrap();
        let Position{offsets:(begin, end), ..} = positions[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].positions,
                   vec![Position::new((0, 2), 0)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].positions,
                   vec![Position::new((0, 2), 0)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].positions,
                   vec![Position::new((0, 2), 0)]);
    }

    #[test]
    fn test_and() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&And(&[Match("today"), Match("you")]));
        let expected: BTreeMap<_, _> = [(doc2,
                                         vec![Position::new((9, 12), 2),
                                              Position::new((13, 18), 3)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.positions,
                       &expected[search_result.doc])
        }
    }

    #[test]
    fn test_and_or() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&Or(&[Match("you"),
                                               And(&[Match("today"), Match("you")])]));
        let expected: BTreeMap<_, _> = [(doc2,
                                         vec![Position::new((9, 12), 2),
                                              Position::new((13, 18), 3)]),
                                        (doc3, vec![Position::new((9, 12), 2)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.positions,
                       &expected[search_result.doc])
        }
    }

    #[test]
    fn test_phrase() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        index.index(doc1.clone());
        println!("{:?}", index.phrase("learn to program"));
        println!("{:?}", index.phrase("learn to pro"));
    }
}
<commit_msg>Factor out query analysis into a function.<commit_after>use std::collections::BTreeMap;
use std::hash::Hasher;
use std::iter;
use std::str::{CharIndices, SplitWhitespace};
use std::ops;

use itertools::{GroupBy, Itertools};

use Query::*;
use super::*;
use util::*;

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    /// Constructs a new, empty InvertedIndex
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        let analyzed = analyze_doc(doc.content());
        for (ngram, position) in analyzed {
            self.index
                .entry(ngram)
                .or_insert_with(BTreeMap::new)
                .entry(doc.id.clone())
                .or_insert_with(Vec::new)
                .search_coalesce(0, position);
        }
    }

    /// Performs a search to the specification of the given query
    pub fn query(&self, query: &Query) -> Vec<SearchResult> {
        let postings = self.query_rec(query);
        self.compute_results(postings)
    }

    /// A helper method for performing a Match query
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        self.query(&Match(query))
    }

    fn postings(&self, query: &str) -> PostingsMap {
        analyze_query(query).unique().flat_map(|word| self.index.get(&word)).merge_postings()
    }

    fn phrase(&self, phrase: &str) -> PostingsMap {
        let terms: Vec<_> = analyze_query(phrase).collect();
        let postings: Vec<_> = terms.windows(2)
            .map(|adjacent_terms| {
            let term0 = &adjacent_terms[0];
            let term1 = &adjacent_terms[1];
            if let (Some(posting0), Some(posting1)) = (self.index.get(term0), self.index.get(term1)) {
                posting0.intersect_positionally(posting1)
            } else {
                PostingsMap::new()
            }
        }).collect();
        postings.intersect_postings()
    }

    fn query_rec(&self, query: &Query) -> PostingsMap {
        match *query {
            Match(query) => self.postings(query),
            And(queries) => {
                let postings: Vec<_> = queries.iter().map(|q| self.query_rec(q)).collect();
                postings.intersect_postings()
            }
            Or(queries) => queries.into_iter().map(|q| self.query_rec(q)).merge_postings(),
            Phrase(phrase) => self.phrase(phrase)
        }
    }

    fn compute_results(&self, postings: PostingsMap) -> Vec<SearchResult> {
        let mut results: Vec<_> = postings.into_iter()
                                          .map(|(doc_id, positions)| {
                                              SearchResult::new(&self.docs[&doc_id], positions)
                                          })
                                          .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

type Tokens<'a> = iter::FlatMap<
                iter::Enumerate<
                    iter::Filter<
                        GroupBy<bool, CharIndices<'a>, fn(&(usize, char)) -> bool>,
                        fn(&(bool, Vec<(usize, char)>)) -> bool>>,
                iter::Map<ops::Range<usize>, Ngrams>,
                fn((usize, (bool, Vec<(usize, char)>))) -> iter::Map<ops::Range<usize>, Ngrams>>;

type QueryTokens<'a> = iter::Map<SplitWhitespace<'a>, fn(&str) -> String>;

fn analyze_query(query: &str) -> QueryTokens {
    query.split_whitespace().map(str::to_lowercase)
}

fn analyze_doc(doc: &str) -> Tokens {
    doc.char_indices()
       .group_by(is_whitespace as fn(&(usize, char)) -> bool)
       .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
       .enumerate()
       .flat_map(ngrams)
}

fn ngrams((position, (_, chars)): (usize, (bool, Vec<(usize, char)>)))
          -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(position, chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    position: usize,
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(position: usize, chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams {
            position: position,
            chars: chars,
        }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, Position) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, Position::new((start, finish), self.position))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, Position) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, Position);
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, Position) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use Query::{And, Match, Or};
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![Position::new((6, 8), 1),
                                                            Position::new((25, 27), 5)]),
                                        (doc2, vec![Position::new((13, 15), 3)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.positions,
                       &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2",
                                 "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref positions, .. } = search_results.iter().next().unwrap();
        let Position{offsets:(begin, end), ..} = positions[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].positions,
                   vec![Position::new((0, 2), 0)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].positions,
                   vec![Position::new((0, 2), 0)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].positions,
                   vec![Position::new((0, 2), 0)]);
    }

    #[test]
    fn test_and() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&And(&[Match("today"), Match("you")]));
        let expected: BTreeMap<_, _> = [(doc2,
                                         vec![Position::new((9, 12), 2),
                                              Position::new((13, 18), 3)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.positions,
                       &expected[search_result.doc])
        }
    }

    #[test]
    fn test_and_or() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&Or(&[Match("you"),
                                               And(&[Match("today"), Match("you")])]));
        let expected: BTreeMap<_, _> = [(doc2,
                                         vec![Position::new((9, 12), 2),
                                              Position::new((13, 18), 3)]),
                                        (doc3, vec![Position::new((9, 12), 2)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.positions,
                       &expected[search_result.doc])
        }
    }

    #[test]
    fn test_phrase() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        index.index(doc1.clone());
        println!("{:?}", index.phrase("learn to program"));
        println!("{:?}", index.phrase("learn to pro"));
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#![feature(borrow_state)]
#![feature(box_syntax)]
#![feature(core_intrinsics)]
#![feature(filling_drop)]
#![feature(iter_arith)]
#![feature(link_args)]
#![feature(plugin)]
#![feature(unicode)]
#![feature(unsafe_no_drop_flag)]

#![allow(non_camel_case_types)]

#![plugin(plugins)]

#[macro_use]
extern crate log;

extern crate servo;
extern crate compositing;

extern crate euclid;
extern crate gleam;
extern crate glutin_app;
extern crate layers;
extern crate rustc_unicode;
extern crate script_traits;

extern crate net_traits;
extern crate msg;
extern crate util;
extern crate style_traits;

extern crate libc;
extern crate url as std_url;

#[cfg(target_os="macos")]
#[link_args="-Xlinker -undefined -Xlinker dynamic_lookup"]
extern { }

#[cfg(target_os="macos")]
extern crate cocoa;
#[cfg(target_os="macos")]
#[macro_use]
extern crate objc;
#[cfg(target_os="linux")] extern crate x11;

// Must come first.
pub mod macros;

pub mod browser;
pub mod browser_host;
pub mod command_line;
pub mod cookie;
pub mod core;
pub mod drag_data;
pub mod eutil;
pub mod frame;
pub mod interfaces;
pub mod print_settings;
pub mod process_message;
pub mod render_handler;
pub mod request;
pub mod request_context;
pub mod response;
pub mod stream;
pub mod string;
pub mod string_list;
pub mod string_map;
pub mod string_multimap;
pub mod stubs;
pub mod switches;
pub mod task;
pub mod types;
pub mod urlrequest;
pub mod v8;
pub mod values;
pub mod window;
pub mod wrappers;
pub mod xml_reader;
pub mod zip_reader;
<commit_msg>Auto merge of #12860 - Coder206:iterWarn, r=Wafflespeanut<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#![feature(borrow_state)]
#![feature(box_syntax)]
#![feature(core_intrinsics)]
#![feature(filling_drop)]
#![feature(link_args)]
#![feature(plugin)]
#![feature(unicode)]
#![feature(unsafe_no_drop_flag)]

#![allow(non_camel_case_types)]

#![plugin(plugins)]

#[macro_use]
extern crate log;

extern crate servo;
extern crate compositing;

extern crate euclid;
extern crate gleam;
extern crate glutin_app;
extern crate layers;
extern crate rustc_unicode;
extern crate script_traits;

extern crate net_traits;
extern crate msg;
extern crate util;
extern crate style_traits;

extern crate libc;
extern crate url as std_url;

#[cfg(target_os="macos")]
#[link_args="-Xlinker -undefined -Xlinker dynamic_lookup"]
extern { }

#[cfg(target_os="macos")]
extern crate cocoa;
#[cfg(target_os="macos")]
#[macro_use]
extern crate objc;
#[cfg(target_os="linux")] extern crate x11;

// Must come first.
pub mod macros;

pub mod browser;
pub mod browser_host;
pub mod command_line;
pub mod cookie;
pub mod core;
pub mod drag_data;
pub mod eutil;
pub mod frame;
pub mod interfaces;
pub mod print_settings;
pub mod process_message;
pub mod render_handler;
pub mod request;
pub mod request_context;
pub mod response;
pub mod stream;
pub mod string;
pub mod string_list;
pub mod string_map;
pub mod string_multimap;
pub mod stubs;
pub mod switches;
pub mod task;
pub mod types;
pub mod urlrequest;
pub mod v8;
pub mod values;
pub mod window;
pub mod wrappers;
pub mod xml_reader;
pub mod zip_reader;
<|endoftext|>