<commit_before>use params::{Map as ParamsMap, Params, Value};
use router::Router;
use iron::prelude::*;
use bodyparser;

use std::str;
use std::num::ParseIntError;

use exonum::api::{Api, ApiError};
use exonum::crypto::{PublicKey, SecretKey, Hash, HexValue};
use exonum::blockchain::{Blockchain, StoredConfiguration, Schema};
use exonum::storage::{Map, StorageValue};
use exonum::node::{TxSender, NodeChannel, TransactionSend};
use exonum::encoding::serialize::json::reexport as serde_json;

use super::{StorageValueConfigProposeData, TxConfigPropose, TxConfigVote, ConfigTx,
            ConfigurationSchema};

pub type ConfigTxSender = TxSender<NodeChannel>;

#[derive(Serialize, Deserialize)]
pub struct ApiResponseConfigHashInfo {
    pub hash: Hash,
    pub config: StoredConfiguration,
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseProposeHashInfo {
    pub hash: Hash,
    pub propose_data: StorageValueConfigProposeData,
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseConfigInfo {
    pub committed_config: Option<StoredConfiguration>,
    pub propose: Option<StorageValueConfigProposeData>,
}

#[derive(Serialize, Deserialize)]
pub enum ApiResponseVotesInfo {
    Votes(Vec<Option<TxConfigVote>>),
    ProposeAbsent(Option<()>),
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseProposePost {
    pub tx_hash: Hash,
    pub cfg_hash: Hash,
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseVotePost {
    pub tx_hash: Hash,
}

#[derive(Clone)]
pub struct PrivateConfigApi<T: TransactionSend + Clone> {
    pub channel: T,
    pub config: (PublicKey, SecretKey),
}

#[derive(Clone)]
pub struct PublicConfigApi {
    pub blockchain: Blockchain,
}

impl PublicConfigApi {
    fn get_actual_config(&self) -> Result<ApiResponseConfigHashInfo, ApiError> {

        let actual_cfg = Schema::new(&self.blockchain.view()).actual_configuration()?;
        let res = ApiResponseConfigHashInfo {
            hash: actual_cfg.hash(),
            config: actual_cfg,
        };
        Ok(res)
    }

    fn get_following_config(&self) -> Result<Option<ApiResponseConfigHashInfo>, ApiError> {

        let following_cfg = Schema::new(&self.blockchain.view())
            .following_configuration()?;
        let res = following_cfg.map(|cfg| {
                                        ApiResponseConfigHashInfo {
                                            hash: cfg.hash(),
                                            config: cfg,
                                        }
                                    });
        Ok(res)
    }

    fn get_config_by_hash(&self, hash: &Hash) -> Result<ApiResponseConfigInfo, ApiError> {
        let view = self.blockchain.view();
        let general_schema = Schema::new(&view);
        let committed_config = general_schema.configs().get(hash)?;

        let configuration_schema = ConfigurationSchema::new(&view);
        let propose = configuration_schema
            .propose_data_by_config_hash()
            .get(hash)?;
        let res = ApiResponseConfigInfo {
            committed_config: committed_config,
            propose: propose,
        };
        Ok(res)
    }

    fn get_votes_for_propose(&self, cfg_hash: &Hash) -> Result<ApiResponseVotesInfo, ApiError> {
        let view = self.blockchain.view();
        let configuration_schema = ConfigurationSchema::new(&view);
        let res = match configuration_schema
                  .propose_data_by_config_hash()
                  .get(cfg_hash)? {
            None => ApiResponseVotesInfo::ProposeAbsent(None),
            Some(_) => ApiResponseVotesInfo::Votes(configuration_schema.get_votes(cfg_hash)?),
        };
        Ok(res)
    }

    fn filter_cfg_predicate(cfg: &StoredConfiguration,
                            previous_cfg_hash_filter: Option<Hash>,
                            actual_from_filter: Option<u64>)
                            -> bool {
        if let Some(prev_ref) = previous_cfg_hash_filter {
            if cfg.previous_cfg_hash != prev_ref {
                return false;
            }
        }
        if let Some(from_height) = actual_from_filter {
            if cfg.actual_from < from_height {
                return false;
            }
        }
        true
    }

    fn get_all_proposes(&self,
                        previous_cfg_hash_filter: Option<Hash>,
                        actual_from_filter: Option<u64>)
                        -> Result<Vec<ApiResponseProposeHashInfo>, ApiError> {
        let view = self.blockchain.view();
        let configuration_schema = ConfigurationSchema::new(&view);
        let proposes: Vec<Hash> = configuration_schema.config_hash_by_ordinal().values()?;
        let mut res: Vec<_> = Vec::new();
        for cfg_hash in proposes {
            let propose_data =
                configuration_schema
                    .propose_data_by_config_hash()
                    .get(&cfg_hash)?
                    .expect(&format!("Not found propose for following cfg_hash: {:?}", cfg_hash));
            let cfg = <StoredConfiguration as StorageValue>::deserialize(propose_data
                                                                             .tx_propose()
                                                                             .cfg()
                                                                             .as_bytes()
                                                                             .to_vec());
            if !PublicConfigApi::filter_cfg_predicate(&cfg,
                                                      previous_cfg_hash_filter,
                                                      actual_from_filter) {
                continue;
            }
            let elem = ApiResponseProposeHashInfo {
                hash: cfg_hash,
                propose_data: propose_data,
            };
            res.push(elem);
        }
        Ok(res)
    }

    fn get_all_committed(&self,
                         previous_cfg_hash_filter: Option<Hash>,
                         actual_from_filter: Option<u64>)
                         -> Result<Vec<ApiResponseConfigHashInfo>, ApiError> {
        let view = self.blockchain.view();
        let general_schema = Schema::new(&view);
        let references = general_schema.configs_actual_from().values()?;
        let mut res: Vec<_> = Vec::new();
        for reference in references {
            let cfg_hash = reference.cfg_hash();
            let cfg =
                general_schema
                    .configs()
                    .get(cfg_hash)?
                    .expect(&format!("Config with hash {:?} is absent in configs table", cfg_hash));
            if !PublicConfigApi::filter_cfg_predicate(&cfg,
                                                      previous_cfg_hash_filter,
                                                      actual_from_filter) {
                continue;
            }
            let elem = ApiResponseConfigHashInfo {
                hash: *cfg_hash,
                config: cfg,
            };
            res.push(elem);
        }
        Ok(res)
    }

    fn retrieve_params(map: &ParamsMap) -> Result<(Option<Hash>, Option<u64>), ApiError> {
        let actual_from: Option<u64>;
        let previous_cfg_hash: Option<Hash>;
        previous_cfg_hash = match map.find(&["previous_cfg_hash"]) {
            Some(&Value::String(ref hash_string)) => {
                Some(Hash::from_hex(hash_string).map_err(ApiError::FromHex)?)
            }
            _ => None,
        };
        actual_from = match map.find(&["actual_from"]) {
            Some(&Value::String(ref from_str)) => {
                Some(from_str
                         .parse()
                         .map_err(|e: ParseIntError| ApiError::IncorrectRequest(Box::new(e)))?)
            }
            _ => None,
        };
        Ok((previous_cfg_hash, actual_from))
    }
}

impl<T> PrivateConfigApi<T>
    where T: TransactionSend + Clone
{
    fn put_config_propose(&self,
                          cfg: StoredConfiguration)
                          -> Result<ApiResponseProposePost, ApiError> {
        let cfg_hash = cfg.hash();
        let config_propose = TxConfigPropose::new(&self.config.0,
                                                  str::from_utf8(cfg.serialize().as_slice())
                                                      .unwrap(),
                                                  &self.config.1);
        let tx_hash = config_propose.hash();
        let ch = self.channel.clone();
        ch.send(ConfigTx::ConfigPropose(config_propose))?;
        let res = ApiResponseProposePost {
            tx_hash: tx_hash,
            cfg_hash: cfg_hash,
        };
        Ok(res)
    }

    fn put_config_vote(&self, cfg_hash: &Hash) -> Result<ApiResponseVotePost, ApiError> {
        let config_vote = TxConfigVote::new(&self.config.0, cfg_hash, &self.config.1);
        let tx_hash = config_vote.hash();
        let ch = self.channel.clone();
        ch.send(ConfigTx::ConfigVote(config_vote))?;
        let res = ApiResponseVotePost { tx_hash: tx_hash };
        Ok(res)
    }
}

impl Api for PublicConfigApi {
    fn wire(&self, router: &mut Router) {

        let _self = self.clone();
        let config_actual = move |_: &mut Request| -> IronResult<Response> {
            let info = _self.get_actual_config()?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };

        let _self = self.clone();
        let config_following = move |_: &mut Request| -> IronResult<Response> {
            let info = _self.get_following_config()?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };

        let _self = self.clone();
        let config_by_hash = move |req: &mut Request| -> IronResult<Response> {
            let params = req.extensions.get::<Router>().unwrap();
            match params.find("hash") {
                Some(hash_str) => {
                    let hash = Hash::from_hex(hash_str).map_err(ApiError::from)?;
                    let info = _self.get_config_by_hash(&hash)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                None => {
                    Err(ApiError::IncorrectRequest("Required route \
                                           parameter of configuration \
                                           'hash' is missing"
                                                           .into()))?
                }
            }
        };

        let _self = self.clone();
        let get_votes_for_propose = move |req: &mut Request| -> IronResult<Response> {
            let params = req.extensions.get::<Router>().unwrap();
            match params.find("hash") {
                Some(hash_str) => {
                    let propose_cfg_hash = Hash::from_hex(hash_str).map_err(ApiError::from)?;
                    let info = _self.get_votes_for_propose(&propose_cfg_hash)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                None => {
                    Err(ApiError::IncorrectRequest("Required route \
                                           parameter of configuration \
                                           'hash' is missing"
                                                           .into()))?
                }
            }
        };

        let _self = self.clone();
        let get_all_proposes = move |req: &mut Request| -> IronResult<Response> {
            let map = req.get_ref::<Params>().unwrap();
            let (previous_cfg_hash, actual_from) = PublicConfigApi::retrieve_params(map)?;
            let info = _self.get_all_proposes(previous_cfg_hash, actual_from)?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };

        let _self = self.clone();
        let get_all_committed = move |req: &mut Request| -> IronResult<Response> {
            let map = req.get_ref::<Params>().unwrap();
            let (previous_cfg_hash, actual_from) = PublicConfigApi::retrieve_params(map)?;
            let info = _self.get_all_committed(previous_cfg_hash, actual_from)?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };
        router.get("/v1/configs/actual", config_actual, "config_actual");
        router.get("/v1/configs/following",
                   config_following,
                   "config_following");
        router.get("/v1/configs/:hash", config_by_hash, "config_by_hash");
        router.get("/v1/configs/:hash/votes",
                   get_votes_for_propose,
                   "get_votes_for_propose");
        router.get("/v1/configs/proposed", get_all_proposes, "get_all_proposes");
        router.get("/v1/configs/committed",
                   get_all_committed,
                   "get_all_committed");

    }
}

impl<T> Api for PrivateConfigApi<T>
    where T: 'static + TransactionSend + Clone
{
    fn wire(&self, router: &mut Router) {
        let _self = self.clone();
        let put_config_propose = move |req: &mut Request| -> IronResult<Response> {
            match req.get::<bodyparser::Struct<StoredConfiguration>>() {
                Ok(Some(cfg)) => {
                    let info = _self.put_config_propose(cfg)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                Ok(None) => Err(ApiError::IncorrectRequest("Empty request body".into()))?,
                Err(e) => Err(ApiError::IncorrectRequest(Box::new(e)))?,
            }
        };

        let _self = self.clone();
        let put_config_vote = move |req: &mut Request| -> IronResult<Response> {
            let params = req.extensions.get::<Router>().unwrap();
            match params.find("hash") {
                Some(hash_str) => {
                    let propose_cfg_hash = Hash::from_hex(hash_str).map_err(ApiError::from)?;
                    let info = _self.put_config_vote(&propose_cfg_hash)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                None => {
                    Err(ApiError::IncorrectRequest("Required route \
                                           parameter of configuration \
                                           'hash' is missing"
                                                           .into()))?
                }
            }
        };
        router.post("/v1/configs/postpropose",
                    put_config_propose,
                    "put_config_propose");
        router.post("/v1/configs/:hash/postvote",
                    put_config_vote,
                    "put_config_vote");
    }
}
<commit_msg>Redefine ApiResponseVotesInfo as option<commit_after>use params::{Map as ParamsMap, Params, Value};
use router::Router;
use iron::prelude::*;
use bodyparser;

use std::str;
use std::num::ParseIntError;

use exonum::api::{Api, ApiError};
use exonum::crypto::{PublicKey, SecretKey, Hash, HexValue};
use exonum::blockchain::{Blockchain, StoredConfiguration, Schema};
use exonum::storage::{Map, StorageValue};
use exonum::node::{TxSender, NodeChannel, TransactionSend};
use exonum::encoding::serialize::json::reexport as serde_json;

use super::{StorageValueConfigProposeData, TxConfigPropose, TxConfigVote, ConfigTx,
            ConfigurationSchema};

pub type ConfigTxSender = TxSender<NodeChannel>;

#[derive(Serialize, Deserialize)]
pub struct ApiResponseConfigHashInfo {
    pub hash: Hash,
    pub config: StoredConfiguration,
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseProposeHashInfo {
    pub hash: Hash,
    pub propose_data: StorageValueConfigProposeData,
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseConfigInfo {
    pub committed_config: Option<StoredConfiguration>,
    pub propose: Option<StorageValueConfigProposeData>,
}

pub type ApiResponseVotesInfo = Option<Vec<Option<TxConfigVote>>>;

#[derive(Serialize, Deserialize)]
pub struct ApiResponseProposePost {
    pub tx_hash: Hash,
    pub cfg_hash: Hash,
}

#[derive(Serialize, Deserialize)]
pub struct ApiResponseVotePost {
    pub tx_hash: Hash,
}

#[derive(Clone)]
pub struct PrivateConfigApi<T: TransactionSend + Clone> {
    pub channel: T,
    pub config: (PublicKey, SecretKey),
}

#[derive(Clone)]
pub struct PublicConfigApi {
    pub blockchain: Blockchain,
}

impl PublicConfigApi {
    fn get_actual_config(&self) -> Result<ApiResponseConfigHashInfo, ApiError> {

        let actual_cfg = Schema::new(&self.blockchain.view()).actual_configuration()?;
        let res = ApiResponseConfigHashInfo {
            hash: actual_cfg.hash(),
            config: actual_cfg,
        };
        Ok(res)
    }

    fn get_following_config(&self) -> Result<Option<ApiResponseConfigHashInfo>, ApiError> {

        let following_cfg = Schema::new(&self.blockchain.view())
            .following_configuration()?;
        let res = following_cfg.map(|cfg| {
                                        ApiResponseConfigHashInfo {
                                            hash: cfg.hash(),
                                            config: cfg,
                                        }
                                    });
        Ok(res)
    }

    fn get_config_by_hash(&self, hash: &Hash) -> Result<ApiResponseConfigInfo, ApiError> {
        let view = self.blockchain.view();
        let general_schema = Schema::new(&view);
        let committed_config = general_schema.configs().get(hash)?;

        let configuration_schema = ConfigurationSchema::new(&view);
        let propose = configuration_schema
            .propose_data_by_config_hash()
            .get(hash)?;
        let res = ApiResponseConfigInfo {
            committed_config: committed_config,
            propose: propose,
        };
        Ok(res)
    }

    fn get_votes_for_propose(&self, cfg_hash: &Hash) -> Result<ApiResponseVotesInfo, ApiError> {
        let view = self.blockchain.view();
        let configuration_schema = ConfigurationSchema::new(&view);
        let res = match configuration_schema
                  .propose_data_by_config_hash()
                  .get(cfg_hash)? {
            None => None,
            Some(_) => Some(configuration_schema.get_votes(cfg_hash)?),
        };

        Ok(res)
    }

    fn filter_cfg_predicate(cfg: &StoredConfiguration,
                            previous_cfg_hash_filter: Option<Hash>,
                            actual_from_filter: Option<u64>)
                            -> bool {
        if let Some(prev_ref) = previous_cfg_hash_filter {
            if cfg.previous_cfg_hash != prev_ref {
                return false;
            }
        }
        if let Some(from_height) = actual_from_filter {
            if cfg.actual_from < from_height {
                return false;
            }
        }
        true
    }

    fn get_all_proposes(&self,
                        previous_cfg_hash_filter: Option<Hash>,
                        actual_from_filter: Option<u64>)
                        -> Result<Vec<ApiResponseProposeHashInfo>, ApiError> {
        let view = self.blockchain.view();
        let configuration_schema = ConfigurationSchema::new(&view);
        let proposes: Vec<Hash> = configuration_schema.config_hash_by_ordinal().values()?;
        let mut res: Vec<_> = Vec::new();
        for cfg_hash in proposes {
            let propose_data =
                configuration_schema
                    .propose_data_by_config_hash()
                    .get(&cfg_hash)?
                    .expect(&format!("Not found propose for following cfg_hash: {:?}", cfg_hash));
            let cfg = <StoredConfiguration as StorageValue>::deserialize(propose_data
                                                                             .tx_propose()
                                                                             .cfg()
                                                                             .as_bytes()
                                                                             .to_vec());
            if !PublicConfigApi::filter_cfg_predicate(&cfg,
                                                      previous_cfg_hash_filter,
                                                      actual_from_filter) {
                continue;
            }
            let elem = ApiResponseProposeHashInfo {
                hash: cfg_hash,
                propose_data: propose_data,
            };
            res.push(elem);
        }
        Ok(res)
    }

    fn get_all_committed(&self,
                         previous_cfg_hash_filter: Option<Hash>,
                         actual_from_filter: Option<u64>)
                         -> Result<Vec<ApiResponseConfigHashInfo>, ApiError> {
        let view = self.blockchain.view();
        let general_schema = Schema::new(&view);
        let references = general_schema.configs_actual_from().values()?;
        let mut res: Vec<_> = Vec::new();
        for reference in references {
            let cfg_hash = reference.cfg_hash();
            let cfg =
                general_schema
                    .configs()
                    .get(cfg_hash)?
                    .expect(&format!("Config with hash {:?} is absent in configs table", cfg_hash));
            if !PublicConfigApi::filter_cfg_predicate(&cfg,
                                                      previous_cfg_hash_filter,
                                                      actual_from_filter) {
                continue;
            }
            let elem = ApiResponseConfigHashInfo {
                hash: *cfg_hash,
                config: cfg,
            };
            res.push(elem);
        }
        Ok(res)
    }

    fn retrieve_params(map: &ParamsMap) -> Result<(Option<Hash>, Option<u64>), ApiError> {
        let actual_from: Option<u64>;
        let previous_cfg_hash: Option<Hash>;
        previous_cfg_hash = match map.find(&["previous_cfg_hash"]) {
            Some(&Value::String(ref hash_string)) => {
                Some(Hash::from_hex(hash_string).map_err(ApiError::FromHex)?)
            }
            _ => None,
        };
        actual_from = match map.find(&["actual_from"]) {
            Some(&Value::String(ref from_str)) => {
                Some(from_str
                         .parse()
                         .map_err(|e: ParseIntError| ApiError::IncorrectRequest(Box::new(e)))?)
            }
            _ => None,
        };
        Ok((previous_cfg_hash, actual_from))
    }
}

impl<T> PrivateConfigApi<T>
    where T: TransactionSend + Clone
{
    fn put_config_propose(&self,
                          cfg: StoredConfiguration)
                          -> Result<ApiResponseProposePost, ApiError> {
        let cfg_hash = cfg.hash();
        let config_propose = TxConfigPropose::new(&self.config.0,
                                                  str::from_utf8(cfg.serialize().as_slice())
                                                      .unwrap(),
                                                  &self.config.1);
        let tx_hash = config_propose.hash();
        let ch = self.channel.clone();
        ch.send(ConfigTx::ConfigPropose(config_propose))?;
        let res = ApiResponseProposePost {
            tx_hash: tx_hash,
            cfg_hash: cfg_hash,
        };
        Ok(res)
    }

    fn put_config_vote(&self, cfg_hash: &Hash) -> Result<ApiResponseVotePost, ApiError> {
        let config_vote = TxConfigVote::new(&self.config.0, cfg_hash, &self.config.1);
        let tx_hash = config_vote.hash();
        let ch = self.channel.clone();
        ch.send(ConfigTx::ConfigVote(config_vote))?;
        let res = ApiResponseVotePost { tx_hash: tx_hash };
        Ok(res)
    }
}

impl Api for PublicConfigApi {
    fn wire(&self, router: &mut Router) {

        let _self = self.clone();
        let config_actual = move |_: &mut Request| -> IronResult<Response> {
            let info = _self.get_actual_config()?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };

        let _self = self.clone();
        let config_following = move |_: &mut Request| -> IronResult<Response> {
            let info = _self.get_following_config()?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };

        let _self = self.clone();
        let config_by_hash = move |req: &mut Request| -> IronResult<Response> {
            let params = req.extensions.get::<Router>().unwrap();
            match params.find("hash") {
                Some(hash_str) => {
                    let hash = Hash::from_hex(hash_str).map_err(ApiError::from)?;
                    let info = _self.get_config_by_hash(&hash)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                None => {
                    Err(ApiError::IncorrectRequest("Required route \
                                           parameter of configuration \
                                           'hash' is missing"
                                                           .into()))?
                }
            }
        };

        let _self = self.clone();
        let get_votes_for_propose = move |req: &mut Request| -> IronResult<Response> {
            let params = req.extensions.get::<Router>().unwrap();
            match params.find("hash") {
                Some(hash_str) => {
                    let propose_cfg_hash = Hash::from_hex(hash_str).map_err(ApiError::from)?;
                    let info = _self.get_votes_for_propose(&propose_cfg_hash)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                None => {
                    Err(ApiError::IncorrectRequest("Required route \
                                           parameter of configuration \
                                           'hash' is missing"
                                                           .into()))?
                }
            }
        };

        let _self = self.clone();
        let get_all_proposes = move |req: &mut Request| -> IronResult<Response> {
            let map = req.get_ref::<Params>().unwrap();
            let (previous_cfg_hash, actual_from) = PublicConfigApi::retrieve_params(map)?;
            let info = _self.get_all_proposes(previous_cfg_hash, actual_from)?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };

        let _self = self.clone();
        let get_all_committed = move |req: &mut Request| -> IronResult<Response> {
            let map = req.get_ref::<Params>().unwrap();
            let (previous_cfg_hash, actual_from) = PublicConfigApi::retrieve_params(map)?;
            let info = _self.get_all_committed(previous_cfg_hash, actual_from)?;
            _self.ok_response(&serde_json::to_value(info).unwrap())
        };
        router.get("/v1/configs/actual", config_actual, "config_actual");
        router.get("/v1/configs/following",
                   config_following,
                   "config_following");
        router.get("/v1/configs/:hash", config_by_hash, "config_by_hash");
        router.get("/v1/configs/:hash/votes",
                   get_votes_for_propose,
                   "get_votes_for_propose");
        router.get("/v1/configs/proposed", get_all_proposes, "get_all_proposes");
        router.get("/v1/configs/committed",
                   get_all_committed,
                   "get_all_committed");

    }
}

impl<T> Api for PrivateConfigApi<T>
    where T: 'static + TransactionSend + Clone
{
    fn wire(&self, router: &mut Router) {
        let _self = self.clone();
        let put_config_propose = move |req: &mut Request| -> IronResult<Response> {
            match req.get::<bodyparser::Struct<StoredConfiguration>>() {
                Ok(Some(cfg)) => {
                    let info = _self.put_config_propose(cfg)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                Ok(None) => Err(ApiError::IncorrectRequest("Empty request body".into()))?,
                Err(e) => Err(ApiError::IncorrectRequest(Box::new(e)))?,
            }
        };

        let _self = self.clone();
        let put_config_vote = move |req: &mut Request| -> IronResult<Response> {
            let params = req.extensions.get::<Router>().unwrap();
            match params.find("hash") {
                Some(hash_str) => {
                    let propose_cfg_hash = Hash::from_hex(hash_str).map_err(ApiError::from)?;
                    let info = _self.put_config_vote(&propose_cfg_hash)?;
                    _self.ok_response(&serde_json::to_value(info).unwrap())
                }
                None => {
                    Err(ApiError::IncorrectRequest("Required route \
                                           parameter of configuration \
                                           'hash' is missing"
                                                           .into()))?
                }
            }
        };
        router.post("/v1/configs/postpropose",
                    put_config_propose,
                    "put_config_propose");
        router.post("/v1/configs/:hash/postvote",
                    put_config_vote,
                    "put_config_vote");
    }
}
<|endoftext|>


<commit_before>use ClientState;
use read::read_varint;
use write::write_varint;

use std::io::{Cursor, Read, Write};
use std::marker::PhantomData;
use std::net::Shutdown;
use std::net::TcpStream;
use std::{io, time};

use netbuf::Buf;

use flate2::Compress;
use flate2::read::ZlibDecoder;
use flate2;

use openssl::symm;

/// Trait for the two enums ClientboundPacket and ServerboundPacket
pub trait Packet: Sized {
    fn deserialize<R: Read>(r: &mut R,
                            state: &ClientState)
                            -> io::Result<Self>;
    fn get_packet_name(&self) -> &str;
    fn get_clientstate(&self) -> ClientState;
    fn get_id(&self) -> i32;
    fn to_u8(&self) -> io::Result<Vec<u8>>;
}

/// Represents a single MC connection, either as client or server
pub struct Connection<I: Packet, O: Packet> {
    stream: TcpStream,
    clientstate: ClientState,
    /* The buffer for clientbound packets */
    buf: Buf,
    /* This tracks the length the next packet in the buffer.
     * If None, then we haven't received enough bytes to completely figure out
     * the packet length (usually means we haven't received anything, but MIGHT
     * also mean we haven't received the full header yet)
     *
     * If Some(x) then we need to read x bytes from the buf to get the complete
     * packet (excluding packet length header, including encryption/compression/
     * packet id header) */
    packet_len: Option<usize>,
    compression: Option<usize>,
    /* Incoming encryption cipher */
    in_encryption: Option<symm::Crypter>,
    /* Outgoing encryption cipher */
    out_encryption: Option<symm::Crypter>,
    /* When we last read something from the server. Use this to timeout the
     * connection if the connection is lost */
    last_read: time::Instant,
    in_type: PhantomData<I>,
    out_type: PhantomData<O>,
}
impl<I: Packet, O: Packet> Connection<I, O> {
    pub fn from_tcpstream(stream: TcpStream) -> io::Result<Self> {
        let conn = Connection {
            stream: stream,
            clientstate: ClientState::Handshake,
            buf: Buf::new(),
            packet_len: None,
            compression: None,
            in_encryption: None,
            out_encryption: None,
            last_read: time::Instant::now(),
            in_type: PhantomData,
            out_type: PhantomData,
        };
        /* Set 30 second timeout */
        conn.stream.set_read_timeout(Some(time::Duration::new(30, 0)))?;
        conn.stream.set_write_timeout(Some(time::Duration::new(30, 0)))?;
        conn.stream.set_nonblocking(true)?;
        Ok(conn)
    }

    pub fn connect_tcp(host: &str, port: u16) -> io::Result<Self> {
        let stream = TcpStream::connect(&format!("{}:{}", host, port))?;
        Ok(Connection::from_tcpstream(stream)?)
    }

    /// Send the given packet
    ///
    /// Will block until the packet has been sent
    pub fn send(&mut self, packet: O) -> io::Result<()> {
        let tmp = packet.to_u8()?;
        let uncompressed_length = tmp.len();
        let mut out = Vec::with_capacity(uncompressed_length);

        match self.compression {
            /* Compression is enabled, and the packet length is over the
             * threshold, thus compress the packet */
            Some(threshold) if uncompressed_length >= threshold => {
                /* We have to copy all the data again, because we need
                 * to prefix the packet with length of the compressed data */
                let mut output = Vec::new();
                let mut compressor = Compress::new(::COMPRESSION_LEVEL, false);
                match compressor.compress(&tmp,
                                          &mut output,
                                          flate2::Flush::Sync) {
                    flate2::Status::Ok => {
                        return io_error!("Got a Status::Ok when trying to compress outgoing packet");
                    },
                    flate2::Status::BufError => {
                        return io_error!("Got a Status::BufError when trying to compress outgoing packet");
                    },
                    flate2::Status::StreamEnd => (),
                }

                write_varint(&(output.len() as i32), &mut out)?;
                write_varint(&(uncompressed_length as i32), &mut out)?;
                out.write_all(&output)?;
            },
            /* Compression is enabled, but the packet length is not over the
             * threshold, thus we don't compress the packet */
            Some(_) => {
                /* Add 1 to the uncompressed length for the 1 byte it takes
                 * to specify no compression */
                write_varint(&((uncompressed_length + 1) as i32), &mut out)?;
                write_varint(&0, &mut out)?;
                out.write_all(&tmp)?;
            },
            /* Compression is not enabled */
            None => {
                write_varint(&(uncompressed_length as i32), &mut out)?;
                out.write_all(&tmp)?;
            },
        }

        if let Some(ref mut enc) = self.out_encryption {
            let mut tmp = vec![0; out.len() + 16];
            let n = match enc.update(&out, &mut tmp) {
                Ok(x) => x,
                Err(_) => return io_error!("client::send error writing encrypted data"),
            };
            let mut i = 0;
            while i < n {
                i += self.stream.write(&tmp[i..n])?;
            }
        } else {
            let mut i = 0;
            while i < out.len() {
                i += self.stream.write(&out[i..])?;
            }
        }

        Ok(())
    }

    /// Attempt to close this connection.
    ///
    /// All future sends and reads to this connection will fail
    pub fn close(&mut self) -> io::Result<()> {
        self.stream.shutdown(Shutdown::Both)
    }

    /// Change the client state of this connection
    pub fn set_clientstate(&mut self, new_state: ClientState) {
        self.clientstate = new_state;
    }

    /// Enable encryption with the given key.
    ///
    /// It is an error to enable encryption if encryption has already been
    /// enabled.
    pub fn enable_encryption(&mut self, key: &[u8; 16]) {
        let out_cipher =
            symm::Crypter::new(symm::Cipher::aes_128_cfb8(),
                               symm::Mode::Encrypt,
                               key,
                               Some(key))
                    .expect("client::enable_encryption error creating cipher");
        let in_cipher =
            symm::Crypter::new(symm::Cipher::aes_128_cfb8(),
                               symm::Mode::Decrypt,
                               key,
                               Some(key))
                    .expect("client::enable_encryption error creating cipher");

        self.out_encryption = Some(out_cipher);
        self.in_encryption = Some(in_cipher);
    }

    /// Enable compression.
    ///
    /// It is generally an error to enable compression if compression has
    /// already been enabled.
    pub fn enable_compression(&mut self, threshold: usize) {
        self.compression = Some(threshold);
    }

    /// Read from the TcpStream and update the incoming buffer.
    ///
    /// This is the only way to actually read from the TcpStream. Unless you
    /// know for sure you need to call this, then you do not need to call this.
    /// I.e. if you're just using client.read(), then you do not need to call
    /// this function.
    pub fn update_inbuf(&mut self) -> io::Result<()> {
        if let Some(ref mut enc) = self.in_encryption {
            let mut enc_buf = Buf::new();
            let n = match enc_buf.read_from(&mut self.stream) {
                Ok(n) => n,
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => 0,
                Err(e) => return Err(e),
            };
            let mut tmp = vec![0; n + 16];
            let n = match enc.update(&enc_buf[..], &mut tmp) {
                Ok(x) => x,
                Err(_) => return io_error!("connection::update_inbuf error reading encrypted data"),
            };
            self.buf.extend(&tmp[..n]);
        } else {
            match self.buf.read_from(&mut self.stream) {
                Ok(_) => (),
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => (),
                Err(e) => return Err(e),
            };
        }
        Ok(())
    }

    /// Read a single packet from the internal buffer.
    ///
    /// This is only really useful if you want finegrained control over the
    /// processing of packets, or if you want to manually authenticate with
    /// the server. In most cases, you'll want to just call client.read().
    ///
    /// You MUST be sure that client.update_inbuf() has been called before this,
    /// this function will not attempt to read from the TcpStream, only from the
    /// internal buffer.
    pub fn read_packet(&mut self) -> io::Result<Option<I>> {
        if let None = self.packet_len {
            self.read_length()?;
        }

        let len = match self.packet_len {
            Some(x) => x,
            None => {
                if self.last_read.elapsed() > time::Duration::new(30, 0) {
                    /* If we haven't read anything for 30 seconds, timeout */
                    self.close()?;
                    return io_error!("Read timeout");
                } else {
                    return Ok(None);
                }
            },
        };

        if self.buf.len() < len {
            /* We haven't received enough yet to read the whole packet */
            if self.last_read.elapsed() > time::Duration::new(30, 0) {
                /* If we haven't read anything for 30 seconds, timeout */
                self.close()?;
                return io_error!("Read timeout");
            } else {
                return Ok(None);
            }
        } else {
            self.last_read = time::Instant::now();
        }

        let packet = {
            let data = &self.buf[..len];
            let mut r = Cursor::new(data);

            match self.compression {
                Some(_) => {
                    let compressed_length = read_varint(&mut r)?;
                    if compressed_length == 0 {
                        /* Compression is enabled, but the given packet
                         * is not compressed */
                        I::deserialize(&mut r, &self.clientstate)?
                    } else {
                        /* Compression is enabled, and the given packet
                         * is compressed */
                        let mut r = ZlibDecoder::new(r);
                        I::deserialize(&mut r, &self.clientstate)?
                    }
                },
                /* Compression is not enabled */
                None => I::deserialize(&mut r, &self.clientstate)?,
            }
        };

        self.buf.consume(len);
        self.packet_len = None;

        Ok(Some(packet))
    }

    /** Tries to read the length of the next packet in the buf, and sets
     * self.packet_len accordingly. It will return Ok(()) as long as it doesn't
     * encounter any io errors, even if it doesn't read the whole length
     * (for example if the buffer is empty.) It will only consume the length
     * header from the buf if it successfully reads the entire length header */
    fn read_length(&mut self) -> io::Result<()> {
        let msb: u8 = 128; /* Only the MSB set */
        let mut i: usize = 0;

        /* The result */
        let res: usize = {
            let mut tmp = match self.buf.get(i) {
                Some(x) => x,
                None => return Ok(()),
            };

            let mut res = (tmp & (!msb)) as usize;

            /* While the previous byte had the MSB set */
            while (tmp & msb) != 0 {
                i += 1;

                /* A varint can be at most 5 bytes, remember it's nullindexed */
                if i >= 5 {
                    return io_error!("Received varint that was too long");
                }

                tmp = match self.buf.get(i) {
                    Some(x) => x,
                    None => return Ok(()),
                };

                res += ((tmp & (!msb)) as usize) << (7 * i);
            }

            res
        };

        self.packet_len = Some(res);
        /* Consume the length header as we no longer need it */
        self.buf.consume(i + 1);
        Ok(())
    }
}
<commit_msg>Disable Nagle's algorithm<commit_after>use ClientState;
use read::read_varint;
use write::write_varint;

use std::io::{Cursor, Read, Write};
use std::marker::PhantomData;
use std::net::Shutdown;
use std::net::TcpStream;
use std::{io, time};

use netbuf::Buf;

use flate2::Compress;
use flate2::read::ZlibDecoder;
use flate2;

use openssl::symm;

/// Trait for the two enums ClientboundPacket and ServerboundPacket
pub trait Packet: Sized {
    fn deserialize<R: Read>(r: &mut R,
                            state: &ClientState)
                            -> io::Result<Self>;
    fn get_packet_name(&self) -> &str;
    fn get_clientstate(&self) -> ClientState;
    fn get_id(&self) -> i32;
    fn to_u8(&self) -> io::Result<Vec<u8>>;
}

/// Represents a single MC connection, either as client or server
pub struct Connection<I: Packet, O: Packet> {
    stream: TcpStream,
    clientstate: ClientState,
    /* The buffer for clientbound packets */
    buf: Buf,
    /* This tracks the length the next packet in the buffer.
     * If None, then we haven't received enough bytes to completely figure out
     * the packet length (usually means we haven't received anything, but MIGHT
     * also mean we haven't received the full header yet)
     *
     * If Some(x) then we need to read x bytes from the buf to get the complete
     * packet (excluding packet length header, including encryption/compression/
     * packet id header) */
    packet_len: Option<usize>,
    compression: Option<usize>,
    /* Incoming encryption cipher */
    in_encryption: Option<symm::Crypter>,
    /* Outgoing encryption cipher */
    out_encryption: Option<symm::Crypter>,
    /* When we last read something from the server. Use this to timeout the
     * connection if the connection is lost */
    last_read: time::Instant,
    in_type: PhantomData<I>,
    out_type: PhantomData<O>,
}
impl<I: Packet, O: Packet> Connection<I, O> {
    pub fn from_tcpstream(stream: TcpStream) -> io::Result<Self> {
        let conn = Connection {
            stream: stream,
            clientstate: ClientState::Handshake,
            buf: Buf::new(),
            packet_len: None,
            compression: None,
            in_encryption: None,
            out_encryption: None,
            last_read: time::Instant::now(),
            in_type: PhantomData,
            out_type: PhantomData,
        };
        /* Set 30 second timeout */
        conn.stream.set_read_timeout(Some(time::Duration::new(30, 0)))?;
        conn.stream.set_write_timeout(Some(time::Duration::new(30, 0)))?;
        conn.stream.set_nonblocking(true)?;
        conn.stream.set_nodelay(true)?;
        Ok(conn)
    }

    pub fn connect_tcp(host: &str, port: u16) -> io::Result<Self> {
        let stream = TcpStream::connect(&format!("{}:{}", host, port))?;
        Ok(Connection::from_tcpstream(stream)?)
    }

    /// Send the given packet
    ///
    /// Will block until the packet has been sent
    pub fn send(&mut self, packet: O) -> io::Result<()> {
        let tmp = packet.to_u8()?;
        let uncompressed_length = tmp.len();
        let mut out = Vec::with_capacity(uncompressed_length);

        match self.compression {
            /* Compression is enabled, and the packet length is over the
             * threshold, thus compress the packet */
            Some(threshold) if uncompressed_length >= threshold => {
                /* We have to copy all the data again, because we need
                 * to prefix the packet with length of the compressed data */
                let mut output = Vec::new();
                let mut compressor = Compress::new(::COMPRESSION_LEVEL, false);
                match compressor.compress(&tmp,
                                          &mut output,
                                          flate2::Flush::Sync) {
                    flate2::Status::Ok => {
                        return io_error!("Got a Status::Ok when trying to compress outgoing packet");
                    },
                    flate2::Status::BufError => {
                        return io_error!("Got a Status::BufError when trying to compress outgoing packet");
                    },
                    flate2::Status::StreamEnd => (),
                }

                write_varint(&(output.len() as i32), &mut out)?;
                write_varint(&(uncompressed_length as i32), &mut out)?;
                out.write_all(&output)?;
            },
            /* Compression is enabled, but the packet length is not over the
             * threshold, thus we don't compress the packet */
            Some(_) => {
                /* Add 1 to the uncompressed length for the 1 byte it takes
                 * to specify no compression */
                write_varint(&((uncompressed_length + 1) as i32), &mut out)?;
                write_varint(&0, &mut out)?;
                out.write_all(&tmp)?;
            },
            /* Compression is not enabled */
            None => {
                write_varint(&(uncompressed_length as i32), &mut out)?;
                out.write_all(&tmp)?;
            },
        }

        if let Some(ref mut enc) = self.out_encryption {
            let mut tmp = vec![0; out.len() + 16];
            let n = match enc.update(&out, &mut tmp) {
                Ok(x) => x,
                Err(_) => return io_error!("client::send error writing encrypted data"),
            };
            let mut i = 0;
            while i < n {
                i += self.stream.write(&tmp[i..n])?;
            }
        } else {
            let mut i = 0;
            while i < out.len() {
                i += self.stream.write(&out[i..])?;
            }
        }

        Ok(())
    }

    /// Attempt to close this connection.
    ///
    /// All future sends and reads to this connection will fail
    pub fn close(&mut self) -> io::Result<()> {
        self.stream.shutdown(Shutdown::Both)
    }

    /// Change the client state of this connection
    pub fn set_clientstate(&mut self, new_state: ClientState) {
        self.clientstate = new_state;
    }

    /// Enable encryption with the given key.
    ///
    /// It is an error to enable encryption if encryption has already been
    /// enabled.
    pub fn enable_encryption(&mut self, key: &[u8; 16]) {
        let out_cipher =
            symm::Crypter::new(symm::Cipher::aes_128_cfb8(),
                               symm::Mode::Encrypt,
                               key,
                               Some(key))
                    .expect("client::enable_encryption error creating cipher");
        let in_cipher =
            symm::Crypter::new(symm::Cipher::aes_128_cfb8(),
                               symm::Mode::Decrypt,
                               key,
                               Some(key))
                    .expect("client::enable_encryption error creating cipher");

        self.out_encryption = Some(out_cipher);
        self.in_encryption = Some(in_cipher);
    }

    /// Enable compression.
    ///
    /// It is generally an error to enable compression if compression has
    /// already been enabled.
    pub fn enable_compression(&mut self, threshold: usize) {
        self.compression = Some(threshold);
    }

    /// Read from the TcpStream and update the incoming buffer.
    ///
    /// This is the only way to actually read from the TcpStream. Unless you
    /// know for sure you need to call this, then you do not need to call this.
    /// I.e. if you're just using client.read(), then you do not need to call
    /// this function.
    pub fn update_inbuf(&mut self) -> io::Result<()> {
        if let Some(ref mut enc) = self.in_encryption {
            let mut enc_buf = Buf::new();
            let n = match enc_buf.read_from(&mut self.stream) {
                Ok(n) => n,
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => 0,
                Err(e) => return Err(e),
            };
            let mut tmp = vec![0; n + 16];
            let n = match enc.update(&enc_buf[..], &mut tmp) {
                Ok(x) => x,
                Err(_) => return io_error!("connection::update_inbuf error reading encrypted data"),
            };
            self.buf.extend(&tmp[..n]);
        } else {
            match self.buf.read_from(&mut self.stream) {
                Ok(_) => (),
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => (),
                Err(e) => return Err(e),
            };
        }
        Ok(())
    }

    /// Read a single packet from the internal buffer.
    ///
    /// This is only really useful if you want finegrained control over the
    /// processing of packets, or if you want to manually authenticate with
    /// the server. In most cases, you'll want to just call client.read().
    ///
    /// You MUST be sure that client.update_inbuf() has been called before this,
    /// this function will not attempt to read from the TcpStream, only from the
    /// internal buffer.
    pub fn read_packet(&mut self) -> io::Result<Option<I>> {
        if let None = self.packet_len {
            self.read_length()?;
        }

        let len = match self.packet_len {
            Some(x) => x,
            None => {
                if self.last_read.elapsed() > time::Duration::new(30, 0) {
                    /* If we haven't read anything for 30 seconds, timeout */
                    self.close()?;
                    return io_error!("Read timeout");
                } else {
                    return Ok(None);
                }
            },
        };

        if self.buf.len() < len {
            /* We haven't received enough yet to read the whole packet */
            if self.last_read.elapsed() > time::Duration::new(30, 0) {
                /* If we haven't read anything for 30 seconds, timeout */
                self.close()?;
                return io_error!("Read timeout");
            } else {
                return Ok(None);
            }
        } else {
            self.last_read = time::Instant::now();
        }

        let packet = {
            let data = &self.buf[..len];
            let mut r = Cursor::new(data);

            match self.compression {
                Some(_) => {
                    let compressed_length = read_varint(&mut r)?;
                    if compressed_length == 0 {
                        /* Compression is enabled, but the given packet
                         * is not compressed */
                        I::deserialize(&mut r, &self.clientstate)?
                    } else {
                        /* Compression is enabled, and the given packet
                         * is compressed */
                        let mut r = ZlibDecoder::new(r);
                        I::deserialize(&mut r, &self.clientstate)?
                    }
                },
                /* Compression is not enabled */
                None => I::deserialize(&mut r, &self.clientstate)?,
            }
        };

        self.buf.consume(len);
        self.packet_len = None;

        Ok(Some(packet))
    }

    /** Tries to read the length of the next packet in the buf, and sets
     * self.packet_len accordingly. It will return Ok(()) as long as it doesn't
     * encounter any io errors, even if it doesn't read the whole length
     * (for example if the buffer is empty.) It will only consume the length
     * header from the buf if it successfully reads the entire length header */
    fn read_length(&mut self) -> io::Result<()> {
        let msb: u8 = 128; /* Only the MSB set */
        let mut i: usize = 0;

        /* The result */
        let res: usize = {
            let mut tmp = match self.buf.get(i) {
                Some(x) => x,
                None => return Ok(()),
            };

            let mut res = (tmp & (!msb)) as usize;

            /* While the previous byte had the MSB set */
            while (tmp & msb) != 0 {
                i += 1;

                /* A varint can be at most 5 bytes, remember it's nullindexed */
                if i >= 5 {
                    return io_error!("Received varint that was too long");
                }

                tmp = match self.buf.get(i) {
                    Some(x) => x,
                    None => return Ok(()),
                };

                res += ((tmp & (!msb)) as usize) << (7 * i);
            }

            res
        };

        self.packet_len = Some(res);
        /* Consume the length header as we no longer need it */
        self.buf.consume(i + 1);
        Ok(())
    }
}
<|endoftext|>


<commit_before>#![allow(dead_code, unused_must_use, unused_imports, unstable)]

extern crate "rustc-serialize" as rustc_serialize;
extern crate log;

use utils;
use std::fmt;
use std::str;
use std::string;
use std::old_io::{File, Open, Append, Read, Write, ReadWrite};
use std::old_io::TempDir;
use std::old_io::fs;
use std::old_io::fs::PathExtensions;
use std::old_io::{BufferedReader, BufferedWriter};
use std::old_path::BytesContainer;
use self::rustc_serialize::base64::{STANDARD, FromBase64, ToBase64};

/// A result type that's specfici to the Reader module.
/// TODO Decide if this is necessary
pub type ReaderResult<T, E> = Result<T, E>;

/// Reader struct of its basic properties.
pub struct Reader {
    /// Path to file where the Reader is created.
    path: Path,
    /// BufferedReader for reading the file. Initialized with the Path.
    read_buffer: BufferedReader<File>,
    /// BufferedWriter for writing to the file. Initialized with the Path.
    write_buffer: BufferedWriter<File>,
    /// Index counter to know how many records exist.
    id_count: u64,
}

/// ReaderFile traits
pub trait ReaderFile {
    // Opens a new File to the Path provided.
    // Returns a boxed File.
    fn open(&self) -> File;
    // Inserts a string to the database.
    fn insert_string(&mut self, String);
}

impl fmt::Debug for Reader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Reader: ( path: {} )", self.path.display())
    }
}

impl ToString for Reader {
    fn to_string(&self) -> String {
        format!("{:?}", self)
    }
}

impl Reader {
    /// Creates a new Reader from the Path provided.
    /// Opens a new BufferedReader and BufferedWriter (with Append mode) to the file.
    /// If the file doesn't exist, it is created.
    // TODO, create a .lock file to let other readers know the database is in use (see: #2).
    pub fn new(apath: &Path) -> Reader {
        Reader::file_lock_create(apath);
        // if file_lock exists, panic and crash with appropriate error.
        // Error: A .lock file already exists, if this is from a previous session, consider
        // deleting the .lock file.

        if !apath.exists() {
            let mut file = File::create(&apath.clone());
            file.write_line("0");
            file.flush();
        }

        let mut buffer_writer = match File::open_mode(&apath.clone(), Append, ReadWrite) {
                                    Ok(file)    => { BufferedWriter::new(file) },
                                    Err(..)     => {
                                        panic!("Failed to create a write buffer to file path: {}",
                                                apath.display())
                                    },
                                };

        let mut buffer_reader = match File::open_mode(&apath.clone(), Open, Read) {
                                    Ok(file)    => BufferedReader::new(file),
                                    Err(..)     => panic!("Failed to create a read buffer to file path: {}",
                                                            apath.display()),
                                };

        let current_record_count = match buffer_reader.read_line() {
                                        Ok(line)    => {
                                            let first_line = utils::string_slice(line);
                                            println!("This is our line read in: {:?}", first_line);
                                            let input = first_line[0].trim().parse::<u64>();
                                            match input {
                                                Ok(num) => num,
                                                Err(..) => {
                                                    println!("We're getting nothing sarge!");
                                                    0},
                                            }
                                        },
                                        Err(..) => {
                                            println!("Can't read a line!");
                                            0
                                        },
                                   };

        Reader {
            path: apath.clone(),
            read_buffer: buffer_reader,
            write_buffer: buffer_writer,
            id_count: current_record_count,
        }
    }

    /// This is a helder function that realistically shouldn't exist in production.
    /// Used primarily for "spilling" the entire database file into a Vec<String>
    fn spill(&mut self) -> Vec<String> {
        let mut result: Vec<String> = Vec::new();
        let mut buffer_reader = BufferedReader::new(File::open(&self.path.clone()));
        for line_iter in buffer_reader.lines() {
            result.push(line_iter.unwrap().trim().to_string());
        }
        return result;
    }

    /// Inserts a &str into the database.
    fn insert_str(&mut self, item: &str) {
        self.write_buffer.write_line(item);
        self.write_buffer.flush();
        self.id_count = self.id_count + 1;
        self.update_counter(self.id_count);
    }

    /// Inserts a byte array into the database.
    fn insert(&mut self, item: &[u8]) {
        self.write_buffer.write(item);
        self.write_buffer.flush();
        self.id_count = self.id_count + 1;
        self.update_counter(self.id_count);
    }

    /// Read a &str from the database
    fn read_line(&mut self) -> String {
        match self.read_buffer.read_line() {
            Ok(string)  => { string.to_string() },
            Err(..)     => {
                error!("Unable to read next line. BufferedReader error.");
                "".to_string()
            },
        }
    }

    fn file_lock_create(lockpath: &Path) -> (bool, Path) {
        if lockpath.exists() {
            return (true, lockpath.clone())
        }

        let mut filelock_path = lockpath.clone();

        // Remove the old name
        filelock_path.pop();
        // Surely, there's a less ugly way to take the filename of a Path and convert it to a string?!
        let mut filename_lock: String = str::from_utf8(lockpath.filename().unwrap()).unwrap().to_string();
        filename_lock.push_str(".lock");
        // Join the new filename with the path
        filelock_path = filelock_path.join(filename_lock);
        println!("{}", filelock_path.display());
        match File::create(&filelock_path) {
            Ok(..)  => (true, filelock_path),
            Err(..) => (false, filelock_path),
        }
    }

    fn file_lock_remove(&self, filelock: &Path) -> bool {
        fs::unlink(&filelock.clone());
        filelock.exists()
    }

    fn update_counter(&self, value: u64) {
        let file = File::open_mode(&self.path.clone(), Open, Write);
        let mut buffer_writer = BufferedWriter::new(file);
        buffer_writer.write_line(value.to_string().as_slice());
        buffer_writer.flush();
    }

}

impl ReaderFile for Reader {

    fn open(&self) -> File {
        match File::open_mode(&self.path, Open, ReadWrite) {
            Ok(file)    => file,
            Err(..)     => { panic!("File {} couldn't be opened!", &self.path.display()); },
        }
    }

    fn insert_string(&mut self, item: String) {
        self.insert_str(&*item);
    }

}

#[test]
fn test_open_file() {
    let reader = Reader::new(&Path::new("tests/base-test.txt"));
}

#[test]
fn test_create_file() {
    use std::rand;
    let mut path_str = String::from_str("tests/");
    path_str.push_str(&*rand::random::<usize>().to_string());
    path_str.push_str(".txt");

    let (tempdir, apath) = setup();
    let path = tempdir.path().join(rand::random::<usize>().to_string());

    //let path = Path::new(path_str);
    assert!(!path.exists());
    let reader = Reader::new(&path.clone());
    assert!(path.exists());
    fs::unlink(&path);
}

#[test]
fn test_read_file() {
    // We should output the entire contents of the database file we open
    // into standard output.
    let (tempdir, path) = setup();
    let mut reader = Reader::new(&path);
    let expected = vec!["10 11".to_string(), "20 21".to_string()];
    assert_eq!(expected, reader.spill());
}

#[test]
fn test_write_string_to_file() {
    let (tempdir, path) = setup();
    let mut reader = Reader::new(&path);
    let expected = vec!["10 11".to_string(), "20 21".to_string(), "30 31".to_string()];
    reader.insert_string("30 31".to_string());
    assert_eq![expected, reader.spill()];
}

#[test]
fn test_write_str_to_file() {
    let (tempdir, path) = setup();
    let mut reader = Reader::new(&path);
    let expected = vec!["10 11".to_string(), "20 21".to_string(), "30 31".to_string()];
    reader.insert_str("30 31");
    assert_eq![expected, reader.spill()];
}

#[test]
fn test_file_path_lock() {

    let (tempdir, path) = setup();

    let mut expected = path.clone();
    expected.pop();

    // Surely, there's a less ugly way to take the filename of a Path and convert it to a string?!
    let mut filename_lock: String = str::from_utf8(path.filename().unwrap()).unwrap().to_string();
    filename_lock.push_str(".lock");
    expected = expected.join(filename_lock);

    let reader = Reader::new(&expected.clone());
    assert!(expected.exists() && expected.is_file());
}

#[test]
fn test_reader_show() {
    let reader: Reader = Reader::new(&Path::new("./tests/file.txt"));
    assert_eq!("Reader: ( path: tests/file.txt )", reader.to_string());
}

/// Test setup code. Current functions:
/// - Create a new file with `TempDir` and a random name.
/// - Write a 2x2 matrix of records into the base-test.txt file
/// - Returns a tuple of `TempDir` and `Path` to the file.
/// - The path is for r/w access and `TempDir` is so that the directory
///   isn't deleted before the test is completed.
#[allow(dead_code, unused_must_use)]
fn setup() -> (TempDir, Path) {
    use std::rand;

    let tmpdir = match TempDir::new("txtdb-tests") {
        Ok(dir) => dir,
        Err(..) => panic!("Cannot create test directory. Tests will fail."),
    };

    let final_dir = tmpdir.path().join(rand::random::<usize>().to_string());

    let mut file = File::create(&final_dir.clone());
    file.write_str("10 11\n20 21\n");

    (tmpdir, final_dir)
}
<commit_msg>Better logging and panics when unable to read db<commit_after>#![allow(dead_code, unused_must_use, unused_imports, unstable)]

extern crate "rustc-serialize" as rustc_serialize;
extern crate log;

use utils;
use std::fmt;
use std::str;
use std::string;
use std::old_io::{File, Open, Append, Read, Write, ReadWrite};
use std::old_io::TempDir;
use std::old_io::fs;
use std::old_io::fs::PathExtensions;
use std::old_io::{BufferedReader, BufferedWriter};
use std::old_path::BytesContainer;
use self::rustc_serialize::base64::{STANDARD, FromBase64, ToBase64};

/// A result type that's specfici to the Reader module.
/// TODO Decide if this is necessary
pub type ReaderResult<T, E> = Result<T, E>;

/// Reader struct of its basic properties.
pub struct Reader {
    /// Path to file where the Reader is created.
    path: Path,
    /// BufferedReader for reading the file. Initialized with the Path.
    read_buffer: BufferedReader<File>,
    /// BufferedWriter for writing to the file. Initialized with the Path.
    write_buffer: BufferedWriter<File>,
    /// Index counter to know how many records exist.
    id_count: u64,
}

/// ReaderFile traits
pub trait ReaderFile {
    // Opens a new File to the Path provided.
    // Returns a boxed File.
    fn open(&self) -> File;
    // Inserts a string to the database.
    fn insert_string(&mut self, String);
}

impl fmt::Debug for Reader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Reader: ( path: {} )", self.path.display())
    }
}

impl ToString for Reader {
    fn to_string(&self) -> String {
        format!("{:?}", self)
    }
}

impl Reader {
    /// Creates a new Reader from the Path provided.
    /// Opens a new BufferedReader and BufferedWriter (with Append mode) to the file.
    /// If the file doesn't exist, it is created.
    // TODO, create a .lock file to let other readers know the database is in use (see: #2).
    pub fn new(apath: &Path) -> Reader {
        Reader::file_lock_create(apath);
        // if file_lock exists, panic and crash with appropriate error.
        // Error: A .lock file already exists, if this is from a previous session, consider
        // deleting the .lock file.

        if !apath.exists() {
            let mut file = File::create(&apath.clone());
            file.write_line("0");
            file.flush();
        }

        let mut buffer_writer = match File::open_mode(&apath.clone(), Append, ReadWrite) {
                                    Ok(file)    => { BufferedWriter::new(file) },
                                    Err(..)     => {
                                        panic!("Failed to create a write buffer to file path: {}",
                                                apath.display())
                                    },
                                };

        let mut buffer_reader = match File::open_mode(&apath.clone(), Open, Read) {
                                    Ok(file)    => BufferedReader::new(file),
                                    Err(..)     => panic!("Failed to create a read buffer to file path: {}",
                                                            apath.display()),
                                };

        let current_record_count = match buffer_reader.read_line() {
                                        Ok(line)    => {
                                            let first_line = utils::string_slice(line);
                                            let input = first_line[0].trim().parse::<u64>();
                                            match input {
                                                Ok(num) => num,
                                                Err(..) => {
                                                    warn!("We can't parse the record count from metadata");
                                                    warn!("Our parsed db metadata: {:?}", first_line);
                                                    // Setting counter to zero because we can't do
                                                    // anything else.
                                                    0
                                                },
                                            }
                                        },
                                        Err(..) => {
                                            panic!("Can't read database metadata! Everybody do the flop!");
                                        },
                                   };

        Reader {
            path: apath.clone(),
            read_buffer: buffer_reader,
            write_buffer: buffer_writer,
            id_count: current_record_count,
        }
    }

    /// This is a helder function that realistically shouldn't exist in production.
    /// Used primarily for "spilling" the entire database file into a Vec<String>
    fn spill(&mut self) -> Vec<String> {
        let mut result: Vec<String> = Vec::new();
        let mut buffer_reader = BufferedReader::new(File::open(&self.path.clone()));
        for line_iter in buffer_reader.lines() {
            result.push(line_iter.unwrap().trim().to_string());
        }
        return result;
    }

    /// Inserts a &str into the database.
    fn insert_str(&mut self, item: &str) {
        self.write_buffer.write_line(item);
        self.write_buffer.flush();
        self.id_count = self.id_count + 1;
        self.update_counter(self.id_count);
    }

    /// Inserts a byte array into the database.
    fn insert(&mut self, item: &[u8]) {
        self.write_buffer.write(item);
        self.write_buffer.flush();
        self.id_count = self.id_count + 1;
        self.update_counter(self.id_count);
    }

    /// Read a &str from the database
    fn read_line(&mut self) -> String {
        match self.read_buffer.read_line() {
            Ok(string)  => { string.to_string() },
            Err(..)     => {
                error!("Unable to read next line. BufferedReader error.");
                "".to_string()
            },
        }
    }

    /// Creates a .lock file to let other processes know that the database is in use.
    /// This is still unfinished and should be considered broken.
    fn file_lock_create(lockpath: &Path) -> (bool, Path) {
        if lockpath.exists() {
            return (true, lockpath.clone())
        }

        let mut filelock_path = lockpath.clone();

        // Remove the old name
        filelock_path.pop();
        // Surely, there's a less ugly way to take the filename of a Path and convert it to a string?!
        let mut filename_lock: String = str::from_utf8(lockpath.filename().unwrap()).unwrap().to_string();
        filename_lock.push_str(".lock");
        // Join the new filename with the path
        filelock_path = filelock_path.join(filename_lock);
        println!("{}", filelock_path.display());
        match File::create(&filelock_path) {
            Ok(..)  => (true, filelock_path),
            Err(..) => (false, filelock_path),
        }
    }

    /// Removes .lock file when the reader process is completed.
    fn file_lock_remove(&self, filelock: &Path) -> bool {
        fs::unlink(&filelock.clone());
        filelock.exists()
    }

    /// Updates database counter on disk.
    fn update_counter(&self, value: u64) {
        let file = File::open_mode(&self.path.clone(), Open, Write);
        let mut buffer_writer = BufferedWriter::new(file);
        buffer_writer.write_line(value.to_string().as_slice());
        buffer_writer.flush();
    }

}

impl ReaderFile for Reader {

    fn open(&self) -> File {
        match File::open_mode(&self.path, Open, ReadWrite) {
            Ok(file)    => file,
            Err(..)     => { panic!("File {} couldn't be opened!", &self.path.display()); },
        }
    }

    fn insert_string(&mut self, item: String) {
        self.insert_str(&*item);
    }

}

#[test]
fn test_open_file() {
    let reader = Reader::new(&Path::new("tests/base-test.txt"));
}

#[test]
fn test_create_file() {
    use std::rand;
    let mut path_str = String::from_str("tests/");
    path_str.push_str(&*rand::random::<usize>().to_string());
    path_str.push_str(".txt");

    let (tempdir, apath) = setup();
    let path = tempdir.path().join(rand::random::<usize>().to_string());

    //let path = Path::new(path_str);
    assert!(!path.exists());
    let reader = Reader::new(&path.clone());
    assert!(path.exists());
    fs::unlink(&path);
}

#[test]
fn test_read_file() {
    // We should output the entire contents of the database file we open
    // into standard output.
    let (tempdir, path) = setup();
    let mut reader = Reader::new(&path);
    let expected = vec!["10 11".to_string(), "20 21".to_string()];
    assert_eq!(expected, reader.spill());
}

#[test]
fn test_write_string_to_file() {
    let (tempdir, path) = setup();
    let mut reader = Reader::new(&path);
    let expected = vec!["10 11".to_string(), "20 21".to_string(), "30 31".to_string()];
    reader.insert_string("30 31".to_string());
    assert_eq![expected, reader.spill()];
}

#[test]
fn test_write_str_to_file() {
    let (tempdir, path) = setup();
    let mut reader = Reader::new(&path);
    let expected = vec!["10 11".to_string(), "20 21".to_string(), "30 31".to_string()];
    reader.insert_str("30 31");
    assert_eq![expected, reader.spill()];
}

#[test]
fn test_file_path_lock() {

    let (tempdir, path) = setup();

    let mut expected = path.clone();
    expected.pop();

    // Surely, there's a less ugly way to take the filename of a Path and convert it to a string?!
    let mut filename_lock: String = str::from_utf8(path.filename().unwrap()).unwrap().to_string();
    filename_lock.push_str(".lock");
    expected = expected.join(filename_lock);

    let reader = Reader::new(&expected.clone());
    assert!(expected.exists() && expected.is_file());
}

#[test]
fn test_reader_show() {
    let reader: Reader = Reader::new(&Path::new("./tests/file.txt"));
    assert_eq!("Reader: ( path: tests/file.txt )", reader.to_string());
}

/// Test setup code. Current functions:
/// - Create a new file with `TempDir` and a random name.
/// - Write a 2x2 matrix of records into the base-test.txt file
/// - Returns a tuple of `TempDir` and `Path` to the file.
/// - The path is for r/w access and `TempDir` is so that the directory
///   isn't deleted before the test is completed.
#[allow(dead_code, unused_must_use)]
fn setup() -> (TempDir, Path) {
    use std::rand;

    let tmpdir = match TempDir::new("txtdb-tests") {
        Ok(dir) => dir,
        Err(..) => panic!("Cannot create test directory. Tests will fail."),
    };

    let final_dir = tmpdir.path().join(rand::random::<usize>().to_string());

    let mut file = File::create(&final_dir.clone());
    file.write_str("10 11\n20 21\n");

    (tmpdir, final_dir)
}
<|endoftext|>


<commit_before>use core::SegmentId;
use error::{ErrorKind, ResultExt};
use schema::Schema;
use serde_json;
use std::borrow::BorrowMut;
use std::fmt;
use std::sync::Arc;
use Result;

use super::pool::LeasedItem;
use super::pool::Pool;
use super::segment::create_segment;
use super::segment::Segment;
use core::searcher::Searcher;
use core::IndexMeta;
use core::SegmentMeta;
use core::SegmentReader;
use core::META_FILEPATH;
use directory::ManagedDirectory;
#[cfg(feature = "mmap")]
use directory::MmapDirectory;
use directory::{Directory, RAMDirectory};
use indexer::index_writer::open_index_writer;
use indexer::segment_updater::save_new_metas;
use indexer::DirectoryLock;
use num_cpus;
use std::path::Path;
use tokenizer::TokenizerManager;
use IndexWriter;

const NUM_SEARCHERS: usize = 12;

fn load_metas(directory: &Directory) -> Result<IndexMeta> {
    let meta_data = directory.atomic_read(&META_FILEPATH)?;
    let meta_string = String::from_utf8_lossy(&meta_data);
    serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone()))
}

/// Search Index
pub struct Index {
    directory: ManagedDirectory,
    schema: Schema,
    searcher_pool: Arc<Pool<Searcher>>,
    tokenizers: TokenizerManager,
}

impl Index {
    /// Creates a new index using the `RAMDirectory`.
    ///
    /// The index will be allocated in anonymous memory.
    /// This should only be used for unit tests.
    pub fn create_in_ram(schema: Schema) -> Index {
        let ram_directory = RAMDirectory::create();
        // unwrap is ok here
        let directory = ManagedDirectory::new(ram_directory).expect(
            "Creating a managed directory from a brand new RAM directory \
             should never fail.",
        );
        Index::from_directory(directory, schema).expect("Creating a RAMDirectory should never fail")
    }

    /// Creates a new index in a given filepath.
    /// The index will use the `MMapDirectory`.
    ///
    /// If a previous index was in this directory, then its meta file will be destroyed.
    #[cfg(feature = "mmap")]
    pub fn create<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        let directory = ManagedDirectory::new(mmap_directory)?;
        Index::from_directory(directory, schema)
    }

    /// Accessor for the tokenizer manager.
    pub fn tokenizers(&self) -> &TokenizerManager {
        &self.tokenizers
    }

    /// Creates a new index in a temp directory.
    ///
    /// The index will use the `MMapDirectory` in a newly created directory.
    /// The temp directory will be destroyed automatically when the `Index` object
    /// is destroyed.
    ///
    /// The temp directory is only used for testing the `MmapDirectory`.
    /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
    #[cfg(feature = "mmap")]
    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::create_from_tempdir()?;
        let directory = ManagedDirectory::new(mmap_directory)?;
        Index::from_directory(directory, schema)
    }

    /// Creates a new index given a directory and an `IndexMeta`.
    fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
        let schema = metas.schema.clone();
        let index = Index {
            directory,
            schema,
            searcher_pool: Arc::new(Pool::new()),
            tokenizers: TokenizerManager::default(),
        };
        index.load_searchers()?;
        Ok(index)
    }

    /// Create a new index from a directory.
    pub fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
        let metas = IndexMeta::with_schema(schema);
        Index::create_from_metas(directory, &metas)
    }

    /// Opens a new directory from an index path.
    #[cfg(feature = "mmap")]
    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        let directory = ManagedDirectory::new(mmap_directory)?;
        let metas = load_metas(&directory)?;
        Index::create_from_metas(directory, &metas)
    }

    /// Reads the index meta file from the directory.
    pub fn load_metas(&self) -> Result<IndexMeta> {
        load_metas(self.directory())
    }

    /// Open a new index writer. Attempts to acquire a lockfile.
    ///
    /// The lockfile should be deleted on drop, but it is possible
    /// that due to a panic or other error, a stale lockfile will be
    /// left in the index directory. If you are sure that no other
    /// `IndexWriter` on the system is accessing the index directory,
    /// it is safe to manually delete the lockfile.
    ///
    /// num_threads specifies the number of indexing workers that
    /// should work at the same time.
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer_with_num_threads(
        &self,
        num_threads: usize,
        heap_size_in_bytes: usize,
    ) -> Result<IndexWriter> {
        let directory_lock = DirectoryLock::lock(self.directory().box_clone())?;
        open_index_writer(self, num_threads, heap_size_in_bytes, directory_lock)
    }

    /// Creates a multithreaded writer
    /// It just calls `writer_with_num_threads` with the number of cores as `num_threads`
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer(&self, heap_size_in_bytes: usize) -> Result<IndexWriter> {
        self.writer_with_num_threads(num_cpus::get(), heap_size_in_bytes)
    }

    /// Accessor to the index schema
    ///
    /// The schema is actually cloned.
    pub fn schema(&self) -> Schema {
        self.schema.clone()
    }

    /// Returns the list of segments that are searchable
    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
        Ok(self.searchable_segment_metas()?
            .into_iter()
            .map(|segment_meta| self.segment(segment_meta))
            .collect())
    }

    #[doc(hidden)]
    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
        create_segment(self.clone(), segment_meta)
    }

    /// Creates a new segment.
    pub fn new_segment(&self) -> Segment {
        let segment_meta = SegmentMeta::new(SegmentId::generate_random());
        create_segment(self.clone(), segment_meta)
    }

    /// Return a reference to the index directory.
    pub fn directory(&self) -> &ManagedDirectory {
        &self.directory
    }

    /// Return a mutable reference to the index directory.
    pub fn directory_mut(&mut self) -> &mut ManagedDirectory {
        &mut self.directory
    }

    /// Reads the meta.json and returns the list of
    /// `SegmentMeta` from the last commit.
    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
        Ok(self.load_metas()?.segments)
    }

    /// Returns the list of segment ids that are searchable.
    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
        Ok(self.searchable_segment_metas()?
            .iter()
            .map(|segment_meta| segment_meta.id())
            .collect())
    }

    /// Creates a new generation of searchers after

    /// a change of the set of searchable indexes.
    ///
    /// This needs to be called when a new segment has been
    /// published or after a merge.
    pub fn load_searchers(&self) -> Result<()> {
        let searchable_segments = self.searchable_segments()?;
        let segment_readers: Vec<SegmentReader> = searchable_segments
            .iter()
            .map(SegmentReader::open)
            .collect::<Result<_>>()?;
        let schema = self.schema();
        let searchers = (0..NUM_SEARCHERS)
            .map(|_| Searcher::new(schema.clone(), segment_readers.clone()))
            .collect();
        self.searcher_pool.publish_new_generation(searchers);
        Ok(())
    }

    /// Returns a searcher
    ///
    /// This method should be called every single time a search
    /// query is performed.
    /// The searchers are taken from a pool of `NUM_SEARCHERS` searchers.
    /// If no searcher is available
    /// this may block.
    ///
    /// The same searcher must be used for a given query, as it ensures
    /// the use of a consistent segment set.
    pub fn searcher(&self) -> LeasedItem<Searcher> {
        self.searcher_pool.acquire()
    }
}

impl fmt::Debug for Index {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Index({:?})", self.directory)
    }
}

impl Clone for Index {
    fn clone(&self) -> Index {
        Index {
            directory: self.directory.clone(),
            schema: self.schema.clone(),
            searcher_pool: Arc::clone(&self.searcher_pool),
            tokenizers: self.tokenizers.clone(),
        }
    }
}
<commit_msg>Add Index::open_directory function (#285)<commit_after>use core::SegmentId;
use error::{ErrorKind, ResultExt};
use schema::Schema;
use serde_json;
use std::borrow::BorrowMut;
use std::fmt;
use std::sync::Arc;
use Result;

use super::pool::LeasedItem;
use super::pool::Pool;
use super::segment::create_segment;
use super::segment::Segment;
use core::searcher::Searcher;
use core::IndexMeta;
use core::SegmentMeta;
use core::SegmentReader;
use core::META_FILEPATH;
use directory::ManagedDirectory;
#[cfg(feature = "mmap")]
use directory::MmapDirectory;
use directory::{Directory, RAMDirectory};
use indexer::index_writer::open_index_writer;
use indexer::segment_updater::save_new_metas;
use indexer::DirectoryLock;
use num_cpus;
use std::path::Path;
use tokenizer::TokenizerManager;
use IndexWriter;

const NUM_SEARCHERS: usize = 12;

fn load_metas(directory: &Directory) -> Result<IndexMeta> {
    let meta_data = directory.atomic_read(&META_FILEPATH)?;
    let meta_string = String::from_utf8_lossy(&meta_data);
    serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone()))
}

/// Search Index
pub struct Index {
    directory: ManagedDirectory,
    schema: Schema,
    searcher_pool: Arc<Pool<Searcher>>,
    tokenizers: TokenizerManager,
}

impl Index {
    /// Creates a new index using the `RAMDirectory`.
    ///
    /// The index will be allocated in anonymous memory.
    /// This should only be used for unit tests.
    pub fn create_in_ram(schema: Schema) -> Index {
        let ram_directory = RAMDirectory::create();
        // unwrap is ok here
        let directory = ManagedDirectory::new(ram_directory).expect(
            "Creating a managed directory from a brand new RAM directory \
             should never fail.",
        );
        Index::from_directory(directory, schema).expect("Creating a RAMDirectory should never fail")
    }

    /// Creates a new index in a given filepath.
    /// The index will use the `MMapDirectory`.
    ///
    /// If a previous index was in this directory, then its meta file will be destroyed.
    #[cfg(feature = "mmap")]
    pub fn create<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        let directory = ManagedDirectory::new(mmap_directory)?;
        Index::from_directory(directory, schema)
    }

    /// Accessor for the tokenizer manager.
    pub fn tokenizers(&self) -> &TokenizerManager {
        &self.tokenizers
    }

    /// Creates a new index in a temp directory.
    ///
    /// The index will use the `MMapDirectory` in a newly created directory.
    /// The temp directory will be destroyed automatically when the `Index` object
    /// is destroyed.
    ///
    /// The temp directory is only used for testing the `MmapDirectory`.
    /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
    #[cfg(feature = "mmap")]
    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::create_from_tempdir()?;
        let directory = ManagedDirectory::new(mmap_directory)?;
        Index::from_directory(directory, schema)
    }

    /// Creates a new index given a directory and an `IndexMeta`.
    fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
        let schema = metas.schema.clone();
        let index = Index {
            directory,
            schema,
            searcher_pool: Arc::new(Pool::new()),
            tokenizers: TokenizerManager::default(),
        };
        index.load_searchers()?;
        Ok(index)
    }

    /// Open the index using the provided directory
    pub fn open_directory<D: Directory>(directory: D) -> Result<Index> {
        let directory = ManagedDirectory::new(directory)?;
        let metas = load_metas(&directory)?;
        Index::create_from_metas(directory, &metas)
    }

    /// Opens a new directory from an index path.
    #[cfg(feature = "mmap")]
    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        Index::open_directory(mmap_directory)
    }

    /// Create a new index from a directory.
    pub fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
        let metas = IndexMeta::with_schema(schema);
        Index::create_from_metas(directory, &metas)
    }

    /// Reads the index meta file from the directory.
    pub fn load_metas(&self) -> Result<IndexMeta> {
        load_metas(self.directory())
    }

    /// Open a new index writer. Attempts to acquire a lockfile.
    ///
    /// The lockfile should be deleted on drop, but it is possible
    /// that due to a panic or other error, a stale lockfile will be
    /// left in the index directory. If you are sure that no other
    /// `IndexWriter` on the system is accessing the index directory,
    /// it is safe to manually delete the lockfile.
    ///
    /// num_threads specifies the number of indexing workers that
    /// should work at the same time.
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer_with_num_threads(
        &self,
        num_threads: usize,
        heap_size_in_bytes: usize,
    ) -> Result<IndexWriter> {
        let directory_lock = DirectoryLock::lock(self.directory().box_clone())?;
        open_index_writer(self, num_threads, heap_size_in_bytes, directory_lock)
    }

    /// Creates a multithreaded writer
    /// It just calls `writer_with_num_threads` with the number of cores as `num_threads`
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer(&self, heap_size_in_bytes: usize) -> Result<IndexWriter> {
        self.writer_with_num_threads(num_cpus::get(), heap_size_in_bytes)
    }

    /// Accessor to the index schema
    ///
    /// The schema is actually cloned.
    pub fn schema(&self) -> Schema {
        self.schema.clone()
    }

    /// Returns the list of segments that are searchable
    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
        Ok(self.searchable_segment_metas()?
            .into_iter()
            .map(|segment_meta| self.segment(segment_meta))
            .collect())
    }

    #[doc(hidden)]
    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
        create_segment(self.clone(), segment_meta)
    }

    /// Creates a new segment.
    pub fn new_segment(&self) -> Segment {
        let segment_meta = SegmentMeta::new(SegmentId::generate_random());
        create_segment(self.clone(), segment_meta)
    }

    /// Return a reference to the index directory.
    pub fn directory(&self) -> &ManagedDirectory {
        &self.directory
    }

    /// Return a mutable reference to the index directory.
    pub fn directory_mut(&mut self) -> &mut ManagedDirectory {
        &mut self.directory
    }

    /// Reads the meta.json and returns the list of
    /// `SegmentMeta` from the last commit.
    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
        Ok(self.load_metas()?.segments)
    }

    /// Returns the list of segment ids that are searchable.
    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
        Ok(self.searchable_segment_metas()?
            .iter()
            .map(|segment_meta| segment_meta.id())
            .collect())
    }

    /// Creates a new generation of searchers after

    /// a change of the set of searchable indexes.
    ///
    /// This needs to be called when a new segment has been
    /// published or after a merge.
    pub fn load_searchers(&self) -> Result<()> {
        let searchable_segments = self.searchable_segments()?;
        let segment_readers: Vec<SegmentReader> = searchable_segments
            .iter()
            .map(SegmentReader::open)
            .collect::<Result<_>>()?;
        let schema = self.schema();
        let searchers = (0..NUM_SEARCHERS)
            .map(|_| Searcher::new(schema.clone(), segment_readers.clone()))
            .collect();
        self.searcher_pool.publish_new_generation(searchers);
        Ok(())
    }

    /// Returns a searcher
    ///
    /// This method should be called every single time a search
    /// query is performed.
    /// The searchers are taken from a pool of `NUM_SEARCHERS` searchers.
    /// If no searcher is available
    /// this may block.
    ///
    /// The same searcher must be used for a given query, as it ensures
    /// the use of a consistent segment set.
    pub fn searcher(&self) -> LeasedItem<Searcher> {
        self.searcher_pool.acquire()
    }
}

impl fmt::Debug for Index {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Index({:?})", self.directory)
    }
}

impl Clone for Index {
    fn clone(&self) -> Index {
        Index {
            directory: self.directory.clone(),
            schema: self.schema.clone(),
            searcher_pool: Arc::clone(&self.searcher_pool),
            tokenizers: self.tokenizers.clone(),
        }
    }
}
<|endoftext|>