<commit_before>// CITA
// Copyright 2016-2017 Cryptape Technologies LLC.

// This program is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any
// later version.

// This program is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

#![allow(unused_variables)]

pub use byteorder::{BigEndian, ByteOrder};
use core::filters::eth_filter::EthFilter;
use core::libchain::call_request::CallRequest;
pub use core::libchain::chain::*;
use jsonrpc_types::rpctypes;
use jsonrpc_types::rpctypes::{Filter as RpcFilter, Log as RpcLog, Receipt as RpcReceipt, CountAndCode, BlockNumber, BlockParamsByNumber, BlockParamsByHash, RpcBlock};
use libproto;
pub use libproto::*;
pub use libproto::request::Request_oneof_req as Request;
use protobuf::Message;
use serde_json;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::sync::mpsc::{Sender, Receiver};
use std::vec::Vec;
use threadpool::*;
use types::filter::Filter;
use types::ids::BlockId;
use util::Address;
use util::H256;
// pub const CHAIN_PUB: u32 = 3;

pub fn chain_pool(pool: &ThreadPool, tx: &Sender<(u32, u32, u32, MsgClass)>, id: u32, msg: Vec<u8>) {
    let tx = tx.clone();
    pool.execute(move || {
                     let (cmd_id, origin, content) = parse_msg(msg.as_slice());
                     tx.send((id, cmd_id, origin, content)).unwrap();
                 });
}

// TODO: RPC Errors
pub fn chain_result(chain: Arc<Chain>, rx: &Receiver<(u32, u32, u32, MsgClass)>, ctx_pub: Sender<(String, Vec<u8>)>) {
    let (id, cmd_id, origin, content_ext) = rx.recv().unwrap();
    trace!("chain_result call {:?} {:?}", id, cmd_id);
    match content_ext {
        MsgClass::REQUEST(mut req) => {
            let mut response = request::Response::new();
            response.set_request_id(req.take_request_id());
            match req.req.clone().unwrap() {
                // TODO: should check the result, parse it first!
                Request::block_number(_) => {
                    // let sys_time = SystemTime::now();
                    let height = chain.get_current_height();
                    response.set_block_number(height);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::block_by_hash(rpc) => {
                    let rpc: BlockParamsByHash = serde_json::from_str(&rpc).expect("Invalid param");
                    let hash = rpc.hash;
                    let include_txs = rpc.include_txs;
                    match chain.block_by_hash(H256::from(hash.as_slice())) {
                        Some(block) => {
                            let rpc_block = RpcBlock::new(hash, include_txs, block.protobuf().write_to_bytes().unwrap());
                            //TODO，发生错误了，应该加错原因给rpc,通知客户
                            serde_json::to_string(&rpc_block)
                                .map(|data| response.set_block(data))
                                .map_err(|_| response.set_none(true));
                        }
                        None => {
                            response.set_none(true);
                        }
                    }
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();

                }

                Request::block_by_height(block_height) => {
                    let block_height: BlockParamsByNumber = serde_json::from_str(&block_height).expect("Invalid param");

                    let include_txs = block_height.include_txs;
                    match chain.block(block_height.block_id.into()) {
                        Some(block) => {
                            let rpc_block = RpcBlock::new(block.hash().to_vec(), include_txs, block.protobuf().write_to_bytes().unwrap());
                            //TODO，发生错误了，应该加错原因给rpc,通知客户
                            serde_json::to_string(&rpc_block)
                                .map(|data| response.set_block(data))
                                .map_err(|_| response.set_none(true));
                        }
                        None => {
                            response.set_none(true);
                        }
                    }
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }
                Request::transaction(hash) => {
                    match chain.full_transaction(H256::from_slice(&hash)) {
                        Some(ts) => {
                            response.set_ts(ts);
                        }
                        None => {
                            response.set_none(true);
                        }
                    }
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }
                Request::transaction_receipt(hash) => {
                    let tx_hash = H256::from_slice(&hash);
                    let receipt = chain.localized_receipt(tx_hash);
                    if let Some(receipt) = receipt {
                        let rpc_receipt: RpcReceipt = receipt.into();
                        let serialized = serde_json::to_string(&rpc_receipt).unwrap();
                        response.set_receipt(serialized);
                    } else {
                        response.set_none(true);
                    }

                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::call(call) => {
                    trace!("Chainvm Call {:?}", call);
                    let block_id: BlockNumber = serde_json::from_str(&(call.height)).expect("Invalid param");
                    let call_request = CallRequest::from(call);
                    let result = chain.eth_call(call_request, block_id.into());
                    response.set_call_result(result.unwrap_or_default());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::filter(encoded) => {
                    trace!("filter: {:?}", encoded);
                    let rpc_filter: RpcFilter = serde_json::from_str(&encoded).expect("Invalid filter");
                    let filter: Filter = rpc_filter.into();
                    let logs = chain.get_logs(filter);
                    let rpc_logs: Vec<RpcLog> = logs.into_iter().map(|x| x.into()).collect();
                    response.set_logs(serde_json::to_string(&rpc_logs).unwrap());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::transaction_count(tx_count) => {
                    trace!("transaction count request from jsonrpc {:?}", tx_count);
                    //TODO 或许有错误返回给用户更好
                    let tx_count: CountAndCode = serde_json::from_str(&tx_count).expect("Invalid param");
                    let address = Address::from_slice(tx_count.address.as_ref());
                    match chain.nonce(&address, tx_count.block_id.into()) {
                        Some(nonce) => {
                            response.set_transaction_count(u64::from(nonce));
                        }
                        None => {
                            response.set_none(true);
                        }
                    };
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();

                }

                Request::code(code_content) => {
                    trace!("code request from josnrpc  {:?}", code_content);
                    let code_content: CountAndCode = serde_json::from_str(&code_content).expect("Invalid param");

                    let address = Address::from_slice(code_content.address.as_ref());
                    match chain.code_at(&address, code_content.block_id.into()) {
                        Some(code) => {
                            match code {
                                Some(code) => {
                                    response.set_code(code);
                                }
                                None => {
                                    response.set_none(true);
                                }
                            }
                        }
                        None => {
                            response.set_none(true);
                        }
                    };
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();

                }

                Request::new_filter(new_filter) => {
                    trace!("new_filter {:?}", new_filter);
                    let new_filter: RpcFilter = serde_json::from_str(&new_filter).expect("Invalid param");
                    trace!("new_filter {:?}", new_filter);
                    response.set_filter_id(chain.new_filter(new_filter) as u64);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::new_block_filter(_) => {
                    let block_filter = chain.new_block_filter();
                    response.set_filter_id(block_filter as u64);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::uninstall_filter(filter_id) => {
                    trace!("uninstall_filter's id is {:?}", filter_id);
                    let index = rpctypes::Index(filter_id as usize);
                    let b = chain.uninstall_filter(index);
                    response.set_uninstall_filter(b);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::filter_changes(filter_id) => {
                    trace!("filter_changes's id is {:?}", filter_id);
                    let index = rpctypes::Index(filter_id as usize);
                    let log = chain.filter_changes(index).unwrap();
                    trace!("Log is: {:?}", log);
                    response.set_filter_changes(serde_json::to_vec(&log).unwrap());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::filter_logs(filter_id) => {
                    trace!("filter_log's id is {:?}", filter_id);
                    let index = rpctypes::Index(filter_id as usize);
                    let log = chain.filter_logs(index).unwrap_or(vec![]);
                    trace!("Log is: {:?}", log);
                    response.set_filter_logs(serde_json::to_vec(&log).unwrap());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                _ => {}
            }
        }
        MsgClass::RESPONSE(rep) => {}
        MsgClass::HEADER(header) => {}
        MsgClass::BODY(body) => {}
        MsgClass::BLOCK(block) => {
            let mut guard = chain.block_map.write();

            let current_height = chain.get_current_height();
            let max_height = chain.get_max_height();
            let blk_heght = block.get_header().get_height();

            let new_map = guard.split_off(&current_height);
            *guard = new_map;

            trace!("received block: block_number:{:?} current_height: {:?} max_height: {:?}", blk_heght, current_height, max_height);
            let source = match id {
                submodules::CONSENSUS => BlockSource::CONSENSUS,
                _ => BlockSource::NET,
            };
            if blk_heght > current_height && blk_heght < current_height + 300 && !guard.contains_key(&blk_heght) {
                trace!("block insert {:?}", blk_heght);
                guard.insert(blk_heght, (source, Block::from(block)));
                let _ = chain.sync_sender.lock().send(blk_heght);
            }

            if !chain.get_current_height() < chain.get_max_height() {
                chain.is_sync.store(false, Ordering::SeqCst);
            }
        }
        MsgClass::TX(content) => {}
        MsgClass::TXRESPONSE(content) => {}
        MsgClass::STATUS(status) => {
            let status_height = status.get_height();
            if status_height > chain.get_max_height() {
                chain.max_height.store(status_height as usize, Ordering::SeqCst);
                trace!("recieved status update max_height: {:?}", status_height);
            }
            let known_max_height = chain.get_max_height();
            let current_height = chain.get_current_height();
            let target_height = ::std::cmp::min(current_height + 100, known_max_height);
            if current_height < target_height && !chain.is_sync.load(Ordering::SeqCst) {
                let mut diff = target_height - current_height;
                let mut start_height = current_height + 1;
                while diff > 0 {
                    let mut wtr = vec![0; 8];
                    trace!("request sync {:?}", start_height);
                    BigEndian::write_u64(&mut wtr, start_height);
                    let msg = factory::create_msg_ex(submodules::CHAIN, topics::SYNC_BLK, communication::MsgType::MSG, communication::OperateType::SINGLE, origin, wtr);
                    trace!("origin {:?}, chain.sync: OperateType {:?}", origin, communication::OperateType::SINGLE);
                    ctx_pub.send(("chain.sync".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                    start_height += 1;
                    diff -= 1;
                }
                if !chain.is_sync.load(Ordering::SeqCst) {
                    chain.is_sync.store(true, Ordering::SeqCst);
                }
            }
        }
        MsgClass::MSG(content) => {
            if libproto::cmd_id(submodules::CHAIN, topics::SYNC_BLK) == cmd_id {
                trace!("Receive sync {:?} from node-{:?}", BigEndian::read_u64(&content), origin);
                if let Some(block) = chain.block(BlockId::Number(BigEndian::read_u64(&content))) {
                    let msg = factory::create_msg_ex(submodules::CHAIN, topics::NEW_BLK, communication::MsgType::BLOCK, communication::OperateType::SINGLE, origin, block.protobuf().write_to_bytes().unwrap());
                    trace!("origin {:?}, chain.blk: OperateType {:?}", origin, communication::OperateType::SINGLE);
                    ctx_pub.send(("chain.blk".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }
            } else {
                warn!("other content.");
            }
        }
        MsgClass::VERIFYREQ(req) => {}
        MsgClass::VERIFYRESP(resp) => {}
    }
}
<commit_msg>proriotize block insertion (#39)<commit_after>// CITA
// Copyright 2016-2017 Cryptape Technologies LLC.

// This program is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any
// later version.

// This program is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

#![allow(unused_variables)]

pub use byteorder::{BigEndian, ByteOrder};
use core::filters::eth_filter::EthFilter;
use core::libchain::call_request::CallRequest;
pub use core::libchain::chain::*;
use jsonrpc_types::rpctypes;
use jsonrpc_types::rpctypes::{Filter as RpcFilter, Log as RpcLog, Receipt as RpcReceipt, CountAndCode, BlockNumber, BlockParamsByNumber, BlockParamsByHash, RpcBlock};
use libproto;
pub use libproto::*;
pub use libproto::request::Request_oneof_req as Request;
use protobuf::Message;
use serde_json;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::sync::mpsc::{Sender, Receiver};
use std::vec::Vec;
use threadpool::*;
use types::filter::Filter;
use types::ids::BlockId;
use util::Address;
use util::H256;
// pub const CHAIN_PUB: u32 = 3;

pub fn chain_pool(pool: &ThreadPool, tx: &Sender<(u32, u32, u32, MsgClass)>, id: u32, msg: Vec<u8>) {
    let tx = tx.clone();
    pool.execute(move || {
                     let (cmd_id, origin, content) = parse_msg(msg.as_slice());
                     tx.send((id, cmd_id, origin, content)).unwrap();
                 });
}

// TODO: RPC Errors
pub fn chain_result(chain: Arc<Chain>, rx: &Receiver<(u32, u32, u32, MsgClass)>, ctx_pub: Sender<(String, Vec<u8>)>) {
    let (id, cmd_id, origin, content_ext) = rx.recv().unwrap();
    trace!("chain_result call {:?} {:?}", id, cmd_id);
    match content_ext {
        MsgClass::REQUEST(mut req) => {
            let mut response = request::Response::new();
            response.set_request_id(req.take_request_id());
            match req.req.clone().unwrap() {
                // TODO: should check the result, parse it first!
                Request::block_number(_) => {
                    // let sys_time = SystemTime::now();
                    let height = chain.get_current_height();
                    response.set_block_number(height);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::block_by_hash(rpc) => {
                    let rpc: BlockParamsByHash = serde_json::from_str(&rpc).expect("Invalid param");
                    let hash = rpc.hash;
                    let include_txs = rpc.include_txs;
                    match chain.block_by_hash(H256::from(hash.as_slice())) {
                        Some(block) => {
                            let rpc_block = RpcBlock::new(hash, include_txs, block.protobuf().write_to_bytes().unwrap());
                            //TODO，发生错误了，应该加错原因给rpc,通知客户
                            serde_json::to_string(&rpc_block)
                                .map(|data| response.set_block(data))
                                .map_err(|_| response.set_none(true));
                        }
                        None => {
                            response.set_none(true);
                        }
                    }
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();

                }

                Request::block_by_height(block_height) => {
                    let block_height: BlockParamsByNumber = serde_json::from_str(&block_height).expect("Invalid param");

                    let include_txs = block_height.include_txs;
                    match chain.block(block_height.block_id.into()) {
                        Some(block) => {
                            let rpc_block = RpcBlock::new(block.hash().to_vec(), include_txs, block.protobuf().write_to_bytes().unwrap());
                            //TODO，发生错误了，应该加错原因给rpc,通知客户
                            serde_json::to_string(&rpc_block)
                                .map(|data| response.set_block(data))
                                .map_err(|_| response.set_none(true));
                        }
                        None => {
                            response.set_none(true);
                        }
                    }
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }
                Request::transaction(hash) => {
                    match chain.full_transaction(H256::from_slice(&hash)) {
                        Some(ts) => {
                            response.set_ts(ts);
                        }
                        None => {
                            response.set_none(true);
                        }
                    }
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }
                Request::transaction_receipt(hash) => {
                    let tx_hash = H256::from_slice(&hash);
                    let receipt = chain.localized_receipt(tx_hash);
                    if let Some(receipt) = receipt {
                        let rpc_receipt: RpcReceipt = receipt.into();
                        let serialized = serde_json::to_string(&rpc_receipt).unwrap();
                        response.set_receipt(serialized);
                    } else {
                        response.set_none(true);
                    }

                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::call(call) => {
                    trace!("Chainvm Call {:?}", call);
                    let block_id: BlockNumber = serde_json::from_str(&(call.height)).expect("Invalid param");
                    let call_request = CallRequest::from(call);
                    let result = chain.eth_call(call_request, block_id.into());
                    response.set_call_result(result.unwrap_or_default());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::filter(encoded) => {
                    trace!("filter: {:?}", encoded);
                    let rpc_filter: RpcFilter = serde_json::from_str(&encoded).expect("Invalid filter");
                    let filter: Filter = rpc_filter.into();
                    let logs = chain.get_logs(filter);
                    let rpc_logs: Vec<RpcLog> = logs.into_iter().map(|x| x.into()).collect();
                    response.set_logs(serde_json::to_string(&rpc_logs).unwrap());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::transaction_count(tx_count) => {
                    trace!("transaction count request from jsonrpc {:?}", tx_count);
                    //TODO 或许有错误返回给用户更好
                    let tx_count: CountAndCode = serde_json::from_str(&tx_count).expect("Invalid param");
                    let address = Address::from_slice(tx_count.address.as_ref());
                    match chain.nonce(&address, tx_count.block_id.into()) {
                        Some(nonce) => {
                            response.set_transaction_count(u64::from(nonce));
                        }
                        None => {
                            response.set_none(true);
                        }
                    };
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();

                }

                Request::code(code_content) => {
                    trace!("code request from josnrpc  {:?}", code_content);
                    let code_content: CountAndCode = serde_json::from_str(&code_content).expect("Invalid param");

                    let address = Address::from_slice(code_content.address.as_ref());
                    match chain.code_at(&address, code_content.block_id.into()) {
                        Some(code) => {
                            match code {
                                Some(code) => {
                                    response.set_code(code);
                                }
                                None => {
                                    response.set_none(true);
                                }
                            }
                        }
                        None => {
                            response.set_none(true);
                        }
                    };
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();

                }

                Request::new_filter(new_filter) => {
                    trace!("new_filter {:?}", new_filter);
                    let new_filter: RpcFilter = serde_json::from_str(&new_filter).expect("Invalid param");
                    trace!("new_filter {:?}", new_filter);
                    response.set_filter_id(chain.new_filter(new_filter) as u64);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::new_block_filter(_) => {
                    let block_filter = chain.new_block_filter();
                    response.set_filter_id(block_filter as u64);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::uninstall_filter(filter_id) => {
                    trace!("uninstall_filter's id is {:?}", filter_id);
                    let index = rpctypes::Index(filter_id as usize);
                    let b = chain.uninstall_filter(index);
                    response.set_uninstall_filter(b);
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::filter_changes(filter_id) => {
                    trace!("filter_changes's id is {:?}", filter_id);
                    let index = rpctypes::Index(filter_id as usize);
                    let log = chain.filter_changes(index).unwrap();
                    trace!("Log is: {:?}", log);
                    response.set_filter_changes(serde_json::to_vec(&log).unwrap());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                Request::filter_logs(filter_id) => {
                    trace!("filter_log's id is {:?}", filter_id);
                    let index = rpctypes::Index(filter_id as usize);
                    let log = chain.filter_logs(index).unwrap_or(vec![]);
                    trace!("Log is: {:?}", log);
                    response.set_filter_logs(serde_json::to_vec(&log).unwrap());
                    let msg: communication::Message = response.into();
                    ctx_pub.send(("chain.rpc".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }

                _ => {}
            }
        }
        MsgClass::RESPONSE(rep) => {}
        MsgClass::HEADER(header) => {}
        MsgClass::BODY(body) => {}
        MsgClass::BLOCK(block) => {
            let mut guard = chain.block_map.write();

            let current_height = chain.get_current_height();
            let max_height = chain.get_max_height();
            let blk_height = block.get_header().get_height();

            let new_map = guard.split_off(&current_height);
            *guard = new_map;

            trace!("received block: block_number:{:?} current_height: {:?} max_height: {:?}", blk_height, current_height, max_height);
            let source = match id {
                submodules::CONSENSUS => BlockSource::CONSENSUS,
                _ => BlockSource::NET,
            };

            if blk_height > current_height && blk_height < current_height + 300 {
                if !guard.contains_key(&blk_height) || (guard.contains_key(&blk_height) && guard[&blk_height].0 == BlockSource::NET && source == BlockSource::CONSENSUS) {
                    trace!("block insert {:?}", blk_height);
                    guard.insert(blk_height, (source, Block::from(block)));
                    let _ = chain.sync_sender.lock().send(blk_height);
                }

            }
        }
        MsgClass::TX(content) => {}
        MsgClass::TXRESPONSE(content) => {}
        MsgClass::STATUS(status) => {
            let status_height = status.get_height();
            if status_height > chain.get_max_height() {
                chain.max_height.store(status_height as usize, Ordering::SeqCst);
                trace!("recieved status update max_height: {:?}", status_height);
            }
            let known_max_height = chain.get_max_height();
            let current_height = chain.get_current_height();
            let target_height = ::std::cmp::min(current_height + 100, known_max_height);
            if current_height < target_height && !chain.is_sync.load(Ordering::SeqCst) {
                let mut diff = target_height - current_height;
                let mut start_height = current_height + 1;
                while diff > 0 {
                    let mut wtr = vec![0; 8];
                    trace!("request sync {:?}", start_height);
                    BigEndian::write_u64(&mut wtr, start_height);
                    let msg = factory::create_msg_ex(submodules::CHAIN, topics::SYNC_BLK, communication::MsgType::MSG, communication::OperateType::SINGLE, origin, wtr);
                    trace!("origin {:?}, chain.sync: OperateType {:?}", origin, communication::OperateType::SINGLE);
                    ctx_pub.send(("chain.sync".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                    start_height += 1;
                    diff -= 1;
                }
                if !chain.is_sync.load(Ordering::SeqCst) {
                    chain.is_sync.store(true, Ordering::SeqCst);
                }
            }
        }
        MsgClass::MSG(content) => {
            if libproto::cmd_id(submodules::CHAIN, topics::SYNC_BLK) == cmd_id {
                trace!("Receive sync {:?} from node-{:?}", BigEndian::read_u64(&content), origin);
                if let Some(block) = chain.block(BlockId::Number(BigEndian::read_u64(&content))) {
                    let msg = factory::create_msg_ex(submodules::CHAIN, topics::NEW_BLK, communication::MsgType::BLOCK, communication::OperateType::SINGLE, origin, block.protobuf().write_to_bytes().unwrap());
                    trace!("origin {:?}, chain.blk: OperateType {:?}", origin, communication::OperateType::SINGLE);
                    ctx_pub.send(("chain.blk".to_string(), msg.write_to_bytes().unwrap())).unwrap();
                }
            } else {
                warn!("other content.");
            }
        }
        MsgClass::VERIFYREQ(req) => {}
        MsgClass::VERIFYRESP(resp) => {}
    }
}
<|endoftext|>


<commit_before>#![feature(default_type_params)]
extern crate curl;
extern crate http;
extern crate hyper;

extern crate test;

use std::fmt::{mod, Show};
use std::from_str::from_str;
use std::io::{IoResult, MemReader};
use std::io::net::ip::SocketAddr;
use std::os;
use std::path::BytesContainer;

use http::connecter::Connecter;

use hyper::net;

static README: &'static [u8] = include_bin!("../README.md");

struct MockStream {
    read: MemReader,
}

impl Clone for MockStream {
    fn clone(&self) -> MockStream {
        MockStream::new()
    }
}

impl MockStream {
    fn new() -> MockStream {
        let head = b"HTTP/1.1 200 OK\r\nServer: Mock\r\n\r\n";
        let mut res = Vec::from_slice(head);
        res.push_all(README);
        MockStream {
            read: MemReader::new(res),
        }
    }
}

impl Reader for MockStream {
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
        self.read.read(buf)
    }
}

impl Writer for MockStream {
    fn write(&mut self, _msg: &[u8]) -> IoResult<()> {
        // we're mocking, what do we care.
        Ok(())
    }
}

#[bench]
fn bench_mock_curl(b: &mut test::Bencher) {
    let mut cwd = os::getcwd();
    cwd.push("README.md");
    let s = format!("file://{}", cwd.container_as_str().unwrap());
    let url = s.as_slice();
    b.iter(|| {
        curl::http::handle()
            .get(url)
            .header("X-Foo", "Bar")
            .exec()
            .unwrap()
    });
}

struct Foo;

impl hyper::header::Header for Foo {
    fn header_name(_: Option<Foo>) -> &'static str {
        "x-foo"
    }
    fn parse_header(_: &[Vec<u8>]) -> Option<Foo> {
        None
    }
    fn fmt_header(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        "Bar".fmt(fmt)
    }
}

impl net::NetworkStream for MockStream {

    fn connect(_host: &str, _port: u16, _scheme: &str) -> IoResult<MockStream> {
        Ok(MockStream::new())
    }

    fn peer_name(&mut self) -> IoResult<SocketAddr> {
        Ok(from_str("127.0.0.1:1337").unwrap())
    }
}

#[bench]
fn bench_mock_hyper(b: &mut test::Bencher) {
    let url = "http://127.0.0.1:1337/";
    b.iter(|| {
        let mut req = hyper::client::Request::with_stream::<MockStream>(
            hyper::Get, hyper::Url::parse(url).unwrap()).unwrap();
        req.headers_mut().set(Foo);

        req
            .start().unwrap()
            .send().unwrap()
            .read_to_string().unwrap()
    });
}

impl Connecter for MockStream {
    fn connect(_addr: SocketAddr, _host: &str, _use_ssl: bool) -> IoResult<MockStream> {
        Ok(MockStream::new())
    }
}

#[bench]
fn bench_mock_http(b: &mut test::Bencher) {
    let url = "http://127.0.0.1:1337/";
    b.iter(|| {
        let mut req: http::client::RequestWriter<MockStream> = http::client::RequestWriter::new(
            http::method::Get,
            hyper::Url::parse(url).unwrap()
        ).unwrap();
        req.headers.extensions.insert("x-foo".to_string(), "Bar".to_string());
        // cant unwrap because Err contains RequestWriter, which does not implement Show
        let mut res = match req.read_response() {
            Ok(res) => res,
            Err(..) => fail!("http response failed")
        };
        res.read_to_string().unwrap();
    });
}

<commit_msg>fix deprecated usage in bench<commit_after>#![feature(default_type_params)]
extern crate curl;
extern crate http;
extern crate hyper;

extern crate test;

use std::fmt::{mod, Show};
use std::from_str::from_str;
use std::io::{IoResult, MemReader};
use std::io::net::ip::SocketAddr;
use std::os;
use std::path::BytesContainer;

use http::connecter::Connecter;

use hyper::net;

static README: &'static [u8] = include_bin!("../README.md");

struct MockStream {
    read: MemReader,
}

impl Clone for MockStream {
    fn clone(&self) -> MockStream {
        MockStream::new()
    }
}

impl MockStream {
    fn new() -> MockStream {
        let head = b"HTTP/1.1 200 OK\r\nServer: Mock\r\n\r\n";
        let mut res = head.to_vec();
        res.push_all(README);
        MockStream {
            read: MemReader::new(res),
        }
    }
}

impl Reader for MockStream {
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
        self.read.read(buf)
    }
}

impl Writer for MockStream {
    fn write(&mut self, _msg: &[u8]) -> IoResult<()> {
        // we're mocking, what do we care.
        Ok(())
    }
}

#[bench]
fn bench_mock_curl(b: &mut test::Bencher) {
    let mut cwd = os::getcwd();
    cwd.push("README.md");
    let s = format!("file://{}", cwd.container_as_str().unwrap());
    let url = s.as_slice();
    b.iter(|| {
        curl::http::handle()
            .get(url)
            .header("X-Foo", "Bar")
            .exec()
            .unwrap()
    });
}

struct Foo;

impl hyper::header::Header for Foo {
    fn header_name(_: Option<Foo>) -> &'static str {
        "x-foo"
    }
    fn parse_header(_: &[Vec<u8>]) -> Option<Foo> {
        None
    }
    fn fmt_header(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        "Bar".fmt(fmt)
    }
}

impl net::NetworkStream for MockStream {

    fn connect(_host: &str, _port: u16, _scheme: &str) -> IoResult<MockStream> {
        Ok(MockStream::new())
    }

    fn peer_name(&mut self) -> IoResult<SocketAddr> {
        Ok(from_str("127.0.0.1:1337").unwrap())
    }
}

#[bench]
fn bench_mock_hyper(b: &mut test::Bencher) {
    let url = "http://127.0.0.1:1337/";
    b.iter(|| {
        let mut req = hyper::client::Request::with_stream::<MockStream>(
            hyper::Get, hyper::Url::parse(url).unwrap()).unwrap();
        req.headers_mut().set(Foo);

        req
            .start().unwrap()
            .send().unwrap()
            .read_to_string().unwrap()
    });
}

impl Connecter for MockStream {
    fn connect(_addr: SocketAddr, _host: &str, _use_ssl: bool) -> IoResult<MockStream> {
        Ok(MockStream::new())
    }
}

#[bench]
fn bench_mock_http(b: &mut test::Bencher) {
    let url = "http://127.0.0.1:1337/";
    b.iter(|| {
        let mut req: http::client::RequestWriter<MockStream> = http::client::RequestWriter::new(
            http::method::Get,
            hyper::Url::parse(url).unwrap()
        ).unwrap();
        req.headers.extensions.insert("x-foo".to_string(), "Bar".to_string());
        // cant unwrap because Err contains RequestWriter, which does not implement Show
        let mut res = match req.read_response() {
            Ok(res) => res,
            Err(..) => fail!("http response failed")
        };
        res.read_to_string().unwrap();
    });
}

<|endoftext|>


<commit_before>// Copyright 2015 Keegan McAllister.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// See `LICENSE` in this repository.

#![feature(plugin, core)]

#![plugin(afl_coverage_plugin)]

// Integer overflow bug.
// Loosely based on:
//   https://github.com/sandstorm-io/capnproto/blob/master/security-advisories/2015-03-02-0-c%2B%2B-integer-overflow.md

extern crate afl_coverage;
extern crate byteorder;

use std::{mem, io, raw};
use std::io::Read;
use byteorder::{ReadBytesExt, LittleEndian, Error};

fn main() {
    let mut stdin = io::stdin();

    // First, the element size.
    let bytes_per_element = stdin.read_u32::<LittleEndian>().unwrap();

    loop {
        let element_count = match stdin.read_u32::<LittleEndian>() {
            Err(Error::UnexpectedEOF) => break,
            Err(e) => panic!(e),
            Ok(n) => n,
        };

        let total_size = element_count.wrapping_mul(bytes_per_element);
        assert!(total_size <= (1 << 20)); // 1MB limit
        let total_size = total_size as usize;

        let mut buf: Vec<u8> = Vec::with_capacity(total_size);

        let dest: &mut [u8] = unsafe {
            mem::transmute(raw::Slice {
                data: buf.as_ptr(),
                len: (element_count as usize) * (bytes_per_element as usize),
            })
        };

        match stdin.by_ref().read(dest) {
            Ok(n) if n == total_size => {
                unsafe {
                    buf.set_len(n);
                    println!("full read: {:?}", buf);
                }
            }
            Ok(n) => println!("partial read: got {}, expected {}", n, total_size),
            Err(_) => println!("error!"),
        }
    }
}
<commit_msg>Use new feature name in example<commit_after>// Copyright 2015 Keegan McAllister.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// See `LICENSE` in this repository.

#![feature(plugin, raw)]

#![plugin(afl_coverage_plugin)]

// Integer overflow bug.
// Loosely based on:
//   https://github.com/sandstorm-io/capnproto/blob/master/security-advisories/2015-03-02-0-c%2B%2B-integer-overflow.md

extern crate afl_coverage;
extern crate byteorder;

use std::{mem, io, raw};
use std::io::Read;
use byteorder::{ReadBytesExt, LittleEndian, Error};

fn main() {
    let mut stdin = io::stdin();

    // First, the element size.
    let bytes_per_element = stdin.read_u32::<LittleEndian>().unwrap();

    loop {
        let element_count = match stdin.read_u32::<LittleEndian>() {
            Err(Error::UnexpectedEOF) => break,
            Err(e) => panic!(e),
            Ok(n) => n,
        };

        let total_size = element_count.wrapping_mul(bytes_per_element);
        assert!(total_size <= (1 << 20)); // 1MB limit
        let total_size = total_size as usize;

        let mut buf: Vec<u8> = Vec::with_capacity(total_size);

        let dest: &mut [u8] = unsafe {
            mem::transmute(raw::Slice {
                data: buf.as_ptr(),
                len: (element_count as usize) * (bytes_per_element as usize),
            })
        };

        match stdin.by_ref().read(dest) {
            Ok(n) if n == total_size => {
                unsafe {
                    buf.set_len(n);
                    println!("full read: {:?}", buf);
                }
            }
            Ok(n) => println!("partial read: got {}, expected {}", n, total_size),
            Err(_) => println!("error!"),
        }
    }
}
<|endoftext|>


<commit_before>/// Return the Hamming distance between the strings,
/// or None if the lengths are mismatched.
pub fn hamming_distance(s1: &str, s2: &str) -> Option<usize> {
    unimplemented!("What is the Hamming Distance between {:?} and {:?}", s1, s2);
}
<commit_msg>hamming: Replaced the Debug formatting of the strands with the Display formatting in the uninplemented message<commit_after>/// Return the Hamming distance between the strings,
/// or None if the lengths are mismatched.
pub fn hamming_distance(s1: &str, s2: &str) -> Option<usize> {
    unimplemented!("What is the Hamming Distance between {} and {}", s1, s2);
}
<|endoftext|>


<commit_before>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use futures::{Future, Stream, Sink, IntoFuture, Poll, Async};
use futures::future::Either;
use futures::sync::mpsc;
use tokio_core::net::{TcpListener, TcpStream, TcpStreamNew};
use tokio_core::reactor::{Core, Timeout, Handle};
use tokio_io::AsyncRead;

use std::cmp;
use std::io;
use std::fmt;
use std::net::SocketAddr;
use std::time::{Duration, SystemTime};
use std::collections::HashMap;

use messages::{Any, Connect, RawMessage};
use node::{ExternalMessage, NodeTimeout};
use helpers::Milliseconds;

use super::EventHandler;
use super::error::{other_error, result_ok, forget_result, into_other, log_error};
use super::codec::MessagesCodec;
use super::EventsAggregator;

#[derive(Debug)]
pub enum NetworkEvent {
    MessageReceived(SocketAddr, RawMessage),
    PeerConnected(SocketAddr, Connect),
    PeerDisconnected(SocketAddr),
}

#[derive(Debug, Clone)]
pub enum NetworkRequest {
    SendMessage(SocketAddr, RawMessage),
    DisconnectWithPeer(SocketAddr),
}

#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub struct NetworkConfiguration {
    // TODO: think more about config parameters
    pub max_incoming_connections: usize,
    pub max_outgoing_connections: usize,
    pub tcp_nodelay: bool,
    pub tcp_keep_alive: Option<u32>,
    pub tcp_reconnect_timeout: u64,
    pub tcp_reconnect_timeout_max: u64,
}

impl Default for NetworkConfiguration {
    fn default() -> NetworkConfiguration {
        NetworkConfiguration {
            max_incoming_connections: 128,
            max_outgoing_connections: 128,
            tcp_keep_alive: None,
            tcp_nodelay: false,
            tcp_reconnect_timeout: 500,
            tcp_reconnect_timeout_max: 600_000,
        }
    }
}

#[derive(Debug)]
pub struct HandlerPart<H: EventHandler> {
    pub core: Core,
    pub handler: H,
    pub timeout_rx: mpsc::Receiver<NodeTimeout>,
    pub network_rx: mpsc::Receiver<NetworkEvent>,
    pub api_rx: mpsc::Receiver<ExternalMessage>,
}

#[derive(Debug)]
pub struct NetworkPart {
    pub listen_address: SocketAddr,
    pub network_config: NetworkConfiguration,
    pub network_requests: (mpsc::Sender<NetworkRequest>, mpsc::Receiver<NetworkRequest>),
    pub network_tx: mpsc::Sender<NetworkEvent>,
}

impl<H: EventHandler> HandlerPart<H> {
    pub fn run(self) -> Result<(), ()> {
        let mut core = self.core;
        let mut handler = self.handler;

        let events_handle = EventsAggregator::new(self.timeout_rx, self.network_rx, self.api_rx)
            .for_each(move |event| {
                handler.handle_event(event);
                Ok(())
            });
        core.run(events_handle)
    }
}

impl NetworkPart {
    pub fn run(self) -> Result<(), ()> {
        let mut core = Core::new().unwrap();

        // Outgoing connections handler
        let mut outgoing_connections: HashMap<SocketAddr, mpsc::Sender<RawMessage>> =
            HashMap::new();

        // Requests handler
        let handle = core.handle();
        let network_tx = self.network_tx.clone();
        let requests_tx = self.network_requests.0.clone();
        let network_config = self.network_config.clone();
        let requests_handle = self.network_requests.1.for_each(|request| {
            match request {
                NetworkRequest::SendMessage(peer, msg) => {
                    let conn_tx = if let Some(conn_tx) = outgoing_connections.get(&peer).cloned() {
                        conn_tx
                    } else {
                        let (conn_tx, conn_rx) = mpsc::channel(10);
                        outgoing_connections.insert(peer, conn_tx.clone());

                        let requests_tx = requests_tx.clone();
                        let connect_handle = NewConnection::create(
                            peer,
                            handle.clone(),
                            network_config.tcp_reconnect_timeout,
                            network_config.tcp_reconnect_timeout_max,
                        ).and_then(move |sock| {
                            sock.set_nodelay(network_config.tcp_nodelay).unwrap();
                            info!("Established connection with peer={}", peer);

                            let stream = sock.framed(MessagesCodec);
                            let (sink, stream) = stream.split();

                            let writer = conn_rx
                                .map_err(|_| other_error("Can't send data into socket"))
                                .forward(sink);
                            let reader = stream.for_each(result_ok).map_err(into_other);

                            reader
                                .select2(writer)
                                .map_err(|_| other_error("Socket error"))
                                .and_then(|res| match res {
                                    Either::A((_, _reader)) => Ok(()).into_future(),
                                    Either::B((_, _writer)) => Ok(()).into_future(),
                                })
                        })
                            .then(move |res| {
                                info!("Connection with peer={} closed, reason={:?}", peer, res);
                                // outgoing_connections.remove(&peer);

                                let request = NetworkRequest::DisconnectWithPeer(peer);
                                requests_tx
                                    .clone()
                                    .send(request)
                                    .map(forget_result)
                                    .map_err(into_other)
                            })
                            .map_err(log_error);
                        handle.spawn(connect_handle);
                        conn_tx
                    };

                    let duration = Duration::from_secs(5);
                    let send_timeout = Timeout::new(duration, &handle)
                        .unwrap()
                        .and_then(result_ok)
                        .map_err(|_| other_error("Can't timeout"));

                    let send_handle = conn_tx.send(msg).map(forget_result).map_err(log_error);

                    let timeouted_connect = send_handle
                        .select2(send_timeout)
                        .map_err(|_| other_error("Unable to send message"))
                        .and_then(move |either| match either {
                            Either::A((send, _timeout_fut)) => Ok(send),
                            Either::B((_, _connect_fut)) => Err(other_error("Send timeout")),
                        })
                        .map_err(log_error);

                    handle.spawn(timeouted_connect);
                }
                NetworkRequest::DisconnectWithPeer(peer) => {
                    outgoing_connections.remove(&peer);

                    let event = NetworkEvent::PeerDisconnected(peer);
                    let event_handle = network_tx.clone().send(event).map(forget_result).map_err(
                        log_error,
                    );
                    handle.spawn(event_handle);
                }
            }

            Ok(())
        });

        // Incoming connections handler
        let listener = TcpListener::bind(&self.listen_address, &core.handle()).unwrap();
        let network_tx = network_tx.clone();
        let server = listener
            .incoming()
            .fold(network_tx, move |network_tx, (sock, addr)| {
                info!("Accepted incoming connection with peer={}", addr);

                let stream = sock.framed(MessagesCodec);
                let (_, stream) = stream.split();
                let network_tx = network_tx.clone();
                stream
                    .into_future()
                    .map_err(|e| e.0)
                    .and_then(move |(raw, stream)| {
                        let msg = raw.map(Any::from_raw);
                        if let Some(Ok(Any::Connect(msg))) = msg {
                            Ok((msg, stream))
                        } else {
                            Err(other_error("First message is not Connect"))
                        }
                    })
                    .and_then(move |(connect, stream)| {
                        info!("Received handshake message={:?}", connect);

                        let event = NetworkEvent::PeerConnected(addr, connect);
                        let connect_event = network_tx.clone().send(event).map_err(into_other);

                        let network_tx = network_tx.clone();
                        let messages_stream = stream.for_each(move |raw| {
                            let event = NetworkEvent::MessageReceived(addr, raw);
                            network_tx.clone().send(event).map(forget_result).map_err(
                                into_other,
                            )
                        });

                        messages_stream
                            .join(connect_event)
                            .map(move |(_, stream)| stream)
                            .map_err(into_other)
                    })
                    .map_err(into_other)
            })
            .map(forget_result)
            .map_err(log_error);
        core.handle().spawn(server);

        core.run(requests_handle)
    }
}

pub struct NewConnection {
    inner: Option<TcpStreamNew>,
    address: SocketAddr,
    handle: Handle,
    groth_factor: f32,
    start_time: SystemTime,
    reconnect_timeout: Milliseconds,
    reconnect_timeout_max: Milliseconds,
}

impl NewConnection {
    pub fn create(
        address: SocketAddr,
        handle: Handle,
        reconnect_timeout: Milliseconds,
        reconnect_timeout_max: Milliseconds,
    ) -> NewConnection {
        NewConnection {
            inner: Some(TcpStream::connect(&address, &handle)),
            address,
            handle,
            groth_factor: 1.5,
            start_time: SystemTime::now(),
            reconnect_timeout,
            reconnect_timeout_max,
        }
    }

    fn next_attempt(&mut self, now: SystemTime) -> io::Result<()> {
        self.start_time = now;
        if self.reconnect_timeout == self.reconnect_timeout_max {
            return Err(other_error(
                &format!("Maximum reconnect timeout reached for connection with the peer: {}",
                self.address,
            ),
            ));
        }

        self.reconnect_timeout = (self.reconnect_timeout as f32 * self.groth_factor) as
            Milliseconds;
        self.reconnect_timeout = cmp::min(self.reconnect_timeout, self.reconnect_timeout_max);
        self.inner = Some(TcpStream::connect(&self.address, &self.handle));
        trace!(
            "Add reconnect timeout to {}, delay = {} ms",
            self.address,
            self.reconnect_timeout
        );
        Ok(())
    }
}

impl Future for NewConnection {
    type Item = TcpStream;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<TcpStream, io::Error> {
        debug!("Poll, stream={}", self.inner.is_some());
        if let Some(mut stream) = self.inner.take() {
            match stream.poll() {
                Ok(Async::Ready(stream)) => Ok(Async::Ready(stream)),
                Ok(Async::NotReady) => {
                    self.inner = Some(stream);
                    Ok(Async::NotReady)
                }
                Err(e) => {
                    error!(
                        "An error occured during connecting to the peer: {}, error: {}",
                        self.address,
                        e
                    );
                    self.inner = None;
                    Ok(Async::NotReady)
                }
            }
        } else {
            let now = SystemTime::now();
            let reconnect_timeout = Duration::from_millis(self.reconnect_timeout);
            if now > self.start_time + reconnect_timeout {
                self.next_attempt(now)?;
            }
            Ok(Async::NotReady)
        }
    }
}

impl fmt::Debug for NewConnection {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.pad("NewConnection { .. }")
    }
}<commit_msg>Temporary remove reconnect future<commit_after>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use futures::{Future, Stream, Sink, IntoFuture, Poll, Async};
use futures::future::Either;
use futures::sync::mpsc;
use tokio_core::net::{TcpListener, TcpStream, TcpStreamNew};
use tokio_core::reactor::{Core, Timeout, Handle};
use tokio_io::AsyncRead;

use std::cmp;
use std::io;
use std::fmt;
use std::net::SocketAddr;
use std::time::{Duration, SystemTime};
use std::collections::HashMap;

use messages::{Any, Connect, RawMessage};
use node::{ExternalMessage, NodeTimeout};
use helpers::Milliseconds;

use super::EventHandler;
use super::error::{other_error, result_ok, forget_result, into_other, log_error};
use super::codec::MessagesCodec;
use super::EventsAggregator;

#[derive(Debug)]
pub enum NetworkEvent {
    MessageReceived(SocketAddr, RawMessage),
    PeerConnected(SocketAddr, Connect),
    PeerDisconnected(SocketAddr),
}

#[derive(Debug, Clone)]
pub enum NetworkRequest {
    SendMessage(SocketAddr, RawMessage),
    DisconnectWithPeer(SocketAddr),
}

#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub struct NetworkConfiguration {
    // TODO: think more about config parameters
    pub max_incoming_connections: usize,
    pub max_outgoing_connections: usize,
    pub tcp_nodelay: bool,
    pub tcp_keep_alive: Option<u32>,
    pub tcp_reconnect_timeout: u64,
    pub tcp_reconnect_timeout_max: u64,
}

impl Default for NetworkConfiguration {
    fn default() -> NetworkConfiguration {
        NetworkConfiguration {
            max_incoming_connections: 128,
            max_outgoing_connections: 128,
            tcp_keep_alive: None,
            tcp_nodelay: false,
            tcp_reconnect_timeout: 500,
            tcp_reconnect_timeout_max: 600_000,
        }
    }
}

#[derive(Debug)]
pub struct HandlerPart<H: EventHandler> {
    pub core: Core,
    pub handler: H,
    pub timeout_rx: mpsc::Receiver<NodeTimeout>,
    pub network_rx: mpsc::Receiver<NetworkEvent>,
    pub api_rx: mpsc::Receiver<ExternalMessage>,
}

#[derive(Debug)]
pub struct NetworkPart {
    pub listen_address: SocketAddr,
    pub network_config: NetworkConfiguration,
    pub network_requests: (mpsc::Sender<NetworkRequest>, mpsc::Receiver<NetworkRequest>),
    pub network_tx: mpsc::Sender<NetworkEvent>,
}

impl<H: EventHandler> HandlerPart<H> {
    pub fn run(self) -> Result<(), ()> {
        let mut core = self.core;
        let mut handler = self.handler;

        let events_handle = EventsAggregator::new(self.timeout_rx, self.network_rx, self.api_rx)
            .for_each(move |event| {
                handler.handle_event(event);
                Ok(())
            });
        core.run(events_handle)
    }
}

impl NetworkPart {
    pub fn run(self) -> Result<(), ()> {
        let mut core = Core::new().unwrap();

        // Outgoing connections handler
        let mut outgoing_connections: HashMap<SocketAddr, mpsc::Sender<RawMessage>> =
            HashMap::new();

        // Requests handler
        let handle = core.handle();
        let network_tx = self.network_tx.clone();
        let requests_tx = self.network_requests.0.clone();
        let network_config = self.network_config.clone();
        let requests_handle = self.network_requests.1.for_each(|request| {
            match request {
                NetworkRequest::SendMessage(peer, msg) => {
                    let conn_tx = if let Some(conn_tx) = outgoing_connections.get(&peer).cloned() {
                        conn_tx
                    } else {
                        let (conn_tx, conn_rx) = mpsc::channel(10);
                        outgoing_connections.insert(peer, conn_tx.clone());

                        let requests_tx = requests_tx.clone();
                        let connect_handle = TcpStream::connect(&peer, &handle).and_then(move |sock| {
                            sock.set_nodelay(network_config.tcp_nodelay).unwrap();
                            info!("Established connection with peer={}", peer);

                            let stream = sock.framed(MessagesCodec);
                            let (sink, stream) = stream.split();

                            let writer = conn_rx
                                .map_err(|_| other_error("Can't send data into socket"))
                                .forward(sink);
                            let reader = stream.for_each(result_ok).map_err(into_other);

                            reader
                                .select2(writer)
                                .map_err(|_| other_error("Socket error"))
                                .and_then(|res| match res {
                                    Either::A((_, _reader)) => Ok(()).into_future(),
                                    Either::B((_, _writer)) => Ok(()).into_future(),
                                })
                        })
                            .then(move |res| {
                                info!("Connection with peer={} closed, reason={:?}", peer, res);
                                // outgoing_connections.remove(&peer);

                                let request = NetworkRequest::DisconnectWithPeer(peer);
                                requests_tx
                                    .clone()
                                    .send(request)
                                    .map(forget_result)
                                    .map_err(into_other)
                            })
                            .map_err(log_error);
                        handle.spawn(connect_handle);
                        conn_tx
                    };

                    let duration = Duration::from_secs(5);
                    let send_timeout = Timeout::new(duration, &handle)
                        .unwrap()
                        .and_then(result_ok)
                        .map_err(|_| other_error("Can't timeout"));

                    let send_handle = conn_tx.send(msg).map(forget_result).map_err(log_error);

                    let timeouted_connect = send_handle
                        .select2(send_timeout)
                        .map_err(|_| other_error("Unable to send message"))
                        .and_then(move |either| match either {
                            Either::A((send, _timeout_fut)) => Ok(send),
                            Either::B((_, _connect_fut)) => Err(other_error("Send timeout")),
                        })
                        .map_err(log_error);

                    handle.spawn(timeouted_connect);
                }
                NetworkRequest::DisconnectWithPeer(peer) => {
                    outgoing_connections.remove(&peer);

                    let event = NetworkEvent::PeerDisconnected(peer);
                    let event_handle = network_tx.clone().send(event).map(forget_result).map_err(
                        log_error,
                    );
                    handle.spawn(event_handle);
                }
            }

            Ok(())
        });

        // Incoming connections handler
        let listener = TcpListener::bind(&self.listen_address, &core.handle()).unwrap();
        let network_tx = network_tx.clone();
        let server = listener
            .incoming()
            .fold(network_tx, move |network_tx, (sock, addr)| {
                info!("Accepted incoming connection with peer={}", addr);

                let stream = sock.framed(MessagesCodec);
                let (_, stream) = stream.split();
                let network_tx = network_tx.clone();
                stream
                    .into_future()
                    .map_err(|e| e.0)
                    .and_then(move |(raw, stream)| {
                        let msg = raw.map(Any::from_raw);
                        if let Some(Ok(Any::Connect(msg))) = msg {
                            Ok((msg, stream))
                        } else {
                            Err(other_error("First message is not Connect"))
                        }
                    })
                    .and_then(move |(connect, stream)| {
                        info!("Received handshake message={:?}", connect);

                        let event = NetworkEvent::PeerConnected(addr, connect);
                        let connect_event = network_tx.clone().send(event).map_err(into_other);

                        let network_tx = network_tx.clone();
                        let messages_stream = stream.for_each(move |raw| {
                            let event = NetworkEvent::MessageReceived(addr, raw);
                            network_tx.clone().send(event).map(forget_result).map_err(
                                into_other,
                            )
                        });

                        messages_stream
                            .join(connect_event)
                            .map(move |(_, stream)| stream)
                            .map_err(into_other)
                    })
                    .map_err(into_other)
            })
            .map(forget_result)
            .map_err(log_error);
        core.handle().spawn(server);

        core.run(requests_handle)
    }
}

pub struct NewConnection {
    inner: Option<TcpStreamNew>,
    address: SocketAddr,
    handle: Handle,
    groth_factor: f32,
    start_time: SystemTime,
    reconnect_timeout: Milliseconds,
    reconnect_timeout_max: Milliseconds,
}

impl NewConnection {
    pub fn create(
        address: SocketAddr,
        handle: Handle,
        reconnect_timeout: Milliseconds,
        reconnect_timeout_max: Milliseconds,
    ) -> NewConnection {
        NewConnection {
            inner: Some(TcpStream::connect(&address, &handle)),
            address,
            handle,
            groth_factor: 1.5,
            start_time: SystemTime::now(),
            reconnect_timeout,
            reconnect_timeout_max,
        }
    }

    fn next_attempt(&mut self, now: SystemTime) -> io::Result<()> {
        self.start_time = now;
        if self.reconnect_timeout == self.reconnect_timeout_max {
            return Err(other_error(
                &format!("Maximum reconnect timeout reached for connection with the peer: {}",
                self.address,
            ),
            ));
        }

        self.reconnect_timeout = (self.reconnect_timeout as f32 * self.groth_factor) as
            Milliseconds;
        self.reconnect_timeout = cmp::min(self.reconnect_timeout, self.reconnect_timeout_max);
        self.inner = Some(TcpStream::connect(&self.address, &self.handle));
        trace!(
            "Add reconnect timeout to {}, delay = {} ms",
            self.address,
            self.reconnect_timeout
        );
        Ok(())
    }
}

impl Future for NewConnection {
    type Item = TcpStream;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<TcpStream, io::Error> {
        debug!("Poll, stream={}", self.inner.is_some());
        if let Some(mut stream) = self.inner.take() {
            match stream.poll() {
                Ok(Async::Ready(stream)) => Ok(Async::Ready(stream)),
                Ok(Async::NotReady) => {
                    self.inner = Some(stream);
                    Ok(Async::NotReady)
                }
                Err(e) => {
                    error!(
                        "An error occured during connecting to the peer: {}, error: {}",
                        self.address,
                        e
                    );
                    self.inner = None;
                    Ok(Async::NotReady)
                }
            }
        } else {
            let now = SystemTime::now();
            let reconnect_timeout = Duration::from_millis(self.reconnect_timeout);
            if now > self.start_time + reconnect_timeout {
                self.next_attempt(now)?;
            }
            Ok(Async::NotReady)
        }
    }
}

impl fmt::Debug for NewConnection {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.pad("NewConnection { .. }")
    }
}<|endoftext|>


<commit_before>// Copyright (c) 2016-2020 Fabian Schuiki

use proc_macro::TokenStream;
use quote::{format_ident, quote};
use syn::DeriveInput;

pub(crate) fn accept_visitor(input: TokenStream) -> TokenStream {
    // Parse the input.
    let input = syn::parse_macro_input!(input as DeriveInput);
    let mut output = proc_macro2::TokenStream::new();

    let name = &input.ident;
    let generics = &input.generics;

    // Generate the match that visits the relevant fields of the input.
    let mut visits = vec![];
    match &input.data {
        syn::Data::Struct(input) => {
            let visit = visit_fields(&input.fields);
            visits.push(quote! {
                #name #visit
            });
        }
        syn::Data::Enum(input) => {
            for variant in &input.variants {
                let variant_name = &variant.ident;
                let visit = visit_fields(&variant.fields);
                visits.push(quote! {
                    #name::#variant_name #visit
                });
            }
        }
        _ => panic!("unsupported item for AcceptVisitor"),
    };

    // Determine the impl generics, which may add another lifetime.
    let mut impl_generics = generics.clone();
    let lt = crate::determine_lifetime(&mut impl_generics);

    // Generate the implementation of the `AcceptVisitor` trait.
    output.extend(quote! {
        impl #impl_generics AcceptVisitor<#lt> for #name #generics {
            fn accept<V: Visitor<#lt> + ?Sized>(&#lt self, visitor: &mut V) {
                match self {
                    #(#visits,)*
                }
            }
        }
    });

    output.into()
}

/// Generate the code to visit fields in a struct-like item.
fn visit_fields(fields: &syn::Fields) -> proc_macro2::TokenStream {
    // Generate a destructuring pattern that assigns predictable names to all
    // fields.
    let mut names = vec![];
    let pat = match fields {
        syn::Fields::Named(ref fields) => {
            let mut mapping = vec![];
            for (i, field) in fields.named.iter().enumerate() {
                let field_name = &field.ident;
                let name = format_ident!("arg{}", i);
                names.push(name.clone());
                mapping.push(quote! {
                    #field_name: #name
                });
            }
            quote! { {#(#mapping),*} }
        }
        syn::Fields::Unnamed(ref fields) => {
            let mut mapping = vec![];
            for (i, _) in fields.unnamed.iter().enumerate() {
                let name = format_ident!("arg{}", i);
                names.push(name.clone());
                mapping.push(name);
            }
            quote! { (#(#mapping),*) }
        }
        syn::Fields::Unit => {
            quote! {}
        }
    };

    // Generate the visit calls for the names that we do not skip or ignore.
    quote! {
        #pat => {
            #(#names.walk(visitor);)*
        }
    }
}
<commit_msg>derive: Honor `#[dont_visit]`<commit_after>// Copyright (c) 2016-2020 Fabian Schuiki

use proc_macro::TokenStream;
use quote::{format_ident, quote, ToTokens};
use syn::DeriveInput;

pub(crate) fn accept_visitor(input: TokenStream) -> TokenStream {
    // Parse the input.
    let input = syn::parse_macro_input!(input as DeriveInput);
    let mut output = proc_macro2::TokenStream::new();

    let name = &input.ident;
    let generics = &input.generics;

    // Generate the match that visits the relevant fields of the input.
    let mut visits = vec![];
    match &input.data {
        syn::Data::Struct(input) => {
            let visit = visit_fields(&input.fields);
            visits.push(quote! {
                #name #visit
            });
        }
        syn::Data::Enum(input) => {
            for variant in &input.variants {
                let variant_name = &variant.ident;
                let visit = visit_fields(&variant.fields);
                visits.push(quote! {
                    #name::#variant_name #visit
                });
            }
        }
        _ => panic!("unsupported item for AcceptVisitor"),
    };

    // Determine the impl generics, which may add another lifetime.
    let mut impl_generics = generics.clone();
    let lt = crate::determine_lifetime(&mut impl_generics);

    // Generate the implementation of the `AcceptVisitor` trait.
    output.extend(quote! {
        impl #impl_generics AcceptVisitor<#lt> for #name #generics {
            fn accept<V: Visitor<#lt> + ?Sized>(&#lt self, visitor: &mut V) {
                match self {
                    #(#visits,)*
                }
            }
        }
    });

    output.into()
}

/// Check if a field has the `#[dont_visit]` attribute.
fn has_dont_visit(attrs: &[syn::Attribute]) -> bool {
    attrs
        .iter()
        .any(|attr| attr.to_token_stream().to_string() == "# [dont_visit]")
}

/// Generate the code to visit fields in a struct-like item.
fn visit_fields(fields: &syn::Fields) -> proc_macro2::TokenStream {
    // Generate a destructuring pattern that assigns predictable names to all
    // fields.
    let mut names = vec![];
    let pat = match fields {
        syn::Fields::Named(ref fields) => {
            let mut mapping = vec![];
            for (i, field) in fields.named.iter().enumerate() {
                let field_name = &field.ident;
                let name = format_ident!("arg{}", i);
                if !has_dont_visit(&field.attrs) {
                    names.push(name.clone());
                }
                mapping.push(quote! {
                    #field_name: #name
                });
            }
            quote! { {#(#mapping),*} }
        }
        syn::Fields::Unnamed(ref fields) => {
            let mut mapping = vec![];
            for (i, field) in fields.unnamed.iter().enumerate() {
                let name = format_ident!("arg{}", i);
                if !has_dont_visit(&field.attrs) {
                    names.push(name.clone());
                }
                mapping.push(name);
            }
            quote! { (#(#mapping),*) }
        }
        syn::Fields::Unit => {
            quote! {}
        }
    };

    // Generate the visit calls for the names that we do not skip or ignore.
    quote! {
        #pat => {
            #(#names.walk(visitor);)*
        }
    }
}
<|endoftext|>


<commit_before>use na::{Mat2, DMat};
use std::collections::HashMap;
use std::num::*;
use itertools::Itertools;
use std::cell::RefCell;
use std::rc::Rc;

mod test;

impl Default for DStreamProps {
    fn default() -> DStreamProps {
        DStreamProps {
            c_m: 3.0,
            c_l: 0.8,
            lambda: 0.998,
            beta: 0.3,
            i_bins: 10 as usize,
            j_bins: 10 as usize,
            i_range: (-10.0, 10.0),
            j_range: (-10.0, 10.0),
        }
    }
}
pub struct DStreamProps {
    c_m: f64,
    c_l: f64,
    lambda: f64,
    beta: f64,
    i_bins: usize,
    j_bins: usize,
    i_range: (f64, f64),
    j_range: (f64, f64),
}

#[derive(Clone)]
pub struct RawData {
    x: f64,
    y: f64,
    v: f64,
}

#[derive(Clone)]
#[derive(Debug)]
#[derive(Copy)]
struct GridPoint {
    t: u32,
    v: f64,
}
#[derive(Clone)]
struct GridData {
    i: usize,
    j: usize,
    v: f64,
}
struct TheWorld {
    g_vec: Vec<((usize, usize), DG)>,
}

#[test]
fn test_new_put_works() {
    let raw_data_1 = RawData{x: 1.0, y: -1.0, v: 123.45};
    let raw_data_2 = RawData{x: 5.0, y: -7.0, v: 23.45};
    let raw_data_3 = RawData{x: 5.0, y: -7.0, v: 83.45};
    let raw_data_4 = RawData{x: 15.0, y: -17.0, v: 83.45};

    let rd_vec = vec!(raw_data_1, raw_data_2, raw_data_3, raw_data_4);

    let t = 1;

    let default_vec : Vec<GridPoint> = Vec::new();
    let mut world = TheWorld{g_vec: Vec::new()};
    world.init(default_vec);
    let res1 = world.put(t, rd_vec.clone());
    let res2 = world.put(t + 1, rd_vec.clone());
    let res3 = world.put(t + 2, rd_vec.clone());

}

impl TheWorld {

    #[test]
    fn test_which_grid() {
        let loc = (-6., 6.);
        let i_rn = (-10.0, 10.0);
        let j_rn = (-10.0, 10.0);
        let i_bins = 10 as usize;
        let j_bins = 10 as usize;

        let result = TheWorld::which_grid(loc, i_rn, j_rn, i_bins, j_bins).unwrap();
        assert_eq!((2, 8), result);

    }
    fn which_grid(val: (f64, f64), i_range: (f64, f64), j_range: (f64, f64), i_bins: usize, j_bins: usize) -> Result<(usize, usize), String> {
        let i_size = (i_range.1 - i_range.0) / i_bins as f64;
        let i_number_of_sizes = ((val.0 - i_range.0) / i_size) as usize;
        let j_size = (j_range.1 - j_range.0) / j_bins as f64;
        let j_number_of_sizes = ((val.1 - j_range.0) / j_size) as usize;

        assert!(i_number_of_sizes < i_bins);
        assert!(j_number_of_sizes < j_bins);

        Ok((i_number_of_sizes, j_number_of_sizes))
    }

    fn init(&mut self, def_bucket: Vec<GridPoint>) {
        println!("init");
        let props: DStreamProps = DStreamProps { ..Default::default() };

        for i in 0..props.i_bins {
            for j in 0..props.j_bins {
                let z_clone = def_bucket.clone();
                let some_default_dg = DG {i: i, j: j,
                    updates_and_vals: z_clone, removed_as_spore_adic: Vec::new(),
                    label: GridLabel::ClassA, status: GridStatus::Normal,};
                (self.g_vec).push(((i as usize, j as usize), some_default_dg));
            }
        }
    }
    fn do_time_steps() {}
    fn do_one_time_step(t: u32, data: Vec<RawData>) {}

    fn get_by_idx(&mut self, idx: (usize, usize))-> DG {
        let  vec_f: Vec<((usize, usize), DG)> = self.g_vec.clone().into_iter().filter(|i| i.0 == idx).collect();
        vec_f[0].1.clone()
    }
    fn update_by_idx(&mut self, idx: (usize, usize), dg: DG) -> Result<(), String> {
        self.g_vec.retain(|i| i.0 != idx);
        self.g_vec.push((idx, dg));
        Ok(())
    }
    fn put(&mut self, t: u32, dat: Vec<RawData>) -> Result<(), String> {

        fn validate_range(loc2d: (f64, f64)) -> bool {

            let props: DStreamProps = DStreamProps { ..Default::default() };

            if ((loc2d.0 <= props.i_range.1) &&
            (loc2d.0 >= props.i_range.0) &&
            (loc2d.1 <= props.j_range.1) &&
            (loc2d.1 >= props.j_range.0)) {
                println!("valid!");
            } else {
                println!("invalid! -- bad input range");
                return false
            }
            return true
        }
        let with_idxs: Vec<GridData> = dat
            .iter()
            .filter(|rd| validate_range((rd.x, rd.y)))
            .map(|rd| self.which_idxs(rd).unwrap())
            .collect();

        let props: DStreamProps = DStreamProps { ..Default::default() };

        for (key, group) in with_idxs.iter().group_by(|gd| (gd.i, gd.j)) {
            println!("--put: key, g size: ({},{}) : {}", key.0, key.1, group.len());

            let the_vec_of_vals: Vec<f64> = group.iter().map(|t| t.v).collect();

            let mut some_default_dg = DG {i: key.0, j: key.1, updates_and_vals: Vec::<GridPoint>::new(), removed_as_spore_adic: Vec::new(), label: GridLabel::ClassA, status: GridStatus::Normal,};

            let teh_dg: &mut DG = &mut self.get_by_idx(key);
            let update_dg_result = teh_dg.update(t, the_vec_of_vals);
            let udpate_world_result = self.update_by_idx(key, teh_dg.clone());
        }
        Ok(())
    }
    fn which_idxs(&self, dat: &RawData) -> Result<GridData, String> {

        let props: DStreamProps = DStreamProps { ..Default::default() };

        let idxs = TheWorld::which_grid((dat.x, dat.y), props.i_range, props.j_range,props.i_bins, props.j_bins).unwrap();

        Ok((GridData{i:idxs.0,j:idxs.1,v:dat.v}))

    }
}

#[test]
fn test_dg_update_and_get() {
    let mut dg = DG {i: 0, j:0, updates_and_vals: Vec::new(), removed_as_spore_adic: Vec::new(), label: GridLabel::ClassA, status: GridStatus::Normal,};
    dg.update(1, vec!(100.0));

    dg.update(10, vec!(200.0));
    dg.update(20, vec!(300.0));

    assert_eq!((1, 100.0), dg.get_last_update_and_value_to(2));
    assert_eq!((10, 200.0), dg.get_last_update_and_value_to(14));
    assert_eq!((20, 300.0), dg.get_last_update_and_value_to(22));

    let v_at_t = dg.get_at_time(30);
    println!("v: {}", v_at_t);
}

#[test]
fn test_removed_as_sporadic() {
    let mut dg = DG {i: 0, j:0, updates_and_vals: Vec::new(), removed_as_spore_adic: vec!(1, 2, 3, 4), label: GridLabel::ClassA, status: GridStatus::Normal,};
    assert_eq!(1, dg.get_last_time_removed_as_sporadic_to(2));
    assert_eq!(4, dg.get_last_time_removed_as_sporadic_to(6));
}

#[derive(Debug)]
#[derive(Clone)]
//#[derive(Copy)] TODO: how to make this work?
struct DG {
    i: usize,
    j: usize,
    updates_and_vals: Vec<GridPoint>,
    removed_as_spore_adic: Vec<u32>,
    label: GridLabel,
    status: GridStatus,
}
#[derive(Debug)]
#[derive(Clone)]
enum GridLabel { ClassA, }
#[derive(Debug)]
#[derive(Clone)]
enum GridStatus { Sporadic, Normal, }
impl DG {

    fn is_dense_at_time(&self, t: u32) -> bool {
        let props: DStreamProps = DStreamProps { ..Default::default() };
        let n_size = (self.i * self.j) as f64;
        let d_m = props.c_m / (n_size * (1.0 - props.lambda));
        self.get_at_time(t) >= d_m
    }

    fn get_at_time(&self, t: u32) -> f64 {
        let last_update_time_and_value = self.get_last_update_and_value_to(t);
        let coeff = self.coeff(t, last_update_time_and_value.0);
        coeff * last_update_time_and_value.1 + 1.0
    }
    fn update(&mut self, t: u32, vals: Vec<f64>) -> Result<(), String> {
        let sum = vals.clone().iter().fold(0.0, |sum, x| sum + x);
        self.updates_and_vals.push(GridPoint {t: t, v: sum});
        println!("  times updated dg: {}", self.updates_and_vals.len());
        Ok(())
    }

    fn get_last_update_and_value_to(&self, t: u32) -> (u32, f64) {
        let a: GridPoint = self.updates_and_vals.clone().into_iter().filter(|bp| bp.t < t).max_by_key(|bp| bp.t).unwrap();
        let t_l = a.t;
        let v_l = a.v;
        println!("  last update relative to {} is (t: {}, v: {})", t, t_l, v_l);
        (t_l, v_l)
    }

    fn get_last_time_removed_as_sporadic_to(&self, t: u32) -> u32 {
        let t = self.removed_as_spore_adic.clone().into_iter().filter(|&the_t| the_t < t).max().unwrap();
        t
    }

    fn coeff(&self, t_n: u32, t_l: u32) -> f64 {
        let props: DStreamProps = DStreamProps { ..Default::default() };
        props.lambda.powf((t_n - t_l) as f64)
    }

 }

#[test]
fn test_initialize_clustering() {

    let result = initialize_clustering(Vec::new());
    assert_eq!(Ok(()), result);
}

pub fn initialize_clustering(grid_list: Vec<RawData>) -> Result<(), String> {
    //update density of all grids in grid_list
    //assign each dense grid to a distinct cluster
    //label all other grids as NO_CLASS; bad grids!
    /*
    do until no change in cluster labels
          foreach cluster c
            foreach outside grid g of c
                foreach neighboring grid h of g
                    if h belongs to cluster c'
                        if |c| > |c'| label all grids in c' as c
                        else label all grids in c as c'
                    else if (h is translational) label h as in c
    end do
    */
    Ok(())
}

pub fn adjust_clustering(grid_list: Vec<RawData>) -> Result<(), String> {
    //update the density of all grids in grid_list
    /*
    foreach grid g whose attribute (dense/sparse/transitional) is changed since last call to adjust_clustering()
        if g is a sparse grid
            delete g from its cluster c, label g as NO_CLASS
            if (c becomes unconnected) split c into two clusters
        else if g is a dense grid
            among all neighboring grids of g, find out the grid h whoe cluster c_h has the largest size
            if h is a dense grid
                if (g is labelled as NO_CLASS) label g as in c_h
                else if (g is in cluster c and |c| > |c_h|)
                    label all grids in c_h as in c
                else if (g is in cluster c and |c| <= |c_h|)
                    label all grids in c as in c_h
            else if h is a transitional grid
                if ((g is NO_CLASS) and (h is an outside grid if g is added to c_h)) label g as in c_h
                else if (g is in cluster c and |c| >= |c_h|)
                    move h from cluster c_h to c
        else if (g is transitional grid)
            among neighboring clusters of g, find the largest one c'
            satisfying that g is an outside grid if added to it and
            label g as in c'
    */

    Ok(())
}<commit_msg>compute grid label<commit_after>use na::{Mat2, DMat};
use std::collections::HashMap;
use std::num::*;
use itertools::Itertools;
use std::cell::RefCell;
use std::rc::Rc;

mod test;

impl Default for DStreamProps {
    fn default() -> DStreamProps {
        DStreamProps {
            c_m: 3.0,
            c_l: 0.8,
            lambda: 0.998,
            beta: 0.3,
            i_bins: 10 as usize,
            j_bins: 10 as usize,
            i_range: (-10.0, 10.0),
            j_range: (-10.0, 10.0),
        }
    }
}
pub struct DStreamProps {
    c_m: f64,
    c_l: f64,
    lambda: f64,
    beta: f64,
    i_bins: usize,
    j_bins: usize,
    i_range: (f64, f64),
    j_range: (f64, f64),
}

#[derive(Clone)]
pub struct RawData {
    x: f64,
    y: f64,
    v: f64,
}

#[derive(Clone)]
#[derive(Debug)]
#[derive(Copy)]
struct GridPoint {
    t: u32,
    v: f64,
}
#[derive(Clone)]
struct GridData {
    i: usize,
    j: usize,
    v: f64,
}
struct TheWorld {
    g_vec: Vec<((usize, usize), DG)>,
}

#[test]
fn test_new_put_works() {
    let raw_data_1 = RawData{x: 1.0, y: -1.0, v: 123.45};
    let raw_data_2 = RawData{x: 5.0, y: -7.0, v: 23.45};
    let raw_data_3 = RawData{x: 5.0, y: -7.0, v: 83.45};
    let raw_data_4 = RawData{x: 15.0, y: -17.0, v: 83.45};

    let rd_vec = vec!(raw_data_1, raw_data_2, raw_data_3, raw_data_4);

    let t = 1;

    let default_vec : Vec<GridPoint> = Vec::new();
    let mut world = TheWorld{g_vec: Vec::new()};
    world.init(default_vec);
    let res1 = world.put(t, rd_vec.clone());
    let res2 = world.put(t + 1, rd_vec.clone());
    let res3 = world.put(t + 2, rd_vec.clone());

}

impl TheWorld {

    #[test]
    fn test_which_grid() {
        let loc = (-6., 6.);
        let i_rn = (-10.0, 10.0);
        let j_rn = (-10.0, 10.0);
        let i_bins = 10 as usize;
        let j_bins = 10 as usize;

        let result = TheWorld::which_grid(loc, i_rn, j_rn, i_bins, j_bins).unwrap();
        assert_eq!((2, 8), result);

    }
    fn which_grid(val: (f64, f64), i_range: (f64, f64), j_range: (f64, f64), i_bins: usize, j_bins: usize) -> Result<(usize, usize), String> {
        let i_size = (i_range.1 - i_range.0) / i_bins as f64;
        let i_number_of_sizes = ((val.0 - i_range.0) / i_size) as usize;
        let j_size = (j_range.1 - j_range.0) / j_bins as f64;
        let j_number_of_sizes = ((val.1 - j_range.0) / j_size) as usize;

        assert!(i_number_of_sizes < i_bins);
        assert!(j_number_of_sizes < j_bins);

        Ok((i_number_of_sizes, j_number_of_sizes))
    }

    fn init(&mut self, def_bucket: Vec<GridPoint>) {
        println!("init");
        let props: DStreamProps = DStreamProps { ..Default::default() };

        for i in 0..props.i_bins {
            for j in 0..props.j_bins {
                let z_clone = def_bucket.clone();
                let some_default_dg = DG {i: i, j: j,
                    updates_and_vals: z_clone, removed_as_spore_adic: Vec::new(),};
                (self.g_vec).push(((i as usize, j as usize), some_default_dg));
            }
        }
    }
    fn do_time_steps() {}
    fn do_one_time_step(t: u32, data: Vec<RawData>) {}

    fn get_by_idx(&mut self, idx: (usize, usize))-> DG {
        let  vec_f: Vec<((usize, usize), DG)> = self.g_vec.clone().into_iter().filter(|i| i.0 == idx).collect();
        vec_f[0].1.clone()
    }
    fn update_by_idx(&mut self, idx: (usize, usize), dg: DG) -> Result<(), String> {
        self.g_vec.retain(|i| i.0 != idx);
        self.g_vec.push((idx, dg));
        Ok(())
    }
    fn put(&mut self, t: u32, dat: Vec<RawData>) -> Result<(), String> {

        fn validate_range(loc2d: (f64, f64)) -> bool {

            let props: DStreamProps = DStreamProps { ..Default::default() };

            if ((loc2d.0 <= props.i_range.1) &&
            (loc2d.0 >= props.i_range.0) &&
            (loc2d.1 <= props.j_range.1) &&
            (loc2d.1 >= props.j_range.0)) {
                println!("valid!");
            } else {
                println!("invalid! -- bad input range");
                return false
            }
            return true
        }
        let with_idxs: Vec<GridData> = dat
            .iter()
            .filter(|rd| validate_range((rd.x, rd.y)))
            .map(|rd| self.which_idxs(rd).unwrap())
            .collect();

        let props: DStreamProps = DStreamProps { ..Default::default() };

        for (key, group) in with_idxs.iter().group_by(|gd| (gd.i, gd.j)) {
            println!("--put: key, g size: ({},{}) : {}", key.0, key.1, group.len());

            let the_vec_of_vals: Vec<f64> = group.iter().map(|t| t.v).collect();

            let mut some_default_dg = DG {i: key.0, j: key.1, updates_and_vals: Vec::<GridPoint>::new(), removed_as_spore_adic: Vec::new(),};

            let teh_dg: &mut DG = &mut self.get_by_idx(key);
            let update_dg_result = teh_dg.update(t, the_vec_of_vals);
            let udpate_world_result = self.update_by_idx(key, teh_dg.clone());
        }
        Ok(())
    }
    fn which_idxs(&self, dat: &RawData) -> Result<GridData, String> {

        let props: DStreamProps = DStreamProps { ..Default::default() };

        let idxs = TheWorld::which_grid((dat.x, dat.y), props.i_range, props.j_range,props.i_bins, props.j_bins).unwrap();

        Ok((GridData{i:idxs.0,j:idxs.1,v:dat.v}))

    }
}

#[test]
fn test_dg_update_and_get() {
    let mut dg = DG {i: 0, j:0, updates_and_vals: Vec::new(), removed_as_spore_adic: Vec::new(),};
    dg.update(1, vec!(100.0));

    dg.update(10, vec!(200.0));
    dg.update(20, vec!(300.0));

    assert_eq!((1, 100.0), dg.get_last_update_and_value_to(2));
    assert_eq!((10, 200.0), dg.get_last_update_and_value_to(14));
    assert_eq!((20, 300.0), dg.get_last_update_and_value_to(22));

    let v_at_t = dg.get_at_time(30);
    println!("v: {}", v_at_t);
}

#[test]
fn test_removed_as_sporadic() {
    let mut dg = DG {i: 0, j:0, updates_and_vals: Vec::new(), removed_as_spore_adic: vec!(1, 2, 3, 4),};
    assert_eq!(1, dg.get_last_time_removed_as_sporadic_to(2));
    assert_eq!(4, dg.get_last_time_removed_as_sporadic_to(6));
}

#[derive(Debug)]
#[derive(Clone)]
//#[derive(Copy)] TODO: how to make this work?
struct DG {
    i: usize,
    j: usize,
    updates_and_vals: Vec<GridPoint>,
    removed_as_spore_adic: Vec<u32>,
}
#[derive(Debug)]
#[derive(Clone)]
enum GridLabel { Dense, Sparse, Transitional, }
#[derive(Debug)]
#[derive(Clone)]
enum GridStatus { Sporadic, Normal, }
impl DG {

    fn get_grid_label_at_time(&self, t:u32) -> GridLabel {
        let props: DStreamProps = DStreamProps { ..Default::default() };
        let n_size = (self.i * self.j) as f64;
        let d_m = props.c_m / (n_size * (1.0 - props.lambda));
        let d_l = props.c_l / (n_size * (1.0 - props.lambda));

        let d_t = self.get_at_time(t);

        if d_t >= d_m {
            GridLabel::Dense
        } else if d_t <= d_l {
            GridLabel::Sparse
        } else {
            assert!(d_t > d_l && d_t < d_t);
            GridLabel::Transitional
        }
    }

    fn get_at_time(&self, t: u32) -> f64 {
        let last_update_time_and_value = self.get_last_update_and_value_to(t);
        let coeff = self.coeff(t, last_update_time_and_value.0);
        coeff * last_update_time_and_value.1 + 1.0
    }
    fn update(&mut self, t: u32, vals: Vec<f64>) -> Result<(), String> {
        let sum = vals.clone().iter().fold(0.0, |sum, x| sum + x);
        self.updates_and_vals.push(GridPoint {t: t, v: sum});
        println!("  times updated dg: {}", self.updates_and_vals.len());
        Ok(())
    }

    fn get_last_update_and_value_to(&self, t: u32) -> (u32, f64) {
        let a: GridPoint = self.updates_and_vals.clone().into_iter().filter(|bp| bp.t < t).max_by_key(|bp| bp.t).unwrap();
        let t_l = a.t;
        let v_l = a.v;
        println!("  last update relative to {} is (t: {}, v: {})", t, t_l, v_l);
        (t_l, v_l)
    }

    fn get_last_time_removed_as_sporadic_to(&self, t: u32) -> u32 {
        let t = self.removed_as_spore_adic.clone().into_iter().filter(|&the_t| the_t < t).max().unwrap();
        t
    }

    fn coeff(&self, t_n: u32, t_l: u32) -> f64 {
        let props: DStreamProps = DStreamProps { ..Default::default() };
        props.lambda.powf((t_n - t_l) as f64)
    }

 }

#[test]
fn test_initialize_clustering() {

    let result = initialize_clustering(Vec::new());
    assert_eq!(Ok(()), result);
}

pub fn initialize_clustering(grid_list: Vec<RawData>) -> Result<(), String> {
    //update density of all grids in grid_list
    //assign each dense grid to a distinct cluster
    //label all other grids as NO_CLASS; bad grids!
    /*
    do until no change in cluster labels
          foreach cluster c
            foreach outside grid g of c
                foreach neighboring grid h of g
                    if h belongs to cluster c'
                        if |c| > |c'| label all grids in c' as c
                        else label all grids in c as c'
                    else if (h is translational) label h as in c
    end do
    */
    Ok(())
}

pub fn adjust_clustering(grid_list: Vec<RawData>) -> Result<(), String> {
    //update the density of all grids in grid_list
    /*
    foreach grid g whose attribute (dense/sparse/transitional) is changed since last call to adjust_clustering()
        if g is a sparse grid
            delete g from its cluster c, label g as NO_CLASS
            if (c becomes unconnected) split c into two clusters
        else if g is a dense grid
            among all neighboring grids of g, find out the grid h whoe cluster c_h has the largest size
            if h is a dense grid
                if (g is labelled as NO_CLASS) label g as in c_h
                else if (g is in cluster c and |c| > |c_h|)
                    label all grids in c_h as in c
                else if (g is in cluster c and |c| <= |c_h|)
                    label all grids in c as in c_h
            else if h is a transitional grid
                if ((g is NO_CLASS) and (h is an outside grid if g is added to c_h)) label g as in c_h
                else if (g is in cluster c and |c| >= |c_h|)
                    move h from cluster c_h to c
        else if (g is transitional grid)
            among neighboring clusters of g, find the largest one c'
            satisfying that g is an outside grid if added to it and
            label g as in c'
    */

    Ok(())
}<|endoftext|>