<commit_before>#[macro_use] extern crate log;
extern crate env_logger;
extern crate clap;
extern crate futures;
extern crate rdkafka;
extern crate byteorder;

use clap::{App, Arg};
use futures::stream::Stream;
use rdkafka::consumer::{Consumer, CommitMode};
use rdkafka::consumer::stream_consumer::StreamConsumer;
use rdkafka::config::{ClientConfig, TopicConfig};
use rdkafka::util::get_rdkafka_version;
use std::str;

use std::io::{self, Cursor, Seek, SeekFrom};
use byteorder::{BigEndian, ReadBytesExt};

#[derive(Debug)]
enum ParserError {
    Format,
    Io(io::Error),
    Utf8(str::Utf8Error),
}

impl From<io::Error> for ParserError {
    fn from(err: io::Error) -> ParserError {
        ParserError::Io(err)
    }
}

impl From<str::Utf8Error> for ParserError {
    fn from(err: str::Utf8Error) -> ParserError {
        ParserError::Utf8(err)
    }
}

fn read_string<'a>(rdr: &'a mut Cursor<&[u8]>) -> Result<&'a str, ParserError> {
    let strlen = try!(rdr.read_i16::<BigEndian>()) as usize;
    let pos = rdr.position() as usize;
    let slice = try!(str::from_utf8(&rdr.get_ref()[pos..(pos+strlen)]));
    rdr.seek(SeekFrom::Current(strlen as i64));
    Ok(slice)
}

fn parse_group_offset(key_rdr: &mut Cursor<&[u8]>, payload: &[u8]) -> Result<(), ParserError> {
    let group = try!(read_string(key_rdr)).to_owned();
    let topic = try!(read_string(key_rdr)).to_owned();
    let partition = try!(key_rdr.read_i32::<BigEndian>());
    println!("group offset {:?} {:?} {}", group, topic, partition);
    Ok(())
}

fn parse_group_metadata(key_rdr: &mut Cursor<&[u8]>, payload: &[u8]) -> Result<(), ParserError> {
    let group = read_string(key_rdr);
    println!("group metadata {:?}", group);
    Ok(())
}

fn parse_message(key: &[u8], payload: &[u8]) -> Result<(), ParserError> {
    let mut key_rdr = Cursor::new(key);
    let key_version = try!(key_rdr.read_i16::<BigEndian>());
    println!(">> {:?}", key_version);
    match key_version {
        0 | 1 => Ok(try!(parse_group_offset(&mut key_rdr, payload))),
        2 => Ok(try!(parse_group_metadata(&mut key_rdr, payload))),
        _ => Err(ParserError::Format),
    }
}

fn consume_and_print(brokers: &str) {
    let mut consumer = ClientConfig::new()
        .set("group.id", "consumer_reader_group")
        .set("bootstrap.servers", brokers)
        .set("enable.partition.eof", "false")
        .set("session.timeout.ms", "6000")
        .set("enable.auto.commit", "false")
        .set_default_topic_config(TopicConfig::new()
            .set("auto.offset.reset", "smallest")
            .finalize())
        .create::<StreamConsumer<_>>()
        .expect("Consumer creation failed");

    consumer.subscribe(&vec!["__consumer_offsets"])
        .expect("Can't subscribe to specified topics");

    for message in consumer.start().wait() {
        match message {
            Err(e) => {
                warn!("Can't receive data from stream: {:?}", e);
            },
            Ok(Ok(m)) => {
                let key = match m.key_view::<[u8]>() {
                    None => &[],
                    Some(Ok(s)) => s,
                    Some(Err(e)) => {
                        println!("Error while deserializing message key: {:?}", e);
                        &[]
                    },
                };
                let payload = match m.payload_view::<[u8]>() {
                    None => &[],
                    Some(Ok(s)) => s,
                    Some(Err(e)) => {
                        println!("Error while deserializing message payload: {:?}", e);
                        &[]
                    },
                };
                println!("\n#### P:{}, o:{}, s:{:.3}KB", m.partition(), m.offset(),
                         (m.payload_len() as f64 / 1000f64));

                parse_message(key, payload);

            },
            Ok(Err(e)) => {
                warn!("Kafka error: {:?}", e);
            },
        };
    }
}

fn main() {
    let matches = App::new("consumer example")
        .version(option_env!("CARGO_PKG_VERSION").unwrap_or(""))
        .about("Simple command line consumer")
        .arg(Arg::with_name("brokers")
             .short("b")
             .long("brokers")
             .help("Broker list in kafka format")
             .takes_value(true)
             .default_value("localhost:9092"))
        .get_matches();

    let (version_n, version_s) = get_rdkafka_version();
    println!("rd_kafka_version: 0x{:08x}, {}", version_n, version_s);

    let brokers = matches.value_of("brokers").unwrap();

    consume_and_print(brokers);
}

<commit_msg>Improve commit offset topic message parsing<commit_after>#[macro_use] extern crate log;
extern crate env_logger;
extern crate clap;
extern crate futures;
extern crate rdkafka;
extern crate byteorder;

use clap::{App, Arg};
use futures::stream::Stream;
use rdkafka::consumer::{Consumer, CommitMode};
use rdkafka::consumer::stream_consumer::StreamConsumer;
use rdkafka::config::{ClientConfig, TopicConfig};
use rdkafka::util::get_rdkafka_version;
use std::str;

use std::io::{self, Cursor, BufRead};
use byteorder::{BigEndian, ReadBytesExt};

#[derive(Debug)]
enum ParserError {
    Format,
    Io(io::Error),
    Utf8(str::Utf8Error),
}

impl From<io::Error> for ParserError {
    fn from(err: io::Error) -> ParserError {
        ParserError::Io(err)
    }
}

impl From<str::Utf8Error> for ParserError {
    fn from(err: str::Utf8Error) -> ParserError {
        ParserError::Utf8(err)
    }
}

#[derive(Debug)]
enum ConsumerUpdate {
    Metadata,
    SetCommit { group: String, topic: String, partition: i32, offset: i64 },
    DeleteCommit { group: String, topic: String, partition: i32 },
}

fn read_str<'a>(rdr: &'a mut Cursor<&[u8]>) -> Result<&'a str, ParserError> {
    let strlen = try!(rdr.read_i16::<BigEndian>()) as usize;
    let pos = rdr.position() as usize;
    let slice = try!(str::from_utf8(&rdr.get_ref()[pos..(pos+strlen)]));
    rdr.consume(strlen);
    Ok(slice)
}

fn parse_group_offset(key_rdr: &mut Cursor<&[u8]>,
                      payload_rdr: &mut Cursor<&[u8]>) -> Result<ConsumerUpdate, ParserError> {
    let group = try!(read_str(key_rdr)).to_owned();
    let topic = try!(read_str(key_rdr)).to_owned();
    let partition = try!(key_rdr.read_i32::<BigEndian>());
    if payload_rdr.get_ref().len() != 0 {
        try!(payload_rdr.read_i16::<BigEndian>());
        let offset = try!(payload_rdr.read_i64::<BigEndian>());
        Ok(ConsumerUpdate::SetCommit { group: group, topic: topic, partition: partition, offset: offset })
    } else {
        Ok(ConsumerUpdate::DeleteCommit { group: group, topic: topic, partition: partition })
    }
}

fn parse_message(key: &[u8], payload: &[u8]) -> Result<ConsumerUpdate, ParserError> {
    let mut key_rdr = Cursor::new(key);
    let mut payload_rdr = Cursor::new(payload);
    let key_version = try!(key_rdr.read_i16::<BigEndian>());
    match key_version {
        0 | 1 => Ok(try!(parse_group_offset(&mut key_rdr, &mut payload_rdr))),
        2 => Ok(ConsumerUpdate::Metadata),
        _ => Err(ParserError::Format),
    }
}

fn consume_and_print(brokers: &str) {
    let mut consumer = ClientConfig::new()
        .set("group.id", "consumer_reader_group")
        .set("bootstrap.servers", brokers)
        .set("enable.partition.eof", "false")
        .set("session.timeout.ms", "6000")
        .set("enable.auto.commit", "false")
        .set_default_topic_config(TopicConfig::new()
            .set("auto.offset.reset", "smallest")
            .finalize())
        .create::<StreamConsumer<_>>()
        .expect("Consumer creation failed");

    consumer.subscribe(&vec!["__consumer_offsets"])
        .expect("Can't subscribe to specified topics");

    for message in consumer.start().wait() {
        match message {
            Err(e) => {
                warn!("Can't receive data from stream: {:?}", e);
            },
            Ok(Ok(m)) => {
                let key = match m.key_view::<[u8]>() {
                    None => &[],
                    Some(Ok(s)) => s,
                    Some(Err(e)) => {
                        println!("Error while deserializing message key: {:?}", e);
                        &[]
                    },
                };
                let payload = match m.payload_view::<[u8]>() {
                    None => &[],
                    Some(Ok(s)) => s,
                    Some(Err(e)) => {
                        println!("Error while deserializing message payload: {:?}", e);
                        &[]
                    },
                };
                println!("\n#### P:{}, o:{}, s:{:.3}KB", m.partition(), m.offset(),
                         (m.payload_len() as f64 / 1000f64));

                let msg = parse_message(key, payload);
                println!("{:?}", msg);
            },
            Ok(Err(e)) => {
                warn!("Kafka error: {:?}", e);
            },
        };
    }
}

fn main() {
    let matches = App::new("consumer example")
        .version(option_env!("CARGO_PKG_VERSION").unwrap_or(""))
        .about("Simple command line consumer")
        .arg(Arg::with_name("brokers")
             .short("b")
             .long("brokers")
             .help("Broker list in kafka format")
             .takes_value(true)
             .default_value("localhost:9092"))
        .get_matches();

    let (version_n, version_s) = get_rdkafka_version();
    println!("rd_kafka_version: 0x{:08x}, {}", version_n, version_s);

    let brokers = matches.value_of("brokers").unwrap();

    consume_and_print(brokers);
}

<|endoftext|>


<commit_before>extern crate timely;
extern crate differential_dataflow;
extern crate rand;

use differential_dataflow::input::InputSession;
use differential_dataflow::operators::{Join,CountTotal,Count};

use rand::{Rng, SeedableRng, StdRng};

fn main() {

    timely::execute_from_args(std::env::args(), move |worker| {

        let mut interactions_input = InputSession::new();

        let probe = worker.dataflow(|scope| {

            let interactions = interactions_input.to_collection(scope);

            // Find all users with more than 5 and less than 500 interactions
            let users_with_enough_interactions = interactions
                .map(|(user, _item)| user)
                .count_total()
                .filter(move |(_user, count): &(u32, isize)| *count > 5 && *count < 500)
                .map(|(user, _count)| user);

            // Remove users with too few or too many interactions
            let remaining_interactions = interactions
                .semijoin(&users_with_enough_interactions);

            let num_interactions_per_item = remaining_interactions
                .map(|(_user, item)| item)
                .count_total();

            // Compute the number of cooccurrences of each item pair
            let cooccurrences = remaining_interactions
                .join_map(&remaining_interactions, |_user, &item_a, &item_b| (item_a, item_b))
                .filter(|&(item_a, item_b)| item_a > item_b)
                .count();

            // Compute the jaccard similarity between item pairs (= number of users that interacted
            // with both items / number of users that interacted with at least one of the items)
            let jaccard_similarities = cooccurrences
                .map(|((item_a, item_b), num_cooc)| (item_a, (item_b, num_cooc)))
                .join_map(&num_interactions_per_item, |&item_a, &(item_b, num_cooc), &occ_a| {
                    (item_b, (item_a, num_cooc, occ_a))
                })
                .join_map(
                    &num_interactions_per_item,
                    |&item_b, &(item_a, num_cooc, occ_a), &occ_b| {
                        ((item_a, item_b), (num_cooc, occ_a, occ_b))
                    },
                )
                // We have to do this calculation in a map due to the lack of a total order for f64
                .map(|((item_a, item_b), (num_cooc, occ_a, occ_b))| {
                    let jaccard = num_cooc as f64 / (occ_a + occ_b - num_cooc) as f64;
                    ((item_a, item_b), jaccard)
                });

            // We threshold the similarity matrix
            let thresholded_similarities = jaccard_similarities
                .filter(|(_item_pair, jaccard)| *jaccard > 0.05);

            thresholded_similarities.probe()
        });

        let num_interactions: usize = std::env::args().nth(1)
            .or_else(|| Some(String::from("10000"))).unwrap().parse().unwrap();

        if worker.index() == 0 {
            println!("Generating {} synthetic interactions...", num_interactions);
        }

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng: StdRng = SeedableRng::from_seed(seed);

        let synthetic_interactions = generate_interactions(num_interactions, &mut rng);

        let num_users = synthetic_interactions.iter().map(|(user, _item)| user).max().unwrap() + 1;
        let users_to_remove = rand::seq::sample_iter(&mut rng, 0..num_users, 20).unwrap();

        let interactions_to_remove: Vec<(u32,u32)> = synthetic_interactions.iter()
            .filter(|(user, _item)| users_to_remove.contains(&user))
            .map(|(user, item)| (*user, *item))
            .collect();

        let timer = worker.timer();

        for (user, item) in synthetic_interactions.iter() {
            if *user as usize % worker.peers() == worker.index() {
                interactions_input.insert((*user, *item));
            }
        }

        interactions_input.advance_to(1);
        interactions_input.flush();

        worker.step_while(|| probe.less_than(interactions_input.time()));

        let initial_model_time = timer.elapsed();
        println!("Model trained in {:?} on worker {:?}", initial_model_time, worker.index());

        if worker.index() == 0 {
            println!("Removing {} interactions...", interactions_to_remove.len());
        }

        for (user, item) in interactions_to_remove.iter() {
            if *user as usize % worker.peers() == worker.index() {
                interactions_input.remove((*user, *item));
            }
        }

        interactions_input.advance_to(2);
        interactions_input.flush();

        worker.step_while(|| probe.less_than(interactions_input.time()));

        let removal_time = timer.elapsed();
        println!(
            "Model updated after {:?} on worker {:?}",
            (removal_time - initial_model_time),
            worker.index(),
        );

    }).unwrap();
}

struct CRP { alpha: f64, discount: f64, weight: f64, weights: Vec<f64> }

impl CRP {
    fn new(alpha: f64, discount: f64) -> Self {
        CRP { alpha, discount, weight: 0.0, weights: Vec::new() }
    }

    fn sample<R>(&mut self, rng: &mut R) -> u32 where R: Rng {
        let mut u = rng.gen::<f64>() * (self.alpha + self.weight);
        for j in 0 .. self.weights.len() {
            if u < self.weights[j] - self.discount {
                self.weights[j] += 1.0;
                self.weight += 1.0;
                return j as u32;
            } else {
                u = u - self.weights[j] - self.discount;
            }
        }

        self.weights.push(1.0);
        self.weight = self.weight + 1.0;

        (self.weights.len() - 1) as u32
    }
}

// Generate synthetic interactions with a skewed distribution
fn generate_interactions<R>(how_many: usize, rng: &mut R) -> Vec<(u32,u32)> where R: Rng {
    let mut interactions = Vec::with_capacity(how_many);

    let mut user_sampler = CRP::new(6000.0, 0.35);
    let mut item_sampler = CRP::new(6000.0, 0.35);

    for _ in 0 .. how_many {
        let user = user_sampler.sample(rng);
        let item = item_sampler.sample(rng);
        interactions.push((user, item));
    }

    interactions
}<commit_msg>first draft done<commit_after>extern crate timely;
extern crate differential_dataflow;
extern crate rand;

use differential_dataflow::input::InputSession;
use differential_dataflow::operators::{Join,CountTotal,Count};

use rand::{Rng, SeedableRng, StdRng};

fn main() {

    timely::execute_from_args(std::env::args(), move |worker| {

        let mut interactions_input = InputSession::new();

        let probe = worker.dataflow(|scope| {

            let interactions = interactions_input.to_collection(scope);

            // Find all users with more than 5 and less than 500 interactions
            let users_with_enough_interactions = interactions
                .map(|(user, _item)| user)
                .count_total()
                .filter(move |(_user, count): &(u32, isize)| *count > 5 && *count < 500)
                .map(|(user, _count)| user);

            // Remove users with too few or too many interactions
            let remaining_interactions = interactions
                .semijoin(&users_with_enough_interactions);

            let num_interactions_per_item = remaining_interactions
                .map(|(_user, item)| item)
                .count_total();

            // Compute the number of cooccurrences of each item pair
            let cooccurrences = remaining_interactions
                .join_map(&remaining_interactions, |_user, &item_a, &item_b| (item_a, item_b))
                .filter(|&(item_a, item_b)| item_a > item_b)
                .count();

            // Compute the jaccard similarity between item pairs (= number of users that interacted
            // with both items / number of users that interacted with at least one of the items)
            let jaccard_similarities = cooccurrences
                .map(|((item_a, item_b), num_cooc)| (item_a, (item_b, num_cooc)))
                .join_map(&num_interactions_per_item, |&item_a, &(item_b, num_cooc), &occ_a| {
                    (item_b, (item_a, num_cooc, occ_a))
                })
                .join_map(
                    &num_interactions_per_item,
                    |&item_b, &(item_a, num_cooc, occ_a), &occ_b| {
                        ((item_a, item_b), (num_cooc, occ_a, occ_b))
                    },
                )
                // We have to do this calculation in a map due to the lack of a total order for f64
                .map(|((item_a, item_b), (num_cooc, occ_a, occ_b))| {
                    let jaccard = num_cooc as f64 / (occ_a + occ_b - num_cooc) as f64;
                    ((item_a, item_b), jaccard)
                });

            // We threshold the similarity matrix
            let thresholded_similarities = jaccard_similarities
                .filter(|(_item_pair, jaccard)| *jaccard > 0.05);

            thresholded_similarities.probe()
        });

        let num_interactions: usize = std::env::args().nth(1)
            .or_else(|| Some(String::from("10000"))).unwrap().parse().unwrap();

        if worker.index() == 0 {
            println!("Generating {} synthetic interactions...", num_interactions);
        }

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng: StdRng = SeedableRng::from_seed(seed);

        let synthetic_interactions = generate_interactions(num_interactions, &mut rng);

        let num_users = synthetic_interactions.iter().map(|(user, _item)| user).max().unwrap() + 1;
        let users_to_remove = rand::seq::sample_iter(&mut rng, 0..num_users, 20).unwrap();

        let interactions_to_remove: Vec<(u32,u32)> = synthetic_interactions.iter()
            .filter(|(user, _item)| users_to_remove.contains(&user))
            .map(|(user, item)| (*user, *item))
            .collect();

        let timer = worker.timer();

        for (user, item) in synthetic_interactions.iter() {
            if *user as usize % worker.peers() == worker.index() {
                interactions_input.insert((*user, *item));
            }
        }

        interactions_input.advance_to(1);
        interactions_input.flush();

        worker.step_while(|| probe.less_than(interactions_input.time()));

        let initial_model_time = timer.elapsed();
        println!("Model trained in {:?} on worker {:?}", initial_model_time, worker.index());

        if worker.index() == 0 {
            println!("Removing {} interactions...", interactions_to_remove.len());
        }

        for (user, item) in interactions_to_remove.iter() {
            if *user as usize % worker.peers() == worker.index() {
                interactions_input.remove((*user, *item));
            }
        }

        interactions_input.advance_to(2);
        interactions_input.flush();

        worker.step_while(|| probe.less_than(interactions_input.time()));

        let removal_time = timer.elapsed();
        println!(
            "Model updated after {:?} on worker {:?}",
            (removal_time - initial_model_time),
            worker.index(),
        );

    }).unwrap();
}

struct CRP { alpha: f64, discount: f64, weight: f64, weights: Vec<f64> }

impl CRP {
    fn new(alpha: f64, discount: f64) -> Self {
        CRP { alpha, discount, weight: 0.0, weights: Vec::new() }
    }

    fn sample<R>(&mut self, rng: &mut R) -> u32 where R: Rng {
        let mut u = rng.gen::<f64>() * (self.alpha + self.weight);
        for j in 0 .. self.weights.len() {
            if u < self.weights[j] - self.discount {
                self.weights[j] += 1.0;
                self.weight += 1.0;
                return j as u32;
            } else {
                u = u - self.weights[j] - self.discount;
            }
        }

        self.weights.push(1.0);
        self.weight = self.weight + 1.0;

        (self.weights.len() - 1) as u32
    }
}

// Generate synthetic interactions with a skewed distribution
fn generate_interactions<R>(how_many: usize, rng: &mut R) -> Vec<(u32,u32)> where R: Rng {
    let mut interactions = Vec::with_capacity(how_many);

    let mut user_sampler = CRP::new(6000.0, 0.35);
    let mut item_sampler = CRP::new(6000.0, 0.35);

    for _ in 0 .. how_many {
        let user = user_sampler.sample(rng);
        let item = item_sampler.sample(rng);
        interactions.push((user, item));
    }

    interactions
}
<|endoftext|>


<commit_before>// Copyright 2019 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use rand::Rng;

use super::{NodeHandler, NodeRole, RequestData};
use crate::crypto::PublicKey;
use crate::events::error::LogError;
use crate::events::network::ConnectedPeerAddr;
use crate::helpers::Height;
use crate::messages::{Connect, Message, PeersRequest, Responses, Service, Status, Verified};

impl NodeHandler {
    /// Redirects message to the corresponding `handle_...` function.
    pub fn handle_message(&mut self, msg: Message) {
        match msg {
            Message::Consensus(msg) => self.handle_consensus(msg),
            Message::Requests(ref msg) => self.handle_request(msg),

            Message::Service(Service::Connect(msg)) => self.handle_connect(msg),
            Message::Service(Service::Status(msg)) => self.handle_status(&msg),
            // ignore tx duplication error,
            Message::Service(Service::AnyTx(msg)) => drop(self.handle_tx(msg)),
            Message::Responses(Responses::BlockResponse(msg)) => {
                self.handle_block(&msg).log_error()
            }
            Message::Responses(Responses::TransactionsResponse(msg)) => {
                self.handle_txs_batch(&msg).log_error()
            }
        }
    }

    /// Handles the `Connected` event. Node's `Connect` message is sent as response
    /// if received `Connect` message is correct.
    pub fn handle_connected(&mut self, address: &ConnectedPeerAddr, connect: Verified<Connect>) {
        info!("Received Connect message from peer: {:?}", address);
        // TODO: use `ConnectInfo` instead of connect-messages. (ECR-1452)
        self.state.add_connection(connect.author(), address.clone());
        self.handle_connect(connect);
    }

    /// Handles the `Disconnected` event. Node will try to connect to that address again if it was
    /// in the validators list.
    pub fn handle_disconnected(&mut self, key: PublicKey) {
        info!("Disconnected from: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Handles the `UnableConnectToPeer` event. Node will try to connect to that address again
    /// if it was in the validators list.
    pub fn handle_unable_to_connect(&mut self, key: PublicKey) {
        info!("Could not connect to: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Removes peer from the state and from the cache. Node will try to connect to that address
    /// again if it was in the validators list.
    fn remove_peer_with_addr(&mut self, key: PublicKey) {
        self.state.remove_peer_with_pubkey(&key);
        self.blockchain.remove_peer_with_pubkey(&key);
        let is_validator = self.state.peer_is_validator(&key);
        let in_connect_list = self.state.peer_in_connect_list(&key);
        if is_validator && in_connect_list {
            self.connect(key);
        }
    }

    /// Handles the `Connect` message and connects to a peer as result.
    pub fn handle_connect(&mut self, message: Verified<Connect>) {
        // TODO Add spam protection (ECR-170)
        // TODO: drop connection if checks have failed. (ECR-1837)
        let address = message.payload().host.clone();
        if address == self.state.our_connect_message().payload().host {
            trace!("Received Connect with same address as our external_address.");
            return;
        }

        let public_key = message.author();
        if public_key == self.state.our_connect_message().author() {
            trace!("Received Connect with same pub_key as ours.");
            return;
        }

        if !self.state.connect_list().is_peer_allowed(&public_key) {
            error!(
                "Received connect message from {:?} peer which not in ConnectList.",
                public_key
            );
            return;
        }

        // Check if we have another connect message from peer with the given public_key.
        let mut need_connect = true;
        if let Some(saved_message) = self.state.peers().get(&public_key) {
            if saved_message.payload().time() > message.payload().time() {
                error!("Received outdated Connect message from {}", address);
                return;
            } else if saved_message.payload().time() < message.payload().time() {
                need_connect = saved_message.payload().host != message.payload().host;
            } else if saved_message.payload().host == message.payload().host {
                need_connect = false;
            } else {
                error!("Received weird Connect message from {}", address);
                return;
            }
            if saved_message.payload().host != message.payload().host {
                info!(
                    "Updating connect list for peer: {} with new addr: {}",
                    public_key,
                    message.payload().host
                );
                self.state
                    .connect_list()
                    .update_peer(&public_key, message.payload().host.to_string())
            }
        }
        self.state.add_peer(public_key, message.clone());
        info!(
            "Received Connect message from {}. Need to connect: {}",
            address, need_connect,
        );
        self.blockchain.save_peer(&public_key, message);
        if need_connect {
            // TODO: reduce double sending of connect message
            info!("Send Connect message to {}", address);
            //TODO: remove responding connect [ECR-2385]
            self.connect(public_key);
        }
    }

    /// Handles the `Status` message. Node sends `BlockRequest` as response if height in the
    /// message is higher than node's height.
    pub fn handle_status(&mut self, msg: &Verified<Status>) {
        let height = self.state.height();
        trace!(
            "HANDLE STATUS: current height = {}, msg height = {}",
            height,
            msg.payload().height()
        );

        if !self.state.connect_list().is_peer_allowed(&msg.author()) {
            error!(
                "Received status message from peer = {:?} which not in ConnectList.",
                msg.author()
            );
            return;
        }

        let peer = msg.author();

        // Handle message from future height
        if msg.payload().height() > height {
            let peer = msg.author();

            // Check validator height info
            if msg.payload().height() > self.state.node_height(&peer) {
                // Update validator height
                self.state.set_node_height(peer, msg.payload().height());
            }

            // Request block
            self.request(RequestData::Block(height), peer);
        }

        if self.uncommitted_txs_count() == 0 && msg.payload().pool_size > 0 {
            self.request(RequestData::PoolTransactions, peer);
        }
    }

    /// Handles the `PeersRequest` message. Node sends `Connect` messages of other peers as result.
    pub fn handle_request_peers(&mut self, msg: &Verified<PeersRequest>) {
        let peers = self.state.peers().values().cloned().collect::<Vec<_>>();

        trace!(
            "HANDLE REQUEST PEERS: Sending {:?} peers to {:?}",
            peers,
            msg.author()
        );

        for peer in peers {
            self.send_to_peer(msg.author(), peer);
        }
    }

    /// Handles `NodeTimeout::Status`, broadcasts the `Status` message if it isn't outdated as
    /// result.
    pub fn handle_status_timeout(&mut self, height: Height) {
        if self.state.height() == height {
            self.broadcast_status();
            self.add_status_timeout();
        }
    }
    /// Handles `NodeTimeout::PeerExchange`. Node sends the `PeersRequest` to a random peer.
    pub fn handle_peer_exchange_timeout(&mut self) {
        if !self.state.peers().is_empty() {
            let to = self.state.peers().len();
            let gen_peer_id = || -> usize {
                let mut rng = rand::thread_rng();
                rng.gen_range(0, to)
            };

            let peer = self
                .state
                .peers()
                .iter()
                .nth(gen_peer_id())
                .map(|x| x.1.clone())
                .unwrap();
            let msg = PeersRequest::new(peer.author());
            trace!(
                "Request peers from peer with addr {:?}",
                peer.payload().host
            );
            let message = self.sign_message(msg);
            self.send_to_peer(peer.author(), message);
        }
        self.add_peer_exchange_timeout();
    }
    /// Handles `NodeTimeout::UpdateApiState`.
    /// Node update internal `ApiState` and `NodeRole`.
    pub fn handle_update_api_state_timeout(&mut self) {
        self.api_state.update_node_state(&self.state);
        self.node_role = NodeRole::new(self.state.validator_id());
        self.add_update_api_state_timeout();
    }

    /// Broadcasts the `Status` message to all peers.
    pub fn broadcast_status(&mut self) {
        let status = Status {
            height: self.state.height(),
            last_hash: self.blockchain.last_hash(),
            pool_size: self.uncommitted_txs_count(),
        };
        trace!("Broadcast status: {:?}", status);

        let message = self.sign_message(status);
        self.broadcast(message);
    }
}
<commit_msg>Update dispatcher info to show changes in list of deployed artifacts (#1431)<commit_after>// Copyright 2019 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use rand::Rng;

use super::{NodeHandler, NodeRole, RequestData};
use crate::crypto::PublicKey;
use crate::events::error::LogError;
use crate::events::network::ConnectedPeerAddr;
use crate::helpers::Height;
use crate::messages::{Connect, Message, PeersRequest, Responses, Service, Status, Verified};

impl NodeHandler {
    /// Redirects message to the corresponding `handle_...` function.
    pub fn handle_message(&mut self, msg: Message) {
        match msg {
            Message::Consensus(msg) => self.handle_consensus(msg),
            Message::Requests(ref msg) => self.handle_request(msg),

            Message::Service(Service::Connect(msg)) => self.handle_connect(msg),
            Message::Service(Service::Status(msg)) => self.handle_status(&msg),
            // ignore tx duplication error,
            Message::Service(Service::AnyTx(msg)) => drop(self.handle_tx(msg)),
            Message::Responses(Responses::BlockResponse(msg)) => {
                self.handle_block(&msg).log_error()
            }
            Message::Responses(Responses::TransactionsResponse(msg)) => {
                self.handle_txs_batch(&msg).log_error()
            }
        }
    }

    /// Handles the `Connected` event. Node's `Connect` message is sent as response
    /// if received `Connect` message is correct.
    pub fn handle_connected(&mut self, address: &ConnectedPeerAddr, connect: Verified<Connect>) {
        info!("Received Connect message from peer: {:?}", address);
        // TODO: use `ConnectInfo` instead of connect-messages. (ECR-1452)
        self.state.add_connection(connect.author(), address.clone());
        self.handle_connect(connect);
    }

    /// Handles the `Disconnected` event. Node will try to connect to that address again if it was
    /// in the validators list.
    pub fn handle_disconnected(&mut self, key: PublicKey) {
        info!("Disconnected from: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Handles the `UnableConnectToPeer` event. Node will try to connect to that address again
    /// if it was in the validators list.
    pub fn handle_unable_to_connect(&mut self, key: PublicKey) {
        info!("Could not connect to: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Removes peer from the state and from the cache. Node will try to connect to that address
    /// again if it was in the validators list.
    fn remove_peer_with_addr(&mut self, key: PublicKey) {
        self.state.remove_peer_with_pubkey(&key);
        self.blockchain.remove_peer_with_pubkey(&key);
        let is_validator = self.state.peer_is_validator(&key);
        let in_connect_list = self.state.peer_in_connect_list(&key);
        if is_validator && in_connect_list {
            self.connect(key);
        }
    }

    /// Handles the `Connect` message and connects to a peer as result.
    pub fn handle_connect(&mut self, message: Verified<Connect>) {
        // TODO Add spam protection (ECR-170)
        // TODO: drop connection if checks have failed. (ECR-1837)
        let address = message.payload().host.clone();
        if address == self.state.our_connect_message().payload().host {
            trace!("Received Connect with same address as our external_address.");
            return;
        }

        let public_key = message.author();
        if public_key == self.state.our_connect_message().author() {
            trace!("Received Connect with same pub_key as ours.");
            return;
        }

        if !self.state.connect_list().is_peer_allowed(&public_key) {
            error!(
                "Received connect message from {:?} peer which not in ConnectList.",
                public_key
            );
            return;
        }

        // Check if we have another connect message from peer with the given public_key.
        let mut need_connect = true;
        if let Some(saved_message) = self.state.peers().get(&public_key) {
            if saved_message.payload().time() > message.payload().time() {
                error!("Received outdated Connect message from {}", address);
                return;
            } else if saved_message.payload().time() < message.payload().time() {
                need_connect = saved_message.payload().host != message.payload().host;
            } else if saved_message.payload().host == message.payload().host {
                need_connect = false;
            } else {
                error!("Received weird Connect message from {}", address);
                return;
            }
            if saved_message.payload().host != message.payload().host {
                info!(
                    "Updating connect list for peer: {} with new addr: {}",
                    public_key,
                    message.payload().host
                );
                self.state
                    .connect_list()
                    .update_peer(&public_key, message.payload().host.to_string())
            }
        }
        self.state.add_peer(public_key, message.clone());
        info!(
            "Received Connect message from {}. Need to connect: {}",
            address, need_connect,
        );
        self.blockchain.save_peer(&public_key, message);
        if need_connect {
            // TODO: reduce double sending of connect message
            info!("Send Connect message to {}", address);
            //TODO: remove responding connect [ECR-2385]
            self.connect(public_key);
        }
    }

    /// Handles the `Status` message. Node sends `BlockRequest` as response if height in the
    /// message is higher than node's height.
    pub fn handle_status(&mut self, msg: &Verified<Status>) {
        let height = self.state.height();
        trace!(
            "HANDLE STATUS: current height = {}, msg height = {}",
            height,
            msg.payload().height()
        );

        if !self.state.connect_list().is_peer_allowed(&msg.author()) {
            error!(
                "Received status message from peer = {:?} which not in ConnectList.",
                msg.author()
            );
            return;
        }

        let peer = msg.author();

        // Handle message from future height
        if msg.payload().height() > height {
            let peer = msg.author();

            // Check validator height info
            if msg.payload().height() > self.state.node_height(&peer) {
                // Update validator height
                self.state.set_node_height(peer, msg.payload().height());
            }

            // Request block
            self.request(RequestData::Block(height), peer);
        }

        if self.uncommitted_txs_count() == 0 && msg.payload().pool_size > 0 {
            self.request(RequestData::PoolTransactions, peer);
        }
    }

    /// Handles the `PeersRequest` message. Node sends `Connect` messages of other peers as result.
    pub fn handle_request_peers(&mut self, msg: &Verified<PeersRequest>) {
        let peers = self.state.peers().values().cloned().collect::<Vec<_>>();

        trace!(
            "HANDLE REQUEST PEERS: Sending {:?} peers to {:?}",
            peers,
            msg.author()
        );

        for peer in peers {
            self.send_to_peer(msg.author(), peer);
        }
    }

    /// Handles `NodeTimeout::Status`, broadcasts the `Status` message if it isn't outdated as
    /// result.
    pub fn handle_status_timeout(&mut self, height: Height) {
        if self.state.height() == height {
            self.broadcast_status();
            self.add_status_timeout();
        }
    }
    /// Handles `NodeTimeout::PeerExchange`. Node sends the `PeersRequest` to a random peer.
    pub fn handle_peer_exchange_timeout(&mut self) {
        if !self.state.peers().is_empty() {
            let to = self.state.peers().len();
            let gen_peer_id = || -> usize {
                let mut rng = rand::thread_rng();
                rng.gen_range(0, to)
            };

            let peer = self
                .state
                .peers()
                .iter()
                .nth(gen_peer_id())
                .map(|x| x.1.clone())
                .unwrap();
            let msg = PeersRequest::new(peer.author());
            trace!(
                "Request peers from peer with addr {:?}",
                peer.payload().host
            );
            let message = self.sign_message(msg);
            self.send_to_peer(peer.author(), message);
        }
        self.add_peer_exchange_timeout();
    }
    /// Handles `NodeTimeout::UpdateApiState`.
    /// Node update internal `ApiState` and `NodeRole`.
    pub fn handle_update_api_state_timeout(&mut self) {
        self.api_state.update_node_state(&self.state);
        // FIXME Add special event to update state [ECR-3222]
        self.api_state.update_dispatcher_state(&self.blockchain);
        self.node_role = NodeRole::new(self.state.validator_id());
        self.add_update_api_state_timeout();
    }

    /// Broadcasts the `Status` message to all peers.
    pub fn broadcast_status(&mut self) {
        let status = Status {
            height: self.state.height(),
            last_hash: self.blockchain.last_hash(),
            pool_size: self.uncommitted_txs_count(),
        };
        trace!("Broadcast status: {:?}", status);

        let message = self.sign_message(status);
        self.broadcast(message);
    }
}
<|endoftext|>


<commit_before>use std::fs;
use std::str::FromStr;

struct Instruction {
    command: String,
    offset: u32,
}

impl FromStr for Instruction {
    type Err = std::num::ParseIntError;
    fn from_str (line: &str) -> Result<Self, Self::Err> {
        let x: Vec<&str> = line.split_whitespace().collect();
        let command: String = x[0].to_string();
        let offset: u32 = u32::from_str(x[1]).unwrap();
        Ok(Instruction { command, offset })
    }
}

fn main() {
    let input = fs::read_to_string("input-1")
        .expect("fail input");

    let mut hpos = 0;
    let mut dpos = 0;
    for line in input.lines() {
        let x = Instruction::from_str(line).unwrap();
        match x.command.as_str() {
            "forward" => hpos += x.offset,
            "up" => dpos -= x.offset,
            "down" => dpos += x.offset,
            _ => unreachable!(),
        }
    }
    println!("{}", hpos * dpos);
}
<commit_msg>expect(), insead of unwrap()<commit_after>use std::fs;
use std::str::FromStr;

struct Instruction {
    command: String,
    offset: u32,
}

impl FromStr for Instruction {
    type Err = std::num::ParseIntError;
    fn from_str (line: &str) -> Result<Self, Self::Err> {
        let x: Vec<&str> = line.split_whitespace().collect();
        let command: String = x[0].to_string();
        let offset: u32 = u32::from_str(x[1]).expect("Err parsing offset");
        Ok(Instruction { command, offset })
    }
}

fn main() {
    let input = fs::read_to_string("input-1")
        .expect("fail input");

    let mut hpos = 0;
    let mut dpos = 0;
    for line in input.lines() {
        let x = Instruction::from_str(line).expect("Error parsing instruction");
        match x.command.as_str() {
            "forward" => hpos += x.offset,
            "up" => dpos -= x.offset,
            "down" => dpos += x.offset,
            _ => unreachable!(),
        }
    }
    println!("{}", hpos * dpos);
}
<|endoftext|>


<commit_before>use sdl2::Sdl;
use sdl2::event::Event;
use sdl2::keyboard::Keycode;

use game::*;

pub struct SdlInputSource {
    sdl: Sdl,
}

impl SdlInputSource {
    pub fn new(sdl: Sdl) -> Self {
        SdlInputSource {
            sdl: sdl,
        }
    }
}

impl InputSource for SdlInputSource {
    fn next_input(&self) -> Option<InputEvent> {
        let mut event_pump = self.sdl.event_pump().expect("Failed to initialise event pump");

        loop {
            let event = event_pump.wait_event();

            match event {
                Event::Quit { .. } => return Some(InputEvent::Quit),
                Event::KeyDown { keycode: Some(keycode), .. } => {
                    return match keycode {
                        Keycode::Up => Some(InputEvent::Up),
                        Keycode::Down => Some(InputEvent::Down),
                        Keycode::Left => Some(InputEvent::Left),
                        Keycode::Right => Some(InputEvent::Right),
                        Keycode::A => Some(InputEvent::Char('a')),
                        Keycode::B => Some(InputEvent::Char('b')),
                        Keycode::C => Some(InputEvent::Char('c')),
                        Keycode::D => Some(InputEvent::Char('d')),
                        Keycode::E => Some(InputEvent::Char('e')),
                        Keycode::F => Some(InputEvent::Char('f')),
                        Keycode::G => Some(InputEvent::Char('g')),
                        Keycode::H => Some(InputEvent::Char('h')),
                        Keycode::I => Some(InputEvent::Char('i')),
                        Keycode::J => Some(InputEvent::Char('j')),
                        Keycode::K => Some(InputEvent::Char('k')),
                        Keycode::L => Some(InputEvent::Char('l')),
                        Keycode::M => Some(InputEvent::Char('m')),
                        Keycode::N => Some(InputEvent::Char('n')),
                        Keycode::O => Some(InputEvent::Char('o')),
                        Keycode::P => Some(InputEvent::Char('p')),
                        Keycode::Q => Some(InputEvent::Char('q')),
                        Keycode::R => Some(InputEvent::Char('r')),
                        Keycode::S => Some(InputEvent::Char('s')),
                        Keycode::T => Some(InputEvent::Char('t')),
                        Keycode::U => Some(InputEvent::Char('u')),
                        Keycode::V => Some(InputEvent::Char('v')),
                        Keycode::W => Some(InputEvent::Char('w')),
                        Keycode::X => Some(InputEvent::Char('x')),
                        Keycode::Y => Some(InputEvent::Char('y')),
                        Keycode::Z => Some(InputEvent::Char('z')),
                        Keycode::Period => Some(InputEvent::Char('.')),
                        Keycode::Escape => Some(InputEvent::Escape),
                        _ => None,
                    };
                },
                _ => (),
            }
        }
    }
}
<commit_msg>Added space to sdl input<commit_after>use sdl2::Sdl;
use sdl2::event::Event;
use sdl2::keyboard::Keycode;

use game::*;

pub struct SdlInputSource {
    sdl: Sdl,
}

impl SdlInputSource {
    pub fn new(sdl: Sdl) -> Self {
        SdlInputSource {
            sdl: sdl,
        }
    }
}

impl InputSource for SdlInputSource {
    fn next_input(&self) -> Option<InputEvent> {
        let mut event_pump = self.sdl.event_pump().expect("Failed to initialise event pump");

        loop {
            let event = event_pump.wait_event();

            match event {
                Event::Quit { .. } => return Some(InputEvent::Quit),
                Event::KeyDown { keycode: Some(keycode), .. } => {
                    return match keycode {
                        Keycode::Up => Some(InputEvent::Up),
                        Keycode::Down => Some(InputEvent::Down),
                        Keycode::Left => Some(InputEvent::Left),
                        Keycode::Right => Some(InputEvent::Right),
                        Keycode::A => Some(InputEvent::Char('a')),
                        Keycode::B => Some(InputEvent::Char('b')),
                        Keycode::C => Some(InputEvent::Char('c')),
                        Keycode::D => Some(InputEvent::Char('d')),
                        Keycode::E => Some(InputEvent::Char('e')),
                        Keycode::F => Some(InputEvent::Char('f')),
                        Keycode::G => Some(InputEvent::Char('g')),
                        Keycode::H => Some(InputEvent::Char('h')),
                        Keycode::I => Some(InputEvent::Char('i')),
                        Keycode::J => Some(InputEvent::Char('j')),
                        Keycode::K => Some(InputEvent::Char('k')),
                        Keycode::L => Some(InputEvent::Char('l')),
                        Keycode::M => Some(InputEvent::Char('m')),
                        Keycode::N => Some(InputEvent::Char('n')),
                        Keycode::O => Some(InputEvent::Char('o')),
                        Keycode::P => Some(InputEvent::Char('p')),
                        Keycode::Q => Some(InputEvent::Char('q')),
                        Keycode::R => Some(InputEvent::Char('r')),
                        Keycode::S => Some(InputEvent::Char('s')),
                        Keycode::T => Some(InputEvent::Char('t')),
                        Keycode::U => Some(InputEvent::Char('u')),
                        Keycode::V => Some(InputEvent::Char('v')),
                        Keycode::W => Some(InputEvent::Char('w')),
                        Keycode::X => Some(InputEvent::Char('x')),
                        Keycode::Y => Some(InputEvent::Char('y')),
                        Keycode::Z => Some(InputEvent::Char('z')),
                        Keycode::Space => Some(InputEvent::Char(' ')),
                        Keycode::Period => Some(InputEvent::Char('.')),
                        Keycode::Escape => Some(InputEvent::Escape),
                        _ => None,
                    };
                },
                _ => (),
            }
        }
    }
}
<|endoftext|>


<commit_before>//! Implements persistence for `BooleanIndex`.
//! e.g. writing the index to a bytestream; reading the index from a bytestream.
//! The API-Entrypoints are defined in the trait `index::PersistentIndex`

use index::{PersistentIndex, ByteEncodable, ByteDecodable};
use index::boolean_index::BooleanIndex;
use index::boolean_index::posting::Posting;

use std::mem::transmute;
use std::io::{Read, Write};
use std::collections::BTreeMap;
use std;

const CHUNKSIZE: usize = 1_000_000;

impl ByteEncodable for String {
    fn encode(&self) -> Vec<u8> {
        let mut result = Vec::with_capacity(self.len());
        result.extend_from_slice(self.as_bytes());
        result
    }
}

impl ByteDecodable for String {
    fn decode(bytes: Vec<u8>) -> Result<Self, String> {
        String::from_utf8(bytes).map_err(|e| format!("{:?}", e))
    }
}

impl ByteEncodable for usize {
    fn encode(&self) -> Vec<u8> {
        let mut result = Vec::with_capacity(8);
        result.extend_from_slice(unsafe { &transmute::<_, [u8; 8]>(*self as u64) });
        result
    }
}

impl ByteDecodable for usize {
    fn decode(bytes: Vec<u8>) -> Result<Self, String> {
        Ok(read_u64(bytes.as_slice()) as usize)
    }
}

impl<TTerm: Ord + ByteDecodable + ByteEncodable> BooleanIndex<TTerm> {
    /// Writes all the terms with postings of the index to specified target
    /// Layout:
    /// [u8; 4] -> Number of bytes term + postings need encoded
    /// [u8] -> term + postings
    fn write_terms<TTarget: Write>(&self, target: &mut TTarget) -> std::io::Result<usize> {
        // Write blocks of 1MB to target
        let mut bytes = Vec::with_capacity(2 * CHUNKSIZE);
        for term in &self.index {
            let term_bytes = encode_term(&term);
            let term_bytes_len: [u8; 4] =
                unsafe { transmute::<_, [u8; 4]>(term_bytes.len() as u32) };

            bytes.extend_from_slice(&term_bytes_len);
            bytes.extend_from_slice(term_bytes.as_slice());
            if bytes.len() > CHUNKSIZE {
                if let Err(e) = target.write(bytes.as_slice()) {
                    return Err(e);
                } else {
                    bytes.clear();
                }
            }
        }
        target.write(bytes.as_slice())
    }

    fn read_terms<TSource: Read>(source: &mut TSource)
                                 -> Result<BTreeMap<TTerm, Vec<Posting>>, String> {
        let mut bytes = Vec::new();
        if let Err(e) = source.read_to_end(&mut bytes) {
            return Err(format!("{:?}", e));
        }

        let mut ptr = 0;
        let mut result = BTreeMap::new();
        while ptr < bytes.len() {
            let entry_size = read_u32(&bytes[ptr..ptr + 4]) as usize;
            ptr += 4;
            match decode_term(&bytes[ptr..ptr + entry_size]) {
                Ok(term_posting) => {
                    result.insert(term_posting.0, term_posting.1);
                    ptr += entry_size;
                }
                Err(e) => return Err(e),
            }
        }
        Ok(result)
    }
}

fn decode_term<TTerm: ByteDecodable>(f: &[u8]) -> Result<(TTerm, Vec<Posting>), String> {
    let term_len: u8 = f[0] + 1;
    let term_bytes_vec = Vec::from(&f[1..(term_len) as usize]);
    match TTerm::decode(term_bytes_vec) {
        Ok(term) => {
            let mut postings = Vec::new();
            let mut decoder = VByteDecoder::new((&f[term_len as usize..]).iter().map(|x| *x));
            while let Some(doc_id) = decoder.next() {
                let positions_len = decoder.next().unwrap();
                let mut positions = Vec::with_capacity(positions_len as usize);
                let mut last_position = 0;
                for _ in 0..positions_len {
                    last_position += decoder.next().unwrap();
                    positions.push(last_position as u32);
                }
                postings.push((doc_id as u64, positions));

            }
            Ok((term, postings))
        }
        Err(e) => Err(e),
    }

}

// Represents a term-entry in the inverted index as a vector of bytes
// Layout:
// [u8; 1] length of term in bytes
// [u8] term
// For every posting:
// [u8] stream of vbyte encoded numbers: doc_id, #positions, [positions]
fn encode_term<TTerm: ByteEncodable>(term: &(&TTerm, &Vec<Posting>)) -> Vec<u8> {
    let mut bytes: Vec<u8> = Vec::new();
    let term_bytes = term.0.encode();
    let term_len: u8 = term_bytes.len() as u8;
    bytes.push(term_len);
    bytes.extend_from_slice(term_bytes.as_slice());
    for posting in term.1.iter() {
        bytes.append(&mut vbyte_encode(posting.0 as usize));
        bytes.append(&mut vbyte_encode(posting.1.len() as usize));
        let mut last_position = 0;
        for position in posting.1.iter() {
            bytes.append(&mut vbyte_encode((*position - last_position) as usize));
            last_position = *position;
        }
    }
    bytes
}

impl<TTerm: ByteDecodable + ByteEncodable + Ord> PersistentIndex for BooleanIndex<TTerm> {
    fn write_to<TTarget: Write>(&self, target: &mut TTarget) -> std::io::Result<usize> {
        self.write_terms(target)
    }

    fn read_from<TSource: Read>(source: &mut TSource) -> Result<Self, String> {
        Self::read_terms(source).map(|btree| {
            BooleanIndex {
                document_count: 0,
                index: btree,
            }
        })
    }
}

fn read_u32(barry: &[u8]) -> u32 {
    let mut array = [0u8; 4];
    for (&x, p) in barry.iter().zip(array.iter_mut()) {
        *p = x;
    }
    unsafe { transmute::<_, u32>(array) }
}

fn read_u64(barry: &[u8]) -> u64 {
    let mut array = [0u8; 8];
    for (&x, p) in barry.iter().zip(array.iter_mut()) {
        *p = x;
    }
    unsafe { transmute::<_, u64>(array) }
}

fn vbyte_encode(mut number: usize) -> Vec<u8> {
    let mut result = Vec::new();
    loop {
        result.insert(0, (number % 128) as u8);
        if number < 128 {
            break;
        } else {
            number /= 128;
        }
    }
    let len = result.len();
    result[len - 1] += 128;
    result
}

macro_rules! unwrap_or_return_none{
    ($operand:expr) => {
        if let Some(x) = $operand {
            x
        } else {
            return None;
        }
    }
}

struct VByteDecoder<T: Iterator<Item = u8>> {
    bytes: T,
}

impl<T: Iterator<Item = u8>> VByteDecoder<T> {
    fn new(bytes: T) -> Self {
        VByteDecoder { bytes: bytes }
    }
}

impl<T: Iterator<Item = u8>> Iterator for VByteDecoder<T> {
    type Item = usize;
    fn next(&mut self) -> Option<Self::Item> {

        let mut result: usize = 0;
        loop {
            result *= 128;
            let val = unwrap_or_return_none!(self.bytes.next());
            result += val as usize;
            if val >= 128 {
                result -= 128;
                break;
            }
        }
        Some(result)
    }
}

#[test]
fn test_vbyte_encode() {
    assert_eq!(vbyte_encode(0), vec![0x80]);
    assert_eq!(vbyte_encode(5), vec![0x85]);
    assert_eq!(vbyte_encode(127), vec![0xFF]);
    assert_eq!(vbyte_encode(128), vec![0x01, 0x80]);
    assert_eq!(vbyte_encode(130), vec![0x01, 0x82]);
    assert_eq!(vbyte_encode(255), vec![0x01, 0xFF]);
    assert_eq!(vbyte_encode(20_000), vec![0x01, 0x1C, 0xA0]);
    assert_eq!(vbyte_encode(0xFFFF), vec![0x03, 0x7F, 0xFF]);
}

#[test]
fn test_vbyte_decode() {
    assert_eq!(VByteDecoder::new(vec![0x80].into_iter()).collect::<Vec<_>>(),
               vec![0]);
    assert_eq!(VByteDecoder::new(vec![0x85].into_iter()).collect::<Vec<_>>(),
               vec![5]);
    assert_eq!(VByteDecoder::new(vec![0xFF].into_iter()).collect::<Vec<_>>(),
               vec![127]);
    assert_eq!(VByteDecoder::new(vec![0x80, 0x81].into_iter()).collect::<Vec<_>>(),
               vec![0, 1]);
    assert_eq!(VByteDecoder::new(vec![0x03, 0x7F, 0xFF, 0x01, 0x82, 0x85].into_iter())
                   .collect::<Vec<_>>(),
               vec![0xFFFF, 130, 5]);
    assert_eq!(VByteDecoder::new(vec![0x80].into_iter()).collect::<Vec<_>>(),
               vec![0]);
}

#[cfg(test)]
mod tests {
    use index::boolean_index::BooleanIndex;
    use index::boolean_index::tests::prepare_index;
    use index::{Index, PersistentIndex};
    use std::io::Cursor;

    #[test]
    fn basic() {
        let index = prepare_index();
        let mut bytes: Vec<u8> = vec![];
        index.write_to(&mut bytes).unwrap();
        let mut buff = Cursor::new(bytes.clone());
        let mut bytes_2: Vec<u8> = vec![];
        BooleanIndex::<usize>::read_from(&mut buff).unwrap().write_to(&mut bytes_2).unwrap();
        assert_eq!(bytes, bytes_2);
    }

    #[test]
    fn length() {
        let index = prepare_index();
        let mut bytes: Vec<u8> = vec![];
        index.write_to(&mut bytes).unwrap();
        let mut buff = Cursor::new(bytes.clone());
        let mut bytes_2: Vec<u8> = vec![];
        let mut read_index = BooleanIndex::<usize>::read_from(&mut buff).unwrap();
        read_index.index_document(1..24);
        read_index.write_to(&mut bytes_2).unwrap();
        assert!(bytes.len() < bytes_2.len());
    }
}
<commit_msg>Further Improve Compression by VByte Encoding Metadata<commit_after>//! Implements persistence for `BooleanIndex`.
//! e.g. writing the index to a bytestream; reading the index from a bytestream.
//! The API-Entrypoints are defined in the trait `index::PersistentIndex`

use index::{PersistentIndex, ByteEncodable, ByteDecodable};
use index::boolean_index::BooleanIndex;
use index::boolean_index::posting::Posting;

use std::io::{Read, Write};
use std::collections::BTreeMap;
use std;

const CHUNKSIZE: usize = 1_000_000;

impl ByteEncodable for String {
    fn encode(&self) -> Vec<u8> {
        let mut result = Vec::with_capacity(self.len());
        result.extend_from_slice(self.as_bytes());
        result
    }
}

impl ByteDecodable for String {
    fn decode(bytes: Vec<u8>) -> Result<Self, String> {
        String::from_utf8(bytes).map_err(|e| format!("{:?}", e))
    }
}

impl ByteEncodable for usize {
    fn encode(&self) -> Vec<u8> {
        vbyte_encode(*self)
    }
}

impl ByteDecodable for usize {
    fn decode(bytes: Vec<u8>) -> Result<Self, String> {
        let mut decoder = VByteDecoder::new(bytes.into_iter());
        if let Some(res) = decoder.next() {
            Ok(res)
        } else {
            Err("Tried to decode bytevector with variable byte code. Failed".to_string())
        }
    }
}

impl<TTerm: Ord + ByteDecodable + ByteEncodable> BooleanIndex<TTerm> {
    /// Writes all the terms with postings of the index to specified target
    /// Layout:
    /// [u8; 4] -> Number of bytes term + postings need encoded
    /// [u8] -> term + postings
    fn write_terms<TTarget: Write>(&self, target: &mut TTarget) -> std::io::Result<usize> {
        // Write blocks of 1MB to target
        let mut bytes = Vec::with_capacity(2 * CHUNKSIZE);
        for term in &self.index {
            let term_bytes = encode_term(&term);
            bytes.extend_from_slice(term_bytes.as_slice());
            if bytes.len() > CHUNKSIZE {
                if let Err(e) = target.write(bytes.as_slice()) {
                    return Err(e);
                } else {
                    bytes.clear();
                }
            }
        }
        target.write(bytes.as_slice())
    }

    fn read_terms<TSource: Read>(source: &mut TSource)
                                 -> Result<BTreeMap<TTerm, Vec<Posting>>, String> {
        let mut bytes = Vec::new();
        if let Err(e) = source.read_to_end(&mut bytes) {
            return Err(format!("{:?}", e));
        }
        let mut decoder = VByteDecoder::new(bytes.into_iter());
        let mut result = BTreeMap::new();
        loop {
            match decode_term(&mut decoder) {
                Ok(Some(term_posting)) => {
                    result.insert(term_posting.0, term_posting.1);
                }
                Ok(None) => break,
                Err(e) => return Err(e),
            }
        }
        Ok(result)
    }
}

fn decode_term<TTerm: ByteDecodable, T: Iterator<Item = u8>>
    (decoder: &mut VByteDecoder<T>)
     -> Result<Option<(TTerm, Vec<Posting>)>, String> {
    if let Some(term_len) = decoder.next() {
        let term_bytes_vec =
            decoder.underlying_iterator().take(term_len as usize).collect::<Vec<_>>();
        match TTerm::decode(term_bytes_vec) {
            Ok(term) => {
                let postings_len = decoder.next().unwrap();
                let mut postings = Vec::with_capacity(postings_len);
                for _ in 0..postings_len {
                    let doc_id = decoder.next().unwrap();
                    let positions_len = decoder.next().unwrap();
                    let mut positions = Vec::with_capacity(positions_len as usize);
                    let mut last_position = 0;
                    for _ in 0..positions_len {
                        last_position += decoder.next().unwrap();
                        positions.push(last_position as u32);
                    }
                    postings.push((doc_id as u64, positions));

                }
                Ok(Some((term, postings)))
            }
            Err(e) => Err(e),
        }
    } else {
        Ok(None)
    }

}

// Represents a term-entry in the inverted index as a vector of bytes
// Layout:
// [u8; 1] length of term in bytes
// [u8] term
// For every posting:
// [u8] stream of vbyte encoded numbers: doc_id, #positions, [positions]
fn encode_term<TTerm: ByteEncodable>(term: &(&TTerm, &Vec<Posting>)) -> Vec<u8> {
    let mut bytes: Vec<u8> = Vec::new();
    let term_bytes = term.0.encode();
    let mut term_len = vbyte_encode(term_bytes.len());
    bytes.append(&mut term_len);
    bytes.extend_from_slice(term_bytes.as_slice());
    bytes.append(&mut vbyte_encode(term.1.len()));
    for posting in term.1.iter() {
        bytes.append(&mut vbyte_encode(posting.0 as usize));
        bytes.append(&mut vbyte_encode(posting.1.len() as usize));
        let mut last_position = 0;
        for position in posting.1.iter() {
            bytes.append(&mut vbyte_encode((*position - last_position) as usize));
            last_position = *position;
        }
    }
    bytes
}

impl<TTerm: ByteDecodable + ByteEncodable + Ord> PersistentIndex for BooleanIndex<TTerm> {
    fn write_to<TTarget: Write>(&self, target: &mut TTarget) -> std::io::Result<usize> {
        self.write_terms(target)
    }

    fn read_from<TSource: Read>(source: &mut TSource) -> Result<Self, String> {
        Self::read_terms(source).map(|btree| {
            BooleanIndex {
                document_count: 0,
                index: btree,
            }
        })
    }
}

fn vbyte_encode(mut number: usize) -> Vec<u8> {
    let mut result = Vec::new();
    loop {
        result.insert(0, (number % 128) as u8);
        if number < 128 {
            break;
        } else {
            number /= 128;
        }
    }
    let len = result.len();
    result[len - 1] += 128;
    result
}

macro_rules! unwrap_or_return_none{
    ($operand:expr) => {
        if let Some(x) = $operand {
            x
        } else {
            return None;
        }
    }
}

struct VByteDecoder<T: Iterator<Item = u8>> {
    bytes: T,
}

impl<T: Iterator<Item = u8>> VByteDecoder<T> {
    fn new(bytes: T) -> Self {
        VByteDecoder { bytes: bytes }
    }

    fn underlying_iterator(&mut self) -> &mut T {
        &mut self.bytes
    }
}

impl<T: Iterator<Item = u8>> Iterator for VByteDecoder<T> {
    type Item = usize;
    fn next(&mut self) -> Option<Self::Item> {

        let mut result: usize = 0;
        loop {
            result *= 128;
            let val = unwrap_or_return_none!(self.bytes.next());
            result += val as usize;
            if val >= 128 {
                result -= 128;
                break;
            }
        }
        Some(result)
    }
}

#[test]
fn test_vbyte_encode() {
    assert_eq!(vbyte_encode(0), vec![0x80]);
    assert_eq!(vbyte_encode(5), vec![0x85]);
    assert_eq!(vbyte_encode(127), vec![0xFF]);
    assert_eq!(vbyte_encode(128), vec![0x01, 0x80]);
    assert_eq!(vbyte_encode(130), vec![0x01, 0x82]);
    assert_eq!(vbyte_encode(255), vec![0x01, 0xFF]);
    assert_eq!(vbyte_encode(20_000), vec![0x01, 0x1C, 0xA0]);
    assert_eq!(vbyte_encode(0xFFFF), vec![0x03, 0x7F, 0xFF]);
}

#[test]
fn test_vbyte_decode() {
    assert_eq!(VByteDecoder::new(vec![0x80].into_iter()).collect::<Vec<_>>(),
               vec![0]);
    assert_eq!(VByteDecoder::new(vec![0x85].into_iter()).collect::<Vec<_>>(),
               vec![5]);
    assert_eq!(VByteDecoder::new(vec![0xFF].into_iter()).collect::<Vec<_>>(),
               vec![127]);
    assert_eq!(VByteDecoder::new(vec![0x80, 0x81].into_iter()).collect::<Vec<_>>(),
               vec![0, 1]);
    assert_eq!(VByteDecoder::new(vec![0x03, 0x7F, 0xFF, 0x01, 0x82, 0x85].into_iter())
                   .collect::<Vec<_>>(),
               vec![0xFFFF, 130, 5]);
    assert_eq!(VByteDecoder::new(vec![0x80].into_iter()).collect::<Vec<_>>(),
               vec![0]);
}

#[cfg(test)]
mod tests {
    use index::boolean_index::BooleanIndex;
    use index::boolean_index::tests::prepare_index;
    use index::{Index, PersistentIndex};
    use std::io::Cursor;

    #[test]
    fn simple() {
        let mut index = BooleanIndex::new();
        index.index_document(0..2);
        let mut bytes: Vec<u8> = vec![];
        index.write_to(&mut bytes).unwrap();
        assert_eq!(bytes,
                   vec![129 /* #TermBytes */, 128 /* Term: 0 */, 129 /* #docs */,
                        128 /* doc_id */, 129 /* #positions */, 128 /* position: 0 */,
                        129 /* #TermBytes */, 129 /* Term: 1 */, 129 /* #docs */,
                        128 /* doc_id */, 129 /* #positions */, 129 /* position */]);
    }

    #[test]
    fn basic() {
        let index = prepare_index();
        let mut bytes: Vec<u8> = vec![];
        index.write_to(&mut bytes).unwrap();
        let mut buff = Cursor::new(bytes.clone());
        let mut bytes_2: Vec<u8> = vec![];
        BooleanIndex::<usize>::read_from(&mut buff).unwrap().write_to(&mut bytes_2).unwrap();
        assert_eq!(bytes, bytes_2);
    }

    #[test]
    fn length() {
        let index = prepare_index();
        let mut bytes: Vec<u8> = vec![];
        index.write_to(&mut bytes).unwrap();
        let mut buff = Cursor::new(bytes.clone());
        let mut bytes_2: Vec<u8> = vec![];
        let mut read_index = BooleanIndex::<usize>::read_from(&mut buff).unwrap();
        read_index.index_document(1..24);
        read_index.write_to(&mut bytes_2).unwrap();
        assert!(bytes.len() < bytes_2.len());
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use kvproto::raftpb::{Entry, Snapshot};

// unstable.entris[i] has raft log position i+unstable.offset.
// Note that unstable.offset may be less than the highest log
// position in storage; this means that the next write to storage
// might need to truncate the log before persisting unstable.entries.
#[derive(Debug, PartialEq, Default)]
pub struct Unstable {
    // the incoming unstable snapshot, if any.
    pub snapshot: Option<Snapshot>,
    // all entries that have not yet been written to storage.
    pub entries: Vec<Entry>,
    pub offset: u64,
}

impl Unstable {
    pub fn new(offset: u64) -> Unstable {
        Unstable {
            offset: offset,
            snapshot: None,
            entries: vec![],
        }
    }
    // maybe_first_index returns the index of the first possible entry in entries
    // if it has a snapshot.
    pub fn maybe_first_index(&self) -> Option<u64> {
        self.snapshot
            .as_ref()
            .map_or(None, |snap| Some(snap.get_metadata().get_index() + 1))
    }

    pub fn get_snapshot(&self) -> Snapshot {
        self.snapshot.as_ref().unwrap().clone()
    }

    // maybe_last_index returns the last index if it has at least one
    // unstable entry or snapshot.
    pub fn maybe_last_index(&self) -> Option<u64> {
        match self.entries.len() {
            0 => {
                self.snapshot
                    .as_ref()
                    .map_or(None, |snap| Some(snap.get_metadata().get_index()))
            }
            len => Some(self.offset + len as u64 - 1),
        }
    }

    // maybe_term returns the term of the entry at index idx, if there
    // is any.
    pub fn maybe_term(&self, idx: u64) -> Option<u64> {
        if idx < self.offset {
            if self.snapshot.is_none() {
                return None;
            }

            let meta = self.snapshot.as_ref().unwrap().get_metadata();
            if idx == meta.get_index() {
                return Some(meta.get_term());
            }
            return None;
        }
        match self.maybe_last_index() {
            None => None,
            Some(last) => {
                if idx > last {
                    return None;
                }
                Some(self.entries[(idx - self.offset) as usize].get_term())
            }
        }
    }

    pub fn stable_to(&mut self, idx: u64, term: u64) {
        let t = self.maybe_term(idx);
        if t.is_none() {
            return;
        }

        if t.unwrap() == term && idx >= self.offset {
            let start = idx + 1 - self.offset;
            self.entries.drain(..start as usize);
            self.offset = idx + 1;
        }
    }

    pub fn stable_snap_to(&mut self, idx: u64) {
        if self.snapshot.is_none() {
            return;
        }
        if idx == self.snapshot.as_ref().unwrap().get_metadata().get_index() {
            self.snapshot = None;
        }
    }

    pub fn restore(&mut self, snap: Snapshot) {
        self.entries.clear();
        self.offset = snap.get_metadata().get_index() + 1;
        self.snapshot = Some(snap);
    }

    // append entries to unstable, truncate local block first if overlapped.
    pub fn truncate_and_append(&mut self, ents: &[Entry]) {
        let after = ents[0].get_index();
        if after == self.offset + self.entries.len() as u64 {
            // after is the next index in the self.entries, append directly
            self.entries.extend_from_slice(ents);
        } else if after <= self.offset {
            // The log is being truncated to before our current offset
            // portion, so set the offset and replace the entries
            self.offset = after;
            self.entries.clear();
            self.entries.extend_from_slice(ents);
        } else {
            // truncate to after and copy to self.entries then append
            let off = self.offset;
            self.entries = {
                let cut_ents = self.slice(off, after);
                let mut entries = Vec::with_capacity(cut_ents.len() + ents.len());
                entries.extend_from_slice(cut_ents);
                entries.extend_from_slice(ents);
                entries
            };
        }
    }

    pub fn slice(&self, lo: u64, hi: u64) -> &[Entry] {
        self.must_check_outofbounds(lo, hi);
        let l = lo as usize;
        let h = hi as usize;
        let off = self.offset as usize;
        &self.entries[l - off..h - off]
    }

    pub fn must_check_outofbounds(&self, lo: u64, hi: u64) {
        if lo > hi {
            panic!("invalid unstable.slice {} > {}", lo, hi)
        }
        let upper = self.offset + self.entries.len() as u64;
        if lo < self.offset || hi > upper {
            panic!("unstable.slice[{}, {}] out of bound[{}, {}]",
                   lo,
                   hi,
                   self.offset,
                   upper)
        }
    }
}

#[cfg(test)]
mod test {
    use kvproto::raftpb::{Entry, Snapshot};
    use raft::log_unstable::Unstable;

    fn new_entry(index: u64, term: u64) -> Entry {
        let mut e = Entry::new();
        e.set_term(term);
        e.set_index(index);
        e
    }

    fn new_snapshot(index: u64, term: u64) -> Snapshot {
        use kvproto::raftpb::{SnapshotMetadata, Snapshot};
        let mut snap = Snapshot::new();
        let mut meta = SnapshotMetadata::new();
        meta.set_index(index);
        meta.set_term(term);
        snap.set_metadata(meta);
        snap
    }

    #[test]
    fn test_maybe_first_index() {
        // entry, offset, snap, wok, windex,
        let tests = vec![
            // no snapshot
            (Some(new_entry(5,1)), 5, None, false, 0),
            (None, 0, None, false, 0),
            // has snapshot
            (Some(new_entry(5, 1)), 5, Some(new_snapshot(4,1)), true, 5),
            (None, 5, Some(new_snapshot(4,1)), true, 5),
        ];

        for (entries, offset, snapshot, wok, windex) in tests {
            let u = Unstable {
                entries: entries.map_or(vec![], |entry| vec![entry]),
                offset: offset,
                snapshot: snapshot,
            };
            let index = u.maybe_first_index();
            match index {
                None => assert!(!wok),
                Some(index) => assert_eq!(index, windex),
            }
        }
    }

    #[test]
    fn test_maybe_last_index() {
        // entry, offset, snap, wok, windex,
        let tests = vec![
            (Some(new_entry(5,1)), 5, None, true, 5),
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), true, 5),
            // last in snapshot
            (None, 5, Some(new_snapshot(4,1)), true, 4),
            // empty unstable
            (None, 0, None, false, 0),
       ];

        for (entries, offset, snapshot, wok, windex) in tests {
            let u = Unstable {
                entries: entries.map_or(vec![], |entry| vec![entry]),
                offset: offset,
                snapshot: snapshot,
            };
            let index = u.maybe_last_index();
            match index {
                None => assert!(!wok),
                Some(index) => assert_eq!(index, windex),
            }
        }
    }

    #[test]
    fn test_maybe_term() {
        // entry, offset, snap, index, wok, wterm
        let tests = vec![
            // term from entries
            (Some(new_entry(5,1)), 5, None, 5, true, 1),
            (Some(new_entry(5,1)), 5, None, 6, false, 0),
            (Some(new_entry(5,1)), 5, None, 4, false, 0),
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), 5, true, 1),
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), 6, false, 0),
            // term from snapshot
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), 4, true, 1),
            (None, 5, Some(new_snapshot(4,1)), 5, false, 0),
            (None, 5, Some(new_snapshot(4,1)), 4, true, 1),
            (None, 0, None, 5, false, 0),
       ];

        for (entries, offset, snapshot, index, wok, wterm) in tests {
            let u = Unstable {
                entries: entries.map_or(vec![], |entry| vec![entry]),
                offset: offset,
                snapshot: snapshot,
            };
            let term = u.maybe_term(index);
            match term {
                None => assert!(!wok),
                Some(term) => assert_eq!(term, wterm),
            }
        }
    }

    #[test]
    fn test_restore() {
        let mut u = Unstable {
            entries: vec![new_entry(5, 1)],
            offset: 5,
            snapshot: Some(new_snapshot(4, 1)),
        };

        let s = new_snapshot(6, 2);
        u.restore(s.clone());

        assert_eq!(u.offset, s.get_metadata().get_index() + 1);
        assert!(u.entries.is_empty());
        assert_eq!(u.snapshot.unwrap(), s);
    }

    #[test]
    fn test_stable_to() {
        // entries, offset, snap, index, term, woffset, wlen
        let tests = vec![
            (vec![], 0, None, 5, 1, 0, 0),
            // stable to the first entry
            (vec![new_entry(5,1)], 5, None, 5, 1, 6, 0),
            (vec![new_entry(5,1), new_entry(6, 0)], 5, None, 5, 1, 6, 1),
            // stable to the first entry and term mismatch
            (vec![new_entry(6,2)], 5, None, 6, 1, 5, 1),
            // stable to old entry
            (vec![new_entry(5,1)], 5, None, 4, 1, 5, 1),
            (vec![new_entry(5,1)], 5, None, 4, 2, 5, 1),
            // with snapshot
            // stable to the first entry
            (vec![new_entry(5,1)], 5, Some(new_snapshot(4,1)), 5, 1, 6, 0),
            // stable to the first entry
            (vec![new_entry(5,1), new_entry(6,1)], 5, Some(new_snapshot(4,1)), 5, 1, 6, 1),
            // stable to the first entry and term mismatch
            (vec![new_entry(6,2)], 5, Some(new_snapshot(5,1)), 6, 1, 5, 1),
            // stable to snapshot
            (vec![new_entry(5,1)], 5, Some(new_snapshot(4,1)), 4, 1, 5, 1),
            // stable to old entry
            (vec![new_entry(5,2)], 5, Some(new_snapshot(4,2)), 4, 1, 5, 1),
       ];

        for (entries, offset, snapshot, index, term, woffset, wlen) in tests {
            let mut u = Unstable {
                entries: entries,
                offset: offset,
                snapshot: snapshot,
            };
            u.stable_to(index, term);
            assert_eq!(u.offset, woffset);
            assert_eq!(u.entries.len(), wlen);
        }
    }

    #[test]
    fn test_truncate_and_append() {
        // entries, offset, snap, to_append, woffset, wentries
        let tests = vec![
            // replace to the end
            (vec![new_entry(5, 1)], 5, None,
                vec![new_entry(6, 1), new_entry(7, 1)],
                    5, vec![new_entry(5,1), new_entry(6,1), new_entry(7,1)]),

            // replace to unstable entries
           (vec![new_entry(5, 1)], 5, None,
               vec![new_entry(5, 2), new_entry(6, 2)],
                   5, vec![new_entry(5, 2), new_entry(6, 2)]),

            (vec![new_entry(5, 1)], 5, None,
                vec![new_entry(4, 2), new_entry(5, 2), new_entry(6, 2)],
                    4, vec![new_entry(4,2), new_entry(5, 2), new_entry(6, 2)]),

            // truncate existing entries and append
           (vec![new_entry(5, 1), new_entry(6, 1), new_entry(7, 1)], 5, None,
               vec![new_entry(6, 2)],
                   5, vec![new_entry(5, 1), new_entry(6, 2)]),

            (vec![new_entry(5, 1), new_entry(6, 1), new_entry(7, 1)], 5, None,
                vec![new_entry(7, 2), new_entry(8, 2)],
                    5, vec![new_entry(5, 1), new_entry(6, 1), new_entry(7, 2), new_entry(8, 2)]),
      ];

        for (entries, offset, snapshot, to_append, woffset, wentries) in tests {
            let mut u = Unstable {
                entries: entries,
                offset: offset,
                snapshot: snapshot,
            };
            u.truncate_and_append(&to_append);
            assert_eq!(u.offset, woffset);
            assert_eq!(u.entries, wentries);
        }
    }
}
<commit_msg>*: cleanup (#808)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use kvproto::raftpb::{Entry, Snapshot};

// unstable.entris[i] has raft log position i+unstable.offset.
// Note that unstable.offset may be less than the highest log
// position in storage; this means that the next write to storage
// might need to truncate the log before persisting unstable.entries.
#[derive(Debug, PartialEq, Default)]
pub struct Unstable {
    // the incoming unstable snapshot, if any.
    pub snapshot: Option<Snapshot>,
    // all entries that have not yet been written to storage.
    pub entries: Vec<Entry>,
    pub offset: u64,
}

impl Unstable {
    pub fn new(offset: u64) -> Unstable {
        Unstable {
            offset: offset,
            snapshot: None,
            entries: vec![],
        }
    }
    // maybe_first_index returns the index of the first possible entry in entries
    // if it has a snapshot.
    pub fn maybe_first_index(&self) -> Option<u64> {
        self.snapshot
            .as_ref()
            .map_or(None, |snap| Some(snap.get_metadata().get_index() + 1))
    }

    pub fn get_snapshot(&self) -> Snapshot {
        self.snapshot.as_ref().unwrap().clone()
    }

    // maybe_last_index returns the last index if it has at least one
    // unstable entry or snapshot.
    pub fn maybe_last_index(&self) -> Option<u64> {
        match self.entries.len() {
            0 => {
                self.snapshot
                    .as_ref()
                    .map_or(None, |snap| Some(snap.get_metadata().get_index()))
            }
            len => Some(self.offset + len as u64 - 1),
        }
    }

    // maybe_term returns the term of the entry at index idx, if there
    // is any.
    pub fn maybe_term(&self, idx: u64) -> Option<u64> {
        if idx < self.offset {
            if self.snapshot.is_none() {
                return None;
            }

            let meta = self.snapshot.as_ref().unwrap().get_metadata();
            if idx == meta.get_index() {
                return Some(meta.get_term());
            }
            return None;
        }
        match self.maybe_last_index() {
            None => None,
            Some(last) => {
                if idx > last {
                    return None;
                }
                Some(self.entries[(idx - self.offset) as usize].get_term())
            }
        }
    }

    pub fn stable_to(&mut self, idx: u64, term: u64) {
        let t = self.maybe_term(idx);
        if t.is_none() {
            return;
        }

        if t.unwrap() == term && idx >= self.offset {
            let start = idx + 1 - self.offset;
            self.entries.drain(..start as usize);
            self.offset = idx + 1;
        }
    }

    pub fn stable_snap_to(&mut self, idx: u64) {
        if self.snapshot.is_none() {
            return;
        }
        if idx == self.snapshot.as_ref().unwrap().get_metadata().get_index() {
            self.snapshot = None;
        }
    }

    pub fn restore(&mut self, snap: Snapshot) {
        self.entries.clear();
        self.offset = snap.get_metadata().get_index() + 1;
        self.snapshot = Some(snap);
    }

    // append entries to unstable, truncate local block first if overlapped.
    pub fn truncate_and_append(&mut self, ents: &[Entry]) {
        let after = ents[0].get_index();
        if after == self.offset + self.entries.len() as u64 {
            // after is the next index in the self.entries, append directly
            self.entries.extend_from_slice(ents);
        } else if after <= self.offset {
            // The log is being truncated to before our current offset
            // portion, so set the offset and replace the entries
            self.offset = after;
            self.entries.clear();
            self.entries.extend_from_slice(ents);
        } else {
            // truncate to after and copy to self.entries then append
            let off = self.offset;
            self.must_check_outofbounds(off, after);
            self.entries.truncate((after - off) as usize);
            self.entries.extend_from_slice(ents);
        }
    }

    pub fn slice(&self, lo: u64, hi: u64) -> &[Entry] {
        self.must_check_outofbounds(lo, hi);
        let l = lo as usize;
        let h = hi as usize;
        let off = self.offset as usize;
        &self.entries[l - off..h - off]
    }

    pub fn must_check_outofbounds(&self, lo: u64, hi: u64) {
        if lo > hi {
            panic!("invalid unstable.slice {} > {}", lo, hi)
        }
        let upper = self.offset + self.entries.len() as u64;
        if lo < self.offset || hi > upper {
            panic!("unstable.slice[{}, {}] out of bound[{}, {}]",
                   lo,
                   hi,
                   self.offset,
                   upper)
        }
    }
}

#[cfg(test)]
mod test {
    use kvproto::raftpb::{Entry, Snapshot};
    use raft::log_unstable::Unstable;

    fn new_entry(index: u64, term: u64) -> Entry {
        let mut e = Entry::new();
        e.set_term(term);
        e.set_index(index);
        e
    }

    fn new_snapshot(index: u64, term: u64) -> Snapshot {
        use kvproto::raftpb::{SnapshotMetadata, Snapshot};
        let mut snap = Snapshot::new();
        let mut meta = SnapshotMetadata::new();
        meta.set_index(index);
        meta.set_term(term);
        snap.set_metadata(meta);
        snap
    }

    #[test]
    fn test_maybe_first_index() {
        // entry, offset, snap, wok, windex,
        let tests = vec![
            // no snapshot
            (Some(new_entry(5,1)), 5, None, false, 0),
            (None, 0, None, false, 0),
            // has snapshot
            (Some(new_entry(5, 1)), 5, Some(new_snapshot(4,1)), true, 5),
            (None, 5, Some(new_snapshot(4,1)), true, 5),
        ];

        for (entries, offset, snapshot, wok, windex) in tests {
            let u = Unstable {
                entries: entries.map_or(vec![], |entry| vec![entry]),
                offset: offset,
                snapshot: snapshot,
            };
            let index = u.maybe_first_index();
            match index {
                None => assert!(!wok),
                Some(index) => assert_eq!(index, windex),
            }
        }
    }

    #[test]
    fn test_maybe_last_index() {
        // entry, offset, snap, wok, windex,
        let tests = vec![
            (Some(new_entry(5,1)), 5, None, true, 5),
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), true, 5),
            // last in snapshot
            (None, 5, Some(new_snapshot(4,1)), true, 4),
            // empty unstable
            (None, 0, None, false, 0),
       ];

        for (entries, offset, snapshot, wok, windex) in tests {
            let u = Unstable {
                entries: entries.map_or(vec![], |entry| vec![entry]),
                offset: offset,
                snapshot: snapshot,
            };
            let index = u.maybe_last_index();
            match index {
                None => assert!(!wok),
                Some(index) => assert_eq!(index, windex),
            }
        }
    }

    #[test]
    fn test_maybe_term() {
        // entry, offset, snap, index, wok, wterm
        let tests = vec![
            // term from entries
            (Some(new_entry(5,1)), 5, None, 5, true, 1),
            (Some(new_entry(5,1)), 5, None, 6, false, 0),
            (Some(new_entry(5,1)), 5, None, 4, false, 0),
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), 5, true, 1),
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), 6, false, 0),
            // term from snapshot
            (Some(new_entry(5,1)), 5, Some(new_snapshot(4,1)), 4, true, 1),
            (None, 5, Some(new_snapshot(4,1)), 5, false, 0),
            (None, 5, Some(new_snapshot(4,1)), 4, true, 1),
            (None, 0, None, 5, false, 0),
       ];

        for (entries, offset, snapshot, index, wok, wterm) in tests {
            let u = Unstable {
                entries: entries.map_or(vec![], |entry| vec![entry]),
                offset: offset,
                snapshot: snapshot,
            };
            let term = u.maybe_term(index);
            match term {
                None => assert!(!wok),
                Some(term) => assert_eq!(term, wterm),
            }
        }
    }

    #[test]
    fn test_restore() {
        let mut u = Unstable {
            entries: vec![new_entry(5, 1)],
            offset: 5,
            snapshot: Some(new_snapshot(4, 1)),
        };

        let s = new_snapshot(6, 2);
        u.restore(s.clone());

        assert_eq!(u.offset, s.get_metadata().get_index() + 1);
        assert!(u.entries.is_empty());
        assert_eq!(u.snapshot.unwrap(), s);
    }

    #[test]
    fn test_stable_to() {
        // entries, offset, snap, index, term, woffset, wlen
        let tests = vec![
            (vec![], 0, None, 5, 1, 0, 0),
            // stable to the first entry
            (vec![new_entry(5,1)], 5, None, 5, 1, 6, 0),
            (vec![new_entry(5,1), new_entry(6, 0)], 5, None, 5, 1, 6, 1),
            // stable to the first entry and term mismatch
            (vec![new_entry(6,2)], 5, None, 6, 1, 5, 1),
            // stable to old entry
            (vec![new_entry(5,1)], 5, None, 4, 1, 5, 1),
            (vec![new_entry(5,1)], 5, None, 4, 2, 5, 1),
            // with snapshot
            // stable to the first entry
            (vec![new_entry(5,1)], 5, Some(new_snapshot(4,1)), 5, 1, 6, 0),
            // stable to the first entry
            (vec![new_entry(5,1), new_entry(6,1)], 5, Some(new_snapshot(4,1)), 5, 1, 6, 1),
            // stable to the first entry and term mismatch
            (vec![new_entry(6,2)], 5, Some(new_snapshot(5,1)), 6, 1, 5, 1),
            // stable to snapshot
            (vec![new_entry(5,1)], 5, Some(new_snapshot(4,1)), 4, 1, 5, 1),
            // stable to old entry
            (vec![new_entry(5,2)], 5, Some(new_snapshot(4,2)), 4, 1, 5, 1),
       ];

        for (entries, offset, snapshot, index, term, woffset, wlen) in tests {
            let mut u = Unstable {
                entries: entries,
                offset: offset,
                snapshot: snapshot,
            };
            u.stable_to(index, term);
            assert_eq!(u.offset, woffset);
            assert_eq!(u.entries.len(), wlen);
        }
    }

    #[test]
    fn test_truncate_and_append() {
        // entries, offset, snap, to_append, woffset, wentries
        let tests = vec![
            // replace to the end
            (vec![new_entry(5, 1)], 5, None,
                vec![new_entry(6, 1), new_entry(7, 1)],
                    5, vec![new_entry(5,1), new_entry(6,1), new_entry(7,1)]),

            // replace to unstable entries
           (vec![new_entry(5, 1)], 5, None,
               vec![new_entry(5, 2), new_entry(6, 2)],
                   5, vec![new_entry(5, 2), new_entry(6, 2)]),

            (vec![new_entry(5, 1)], 5, None,
                vec![new_entry(4, 2), new_entry(5, 2), new_entry(6, 2)],
                    4, vec![new_entry(4,2), new_entry(5, 2), new_entry(6, 2)]),

            // truncate existing entries and append
           (vec![new_entry(5, 1), new_entry(6, 1), new_entry(7, 1)], 5, None,
               vec![new_entry(6, 2)],
                   5, vec![new_entry(5, 1), new_entry(6, 2)]),

            (vec![new_entry(5, 1), new_entry(6, 1), new_entry(7, 1)], 5, None,
                vec![new_entry(7, 2), new_entry(8, 2)],
                    5, vec![new_entry(5, 1), new_entry(6, 1), new_entry(7, 2), new_entry(8, 2)]),
      ];

        for (entries, offset, snapshot, to_append, woffset, wentries) in tests {
            let mut u = Unstable {
                entries: entries,
                offset: offset,
                snapshot: snapshot,
            };
            u.truncate_and_append(&to_append);
            assert_eq!(u.offset, woffset);
            assert_eq!(u.entries, wentries);
        }
    }
}
<|endoftext|>