<commit_before>
extern crate rustc_serialize;
extern crate amqp;
#[macro_use]
extern crate log;
extern crate argparse;
extern crate rand;

use amqp::{ConsumerCallBackFn, Session, Table, Basic, Channel, Options, Consumer};
use amqp::protocol;
use std::default::Default;
use rustc_serialize::json;
use std::slice;

// for debugging
fn listen_discovery(channel: &mut Channel) {
    fn participant_discovered(channel: &mut Channel, deliver: protocol::basic::Deliver,
                            headers: protocol::basic::BasicProperties, body: Vec<u8>){
        let s = std::str::from_utf8(&body).unwrap();
        let info: ParticipantInfo = json::decode(s).unwrap();
        info!("MsgFlo participant discovered: {:?}", info);
        channel.basic_ack(deliver.delivery_tag, false);
    }

    let queue_name = "fbp";
    let consumer_name = channel.basic_consume(participant_discovered as ConsumerCallBackFn, queue_name,
                                                "", false, false, false, false, Table::new());
    info!("listening for discovery messages");
}

#[derive(Debug, Default, RustcDecodable, RustcEncodable)]
pub struct ParticipantPort {
    pub id: String, // port name
    pub queue: String, // the associated message queue
    // FIXME: support. Is a keyword so needs some special handling   type: String, // datatype, ex: "boolean"
    // options: queue options as specified by the message queue implementation
    // description?? TODO: standardize
}

#[derive(Debug, Default, RustcDecodable, RustcEncodable)]
pub struct ParticipantInfo {
    pub id: String, // unique name
    pub role: String ,// role participant has
    pub component: String, // component the participant is instance of
    pub label: Option<String>, // (optional) short human-readable description
    pub icon: Option<String>, // (optional)
    pub inports: Vec<ParticipantPort>,
    pub outports: Vec<ParticipantPort>,
}

type SendFunction = fn(String, Vec<u8>);
pub type ProcessFunction = fn(Vec<u8>) -> Result<Vec<u8>, Vec<u8>>;
pub struct Participant {
    pub info: ParticipantInfo,
    pub process: ProcessFunction,
}

struct Connection {
    session: Session,
    channel: Channel,
}

// FIXME: wrap in protocol/command/payload envelope
fn create_queue_and_send(channel: &mut Channel, queue_name: &str, payload: String) {

    let queue_declare = channel.queue_declare(queue_name, false, true, false, false, false, Table::new());
    queue_declare.expect("queue creation failed");
    let content_type = Some("application/json".to_string());
    let props = protocol::basic::BasicProperties { content_type: content_type, ..Default::default() };
    debug!("sending on {}: {}", queue_name, payload);
    let res = channel.basic_publish("", queue_name, true, false, props, payload.into_bytes());
    res.expect("send on new queue failed");
}

fn send_discovery(channel: &mut Channel, info: &ParticipantInfo) {
    let queue_name = "fbp"; // TODO: use an exchange istead, requires protocol change in msgflo

    let payload = json::encode(&info).unwrap();
    create_queue_and_send(channel, queue_name, payload);
}

struct PortConsumer {
    process: ProcessFunction,
    portname: String,
    outqueue: String, // FIXME: allow sending on any port, also multiple times
}

fn send_out(channel: &mut Channel, exchange: String, data: Vec<u8>) {

    let routing_key = "".to_string();
    let content_type = Some("application/json".to_string()); // TODO: should be parameter
    let props = protocol::basic::BasicProperties { content_type: content_type, ..Default::default() };
    let s = channel.basic_publish(exchange, routing_key, true, false, props, data);
    s.expect("failed to send");
    debug!("sent output");
}

impl Consumer for PortConsumer {
    fn handle_delivery(&mut self,
                       channel: &mut Channel,
                       deliver: protocol::basic::Deliver,
                       headers: protocol::basic::BasicProperties,
                       body: Vec<u8>) {

        debug!("calling process()");
        let f = self.process;
        let res = f(body);
        debug!("process() returned");

        if res.is_ok() {
            debug!("ACKing and sending");
            send_out(channel, self.outqueue.to_string(), res.unwrap());
            let r = channel.basic_ack(deliver.delivery_tag, false);
        } else {
            error!("process() errored");
            let r = channel.basic_nack(deliver.delivery_tag, true, true);
        }
        // FIXME: send error data
    }
}

// FIXME: actually call ProcessFunction
fn setup_inport(participant: &Participant, port: &ParticipantPort, connection: &mut Connection) {
    debug!("setup inport: {}", port.queue.to_string());

    let consumer = PortConsumer {
        process: participant.process,
        portname: port.id.to_string(),
        outqueue: participant.info.outports[0].queue.to_string(),
    };

    // create
    let declare = connection.channel.queue_declare(port.queue.to_string(), false, true, false, false, false, Table::new());
    declare.expect("inport queue creation failed");

    // subscribe
    let q = port.queue.to_string();
    let cons = connection.channel.basic_consume(consumer, q,
                                                "".to_string(), false, false, false, false, Table::new());

    cons.expect("iport setup failed");

    debug!("inport setup done: {:?}, {:?}", port.id.to_string(), port.queue.to_string());
}

fn setup_outport(participant: &Participant, port: &ParticipantPort, connection: &mut Connection) {

    let exchange_type = "fanout".to_string();
    let declare = connection.channel.exchange_declare(port.queue.to_string(), exchange_type,
                                                    false, true, false, false, false, Table::new());
    declare.expect("outport setup failed");

    debug!("setup outport done: {:?}, {:?}", port.id.to_string(), port.queue.to_string());
}

fn start_participant(participant: &Participant, options: &ParticipantOptions) -> Connection {

    let mut session = Session::open_url(&options.broker).expect("Can't create AMQP session");
    let mut channel = session.open_channel(1).expect("could not open AMQP channel");

    let mut conn = Connection { session: session, channel: channel };

    // setup ports
    setup_inport(&participant, &participant.info.inports[0], &mut conn);
    setup_outport(&participant, &participant.info.outports[0], &mut conn);

    // send MsgFlo participant discovery message
    send_discovery(&mut conn.channel, &participant.info);

    return conn;
}

fn stop_participant(participant: &Participant, connection: &mut Connection) {

    let closed = connection.channel.close(200, "Bye".to_string());
    connection.session.close(200, "Good Bye".to_string());
}

use log::{LogRecord, LogLevel, LogMetadata, LogLevelFilter};

struct SimpleLogger {
    showlevel: LogLevel,
}

impl log::Log for SimpleLogger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= self.showlevel
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            println!("{} - {}", record.level(), record.args());
        }
    }
}

#[derive(Debug)]
struct ParticipantOptions {
    role: String,
    broker: String,
    log: String,
}

impl Default for ParticipantOptions {
    fn default() -> ParticipantOptions {
        use rand::{thread_rng, Rng};

        let id: String = thread_rng().gen_ascii_chars().take(5).collect();
        ParticipantOptions {
            broker: "amqp://localhost//".to_string(),
            role: format!("msgflo-rust-{}", id),
        }
    }
}

fn parse(options: &mut ParticipantOptions) {

    use argparse::{StoreTrue, Store};
    let mut parser = argparse::ArgumentParser::new();

    parser.refer(&mut options.role)
        .add_option(&["--role"], Store, "Participant role name");
    parser.refer(&mut options.broker)
        .add_option(&["--broker"], Store, "Address of messaging broker")
        .envvar("MSGFLO_BROKER");

    parser.parse_args_or_exit(); // XXX: should return out
}

// XXX: seems rust-amqp makes program hangs forever if error occurs / channel is borked?
// TODO: pass port info in/out of process()
// TODO: nicer way to declare ports? ideally they are enums not stringly typed?
pub fn participant_main(p: Participant) {

    let mut options = ParticipantOptions { .. Default::default() };
    parse(&mut options);

    let mut c = start_participant(&p, &options);
    c.channel.start_consuming();
    println!("{}({}) started", &options.role, &p.info.component);

    stop_participant(&p, &mut c);
}

<commit_msg>Remove obsolete log member<commit_after>
extern crate rustc_serialize;
extern crate amqp;
#[macro_use]
extern crate log;
extern crate argparse;
extern crate rand;

use amqp::{ConsumerCallBackFn, Session, Table, Basic, Channel, Options, Consumer};
use amqp::protocol;
use std::default::Default;
use rustc_serialize::json;
use std::slice;

// for debugging
fn listen_discovery(channel: &mut Channel) {
    fn participant_discovered(channel: &mut Channel, deliver: protocol::basic::Deliver,
                            headers: protocol::basic::BasicProperties, body: Vec<u8>){
        let s = std::str::from_utf8(&body).unwrap();
        let info: ParticipantInfo = json::decode(s).unwrap();
        info!("MsgFlo participant discovered: {:?}", info);
        channel.basic_ack(deliver.delivery_tag, false);
    }

    let queue_name = "fbp";
    let consumer_name = channel.basic_consume(participant_discovered as ConsumerCallBackFn, queue_name,
                                                "", false, false, false, false, Table::new());
    info!("listening for discovery messages");
}

#[derive(Debug, Default, RustcDecodable, RustcEncodable)]
pub struct ParticipantPort {
    pub id: String, // port name
    pub queue: String, // the associated message queue
    // FIXME: support. Is a keyword so needs some special handling   type: String, // datatype, ex: "boolean"
    // options: queue options as specified by the message queue implementation
    // description?? TODO: standardize
}

#[derive(Debug, Default, RustcDecodable, RustcEncodable)]
pub struct ParticipantInfo {
    pub id: String, // unique name
    pub role: String ,// role participant has
    pub component: String, // component the participant is instance of
    pub label: Option<String>, // (optional) short human-readable description
    pub icon: Option<String>, // (optional)
    pub inports: Vec<ParticipantPort>,
    pub outports: Vec<ParticipantPort>,
}

type SendFunction = fn(String, Vec<u8>);
pub type ProcessFunction = fn(Vec<u8>) -> Result<Vec<u8>, Vec<u8>>;
pub struct Participant {
    pub info: ParticipantInfo,
    pub process: ProcessFunction,
}

struct Connection {
    session: Session,
    channel: Channel,
}

// FIXME: wrap in protocol/command/payload envelope
fn create_queue_and_send(channel: &mut Channel, queue_name: &str, payload: String) {

    let queue_declare = channel.queue_declare(queue_name, false, true, false, false, false, Table::new());
    queue_declare.expect("queue creation failed");
    let content_type = Some("application/json".to_string());
    let props = protocol::basic::BasicProperties { content_type: content_type, ..Default::default() };
    debug!("sending on {}: {}", queue_name, payload);
    let res = channel.basic_publish("", queue_name, true, false, props, payload.into_bytes());
    res.expect("send on new queue failed");
}

fn send_discovery(channel: &mut Channel, info: &ParticipantInfo) {
    let queue_name = "fbp"; // TODO: use an exchange istead, requires protocol change in msgflo

    let payload = json::encode(&info).unwrap();
    create_queue_and_send(channel, queue_name, payload);
}

struct PortConsumer {
    process: ProcessFunction,
    portname: String,
    outqueue: String, // FIXME: allow sending on any port, also multiple times
}

fn send_out(channel: &mut Channel, exchange: String, data: Vec<u8>) {

    let routing_key = "".to_string();
    let content_type = Some("application/json".to_string()); // TODO: should be parameter
    let props = protocol::basic::BasicProperties { content_type: content_type, ..Default::default() };
    let s = channel.basic_publish(exchange, routing_key, true, false, props, data);
    s.expect("failed to send");
    debug!("sent output");
}

impl Consumer for PortConsumer {
    fn handle_delivery(&mut self,
                       channel: &mut Channel,
                       deliver: protocol::basic::Deliver,
                       headers: protocol::basic::BasicProperties,
                       body: Vec<u8>) {

        debug!("calling process()");
        let f = self.process;
        let res = f(body);
        debug!("process() returned");

        if res.is_ok() {
            debug!("ACKing and sending");
            send_out(channel, self.outqueue.to_string(), res.unwrap());
            let r = channel.basic_ack(deliver.delivery_tag, false);
        } else {
            error!("process() errored");
            let r = channel.basic_nack(deliver.delivery_tag, true, true);
        }
        // FIXME: send error data
    }
}

// FIXME: actually call ProcessFunction
fn setup_inport(participant: &Participant, port: &ParticipantPort, connection: &mut Connection) {
    debug!("setup inport: {}", port.queue.to_string());

    let consumer = PortConsumer {
        process: participant.process,
        portname: port.id.to_string(),
        outqueue: participant.info.outports[0].queue.to_string(),
    };

    // create
    let declare = connection.channel.queue_declare(port.queue.to_string(), false, true, false, false, false, Table::new());
    declare.expect("inport queue creation failed");

    // subscribe
    let q = port.queue.to_string();
    let cons = connection.channel.basic_consume(consumer, q,
                                                "".to_string(), false, false, false, false, Table::new());

    cons.expect("iport setup failed");

    debug!("inport setup done: {:?}, {:?}", port.id.to_string(), port.queue.to_string());
}

fn setup_outport(participant: &Participant, port: &ParticipantPort, connection: &mut Connection) {

    let exchange_type = "fanout".to_string();
    let declare = connection.channel.exchange_declare(port.queue.to_string(), exchange_type,
                                                    false, true, false, false, false, Table::new());
    declare.expect("outport setup failed");

    debug!("setup outport done: {:?}, {:?}", port.id.to_string(), port.queue.to_string());
}

fn start_participant(participant: &Participant, options: &ParticipantOptions) -> Connection {

    let mut session = Session::open_url(&options.broker).expect("Can't create AMQP session");
    let mut channel = session.open_channel(1).expect("could not open AMQP channel");

    let mut conn = Connection { session: session, channel: channel };

    // setup ports
    setup_inport(&participant, &participant.info.inports[0], &mut conn);
    setup_outport(&participant, &participant.info.outports[0], &mut conn);

    // send MsgFlo participant discovery message
    send_discovery(&mut conn.channel, &participant.info);

    return conn;
}

fn stop_participant(participant: &Participant, connection: &mut Connection) {

    let closed = connection.channel.close(200, "Bye".to_string());
    connection.session.close(200, "Good Bye".to_string());
}

use log::{LogRecord, LogLevel, LogMetadata, LogLevelFilter};

struct SimpleLogger {
    showlevel: LogLevel,
}

impl log::Log for SimpleLogger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= self.showlevel
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            println!("{} - {}", record.level(), record.args());
        }
    }
}

#[derive(Debug)]
struct ParticipantOptions {
    role: String,
    broker: String,
}

impl Default for ParticipantOptions {
    fn default() -> ParticipantOptions {
        use rand::{thread_rng, Rng};

        let id: String = thread_rng().gen_ascii_chars().take(5).collect();
        ParticipantOptions {
            broker: "amqp://localhost//".to_string(),
            role: format!("msgflo-rust-{}", id),
        }
    }
}

fn parse(options: &mut ParticipantOptions) {

    use argparse::{StoreTrue, Store};
    let mut parser = argparse::ArgumentParser::new();

    parser.refer(&mut options.role)
        .add_option(&["--role"], Store, "Participant role name");
    parser.refer(&mut options.broker)
        .add_option(&["--broker"], Store, "Address of messaging broker")
        .envvar("MSGFLO_BROKER");

    parser.parse_args_or_exit(); // XXX: should return out
}

// XXX: seems rust-amqp makes program hangs forever if error occurs / channel is borked?
// TODO: pass port info in/out of process()
// TODO: nicer way to declare ports? ideally they are enums not stringly typed?
pub fn participant_main(p: Participant) {

    let mut options = ParticipantOptions { .. Default::default() };
    parse(&mut options);

    let mut c = start_participant(&p, &options);
    c.channel.start_consuming();
    println!("{}({}) started", &options.role, &p.info.component);

    stop_participant(&p, &mut c);
}

<|endoftext|>


<commit_before>extern crate time;
extern crate byteorder;

mod ntp;
mod error;

use std::net::UdpSocket;
use time::Timespec;
pub use error::Error;
pub use ntp::NTPHeader;

const NTP_PORT: u16 = 123;
const UDP_LOCAL: &'static str = "0.0.0.0:35000";

/// `retrieve_ntp_timestamp` retrieves the current time from a NTP server.
///
/// # Arguments
///
/// * `host` - The NTP server (i.e. sundial.columbia.edu).
pub fn retrieve_ntp_timestamp(host: &str) -> Result<Timespec, Error> {
    let header = NTPHeader::new();
    let message = try!(header.encode());

    let socket = try!(UdpSocket::bind(UDP_LOCAL));

    let host = format!("{host}:{port}", host=host, port=NTP_PORT);
    try!(socket.send_to(&message[..], &host[..]));

    let mut buf = [0u8; 1000];

    // TODO: Rust doesn't support timeouts yet
    let (amt, _) = try!(socket.recv_from(&mut buf));

    drop(socket);

    let header = try!(ntp::NTPHeader::decode(amt, &buf));

    Ok(header.transmit_timestamp.as_timespec())
}

#[test]
fn receive_timestamp() {
    const NTP_SERVER: &'static str = "ntp.xs4all.nl";

    let t1 = retrieve_ntp_timestamp(NTP_SERVER).unwrap();
    let t2 = retrieve_ntp_timestamp(NTP_SERVER).unwrap();

    assert!(t2 > t1);
}
<commit_msg>Better choice of server<commit_after>extern crate time;
extern crate byteorder;

mod ntp;
mod error;

use std::net::UdpSocket;
use time::Timespec;
pub use error::Error;
pub use ntp::NTPHeader;

const NTP_PORT: u16 = 123;
const UDP_LOCAL: &'static str = "0.0.0.0:35000";

/// `retrieve_ntp_timestamp` retrieves the current time from a NTP server.
///
/// # Arguments
///
/// * `host` - The NTP server (i.e. sundial.columbia.edu).
pub fn retrieve_ntp_timestamp(host: &str) -> Result<Timespec, Error> {
    let header = NTPHeader::new();
    let message = try!(header.encode());

    let socket = try!(UdpSocket::bind(UDP_LOCAL));

    let host = format!("{host}:{port}", host=host, port=NTP_PORT);
    try!(socket.send_to(&message[..], &host[..]));

    let mut buf = [0u8; 1000];

    // TODO: Rust doesn't support timeouts yet
    let (amt, _) = try!(socket.recv_from(&mut buf));

    drop(socket);

    let header = try!(ntp::NTPHeader::decode(amt, &buf));

    Ok(header.transmit_timestamp.as_timespec())
}

#[test]
fn receive_timestamp() {
    const NTP_SERVER: &'static str = "pool.ntp.org";

    let t1 = retrieve_ntp_timestamp(NTP_SERVER).unwrap();
    let t2 = retrieve_ntp_timestamp(NTP_SERVER).unwrap();

    assert!(t2 > t1);
}
<|endoftext|>


<commit_before>//! Bit utilities for Rust
//!
//! This will eventually expand to include Nibbles and packing them into bytes and integers.

pub trait BitInformation {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize;

    /// Checks to see if bit X is set
    fn has_x_bit(&self, position: usize) -> bool;

    /// Checks to see if the requested bit position is in bounds
    fn is_bit_in_bounds(&self, position: usize) -> bool {
        position <= self.number_of_bits()
    }

    /// Checks to see if the most signifigant bit is set
    fn has_most_signifigant_bit(&self) -> bool {
        self.has_x_bit(self.number_of_bits() - 1) //8 bits? position 7.
    }

}

impl BitInformation for u8 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        8
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b00000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for i8 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        8
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b00000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for u16 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        16
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b0000000000000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for i16 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        16
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b0000000000000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

#[allow(overflowing_literals)] //Just for tests, don't do this in real life, kids!
#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn test_number_of_bits_i8() {
        assert_eq!(8, 1i8.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_i8() {
        assert!(1i8.has_most_signifigant_bit() == false);
        assert!(0b10101010i8.has_most_signifigant_bit() == true);
        assert!(0b10111000i8.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_i8() {
        assert!(25i8.is_bit_in_bounds(1));
        assert!(25i8.is_bit_in_bounds(9) == false);
    }

    #[test]
    fn test_has_x_bit_i8() {
        let num: i8 = 0b10101010i8;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
    }

    #[test]
    fn test_number_of_bits_u8() {
        assert_eq!(8, 1u8.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_u8() {
        assert!(1u8.has_most_signifigant_bit() == false);
        assert!(0b10101010u8.has_most_signifigant_bit() == true);
        assert!(0b10111000u8.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_u8() {
        assert!(25u8.is_bit_in_bounds(1));
        assert!(25u8.is_bit_in_bounds(9) == false);
    }

    #[test]
    fn test_has_x_bit_u8() {
        let num: u8 = 0b10101010u8;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
    }

    #[test]
    fn test_number_of_bits_u16() {
        assert_eq!(16, 1u16.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_u16() {
        assert!(1u16.has_most_signifigant_bit() == false);
        assert!(0b10101010u8.has_most_signifigant_bit() == true);
        assert!(0b10111000u8.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_u16() {
        assert!(25u16.is_bit_in_bounds(12));
        assert!(25u16.is_bit_in_bounds(18) == false);
    }

    #[test]
    fn test_has_x_bit_u16() {
        let num: u16 = 0b1010101010101010u16;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
        assert!(num.has_x_bit(8) == false);
        assert!(num.has_x_bit(9) == true);
        assert!(num.has_x_bit(10) == false);
        assert!(num.has_x_bit(11) == true);
        assert!(num.has_x_bit(12) == false);
        assert!(num.has_x_bit(13) == true);
        assert!(num.has_x_bit(14) == false);
        assert!(num.has_x_bit(15) == true);
    }

    #[test]
    fn test_number_of_bits_i16() {
        assert_eq!(16, 1i16.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_i16() {
        assert!(1i16.has_most_signifigant_bit() == false);
        assert!(0b10101010i8.has_most_signifigant_bit() == true);
        assert!(0b10111000i8.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_i16() {
        assert!(25i16.is_bit_in_bounds(12));
        assert!(25i16.is_bit_in_bounds(18) == false);
    }

    #[test]
    fn test_has_x_bit_i16() {
        let num: i16 = 0b1010101010101010i16;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
        assert!(num.has_x_bit(8) == false);
        assert!(num.has_x_bit(9) == true);
        assert!(num.has_x_bit(10) == false);
        assert!(num.has_x_bit(11) == true);
        assert!(num.has_x_bit(12) == false);
        assert!(num.has_x_bit(13) == true);
        assert!(num.has_x_bit(14) == false);
        assert!(num.has_x_bit(15) == true);
    }

}<commit_msg>Implement for i32 and u32<commit_after>//! Bit utilities for Rust
//!
//! This will eventually expand to include Nibbles and packing them into bytes and integers.

pub trait BitInformation {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize;

    /// Checks to see if bit X is set
    fn has_x_bit(&self, position: usize) -> bool;

    /// Checks to see if the requested bit position is in bounds
    fn is_bit_in_bounds(&self, position: usize) -> bool {
        position <= self.number_of_bits()
    }

    /// Checks to see if the most signifigant bit is set
    fn has_most_signifigant_bit(&self) -> bool {
        self.has_x_bit(self.number_of_bits() - 1) //8 bits? position 7.
    }

}

impl BitInformation for u8 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        8
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b00000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for i8 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        8
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b00000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for u16 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        16
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b0000000000000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for i16 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        16
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b0000000000000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for u32 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        32
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b00000000000000000000000000000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

impl BitInformation for i32 {

    /// Gets the number of bits contained in this type
    fn number_of_bits(&self) -> usize {
        32
    }

    fn has_x_bit(&self, position: usize) -> bool {
        if self.is_bit_in_bounds(position) {
            return (self & (0b00000000000000000000000000000001 << position)) != 0;
        } else {
            return false; //Huh
        }
    }

}

#[allow(overflowing_literals)] //Just for tests, don't do this in real life, kids!
#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn test_number_of_bits_i8() {
        assert_eq!(8, 1i8.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_i8() {
        assert!(1i8.has_most_signifigant_bit() == false);
        assert!(0b10101010i8.has_most_signifigant_bit() == true);
        assert!(0b10111000i8.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_i8() {
        assert!(25i8.is_bit_in_bounds(1));
        assert!(25i8.is_bit_in_bounds(9) == false);
    }

    #[test]
    fn test_has_x_bit_i8() {
        let num: i8 = 0b10101010i8;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
    }

    #[test]
    fn test_number_of_bits_u8() {
        assert_eq!(8, 1u8.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_u8() {
        assert!(1u8.has_most_signifigant_bit() == false);
        assert!(0b10101010u8.has_most_signifigant_bit() == true);
        assert!(0b10111000u8.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_u8() {
        assert!(25u8.is_bit_in_bounds(1));
        assert!(25u8.is_bit_in_bounds(9) == false);
    }

    #[test]
    fn test_has_x_bit_u8() {
        let num: u8 = 0b10101010u8;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
    }

    #[test]
    fn test_number_of_bits_u16() {
        assert_eq!(16, 1u16.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_u16() {
        assert!(1u16.has_most_signifigant_bit() == false);
        assert!(0b1010101010101010u16.has_most_signifigant_bit() == true);
        assert!(0b1011100000000000u16.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_u16() {
        assert!(25u16.is_bit_in_bounds(12));
        assert!(25u16.is_bit_in_bounds(18) == false);
    }

    #[test]
    fn test_has_x_bit_u16() {
        let num: u16 = 0b1010101010101010u16;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
        assert!(num.has_x_bit(8) == false);
        assert!(num.has_x_bit(9) == true);
        assert!(num.has_x_bit(10) == false);
        assert!(num.has_x_bit(11) == true);
        assert!(num.has_x_bit(12) == false);
        assert!(num.has_x_bit(13) == true);
        assert!(num.has_x_bit(14) == false);
        assert!(num.has_x_bit(15) == true);
    }

    #[test]
    fn test_number_of_bits_i16() {
        assert_eq!(16, 1i16.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_i16() {
        assert!(1i16.has_most_signifigant_bit() == false);
        assert!(0b1010101010101010i16.has_most_signifigant_bit() == true);
        assert!(0b1011100000000000i16.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_i16() {
        assert!(25i16.is_bit_in_bounds(12));
        assert!(25i16.is_bit_in_bounds(18) == false);
    }

    #[test]
    fn test_has_x_bit_i16() {
        let num: i16 = 0b1010101010101010i16;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
        assert!(num.has_x_bit(8) == false);
        assert!(num.has_x_bit(9) == true);
        assert!(num.has_x_bit(10) == false);
        assert!(num.has_x_bit(11) == true);
        assert!(num.has_x_bit(12) == false);
        assert!(num.has_x_bit(13) == true);
        assert!(num.has_x_bit(14) == false);
        assert!(num.has_x_bit(15) == true);
    }

    #[test]
    fn test_number_of_bits_u32() {
        assert_eq!(32, 1u32.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_u32() {
        assert!(1u32.has_most_signifigant_bit() == false);
        assert!(0b10101010101010101010101010101010u32.has_most_signifigant_bit() == true);
        assert!(0b10111000000000000000000000000000u32.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_u32() {
        assert!(25u32.is_bit_in_bounds(30));
        assert!(25u32.is_bit_in_bounds(57) == false);
    }

    #[test]
    fn test_has_x_bit_u32() {
        let num: u32 = 0b10101010101010101010101010101010u32;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
        assert!(num.has_x_bit(8) == false);
        assert!(num.has_x_bit(9) == true);
        assert!(num.has_x_bit(10) == false);
        assert!(num.has_x_bit(11) == true);
        assert!(num.has_x_bit(12) == false);
        assert!(num.has_x_bit(13) == true);
        assert!(num.has_x_bit(14) == false);
        assert!(num.has_x_bit(15) == true);
        assert!(num.has_x_bit(16) == false);
        assert!(num.has_x_bit(17) == true);
        assert!(num.has_x_bit(18) == false);
        assert!(num.has_x_bit(19) == true);
        assert!(num.has_x_bit(20) == false);
        assert!(num.has_x_bit(21) == true);
        assert!(num.has_x_bit(22) == false);
        assert!(num.has_x_bit(23) == true);
        assert!(num.has_x_bit(24) == false);
        assert!(num.has_x_bit(25) == true);
        assert!(num.has_x_bit(26) == false);
        assert!(num.has_x_bit(27) == true);
        assert!(num.has_x_bit(28) == false);
        assert!(num.has_x_bit(29) == true);
        assert!(num.has_x_bit(30) == false);
        assert!(num.has_x_bit(31) == true);
    }

    #[test]
    fn test_number_of_bits_i32() {
        assert_eq!(32, 1i32.number_of_bits());
    }

    #[test]
    fn test_most_signifigant_bit_i32() {
        assert!(1i32.has_most_signifigant_bit() == false);
        assert!(0b10101010101010101010101010101010i32.has_most_signifigant_bit() == true);
        assert!(0b10111000000000000000000000000000i32.has_most_signifigant_bit() == true);
    }

    #[test]
    fn test_bit_in_bounds_i32() {
        assert!(25i32.is_bit_in_bounds(30));
        assert!(25i32.is_bit_in_bounds(57) == false);
    }

    #[test]
    fn test_has_x_bit_i32() {
        let num: i32 = 0b10101010101010101010101010101010i32;

        assert!(num.has_x_bit(0) == false);
        assert!(num.has_x_bit(1) == true);
        assert!(num.has_x_bit(2) == false);
        assert!(num.has_x_bit(3) == true);
        assert!(num.has_x_bit(4) == false);
        assert!(num.has_x_bit(5) == true);
        assert!(num.has_x_bit(6) == false);
        assert!(num.has_x_bit(7) == true);
        assert!(num.has_x_bit(8) == false);
        assert!(num.has_x_bit(9) == true);
        assert!(num.has_x_bit(10) == false);
        assert!(num.has_x_bit(11) == true);
        assert!(num.has_x_bit(12) == false);
        assert!(num.has_x_bit(13) == true);
        assert!(num.has_x_bit(14) == false);
        assert!(num.has_x_bit(15) == true);
        assert!(num.has_x_bit(16) == false);
        assert!(num.has_x_bit(17) == true);
        assert!(num.has_x_bit(18) == false);
        assert!(num.has_x_bit(19) == true);
        assert!(num.has_x_bit(20) == false);
        assert!(num.has_x_bit(21) == true);
        assert!(num.has_x_bit(22) == false);
        assert!(num.has_x_bit(23) == true);
        assert!(num.has_x_bit(24) == false);
        assert!(num.has_x_bit(25) == true);
        assert!(num.has_x_bit(26) == false);
        assert!(num.has_x_bit(27) == true);
        assert!(num.has_x_bit(28) == false);
        assert!(num.has_x_bit(29) == true);
        assert!(num.has_x_bit(30) == false);
        assert!(num.has_x_bit(31) == true);
    }

}<|endoftext|>


<commit_before>#![feature(custom_derive)]
#![feature(proc_macro)]

#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

//pub mod reference;

use serde_json::Value;

#[derive(Deserialize, Debug)]
pub struct Response {
    took: u64,
    timed_out: bool,
    _shards: Shards,
    hits: Hits,
    aggregations: Option<Aggregations>,
    status: Option<u16>
}

#[derive(Deserialize, Debug)]
pub struct Aggregations(Value);

impl<'a> IntoIterator for Aggregations {
    type Item = Value;
    type IntoIter = AggregationIterator<'a>;

    fn into_iter(self) -> Self::IntoIter {
        AggregationIterator::new(self)
    }
}

#[derive(Debug)]
pub struct AggregationIterator<'a> {
    start_at: Option<&'a Value>,
    count: u64,
    aggregations: Aggregations
}

impl<'a> AggregationIterator<'a> {
    fn new(a: Aggregations) -> AggregationIterator<'a> {

        let v : &Value = match a {
            Aggregations(v) => v
        };

        AggregationIterator {
            start_at: Some(v),
            count: 0,
            aggregations: a
        }
    }
}

impl<'a> Iterator for AggregationIterator<'a> {
    type Item = Value;

    fn next(&mut self) -> Option<Value> {

        if self.count < 6 {
            Some(Value::U64(0))
        } else {
            None
        }
    }
}

#[derive(Deserialize, Debug)]
struct Shards {
    total: u32,
    successful: u32,
    failed: u32
}

#[derive(Deserialize, Debug)]
pub struct Hits {
    total: u64,
    max_score: u64,
    hits: Vec<Value>
}

impl Response {
    pub fn hits(&self) -> &Vec<Value> {
        &self.hits.hits()
    }

    pub fn aggs(self) -> std::option::Option<Aggregations> {
        self.aggregations
    }
}

impl Hits {
    pub fn hits(&self) -> &Vec<Value> {
        &self.hits
    }
}

#[derive(Deserialize, Debug)]
struct Hit {
    _index: String
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {}
}
<commit_msg>Initialisation finally works<commit_after>#![feature(custom_derive)]
#![feature(proc_macro)]

#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

//pub mod reference;

use serde_json::Value;

#[derive(Deserialize, Debug)]
pub struct Response {
    took: u64,
    timed_out: bool,
    _shards: Shards,
    hits: Hits,
    aggregations: Option<Aggregations>,
    status: Option<u16>
}

#[derive(Deserialize, Debug)]
pub struct Aggregations(Value);

impl<'a> IntoIterator for &'a Aggregations {
    type Item = &'a Value;
    type IntoIter = AggregationIterator<'a>;

    fn into_iter(self) -> AggregationIterator<'a> {
        AggregationIterator::new(self)
    }
}

#[derive(Debug)]
pub struct AggregationIterator<'a> {
    start_at: Option<&'a Value>,
    count: u64,
    aggregations: &'a Aggregations
}

impl<'a> AggregationIterator<'a> {
    fn new(a: &'a Aggregations) -> AggregationIterator<'a> {

        let v = match a {
            &Aggregations(ref v) => v
        };

//        println!("{:#?}", v);

        AggregationIterator {
            start_at: Some(v),
            count: 0,
            aggregations: a
        }
    }
}

impl<'a> Iterator for AggregationIterator<'a> {
    type Item = &'a Value;

    fn next(&mut self) -> Option<&'a Value> {

        if self.count < 6 {
            Some(self.start_at.unwrap())
        } else {
            None
        }
    }
}

#[derive(Deserialize, Debug)]
struct Shards {
    total: u32,
    successful: u32,
    failed: u32
}

#[derive(Deserialize, Debug)]
pub struct Hits {
    total: u64,
    max_score: u64,
    hits: Vec<Value>
}

impl Response {
    pub fn hits(&self) -> &Vec<Value> {
        &self.hits.hits()
    }

    pub fn aggs(self) -> std::option::Option<Aggregations> {
        self.aggregations
    }
}

impl Hits {
    pub fn hits(&self) -> &Vec<Value> {
        &self.hits
    }
}

#[derive(Deserialize, Debug)]
struct Hit {
    _index: String
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {}
}
<|endoftext|>


<commit_before>extern crate crc;
extern crate memmap;
extern crate byteorder;

use std::path::Path;
use std::fs::{OpenOptions, File};
use crc::crc32::checksum_ieee;
use memmap::{Mmap, Protection};
use byteorder::{BigEndian, ByteOrder};
use std::io::{self, Write};

pub struct Message {
    crc: u32,
    payload: Vec<u8>,
}

impl Message {
    pub fn new(payload: Vec<u8>) -> Message {
        let payload_crc = checksum_ieee(&payload);
        Message {
            crc: payload_crc,
            payload: payload,
        }
    }

    pub fn payload(&self) -> &[u8] {
        &self.payload
    }

    pub fn crc(&self) -> u32 {
        self.crc
    }
}

pub struct MessageSet {
    bytes: Vec<u8>,
}

impl MessageSet {
    pub fn new(msgs: Vec<Message>, offset: u64) -> MessageSet {
        // encode bytes with
        // 8: offset
        // 4: size
        // .: messages
        let size: usize = msgs.iter().map(|m| 4 + m.payload().len()).sum();
        let mut bytes = vec![0; 12 + size];
        BigEndian::write_u64(&mut bytes[0..8], offset);
        BigEndian::write_u32(&mut bytes[8..12], size as u32);

        let mut pos = 12;
        for m in msgs.iter() {
            // serialize the message CRC32
            BigEndian::write_u32(&mut bytes[pos..pos+4], m.crc());
            pos += 4;

            let mut copy_slice = &mut bytes[pos..pos+m.payload.len()];

            // copy the bytes of the message payload
            copy_slice.write_all(m.payload()).unwrap();
            pos += m.payload.len();
        }

        MessageSet {
            bytes: bytes,
        }
    }

    pub fn bytes(&self) -> &[u8] {
        &self.bytes
    }
}

/// An index is a file with pairs of relative offset to file position offset
/// messages. Both are stored as 4 bytes.
pub struct Index {
    file: File,
    mmap: Mmap,
    /// next starting byte in index file offset to write
    next_write_offset: usize,
    base_offset: u64,
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub struct IndexEntry {
    rel_offset: u32,
    base_offset: u64,
    file_pos: u32,
}

impl IndexEntry {
    pub fn relative_offset(&self) -> u32 {
        self.rel_offset
    }

    pub fn offset(&self) -> u64 {
        self.base_offset + (self.rel_offset as u64)
    }

    pub fn file_position(&self) -> u32 {
        self.file_pos
    }
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub enum IndexWriteError {
    IndexFull,
    OffsetLessThanBase,
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub enum IndexReadError {
    OutOfBounds
}

impl Index {
    pub fn new<P>(path: P, file_bytes: u64, base_offset: u64) -> io::Result<Index>
        where P: AsRef<Path>
    {
        // open the file, expecting to create it
        let index_file = try!(OpenOptions::new()
            .read(true)
            .write(true)
            .append(true)
            .create(true)
            .open(path));

        // read the metadata and truncate
        let meta = try!(index_file.metadata());
        let len = meta.len();
        if len == 0 {
            try!(index_file.set_len(file_bytes));
        }

        let mmap = try!(Mmap::open(&index_file, Protection::ReadWrite));

        Ok(Index {
            file: index_file,
            mmap: mmap,
            next_write_offset: len as usize,
            base_offset: base_offset,
        })
    }

    pub fn can_write(&self) -> bool {
        self.len() >= (self.next_write_offset + 8)
    }

    pub fn len(&self) -> usize {
        self.mmap.len()
    }

    pub fn append(&mut self, abs_offset: u64, position: u32) -> Result<(), IndexWriteError> {
        // TODO: test to ensure abs_offset is < last offset written (invariant of the index)
        if !self.can_write() {
            return Err(IndexWriteError::IndexFull);
        }

        if abs_offset < self.base_offset {
            return Err(IndexWriteError::OffsetLessThanBase);
        }

        unsafe {
            let mem_slice: &mut [u8] = self.mmap.as_mut_slice();
            let offset = (abs_offset - self.base_offset) as u32;
            let buf_pos = self.next_write_offset;

            BigEndian::write_u32(&mut mem_slice[buf_pos..buf_pos+4], offset);
            BigEndian::write_u32(&mut mem_slice[buf_pos+4..buf_pos+8], position);

            self.next_write_offset += 8;

            Ok(())
        }
    }

    pub fn flush_sync(&mut self) -> io::Result<()> {
        try!(self.mmap.flush());
        self.file.flush()
    }

    pub fn read_entry(&self, i: usize) -> Result<Option<IndexEntry>, IndexReadError> {
        if self.len() < (i+1)*8 {
            return Err(IndexReadError::OutOfBounds);
        }

        unsafe {
            let mem_slice = self.mmap.as_slice();
            let start = i*8;
            let offset = BigEndian::read_u32(&mem_slice[start..start+4]);
            if offset == 0 {
                Ok(None)
            } else {
                let pos = BigEndian::read_u32(&mem_slice[start+4..start+8]);
                Ok(Some(IndexEntry {
                    rel_offset: offset,
                    base_offset: self.base_offset,
                    file_pos: pos,
                }))
            }
        }
    }
}

pub struct Log {
    f: File,
    max_bytes: usize,
    pos: usize,
}

#[derive(Debug)]
pub enum LogAppendError {
    LogFull,
    IoError(io::Error),
}

impl From<std::io::Error> for LogAppendError {
    fn from(e: std::io::Error) -> LogAppendError {
        LogAppendError::IoError(e)
    }
}

impl Log {
    pub fn new<P>(p: P, max_bytes: usize) -> io::Result<Log>
        where P: AsRef<Path>
    {
        let f = try!(OpenOptions::new()
                     .write(true)
                     .read(true)
                     .create(true)
                     .append(true)
                     .open(p));
        Ok(Log {
            f: f,
            max_bytes: max_bytes,
            pos: 0,
        })
    }

    pub fn can_write(&self) -> bool {
        self.pos < self.max_bytes
    }

    pub fn append(&mut self, msgs: MessageSet) -> Result<u32, LogAppendError> {
        // ensure we won't go over the max
        if msgs.bytes().len() + self.pos > self.max_bytes {
            return Err(LogAppendError::LogFull);
        }

        try!(self.f.write_all(msgs.bytes()));
        let write_pos = self.pos;
        self.pos += msgs.bytes().len();
        Ok(write_pos as u32)
    }

    pub fn flush(&mut self) -> io::Result<()> {
        self.f.flush()
    }
}

#[cfg(test)]
mod tests {
    use std::fs;
    use std::io::Write;
    use super::*;

    #[test]
    fn message_construction() {
        let msg = Message::new(b"123456789".to_vec());
        assert_eq!(msg.payload(), b"123456789");
        assert_eq!(msg.crc(), 0xcbf43926);
    }

    #[test]
    fn index() {
        let index_path = "target/.test-index";

        // remove before running (may be around from failing test run)
        fs::remove_file(index_path).unwrap_or(());

        {
            let mut index = Index::new(index_path, 1000u64, 10u64).unwrap();
            assert_eq!(1000, index.len());
            index.append(11u64, 0xffff).unwrap();
            index.append(12u64, 0xeeee).unwrap();
            index.flush_sync().unwrap();
        }

        // reopen it
        {
            let index = Index::new(index_path, 1000u64, 10u64).unwrap();

            let e0 = index.read_entry(0).unwrap().unwrap();
            assert_eq!(1u32, e0.relative_offset());
            assert_eq!(11u64, e0.offset());
            assert_eq!(0xffff, e0.file_position());

            let e1 = index.read_entry(1).unwrap().unwrap();
            assert_eq!(2u32, e1.relative_offset());
            assert_eq!(12u64, e1.offset());
            assert_eq!(0xeeee, e1.file_position());

            // read an entry that does not exist
            let e2 = index.read_entry(2).unwrap();
            assert_eq!(None, e2);
        }
        fs::remove_file(index_path).unwrap();
    }

    #[test]
    pub fn log() {
        let log_path = "target/.test-log";
        fs::remove_file(log_path);
        {
            let mut f = Log::new(log_path, 1024).unwrap();

            let m0 = Message::new(b"12345".to_vec());
            let ms0 = MessageSet::new(vec![m0], 1000);
            assert_eq!(ms0.bytes().len(), 21);
            let p0 = f.append(ms0).unwrap();
            assert_eq!(p0, 0);

            let m1 = Message::new(b"66666".to_vec());
            let m2 = Message::new(b"77777".to_vec());
            let p1 = f.append(MessageSet::new(vec![m1, m2], 1001)).unwrap();
            assert_eq!(p1, 21);

            f.flush().unwrap();
        }
        fs::remove_file(log_path);
    }
}
<commit_msg>Adding segment<commit_after>extern crate crc;
extern crate memmap;
extern crate byteorder;

use std::path::{Path, PathBuf};
use std::fs::{OpenOptions, File};
use crc::crc32::checksum_ieee;
use memmap::{Mmap, Protection};
use byteorder::{BigEndian, ByteOrder};
use std::io::{self, Write};

// TODO: ...
// pub type Offset(u64);

pub struct Message {
    crc: u32,
    payload: Vec<u8>,
}

impl Message {
    pub fn new(payload: Vec<u8>) -> Message {
        let payload_crc = checksum_ieee(&payload);
        Message {
            crc: payload_crc,
            payload: payload,
        }
    }

    pub fn payload(&self) -> &[u8] {
        &self.payload
    }

    pub fn crc(&self) -> u32 {
        self.crc
    }
}

pub struct MessageSet {
    bytes: Vec<u8>,
}

impl MessageSet {
    pub fn new(msgs: Vec<Message>, offset: u64) -> MessageSet {
        // encode bytes with
        // 8: offset
        // 4: size
        // .: messages
        let size: usize = msgs.iter().map(|m| 4 + m.payload().len()).sum();
        let mut bytes = vec![0; 12 + size];
        BigEndian::write_u64(&mut bytes[0..8], offset);
        BigEndian::write_u32(&mut bytes[8..12], size as u32);

        let mut pos = 12;
        for m in msgs.iter() {
            // serialize the message CRC32
            BigEndian::write_u32(&mut bytes[pos..pos+4], m.crc());
            pos += 4;

            let mut copy_slice = &mut bytes[pos..pos+m.payload.len()];

            // copy the bytes of the message payload
            copy_slice.write_all(m.payload()).unwrap();
            pos += m.payload.len();
        }

        MessageSet {
            bytes: bytes,
        }
    }

    pub fn bytes(&self) -> &[u8] {
        &self.bytes
    }
}

/// An index is a file with pairs of relative offset to file position offset
/// messages. Both are stored as 4 bytes.
pub struct Index {
    file: File,
    mmap: Mmap,
    /// next starting byte in index file offset to write
    next_write_offset: usize,
    base_offset: u64,
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub struct IndexEntry {
    rel_offset: u32,
    base_offset: u64,
    file_pos: u32,
}

impl IndexEntry {
    pub fn relative_offset(&self) -> u32 {
        self.rel_offset
    }

    pub fn offset(&self) -> u64 {
        self.base_offset + (self.rel_offset as u64)
    }

    pub fn file_position(&self) -> u32 {
        self.file_pos
    }
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub enum IndexWriteError {
    IndexFull,
    OffsetLessThanBase,
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub enum IndexReadError {
    OutOfBounds
}

impl Index {
    pub fn new<P>(path: P, file_bytes: u64, base_offset: u64) -> io::Result<Index>
        where P: AsRef<Path>
    {
        // open the file, expecting to create it
        let index_file = try!(OpenOptions::new()
            .read(true)
            .write(true)
            .append(true)
            .create(true)
            .open(path));

        // read the metadata and truncate
        let meta = try!(index_file.metadata());
        let len = meta.len();
        if len == 0 {
            try!(index_file.set_len(file_bytes));
        }

        let mmap = try!(Mmap::open(&index_file, Protection::ReadWrite));

        Ok(Index {
            file: index_file,
            mmap: mmap,
            next_write_offset: len as usize,
            base_offset: base_offset,
        })
    }

    pub fn can_write(&self) -> bool {
        self.len() >= (self.next_write_offset + 8)
    }

    pub fn len(&self) -> usize {
        self.mmap.len()
    }

    pub fn append(&mut self, abs_offset: u64, position: u32) -> Result<(), IndexWriteError> {
        // TODO: test to ensure abs_offset is < last offset written (invariant of the index)
        if !self.can_write() {
            return Err(IndexWriteError::IndexFull);
        }

        if abs_offset < self.base_offset {
            return Err(IndexWriteError::OffsetLessThanBase);
        }

        unsafe {
            let mem_slice: &mut [u8] = self.mmap.as_mut_slice();
            let offset = (abs_offset - self.base_offset) as u32;
            let buf_pos = self.next_write_offset;

            BigEndian::write_u32(&mut mem_slice[buf_pos..buf_pos+4], offset);
            BigEndian::write_u32(&mut mem_slice[buf_pos+4..buf_pos+8], position);

            self.next_write_offset += 8;

            Ok(())
        }
    }

    pub fn flush_sync(&mut self) -> io::Result<()> {
        try!(self.mmap.flush());
        self.file.flush()
    }

    pub fn read_entry(&self, i: usize) -> Result<Option<IndexEntry>, IndexReadError> {
        if self.len() < (i+1)*8 {
            return Err(IndexReadError::OutOfBounds);
        }

        unsafe {
            let mem_slice = self.mmap.as_slice();
            let start = i*8;
            let offset = BigEndian::read_u32(&mem_slice[start..start+4]);
            if offset == 0 {
                Ok(None)
            } else {
                let pos = BigEndian::read_u32(&mem_slice[start+4..start+8]);
                Ok(Some(IndexEntry {
                    rel_offset: offset,
                    base_offset: self.base_offset,
                    file_pos: pos,
                }))
            }
        }
    }
}

pub struct Log {
    f: File,
    max_bytes: usize,
    pos: usize,
}

#[derive(Debug)]
pub enum LogAppendError {
    LogFull,
    IoError(io::Error),
}

impl From<std::io::Error> for LogAppendError {
    fn from(e: std::io::Error) -> LogAppendError {
        LogAppendError::IoError(e)
    }
}

impl Log {
    pub fn new<P>(p: P, max_bytes: usize) -> io::Result<Log>
        where P: AsRef<Path>
    {
        let f = try!(OpenOptions::new()
                     .write(true)
                     .read(true)
                     .create(true)
                     .append(true)
                     .open(p));
        Ok(Log {
            f: f,
            max_bytes: max_bytes,
            pos: 0,
        })
    }

    pub fn can_write(&self) -> bool {
        self.pos < self.max_bytes
    }

    pub fn append(&mut self, msgs: MessageSet) -> Result<u32, LogAppendError> {
        // ensure we won't go over the max
        if msgs.bytes().len() + self.pos > self.max_bytes {
            return Err(LogAppendError::LogFull);
        }

        try!(self.f.write_all(msgs.bytes()));
        let write_pos = self.pos;
        self.pos += msgs.bytes().len();
        Ok(write_pos as u32)
    }

    pub fn flush(&mut self) -> io::Result<()> {
        self.f.flush()
    }
}

pub struct Segment {
    log: Log,
    index: Index,
    next_offset: u64,
}

#[derive(Debug)]
pub enum SegmentAppendError {
    LogFull,
    IndexError(IndexWriteError),
    IoError(io::Error),
}

impl From<io::Error> for SegmentAppendError {
    fn from(e: io::Error) -> SegmentAppendError {
        SegmentAppendError::IoError(e)
    }
}

impl From<LogAppendError> for SegmentAppendError {
    fn from(e: LogAppendError) -> SegmentAppendError {
        match e {
            LogAppendError::LogFull => SegmentAppendError::LogFull,
            LogAppendError::IoError(e) => SegmentAppendError::IoError(e),
        }
    }
}

impl From<IndexWriteError> for SegmentAppendError {
    fn from(e: IndexWriteError) -> SegmentAppendError {
        SegmentAppendError::IndexError(e)
    }
}

impl Segment {
    // TODO: open variant for reading

    pub fn new<P>(log_dir: P, base_offset: u64, max_bytes: usize, index_bytes: usize) -> io::Result<Segment>
        where P: AsRef<Path>
    {
        let log = {
            let mut path_buf = PathBuf::new();
            path_buf.push(&log_dir);
            path_buf.push(format!("{:020}", base_offset));
            path_buf.set_extension(".log");
            try!(Log::new(&path_buf, max_bytes))
        };

        let index = {
            let mut path_buf = PathBuf::new();
            path_buf.push(&log_dir);
            path_buf.push(format!("{:020}", base_offset));
            path_buf.set_extension(".index");
            try!(Index::new(&path_buf, index_bytes as u64, base_offset))
        };

        Ok(Segment {
            log: log,
            index: index,
            next_offset: base_offset,
        })
    }

    pub fn next_offset(&self) -> u64{
        self.next_offset
    }

    pub fn append(&mut self, msgs: Vec<Message>) -> Result<u64, SegmentAppendError> {
        if !self.index.can_write() || !self.log.can_write() {
            return Err(SegmentAppendError::LogFull);
        }

        let off = self.next_offset;
        let pos = try!(self.log.append(MessageSet::new(msgs, self.next_offset)));
        try!(self.index.append(off, pos));

        self.next_offset += 1;
        Ok(off)
    }

}

#[cfg(test)]
mod tests {
    use std::fs;
    use std::io::Write;
    use super::*;

    #[test]
    fn message_construction() {
        let msg = Message::new(b"123456789".to_vec());
        assert_eq!(msg.payload(), b"123456789");
        assert_eq!(msg.crc(), 0xcbf43926);
    }

    #[test]
    fn index() {
        let index_path = "target/.test-index";

        // remove before running (may be around from failing test run)
        fs::remove_file(index_path).unwrap_or(());

        {
            let mut index = Index::new(index_path, 1000u64, 10u64).unwrap();
            assert_eq!(1000, index.len());
            index.append(11u64, 0xffff).unwrap();
            index.append(12u64, 0xeeee).unwrap();
            index.flush_sync().unwrap();
        }

        // reopen it for read
        {
            let index = Index::new(index_path, 1000u64, 10u64).unwrap();

            let e0 = index.read_entry(0).unwrap().unwrap();
            assert_eq!(1u32, e0.relative_offset());
            assert_eq!(11u64, e0.offset());
            assert_eq!(0xffff, e0.file_position());

            let e1 = index.read_entry(1).unwrap().unwrap();
            assert_eq!(2u32, e1.relative_offset());
            assert_eq!(12u64, e1.offset());
            assert_eq!(0xeeee, e1.file_position());

            // read an entry that does not exist
            let e2 = index.read_entry(2).unwrap();
            assert_eq!(None, e2);
        }
        fs::remove_file(index_path).unwrap();
    }

    #[test]
    pub fn log() {
        let log_path = "target/.test-log";
        fs::remove_file(log_path);
        {
            let mut f = Log::new(log_path, 1024).unwrap();

            let m0 = Message::new(b"12345".to_vec());
            let ms0 = MessageSet::new(vec![m0], 1000);
            assert_eq!(ms0.bytes().len(), 21);
            let p0 = f.append(ms0).unwrap();
            assert_eq!(p0, 0);

            let m1 = Message::new(b"66666".to_vec());
            let m2 = Message::new(b"77777".to_vec());
            let p1 = f.append(MessageSet::new(vec![m1, m2], 1001)).unwrap();
            assert_eq!(p1, 21);

            f.flush().unwrap();
        }
        fs::remove_file(log_path);
    }
}
<|endoftext|>


<commit_before>#![feature(plugin_registrar, quote, rustc_private, custom_attribute)]

extern crate rustc_plugin;
extern crate syntax;

use rustc_plugin::registry::Registry;
use syntax::ast::{Attribute, Block, Expr, ExprKind, Ident, Item, ItemKind, Mac,
                  MetaItem, MetaItemKind};
use syntax::fold::{self, Folder};
use syntax::ptr::P;
use syntax::codemap::{DUMMY_SP, Span};
use syntax::ext::base::{Annotatable, ExtCtxt, SyntaxExtension};
use syntax::ext::build::AstBuilder;
use syntax::feature_gate::AttributeType;
use syntax::parse::token;
use syntax::util::small_vector::SmallVector;

pub fn insert_flame_guard(cx: &mut ExtCtxt, _span: Span, _mi: &MetaItem,
                          a: Annotatable) -> Annotatable {
    match a {
        Annotatable::Item(i) => Annotatable::Item(
            Flamer { cx: cx, ident: i.ident }.fold_item(i).expect_one("expected exactly one item")),
        Annotatable::TraitItem(i) => Annotatable::TraitItem(
            i.map(|i| Flamer { cx: cx, ident: i.ident }.fold_trait_item(i).expect_one("expected exactly one item"))),
        Annotatable::ImplItem(i) => Annotatable::ImplItem(
            i.map(|i| Flamer { cx: cx, ident: i.ident }.fold_impl_item(i).expect_one("expected exactly one item"))),
    }
}

struct Flamer<'a, 'cx: 'a> {
    ident: Ident,
    cx: &'a mut ExtCtxt<'cx>,
}

impl<'a, 'cx> Folder for Flamer<'a, 'cx> {
    fn fold_item(&mut self, item: P<Item>) -> SmallVector<P<Item>> {
        if let ItemKind::Mac(_) = item.node {
            let expanded = self.cx.expander().fold_item(item);
            expanded.into_iter()
                    .flat_map(|i| fold::noop_fold_item(i, self).into_iter())
                    .collect()
        } else {
            fold::noop_fold_item(item, self)
        }
    }

    fn fold_item_simple(&mut self, i: Item) -> Item {
        fn is_flame_annotation(attr: &Attribute) -> bool {
            if let MetaItemKind::Word(ref name) = attr.node.value.node {
                name == "flame" || name == "noflame"
            } else {
                false
            }
        }
        // don't double-flame nested annotations
        if i.attrs.iter().any(is_flame_annotation) { return i; }
        if let ItemKind::Mac(_) = i.node {
            return i;
        }else {
            self.ident = i.ident; // update in case of nested items
            fold::noop_fold_item_simple(i, self)
        }
    }

    fn fold_block(&mut self, block: P<Block>) -> P<Block> {
        block.map(|block| {
            let name = self.cx.expr_str(DUMMY_SP, self.ident.name.as_str());
            quote_block!(self.cx, {
                let g = ::flame::start_guard($name);
                let r = $block;
                g.end();
                r
            }).unwrap()
        })
    }

    fn fold_expr(&mut self, expr: P<Expr>) -> P<Expr> {
        if let ExprKind::Mac(_) = expr.node {
            self.cx.expander().fold_expr(expr)
                              .map(|e| fold::noop_fold_expr(e, self))
        } else {
            expr
        }
    }

    fn fold_mac(&mut self, mac: Mac) -> Mac {
        mac
    }
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_syntax_extension(token::intern("flame"),
        SyntaxExtension::MultiModifier(Box::new(insert_flame_guard)));
    reg.register_attribute(String::from("noflame"), AttributeType::Whitelisted);
}
<commit_msg>switch from parse to symbol<commit_after>#![feature(plugin_registrar, quote, rustc_private, custom_attribute)]

extern crate rustc_plugin;
extern crate syntax;

use rustc_plugin::registry::Registry;
use syntax::ast::{Attribute, Block, Expr, ExprKind, Ident, Item, ItemKind, Mac,
                  MetaItem, MetaItemKind};
use syntax::fold::{self, Folder};
use syntax::ptr::P;
use syntax::codemap::{DUMMY_SP, Span};
use syntax::ext::base::{Annotatable, ExtCtxt, SyntaxExtension};
use syntax::ext::build::AstBuilder;
use syntax::feature_gate::AttributeType;
use syntax::symbol::Symbol;
use syntax::util::small_vector::SmallVector;

pub fn insert_flame_guard(cx: &mut ExtCtxt, _span: Span, _mi: &MetaItem,
                          a: Annotatable) -> Annotatable {
    match a {
        Annotatable::Item(i) => Annotatable::Item(
            Flamer { cx: cx, ident: i.ident }.fold_item(i).expect_one("expected exactly one item")),
        Annotatable::TraitItem(i) => Annotatable::TraitItem(
            i.map(|i| Flamer { cx: cx, ident: i.ident }.fold_trait_item(i).expect_one("expected exactly one item"))),
        Annotatable::ImplItem(i) => Annotatable::ImplItem(
            i.map(|i| Flamer { cx: cx, ident: i.ident }.fold_impl_item(i).expect_one("expected exactly one item"))),
    }
}

struct Flamer<'a, 'cx: 'a> {
    ident: Ident,
    cx: &'a mut ExtCtxt<'cx>,
}

impl<'a, 'cx> Folder for Flamer<'a, 'cx> {
    fn fold_item(&mut self, item: P<Item>) -> SmallVector<P<Item>> {
        if let ItemKind::Mac(_) = item.node {
            let expanded = self.cx.expander().fold_item(item);
            expanded.into_iter()
                    .flat_map(|i| fold::noop_fold_item(i, self).into_iter())
                    .collect()
        } else {
            fold::noop_fold_item(item, self)
        }
    }

    fn fold_item_simple(&mut self, i: Item) -> Item {
        fn is_flame_annotation(attr: &Attribute) -> bool {
            if let MetaItemKind::Word(ref name) = attr.node.value.node {
                name == "flame" || name == "noflame"
            } else {
                false
            }
        }
        // don't double-flame nested annotations
        if i.attrs.iter().any(is_flame_annotation) { return i; }
        if let ItemKind::Mac(_) = i.node {
            return i;
        }else {
            self.ident = i.ident; // update in case of nested items
            fold::noop_fold_item_simple(i, self)
        }
    }

    fn fold_block(&mut self, block: P<Block>) -> P<Block> {
        block.map(|block| {
            let name = self.cx.expr_str(DUMMY_SP, self.ident.name.as_str());
            quote_block!(self.cx, {
                let g = ::flame::start_guard($name);
                let r = $block;
                g.end();
                r
            }).unwrap()
        })
    }

    fn fold_expr(&mut self, expr: P<Expr>) -> P<Expr> {
        if let ExprKind::Mac(_) = expr.node {
            self.cx.expander().fold_expr(expr)
                              .map(|e| fold::noop_fold_expr(e, self))
        } else {
            expr
        }
    }

    fn fold_mac(&mut self, mac: Mac) -> Mac {
        mac
    }
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_syntax_extension(Symbol::intern("flame"),
        SyntaxExtension::MultiModifier(Box::new(insert_flame_guard)));
    reg.register_attribute(String::from("noflame"), AttributeType::Whitelisted);
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017-2019 Ben Ashford
 *
 * Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 * http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
 * <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 * option. This file may not be copied, modified, or distributed
 * except according to those terms.
 */

//! A client for Redis using Tokio and Futures.
//!
//! Three interfaces are provided: one low-level, that makes no assumptions about how Redis is used; a high-level client,
//! suitable for the vast majority of use-cases; a PUBSUB client specifically for Redis's PUBSUB functionality.
//!
//! ## Low-level
//!
//! [`client::connect`](client/connect/fn.connect.html) returns a pair of `Sink` and `Stream` (see [futures](https://github.com/alexcrichton/futures-rs)) which
//! both transport [`resp::RespValue`](resp/enum.RespValue.html)s between client and Redis, these work independently of one another
//! to allow pipelining.  It is the responsibility of the caller to match responses to requests.  It is also the
//! responsibility of the client to convert application data into instances of [`resp::RespValue`](resp/enum.RespValue.html) and
//! back (there are conversion traits available for common examples).
//!
//! This is a very low-level API compared to most Redis clients, but is done so intentionally, for two reasons: 1) it is
//! the common demoniator between a functional Redis client (i.e. is able to support all types of requests, including those
//! that block and have streaming responses), and 2) it results in clean `Sink`s and `Stream`s which will be composable
//! with other Tokio-based libraries.
//!
//! For most practical purposes this low-level interface will not be used, the only exception possibly being the
//! [`MONITOR`](https://redis.io/commands/monitor) command.
//!
//! ## High-level
//!
//! [`client::paired_connect`](client/paired/fn.paired_connect.html) is used for most Redis commands (those for which one command
//! returns one response, it's not suitable for PUBSUB, `MONITOR` or other similar commands).  It allows a Redis command to
//! be sent and a Future returned for each command.
//!
//! Commands will be sent in the order that [`send`](client/paired/struct.PairedConnection.html#method.send) is called, regardless
//! of how the future is realised.  This is to allow us to take advantage of Redis's features by implicitly pipelining
//! commands where appropriate.  One side-effect of this is that for many commands, e.g. `SET` we don't need to realise the
//! future at all, it can be assumed to be fire-and-forget; but, the final future of the final command does need to be
//! realised (at least) to ensure that the correct behaviour is observed.
//!
//! ## PUBSUB
//!
//! PUBSUB in Redis works differently.  A connection will subscribe to one or more topics, then receive all messages that
//! are published to that topic.  As such the single-request/single-response model of
//! [`paired_connect`](client/paired/fn.paired_connect.html) will not work.  A specific
//! [`client::pubsub_connect`](client/pubsub/fn.pubsub_connect.html) is provided for this purpose.
//!
//! It returns a future which resolves to a [`PubsubConnection`](client/pubsub/struct.PubsubConnection.html), this provides a
//! [`subscribe`](client/pubsub/struct.PubsubConnection.html#method.subscribe) function that takes a topic as a parameter and
//! returns a future which, once the subscription is confirmed, resolves to a stream that contains all messages published
//! to that topic.
#[macro_use]
pub mod resp;

#[macro_use]
pub mod client;

pub mod error;

pub mod reconnect;
<commit_msg>Make reconnect crate public<commit_after>/*
 * Copyright 2017-2019 Ben Ashford
 *
 * Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 * http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
 * <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 * option. This file may not be copied, modified, or distributed
 * except according to those terms.
 */

//! A client for Redis using Tokio and Futures.
//!
//! Three interfaces are provided: one low-level, that makes no assumptions about how Redis is used; a high-level client,
//! suitable for the vast majority of use-cases; a PUBSUB client specifically for Redis's PUBSUB functionality.
//!
//! ## Low-level
//!
//! [`client::connect`](client/connect/fn.connect.html) returns a pair of `Sink` and `Stream` (see [futures](https://github.com/alexcrichton/futures-rs)) which
//! both transport [`resp::RespValue`](resp/enum.RespValue.html)s between client and Redis, these work independently of one another
//! to allow pipelining.  It is the responsibility of the caller to match responses to requests.  It is also the
//! responsibility of the client to convert application data into instances of [`resp::RespValue`](resp/enum.RespValue.html) and
//! back (there are conversion traits available for common examples).
//!
//! This is a very low-level API compared to most Redis clients, but is done so intentionally, for two reasons: 1) it is
//! the common demoniator between a functional Redis client (i.e. is able to support all types of requests, including those
//! that block and have streaming responses), and 2) it results in clean `Sink`s and `Stream`s which will be composable
//! with other Tokio-based libraries.
//!
//! For most practical purposes this low-level interface will not be used, the only exception possibly being the
//! [`MONITOR`](https://redis.io/commands/monitor) command.
//!
//! ## High-level
//!
//! [`client::paired_connect`](client/paired/fn.paired_connect.html) is used for most Redis commands (those for which one command
//! returns one response, it's not suitable for PUBSUB, `MONITOR` or other similar commands).  It allows a Redis command to
//! be sent and a Future returned for each command.
//!
//! Commands will be sent in the order that [`send`](client/paired/struct.PairedConnection.html#method.send) is called, regardless
//! of how the future is realised.  This is to allow us to take advantage of Redis's features by implicitly pipelining
//! commands where appropriate.  One side-effect of this is that for many commands, e.g. `SET` we don't need to realise the
//! future at all, it can be assumed to be fire-and-forget; but, the final future of the final command does need to be
//! realised (at least) to ensure that the correct behaviour is observed.
//!
//! ## PUBSUB
//!
//! PUBSUB in Redis works differently.  A connection will subscribe to one or more topics, then receive all messages that
//! are published to that topic.  As such the single-request/single-response model of
//! [`paired_connect`](client/paired/fn.paired_connect.html) will not work.  A specific
//! [`client::pubsub_connect`](client/pubsub/fn.pubsub_connect.html) is provided for this purpose.
//!
//! It returns a future which resolves to a [`PubsubConnection`](client/pubsub/struct.PubsubConnection.html), this provides a
//! [`subscribe`](client/pubsub/struct.PubsubConnection.html#method.subscribe) function that takes a topic as a parameter and
//! returns a future which, once the subscription is confirmed, resolves to a stream that contains all messages published
//! to that topic.
#[macro_use]
pub mod resp;

#[macro_use]
pub mod client;

pub mod error;

pub(crate) mod reconnect;
<|endoftext|>


<commit_before>//! Quick Csv reader which performs **very** well.
//!
//! ## Example
//!
//! First, create a `Csv` from a `BufRead` reader, a file or a string
//!
//! ```rust
//! extern crate quick_csv;
//!
//! fn main() {
//!     let data = "a,b\r\nc,d\r\ne,f";
//!     let csv = quick_csv::Csv::from_string(data);
//!     for row in csv.into_iter() {
//!         // work on csv row ...
//!         if let Ok(_) = row {
//!             println!("new row!");
//!         } else {
//!             println!("cannot read next line");
//!         }
//!     }
//! }
//! ```
//!
//! `Row` is on the other hand provides 3 methods to access csv columns:
//! - `columns`:
//!   - iterator over columns.
//!   - Iterator item is a `&str`, which means you only have to `parse()` it to the needed type and you're done
//!
//!   ```rust
//!   # let row = quick_csv::Csv::from_string("a,b,c,d,e,38,f").next().unwrap().unwrap();
//!   let mut cols = row.columns().expect("cannot convert to utf8");
//!   let fifth = cols.nth(5).unwrap().parse::<f64>().unwrap();
//!   println!("Doubled fifth column: {}", fifth * 2.0);
//!   ```
//!
//! - `decode`:
//!   - deserialize into you `Decodable` struct, a-la rust-csv.
//!   - most convenient way to deal with your csv data
//!
//!   ```rust
//!   let row = quick_csv::Csv::from_string("a,b,54").next().unwrap().unwrap();
//!   if let Ok((col1, col2, col3)) = row.decode::<(String, u64, f64)>() {
//!       println!("col1: '{}', col2: {}, col3: {}", col1, col2, col3);
//!   }
//!   ```
//!
//! - `bytes_columns`:
//!   - similar to `columns` but columns are of type `&[u8]`, which means you may want to convert it to &str first
//!   - performance gain compared to `columns` is minimal, use it only if you *really* need to as it is less convenient

#![deny(missing_docs)]

extern crate rustc_serialize;

pub mod columns;
pub mod error;

use self::columns::{Columns, BytesColumns};
use std::fs::File;
use std::io::{self, BufRead, BufReader};
use std::iter::Iterator;
use std::path::Path;

use error::{Error, Result};
use rustc_serialize::Decodable;

#[cfg(test)] mod test;

/// Csv reader
///
/// Iterates over the rows of the csv
///
/// # Example
///
/// ```rust
/// let csv = quick_csv::Csv::from_file("./examples/data/bench.csv").unwrap();
/// for row in csv.into_iter() {
///     let row = row.unwrap(); // unwrap result, panic if not utf8
///     {
///         // either use columns iterator directly (Item = &str)
///         if let Ok(mut columns) = row.columns() {
///             println!("Column 1: '{:?}', Column 2: '{:?}'", columns.next(), columns.next());
///         }
///     }
///
///     {
///         // or decode it directly into something simpler
///         if let Ok((col1, col2)) = row.decode::<(String, u64)>() {
///             println!("Column 1: '{:?}', Column 2: '{:?}'", &col1, &col2);
///         }
///     }
///
/// }
/// ```
pub struct Csv<B: BufRead> {
    /// delimiter
    delimiter: u8,
    /// reader
    reader: B,
    /// header
    has_header: bool,
    /// header
    headers: Option<Vec<String>>,
    /// flexible column count
    flexible: bool,
    /// column count
    len: Option<usize>,
    /// if was error, exit next
    exit: bool,
    /// line count
    current_line: usize,
}

impl<B: BufRead> Csv<B> {

    /// Creates a Csv from a generic BufReader
    ///
    /// Note: default delimiter = ','
    pub fn from_reader(reader: B) -> Csv<B> {
        Csv {
            reader: reader,
            delimiter: b',',
            has_header: false,
            headers: None,
            flexible: false,
            len: None,
            exit: false,
            current_line: 0,
        }
    }

    /// Sets a new delimiter
    pub fn delimiter(mut self, delimiter: u8) -> Csv<B> {
        self.delimiter = delimiter;
        self
    }

    /// Sets flexible columns
    pub fn flexible(mut self, flexible: bool) -> Csv<B> {
        self.flexible = flexible;
        self
    }

    /// Defines whether there is a header or not
    pub fn has_header(mut self, has_header: bool) -> Csv<B> {
        self.has_header = has_header;
        let _ = self.headers();
        self
    }

   /// gets first row as Vec<String>
    pub fn headers(&mut self) -> Vec<String> {
        if let Some(ref h) = self.headers {
            return h.clone();
        }
        if self.has_header {
            if let Some(r) = self.next() {
                if let Ok(r) = r {
                    let h = r.decode().ok().unwrap_or_else(Vec::new);
                    self.headers = Some(h.clone());
                    return h;
                }
            }
        }
        Vec::new()
    }

    /// Get column count
    pub fn column_count(&self) -> Option<usize> {
        self.len
    }

    /// Gets the current line number
    ///
    /// Useful if you get an error and want to investigate the source
    pub fn current_line(&self) -> usize {
        self.current_line
    }

}

impl Csv<BufReader<File>> {
    /// Creates a csv from a file path
    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Csv<BufReader<File>>>
    {
        let reader = BufReader::new(try!(File::open(path)));
        Ok(Csv::from_reader(reader))
    }
}

impl<'a> Csv<&'a [u8]> {
    /// Creates a CSV reader for an in memory string buffer.
    pub fn from_string(s: &'a str) -> Csv<&'a [u8]> {
        Csv::from_reader(s.as_bytes())
    }
}

/// Iterator on csv `Row`s
impl<B: BufRead> Iterator for Csv<B> {
    type Item = Result<Row>;
    fn next(&mut self) -> Option<Result<Row>> {
        if self.exit { return None; }
        let mut buf = Vec::new();
        let mut cols = self.len.map_or_else(Vec::new, Vec::with_capacity);
        match read_line(&mut self.reader, &mut buf, self.delimiter, &mut cols, self.len == None) {
            Ok(0) => None,
            Ok(_n) => {
                if buf.ends_with(&[b'\r']) {
                    buf.pop();
                }
                cols.push(buf.len());
                let c = cols.len();
                if let Some(n) = self.len {
                    if n != c && !self.flexible {
                        self.exit = true;
                        return Some(Err(Error::ColumnMismatch(n, c)));
                    }
                } else {
                    self.len = Some(c);
                }
                self.current_line += 1;
                Some(Ok(Row {
                    line: buf,
                    cols: cols,
                }))
            }
            Err(e) => {
                self.exit = true;
                Some(Err(e))
            },
        }
    }
}

/// Row struct used as Csv iterator Item
///
/// Row can be decoded into a Result<T: Decodable>
pub struct Row {
    line: Vec<u8>,
    cols: Vec<usize>,
}

impl Row {

    /// Gets an iterator over columns
    pub fn columns(&self) -> Result<Columns> {
        match ::std::str::from_utf8(&self.line) {
            Err(_) => Err(Error::Io(io::Error::new(io::ErrorKind::InvalidData,
                                    "stream did not contain valid UTF-8"))),
            Ok(s) => Ok(Columns::new(s, &self.cols)),
        }
    }

    ///  Creates a new BytesColumns iterator over &[u8]
    pub fn bytes_columns(&self) -> BytesColumns {
        BytesColumns::new(&self.line, &self.cols)
    }

    /// Decode row into custom decodable type
    pub fn decode<T: Decodable>(&self) -> Result<T> {
        let mut columns = try!(self.columns());
        Decodable::decode(&mut columns)
    }

    /// Gets columns count
    pub fn len(&self) -> usize {
        self.cols.len()
    }

    /// `Row` is empty if there is no columns
    pub fn is_empty(&self) -> bool {
        self.cols.is_empty()
    }

}

/// Consumes bytes as long as they are within quotes
/// manages "" as quote escape
/// returns
/// - Ok(true) if entirely consumed
/// - Ok(false) if no issue but it reached end of buffer
/// - Err(Error::UnescapeQuote) if a quote if found within the column
macro_rules! consume_quote {
    ($bytes: expr, $delimiter: expr, $in_quote: expr,
     $start: expr, $buf: expr, $available: expr, $quote_count: expr) => {
        $in_quote = false;
        loop {
            match $bytes.next() {
                Some((_, &b'\"')) => {
                    match $bytes.clone().next() {
                        Some((i, &b'\"')) => {
                            $bytes.next(); // escaping quote
                            $buf.extend_from_slice(&$available[$start..i]);
                            $start = i + 1;
                            $quote_count += 1;
                        },
                        None | Some((_, &b'\r')) | Some((_, &b'\n')) => break,
                        Some((_, d)) if *d == $delimiter => break,
                        Some((_, _)) => return Err(Error::UnescapedQuote),
                    }
                },
                None => {
                    $in_quote = true;
                    break;
                },
                _ => (),
            }
        }
    }
}

/// Reads an entire line into memory
fn read_line<R: BufRead>(r: &mut R, buf: &mut Vec<u8>,
    delimiter: u8, cols: &mut Vec<usize>, first_read: bool) -> Result<usize>
{
    const UTF8_BOM: &'static [u8] = b"\xef\xbb\xbf";
    let mut read = 0;
    let mut in_quote = false;
    let mut done = false;
    let mut quote_count = 0;
    while !done {
        let used = {
            let mut available = match r.fill_buf() {
                Ok(n) if n.is_empty() => return Ok(read),
                Ok(n) => n,
                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
                Err(e) => return Err(Error::from(e)),
            };

            // check if the file starts with a UTF8 BOM and skip it if present
            if first_read && available.starts_with(UTF8_BOM) {
                available = &available[UTF8_BOM.len()..];
            }

            let mut bytes = available.iter().enumerate();
            let mut start = 0;

            // previous buffer was exhausted without exiting from quotes
            if in_quote {
                consume_quote!(bytes, delimiter, in_quote, start, buf, available, quote_count);
            }

            // use a simple loop instead of for loop to allow nested loop
            let used: usize;
            loop {
                match bytes.next() {
                    Some((i, &b'\"')) => {
                        if i == 0 || available[i - 1] == delimiter {
                            consume_quote!(bytes, delimiter, in_quote, start, buf, available, quote_count);
                        } else {
                            return Err(Error::UnexpextedQuote);
                        }
                    },
                    Some((i, &b'\n')) => {
                        done = true;
                        used = i + 1;
                        buf.extend_from_slice(&available[start..i]);
                        break;
                    },
                    Some((i, &d)) => {
                        if d == delimiter { cols.push(read + i - quote_count); }
                    },
                    None => {
                        used = available.len();
                        buf.extend_from_slice(&available[start..used]);
                        break;
                    },
                }
            }
            used
        };
        r.consume(used);
        read += used;
    }
    Ok(read)
}
<commit_msg>Only try to skip the BOM once<commit_after>//! Quick Csv reader which performs **very** well.
//!
//! ## Example
//!
//! First, create a `Csv` from a `BufRead` reader, a file or a string
//!
//! ```rust
//! extern crate quick_csv;
//!
//! fn main() {
//!     let data = "a,b\r\nc,d\r\ne,f";
//!     let csv = quick_csv::Csv::from_string(data);
//!     for row in csv.into_iter() {
//!         // work on csv row ...
//!         if let Ok(_) = row {
//!             println!("new row!");
//!         } else {
//!             println!("cannot read next line");
//!         }
//!     }
//! }
//! ```
//!
//! `Row` is on the other hand provides 3 methods to access csv columns:
//! - `columns`:
//!   - iterator over columns.
//!   - Iterator item is a `&str`, which means you only have to `parse()` it to the needed type and you're done
//!
//!   ```rust
//!   # let row = quick_csv::Csv::from_string("a,b,c,d,e,38,f").next().unwrap().unwrap();
//!   let mut cols = row.columns().expect("cannot convert to utf8");
//!   let fifth = cols.nth(5).unwrap().parse::<f64>().unwrap();
//!   println!("Doubled fifth column: {}", fifth * 2.0);
//!   ```
//!
//! - `decode`:
//!   - deserialize into you `Decodable` struct, a-la rust-csv.
//!   - most convenient way to deal with your csv data
//!
//!   ```rust
//!   let row = quick_csv::Csv::from_string("a,b,54").next().unwrap().unwrap();
//!   if let Ok((col1, col2, col3)) = row.decode::<(String, u64, f64)>() {
//!       println!("col1: '{}', col2: {}, col3: {}", col1, col2, col3);
//!   }
//!   ```
//!
//! - `bytes_columns`:
//!   - similar to `columns` but columns are of type `&[u8]`, which means you may want to convert it to &str first
//!   - performance gain compared to `columns` is minimal, use it only if you *really* need to as it is less convenient

#![deny(missing_docs)]

extern crate rustc_serialize;

pub mod columns;
pub mod error;

use self::columns::{Columns, BytesColumns};
use std::fs::File;
use std::io::{self, BufRead, BufReader};
use std::iter::Iterator;
use std::path::Path;

use error::{Error, Result};
use rustc_serialize::Decodable;

#[cfg(test)] mod test;

const UTF8_BOM: &'static [u8] = b"\xef\xbb\xbf";

/// Csv reader
///
/// Iterates over the rows of the csv
///
/// # Example
///
/// ```rust
/// let csv = quick_csv::Csv::from_file("./examples/data/bench.csv").unwrap();
/// for row in csv.into_iter() {
///     let row = row.unwrap(); // unwrap result, panic if not utf8
///     {
///         // either use columns iterator directly (Item = &str)
///         if let Ok(mut columns) = row.columns() {
///             println!("Column 1: '{:?}', Column 2: '{:?}'", columns.next(), columns.next());
///         }
///     }
///
///     {
///         // or decode it directly into something simpler
///         if let Ok((col1, col2)) = row.decode::<(String, u64)>() {
///             println!("Column 1: '{:?}', Column 2: '{:?}'", &col1, &col2);
///         }
///     }
///
/// }
/// ```
pub struct Csv<B: BufRead> {
    /// delimiter
    delimiter: u8,
    /// reader
    reader: B,
    /// header
    has_header: bool,
    /// header
    headers: Option<Vec<String>>,
    /// flexible column count
    flexible: bool,
    /// column count
    len: Option<usize>,
    /// if was error, exit next
    exit: bool,
    /// line count
    current_line: usize,
}

impl<B: BufRead> Csv<B> {

    /// Creates a Csv from a generic BufReader
    ///
    /// Note: default delimiter = ','
    pub fn from_reader(mut reader: B) -> Csv<B> {
        let result = try_consume_utf8_bom(&mut reader);

        Csv {
            reader: reader,
            delimiter: b',',
            has_header: false,
            headers: None,
            flexible: false,
            len: None,
            exit: result.is_err(),
            current_line: 0,
        }
    }

    /// Sets a new delimiter
    pub fn delimiter(mut self, delimiter: u8) -> Csv<B> {
        self.delimiter = delimiter;
        self
    }

    /// Sets flexible columns
    pub fn flexible(mut self, flexible: bool) -> Csv<B> {
        self.flexible = flexible;
        self
    }

    /// Defines whether there is a header or not
    pub fn has_header(mut self, has_header: bool) -> Csv<B> {
        self.has_header = has_header;
        let _ = self.headers();
        self
    }

   /// gets first row as Vec<String>
    pub fn headers(&mut self) -> Vec<String> {
        if let Some(ref h) = self.headers {
            return h.clone();
        }
        if self.has_header {
            if let Some(r) = self.next() {
                if let Ok(r) = r {
                    let h = r.decode().ok().unwrap_or_else(Vec::new);
                    self.headers = Some(h.clone());
                    return h;
                }
            }
        }
        Vec::new()
    }

    /// Get column count
    pub fn column_count(&self) -> Option<usize> {
        self.len
    }

    /// Gets the current line number
    ///
    /// Useful if you get an error and want to investigate the source
    pub fn current_line(&self) -> usize {
        self.current_line
    }

    fn try_consume_utf8_bom(reader: &mut B) -> Result<()> {
        if try!(reader.fill_buf()).starts_with(UTF8_BOM) {
            reader.consume(UTF8_BOM.len());
        }

        Ok(())
    }
}

impl Csv<BufReader<File>> {
    /// Creates a csv from a file path
    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Csv<BufReader<File>>>
    {
        let reader = BufReader::new(try!(File::open(path)));
        Ok(Csv::from_reader(reader))
    }
}

impl<'a> Csv<&'a [u8]> {
    /// Creates a CSV reader for an in memory string buffer.
    pub fn from_string(s: &'a str) -> Csv<&'a [u8]> {
        Csv::from_reader(s.as_bytes())
    }
}

/// Iterator on csv `Row`s
impl<B: BufRead> Iterator for Csv<B> {
    type Item = Result<Row>;
    fn next(&mut self) -> Option<Result<Row>> {
        if self.exit { return None; }
        let mut buf = Vec::new();
        let mut cols = self.len.map_or_else(Vec::new, Vec::with_capacity);
        match read_line(&mut self.reader, &mut buf, self.delimiter, &mut cols) {
            Ok(0) => None,
            Ok(_n) => {
                if buf.ends_with(&[b'\r']) {
                    buf.pop();
                }
                cols.push(buf.len());
                let c = cols.len();
                if let Some(n) = self.len {
                    if n != c && !self.flexible {
                        self.exit = true;
                        return Some(Err(Error::ColumnMismatch(n, c)));
                    }
                } else {
                    self.len = Some(c);
                }
                self.current_line += 1;
                Some(Ok(Row {
                    line: buf,
                    cols: cols,
                }))
            }
            Err(e) => {
                self.exit = true;
                Some(Err(e))
            },
        }
    }
}

/// Row struct used as Csv iterator Item
///
/// Row can be decoded into a Result<T: Decodable>
pub struct Row {
    line: Vec<u8>,
    cols: Vec<usize>,
}

impl Row {

    /// Gets an iterator over columns
    pub fn columns(&self) -> Result<Columns> {
        match ::std::str::from_utf8(&self.line) {
            Err(_) => Err(Error::Io(io::Error::new(io::ErrorKind::InvalidData,
                                    "stream did not contain valid UTF-8"))),
            Ok(s) => Ok(Columns::new(s, &self.cols)),
        }
    }

    ///  Creates a new BytesColumns iterator over &[u8]
    pub fn bytes_columns(&self) -> BytesColumns {
        BytesColumns::new(&self.line, &self.cols)
    }

    /// Decode row into custom decodable type
    pub fn decode<T: Decodable>(&self) -> Result<T> {
        let mut columns = try!(self.columns());
        Decodable::decode(&mut columns)
    }

    /// Gets columns count
    pub fn len(&self) -> usize {
        self.cols.len()
    }

    /// `Row` is empty if there is no columns
    pub fn is_empty(&self) -> bool {
        self.cols.is_empty()
    }

}

/// Consumes bytes as long as they are within quotes
/// manages "" as quote escape
/// returns
/// - Ok(true) if entirely consumed
/// - Ok(false) if no issue but it reached end of buffer
/// - Err(Error::UnescapeQuote) if a quote if found within the column
macro_rules! consume_quote {
    ($bytes: expr, $delimiter: expr, $in_quote: expr,
     $start: expr, $buf: expr, $available: expr, $quote_count: expr) => {
        $in_quote = false;
        loop {
            match $bytes.next() {
                Some((_, &b'\"')) => {
                    match $bytes.clone().next() {
                        Some((i, &b'\"')) => {
                            $bytes.next(); // escaping quote
                            $buf.extend_from_slice(&$available[$start..i]);
                            $start = i + 1;
                            $quote_count += 1;
                        },
                        None | Some((_, &b'\r')) | Some((_, &b'\n')) => break,
                        Some((_, d)) if *d == $delimiter => break,
                        Some((_, _)) => return Err(Error::UnescapedQuote),
                    }
                },
                None => {
                    $in_quote = true;
                    break;
                },
                _ => (),
            }
        }
    }
}

/// Reads an entire line into memory
fn read_line<R: BufRead>(r: &mut R, buf: &mut Vec<u8>,
                         delimiter: u8, cols: &mut Vec<usize>) -> Result<usize>
{
    let mut read = 0;
    let mut in_quote = false;
    let mut done = false;
    let mut quote_count = 0;
    while !done {
        let used = {
            let available = match r.fill_buf() {
                Ok(n) if n.is_empty() => return Ok(read),
                Ok(n) => n,
                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
                Err(e) => return Err(Error::from(e)),
            };

            let mut bytes = available.iter().enumerate();
            let mut start = 0;

            // previous buffer was exhausted without exiting from quotes
            if in_quote {
                consume_quote!(bytes, delimiter, in_quote, start, buf, available, quote_count);
            }

            // use a simple loop instead of for loop to allow nested loop
            let used: usize;
            loop {
                match bytes.next() {
                    Some((i, &b'\"')) => {
                        if i == 0 || available[i - 1] == delimiter {
                            consume_quote!(bytes, delimiter, in_quote, start, buf, available, quote_count);
                        } else {
                            return Err(Error::UnexpextedQuote);
                        }
                    },
                    Some((i, &b'\n')) => {
                        done = true;
                        used = i + 1;
                        buf.extend_from_slice(&available[start..i]);
                        break;
                    },
                    Some((i, &d)) => {
                        if d == delimiter { cols.push(read + i - quote_count); }
                    },
                    None => {
                        used = available.len();
                        buf.extend_from_slice(&available[start..used]);
                        break;
                    },
                }
            }
            used
        };
        r.consume(used);
        read += used;
    }
    Ok(read)
}

fn try_consume_utf8_bom<B: BufRead>(reader: &mut B) -> Result<()> {
    if try!(reader.fill_buf()).starts_with(UTF8_BOM) {
        reader.consume(UTF8_BOM.len());
    }

    Ok(())
}<|endoftext|>