<commit_before>// Copyright 2015 Nathan Sizemore <nathanrsizemore@gmail.com>
//
// This Source Code Form is subject to the terms of the
// Mozilla Public License, v. 2.0. If a copy of the MPL was not
// distributed with this file, You can obtain one at
// http://mozilla.org/MPL/2.0/.

use std::io::Error;
use std::ops::DerefMut;
use std::sync::{Arc, Mutex};
use std::{ptr, mem, thread};
use std::os::unix::io::{RawFd, AsRawFd};
use std::collections::LinkedList;

use libc;
use errno::errno;
use epoll;
use epoll::util::*;
use epoll::EpollEvent;
use stream::socket::Socket;
use stream::nbstream::Nbstream;
use stream::securestream::SecureStream;
use stream::{Stream, HRecv, HSend, CloneHStream};
use stats;
use types::*;
use resources::ResourcePool;
use libc::{c_ushort, c_ulong, c_uint, c_int, c_void};
use config::Config;
use openssl::ssl::SslContext;

extern "C" {
    fn shim_htons(hostshort: c_ushort) -> c_ushort;
    fn shim_htonl(addr: c_uint) -> c_uint;
    fn shim_inaddr_any() -> c_ulong;
}

// We need to be able to access our resource pool from several methods
static mut pool: *mut ResourcePool = 0 as *mut ResourcePool;

// Global SslContext
static mut ssl_context: *mut SslContext = 0 as *mut SslContext;

// When added to epoll, these will be the conditions of kernel notification:
//
// EPOLLET  - Fd is in EdgeTriggered mode (notification on state changes)
// EPOLLIN  - Data is available in kerndl buffer
const EVENTS: u32 = event_type::EPOLLET | event_type::EPOLLIN;

/// Starts the epoll wait and incoming connection listener threads.
pub fn begin(config: Config, handler: Box<EventHandler>) {
    // Master socket list
    let sockets = Arc::new(Mutex::new(LinkedList::<Stream>::new()));

    // Resource pool
    let mut rp = ResourcePool::new(config.workers);
    unsafe {
        pool = &mut rp;
    }

    // Wrap our event handler into something that can be safely shared
    // between threads.
    let e_handler = Handler(Box::into_raw(handler));

    // Epoll instance
    let result = epoll::create1(0);
    if result.is_err() {
        let err = result.unwrap_err();
        error!("Unable to create epoll instance: {}", err);
        panic!()
    }
    let epfd = result.unwrap();

    // Epoll wait thread
    let epfd2 = epfd.clone();
    let streams2 = sockets.clone();
    thread::Builder::new()
        .name("Epoll Wait".to_string())
        .spawn(move || {
            event_loop(epfd2, streams2, e_handler);
        })
        .unwrap();

    // New connection thread
    let epfd3 = epfd.clone();
    let streams3 = sockets.clone();
    let prox = thread::Builder::new()
                   .name("TCP Incoming Listener".to_string())
                   .spawn(move || {
                       listen(config, epfd3, streams3);
                   })
                   .unwrap();

    // Stay alive forever, or at least we hope
    let _ = prox.join();
}

/// Incoming connection listening thread
fn listen(config: Config, epfd: RawFd, streams: StreamList) {
    unsafe {
        let server_addr = libc::sockaddr_in {
            sin_family: libc::AF_INET as u16,
            sin_port: shim_htons(config.port),
            sin_addr: libc::in_addr { s_addr: shim_htonl(shim_inaddr_any() as u32) },
            sin_zero: [0u8; 8],
        };

        // Create a server socket with the passed info for TCP/IP
        let server_fd = libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0);

        // Set the SO_REUSEADDR so that restarts after crashes do not take 5min to unbind
        // the initial port
        let optval: c_int = 1;
        let opt_result = libc::setsockopt(server_fd,
                                          libc::SOL_SOCKET,
                                          libc::SO_REUSEADDR,
                                          &optval as *const _ as *const c_void,
                                          mem::size_of::<c_int>() as u32);
        if opt_result < 0 {
            error!("Setting SO_REUSEADDR: {}",
                   Error::from_raw_os_error(errno().0 as i32));
            return;
        }

        let serv_addr: *const libc::sockaddr = mem::transmute(&server_addr);
        let bind_result = libc::bind(server_fd,
                                     serv_addr,
                                     mem::size_of::<libc::sockaddr_in>() as u32);
        if bind_result < 0 {
            error!("Binding fd: {} - {}",
                   server_fd,
                   Error::from_raw_os_error(errno().0 as i32));
            return;
        }

        let listen_result = libc::listen(server_fd, 100);
        if listen_result < 0 {
            error!("Listening: {}", Error::from_raw_os_error(errno().0 as i32));
            return;
        }

        // If we're using SSL, setup our context
        let using_ssl;
        let mut config = config;
        match config.ssl.as_mut() {
            Some(context) => {
                ssl_context = context;
                using_ssl = true;
            }
            None => {
                using_ssl = false;
            }
        };

        loop {
            // Accept new client
            let result = libc::accept(server_fd,
                                      ptr::null_mut() as *mut libc::sockaddr,
                                      ptr::null_mut() as *mut u32);
            if result < 0 {
                error!("Accepting new connection: {}",
                       Error::from_raw_os_error(result as i32));
            }

            // Setup new socket
            let socket = Socket {
                fd: result
            };
            socket.set_tcp_keepalive(true);
            socket.set_tcp_nodelay(true);

            // Create new stream and add to server
            if using_ssl {
                let _ = SecureStream::new(&(*ssl_context), socket).map(|securestream| {
                    let stream = Stream {
                        inner: Box::new(securestream)
                    };
                    let fd = stream.as_raw_fd();
                    add_stream_to_master_list(stream, streams.clone());
                    add_to_epoll(epfd, fd, streams.clone());
                }).map_err(|e| {
                    // If we ever fail here, it is safe to assume everything else has gone to
                    // shit. No way to clean it up, so we'll just leave. The main process thread
                    // is joined to this tread, so panic-ing will cause our parent process
                    // to close out completely.
                    error!("Creating stream: {}", e);
                    panic!()
                });
            } else {
                let _ = Nbstream::new(socket).map(|nbstream| {
                    let stream = Stream {
                        inner: Box::new(nbstream)
                    };
                    let fd = stream.as_raw_fd();
                    add_stream_to_master_list(stream, streams.clone());
                    add_to_epoll(epfd, fd, streams.clone());
                }).map_err(|e| {
                    // If we ever fail here, it is safe to assume everything else has gone to
                    // shit. No way to clean it up, so we'll just leave. The main process thread
                    // is joined to this tread, so panic-ing will cause our parent process
                    // to close out completely.
                    error!("Creating stream: {}", e);
                    panic!()
                });
            }
        }
    }
}

/// Event loop for handling all epoll events
fn event_loop(epfd: RawFd, streams: StreamList, handler: Handler) {
    let mut events = Vec::<EpollEvent>::with_capacity(100);
    unsafe {
        events.set_len(100);
    }

    loop {
        match epoll::wait(epfd, &mut events[..], -1) {
            Ok(num_events) => {
                for x in 0..num_events as usize {
                    handle_epoll_event(epfd, &events[x], streams.clone(), handler.clone());
                }
            }
            Err(e) => {
                error!("Error on epoll::wait(): {}", e);
                panic!()
            }
        };
    }
}

/// Finds the stream the epoll event is associated with and parses the event type
/// to hand off to specific handlers
fn handle_epoll_event(epfd: RawFd, event: &EpollEvent, streams: StreamList, handler: Handler) {
    const READ_EVENT: u32 = event_type::EPOLLIN;

    // Locate the stream the event was for
    let mut stream;
    {
        // Mutex lock
        // Find the stream the event was for
        let mut guard = match streams.lock() {
            Ok(guard) => guard,
            Err(poisoned) => {
                warn!("StreamList Mutex was poisoned, using anyway");
                poisoned.into_inner()
            }
        };
        let list = guard.deref_mut();

        let mut found = false;
        let mut index = 1usize;
        for s in list.iter() {
            if s.as_raw_fd() == event.data as RawFd {
                found = true;
                break;
            }
            index += 1;
        }

        if !found {
            error!("Could not find fd in stream list? Removing from epoll and closing fd");

            let fd = event.data as RawFd;
            remove_fd_from_epoll(epfd, fd);
            close_fd(fd);
            return;
        }

        if index == 1 {
            stream = list.pop_front().unwrap();
        } else {
            let mut split = list.split_off(index - 1);
            stream = split.pop_front().unwrap();
            list.append(&mut split);
        }
    } // Mutex unlock

    if (event.events & READ_EVENT) > 0 {
        trace!("event was read event");
        let _ = handle_read_event(epfd, &mut stream, handler).map(|_| {
            add_stream_to_master_list(stream, streams.clone());
        });
    } else {
        trace!("event was drop event");

        let fd = stream.as_raw_fd();
        remove_fd_from_epoll(epfd, fd);
        close_fd(fd);

        let stream_fd = stream.as_raw_fd();
        unsafe {
            (*pool).run(move || {
                let Handler(ptr) = handler;
                (*ptr).on_stream_closed(stream_fd);
            });
        }
    }
}

/// Reads all available data on the stream.
///
/// If a complete message(s) is available, each message will be routed through the
/// resource pool.
///
/// If an error occurs during the read, the stream is dropped from the server.
fn handle_read_event(epfd: RawFd, stream: &mut Stream, handler: Handler) -> Result<(), ()> {
    trace!("handle read event");

    match stream.recv() {
        Ok(_) => {
            let mut rx_queue = stream.drain_rx_queue();
            for payload in rx_queue.iter_mut() {
                // Check if this is a request for stats
                if payload.len() == 6 && payload[0] == 0x04 && payload[1] == 0x04 {
                    let u8ptr: *const u8 = &payload[2] as *const _;
                    let f32ptr: *const f32 = u8ptr as *const _;
                    let sec = unsafe { *f32ptr };
                    let stream_cpy = stream.clone();
                    unsafe {
                        (*pool).run(move || {
                            let mut s = stream_cpy.clone();
                            let result = stats::as_serialized_buffer(sec);
                            if result.is_ok() {
                                let _ = s.send(&result.unwrap()[..]);
                            }
                        });
                    }
                    return Ok(());
                }

                // TODO - Refactor once better function passing traits are available in stable.
                let buf_len = payload.len();
                let handler_cpy = handler.clone();
                let stream_cpy = Stream { inner: stream.inner.clone_h_stream() };
                let payload_cpy = payload.clone();
                unsafe {
                    (*pool).run(move || {
                        let Handler(ptr) = handler_cpy;
                        (*ptr).on_data_received(stream_cpy.clone(), payload_cpy.clone());
                    });
                }
            }
            Ok(())
        }
        Err(e) => {
            error!("stream.fd: {} during read: {}", stream.as_raw_fd(), e);

            remove_fd_from_epoll(epfd, stream.as_raw_fd());
            close_fd(stream.as_raw_fd());

            let stream_fd = stream.as_raw_fd();
            unsafe {
                (*pool).run(move || {
                    let Handler(ptr) = handler;
                    (*ptr).on_stream_closed(stream_fd.clone());
                });
            }

            Err(())
        }
    }
}

/// Inserts the stream back into the master list of streams
fn add_stream_to_master_list(stream: Stream, streams: StreamList) {
    let mut guard = match streams.lock() {
        Ok(guard) => guard,
        Err(poisoned) => {
            warn!("StreamList Mutex failed, using anyway...");
            poisoned.into_inner()
        }
    };
    let stream_list = guard.deref_mut();
    stream_list.push_back(stream);
    stats::conn_recv();
}

/// Adds a new fd to the epoll instance
fn add_to_epoll(epfd: RawFd, fd: RawFd, streams: StreamList) {
    let result = epoll::ctl(epfd,
                            ctl_op::ADD,
                            fd,
                            &mut EpollEvent {
                                data: fd as u64,
                                events: EVENTS,
                            });

    if result.is_ok() {
        trace!("New fd added to epoll");
    } else {
        let e = result.unwrap_err();
        error!("CtrlError during add: {}", e);
        remove_fd_from_list(fd, streams.clone());
        close_fd(fd);
    }
}

/// Removes a fd from the epoll instance
fn remove_fd_from_epoll(epfd: RawFd, fd: RawFd) {
    debug!("removing fd from epoll");

    // In kernel versions before 2.6.9, the EPOLL_CTL_DEL operation required
    // a non-null pointer in event, even though this argument is ignored.
    // Since Linux 2.6.9, event can be specified as NULL when using
    // EPOLL_CTL_DEL. We'll be as backwards compatible as possible.
    let _ = epoll::ctl(epfd,
                       ctl_op::DEL,
                       fd,
                       &mut EpollEvent {
                           data: 0 as u64,
                           events: 0 as u32,
                       })
                .map_err(|e| warn!("Epoll CtrlError during del: {}", e));
}

/// Removes stream with id from master list
fn remove_fd_from_list(fd: RawFd, streams: StreamList) {
    trace!("removing fd: {} from master list", fd);

    let mut guard = match streams.lock() {
        Ok(guard) => guard,
        Err(poisoned) => {
            warn!("StreamList Mutex was poisoned, using anyway");
            poisoned.into_inner()
        }
    };
    let list = guard.deref_mut();

    let mut found = false;
    let mut index = 1usize;
    for s in list.iter() {
        if s.as_raw_fd() == fd {
            found = true;
            break;
        }
        index += 1;
    }

    if !found {
        trace!("fd: {} not found in list", fd);
        return;
    }

    if index == 1 {
        list.pop_front();
    } else {
        let mut split = list.split_off(index - 1);
        split.pop_front();
        list.append(&mut split);
    }

    stats::conn_lost();
}

/// Closes a fd with the kernel
fn close_fd(fd: RawFd) {
    unsafe {
        let result = libc::close(fd);
        if result < 0 {
            error!("Error closing fd: {}",
                   Error::from_raw_os_error(result as i32));
            return;
        }
    }
}
<commit_msg>Remote compile<commit_after>// Copyright 2015 Nathan Sizemore <nathanrsizemore@gmail.com>
//
// This Source Code Form is subject to the terms of the
// Mozilla Public License, v. 2.0. If a copy of the MPL was not
// distributed with this file, You can obtain one at
// http://mozilla.org/MPL/2.0/.

use std::io::Error;
use std::ops::DerefMut;
use std::sync::{Arc, Mutex};
use std::{ptr, mem, thread};
use std::os::unix::io::{RawFd, AsRawFd};
use std::collections::LinkedList;

use libc;
use errno::errno;
use epoll;
use epoll::util::*;
use epoll::EpollEvent;
use stream::socket::Socket;
use stream::nbstream::Nbstream;
use stream::securestream::SecureStream;
use stream::{Stream, HRecv, HSend, CloneHStream};
use stats;
use types::*;
use resources::ResourcePool;
use libc::{c_ushort, c_ulong, c_uint, c_int, c_void};
use config::Config;
use openssl::ssl::SslContext;

extern "C" {
    fn shim_htons(hostshort: c_ushort) -> c_ushort;
    fn shim_htonl(addr: c_uint) -> c_uint;
    fn shim_inaddr_any() -> c_ulong;
}

// We need to be able to access our resource pool from several methods
static mut pool: *mut ResourcePool = 0 as *mut ResourcePool;

// Global SslContext
static mut ssl_context: *mut SslContext = 0 as *mut SslContext;

// When added to epoll, these will be the conditions of kernel notification:
//
// EPOLLET  - Fd is in EdgeTriggered mode (notification on state changes)
// EPOLLIN  - Data is available in kerndl buffer
const EVENTS: u32 = event_type::EPOLLET | event_type::EPOLLIN;

/// Starts the epoll wait and incoming connection listener threads.
pub fn begin(config: Config, handler: Box<EventHandler>) {
    // Master socket list
    let sockets = Arc::new(Mutex::new(LinkedList::<Stream>::new()));

    // Resource pool
    let mut rp = ResourcePool::new(config.workers);
    unsafe {
        pool = &mut rp;
    }

    // Wrap our event handler into something that can be safely shared
    // between threads.
    let e_handler = Handler(Box::into_raw(handler));

    // Epoll instance
    let result = epoll::create1(0);
    if result.is_err() {
        let err = result.unwrap_err();
        error!("Unable to create epoll instance: {}", err);
        panic!()
    }
    let epfd = result.unwrap();

    // Epoll wait thread
    let epfd2 = epfd.clone();
    let streams2 = sockets.clone();
    thread::Builder::new()
        .name("Epoll Wait".to_string())
        .spawn(move || {
            event_loop(epfd2, streams2, e_handler);
        })
        .unwrap();

    // New connection thread
    let epfd3 = epfd.clone();
    let streams3 = sockets.clone();
    let prox = thread::Builder::new()
                   .name("TCP Incoming Listener".to_string())
                   .spawn(move || {
                       listen(config, epfd3, streams3);
                   })
                   .unwrap();

    // Stay alive forever, or at least we hope
    let _ = prox.join();
}

/// Incoming connection listening thread
fn listen(config: Config, epfd: RawFd, streams: StreamList) {
    unsafe {
        let server_addr = libc::sockaddr_in {
            sin_family: libc::AF_INET as u16,
            sin_port: shim_htons(config.port),
            sin_addr: libc::in_addr { s_addr: shim_htonl(shim_inaddr_any() as u32) },
            sin_zero: [0u8; 8],
        };

        // Create a server socket with the passed info for TCP/IP
        let server_fd = libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0);

        // Set the SO_REUSEADDR so that restarts after crashes do not take 5min to unbind
        // the initial port
        let optval: c_int = 1;
        let opt_result = libc::setsockopt(server_fd,
                                          libc::SOL_SOCKET,
                                          libc::SO_REUSEADDR,
                                          &optval as *const _ as *const c_void,
                                          mem::size_of::<c_int>() as u32);
        if opt_result < 0 {
            error!("Setting SO_REUSEADDR: {}",
                   Error::from_raw_os_error(errno().0 as i32));
            return;
        }

        let serv_addr: *const libc::sockaddr = mem::transmute(&server_addr);
        let bind_result = libc::bind(server_fd,
                                     serv_addr,
                                     mem::size_of::<libc::sockaddr_in>() as u32);
        if bind_result < 0 {
            error!("Binding fd: {} - {}",
                   server_fd,
                   Error::from_raw_os_error(errno().0 as i32));
            return;
        }

        let listen_result = libc::listen(server_fd, 100);
        if listen_result < 0 {
            error!("Listening: {}", Error::from_raw_os_error(errno().0 as i32));
            return;
        }

        // If we're using SSL, setup our context
        let using_ssl;
        let mut config = config;
        match config.ssl.as_mut() {
            Some(context) => {
                ssl_context = context;
                using_ssl = true;
            }
            None => {
                using_ssl = false;
            }
        };

        loop {
            // Accept new client
            let result = libc::accept(server_fd,
                                      ptr::null_mut() as *mut libc::sockaddr,
                                      ptr::null_mut() as *mut u32);
            if result < 0 {
                error!("Accepting new connection: {}",
                       Error::from_raw_os_error(result as i32));
            }

            // Setup new socket
            let socket = Socket {
                fd: result
            };
            socket.set_tcp_keepalive(true);
            socket.set_tcp_nodelay(true);

            // Create new stream and add to server
            if using_ssl {
                let _ = SecureStream::new(&(*ssl_context), socket).map(|securestream| {
                    let stream = Stream {
                        inner: Box::new(securestream)
                    };
                    let fd = stream.as_raw_fd();
                    add_stream_to_master_list(stream, streams.clone());
                    add_to_epoll(epfd, fd, streams.clone());
                }).map_err(|e| {
                    // If we ever fail here, it is safe to assume everything else has gone to
                    // shit. No way to clean it up, so we'll just leave. The main process thread
                    // is joined to this tread, so panic-ing will cause our parent process
                    // to close out completely.
                    error!("Creating stream: {}", e);
                    panic!()
                });
            } else {
                let _ = Nbstream::new(socket).map(|nbstream| {
                    let stream = Stream {
                        inner: Box::new(nbstream)
                    };
                    let fd = stream.as_raw_fd();
                    add_stream_to_master_list(stream, streams.clone());
                    add_to_epoll(epfd, fd, streams.clone());
                }).map_err(|e| {
                    // If we ever fail here, it is safe to assume everything else has gone to
                    // shit. No way to clean it up, so we'll just leave. The main process thread
                    // is joined to this tread, so panic-ing will cause our parent process
                    // to close out completely.
                    error!("Creating stream: {}", e);
                    panic!()
                });
            }
        }
    }
}

/// Event loop for handling all epoll events
fn event_loop(epfd: RawFd, streams: StreamList, handler: Handler) {
    let mut events = Vec::<EpollEvent>::with_capacity(100);
    unsafe {
        events.set_len(100);
    }

    loop {
        match epoll::wait(epfd, &mut events[..], -1) {
            Ok(num_events) => {
                for x in 0..num_events as usize {
                    handle_epoll_event(epfd, &events[x], streams.clone(), handler.clone());
                }
            }
            Err(e) => {
                error!("Error on epoll::wait(): {}", e);
                panic!()
            }
        };
    }
}

/// Finds the stream the epoll event is associated with and parses the event type
/// to hand off to specific handlers
fn handle_epoll_event(epfd: RawFd, event: &EpollEvent, streams: StreamList, handler: Handler) {
    const READ_EVENT: u32 = event_type::EPOLLIN;

    // Locate the stream the event was for
    let mut stream;
    {
        // Mutex lock
        // Find the stream the event was for
        let mut guard = match streams.lock() {
            Ok(guard) => guard,
            Err(poisoned) => {
                warn!("StreamList Mutex was poisoned, using anyway");
                poisoned.into_inner()
            }
        };
        let list = guard.deref_mut();

        let mut found = false;
        let mut index = 1usize;
        for s in list.iter() {
            if s.as_raw_fd() == event.data as RawFd {
                found = true;
                break;
            }
            index += 1;
        }

        if !found {
            error!("Could not find fd in stream list? Removing from epoll and closing fd");

            let fd = event.data as RawFd;
            remove_fd_from_epoll(epfd, fd);
            close_fd(fd);
            return;
        }

        if index == 1 {
            stream = list.pop_front().unwrap();
        } else {
            let mut split = list.split_off(index - 1);
            stream = split.pop_front().unwrap();
            list.append(&mut split);
        }
    } // Mutex unlock

    if (event.events & READ_EVENT) > 0 {
        trace!("event was read event");
        let _ = handle_read_event(epfd, &mut stream, handler).map(|_| {
            add_stream_to_master_list(stream, streams.clone());
        });
    } else {
        trace!("event was drop event");

        let fd = stream.as_raw_fd();
        remove_fd_from_epoll(epfd, fd);
        close_fd(fd);

        let stream_fd = stream.as_raw_fd();
        unsafe {
            (*pool).run(move || {
                let Handler(ptr) = handler;
                (*ptr).on_stream_closed(stream_fd);
            });
        }
    }
}

/// Reads all available data on the stream.
///
/// If a complete message(s) is available, each message will be routed through the
/// resource pool.
///
/// If an error occurs during the read, the stream is dropped from the server.
fn handle_read_event(epfd: RawFd, stream: &mut Stream, handler: Handler) -> Result<(), ()> {
    trace!("handle read event");

    match stream.recv() {
        Ok(_) => {
            let mut rx_queue = stream.drain_rx_queue();
            for payload in rx_queue.iter_mut() {
                // Check if this is a request for stats
                if payload.len() == 6 && payload[0] == 0x04 && payload[1] == 0x04 {
                    let u8ptr: *const u8 = &payload[2] as *const _;
                    let f32ptr: *const f32 = u8ptr as *const _;
                    let sec = unsafe { *f32ptr };
                    let stream_cpy = stream.clone();
                    unsafe {
                        (*pool).run(move || {
                            let mut s = stream_cpy.clone();
                            let result = stats::as_serialized_buffer(sec);
                            if result.is_ok() {
                                let _ = s.send(&result.unwrap()[..]);
                            }
                        });
                    }
                    return Ok(());
                }

                // TODO - Refactor once better function passing traits are available in stable.
                let handler_cpy = handler.clone();
                let stream_cpy = Stream { inner: stream.inner.clone_h_stream() };
                let payload_cpy = payload.clone();
                unsafe {
                    (*pool).run(move || {
                        let Handler(ptr) = handler_cpy;
                        (*ptr).on_data_received(stream_cpy.clone(), payload_cpy.clone());
                    });
                }
            }
            Ok(())
        }
        Err(e) => {
            error!("stream.fd: {} during read: {}", stream.as_raw_fd(), e);

            remove_fd_from_epoll(epfd, stream.as_raw_fd());
            close_fd(stream.as_raw_fd());

            let stream_fd = stream.as_raw_fd();
            unsafe {
                (*pool).run(move || {
                    let Handler(ptr) = handler;
                    (*ptr).on_stream_closed(stream_fd.clone());
                });
            }

            Err(())
        }
    }
}

/// Inserts the stream back into the master list of streams
fn add_stream_to_master_list(stream: Stream, streams: StreamList) {
    let mut guard = match streams.lock() {
        Ok(guard) => guard,
        Err(poisoned) => {
            warn!("StreamList Mutex failed, using anyway...");
            poisoned.into_inner()
        }
    };
    let stream_list = guard.deref_mut();
    stream_list.push_back(stream);
    stats::conn_recv();
}

/// Adds a new fd to the epoll instance
fn add_to_epoll(epfd: RawFd, fd: RawFd, streams: StreamList) {
    let result = epoll::ctl(epfd,
                            ctl_op::ADD,
                            fd,
                            &mut EpollEvent {
                                data: fd as u64,
                                events: EVENTS,
                            });

    if result.is_ok() {
        trace!("New fd added to epoll");
    } else {
        let e = result.unwrap_err();
        error!("CtrlError during add: {}", e);
        remove_fd_from_list(fd, streams.clone());
        close_fd(fd);
    }
}

/// Removes a fd from the epoll instance
fn remove_fd_from_epoll(epfd: RawFd, fd: RawFd) {
    debug!("removing fd from epoll");

    // In kernel versions before 2.6.9, the EPOLL_CTL_DEL operation required
    // a non-null pointer in event, even though this argument is ignored.
    // Since Linux 2.6.9, event can be specified as NULL when using
    // EPOLL_CTL_DEL. We'll be as backwards compatible as possible.
    let _ = epoll::ctl(epfd,
                       ctl_op::DEL,
                       fd,
                       &mut EpollEvent {
                           data: 0 as u64,
                           events: 0 as u32,
                       })
                .map_err(|e| warn!("Epoll CtrlError during del: {}", e));
}

/// Removes stream with id from master list
fn remove_fd_from_list(fd: RawFd, streams: StreamList) {
    trace!("removing fd: {} from master list", fd);

    let mut guard = match streams.lock() {
        Ok(guard) => guard,
        Err(poisoned) => {
            warn!("StreamList Mutex was poisoned, using anyway");
            poisoned.into_inner()
        }
    };
    let list = guard.deref_mut();

    let mut found = false;
    let mut index = 1usize;
    for s in list.iter() {
        if s.as_raw_fd() == fd {
            found = true;
            break;
        }
        index += 1;
    }

    if !found {
        trace!("fd: {} not found in list", fd);
        return;
    }

    if index == 1 {
        list.pop_front();
    } else {
        let mut split = list.split_off(index - 1);
        split.pop_front();
        list.append(&mut split);
    }

    stats::conn_lost();
}

/// Closes a fd with the kernel
fn close_fd(fd: RawFd) {
    unsafe {
        let result = libc::close(fd);
        if result < 0 {
            error!("Error closing fd: {}",
                   Error::from_raw_os_error(result as i32));
            return;
        }
    }
}
<|endoftext|>


<commit_before>use std::thread;
use std::collections::HashSet;
use std::net::SocketAddr;
use std::collections::VecDeque;
use std::io::BufReader;

// MIO
use mio::tcp::{listen, TcpListener, TcpStream};
use mio::util::Slab;
use mio::Socket;
use mio::buf::{RingBuf};
use mio::{Interest, PollOpt, NonBlock, Token, EventLoop, Handler, ReadHint};
use mio::buf::Buf;
use mio::{TryRead, TryWrite};

use rand::{self, Rng};

// Data structures.
use store::Store;
use replica::{Replica, Emit, Broadcast};
use state_machine::StateMachine;

// Cap'n Proto
use capnp::serialize_packed;
use capnp::{
    MessageBuilder,
    MessageReader,
    ReaderOptions,
    MallocMessageBuilder,
    OwnedSpaceMessageReader,
};
use messages_capnp::{
    rpc_request,
    rpc_response,
    client_request,
    client_response,
    request_vote_response,
    append_entries_response,
    append_entries_request,
};
use super::{Error, Result};

// MIO Tokens
const ELECTION_TIMEOUT: Token = Token(0);
const HEARTBEAT_TIMEOUT: Token = Token(1);
const LISTENER:  Token = Token(2);

const ELECTION_MIN: u64 = 150;
const ELECTION_MAX: u64 = 300;
const HEARTBEAT_DURATION: u64 = 50;
const RINGBUF_SIZE: usize = 4096;

/// The Raft Distributed Consensus Algorithm requires two RPC calls to be available:
///
///   * `append_entries` which is used as both a heartbeat (with no payload) and the primary
///     interface for requests.
///   * `request_vote` which is used by candidates during campaigns to obtain a vote.
///
/// A `Server` acts as a replicated state machine. The server's role in the cluster depends on it's
/// own status. It will maintain both volatile state (which can be safely lost) and persistent
/// state (which must be carefully stored and kept safe).
///
/// Currently, the `Server` API is not well defined. **We are looking for feedback and suggestions.**
pub struct Server<S, M> where S: Store, M: StateMachine {
    replica: Replica<S, M>,
    // Channels and Sockets
    listener: NonBlock<TcpListener>,
    connections: Slab<Connection>,
}

/// The implementation of the Server. In most use cases, creating a `Server` should just be
/// done via `::new()`.
impl<S, M> Server<S, M> where S: Store, M: StateMachine {

    /// Creates a new Raft node with the cluster members specified.
    ///
    /// # Arguments
    ///
    /// * `addr` - The address of the new node.
    /// * `peers` - The address of all peers in the Raft cluster.
    /// * `store` - The persitent log store.
    /// * `state_machine` - The client state machine to which client commands will be applied.
    pub fn spawn(addr: SocketAddr,
                 peers: HashSet<SocketAddr>,
                 store: S,
                 state_machine: M) {
        debug!("Spawning Server");
        // Create an event loop
        let mut event_loop = EventLoop::<Server<S, M>>::new().unwrap();
        // Setup the socket, make it not block.
        let listener = listen(&addr).unwrap();
        listener.set_reuseaddr(true).unwrap();
        event_loop.register(&listener, LISTENER).unwrap();
        let timeout = rand::thread_rng().gen_range::<u64>(ELECTION_MIN, ELECTION_MAX);
        event_loop.timeout_ms(ELECTION_TIMEOUT, timeout).unwrap();
        event_loop.timeout_ms(HEARTBEAT_TIMEOUT, HEARTBEAT_DURATION).unwrap();
        let replica = Replica::new(addr, peers, store, state_machine);
        // Fire up the thread.
        thread::Builder::new().name(format!("Server {}", addr)).spawn(move || {
            let mut raft_node = Server {
                listener: listener,
                replica: replica,
                connections: Slab::new_starting_at(Token(2), 128),
            };
            event_loop.run(&mut raft_node).unwrap();
        }).unwrap();
    }
}

impl<S, M> Handler for Server<S, M> where S: Store, M: StateMachine {

    type Message = RingBuf;
    type Timeout = Token;

    /// A registered IoHandle has available writing space.
    fn writable(&mut self, reactor: &mut EventLoop<Server<S, M>>, token: Token) {
        debug!("Writeable");
        match token {
            ELECTION_TIMEOUT => unreachable!(),
            HEARTBEAT_TIMEOUT => unreachable!(),
            LISTENER => unreachable!(),
            tok => {
                self.connections[tok].writable(reactor, &mut self.replica).unwrap();
            }
        }
    }

    /// A registered IoHandle has available data to read
    fn readable(&mut self, reactor: &mut EventLoop<Server<S, M>>, token: Token, _hint: ReadHint) {
        debug!("Readable");
        match token {
            ELECTION_TIMEOUT => unreachable!(),
            HEARTBEAT_TIMEOUT => unreachable!(),
            LISTENER => {
                let stream = match self.listener.accept().unwrap() {
                    Some(s) => s,
                    None => return, // Socket isn't quite ready.
                }; // Result<Option<_>,_>
                let conn = Connection::new(stream);
                let tok = self.connections.insert(conn)
                    .ok().expect("Could not add connection to slab.");

                // Register the connection
                self.connections[tok].token = tok;
                reactor.register_opt(&self.connections[tok].stream, tok, Interest::readable(), PollOpt::edge() | PollOpt::oneshot())
                    .ok().expect("Could not register socket with event loop.");
            },
            tok => {
                self.connections[tok].readable(reactor, &mut self.replica).unwrap();
            }
        }
    }

    /// A registered timer has expired. This is either:
    ///
    /// * An election timeout, when a `Follower` node has waited too long for a heartbeat and doing
    /// to become a `Candidate`.
    /// * A heartbeat timeout, when the `Leader` node needs to refresh it's authority over the
    /// followers. Initializes and sends an `AppendEntries` request to all followers.
    fn timeout(&mut self, reactor: &mut EventLoop<Server<S, M>>, token: Token) {
        debug!("Timeout");
        let mut message = MallocMessageBuilder::new_default();
        let mut send_message = None;
        match token {
            ELECTION_TIMEOUT => {
                let request = message.init_root::<rpc_request::Builder>();
                send_message = self.replica.election_timeout(request.init_request_vote());
                // Set timeout.
                let timeout = rand::thread_rng().gen_range::<u64>(ELECTION_MIN, ELECTION_MAX);
                reactor.timeout_ms(ELECTION_TIMEOUT, timeout).unwrap();

            },
            HEARTBEAT_TIMEOUT => {
                let request = message.init_root::<rpc_request::Builder>();
                send_message = self.replica.heartbeat_timeout(request.init_append_entries());
                // Set Timeout
                reactor.timeout_ms(HEARTBEAT_TIMEOUT, HEARTBEAT_DURATION).unwrap();
            },
            _ => unreachable!(),
        }
        // Send if necessary.
        match send_message {
            Some(Broadcast) => {
                let mut buf = RingBuf::new(RINGBUF_SIZE);
                serialize_packed::write_message(
                    &mut buf,
                    &mut message
                ).unwrap();
                for connection in self.connections.iter_mut() {
                    connection.add_write(buf.clone());
                }
            },
            None => (),
        }
    }
}

struct Connection {
    stream: NonBlock<TcpStream>,
    token: Token,
    interest: Interest,
    current_read: BufReader<RingBuf>,
    current_write: BufReader<RingBuf>,
    next_write: VecDeque<RingBuf>,
}

impl Connection {
    /// Note: The caller must manually assign `token` to what is desired.
    fn new(sock: NonBlock<TcpStream>) -> Connection {
        Connection {
            stream: sock,
            token: Token(0), // Effectively a `null`. This needs to be assigned by the caller.
            interest: Interest::hup(),
            current_read: BufReader::new(RingBuf::new(4096)),
            current_write: BufReader::new(RingBuf::new(4096)),
            next_write: VecDeque::with_capacity(10),
        }
    }

    /// A registered IoHandle has available writing space.
    fn writable<S, M>(&mut self, event_loop: &mut EventLoop<Server<S, M>>, replica: &mut Replica<S,M>)
                      -> Result<()>
    where S: Store, M: StateMachine {
        // Attempt to write data.
        // The `current_write` buffer will be advanced based on how much we wrote.
        match self.stream.write(self.current_write.get_mut()) {
            Ok(None) => {
                // This is a buffer flush. WOULDBLOCK
                self.interest.insert(Interest::writable());
            },
            Ok(Some(r)) => {
                // We managed to write data!
                match (self.current_write.get_ref().has_remaining(), self.next_write.is_empty()) {
                    // Need to write more of what we have.
                    (true, _) => (),
                    // Need to roll over.
                    (false, false) => self.current_write = BufReader::new(self.next_write.pop_front().unwrap()),
                    // We're done writing for now.
                    (false, true) => self.interest.remove(Interest::writable()),

                }
            },
            Err(e) => return Err(Error::from(e)),
        }

        match event_loop.reregister(&self.stream, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()) {
            Ok(()) => Ok(()),
            Err(e) => Err(Error::from(e)),
        }
    }

    /// A registered IoHandle has available data to read.
    /// This does not necessarily mean that there is an entire packed item on the stream. We could
    /// get some, all of it, or none. We'll use the buffer to read in until we can find one.
    fn readable<S, M>(&mut self, event_loop: &mut EventLoop<Server<S, M>>, replica: &mut Replica<S,M>)
                      -> Result<()>
    where S: Store, M: StateMachine {
        let mut read = 0;
        match self.stream.read(self.current_read.get_mut()) {
            Ok(Some(r)) => {
                // Just read `r` bytes.
                read = r;
            },
            Ok(None) => panic!("We just got readable, but were unable to read from the socket?"),
            Err(e) => return Err(Error::from(e)),
        };
        if read > 0 {
            match serialize_packed::read_message(&mut self.current_read, ReaderOptions::new()) {
                // We have something reasonably interesting in the buffer!
                Ok(reader) => {
                    self.handle_reader(reader, event_loop, replica);
                },
                // It's not read entirely yet.
                // Should roll back, pending changes to bytes upstream.
                // TODO: This was fixed.
                Err(_) => unimplemented!(),
            }
        }
        match event_loop.reregister(&self.stream, self.token, self.interest, PollOpt::edge()) {
            Ok(()) => Ok(()),
            Err(e) => Err(Error::from(e)),
        }
    }

    /// This is called when there is a full reader available in the buffer.
    /// It handles what to do with the data.
    fn handle_reader<S, M>(&mut self, reader: OwnedSpaceMessageReader,
                           event_loop: &mut EventLoop<Server<S, M>>, replica: &mut Replica<S,M>)
    where S: Store, M: StateMachine {
        let mut builder_message = MallocMessageBuilder::new_default();
        let from = self.stream.peer_addr().unwrap();
        if let Ok(request) = reader.get_root::<rpc_request::Reader>() {
            match request.which().unwrap() {
                // TODO: Move these into replica?
                rpc_request::Which::AppendEntries(Ok(call)) => {
                    let builder = builder_message.init_root::<append_entries_response::Builder>();
                    match replica.append_entries_request(from, call, builder) {
                        Some(Emit) => {
                            // Special Cirsumstance Detection
                            unimplemented!();
                        },
                        None => (),
                    }
                },
                rpc_request::Which::RequestVote(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<request_vote_response::Builder>();
                        replica.request_vote_request(from, call, builder)
                    };
                    match respond {
                        Some(Emit) => {
                            // TODO
                            self.emit(builder_message);
                        },
                        None            => (),
                    }
                },
                _ => unimplemented!(),
            };
        } else if let Ok(response) = reader.get_root::<rpc_response::Reader>() {
            // We won't be responding. This is already a response.
            match response.which().unwrap() {
                rpc_response::Which::AppendEntries(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<append_entries_request::Builder>();
                        replica.append_entries_response(from, call, builder)
                    };
                    match respond {
                        Some(Emit) => {
                            // TODO
                            self.emit(builder_message);
                        },
                        None => (),
                    }
                },
                rpc_response::Which::RequestVote(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<append_entries_request::Builder>();
                        replica.request_vote_response(from, call, builder)
                    };
                    match respond {
                        Some(Broadcast) => {
                            // Won an election!
                            self.broadcast(builder_message);
                        },
                        None => (),
                    }

                },
                _ => unimplemented!(),
            }
        } else if let Ok(client_req) = reader.get_root::<client_request::Reader>() {
            let mut should_die = false;
            // We will be responding.
            match client_req.which().unwrap() {
                client_request::Which::Append(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<client_response::Builder>();
                        replica.client_append(from, call, builder)
                    };
                    match respond {
                        Some(emit) => {
                            self.emit(builder_message);
                        },
                        None => (),
                    }

                },
                client_request::Which::Die(Ok(call)) => {
                    should_die = true;
                    let mut builder = builder_message.init_root::<client_response::Builder>();
                    builder.set_success(());
                    self.interest.insert(Interest::writable());
                    debug!("Got a Die request from Client({}). Reason: {}", from, call);
                },
                client_request::Which::LeaderRefresh(()) => {
                    let respond = {
                        let builder = builder_message.init_root::<client_response::Builder>();
                        replica.client_leader_refresh(from, builder)
                    };
                    match respond {
                        Some(Emit) => {
                            self.emit(builder_message);
                        },
                        None => (),
                    }
                },
                _ => unimplemented!(),
            };

            // Do this here so that we can send the response.
            if should_die {
                panic!("Got a Die request.");
            }

        } else {
            // It's something we don't understand.
            unimplemented!();
        }
    }

    fn broadcast(&mut self, builder: MallocMessageBuilder) {
        unimplemented!();
    }

    /// Push the new message into `self.next_write`. This does not actually send the message, it
    /// just queues it up.
    pub fn emit(&mut self, mut builder: MallocMessageBuilder) {
        let mut buf = RingBuf::new(RINGBUF_SIZE);
        serialize_packed::write_message(
            &mut buf,
            &mut builder
        ).unwrap();
        self.add_write(buf);
    }

    /// This queues a byte buffer into the write queue. This is used primarily when message has
    /// already been packed.
    pub fn add_write(&mut self, buf: RingBuf) {
        self.next_write.push_back(buf);
    }
}
<commit_msg>Update mio<commit_after>use std::thread;
use std::collections::HashSet;
use std::net::SocketAddr;
use std::collections::VecDeque;
use std::io::BufReader;

// MIO
use mio::tcp::{TcpListener, TcpStream};
use mio::util::Slab;
use mio::Socket;
use mio::buf::{RingBuf};
use mio::{Interest, PollOpt, Token, EventLoop, Handler, ReadHint};
use mio::buf::Buf;
use mio::{TryRead, TryWrite};

use rand::{self, Rng};

// Data structures.
use store::Store;
use replica::{Replica, Emit, Broadcast};
use state_machine::StateMachine;

// Cap'n Proto
use capnp::serialize_packed;
use capnp::{
    MessageBuilder,
    MessageReader,
    ReaderOptions,
    MallocMessageBuilder,
    OwnedSpaceMessageReader,
};
use messages_capnp::{
    rpc_request,
    rpc_response,
    client_request,
    client_response,
    request_vote_response,
    append_entries_response,
    append_entries_request,
};
use super::{Error, Result};

// MIO Tokens
const ELECTION_TIMEOUT: Token = Token(0);
const HEARTBEAT_TIMEOUT: Token = Token(1);
const LISTENER:  Token = Token(2);

const ELECTION_MIN: u64 = 150;
const ELECTION_MAX: u64 = 300;
const HEARTBEAT_DURATION: u64 = 50;
const RINGBUF_SIZE: usize = 4096;

/// The Raft Distributed Consensus Algorithm requires two RPC calls to be available:
///
///   * `append_entries` which is used as both a heartbeat (with no payload) and the primary
///     interface for requests.
///   * `request_vote` which is used by candidates during campaigns to obtain a vote.
///
/// A `Server` acts as a replicated state machine. The server's role in the cluster depends on it's
/// own status. It will maintain both volatile state (which can be safely lost) and persistent
/// state (which must be carefully stored and kept safe).
///
/// Currently, the `Server` API is not well defined. **We are looking for feedback and suggestions.**
pub struct Server<S, M> where S: Store, M: StateMachine {
    replica: Replica<S, M>,
    // Channels and Sockets
    listener: TcpListener,
    connections: Slab<Connection>,
}

/// The implementation of the Server. In most use cases, creating a `Server` should just be
/// done via `::new()`.
impl<S, M> Server<S, M> where S: Store, M: StateMachine {

    /// Creates a new Raft node with the cluster members specified.
    ///
    /// # Arguments
    ///
    /// * `addr` - The address of the new node.
    /// * `peers` - The address of all peers in the Raft cluster.
    /// * `store` - The persitent log store.
    /// * `state_machine` - The client state machine to which client commands will be applied.
    pub fn spawn(addr: SocketAddr,
                 peers: HashSet<SocketAddr>,
                 store: S,
                 state_machine: M) {
        debug!("Spawning Server");
        // Create an event loop
        let mut event_loop = EventLoop::<Server<S, M>>::new().unwrap();
        // Setup the socket, make it not block.
        let listener = TcpListener::bind(&addr).unwrap();
        listener.set_reuseaddr(true).unwrap();
        event_loop.register(&listener, LISTENER).unwrap();
        let timeout = rand::thread_rng().gen_range::<u64>(ELECTION_MIN, ELECTION_MAX);
        event_loop.timeout_ms(ELECTION_TIMEOUT, timeout).unwrap();
        event_loop.timeout_ms(HEARTBEAT_TIMEOUT, HEARTBEAT_DURATION).unwrap();
        let replica = Replica::new(addr, peers, store, state_machine);
        // Fire up the thread.
        thread::Builder::new().name(format!("Server {}", addr)).spawn(move || {
            let mut raft_node = Server {
                listener: listener,
                replica: replica,
                connections: Slab::new_starting_at(Token(2), 128),
            };
            event_loop.run(&mut raft_node).unwrap();
        }).unwrap();
    }
}

impl<S, M> Handler for Server<S, M> where S: Store, M: StateMachine {

    type Message = RingBuf;
    type Timeout = Token;

    /// A registered IoHandle has available writing space.
    fn writable(&mut self, reactor: &mut EventLoop<Server<S, M>>, token: Token) {
        debug!("Writeable");
        match token {
            ELECTION_TIMEOUT => unreachable!(),
            HEARTBEAT_TIMEOUT => unreachable!(),
            LISTENER => unreachable!(),
            tok => {
                self.connections[tok].writable(reactor, &mut self.replica).unwrap();
            }
        }
    }

    /// A registered IoHandle has available data to read
    fn readable(&mut self, reactor: &mut EventLoop<Server<S, M>>, token: Token, _hint: ReadHint) {
        debug!("Readable");
        match token {
            ELECTION_TIMEOUT => unreachable!(),
            HEARTBEAT_TIMEOUT => unreachable!(),
            LISTENER => {
                let stream = match self.listener.accept().unwrap() {
                    Some(s) => s,
                    None => return, // Socket isn't quite ready.
                }; // Result<Option<_>,_>
                let conn = Connection::new(stream);
                let tok = self.connections.insert(conn)
                    .ok().expect("Could not add connection to slab.");

                // Register the connection
                self.connections[tok].token = tok;
                reactor.register_opt(&self.connections[tok].stream, tok, Interest::readable(), PollOpt::edge() | PollOpt::oneshot())
                    .ok().expect("Could not register socket with event loop.");
            },
            tok => {
                self.connections[tok].readable(reactor, &mut self.replica).unwrap();
            }
        }
    }

    /// A registered timer has expired. This is either:
    ///
    /// * An election timeout, when a `Follower` node has waited too long for a heartbeat and doing
    /// to become a `Candidate`.
    /// * A heartbeat timeout, when the `Leader` node needs to refresh it's authority over the
    /// followers. Initializes and sends an `AppendEntries` request to all followers.
    fn timeout(&mut self, reactor: &mut EventLoop<Server<S, M>>, token: Token) {
        debug!("Timeout");
        let mut message = MallocMessageBuilder::new_default();
        let mut send_message = None;
        match token {
            ELECTION_TIMEOUT => {
                let request = message.init_root::<rpc_request::Builder>();
                send_message = self.replica.election_timeout(request.init_request_vote());
                // Set timeout.
                let timeout = rand::thread_rng().gen_range::<u64>(ELECTION_MIN, ELECTION_MAX);
                reactor.timeout_ms(ELECTION_TIMEOUT, timeout).unwrap();

            },
            HEARTBEAT_TIMEOUT => {
                let request = message.init_root::<rpc_request::Builder>();
                send_message = self.replica.heartbeat_timeout(request.init_append_entries());
                // Set Timeout
                reactor.timeout_ms(HEARTBEAT_TIMEOUT, HEARTBEAT_DURATION).unwrap();
            },
            _ => unreachable!(),
        }
        // Send if necessary.
        match send_message {
            Some(Broadcast) => {
                let mut buf = RingBuf::new(RINGBUF_SIZE);
                serialize_packed::write_message(
                    &mut buf,
                    &mut message
                ).unwrap();
                for connection in self.connections.iter_mut() {
                    connection.add_write(buf.clone());
                }
            },
            None => (),
        }
    }
}

struct Connection {
    stream: TcpStream,
    token: Token,
    interest: Interest,
    current_read: BufReader<RingBuf>,
    current_write: BufReader<RingBuf>,
    next_write: VecDeque<RingBuf>,
}

impl Connection {
    /// Note: The caller must manually assign `token` to what is desired.
    fn new(sock: TcpStream) -> Connection {
        Connection {
            stream: sock,
            token: Token(0), // Effectively a `null`. This needs to be assigned by the caller.
            interest: Interest::hup(),
            current_read: BufReader::new(RingBuf::new(4096)),
            current_write: BufReader::new(RingBuf::new(4096)),
            next_write: VecDeque::with_capacity(10),
        }
    }

    /// A registered IoHandle has available writing space.
    fn writable<S, M>(&mut self, event_loop: &mut EventLoop<Server<S, M>>, replica: &mut Replica<S,M>)
                      -> Result<()>
    where S: Store, M: StateMachine {
        // Attempt to write data.
        // The `current_write` buffer will be advanced based on how much we wrote.
        match self.stream.write(self.current_write.get_mut()) {
            Ok(None) => {
                // This is a buffer flush. WOULDBLOCK
                self.interest.insert(Interest::writable());
            },
            Ok(Some(r)) => {
                // We managed to write data!
                match (self.current_write.get_ref().has_remaining(), self.next_write.is_empty()) {
                    // Need to write more of what we have.
                    (true, _) => (),
                    // Need to roll over.
                    (false, false) => self.current_write = BufReader::new(self.next_write.pop_front().unwrap()),
                    // We're done writing for now.
                    (false, true) => self.interest.remove(Interest::writable()),

                }
            },
            Err(e) => return Err(Error::from(e)),
        }

        match event_loop.reregister(&self.stream, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()) {
            Ok(()) => Ok(()),
            Err(e) => Err(Error::from(e)),
        }
    }

    /// A registered IoHandle has available data to read.
    /// This does not necessarily mean that there is an entire packed item on the stream. We could
    /// get some, all of it, or none. We'll use the buffer to read in until we can find one.
    fn readable<S, M>(&mut self, event_loop: &mut EventLoop<Server<S, M>>, replica: &mut Replica<S,M>)
                      -> Result<()>
    where S: Store, M: StateMachine {
        let mut read = 0;
        match self.stream.read(self.current_read.get_mut()) {
            Ok(Some(r)) => {
                // Just read `r` bytes.
                read = r;
            },
            Ok(None) => panic!("We just got readable, but were unable to read from the socket?"),
            Err(e) => return Err(Error::from(e)),
        };
        if read > 0 {
            match serialize_packed::read_message(&mut self.current_read, ReaderOptions::new()) {
                // We have something reasonably interesting in the buffer!
                Ok(reader) => {
                    self.handle_reader(reader, event_loop, replica);
                },
                // It's not read entirely yet.
                // Should roll back, pending changes to bytes upstream.
                // TODO: This was fixed.
                Err(_) => unimplemented!(),
            }
        }
        match event_loop.reregister(&self.stream, self.token, self.interest, PollOpt::edge()) {
            Ok(()) => Ok(()),
            Err(e) => Err(Error::from(e)),
        }
    }

    /// This is called when there is a full reader available in the buffer.
    /// It handles what to do with the data.
    fn handle_reader<S, M>(&mut self, reader: OwnedSpaceMessageReader,
                           event_loop: &mut EventLoop<Server<S, M>>, replica: &mut Replica<S,M>)
    where S: Store, M: StateMachine {
        let mut builder_message = MallocMessageBuilder::new_default();
        let from = self.stream.peer_addr().unwrap();
        if let Ok(request) = reader.get_root::<rpc_request::Reader>() {
            match request.which().unwrap() {
                // TODO: Move these into replica?
                rpc_request::Which::AppendEntries(Ok(call)) => {
                    let builder = builder_message.init_root::<append_entries_response::Builder>();
                    match replica.append_entries_request(from, call, builder) {
                        Some(Emit) => {
                            // Special Cirsumstance Detection
                            unimplemented!();
                        },
                        None => (),
                    }
                },
                rpc_request::Which::RequestVote(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<request_vote_response::Builder>();
                        replica.request_vote_request(from, call, builder)
                    };
                    match respond {
                        Some(Emit) => {
                            // TODO
                            self.emit(builder_message);
                        },
                        None            => (),
                    }
                },
                _ => unimplemented!(),
            };
        } else if let Ok(response) = reader.get_root::<rpc_response::Reader>() {
            // We won't be responding. This is already a response.
            match response.which().unwrap() {
                rpc_response::Which::AppendEntries(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<append_entries_request::Builder>();
                        replica.append_entries_response(from, call, builder)
                    };
                    match respond {
                        Some(Emit) => {
                            // TODO
                            self.emit(builder_message);
                        },
                        None => (),
                    }
                },
                rpc_response::Which::RequestVote(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<append_entries_request::Builder>();
                        replica.request_vote_response(from, call, builder)
                    };
                    match respond {
                        Some(Broadcast) => {
                            // Won an election!
                            self.broadcast(builder_message);
                        },
                        None => (),
                    }

                },
                _ => unimplemented!(),
            }
        } else if let Ok(client_req) = reader.get_root::<client_request::Reader>() {
            let mut should_die = false;
            // We will be responding.
            match client_req.which().unwrap() {
                client_request::Which::Append(Ok(call)) => {
                    let respond = {
                        let builder = builder_message.init_root::<client_response::Builder>();
                        replica.client_append(from, call, builder)
                    };
                    match respond {
                        Some(emit) => {
                            self.emit(builder_message);
                        },
                        None => (),
                    }

                },
                client_request::Which::Die(Ok(call)) => {
                    should_die = true;
                    let mut builder = builder_message.init_root::<client_response::Builder>();
                    builder.set_success(());
                    self.interest.insert(Interest::writable());
                    debug!("Got a Die request from Client({}). Reason: {}", from, call);
                },
                client_request::Which::LeaderRefresh(()) => {
                    let respond = {
                        let builder = builder_message.init_root::<client_response::Builder>();
                        replica.client_leader_refresh(from, builder)
                    };
                    match respond {
                        Some(Emit) => {
                            self.emit(builder_message);
                        },
                        None => (),
                    }
                },
                _ => unimplemented!(),
            };

            // Do this here so that we can send the response.
            if should_die {
                panic!("Got a Die request.");
            }

        } else {
            // It's something we don't understand.
            unimplemented!();
        }
    }

    fn broadcast(&mut self, builder: MallocMessageBuilder) {
        unimplemented!();
    }

    /// Push the new message into `self.next_write`. This does not actually send the message, it
    /// just queues it up.
    pub fn emit(&mut self, mut builder: MallocMessageBuilder) {
        let mut buf = RingBuf::new(RINGBUF_SIZE);
        serialize_packed::write_message(
            &mut buf,
            &mut builder
        ).unwrap();
        self.add_write(buf);
    }

    /// This queues a byte buffer into the write queue. This is used primarily when message has
    /// already been packed.
    pub fn add_write(&mut self, buf: RingBuf) {
        self.next_write.push_back(buf);
    }
}
<|endoftext|>


<commit_before>//! `Server` listens for HTTP requests.
//!
//!```rust
//!# use rustful::server::Server;
//!# use rustful::router::Router;
//!# let routes = [];
//!let server = Server {
//!	router: ~Router::from_routes(routes),
//!	port: 8080
//!};
//!
//!server.run();
//!```

use router::Router;
use request::Request;
use response::Response;
use response::status::NotFound;

use http;
use http::server::{ResponseWriter, Config};
use http::server::request::{AbsoluteUri, AbsolutePath};
use http::headers::content_type::MediaType;
use http::method::Post;

use std::io::net::ip::{SocketAddr, Ipv4Addr, Port};
use std::hashmap::HashMap;
use std::str::from_utf8;
use std::uint;
use std::io::BufReader;

use extra::time;

use HTTP = http::server::Server;

///A handler function for routing.
pub type HandlerFn = fn(&Request, &mut Response);

#[deriving(Clone)]
pub struct Server {
	///A routing tree with response handlers
	router: ~Router<HandlerFn>,

	///The port where the server will listen for requests
	port: Port
}

impl Server {
	///Start the server and run forever.
	///This will only return if the initial connection fails.
	pub fn run(&self) {
		self.clone().serve_forever();
	}
}

impl HTTP for Server {
	fn get_config(&self) -> Config {
		Config {
			bind_address: SocketAddr {
				ip: Ipv4Addr(0, 0, 0, 0),
				port: self.port
			}
		}
	}

	fn handle_request(&self, request: &http::server::request::Request, writer: &mut ResponseWriter) {
		let mut response = Response::new(writer);
		response.headers.date = Some(time::now_utc());
		response.headers.content_type = Some(MediaType {
			type_: ~"text",
			subtype: ~"plain",
			parameters: ~[(~"charset", ~"UTF-8")]
		});
		response.headers.server = Some(~"rustful");

		let found = match build_request(request) {
			Some(mut request) => match self.router.find(request.method.clone(), request.path) {
				Some((&handler, variables)) => {
					request.variables = variables;
					handler(&request, response);
					true
				},
				None => false
			},
			None => false
		};

		if !found {
			let content = bytes!("File not found");
			response.headers.content_length = Some(content.len());
			response.status = NotFound;
			match response.write(content.to_owned()) {
				Err(e) => println!("error while writing 404: {}", e),
				_ => {}
			}
		}
	}
}

fn build_request(request: &http::server::request::Request) -> Option<Request> {
	let path = match request.request_uri {
		AbsoluteUri(ref url) => {
			let query = ~url.query.iter().map(|&(ref a, ref b)| {
				(a.to_str(), b.to_str())
			}).collect();
			Some((url.path.to_str(), query, url.fragment.to_str()))
		},
		AbsolutePath(ref path) => Some(parse_path(path.to_str())),
		_ => None
	};

	match path {
		Some((path, query, fragment)) => {
			let post = if request.method == Post {
				parse_parameters(request.body)
			} else {
				~HashMap::new()
			};

			Some(Request {
				headers: request.headers.clone(),
				method: request.method.clone(),
				path: path.to_owned(),
				variables: ~HashMap::new(),
				post: post,
				query: query,
				fragment: fragment,
				body: request.body.to_str()
			})
		}
		None => None
	}
}

fn parse_parameters(source: &str) -> ~HashMap<~str, ~str> {
	let mut parameters = ~HashMap::new();
	for parameter in source.split('&') {
		let parts = parameter.split('=').collect::<~[&str]>();
		match parts.as_slice() {
			[name, value] => {
				parameters.insert(url_decode(name), url_decode(value));
			},
			[name] => {
				parameters.insert(url_decode(name), ~"");
			},
			[name, value, ..] => {
				parameters.insert(url_decode(name), url_decode(value));
			},
			[] => {}
		}
	}

	parameters
}

fn parse_path(path: &str) -> (~str, ~HashMap<~str, ~str>, ~str) {
	match path.find('?') {
		Some(index) => {
			let (query, fragment) = parse_fragment(path.slice(index+1, path.len()));
			(path.slice(0, index).to_str(), parse_parameters(query), fragment)
		},
		None => {
			let (path, fragment) = parse_fragment(path);
			(path, ~HashMap::new(), fragment)
		}
	}
}

fn parse_fragment(path: &str) -> (~str, ~str) {
	match path.find('#') {
		Some(index) => (path.slice(0, index).to_str(), path.slice(index+1, path.len()).to_str()),
		None => (path.to_str(), ~"")
	}
}

fn url_decode(string: &str) -> ~str {
	let mut rdr = BufReader::new(string.as_bytes());
	let mut out = ~[];

	loop {
		let mut buf = [0];
		let ch = match rdr.read(buf) {
			Err(..) => break,
			Ok(..) => buf[0] as char
		};
		match ch {
			'%' => {
				let mut bytes = [0, 0];
				match rdr.read(bytes) {
					Ok(2) => {}
					_ => fail!()
				}
				let ch = uint::parse_bytes(bytes, 16u).unwrap() as u8;
				out.push(ch);
			}
			ch => out.push(ch as u8)
		}
	}

	match from_utf8(out) {
		Some(result) => result.to_str(),
		None => string.to_str()
	}
}

#[test]
fn parsing_parameters() {
	let parameters = parse_parameters("a=1&aa=2&ab=202");
	let a = ~"1";
	let aa = ~"2";
	let ab = ~"202";
	assert_eq!(parameters.find(&~"a"), Some(&a));
	assert_eq!(parameters.find(&~"aa"), Some(&aa));
	assert_eq!(parameters.find(&~"ab"), Some(&ab));
}

#[test]
fn parsing_strange_parameters() {
	let parameters = parse_parameters("a=1=2&=2&ab=");
	let a = ~"1";
	let aa = ~"2";
	let ab = ~"";
	assert_eq!(parameters.find(&~"a"), Some(&a));
	assert_eq!(parameters.find(&~""), Some(&aa));
	assert_eq!(parameters.find(&~"ab"), Some(&ab));
}

#[test]
fn parse_path_parts() {
	let (path, query, fragment) = parse_path("/path/to/something?whith=this&and=that#lol");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query, ~"whith=this&and=that");
	assert_eq!(fragment, ~"lol");
}

#[test]
fn parse_strange_path() {
	let (path, query, fragment) = parse_path("/path/to/something?whith=this&and=what?#");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query, ~"whith=this&and=what?");
	assert_eq!(fragment, ~"");
}

#[test]
fn parse_missing_path_parts() {
	let (path, query, fragment) = parse_path("/path/to/something?whith=this&and=that");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query, ~"whith=this&and=that");
	assert_eq!(fragment, ~"");

	let (path, query, fragment) = parse_path("/path/to/something#lol");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query, ~"");
	assert_eq!(fragment, ~"lol");

	let (path, query, fragment) = parse_path("?whith=this&and=that#lol");
	assert_eq!(path, ~"");
	assert_eq!(query, ~"whith=this&and=that");
	assert_eq!(fragment, ~"lol");
}<commit_msg>Fixed errors in tests<commit_after>//! `Server` listens for HTTP requests.
//!
//!```rust
//!# use rustful::server::Server;
//!# use rustful::router::Router;
//!# let routes = [];
//!let server = Server {
//!	router: ~Router::from_routes(routes),
//!	port: 8080
//!};
//!
//!server.run();
//!```

use router::Router;
use request::Request;
use response::Response;
use response::status::NotFound;

use http;
use http::server::{ResponseWriter, Config};
use http::server::request::{AbsoluteUri, AbsolutePath};
use http::headers::content_type::MediaType;
use http::method::Post;

use std::io::net::ip::{SocketAddr, Ipv4Addr, Port};
use std::hashmap::HashMap;
use std::str::from_utf8;
use std::uint;
use std::io::BufReader;

use extra::time;

use HTTP = http::server::Server;

///A handler function for routing.
pub type HandlerFn = fn(&Request, &mut Response);

#[deriving(Clone)]
pub struct Server {
	///A routing tree with response handlers
	router: ~Router<HandlerFn>,

	///The port where the server will listen for requests
	port: Port
}

impl Server {
	///Start the server and run forever.
	///This will only return if the initial connection fails.
	pub fn run(&self) {
		self.clone().serve_forever();
	}
}

impl HTTP for Server {
	fn get_config(&self) -> Config {
		Config {
			bind_address: SocketAddr {
				ip: Ipv4Addr(0, 0, 0, 0),
				port: self.port
			}
		}
	}

	fn handle_request(&self, request: &http::server::request::Request, writer: &mut ResponseWriter) {
		let mut response = Response::new(writer);
		response.headers.date = Some(time::now_utc());
		response.headers.content_type = Some(MediaType {
			type_: ~"text",
			subtype: ~"plain",
			parameters: ~[(~"charset", ~"UTF-8")]
		});
		response.headers.server = Some(~"rustful");

		let found = match build_request(request) {
			Some(mut request) => match self.router.find(request.method.clone(), request.path) {
				Some((&handler, variables)) => {
					request.variables = variables;
					handler(&request, response);
					true
				},
				None => false
			},
			None => false
		};

		if !found {
			let content = bytes!("File not found");
			response.headers.content_length = Some(content.len());
			response.status = NotFound;
			match response.write(content.to_owned()) {
				Err(e) => println!("error while writing 404: {}", e),
				_ => {}
			}
		}
	}
}

fn build_request(request: &http::server::request::Request) -> Option<Request> {
	let path = match request.request_uri {
		AbsoluteUri(ref url) => {
			let query = ~url.query.iter().map(|&(ref a, ref b)| {
				(a.to_str(), b.to_str())
			}).collect();
			Some((url.path.to_str(), query, url.fragment.to_str()))
		},
		AbsolutePath(ref path) => Some(parse_path(path.to_str())),
		_ => None
	};

	match path {
		Some((path, query, fragment)) => {
			let post = if request.method == Post {
				parse_parameters(request.body)
			} else {
				~HashMap::new()
			};

			Some(Request {
				headers: request.headers.clone(),
				method: request.method.clone(),
				path: path.to_owned(),
				variables: ~HashMap::new(),
				post: post,
				query: query,
				fragment: fragment,
				body: request.body.to_str()
			})
		}
		None => None
	}
}

fn parse_parameters(source: &str) -> ~HashMap<~str, ~str> {
	let mut parameters = ~HashMap::new();
	for parameter in source.split('&') {
		let parts = parameter.split('=').collect::<~[&str]>();
		match parts.as_slice() {
			[name, value] => {
				parameters.insert(url_decode(name), url_decode(value));
			},
			[name] => {
				parameters.insert(url_decode(name), ~"");
			},
			[name, value, ..] => {
				parameters.insert(url_decode(name), url_decode(value));
			},
			[] => {}
		}
	}

	parameters
}

fn parse_path(path: &str) -> (~str, ~HashMap<~str, ~str>, ~str) {
	match path.find('?') {
		Some(index) => {
			let (query, fragment) = parse_fragment(path.slice(index+1, path.len()));
			(path.slice(0, index).to_str(), parse_parameters(query), fragment)
		},
		None => {
			let (path, fragment) = parse_fragment(path);
			(path, ~HashMap::new(), fragment)
		}
	}
}

fn parse_fragment(path: &str) -> (~str, ~str) {
	match path.find('#') {
		Some(index) => (path.slice(0, index).to_str(), path.slice(index+1, path.len()).to_str()),
		None => (path.to_str(), ~"")
	}
}

fn url_decode(string: &str) -> ~str {
	let mut rdr = BufReader::new(string.as_bytes());
	let mut out = ~[];

	loop {
		let mut buf = [0];
		let ch = match rdr.read(buf) {
			Err(..) => break,
			Ok(..) => buf[0] as char
		};
		match ch {
			'%' => {
				let mut bytes = [0, 0];
				match rdr.read(bytes) {
					Ok(2) => {}
					_ => fail!()
				}
				let ch = uint::parse_bytes(bytes, 16u).unwrap() as u8;
				out.push(ch);
			}
			ch => out.push(ch as u8)
		}
	}

	match from_utf8(out) {
		Some(result) => result.to_str(),
		None => string.to_str()
	}
}

#[test]
fn parsing_parameters() {
	let parameters = parse_parameters("a=1&aa=2&ab=202");
	let a = ~"1";
	let aa = ~"2";
	let ab = ~"202";
	assert_eq!(parameters.find(&~"a"), Some(&a));
	assert_eq!(parameters.find(&~"aa"), Some(&aa));
	assert_eq!(parameters.find(&~"ab"), Some(&ab));
}

#[test]
fn parsing_strange_parameters() {
	let parameters = parse_parameters("a=1=2&=2&ab=");
	let a = ~"1";
	let aa = ~"2";
	let ab = ~"";
	assert_eq!(parameters.find(&~"a"), Some(&a));
	assert_eq!(parameters.find(&~""), Some(&aa));
	assert_eq!(parameters.find(&~"ab"), Some(&ab));
}

#[test]
fn parse_path_parts() {
	let with = ~"this";
	let and = ~"that";
	let (path, query, fragment) = parse_path("/path/to/something?with=this&and=that#lol");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query.find(&~"with"), Some(&with));
	assert_eq!(query.find(&~"and"), Some(&and));
	assert_eq!(fragment, ~"lol");
}

#[test]
fn parse_strange_path() {
	let with = ~"this";
	let and = ~"what?";
	let (path, query, fragment) = parse_path("/path/to/something?with=this&and=what?#");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query.find(&~"with"), Some(&with));
	assert_eq!(query.find(&~"and"), Some(&and));
	assert_eq!(fragment, ~"");
}

#[test]
fn parse_missing_path_parts() {
	let with = ~"this";
	let and = ~"that";
	let (path, query, fragment) = parse_path("/path/to/something?with=this&and=that");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query.find(&~"with"), Some(&with));
	assert_eq!(query.find(&~"and"), Some(&and));
	assert_eq!(fragment, ~"");

	let (path, query, fragment) = parse_path("/path/to/something#lol");
	assert_eq!(path, ~"/path/to/something");
	assert_eq!(query.len(), 0);
	assert_eq!(fragment, ~"lol");

	let (path, query, fragment) = parse_path("?with=this&and=that#lol");
	assert_eq!(path, ~"");
	assert_eq!(query.find(&~"with"), Some(&with));
	assert_eq!(query.find(&~"and"), Some(&and));
	assert_eq!(fragment, ~"lol");
}<|endoftext|>


<commit_before>pub static VS_COLORED_120: &'static str = "
#version 120
uniform vec4 color;

attribute vec2 position;

void main()
{
    gl_Position = vec4(position, 0.0, 1.0);
}
";

pub static FS_COLORED_120: &'static str = "
#version 120
uniform vec4 color;

void main()
{
    gl_FragColor = color;
}
";

pub static VS_TEXTURED_120: &'static str = "
#version 120
uniform vec4 color;

attribute vec2 position;
attribute vec2 texcoord;

uniform sampler2D s_texture;

varying vec2 v_texcoord;

void main()
{
    v_texcoord = texcoord;
    gl_Position = vec4(position, 0.0, 1.0);
}
";

pub static FS_TEXTURED_120: &'static str = "
#version 120
uniform vec4 color;
uniform sampler2D s_texture;

varying vec2 v_texcoord;

void main()
{
    gl_FragColor = texture2D(s_texture, v_texcoord) * color;
}
";
<commit_msg>Add GLSL 1.50 shaders<commit_after>pub static VS_COLORED_120: &'static str = "
#version 120
uniform vec4 color;

attribute vec2 position;

void main()
{
    gl_Position = vec4(position, 0.0, 1.0);
}
";

pub static VS_COLORED_150: &'static str = "
#version 150 core
uniform vec4 color;

in vec2 position;

void main()
{
    gl_Position = vec4(position, 0.0, 1.0);
}
";

pub static FS_COLORED_120: &'static str = "
#version 120
uniform vec4 color;

void main()
{
    gl_FragColor = color;
}
";

pub static FS_COLORED_150: &'static str = "
#version 150 core
uniform vec4 color;

out vec4 o_Color;

void main()
{
    o_Color = color;
}
";

pub static VS_TEXTURED_120: &'static str = "
#version 120
uniform vec4 color;

attribute vec2 position;
attribute vec2 texcoord;

uniform sampler2D s_texture;

varying vec2 v_texcoord;

void main()
{
    v_texcoord = texcoord;
    gl_Position = vec4(position, 0.0, 1.0);
}
";

pub static VS_TEXTURED_150: &'static str = "
#version 150 core
uniform sampler2D s_texture;
uniform vec4 color;

in vec2 position;
in vec2 texcoord;

out vec2 v_texcoord;

void main()
{
    v_texcoord = texcoord;
    gl_Position = vec4(position, 0.0, 1.0);
}
";

pub static FS_TEXTURED_120: &'static str = "
#version 120
uniform vec4 color;
uniform sampler2D s_texture;

varying vec2 v_texcoord;

void main()
{
    gl_FragColor = texture2D(s_texture, v_texcoord) * color;
}
";

pub static FS_TEXTURED_150: &'static str = "
#version 150 core
uniform sampler2D s_texture;
uniform vec4 color;

out vec4 o_Color;

in vec2 v_texcoord;

void main()
{
    o_Color = texture(s_texture, v_texcoord) * color;
}
";
<|endoftext|>