<commit_before>use std::mem;
use std::ptr;
use std::io::{File, BufferedReader};

use utils;
use cursor::Cursor;

pub struct Buffer {
    length: uint,
    pub first_line: Link,
    pub last_line: Rawlink,

    pub cursor: Cursor,
}

impl Buffer {
    pub fn new() -> Buffer {
        Buffer {
            length: 0,
            first_line: None,
            last_line: Rawlink::none(),
            cursor: Cursor::new(),
        }
    }

    pub fn new_from_file(path: &Path) -> Buffer {
        let mut file = BufferedReader::new(File::open(path));
        let lines: Vec<String> = file.lines().map(|x| x.unwrap()).collect();
        let mut buffer = Buffer::new();

        for line in lines.iter() {
            buffer.add_line(line.clone());
        }

        // FIXME: This doesn't seem like a good idea..
        // Update the `cursor.line` to point to the `buffer.first_line`
        let b = buffer.first_line.clone();
        let cursor = Cursor{x:0, y:0, line: b};
        mem::replace(&mut buffer.cursor, cursor);

        buffer
    }

    pub fn draw_contents(&self) {
        for (index, line) in self.iter_lines().enumerate() {
            utils::draw(index, line.value.clone());
        }
    }

    pub fn push_front_line(&mut self, mut new_head: Box<Line>) {
        match self.first_line {
            None => {
                self.last_line = Rawlink::some(&mut *new_head);
                self.first_line = link_with_prev(new_head, Rawlink::none());
            }
            Some(ref mut head) => {
                new_head.prev = Rawlink::none();
                head.prev = Rawlink::some(&mut *new_head);
                mem::swap(head, &mut new_head);
                head.next = Some(new_head);
            }
        }
        self.length += 1;
    }

    pub fn push_back_line(&mut self, mut new_tail: Box<Line>) {
        match self.last_line.resolve() {
            None => return self.push_front_line(new_tail),
            Some(tail) => {
                self.last_line = Rawlink::some(&mut *new_tail);
                tail.next = link_with_prev(new_tail, Rawlink::some(tail));
            }
        }
        self.length += 1
    }

    pub fn add_line(&mut self, elt: String) {
        self.push_back_line(box Line::new(elt));
    }

    pub fn len(&self) -> uint {
        self.length
    }

    pub fn iter_lines<'a>(&'a self) -> Items<'a> {
        Items { nelem: self.len(), head: &self.first_line, tail: self.last_line }
    }
}

fn link_with_prev(mut next: Box<Line>, prev: Rawlink) -> Link {
    next.prev = prev;
    Some(next)
}

struct Items<'a> {
    head: &'a Link,
    tail: Rawlink,
    nelem: uint,
}

impl<'a> Iterator<&'a Line> for Items<'a> {
    #[inline]
    fn next(&mut self) -> Option<&'a Line> {
        if self.nelem == 0 {
            return None;
        }
        self.head.as_ref().map(|head| {
            self.nelem -= 1;
            self.head = &head.next;
            &**head
        })
    }
}

#[deriving(Clone)]
pub struct Line {
    pub next: Link,
    pub prev: Rawlink,
    value: String,
}

impl Line {
    pub fn new(v: String) -> Line {
        Line{value: v, next: None, prev: Rawlink::none()}
    }

    pub fn len(&self) -> uint {
        self.value.len()
    }
}

pub type Link = Option<Box<Line>>;

#[deriving(Clone)]
struct Rawlink {
    p: *mut Line,
}

impl Rawlink {
    fn none() -> Rawlink {
        Rawlink{p: ptr::null_mut()}
    }

    fn some(n: &mut Line) -> Rawlink {
        Rawlink{p: n}
    }

    pub fn resolve<'a>(&mut self) -> Option<&'a mut Line> {
        if self.p.is_null() {
            None
        } else {
            Some(unsafe { mem::transmute(self.p) })
        }
    }
}
<commit_msg>Added some comments to the buffer module<commit_after>use std::mem;
use std::ptr;
use std::io::{File, BufferedReader};

use utils;
use cursor::Cursor;

pub struct Buffer {
    length: uint,
    pub first_line: Link,
    pub last_line: Rawlink,

    pub cursor: Cursor,
}

impl Buffer {
    /// Create a new buffer instance
    pub fn new() -> Buffer {
        Buffer {
            length: 0,
            first_line: None,
            last_line: Rawlink::none(),
            cursor: Cursor::new(),
        }
    }

    /// Create a new buffer instance and load the given file
    pub fn new_from_file(path: &Path) -> Buffer {
        let mut file = BufferedReader::new(File::open(path));
        let lines: Vec<String> = file.lines().map(|x| x.unwrap()).collect();
        let mut buffer = Buffer::new();

        // for every line in the file we add a corresponding line to the buffer
        for line in lines.iter() {
            buffer.add_line(line.clone());
        }

        // FIXME: This doesn't seem like a good idea..
        // Update the `cursor.line` to point to the `buffer.first_line`
        let b = buffer.first_line.clone();
        let cursor = Cursor{x:0, y:0, line: b};
        mem::replace(&mut buffer.cursor, cursor);

        buffer
    }

    /// Draw the contents of the buffer
    ///
    /// Loops over each line in the buffer and draws it to the screen
    pub fn draw_contents(&self) {
        for (index, line) in self.iter_lines().enumerate() {
            utils::draw(index, line.value.clone());
        }
    }

    /// Add a line to the front/top of the buffer
    pub fn push_front_line(&mut self, mut new_head: Box<Line>) {
        match self.first_line {
            None => {
                self.last_line = Rawlink::some(&mut *new_head);
                self.first_line = link_with_prev(new_head, Rawlink::none());
            }
            Some(ref mut head) => {
                new_head.prev = Rawlink::none();
                head.prev = Rawlink::some(&mut *new_head);
                mem::swap(head, &mut new_head);
                head.next = Some(new_head);
            }
        }
        self.length += 1;
    }

    /// Add a line to the back/end of the buffer
    ///
    /// If there is no last_line, ie the buffer is empty, then it will
    /// defer to `push_front_line`.
    pub fn push_back_line(&mut self, mut new_tail: Box<Line>) {
        match self.last_line.resolve() {
            None => return self.push_front_line(new_tail),
            Some(tail) => {
                self.last_line = Rawlink::some(&mut *new_tail);
                tail.next = link_with_prev(new_tail, Rawlink::some(tail));
            }
        }
        self.length += 1
    }

    /// Add a line to the buffer
    pub fn add_line(&mut self, elt: String) {
        self.push_back_line(box Line::new(elt));
    }

    /// Get the length of the buffer - number of lines
    pub fn len(&self) -> uint {
        self.length
    }

    /// Iterate over each line in the buffer
    pub fn iter_lines<'a>(&'a self) -> Items<'a> {
        Items { nelem: self.len(), head: &self.first_line, tail: self.last_line }
    }
}

fn link_with_prev(mut next: Box<Line>, prev: Rawlink) -> Link {
    next.prev = prev;
    Some(next)
}

struct Items<'a> {
    head: &'a Link,
    tail: Rawlink,
    nelem: uint,
}

impl<'a> Iterator<&'a Line> for Items<'a> {
    #[inline]
    fn next(&mut self) -> Option<&'a Line> {
        if self.nelem == 0 {
            return None;
        }
        self.head.as_ref().map(|head| {
            self.nelem -= 1;
            self.head = &head.next;
            &**head
        })
    }
}

#[deriving(Clone)]
pub struct Line {
    pub next: Link,
    pub prev: Rawlink,
    value: String,
}

impl Line {
    /// Create a new line instance
    pub fn new(v: String) -> Line {
        Line{value: v, next: None, prev: Rawlink::none()}
    }

    /// Get the length of the current line
    pub fn len(&self) -> uint {
        self.value.len()
    }
}

pub type Link = Option<Box<Line>>;

#[deriving(Clone)]
struct Rawlink {
    p: *mut Line,
}

impl Rawlink {
    fn none() -> Rawlink {
        Rawlink{p: ptr::null_mut()}
    }

    fn some(n: &mut Line) -> Rawlink {
        Rawlink{p: n}
    }

    pub fn resolve<'a>(&mut self) -> Option<&'a mut Line> {
        if self.p.is_null() {
            None
        } else {
            Some(unsafe { mem::transmute(self.p) })
        }
    }
}
<|endoftext|>


<commit_before>//! The SpaceAPI server struct.

use std::net::ToSocketAddrs;
use std::sync::Arc;
use std::time::Duration;

use r2d2;
use r2d2_redis::RedisConnectionManager;
use iron::Iron;
use router::Router;
use redis::{IntoConnectionInfo, ConnectionInfo};

use serde_json::Value;
use serde_json::map::Map;

mod handlers;

use crate::api;

use crate::sensors;
use crate::modifiers;
use crate::errors::SpaceapiServerError;
use crate::types::RedisPool;

pub struct SpaceapiServerBuilder {
    status: api::Status,
    pool: Result<r2d2::Pool<r2d2_redis::RedisConnectionManager>, SpaceapiServerError>,
    sensor_specs: Vec<sensors::SensorSpec>,
    status_modifiers: Vec<Box<modifiers::StatusModifier>>,
}

impl SpaceapiServerBuilder {

    pub fn new(mut status: api::Status) -> SpaceapiServerBuilder {
        // Instantiate versions object
        let mut versions = Map::new();
        versions.insert("spaceapi-rs".into(), api::get_version().into());
        versions.insert("spaceapi-server-rs".into(), crate::get_version().into());

        // Add to extensions
        status.extensions.insert("versions".into(), Value::Object(versions));

        SpaceapiServerBuilder {
            status,
            pool: Err("redis_connection_info missing".into()),
            sensor_specs: vec![],
            status_modifiers: vec![],
        }
    }

    pub fn redis_connection_info<R: IntoConnectionInfo>(mut self, redis_connection_info: R) -> Self {
        let redis_connection_info  = redis_connection_info
            .into_connection_info();

        // Log some useful debug information
        debug!("Redis connection info: {:?}", &redis_connection_info);

        // Create redis pool
        let redis_config = r2d2::Config::builder()
            // Provide up to 6 connections in connection pool
            .pool_size(6)
            // At least 1 connection must be active
            .min_idle(Some(2))
            // Initialize connection pool lazily. This allows the SpaceAPI
            // server to work even without a database connection.
            .initialization_fail_fast(false)
            // Try to get a connection for max 1 second
            .connection_timeout(Duration::from_secs(1))
            // Don't log errors directly.
            // They can get quite verbose, and we're already catching and
            // logging the corresponding results anyways.
            .error_handler(Box::new(r2d2::NopErrorHandler))
            .build();
        let redis_manager = RedisConnectionManager::new(redis_connection_info.unwrap());
        self.pool = r2d2::Pool::new(redis_config, redis_manager.unwrap()).or(Err("Pool creation failed".into()));
        self
    }

    pub fn redis_pool(mut self, redis_pool: r2d2::Pool<r2d2_redis::RedisConnectionManager>) -> Self {
        self.pool = Ok(redis_pool);
        self
    }

    pub fn add_status_modifier<M: modifiers::StatusModifier + 'static>(mut self, modifier: M) -> Self {
        self.status_modifiers.push(Box::new(modifier));
        self
    }

    /// Add a new sensor.
    ///
    /// The first argument is a ``api::SensorTemplate`` instance containing all static data.
    /// The second argument specifies how to get the actual sensor value from Redis.
    pub fn add_sensor<T: api::SensorTemplate + 'static>(mut self, template: T, data_key: String) -> Self {
        self.sensor_specs.push(
            sensors::SensorSpec { template: Box::new(template), data_key }
        );
        self
    }

    pub fn build(self) -> Result<SpaceapiServer, SpaceapiServerError> {

        Ok(SpaceapiServer {
            status: self.status,
            redis_pool: self.pool.expect("Error initializing redis pool"),
            sensor_specs: Arc::new(self.sensor_specs),
            status_modifiers: self.status_modifiers,
        })
    }
}

/// A Space API server instance.
///
/// You can create a new instance using the ``new`` constructor method by
/// passing it the host, the port, the ``Status`` object and a redis connection info object.
///
/// The ``SpaceapiServer`` includes a web server through
/// [Hyper](http://hyper.rs/hyper/hyper/server/index.html). Simply call the ``serve`` method.
pub struct SpaceapiServer {
    status: api::Status,
    redis_pool: RedisPool,
    sensor_specs: sensors::SafeSensorSpecs,
    status_modifiers: Vec<Box<modifiers::StatusModifier>>,
}

impl SpaceapiServer {

    /// Create and return a Router instance.
    fn route(self) -> Router {
        let mut router = Router::new();

        router.get("/", handlers::ReadHandler::new(
            self.status.clone(), self.redis_pool.clone(),
            self.sensor_specs.clone(), self.status_modifiers), "root");

        router.put("/sensors/:sensor/", handlers::UpdateHandler::new(
            self.redis_pool.clone(), self.sensor_specs.clone()), "sensors");

        router
    }

    /// Start a HTTP server listening on ``self.host:self.port``.
    ///
    /// The call returns an `HttpResult<Listening>` object, see
    /// http://ironframework.io/doc/hyper/server/struct.Listening.html
    /// for more information.
    pub fn serve<S: ToSocketAddrs>(self, socket_addr: S) -> crate::HttpResult<crate::Listening> {
        // Launch server process
        let router = self.route();
        println!("Starting HTTP server on:");
        for a in socket_addr.to_socket_addrs()? {
            println!("\thttp://{}", a);
        }
        Iron::new(router).http(socket_addr)
    }
}
<commit_msg>Use a custom enum to handle errors more elegantly<commit_after>//! The SpaceAPI server struct.

use std::net::ToSocketAddrs;
use std::sync::Arc;
use std::time::Duration;

use r2d2;
use r2d2_redis::RedisConnectionManager;
use iron::Iron;
use router::Router;
use redis::{IntoConnectionInfo, ConnectionInfo};

use serde_json::Value;
use serde_json::map::Map;

mod handlers;

use crate::api;

use crate::sensors;
use crate::modifiers;
use crate::errors::SpaceapiServerError;
use crate::types::RedisPool;

enum RedisInfo {
    None,
    Pool(r2d2::Pool<r2d2_redis::RedisConnectionManager>),
    ConnectionInfo(ConnectionInfo),
    Err(SpaceapiServerError),
}

pub struct SpaceapiServerBuilder {
    status: api::Status,
    redis_info: RedisInfo,
    sensor_specs: Vec<sensors::SensorSpec>,
    status_modifiers: Vec<Box<modifiers::StatusModifier>>,
}

impl SpaceapiServerBuilder {

    pub fn new(mut status: api::Status) -> SpaceapiServerBuilder {
        // Instantiate versions object
        let mut versions = Map::new();
        versions.insert("spaceapi-rs".into(), api::get_version().into());
        versions.insert("spaceapi-server-rs".into(), crate::get_version().into());

        // Add to extensions
        status.extensions.insert("versions".into(), Value::Object(versions));

        SpaceapiServerBuilder {
            status,
            redis_info: RedisInfo::None,
            sensor_specs: vec![],
            status_modifiers: vec![],
        }
    }

    pub fn redis_connection_info<R: IntoConnectionInfo>(mut self, redis_connection_info: R) -> Self {
        self.redis_info = match redis_connection_info.into_connection_info() {
            Ok(ci) => RedisInfo::ConnectionInfo(ci),
            Err(e) => RedisInfo::Err(e.into()),
        };
        self
    }

    pub fn redis_pool(mut self, redis_pool: r2d2::Pool<r2d2_redis::RedisConnectionManager>) -> Self {
        self.redis_info = RedisInfo::Pool(redis_pool);
        self
    }

    pub fn add_status_modifier<M: modifiers::StatusModifier + 'static>(mut self, modifier: M) -> Self {
        self.status_modifiers.push(Box::new(modifier));
        self
    }

    /// Add a new sensor.
    ///
    /// The first argument is a ``api::SensorTemplate`` instance containing all static data.
    /// The second argument specifies how to get the actual sensor value from Redis.
    pub fn add_sensor<T: api::SensorTemplate + 'static>(mut self, template: T, data_key: String) -> Self {
        self.sensor_specs.push(
            sensors::SensorSpec { template: Box::new(template), data_key }
        );
        self
    }

    pub fn build(self) -> Result<SpaceapiServer, SpaceapiServerError> {

        let pool = match self.redis_info {
            RedisInfo::None => Err("No redis connection defined".into()),
            RedisInfo::Err(e) => Err(e),
            RedisInfo::Pool(p) => Ok(p),
            RedisInfo::ConnectionInfo(ci) => {
                // Log some useful debug information
                debug!("Connecting to redis database {} at {:?}",
                       ci.db, ci.addr);

                // Create redis pool
                let redis_config = r2d2::Config::builder()
                    // Provide up to 6 connections in connection pool
                    .pool_size(6)
                    // At least 1 connection must be active
                    .min_idle(Some(2))
                    // Initialize connection pool lazily. This allows the SpaceAPI
                    // server to work even without a database connection.
                    .initialization_fail_fast(false)
                    // Try to get a connection for max 1 second
                    .connection_timeout(Duration::from_secs(1))
                    // Don't log errors directly.
                    // They can get quite verbose, and we're already catching and
                    // logging the corresponding results anyways.
                    .error_handler(Box::new(r2d2::NopErrorHandler))
                    .build();
                let redis_manager = RedisConnectionManager::new(ci)?;
                Ok(r2d2::Pool::new(redis_config, redis_manager)?)
            }
        };

        Ok(SpaceapiServer {
            status: self.status,
            redis_pool: pool?,
            sensor_specs: Arc::new(self.sensor_specs),
            status_modifiers: self.status_modifiers,
        })
    }
}

/// A Space API server instance.
///
/// You can create a new instance using the ``new`` constructor method by
/// passing it the host, the port, the ``Status`` object and a redis connection info object.
///
/// The ``SpaceapiServer`` includes a web server through
/// [Hyper](http://hyper.rs/hyper/hyper/server/index.html). Simply call the ``serve`` method.
pub struct SpaceapiServer {
    status: api::Status,
    redis_pool: RedisPool,
    sensor_specs: sensors::SafeSensorSpecs,
    status_modifiers: Vec<Box<modifiers::StatusModifier>>,
}

impl SpaceapiServer {

    /// Create and return a Router instance.
    fn route(self) -> Router {
        let mut router = Router::new();

        router.get("/", handlers::ReadHandler::new(
            self.status.clone(), self.redis_pool.clone(),
            self.sensor_specs.clone(), self.status_modifiers), "root");

        router.put("/sensors/:sensor/", handlers::UpdateHandler::new(
            self.redis_pool.clone(), self.sensor_specs.clone()), "sensors");

        router
    }

    /// Start a HTTP server listening on ``self.host:self.port``.
    ///
    /// The call returns an `HttpResult<Listening>` object, see
    /// http://ironframework.io/doc/hyper/server/struct.Listening.html
    /// for more information.
    pub fn serve<S: ToSocketAddrs>(self, socket_addr: S) -> crate::HttpResult<crate::Listening> {
        // Launch server process
        let router = self.route();
        println!("Starting HTTP server on:");
        for a in socket_addr.to_socket_addrs()? {
            println!("\thttp://{}", a);
        }
        Iron::new(router).http(socket_addr)
    }
}
<|endoftext|>


<commit_before>//! Updates crates.io index and builds new packages

use super::{DocBuilder, RustwideBuilder};
use crate::db::connect_db;
use crate::error::Result;
use crate::utils::{add_crate_to_queue, get_crate_priority};
use crates_index_diff::{ChangeKind, Index};
use log::{debug, error};

impl DocBuilder {
    /// Updates crates.io-index repository and adds new crates into build queue.
    /// Returns size of queue
    pub fn get_new_crates(&mut self) -> Result<usize> {
        let conn = connect_db()?;
        let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?;
        let (mut changes, oid) = index.peek_changes()?;

        // I believe this will fix ordering of queue if we get more than one crate from changes
        changes.reverse();

        for krate in &changes {
            match krate.kind {
                ChangeKind::Yanked => {
                    let res = conn.execute(
                        "
                        UPDATE releases
                            SET yanked = TRUE
                        FROM crates
                        WHERE crates.id = releases.crate_id
                            AND name = $1
                            AND version = $2
                        ",
                        &[&krate.name, &krate.version],
                    );
                    match res {
                        Ok(_) => debug!("{}-{} yanked", krate.name, krate.version),
                        Err(err) => error!(
                            "error while setting {}-{} to yanked: {}",
                            krate.name, krate.version, err
                        ),
                    }
                }

                ChangeKind::Added => {
                    let priority = get_crate_priority(&conn, &krate.name)?;

                    match add_crate_to_queue(&conn, &krate.name, &krate.version, priority) {
                        Ok(()) => debug!("{}-{} added into build queue", krate.name, krate.version),
                        Err(err) => error!(
                            "failed adding {}-{} into build queue: {}",
                            krate.name, krate.version, err
                        ),
                    }
                }
            }
        }

        index.set_last_seen_reference(oid)?;

        // Get the size of the queue
        let query = conn.query("SELECT COUNT(*) as size FROM queue", &[])?;
        let queue_size = query.iter().next().unwrap().get::<_, i64>("size") as usize;

        Ok(queue_size)
    }

    pub fn get_queue_count(&self) -> Result<i64> {
        let conn = connect_db()?;

        Ok(conn
            .query("SELECT COUNT(*) FROM queue WHERE attempt < 5", &[])?
            .get(0)
            .get(0))
    }

    /// Builds the top package from the queue. Returns whether the queue was empty.
    pub(crate) fn build_next_queue_package(
        &mut self,
        builder: &mut RustwideBuilder,
    ) -> Result<bool> {
        let conn = connect_db()?;

        let query = conn.query(
            "SELECT id, name, version
                                     FROM queue
                                     WHERE attempt < 5
                                     ORDER BY priority ASC, attempt ASC, id ASC
                                     LIMIT 1",
            &[],
        )?;

        if query.is_empty() {
            // nothing in the queue; bail
            return Ok(false);
        }

        let id: i32 = query.get(0).get(0);
        let name: String = query.get(0).get(1);
        let version: String = query.get(0).get(2);

        match builder.build_package(self, &name, &version, None) {
            Ok(_) => {
                let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
                crate::web::metrics::TOTAL_BUILDS.inc();
            }
            Err(e) => {
                // Increase attempt count
                let rows = conn.query(
                    "UPDATE queue SET attempt = attempt + 1 WHERE id = $1 RETURNING attempt",
                    &[&id],
                )?;
                let attempt: i32 = rows.get(0).get(0);
                if attempt >= 5 {
                    crate::web::metrics::FAILED_BUILDS.inc();
                    crate::web::metrics::TOTAL_BUILDS.inc();
                }
                error!(
                    "Failed to build package {}-{} from queue: {}",
                    name, version, e
                )
            }
        }

        Ok(true)
    }
}

#[cfg(test)]
mod test {
    use crate::{DocBuilder, DocBuilderOptions};
    use log::error;
    use std::path::PathBuf;

    #[test]
    #[ignore]
    fn test_get_new_crates() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.get_new_crates();
        if res.is_err() {
            error!("{:?}", res);
        }
        assert!(res.is_ok());
    }
}
<commit_msg>Changed comment to reflect code & added back the number of crates added<commit_after>//! Updates crates.io index and builds new packages

use super::{DocBuilder, RustwideBuilder};
use crate::db::connect_db;
use crate::error::Result;
use crate::utils::{add_crate_to_queue, get_crate_priority};
use crates_index_diff::{ChangeKind, Index};
use log::{debug, error};

impl DocBuilder {
    /// Updates crates.io-index repository and adds new crates into build queue.
    /// Returns the number of crates added
    pub fn get_new_crates(&mut self) -> Result<usize> {
        let conn = connect_db()?;
        let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?;
        let (mut changes, oid) = index.peek_changes()?;
        let mut crates_added = 0;

        // I believe this will fix ordering of queue if we get more than one crate from changes
        changes.reverse();

        for krate in &changes {
            match krate.kind {
                ChangeKind::Yanked => {
                    let res = conn.execute(
                        "
                        UPDATE releases
                            SET yanked = TRUE
                        FROM crates
                        WHERE crates.id = releases.crate_id
                            AND name = $1
                            AND version = $2
                        ",
                        &[&krate.name, &krate.version],
                    );
                    match res {
                        Ok(_) => debug!("{}-{} yanked", krate.name, krate.version),
                        Err(err) => error!(
                            "error while setting {}-{} to yanked: {}",
                            krate.name, krate.version, err
                        ),
                    }
                }

                ChangeKind::Added => {
                    let priority = get_crate_priority(&conn, &krate.name)?;

                    match add_crate_to_queue(&conn, &krate.name, &krate.version, priority) {
                        Ok(()) => {
                            debug!("{}-{} added into build queue", krate.name, krate.version);
                            crates_added += 1;
                        }
                        Err(err) => error!(
                            "failed adding {}-{} into build queue: {}",
                            krate.name, krate.version, err
                        ),
                    }
                }
            }
        }

        index.set_last_seen_reference(oid)?;

        Ok(crates_added)
    }

    pub fn get_queue_count(&self) -> Result<i64> {
        let conn = connect_db()?;

        Ok(conn
            .query("SELECT COUNT(*) FROM queue WHERE attempt < 5", &[])?
            .get(0)
            .get(0))
    }

    /// Builds the top package from the queue. Returns whether the queue was empty.
    pub(crate) fn build_next_queue_package(
        &mut self,
        builder: &mut RustwideBuilder,
    ) -> Result<bool> {
        let conn = connect_db()?;

        let query = conn.query(
            "SELECT id, name, version
                                     FROM queue
                                     WHERE attempt < 5
                                     ORDER BY priority ASC, attempt ASC, id ASC
                                     LIMIT 1",
            &[],
        )?;

        if query.is_empty() {
            // nothing in the queue; bail
            return Ok(false);
        }

        let id: i32 = query.get(0).get(0);
        let name: String = query.get(0).get(1);
        let version: String = query.get(0).get(2);

        match builder.build_package(self, &name, &version, None) {
            Ok(_) => {
                let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
                crate::web::metrics::TOTAL_BUILDS.inc();
            }
            Err(e) => {
                // Increase attempt count
                let rows = conn.query(
                    "UPDATE queue SET attempt = attempt + 1 WHERE id = $1 RETURNING attempt",
                    &[&id],
                )?;
                let attempt: i32 = rows.get(0).get(0);
                if attempt >= 5 {
                    crate::web::metrics::FAILED_BUILDS.inc();
                    crate::web::metrics::TOTAL_BUILDS.inc();
                }
                error!(
                    "Failed to build package {}-{} from queue: {}",
                    name, version, e
                )
            }
        }

        Ok(true)
    }
}

#[cfg(test)]
mod test {
    use crate::{DocBuilder, DocBuilderOptions};
    use log::error;
    use std::path::PathBuf;

    #[test]
    #[ignore]
    fn test_get_new_crates() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.get_new_crates();
        if res.is_err() {
            error!("{:?}", res);
        }
        assert!(res.is_ok());
    }
}
<|endoftext|>


<commit_before>use std::collections::BTreeMap;
use std::io::{self, Read, Write, BufRead, Lines};
use std::convert::From;
use std::fmt::Arguments;

use time::Duration;
use bufstream::BufStream;
use version::Version;
use error::{ProtoError, Error, Result};
use reply::Reply;
use status::Status;
use replaygain::ReplayGain;
use song::Song;

// Iterator {{{
struct Pairs<I: Iterator<Item=io::Result<String>>>(I);

impl<I> Iterator for Pairs<I> where I: Iterator<Item=io::Result<String>> {
    type Item = Result<(String, String)>;
    fn next(&mut self) -> Option<Result<(String, String)>> {
        let reply: Option<Result<Reply>> = self.0.next().map(|v| v.map_err(Error::Io).and_then(|s| s.parse::<Reply>().map_err(Error::Parse)));
        match reply {
            Some(Ok(Reply::Pair(a, b))) => Some(Ok((a, b))),
            None | Some(Ok(Reply::Ok)) => None,
            Some(Ok(Reply::Ack(e))) => Some(Err(Error::Server(e))),
            Some(Err(e)) => Some(Err(e)),
        }
    }
}
// }}}

// Client {{{
#[derive(Debug)]
pub struct Client<S: Read+Write> {
    socket: BufStream<S>,
    pub version: Version
}

impl<S: Read+Write> Client<S> {
    pub fn new(socket: S) -> Result<Client<S>> {
        let mut socket = BufStream::new(socket);

        let mut banner = String::new();
        try!(socket.read_line(&mut banner));

        if !banner.starts_with("OK MPD ") {
            return Err(From::from(ProtoError::BadBanner));
        }

        let version = try!(banner[7..].trim().parse::<Version>());

        Ok(Client {
            socket: socket,
            version: version
        })
    }

    fn read_line(&mut self) -> Result<String> {
        let mut buf = String::new();
        try!(self.socket.read_line(&mut buf));
        if buf.ends_with("\n") {
            buf.pop();
        }
        Ok(buf)
    }

    fn read_pairs(&mut self) -> Pairs<Lines<&mut BufStream<S>>> {
        Pairs((&mut self.socket).lines())
    }

    fn read_map(&mut self) -> Result<BTreeMap<String, String>> {
        self.read_pairs().collect()
    }

    fn write_command(&mut self, command: &str) -> Result<()> {
        self.socket.write_all(command.as_bytes())
            .and_then(|_| self.socket.write(&[0x0a]))
            .and_then(|_| self.socket.flush())
            .map_err(From::from)
    }

    fn write_command_args(&mut self, command: Arguments) -> Result<()> {
        self.socket.write_fmt(command)
            .and_then(|_| self.socket.write(&[0x0a]))
            .and_then(|_| self.socket.flush())
            .map_err(From::from)
    }

    fn expect_ok(&mut self) -> Result<()> {
        let line = try!(self.read_line());

        match line.parse::<Reply>() {
            Ok(Reply::Ok) => Ok(()),
            Ok(Reply::Ack(e)) => Err(Error::Server(e)),
            Ok(_) => Err(Error::Proto(ProtoError::NotOk)),
            Err(e) => Err(From::from(e)),
        }
    }

    pub fn status(&mut self) -> Result<Status> {
        self.write_command("status")
            .and_then(|_| self.read_map())
            .and_then(Status::from_map)
    }

    pub fn clearerror(&mut self) -> Result<()> {
        self.write_command("clearerror")
            .and_then(|_| self.expect_ok())
    }

    pub fn volume(&mut self, volume: i8) -> Result<()> {
        self.write_command_args(format_args!("setvol {}", volume))
            .and_then(|_| self.expect_ok())
    }

    pub fn repeat(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("repeat {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn random(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("random {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn single(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("single {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn consume(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("consume {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn crossfade(&mut self, value: u64) -> Result<()> {
        self.write_command_args(format_args!("crossfade {}", value))
            .and_then(|_| self.expect_ok())
    }

    pub fn mixrampdb(&mut self, value: f32) -> Result<()> {
        self.write_command_args(format_args!("mixrampdb {}", value))
            .and_then(|_| self.expect_ok())
    }

    pub fn mixrampdelay<T: IntoSeconds>(&mut self, value: T) -> Result<()> {
        self.write_command_args(format_args!("mixrampdelay {}", value.into_seconds()))
            .and_then(|_| self.expect_ok())
    }

    pub fn replaygain(&mut self, gain: ReplayGain) -> Result<()> {
        self.write_command_args(format_args!("replay_gain_mode {}", gain))
            .and_then(|_| self.expect_ok())
    }

    pub fn get_replaygain(&mut self) -> Result<ReplayGain> {
        try!(self.write_command("replay_gain_status"));

        let buf = try!(self.read_line());

        let reply = try!(buf.parse::<Reply>());
        try!(self.expect_ok());

        match reply {
            Reply::Ack(e) => Err(Error::Server(e)),
            Reply::Pair(ref a, ref b) if &*a == "replay_gain_mode" => b.parse().map_err(From::from),
            _ => Err(Error::Proto(ProtoError::NoField("replay_gain_mode")))
        }
    }

    pub fn play(&mut self) -> Result<()> {
        self.write_command("play")
            .and_then(|_| self.expect_ok())
    }

    pub fn next(&mut self) -> Result<()> {
        self.write_command("next")
            .and_then(|_| self.expect_ok())
    }

    pub fn prev(&mut self) -> Result<()> {
        self.write_command("previous")
            .and_then(|_| self.expect_ok())
    }

    pub fn stop(&mut self) -> Result<()> {
        self.write_command("stop")
            .and_then(|_| self.expect_ok())
    }

    pub fn pause(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("pause {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn seek<T: IntoSeconds>(&mut self, pos: T) -> Result<()> {
        self.write_command_args(format_args!("seekcur {}", pos.into_seconds()))
            .and_then(|_| self.expect_ok())
    }

    pub fn currentsong(&mut self) -> Result<Option<Song>> {
        self.write_command("currentsong")
            .and_then(|_| self.read_map())
            .and_then(|m| if m.is_empty() {
                Ok(None)
            } else {
                Song::from_map(m).map(Some)
            })
    }
}

// }}}

pub trait IntoSeconds {
    fn into_seconds(self) -> f64;
}

impl IntoSeconds for i64 {
    fn into_seconds(self) -> f64 {
        self as f64
    }
}

impl IntoSeconds for f64 {
    fn into_seconds(self) -> f64 {
        self
    }
}

impl IntoSeconds for Duration {
    fn into_seconds(self) -> f64 {
        self.num_milliseconds() as f64 / 1000.0
    }
}
<commit_msg>add convinience methods for default connections<commit_after>use std::collections::BTreeMap;
use std::io::{self, Read, Write, BufRead, Lines};
use std::convert::From;
use std::fmt::Arguments;
use std::net::{TcpStream, ToSocketAddrs};

use time::Duration;
use bufstream::BufStream;
use version::Version;
use error::{ProtoError, Error, Result};
use reply::Reply;
use status::Status;
use replaygain::ReplayGain;
use song::Song;

// Iterator {{{
struct Pairs<I: Iterator<Item=io::Result<String>>>(I);

impl<I> Iterator for Pairs<I> where I: Iterator<Item=io::Result<String>> {
    type Item = Result<(String, String)>;
    fn next(&mut self) -> Option<Result<(String, String)>> {
        let reply: Option<Result<Reply>> = self.0.next().map(|v| v.map_err(Error::Io).and_then(|s| s.parse::<Reply>().map_err(Error::Parse)));
        match reply {
            Some(Ok(Reply::Pair(a, b))) => Some(Ok((a, b))),
            None | Some(Ok(Reply::Ok)) => None,
            Some(Ok(Reply::Ack(e))) => Some(Err(Error::Server(e))),
            Some(Err(e)) => Some(Err(e)),
        }
    }
}
// }}}

// Client {{{
#[derive(Debug)]
pub struct Client<S=TcpStream> where S: Read+Write {
    socket: BufStream<S>,
    pub version: Version
}

impl Default for Client<TcpStream> {
    fn default() -> Client<TcpStream> {
        Client::<TcpStream>::connect("127.0.0.1:6600").unwrap()
    }
}

impl<S: Read+Write> Client<S> {
    pub fn connect<A: ToSocketAddrs>(addr: A) -> Result<Client<TcpStream>> {
        TcpStream::connect(addr).map_err(Error::Io).and_then(Client::new)
    }

    pub fn new(socket: S) -> Result<Client<S>> {
        let mut socket = BufStream::new(socket);

        let mut banner = String::new();
        try!(socket.read_line(&mut banner));

        if !banner.starts_with("OK MPD ") {
            return Err(From::from(ProtoError::BadBanner));
        }

        let version = try!(banner[7..].trim().parse::<Version>());

        Ok(Client {
            socket: socket,
            version: version
        })
    }

    fn read_line(&mut self) -> Result<String> {
        let mut buf = String::new();
        try!(self.socket.read_line(&mut buf));
        if buf.ends_with("\n") {
            buf.pop();
        }
        Ok(buf)
    }

    fn read_pairs(&mut self) -> Pairs<Lines<&mut BufStream<S>>> {
        Pairs((&mut self.socket).lines())
    }

    fn read_map(&mut self) -> Result<BTreeMap<String, String>> {
        self.read_pairs().collect()
    }

    fn write_command(&mut self, command: &str) -> Result<()> {
        self.socket.write_all(command.as_bytes())
            .and_then(|_| self.socket.write(&[0x0a]))
            .and_then(|_| self.socket.flush())
            .map_err(From::from)
    }

    fn write_command_args(&mut self, command: Arguments) -> Result<()> {
        self.socket.write_fmt(command)
            .and_then(|_| self.socket.write(&[0x0a]))
            .and_then(|_| self.socket.flush())
            .map_err(From::from)
    }

    fn expect_ok(&mut self) -> Result<()> {
        let line = try!(self.read_line());

        match line.parse::<Reply>() {
            Ok(Reply::Ok) => Ok(()),
            Ok(Reply::Ack(e)) => Err(Error::Server(e)),
            Ok(_) => Err(Error::Proto(ProtoError::NotOk)),
            Err(e) => Err(From::from(e)),
        }
    }

    pub fn status(&mut self) -> Result<Status> {
        self.write_command("status")
            .and_then(|_| self.read_map())
            .and_then(Status::from_map)
    }

    pub fn clearerror(&mut self) -> Result<()> {
        self.write_command("clearerror")
            .and_then(|_| self.expect_ok())
    }

    pub fn volume(&mut self, volume: i8) -> Result<()> {
        self.write_command_args(format_args!("setvol {}", volume))
            .and_then(|_| self.expect_ok())
    }

    pub fn repeat(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("repeat {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn random(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("random {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn single(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("single {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn consume(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("consume {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn crossfade(&mut self, value: u64) -> Result<()> {
        self.write_command_args(format_args!("crossfade {}", value))
            .and_then(|_| self.expect_ok())
    }

    pub fn mixrampdb(&mut self, value: f32) -> Result<()> {
        self.write_command_args(format_args!("mixrampdb {}", value))
            .and_then(|_| self.expect_ok())
    }

    pub fn mixrampdelay<T: IntoSeconds>(&mut self, value: T) -> Result<()> {
        self.write_command_args(format_args!("mixrampdelay {}", value.into_seconds()))
            .and_then(|_| self.expect_ok())
    }

    pub fn replaygain(&mut self, gain: ReplayGain) -> Result<()> {
        self.write_command_args(format_args!("replay_gain_mode {}", gain))
            .and_then(|_| self.expect_ok())
    }

    pub fn get_replaygain(&mut self) -> Result<ReplayGain> {
        try!(self.write_command("replay_gain_status"));

        let buf = try!(self.read_line());

        let reply = try!(buf.parse::<Reply>());
        try!(self.expect_ok());

        match reply {
            Reply::Ack(e) => Err(Error::Server(e)),
            Reply::Pair(ref a, ref b) if &*a == "replay_gain_mode" => b.parse().map_err(From::from),
            _ => Err(Error::Proto(ProtoError::NoField("replay_gain_mode")))
        }
    }

    pub fn play(&mut self) -> Result<()> {
        self.write_command("play")
            .and_then(|_| self.expect_ok())
    }

    pub fn next(&mut self) -> Result<()> {
        self.write_command("next")
            .and_then(|_| self.expect_ok())
    }

    pub fn prev(&mut self) -> Result<()> {
        self.write_command("previous")
            .and_then(|_| self.expect_ok())
    }

    pub fn stop(&mut self) -> Result<()> {
        self.write_command("stop")
            .and_then(|_| self.expect_ok())
    }

    pub fn pause(&mut self, value: bool) -> Result<()> {
        self.write_command_args(format_args!("pause {}", value as u8))
            .and_then(|_| self.expect_ok())
    }

    pub fn seek<T: IntoSeconds>(&mut self, pos: T) -> Result<()> {
        self.write_command_args(format_args!("seekcur {}", pos.into_seconds()))
            .and_then(|_| self.expect_ok())
    }

    pub fn currentsong(&mut self) -> Result<Option<Song>> {
        self.write_command("currentsong")
            .and_then(|_| self.read_map())
            .and_then(|m| if m.is_empty() {
                Ok(None)
            } else {
                Song::from_map(m).map(Some)
            })
    }
}

// }}}

pub trait IntoSeconds {
    fn into_seconds(self) -> f64;
}

impl IntoSeconds for i64 {
    fn into_seconds(self) -> f64 {
        self as f64
    }
}

impl IntoSeconds for f64 {
    fn into_seconds(self) -> f64 {
        self
    }
}

impl IntoSeconds for Duration {
    fn into_seconds(self) -> f64 {
        self.num_milliseconds() as f64 / 1000.0
    }
}
<|endoftext|>


<commit_before>use std::io;
use std::io::{BufferedReader, BufferedWriter, IoResult};
use std::io::net::ip::SocketAddr;
use common::{Header, HTTPVersion, Method};
use Request;
use url::Path;
use sequential::{SequentialReader, SequentialReaderBuilder, SequentialWriterBuilder};
use util::ClosableTcpStream;

/// A ClientConnection is an object that will store a socket to a client
/// and return Request objects.
pub struct ClientConnection {
    // address of the client
    remote_addr: io::IoResult<SocketAddr>,

    // sequence of Readers to the stream, so that the data is not read in
    //  the wrong order
    source: SequentialReaderBuilder<BufferedReader<ClosableTcpStream>>,

    // sequence of Writers to the stream, to avoid writing response #2 before
    //  response #1
    sink: SequentialWriterBuilder<BufferedWriter<ClosableTcpStream>>,

    // Reader to read the next header from
	next_header_source: SequentialReader<BufferedReader<ClosableTcpStream>>,

    // set to true if the client sent a "Connection: close" in the previous request
    connection_must_close: bool,
}

impl ClientConnection {
    /// Creates a new ClientConnection that takes ownership of the TcpStream.
    pub fn new(write_socket: ClosableTcpStream, mut read_socket: ClosableTcpStream)
        -> ClientConnection
    {
        let remote_addr = read_socket.peer_name();

        let mut source = SequentialReaderBuilder::new(BufferedReader::new(read_socket));
        let first_header = source.next().unwrap();

        ClientConnection {
            source: source,
            sink: SequentialWriterBuilder::new(BufferedWriter::new(write_socket)),
            remote_addr: remote_addr,
            next_header_source: first_header,
            connection_must_close: false,
        }
    }

    /// Reads the next line from self.next_header_source.
    ///
    /// Reads until `CRLF` is reached. The next read will start
    ///  at the first byte of the new line.
    fn read_next_line(&mut self) -> IoResult<String> {
        use std::io;
        use std::path::BytesContainer;

        let mut buf = Vec::new();
        let mut prev_byte_was_cr = false;

        loop {
            let byte = try!(self.next_header_source.read_byte());

            if byte == b'\n' && prev_byte_was_cr {
                return match buf.container_as_str() {
                    Some(s) => Ok(s.to_string()),
                    None => Err(io::standard_error(io::InvalidInput))
                }
            }

            if byte == b'\r' {
                prev_byte_was_cr = true;
            } else {
                prev_byte_was_cr = false;
            }

            buf.push(byte);
        }
    }

    /// Reads a request from the stream.
    /// Blocks until the header has been read.
    fn read(&mut self) -> io::IoResult<Request> {
        use util::EqualReader;

        let (method, path, version, headers) = {
            // reading the request line
            let (method, path, version) = {
                let line = try!(self.read_next_line());

                try!(parse_request_line(
                    line.as_slice().trim()
                ))
            };

            // getting all headers
            let headers = {
                let mut headers = Vec::new();
                loop {
                    let line = try!(self.read_next_line());

                    if line.as_slice().trim().len() == 0 { break };
                    headers.push(
                        match from_str(line.as_slice().trim()) {
                            Some(h) => h,
                            None => return Err(gen_invalid_input(
                                "Could not parse header"))
                        }
                    );
                }
                headers
            };

            (method, path, version, headers)
        };

        // finding length of body
        // TODO: handle transfer-encoding
        let body_length = headers.iter()
            .find(|h: &&Header| h.field.equiv(&"Content-Length"))
            .and_then(|h| from_str::<uint>(h.value.as_slice()))
            .unwrap_or(0u);

        // building the next reader
        let request_body_reader =
            if body_length == 0 {
                use std::io::util::NullReader;
                box NullReader as Box<Reader + Send>

            } else if body_length <= 1024 {
                use std::io::MemReader;
                let data = try!(self.next_header_source.read_exact(body_length));
                box MemReader::new(data) as Box<Reader + Send>

            } else {
                let data_reader = self.source.next().unwrap();
                let (data_reader, _) = EqualReader::new(data_reader, body_length);   // TODO:
                box data_reader as Box<Reader + Send>
            };

        // follow-up for next potential request
        self.next_header_source = self.source.next().unwrap();

        // building the writer
        let writer = self.sink.next().unwrap();

        // building the request
        Ok(Request {
            data_reader: request_body_reader,
            response_writer: box writer,
            remote_addr: self.remote_addr.clone().unwrap(),     // TODO: could fail
            method: method,
            path: path,
            http_version: version,
            headers: headers,
            body_length: body_length,
        })
    }
}

impl Iterator<Request> for ClientConnection {
    /// Blocks until the next Request is available.
    /// Returns None when no new Requests will come from the client.
    fn next(&mut self) -> Option<Request> {
        // the client sent a "connection: close" header in this previous request
        //  or is using HTTP 1.0, meaning that no new request will come
        if self.connection_must_close {
            return None
        }

        // TODO: send back message to client in case of parsing error
        loop {
            let rq = match self.read() {
                Err(_) => return None,
                Ok(rq) => rq
            };

            // updating the status of the connection
            {
                let connection_header = rq.headers.iter()
                    .find(|h| h.field.equiv(&"Connection")).map(|h| h.value.as_slice());

                if connection_header == Some("close") {
                    self.connection_must_close = true;
                } else if rq.http_version == HTTPVersion(1, 0) &&
                        connection_header != Some("keep-alive")
                {
                    self.connection_must_close = true;
                }
            }

            // returning the request
            return Some(rq);
        }
    }
}

/// Generates an IoError with invalid input.
/// This function is here because it is incredibly annoying to create this error.
fn gen_invalid_input(desc: &'static str) -> io::IoError {
    io::IoError {
        kind: io::InvalidInput,
        desc: desc,
        detail: None
    }
}

/// Parses a "HTTP/1.1" string.
fn parse_http_version(version: &str) -> io::IoResult<HTTPVersion> {
    let elems = version.splitn('/', 1).map(|e| e.to_string()).collect::<Vec<String>>();
    if elems.len() != 2 {
        return Err(gen_invalid_input("Wrong HTTP version format"))
    }

    let elems = elems.get(1).as_slice().splitn('.', 1)
        .map(|e| e.to_string()).collect::<Vec<String>>();
    if elems.len() != 2 {
        return Err(gen_invalid_input("Wrong HTTP version format"))
    }

    match (from_str(elems.get(0).as_slice()), from_str(elems.get(1).as_slice())) {
        (Some(major), Some(minor)) =>
            Ok(HTTPVersion(major, minor)),
        _ => Err(gen_invalid_input("Wrong HTTP version format"))
    }
}

/// Parses the request line of the request.
/// eg. GET / HTTP/1.1
fn parse_request_line(line: &str) -> io::IoResult<(Method, Path, HTTPVersion)> {
    let mut words = line.words();

    let method = words.next();
    let path = words.next();
    let version = words.next();

    let (method, path, version) = match (method, path, version) {
        (Some(m), Some(p), Some(v)) => (m, p, v),
        _ => return Err(gen_invalid_input("Missing element in request line"))
    };

    let method = match from_str(method) {
        Some(method) => method,
        None => return Err(gen_invalid_input("Could not parse method"))
    };

    let path = match Path::parse(path) {
        Ok(p) => p,
        Err(_) => return Err(gen_invalid_input("Wrong requested URL"))
    };

    let version = try!(parse_http_version(version));

    Ok((method, path, version))
}

#[cfg(test)]
mod test {
    #[test]
    fn test_parse_request_line() {
        let (method, path, ver) =
            super::parse_request_line("GET /hello HTTP/1.1").unwrap();

        assert!(method.equiv(&"get"));
        assert!(path == from_str("/hello").unwrap());
        assert!(ver == ::common::HTTPVersion(1, 1));

        assert!(super::parse_request_line("GET /hello").is_err());
        assert!(super::parse_request_line("qsd qsd qsd").is_err());
    }
}
<commit_msg>Sending back a 400 message in case of wrong request<commit_after>use std::io;
use std::io::{BufferedReader, BufferedWriter, IoError, IoResult};
use std::io::net::ip::SocketAddr;
use common::{Header, HTTPVersion, Method};
use Request;
use url::Path;
use sequential::{SequentialReader, SequentialReaderBuilder, SequentialWriterBuilder};
use util::ClosableTcpStream;

/// A ClientConnection is an object that will store a socket to a client
/// and return Request objects.
pub struct ClientConnection {
    // address of the client
    remote_addr: io::IoResult<SocketAddr>,

    // sequence of Readers to the stream, so that the data is not read in
    //  the wrong order
    source: SequentialReaderBuilder<BufferedReader<ClosableTcpStream>>,

    // sequence of Writers to the stream, to avoid writing response #2 before
    //  response #1
    sink: SequentialWriterBuilder<BufferedWriter<ClosableTcpStream>>,

    // Reader to read the next header from
	next_header_source: SequentialReader<BufferedReader<ClosableTcpStream>>,

    // set to true if the client sent a "Connection: close" in the previous request
    connection_must_close: bool,
}

/// Error that can happen when reading a request.
enum ReadError {
    WrongRequestLine,
    WrongHeader(HTTPVersion),
    ReadIoError(IoError),
}

impl ClientConnection {
    /// Creates a new ClientConnection that takes ownership of the TcpStream.
    pub fn new(write_socket: ClosableTcpStream, mut read_socket: ClosableTcpStream)
        -> ClientConnection
    {
        let remote_addr = read_socket.peer_name();

        let mut source = SequentialReaderBuilder::new(BufferedReader::new(read_socket));
        let first_header = source.next().unwrap();

        ClientConnection {
            source: source,
            sink: SequentialWriterBuilder::new(BufferedWriter::new(write_socket)),
            remote_addr: remote_addr,
            next_header_source: first_header,
            connection_must_close: false,
        }
    }

    /// Reads the next line from self.next_header_source.
    ///
    /// Reads until `CRLF` is reached. The next read will start
    ///  at the first byte of the new line.
    fn read_next_line(&mut self) -> IoResult<String> {
        use std::io;
        use std::path::BytesContainer;

        let mut buf = Vec::new();
        let mut prev_byte_was_cr = false;

        loop {
            let byte = try!(self.next_header_source.read_byte());

            if byte == b'\n' && prev_byte_was_cr {
                return match buf.container_as_str() {
                    Some(s) => Ok(s.to_string()),
                    None => Err(io::standard_error(io::InvalidInput))
                }
            }

            if byte == b'\r' {
                prev_byte_was_cr = true;
            } else {
                prev_byte_was_cr = false;
            }

            buf.push(byte);
        }
    }

    /// Reads a request from the stream.
    /// Blocks until the header has been read.
    fn read(&mut self) -> Result<Request, ReadError> {
        use util::EqualReader;

        let (method, path, version, headers) = {
            // reading the request line
            let (method, path, version) = {
                let line = try!(self.read_next_line().map_err(|e| ReadIoError(e)));

                try!(parse_request_line(
                    line.as_slice().trim()
                ))
            };

            // getting all headers
            let headers = {
                let mut headers = Vec::new();
                loop {
                    let line = try!(self.read_next_line().map_err(|e| ReadIoError(e)));

                    if line.as_slice().trim().len() == 0 { break };
                    headers.push(
                        match from_str(line.as_slice().trim()) {
                            Some(h) => h,
                            None => return Err(WrongHeader(version))
                        }
                    );
                }

                headers
            };

            (method, path, version, headers)
        };

        // finding length of body
        // TODO: handle transfer-encoding
        let body_length = headers.iter()
            .find(|h: &&Header| h.field.equiv(&"Content-Length"))
            .and_then(|h| from_str::<uint>(h.value.as_slice()))
            .unwrap_or(0u);

        // building the next reader
        let request_body_reader =
            if body_length == 0 {
                use std::io::util::NullReader;
                box NullReader as Box<Reader + Send>

            } else if body_length <= 1024 {
                use std::io::MemReader;
                let data = try!(self.next_header_source.read_exact(body_length)
                    .map_err(|e| ReadIoError(e)));
                box MemReader::new(data) as Box<Reader + Send>

            } else {
                let data_reader = self.source.next().unwrap();
                let (data_reader, _) = EqualReader::new(data_reader, body_length);   // TODO:
                box data_reader as Box<Reader + Send>
            };

        // follow-up for next potential request
        self.next_header_source = self.source.next().unwrap();

        // building the writer
        let writer = self.sink.next().unwrap();

        // building the request
        Ok(Request {
            data_reader: request_body_reader,
            response_writer: box writer,
            remote_addr: self.remote_addr.clone().unwrap(),     // TODO: could fail
            method: method,
            path: path,
            http_version: version,
            headers: headers,
            body_length: body_length,
        })
    }
}

impl Iterator<Request> for ClientConnection {
    /// Blocks until the next Request is available.
    /// Returns None when no new Requests will come from the client.
    fn next(&mut self) -> Option<Request> {
        use {Response, StatusCode};

        // the client sent a "connection: close" header in this previous request
        //  or is using HTTP 1.0, meaning that no new request will come
        if self.connection_must_close {
            return None
        }

        // TODO: send back message to client in case of parsing error
        loop {
            let rq = match self.read() {
                Err(WrongRequestLine) => {
                    let writer = self.sink.next().unwrap();
                    let response = Response::new_empty(StatusCode(400));
                    response.raw_print(writer, HTTPVersion(1, 1), &[], false).ok();
                    return None;    // we don't know where the next request would start,
                                    // se we have to close
                },

                Err(WrongHeader(ver)) => {
                    let writer = self.sink.next().unwrap();
                    let response = Response::new_empty(StatusCode(400));
                    response.raw_print(writer, ver, &[], false).ok();
                    return None;    // we don't know where the next request would start,
                                    // se we have to close
                },

                Err(ReadIoError(_)) =>
                    return None,

                Ok(rq) => rq
            };

            // updating the status of the connection
            {
                let connection_header = rq.headers.iter()
                    .find(|h| h.field.equiv(&"Connection")).map(|h| h.value.as_slice());

                if connection_header == Some("close") {
                    self.connection_must_close = true;
                } else if rq.http_version == HTTPVersion(1, 0) &&
                        connection_header != Some("keep-alive")
                {
                    self.connection_must_close = true;
                }
            }

            // returning the request
            return Some(rq);
        }
    }
}

/// Parses a "HTTP/1.1" string.
fn parse_http_version(version: &str) -> Result<HTTPVersion, ReadError> {
    let elems = version.splitn('/', 1).map(|e| e.to_string()).collect::<Vec<String>>();
    if elems.len() != 2 {
        return Err(WrongRequestLine)
    }

    let elems = elems.get(1).as_slice().splitn('.', 1)
        .map(|e| e.to_string()).collect::<Vec<String>>();
    if elems.len() != 2 {
        return Err(WrongRequestLine)
    }

    match (from_str(elems.get(0).as_slice()), from_str(elems.get(1).as_slice())) {
        (Some(major), Some(minor)) =>
            Ok(HTTPVersion(major, minor)),
        _ => Err(WrongRequestLine)
    }
}

/// Parses the request line of the request.
/// eg. GET / HTTP/1.1
fn parse_request_line(line: &str) -> Result<(Method, Path, HTTPVersion), ReadError> {
    let mut words = line.words();

    let method = words.next();
    let path = words.next();
    let version = words.next();

    let (method, path, version) = match (method, path, version) {
        (Some(m), Some(p), Some(v)) => (m, p, v),
        _ => return Err(WrongRequestLine)
    };

    let method = match from_str(method) {
        Some(method) => method,
        None => return Err(WrongRequestLine)
    };

    let path = match Path::parse(path) {
        Ok(p) => p,
        Err(_) => return Err(WrongRequestLine)
    };

    let version = try!(parse_http_version(version));

    Ok((method, path, version))
}

#[cfg(test)]
mod test {
    #[test]
    fn test_parse_request_line() {
        let (method, path, ver) =
            match super::parse_request_line("GET /hello HTTP/1.1") {
                Err(_) => fail!(),
                Ok(v) => v
            };

        assert!(method.equiv(&"get"));
        assert!(path == from_str("/hello").unwrap());
        assert!(ver == ::common::HTTPVersion(1, 1));

        assert!(super::parse_request_line("GET /hello").is_err());
        assert!(super::parse_request_line("qsd qsd qsd").is_err());
    }
}
<|endoftext|>


<commit_before>use std::collections::{HashMap, VecDeque};
use std::collections::hash_map::Entry;
use std::io;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};

use futures::{Future, Sink, Stream};
use futures::future;
use futures::sync::{mpsc, oneshot};

use tokio_core::net::TcpStream;
use tokio_core::reactor::Handle;
use tokio_io::AsyncRead;

use super::{error, resp};

/// TODO: comeback and optimise this number
const DEFAULT_BUFFER_SIZE:usize = 100;

/// Connect to a Redis server and return paired Sink and Stream for reading and writing
/// asynchronously.
fn connect(addr: &SocketAddr, handle: &Handle) -> Box<Future<Item=ClientConnection, Error=io::Error>> {
    TcpStream::connect(addr, handle).map(move |socket| {
        let framed = socket.framed(resp::RespCodec);
        let (write_f, read_f) = framed.split();
        let write_b = write_f.buffer(DEFAULT_BUFFER_SIZE);
        ClientConnection {
            sender: ClientSink(Box::new(write_b)),
            receiver: ClientStream(Box::new(read_f))
        }
    }).boxed()
}

/// TODO - is the boxing necessary?  It makes the type signature much simpler
struct ClientSink(Box<Sink<SinkItem=resp::RespValue, SinkError=io::Error>>);
struct ClientStream(Box<Stream<Item=resp::RespValue, Error=error::Error>>);

/// A low-level client connection representing a sender and a receiver.
///
/// The two halves operate independently from one another
struct ClientConnection {
    sender: ClientSink,
    receiver: ClientStream
}

fn paired_connect(addr: &SocketAddr, handle: &Handle) -> Box<Future<Item=PairedConnection, Error=error::Error>> {
    let handle = handle.clone();
    let paired_con = connect(addr, &handle).map(move |connection| {
        let ClientConnection { sender, receiver } = connection;
        let (out_tx, out_rx) = mpsc::unbounded();
        let sender = out_rx.fold(sender.0, |sender, msg| {
            sender.send(msg).map_err(|_| ())
        });
        let resp_queue:Arc<Mutex<VecDeque<oneshot::Sender<resp::RespValue>>>> = Arc::new(Mutex::new(VecDeque::new()));
        let receiver_queue = resp_queue.clone();
        let receiver = receiver.0.for_each(move |msg| {
            let mut queue = receiver_queue.lock().expect("Lock is tainted");
            let dest = queue.pop_front().expect("Queue is empty");
            match dest.send(msg) {
                Ok(()) => Ok(()),
                // Ignore error as the channel may have been legitimately closed in the meantime
                Err(_) => Ok(())
            }
        });
        handle.spawn(sender.map(|_| ()));
        handle.spawn(receiver.map_err(|_| ()));
        PairedConnection {
            out_tx: out_tx,
            resp_queue: resp_queue
        }
    }).map_err(|e| e.into());
    Box::new(paired_con)
}

struct PairedConnection {
    out_tx: mpsc::UnboundedSender<resp::RespValue>,
    resp_queue: Arc<Mutex<VecDeque<oneshot::Sender<resp::RespValue>>>>
}

impl PairedConnection {
    fn send<R>(&self, msg: R) -> Box<Future<Item=resp::RespValue, Error=error::Error>>
    where R: Into<resp::RespValue> {
        let (tx, rx) = oneshot::channel();
        let mut queue = self.resp_queue.lock().expect("Tainted queue");
        queue.push_back(tx);
        mpsc::UnboundedSender::send(&self.out_tx, msg.into()).expect("Failed to send");
        Box::new(rx.map_err(|e| e.into()))
    }
}

fn pubsub_connect(addr: &SocketAddr, handle: &Handle) -> Box<Future<Item=PubsubConnection, Error=error::Error>> {
    let handle = handle.clone();
    let pubsub_con = connect(addr, &handle).map(move |connection| {
        let ClientConnection { sender, receiver } = connection;
        let (out_tx, out_rx) = mpsc::unbounded();
        let sender = out_rx.fold(sender.0, |sender, msg| {
            sender.send(msg).map_err(|_| ())
        });
        let subs = Arc::new(Mutex::new(PubsubSubscriptions {
            pending: HashMap::new(),
            confirmed: HashMap::new()
        }));
        let subs_reader = subs.clone();
        let receiver = receiver.0.for_each(move |msg| {
            // TODO: check message type - and handle accordingly.
            let (message_type, topic, msg) = if let resp::RespValue::Array(mut messages) = msg {
                assert_eq!(messages.len(), 3);
                let msg = messages.pop().expect("No message");
                let topic = messages.pop().expect("No topic");
                let message_type = messages.pop().expect("No type");
                (message_type, topic.into_string().expect("Topic should be a string"), msg)
            } else {
                panic!("incorrect type");
            };
            let mut subs = subs_reader.lock().expect("Lock is tainted");
            if let resp::RespValue::BulkString(ref bytes) = message_type {
                if bytes == b"subscribe" {
                    if let Some(pending) = subs.pending.remove(&topic) {
                        let mut txes = Vec::with_capacity(pending.len());
                        let mut futures = Vec::with_capacity(pending.len());
                        for (tx, notification_tx) in pending {
                            txes.push(tx);
                            futures.push(notification_tx.send(()));
                        }
                        subs.confirmed.entry(topic).or_insert(vec![]).extend(txes);
                        future::join_all(futures).map(|_| ()).map_err(|_| error::internal("unreachable")).boxed()
                    } else {
                        future::ok(()).map_err(|_:()| error::internal("unreachable")).boxed()
                    }
                } else if bytes == b"message" {
                    match subs.confirmed.get(&topic) {
                        Some(txes) => {
                            let futures:Vec<_> = txes.iter().map(|tx| {
                                let tx = tx.clone();
                                tx.send(msg.clone())
                            }).collect();
                            future::join_all(futures).map(|_| ()).map_err(|e| e.into()).boxed()
                        }
                        None => future::ok(()).map_err(|_:()| error::internal("unreachable")).boxed()
                    }
                } else {
                    panic!("Unexpected bytes: {:?}", bytes);
                }
            } else {
                panic!("Message format error: {:?}", message_type);
            }
        });
        handle.spawn(sender.map(|_| ()));
        handle.spawn(receiver.map_err(|_| ()));
        PubsubConnection { out_tx: out_tx, subscriptions: subs }
    }).map_err(|e| e.into());
    Box::new(pubsub_con)
}

struct PubsubSubscriptions {
    pending: HashMap<String, Vec<(mpsc::Sender<resp::RespValue>, oneshot::Sender<()>)>>,
    confirmed: HashMap<String, Vec<mpsc::Sender<resp::RespValue>>>
}

#[derive(Clone)]
struct PubsubConnection {
    out_tx: mpsc::UnboundedSender<resp::RespValue>,
    subscriptions: Arc<Mutex<PubsubSubscriptions>>
}

impl PubsubConnection {
    fn subscribe<T: Into<String>>(&self, topic: T) -> Box<Future<Item=Box<Stream<Item=resp::RespValue, Error=()>>, Error=error::Error>> {
        let topic = topic.into();
        let mut subs = self.subscriptions.lock().expect("Lock is tainted");
        let key = topic.clone();
        // TODO - check arbitrary buffer size
        let (tx, rx) = mpsc::channel(10);
        let stream = Box::new(rx) as Box<Stream<Item=resp::RespValue, Error=()>>;
        if let Entry::Occupied(ref mut entry) = subs.confirmed.entry(key) {
            entry.get_mut().push(tx);
            return Box::new(future::ok(stream))
        }

        let (notification_tx, notification_rx) = oneshot::channel();
        subs.pending.entry(topic.clone()).or_insert(Vec::new()).push((tx, notification_tx));

        let subscribe_msg = vec!["SUBSCRIBE", &topic];
        mpsc::UnboundedSender::send(&self.out_tx, subscribe_msg.into()).expect("Failed to send");

        let done = notification_rx.map(|_| stream).map_err(|e| e.into());
        Box::new(done)
    }
}

#[cfg(test)]
mod test {
    use std::io;
    use std::rc::Rc;

    use futures::{Future, Sink, Stream};
    use futures::{future, stream};

    use tokio_core::reactor::Core;

    use super::{error, resp};

    #[test]
    fn can_connect() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let connection = super::connect(&addr, &core.handle()).map_err(|e| e.into()).and_then(|connection| {
            let a = connection.sender.0.send(["PING", "TEST"].as_ref().into()).map_err(|e| e.into());
            let b = connection.receiver.0.take(1).collect();
            a.join(b)
        });

        let (_, values) = core.run(connection).unwrap();
        assert_eq!(values.len(), 1);
        assert_eq!(values[0], "TEST".into());
    }

    #[test]
    fn complex_test() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();
        let connection = super::connect(&addr, &core.handle()).map_err(|e| e.into()).and_then(|connection| {
            let mut ops = Vec::<resp::RespValue>::new();
            ops.push(["FLUSH"].as_ref().into());
            ops.extend((0..1000).map(|i| ["SADD", "test_set", &format!("VALUE: {}", i)].as_ref().into()));
            ops.push(["SMEMBERS", "test_set"].as_ref().into());
            let ops_r:Vec<Result<resp::RespValue, io::Error>> = ops.into_iter().map(Result::Ok).collect();
            let send = connection.sender.0.send_all(stream::iter(ops_r)).map_err(|e| e.into());
            let receive = connection.receiver.0.skip(1001).take(1).collect();
            send.join(receive)
        });
        let (_, values) = core.run(connection).unwrap();
        assert_eq!(values.len(), 1);
        let values = match &values[0] {
            &resp::RespValue::Array(ref values) => values.clone(),
            _ => panic!("Not an array")
        };
        assert_eq!(values.len(), 1000);
    }

    #[test]
    fn can_paired_connect() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let connect_f = super::paired_connect(&addr, &core.handle()).and_then(|connection| {
            let res_f = connection.send(vec!["PING", "TEST"]);
            connection.send(vec!["SET", "X", "123"]);
            let wait_f = connection.send(vec!["GET", "X"]);
            res_f.join(wait_f)
        });
        let (result_1, result_2) = core.run(connect_f).unwrap();
        assert_eq!(result_1, "TEST".into());
        assert_eq!(result_2, "123".into());
    }

    #[test]
    fn complex_paired_connect() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let connect_f = super::paired_connect(&addr, &core.handle()).and_then(|connection| {
            connection.send(vec!["INCR", "CTR"]).and_then(move |value| {
                let value_str = value.into_string().expect("A string");
                connection.send(vec!["SET", "LASTCTR", &value_str])
            })
        });
        let result = core.run(connect_f).unwrap();
        assert_eq!(result, resp::RespValue::SimpleString("OK".into()));
    }

    #[test]
    fn sending_a_lot_of_data_test() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let test_f = super::paired_connect(&addr, &core.handle());
        let send_data = test_f.and_then(|connection| {
            let mut futures = Vec::with_capacity(1000);
            for i in 0..1000 {
                let key = format!("X_{}", i);
                connection.send(vec!["SET", &key, &i.to_string()]);
                futures.push(connection.send(vec!["GET", &key]));
            }
            futures.remove(999)
        });
        let result = core.run(send_data).unwrap();
        assert_eq!(result.into_string().unwrap(), "999");
    }

    #[test]
    fn realistic_test() {
        let test_data_size = 100;
        let test_data:Vec<_> = (0..test_data_size).map(|x| (x, x.to_string())).collect();
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();
        let test_f = super::paired_connect(&addr, &core.handle());
        let send_data = test_f.and_then(|connection| {
            let connection = Rc::new(connection);
            let futures:Vec<_> = test_data.into_iter().map(move |data| {
                let connection_inner = connection.clone();
                connection.send(vec!["INCR", "realistic_test_ctr"]).and_then(move |ctr| {
                    let key = format!("rt_{}", ctr.into_string().unwrap());
                    let d_val = data.0.to_string();
                    connection_inner.send(vec!["SET", &key, &d_val]);
                    connection_inner.send(vec!["SET", &data.1, &key])
                })
            }).collect();
            future::join_all(futures)
        });
        let result = core.run(send_data).unwrap();
        assert_eq!(result.len(), 100);
    }

    #[test]
    fn pubsub_test() {
        let mut core = Core::new().unwrap();
        let handle = core.handle();
        let addr = "127.0.0.1:6379".parse().unwrap();
        let paired_c = super::paired_connect(&addr, &handle);
        let pubsub_c = super::pubsub_connect(&addr, &handle);
        let msgs = paired_c.join(pubsub_c).and_then(|(paired, pubsub)| {
            let subscribe = pubsub.subscribe("test-topic");
            subscribe.and_then(move |msgs| {
                paired.send(vec!["PUBLISH", "test-topic", "test-message"]);
                paired.send(vec!["PUBLISH", "test-not-topic", "test-message-1.5"]);
                paired.send(vec!["PUBLISH", "test-topic", "test-message2"]).map(|_| msgs)
            })
        });
        let tst = msgs.and_then(|msgs| msgs.take(2).collect().map_err(|_| error::internal("unreachable")));
        let result = core.run(tst).unwrap();
        assert_eq!(result.len(), 2);
        assert_eq!(result[0], "test-message".into());
        assert_eq!(result[1], "test-message2".into());
    }
}<commit_msg>Unused import<commit_after>use std::collections::{HashMap, VecDeque};
use std::io;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};

use futures::{Future, Sink, Stream};
use futures::future;
use futures::sync::{mpsc, oneshot};

use tokio_core::net::TcpStream;
use tokio_core::reactor::Handle;
use tokio_io::AsyncRead;

use super::{error, resp};

/// TODO: comeback and optimise this number
const DEFAULT_BUFFER_SIZE:usize = 100;

/// Connect to a Redis server and return paired Sink and Stream for reading and writing
/// asynchronously.
fn connect(addr: &SocketAddr, handle: &Handle) -> Box<Future<Item=ClientConnection, Error=io::Error>> {
    TcpStream::connect(addr, handle).map(move |socket| {
        let framed = socket.framed(resp::RespCodec);
        let (write_f, read_f) = framed.split();
        let write_b = write_f.buffer(DEFAULT_BUFFER_SIZE);
        ClientConnection {
            sender: ClientSink(Box::new(write_b)),
            receiver: ClientStream(Box::new(read_f))
        }
    }).boxed()
}

/// TODO - is the boxing necessary?  It makes the type signature much simpler
struct ClientSink(Box<Sink<SinkItem=resp::RespValue, SinkError=io::Error>>);
struct ClientStream(Box<Stream<Item=resp::RespValue, Error=error::Error>>);

/// A low-level client connection representing a sender and a receiver.
///
/// The two halves operate independently from one another
struct ClientConnection {
    sender: ClientSink,
    receiver: ClientStream
}

fn paired_connect(addr: &SocketAddr, handle: &Handle) -> Box<Future<Item=PairedConnection, Error=error::Error>> {
    let handle = handle.clone();
    let paired_con = connect(addr, &handle).map(move |connection| {
        let ClientConnection { sender, receiver } = connection;
        let (out_tx, out_rx) = mpsc::unbounded();
        let sender = out_rx.fold(sender.0, |sender, msg| {
            sender.send(msg).map_err(|_| ())
        });
        let resp_queue:Arc<Mutex<VecDeque<oneshot::Sender<resp::RespValue>>>> = Arc::new(Mutex::new(VecDeque::new()));
        let receiver_queue = resp_queue.clone();
        let receiver = receiver.0.for_each(move |msg| {
            let mut queue = receiver_queue.lock().expect("Lock is tainted");
            let dest = queue.pop_front().expect("Queue is empty");
            match dest.send(msg) {
                Ok(()) => Ok(()),
                // Ignore error as the channel may have been legitimately closed in the meantime
                Err(_) => Ok(())
            }
        });
        handle.spawn(sender.map(|_| ()));
        handle.spawn(receiver.map_err(|_| ()));
        PairedConnection {
            out_tx: out_tx,
            resp_queue: resp_queue
        }
    }).map_err(|e| e.into());
    Box::new(paired_con)
}

struct PairedConnection {
    out_tx: mpsc::UnboundedSender<resp::RespValue>,
    resp_queue: Arc<Mutex<VecDeque<oneshot::Sender<resp::RespValue>>>>
}

impl PairedConnection {
    fn send<R>(&self, msg: R) -> Box<Future<Item=resp::RespValue, Error=error::Error>>
    where R: Into<resp::RespValue> {
        let (tx, rx) = oneshot::channel();
        let mut queue = self.resp_queue.lock().expect("Tainted queue");
        queue.push_back(tx);
        mpsc::UnboundedSender::send(&self.out_tx, msg.into()).expect("Failed to send");
        Box::new(rx.map_err(|e| e.into()))
    }
}

fn pubsub_connect(addr: &SocketAddr, handle: &Handle) -> Box<Future<Item=PubsubConnection, Error=error::Error>> {
    let handle = handle.clone();
    let pubsub_con = connect(addr, &handle).map(move |connection| {
        let ClientConnection { sender, receiver } = connection;
        let (out_tx, out_rx) = mpsc::unbounded();
        let sender = out_rx.fold(sender.0, |sender, msg| {
            sender.send(msg).map_err(|_| ())
        });
        let subs = Arc::new(Mutex::new(PubsubSubscriptions {
            pending: HashMap::new(),
            confirmed: HashMap::new()
        }));
        let subs_reader = subs.clone();
        let receiver = receiver.0.for_each(move |msg| {
            // TODO: check message type - and handle accordingly.
            let (message_type, topic, msg) = if let resp::RespValue::Array(mut messages) = msg {
                assert_eq!(messages.len(), 3);
                let msg = messages.pop().expect("No message");
                let topic = messages.pop().expect("No topic");
                let message_type = messages.pop().expect("No type");
                (message_type, topic.into_string().expect("Topic should be a string"), msg)
            } else {
                panic!("incorrect type");
            };
            let mut subs = subs_reader.lock().expect("Lock is tainted");
            if let resp::RespValue::BulkString(ref bytes) = message_type {
                if bytes == b"subscribe" {
                    if let Some(pending) = subs.pending.remove(&topic) {
                        let mut txes = Vec::with_capacity(pending.len());
                        let mut futures = Vec::with_capacity(pending.len());
                        for (tx, notification_tx) in pending {
                            txes.push(tx);
                            futures.push(notification_tx.send(()));
                        }
                        subs.confirmed.entry(topic).or_insert(vec![]).extend(txes);
                        future::join_all(futures).map(|_| ()).map_err(|_| error::internal("unreachable")).boxed()
                    } else {
                        future::ok(()).map_err(|_:()| error::internal("unreachable")).boxed()
                    }
                } else if bytes == b"message" {
                    match subs.confirmed.get(&topic) {
                        Some(txes) => {
                            let futures:Vec<_> = txes.iter().map(|tx| {
                                let tx = tx.clone();
                                tx.send(msg.clone())
                            }).collect();
                            future::join_all(futures).map(|_| ()).map_err(|e| e.into()).boxed()
                        }
                        None => future::ok(()).map_err(|_:()| error::internal("unreachable")).boxed()
                    }
                } else {
                    panic!("Unexpected bytes: {:?}", bytes);
                }
            } else {
                panic!("Message format error: {:?}", message_type);
            }
        });
        handle.spawn(sender.map(|_| ()));
        handle.spawn(receiver.map_err(|_| ()));
        PubsubConnection { out_tx: out_tx, subscriptions: subs }
    }).map_err(|e| e.into());
    Box::new(pubsub_con)
}

struct PubsubSubscriptions {
    pending: HashMap<String, Vec<(mpsc::Sender<resp::RespValue>, oneshot::Sender<()>)>>,
    confirmed: HashMap<String, Vec<mpsc::Sender<resp::RespValue>>>
}

#[derive(Clone)]
struct PubsubConnection {
    out_tx: mpsc::UnboundedSender<resp::RespValue>,
    subscriptions: Arc<Mutex<PubsubSubscriptions>>
}

impl PubsubConnection {
    fn subscribe<T: Into<String>>(&self, topic: T) -> Box<Future<Item=Box<Stream<Item=resp::RespValue, Error=()>>, Error=error::Error>> {
        let topic = topic.into();
        let mut subs = self.subscriptions.lock().expect("Lock is tainted");

        // TODO - check arbitrary buffer size
        let (tx, rx) = mpsc::channel(10);
        let stream = Box::new(rx) as Box<Stream<Item=resp::RespValue, Error=()>>;
        if let Some(ref mut entry) = subs.confirmed.get_mut(&topic) {
            entry.push(tx);
            return Box::new(future::ok(stream))
        }

        let (notification_tx, notification_rx) = oneshot::channel();
        let subscribe_msg = vec!["SUBSCRIBE", &topic].into();
        subs.pending.entry(topic).or_insert(Vec::new()).push((tx, notification_tx));
        mpsc::UnboundedSender::send(&self.out_tx, subscribe_msg).expect("Failed to send");

        let done = notification_rx.map(|_| stream).map_err(|e| e.into());
        Box::new(done)
    }
}

#[cfg(test)]
mod test {
    use std::io;
    use std::rc::Rc;

    use futures::{Future, Sink, Stream};
    use futures::{future, stream};

    use tokio_core::reactor::Core;

    use super::{error, resp};

    #[test]
    fn can_connect() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let connection = super::connect(&addr, &core.handle()).map_err(|e| e.into()).and_then(|connection| {
            let a = connection.sender.0.send(["PING", "TEST"].as_ref().into()).map_err(|e| e.into());
            let b = connection.receiver.0.take(1).collect();
            a.join(b)
        });

        let (_, values) = core.run(connection).unwrap();
        assert_eq!(values.len(), 1);
        assert_eq!(values[0], "TEST".into());
    }

    #[test]
    fn complex_test() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();
        let connection = super::connect(&addr, &core.handle()).map_err(|e| e.into()).and_then(|connection| {
            let mut ops = Vec::<resp::RespValue>::new();
            ops.push(["FLUSH"].as_ref().into());
            ops.extend((0..1000).map(|i| ["SADD", "test_set", &format!("VALUE: {}", i)].as_ref().into()));
            ops.push(["SMEMBERS", "test_set"].as_ref().into());
            let ops_r:Vec<Result<resp::RespValue, io::Error>> = ops.into_iter().map(Result::Ok).collect();
            let send = connection.sender.0.send_all(stream::iter(ops_r)).map_err(|e| e.into());
            let receive = connection.receiver.0.skip(1001).take(1).collect();
            send.join(receive)
        });
        let (_, values) = core.run(connection).unwrap();
        assert_eq!(values.len(), 1);
        let values = match &values[0] {
            &resp::RespValue::Array(ref values) => values.clone(),
            _ => panic!("Not an array")
        };
        assert_eq!(values.len(), 1000);
    }

    #[test]
    fn can_paired_connect() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let connect_f = super::paired_connect(&addr, &core.handle()).and_then(|connection| {
            let res_f = connection.send(vec!["PING", "TEST"]);
            connection.send(vec!["SET", "X", "123"]);
            let wait_f = connection.send(vec!["GET", "X"]);
            res_f.join(wait_f)
        });
        let (result_1, result_2) = core.run(connect_f).unwrap();
        assert_eq!(result_1, "TEST".into());
        assert_eq!(result_2, "123".into());
    }

    #[test]
    fn complex_paired_connect() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let connect_f = super::paired_connect(&addr, &core.handle()).and_then(|connection| {
            connection.send(vec!["INCR", "CTR"]).and_then(move |value| {
                let value_str = value.into_string().expect("A string");
                connection.send(vec!["SET", "LASTCTR", &value_str])
            })
        });
        let result = core.run(connect_f).unwrap();
        assert_eq!(result, resp::RespValue::SimpleString("OK".into()));
    }

    #[test]
    fn sending_a_lot_of_data_test() {
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();

        let test_f = super::paired_connect(&addr, &core.handle());
        let send_data = test_f.and_then(|connection| {
            let mut futures = Vec::with_capacity(1000);
            for i in 0..1000 {
                let key = format!("X_{}", i);
                connection.send(vec!["SET", &key, &i.to_string()]);
                futures.push(connection.send(vec!["GET", &key]));
            }
            futures.remove(999)
        });
        let result = core.run(send_data).unwrap();
        assert_eq!(result.into_string().unwrap(), "999");
    }

    #[test]
    fn realistic_test() {
        let test_data_size = 100;
        let test_data:Vec<_> = (0..test_data_size).map(|x| (x, x.to_string())).collect();
        let mut core = Core::new().unwrap();
        let addr = "127.0.0.1:6379".parse().unwrap();
        let test_f = super::paired_connect(&addr, &core.handle());
        let send_data = test_f.and_then(|connection| {
            let connection = Rc::new(connection);
            let futures:Vec<_> = test_data.into_iter().map(move |data| {
                let connection_inner = connection.clone();
                connection.send(vec!["INCR", "realistic_test_ctr"]).and_then(move |ctr| {
                    let key = format!("rt_{}", ctr.into_string().unwrap());
                    let d_val = data.0.to_string();
                    connection_inner.send(vec!["SET", &key, &d_val]);
                    connection_inner.send(vec!["SET", &data.1, &key])
                })
            }).collect();
            future::join_all(futures)
        });
        let result = core.run(send_data).unwrap();
        assert_eq!(result.len(), 100);
    }

    #[test]
    fn pubsub_test() {
        let mut core = Core::new().unwrap();
        let handle = core.handle();
        let addr = "127.0.0.1:6379".parse().unwrap();
        let paired_c = super::paired_connect(&addr, &handle);
        let pubsub_c = super::pubsub_connect(&addr, &handle);
        let msgs = paired_c.join(pubsub_c).and_then(|(paired, pubsub)| {
            let subscribe = pubsub.subscribe("test-topic");
            subscribe.and_then(move |msgs| {
                paired.send(vec!["PUBLISH", "test-topic", "test-message"]);
                paired.send(vec!["PUBLISH", "test-not-topic", "test-message-1.5"]);
                paired.send(vec!["PUBLISH", "test-topic", "test-message2"]).map(|_| msgs)
            })
        });
        let tst = msgs.and_then(|msgs| msgs.take(2).collect().map_err(|_| error::internal("unreachable")));
        let result = core.run(tst).unwrap();
        assert_eq!(result.len(), 2);
        assert_eq!(result[0], "test-message".into());
        assert_eq!(result[1], "test-message2".into());
    }
}<|endoftext|>


<commit_before>use ws;
use ws::{CloseCode, Handler, Message , Sender, listen};
use std::thread;

#[derive(Debug, PartialEq)]
enum Action {
    Next,
    Step(u64),
    AddBreakPtn(u64),
    DelBreakPtn(u64),
    Err
}

pub struct Debugger {
    pub step: u64,
}

struct Websocket(Sender);

impl Debugger {
    pub fn new() -> Debugger {
        let debugger = Debugger { step: 0 };
        debugger.listen();
        debugger
    }

    fn listen(&self) {
        thread::spawn(move || { listen("127.0.0.1:3012", |out| Websocket(out)).unwrap(); });
    }
}

impl Handler for Websocket {
    fn on_message(&mut self, msg: Message) -> ws::Result<()> {
        println!("{:?}", handle(msg.as_text().unwrap()));
        Ok(())
    }

    fn on_close(&mut self, code: CloseCode, reason: &str) {
        println!("WebSocket closing for ({:?}) {}", code, reason);
    }
}

//TODO: Better error handling
fn handle(s: &str) -> Result<Action, ()> {
    let mut split = s.split(",");

    match split.next().unwrap() {
        "Next" => Ok(Action::Next),
        "Step" => Ok(Action::Step(split.next().unwrap().parse::<u64>().unwrap())),
        "AddBreakPtn" => Ok(Action::AddBreakPtn(split.next().unwrap().parse::<u64>().unwrap())),
        "DelBreakPtn" => Ok(Action::DelBreakPtn(split.next().unwrap().parse::<u64>().unwrap())),
        _ => Err(())
    }
}

#[test]
fn handle_cmd() {
    assert_eq!(handle("Next"), Ok(Action::Next));
    assert_eq!(handle("Step,45"), Ok(Action::Step(45)));
    assert_eq!(handle("AddBreakPtn,678"), Ok(Action::AddBreakPtn(678)));
    assert_eq!(handle("DelBreakPtn,654"), Ok(Action::DelBreakPtn(654)));

    assert_eq!(handle("Foo"), Err(()));
    assert_eq!(handle("Step,-45"), Err(()));
    assert_eq!(handle("AddBreakPtn"), Err(()));
    assert_eq!(handle("DelBreakPtn,0"), Err(()));

}
<commit_msg>parse debugger command<commit_after>
use std::thread;
use ws;
use ws::{CloseCode, Handler, Message, Sender, listen};

#[derive(Debug, PartialEq)]
enum Action {
    Next,
    Step(u64),
    AddBreakPtn(u64),
    DelBreakPtn(u64),
    Err,
}

pub struct Debugger {
    pub step: u64,
}

struct Websocket(Sender);

impl Debugger {
    pub fn new() -> Debugger {
        let debugger = Debugger { step: 0 };
        debugger.listen();
        debugger
    }

    fn listen(&self) {
        thread::spawn(move || { listen("127.0.0.1:3012", |out| Websocket(out)).unwrap(); });
    }
}

impl Handler for Websocket {
    fn on_message(&mut self, msg: Message) -> ws::Result<()> {
        println!("{:?}", handle(msg.as_text().unwrap()));
        Ok(())
    }

    fn on_close(&mut self, code: CloseCode, reason: &str) {
        println!("WebSocket closing for ({:?}) {}", code, reason);
    }
}

//TODO: Better error handling
fn handle(s: &str) -> Result<Action, ()> {
    let mut split = s.split(",");

    let cmd: &str = match split.next() {
        Some(e) => e,
        None => return Err(()),
    };

    let opt: u64 = match (split.next(), cmd) {
        (None, "Next") => 0,
        (Some(e), _) => {
            match e.parse::<u64>() {
                Ok(u) => match u {
                    0 => return Err(()),
                    _ => u
                },
                Err(_) => return Err(()),
            }
        }
        (None, _) => return Err(()),
    };

    match cmd {
        "Next" => Ok(Action::Next),
        "Step" => Ok(Action::Step(opt)),
        "AddBreakPtn" => Ok(Action::AddBreakPtn(opt)),
        "DelBreakPtn" => Ok(Action::DelBreakPtn(opt)),
        _ => Err(()),
    }
}

#[test]
fn handle_cmd() {
    assert_eq!(handle("Next"), Ok(Action::Next));
    assert_eq!(handle("Step,45"), Ok(Action::Step(45)));
    assert_eq!(handle("AddBreakPtn,678"), Ok(Action::AddBreakPtn(678)));
    assert_eq!(handle("DelBreakPtn,654"), Ok(Action::DelBreakPtn(654)));

    assert_eq!(handle("Foo"), Err(()));
    assert_eq!(handle("Step,-45"), Err(()));
    assert_eq!(handle("AddBreakPtn"), Err(()));
    assert_eq!(handle("DelBreakPtn,0"), Err(()));

}
<|endoftext|>