<commit_before>#![crate_name = "rustful"]

#![crate_type = "rlib"]

#![doc(html_root_url = "http://ogeon.github.io/docs/rustful/master/")]

#![cfg_attr(feature = "nightly", feature(fs_time, path_ext, std_misc))]
#![cfg_attr(test, cfg_attr(feature = "nightly", feature(test)))]

#[cfg(test)]
#[cfg(feature = "nightly")]
extern crate test;

#[cfg(test)]
extern crate tempdir;

extern crate url;
extern crate time;
extern crate hyper;
extern crate anymap;

pub use hyper::mime;
pub use hyper::method::Method;
pub use hyper::status::StatusCode;
pub use hyper::header;
pub use hyper::HttpResult;
pub use hyper::HttpError;
pub use hyper::version::HttpVersion;

pub use self::server::Server;
pub use self::context::Context;
pub use self::response::Response;
pub use self::response::Error;
pub use self::handler::Handler;
pub use self::router::Router;
pub use self::log::Log;
pub use self::router::TreeRouter;

mod utils;
#[macro_use] mod macros;

pub mod server;
pub mod router;
pub mod handler;
pub mod context;
pub mod response;
pub mod filter;
pub mod log;

#[cfg(feature = "nightly")]
pub mod cache;

use std::path::Path;
use std::net::{SocketAddr, SocketAddrV4, SocketAddrV6, Ipv4Addr};
use std::str::FromStr;
use std::any::Any;

///HTTP or HTTPS.
pub enum Scheme<'a> {
    ///Standard HTTP.
    Http,

    ///HTTP with SSL encryption.
    Https {
        ///Path to SSL certificate.
        cert: &'a Path,

        ///Path to key file.
        key: &'a Path
    }
}

///A host address and a port.
///
///Can be conveniently converted from an existing address-port pair or just a port:
///
///```
///use std::net::Ipv4Addr;
///use rustful::Host;
///
///let host1: Host = (Ipv4Addr::new(0, 0, 0, 0), 80).into();
///let host2: Host = 80.into();
///
///assert_eq!(host1, host2);
///```
#[derive(Eq, PartialEq, Debug, Hash, Clone, Copy)]
pub struct Host(SocketAddr);

impl Host {
    ///Create a `Host` with the address `0.0.0.0:port`. This is the same as `port.into()`.
    pub fn any_v4(port: u16) -> Host {
        Host(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), port)))
    }

    ///Change the port of the host address.
    pub fn port(&mut self, port: u16) {
        self.0 = match self.0 {
            SocketAddr::V4(addr) => SocketAddr::V4(SocketAddrV4::new(addr.ip().clone(), port)),
            SocketAddr::V6(addr) => {
                SocketAddr::V6(SocketAddrV6::new(addr.ip().clone(), port, addr.flowinfo(), addr.scope_id()))
            }
        };
    }
}

impl From<Host> for SocketAddr {
    fn from(host: Host) -> SocketAddr {
        host.0
    }
}

impl From<u16> for Host {
    fn from(port: u16) -> Host {
        Host::any_v4(port)
    }
}

impl From<SocketAddr> for Host {
    fn from(addr: SocketAddr) -> Host {
        Host(addr)
    }
}

impl From<SocketAddrV4> for Host {
    fn from(addr: SocketAddrV4) -> Host {
        Host(SocketAddr::V4(addr))
    }
}

impl From<SocketAddrV6> for Host {
    fn from(addr: SocketAddrV6) -> Host {
        Host(SocketAddr::V6(addr))
    }
}

impl From<(Ipv4Addr, u16)> for Host {
    fn from((ip, port): (Ipv4Addr, u16)) -> Host {
        Host(SocketAddr::V4(SocketAddrV4::new(ip, port)))
    }
}

impl FromStr for Host {
    type Err = <SocketAddr as FromStr>::Err;

    fn from_str(s: &str) -> Result<Host, Self::Err> {
        s.parse().map(|s| Host(s))
    }
}

///Globally accessible data.
///
///This structure can currently only hold either one item or nothing.
///A future version will be able to hold multiple items.
pub enum Global {
    None,
    One(Box<Any + Send + Sync>),
}

impl Global {
    ///Borrow a value of type `T` if the there is one.
    pub fn get<T: Any + Send + Sync>(&self) -> Option<&T> {
        match self {
            &Global::None => None,
            &Global::One(ref a) => (&**a as &Any).downcast_ref(),
        }
    }
}

impl<T: Any + Send + Sync> From<Box<T>> for Global {
    fn from(data: Box<T>) -> Global {
        Global::One(data)
    }
}

impl Default for Global {
    fn default() -> Global {
        Global::None
    }
}
<commit_msg>Write a better description for Global and add an example<commit_after>#![crate_name = "rustful"]

#![crate_type = "rlib"]

#![doc(html_root_url = "http://ogeon.github.io/docs/rustful/master/")]

#![cfg_attr(feature = "nightly", feature(fs_time, path_ext, std_misc))]
#![cfg_attr(test, cfg_attr(feature = "nightly", feature(test)))]

#[cfg(test)]
#[cfg(feature = "nightly")]
extern crate test;

#[cfg(test)]
extern crate tempdir;

extern crate url;
extern crate time;
extern crate hyper;
extern crate anymap;

pub use hyper::mime;
pub use hyper::method::Method;
pub use hyper::status::StatusCode;
pub use hyper::header;
pub use hyper::HttpResult;
pub use hyper::HttpError;
pub use hyper::version::HttpVersion;

pub use self::server::Server;
pub use self::context::Context;
pub use self::response::Response;
pub use self::response::Error;
pub use self::handler::Handler;
pub use self::router::Router;
pub use self::log::Log;
pub use self::router::TreeRouter;

mod utils;
#[macro_use] mod macros;

pub mod server;
pub mod router;
pub mod handler;
pub mod context;
pub mod response;
pub mod filter;
pub mod log;

#[cfg(feature = "nightly")]
pub mod cache;

use std::path::Path;
use std::net::{SocketAddr, SocketAddrV4, SocketAddrV6, Ipv4Addr};
use std::str::FromStr;
use std::any::Any;

///HTTP or HTTPS.
pub enum Scheme<'a> {
    ///Standard HTTP.
    Http,

    ///HTTP with SSL encryption.
    Https {
        ///Path to SSL certificate.
        cert: &'a Path,

        ///Path to key file.
        key: &'a Path
    }
}

///A host address and a port.
///
///Can be conveniently converted from an existing address-port pair or just a port:
///
///```
///use std::net::Ipv4Addr;
///use rustful::Host;
///
///let host1: Host = (Ipv4Addr::new(0, 0, 0, 0), 80).into();
///let host2: Host = 80.into();
///
///assert_eq!(host1, host2);
///```
#[derive(Eq, PartialEq, Debug, Hash, Clone, Copy)]
pub struct Host(SocketAddr);

impl Host {
    ///Create a `Host` with the address `0.0.0.0:port`. This is the same as `port.into()`.
    pub fn any_v4(port: u16) -> Host {
        Host(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), port)))
    }

    ///Change the port of the host address.
    pub fn port(&mut self, port: u16) {
        self.0 = match self.0 {
            SocketAddr::V4(addr) => SocketAddr::V4(SocketAddrV4::new(addr.ip().clone(), port)),
            SocketAddr::V6(addr) => {
                SocketAddr::V6(SocketAddrV6::new(addr.ip().clone(), port, addr.flowinfo(), addr.scope_id()))
            }
        };
    }
}

impl From<Host> for SocketAddr {
    fn from(host: Host) -> SocketAddr {
        host.0
    }
}

impl From<u16> for Host {
    fn from(port: u16) -> Host {
        Host::any_v4(port)
    }
}

impl From<SocketAddr> for Host {
    fn from(addr: SocketAddr) -> Host {
        Host(addr)
    }
}

impl From<SocketAddrV4> for Host {
    fn from(addr: SocketAddrV4) -> Host {
        Host(SocketAddr::V4(addr))
    }
}

impl From<SocketAddrV6> for Host {
    fn from(addr: SocketAddrV6) -> Host {
        Host(SocketAddr::V6(addr))
    }
}

impl From<(Ipv4Addr, u16)> for Host {
    fn from((ip, port): (Ipv4Addr, u16)) -> Host {
        Host(SocketAddr::V4(SocketAddrV4::new(ip, port)))
    }
}

impl FromStr for Host {
    type Err = <SocketAddr as FromStr>::Err;

    fn from_str(s: &str) -> Result<Host, Self::Err> {
        s.parse().map(|s| Host(s))
    }
}

///Container for globally accessible data.
///
///It should only be created using `Global::default()`/`Default::default()` or
///`Box::new(v).into()`.
///
///```
///# use rustful::Global;
///let g: Global = Box::new(5).into();
///assert_eq!(g.get(), Some(&5));
///```
///
///Direct manipulation of its variants may cause an undesired state.
///
///_Note: This structure can currently only hold either one item or nothing.
///A future version will be able to hold multiple items._
pub enum Global {
    #[doc(hidden)]
    None,
    #[doc(hidden)]
    One(Box<Any + Send + Sync>),
}

impl Global {
    ///Borrow a value of type `T` if the there is one.
    pub fn get<T: Any + Send + Sync>(&self) -> Option<&T> {
        match self {
            &Global::None => None,
            &Global::One(ref a) => (&**a as &Any).downcast_ref(),
        }
    }
}

impl<T: Any + Send + Sync> From<Box<T>> for Global {
    fn from(data: Box<T>) -> Global {
        Global::One(data)
    }
}

impl Default for Global {
    fn default() -> Global {
        Global::None
    }
}
<|endoftext|>


<commit_before>use std::io;
use std::fs::File;
use std::thread;
use std::path::{Path, PathBuf};
use futures::sync::oneshot;
use futures::future::{self, Future};
use hyper::{self, StatusCode};
use hyper::header::ContentLength;
use hyper::server::{Request, Response, Service};

static NOTFOUND: &[u8] = b"Not Found";

/// Http Service for static file server
pub struct HttpService {
    pub root: PathBuf,
}

impl Service for HttpService {
    type Request = Request;
    type Response = Response;
    type Error = hyper::Error;
    type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;

    fn call(&self, req: Request) -> Self::Future {
        let uri_path = req.uri().path();
        match local_path_for_request(&self.root, &uri_path) {
            None => Box::new(future::ok(not_found_response())),
            Some(path) => {
                info!("{:20} ==> {}", uri_path, path.display());
                let (tx, rx) = oneshot::channel();
                thread::spawn(move || {
                    let mut file = match File::open(path) {
                        Ok(f) => f,
                        Err(_) => {
                            tx.send(not_found_response()).expect("Send error on open");
                            return;
                        },
                    };
                    let mut buf: Vec<u8> = Vec::new();
                    match io::copy(&mut file, &mut buf) {
                        Ok(_) => {
                            let res = Response::new()
                                .with_header(ContentLength(buf.len() as u64))
                                .with_body(buf);
                            tx.send(res)
                              .expect("Send error on successful file read");
                        },
                        Err(_) => {
                            tx.send(Response::new().with_status(StatusCode::InternalServerError))
                              .expect("Send error on error reading file");
                        },
                    };
                });
                Box::new(rx.map_err(|e| io::Error::new(io::ErrorKind::Other, e).into()))
            }
        }
    }
}

fn not_found_response() -> Response {
    Response::new()
            .with_status(StatusCode::NotFound)
            .with_header(ContentLength(NOTFOUND.len() as u64))
            .with_body(NOTFOUND)
}

fn local_path_for_request<P: AsRef<Path>>(root_dir: P, request_path: &str) -> Option<PathBuf> {
    if !request_path.starts_with("/") {
        return None;
    }
    // skip query string
    let end = request_path.find('?').unwrap_or(request_path.len());
    let request_path = &request_path[0..end];

    let mut path = root_dir.as_ref().to_owned();
    // request path start with "/"
    path.push("_builds");
    path.push(&request_path[1..]);
    if request_path == "/" {
        path.push("index.html");
    }

    Some(path)
}
<commit_msg>change service log format<commit_after>use std::io;
use std::fs::File;
use std::thread;
use std::path::{Path, PathBuf};
use futures::sync::oneshot;
use futures::future::{self, Future};
use hyper::{self, StatusCode};
use hyper::header::ContentLength;
use hyper::server::{Request, Response, Service};

static NOTFOUND: &[u8] = b"Not Found";

/// Http Service for static file server
pub struct HttpService {
    pub root: PathBuf,
}

impl Service for HttpService {
    type Request = Request;
    type Response = Response;
    type Error = hyper::Error;
    type Future = Box<Future<Item=Self::Response, Error=Self::Error>>;

    fn call(&self, req: Request) -> Self::Future {
        let uri_path = req.uri().path();
        match local_path_for_request(&self.root, &uri_path) {
            None => Box::new(future::ok(not_found_response())),
            Some(path) => {
                info!("{}", uri_path);
                let (tx, rx) = oneshot::channel();
                thread::spawn(move || {
                    let mut file = match File::open(path) {
                        Ok(f) => f,
                        Err(_) => {
                            tx.send(not_found_response()).expect("Send error on open");
                            return;
                        },
                    };
                    let mut buf: Vec<u8> = Vec::new();
                    match io::copy(&mut file, &mut buf) {
                        Ok(_) => {
                            let res = Response::new()
                                .with_header(ContentLength(buf.len() as u64))
                                .with_body(buf);
                            tx.send(res)
                              .expect("Send error on successful file read");
                        },
                        Err(_) => {
                            tx.send(Response::new().with_status(StatusCode::InternalServerError))
                              .expect("Send error on error reading file");
                        },
                    };
                });
                Box::new(rx.map_err(|e| io::Error::new(io::ErrorKind::Other, e).into()))
            }
        }
    }
}

fn not_found_response() -> Response {
    Response::new()
            .with_status(StatusCode::NotFound)
            .with_header(ContentLength(NOTFOUND.len() as u64))
            .with_body(NOTFOUND)
}

fn local_path_for_request<P: AsRef<Path>>(root_dir: P, request_path: &str) -> Option<PathBuf> {
    if !request_path.starts_with("/") {
        return None;
    }
    // skip query string
    let end = request_path.find('?').unwrap_or(request_path.len());
    let request_path = &request_path[0..end];

    let mut path = root_dir.as_ref().to_owned();
    // request path start with "/"
    path.push("_builds");
    path.push(&request_path[1..]);
    if request_path == "/" {
        path.push("index.html");
    }

    Some(path)
}
<|endoftext|>


<commit_before>#![crate_name = "macaroons"]
#![crate_type = "lib"]

#![feature(core)]
#![feature(collections)]

use std::slice::bytes;

pub mod caveat;
pub use caveat::{Caveat, Predicate};
pub use sodiumoxide::crypto::auth::hmacsha256::{Key, Tag, TAGBYTES};

extern crate sodiumoxide;
use sodiumoxide::crypto::auth::hmacsha256::authenticate;

extern crate "rustc-serialize" as serialize;
use serialize::base64::{self, FromBase64, ToBase64};

// Macaroons personalize the HMAC key using this string
// "macaroons-key-generator" padded to 32-bytes with zeroes
const KEY_GENERATOR: &'static [u8; 32] = b"macaroons-key-generator\0\0\0\0\0\0\0\0\0";

const PACKET_PREFIX_LENGTH: usize = 4;
const MAX_PACKET_LENGTH:    usize = 65535;

pub struct Token {
  pub location:   Vec<u8>,
  pub identifier: Vec<u8>,
  pub caveats:    Vec<Caveat>,
  pub tag:        Tag
}

struct Packet {
  pub id:     Vec<u8>,
  pub value:  Vec<u8>,
  pub length: usize
}

impl Token {
  pub fn new(key: Vec<u8>, identifier: Vec<u8>, location: Vec<u8>) -> Token {
    let Tag(personalized_key) = authenticate(&key, &Key(*KEY_GENERATOR));
    let tag = authenticate(&identifier, &Key(personalized_key));

    Token {
      location:   location,
      identifier: identifier,
      caveats:    Vec::new(),
      tag:        tag
    }
  }

  pub fn deserialize(macaroon: Vec<u8>) -> Result<Token, &'static str> {
    let mut location:   Option<Vec<u8>> = None;
    let mut identifier: Option<Vec<u8>> = None;
    let mut caveats:    Vec<Caveat>     = Vec::new();
    let mut tag:        Option<Tag>     = None;

    let token_data = match macaroon.as_slice().from_base64() {
      Ok(bytes) => bytes,
      _         => return Err("couldn't parse base64")
    };

    let mut index: usize = 0;

    while index < token_data.len() {
      let packet = match Token::depacketize(&token_data, index) {
        Ok(p)       => p,
        Err(reason) => return Err(reason)
      };

      index += packet.length;

      match packet.id.as_slice() {
        b"location"   => location   = Some(packet.value),
        b"identifier" => identifier = Some(packet.value),
        b"cid"        => caveats.push(Caveat::new(Predicate(packet.value))),
        b"signature"  => {
          if packet.value.len() != TAGBYTES {
            return Err("invalid signature length")
          }

          let mut signature_bytes = [0u8; TAGBYTES];
          bytes::copy_memory(&mut signature_bytes, &packet.value[..TAGBYTES]);

          tag = Some(Tag(signature_bytes))
        },
        _ => return Err("unrecognized packet type")
      }
    }

    if location   == None { return Err("no 'location' found"); }
    if identifier == None { return Err("no 'identifier' found"); }
    if tag        == None { return Err("no 'signature' found"); }

    let token = Token {
      location:   location.unwrap(),
      identifier: identifier.unwrap(),
      caveats:    caveats,
      tag:        tag.unwrap()
    };

    Ok(token)
  }

  fn depacketize(data: &Vec<u8>, index: usize) -> Result<Packet, &'static str> {
    // TODO: parse this length without involving any UTF-8 conversions
    let length_str = match std::str::from_utf8(&data[index .. index + PACKET_PREFIX_LENGTH]) {
      Ok(string) => string,
      _          => return Err("couldn't stringify packet length")
    };

    let packet_length: usize = match std::num::from_str_radix(length_str, 16) {
      Ok(length) => length,
      _          => return Err("couldn't parse packet length")
    };

    let mut packet_bytes = data[index + PACKET_PREFIX_LENGTH .. index + packet_length].to_vec();

    let pos = match packet_bytes.iter().position(|&byte| byte == b' ') {
      Some(i) => i,
      None    => return Err("malformed packet")
    };

    let mut value = packet_bytes.split_off(pos);
    value.remove(0);

    match value.pop().unwrap() {
      b'\n' => (),
      _     => return Err("packet not newline terminated")
    }

    Ok(Packet { id: packet_bytes, value: value, length: packet_length })
  }

  pub fn add_caveat(&self, caveat: Caveat) -> Token {
    let Tag(key_bytes) = self.tag;
    let Predicate(predicate_bytes) = caveat.predicate.clone();
    let tag = authenticate(&predicate_bytes, &Key(key_bytes));

    let mut new_caveats = self.caveats.to_vec();
    new_caveats.push(caveat);

    Token {
      identifier: self.identifier.clone(),
      location:   self.location.clone(),
      caveats:    new_caveats,
      tag:        tag
    }
  }

  pub fn serialize(&self) -> Vec<u8> {
    // TODO: estimate capacity and use Vec::with_capacity
    let mut result: Vec<u8> = Vec::new();

    Token::packetize(&mut result, "location",   &self.location);
    Token::packetize(&mut result, "identifier", &self.identifier);

    for caveat in self.caveats.iter() {
      let Predicate(predicate_bytes) = caveat.predicate.clone();
      Token::packetize(&mut result, "cid", &predicate_bytes);
    }

    let Tag(signature_bytes) = self.tag;
    let mut signature_vec = Vec::new();
    signature_vec.push_all(&signature_bytes);

    Token::packetize(&mut result, "signature", &signature_vec);

    result.to_base64(base64::URL_SAFE).into_bytes()
  }

  fn packetize(result: &mut Vec<u8>, field: &str, value: &Vec<u8>) {
    let field_bytes: Vec<u8> = String::from_str(field).into_bytes();
    let packet_length = PACKET_PREFIX_LENGTH + field_bytes.len() + value.len() + 2;

    if packet_length > MAX_PACKET_LENGTH {
      panic!("packet too large to serialize");
    }

    let mut pkt_line = format!("{:04x}{} ", packet_length, field).into_bytes();
    result.append(&mut pkt_line);
    result.append(&mut value.clone());
    result.push('\n' as u8);
  }
}
<commit_msg>this -> the<commit_after>#![crate_name = "macaroons"]
#![crate_type = "lib"]

#![feature(core)]
#![feature(collections)]

use std::slice::bytes;

pub mod caveat;
pub use caveat::{Caveat, Predicate};
pub use sodiumoxide::crypto::auth::hmacsha256::{Key, Tag, TAGBYTES};

extern crate sodiumoxide;
use sodiumoxide::crypto::auth::hmacsha256::authenticate;

extern crate "rustc-serialize" as serialize;
use serialize::base64::{self, FromBase64, ToBase64};

// Macaroons personalize the HMAC key using the string
// "macaroons-key-generator" padded to 32-bytes with zeroes
const KEY_GENERATOR: &'static [u8; 32] = b"macaroons-key-generator\0\0\0\0\0\0\0\0\0";

const PACKET_PREFIX_LENGTH: usize = 4;
const MAX_PACKET_LENGTH:    usize = 65535;

pub struct Token {
  pub location:   Vec<u8>,
  pub identifier: Vec<u8>,
  pub caveats:    Vec<Caveat>,
  pub tag:        Tag
}

struct Packet {
  pub id:     Vec<u8>,
  pub value:  Vec<u8>,
  pub length: usize
}

impl Token {
  pub fn new(key: Vec<u8>, identifier: Vec<u8>, location: Vec<u8>) -> Token {
    let Tag(personalized_key) = authenticate(&key, &Key(*KEY_GENERATOR));
    let tag = authenticate(&identifier, &Key(personalized_key));

    Token {
      location:   location,
      identifier: identifier,
      caveats:    Vec::new(),
      tag:        tag
    }
  }

  pub fn deserialize(macaroon: Vec<u8>) -> Result<Token, &'static str> {
    let mut location:   Option<Vec<u8>> = None;
    let mut identifier: Option<Vec<u8>> = None;
    let mut caveats:    Vec<Caveat>     = Vec::new();
    let mut tag:        Option<Tag>     = None;

    let token_data = match macaroon.as_slice().from_base64() {
      Ok(bytes) => bytes,
      _         => return Err("couldn't parse base64")
    };

    let mut index: usize = 0;

    while index < token_data.len() {
      let packet = match Token::depacketize(&token_data, index) {
        Ok(p)       => p,
        Err(reason) => return Err(reason)
      };

      index += packet.length;

      match packet.id.as_slice() {
        b"location"   => location   = Some(packet.value),
        b"identifier" => identifier = Some(packet.value),
        b"cid"        => caveats.push(Caveat::new(Predicate(packet.value))),
        b"signature"  => {
          if packet.value.len() != TAGBYTES {
            return Err("invalid signature length")
          }

          let mut signature_bytes = [0u8; TAGBYTES];
          bytes::copy_memory(&mut signature_bytes, &packet.value[..TAGBYTES]);

          tag = Some(Tag(signature_bytes))
        },
        _ => return Err("unrecognized packet type")
      }
    }

    if location   == None { return Err("no 'location' found"); }
    if identifier == None { return Err("no 'identifier' found"); }
    if tag        == None { return Err("no 'signature' found"); }

    let token = Token {
      location:   location.unwrap(),
      identifier: identifier.unwrap(),
      caveats:    caveats,
      tag:        tag.unwrap()
    };

    Ok(token)
  }

  fn depacketize(data: &Vec<u8>, index: usize) -> Result<Packet, &'static str> {
    // TODO: parse this length without involving any UTF-8 conversions
    let length_str = match std::str::from_utf8(&data[index .. index + PACKET_PREFIX_LENGTH]) {
      Ok(string) => string,
      _          => return Err("couldn't stringify packet length")
    };

    let packet_length: usize = match std::num::from_str_radix(length_str, 16) {
      Ok(length) => length,
      _          => return Err("couldn't parse packet length")
    };

    let mut packet_bytes = data[index + PACKET_PREFIX_LENGTH .. index + packet_length].to_vec();

    let pos = match packet_bytes.iter().position(|&byte| byte == b' ') {
      Some(i) => i,
      None    => return Err("malformed packet")
    };

    let mut value = packet_bytes.split_off(pos);
    value.remove(0);

    match value.pop().unwrap() {
      b'\n' => (),
      _     => return Err("packet not newline terminated")
    }

    Ok(Packet { id: packet_bytes, value: value, length: packet_length })
  }

  pub fn add_caveat(&self, caveat: Caveat) -> Token {
    let Tag(key_bytes) = self.tag;
    let Predicate(predicate_bytes) = caveat.predicate.clone();
    let tag = authenticate(&predicate_bytes, &Key(key_bytes));

    let mut new_caveats = self.caveats.to_vec();
    new_caveats.push(caveat);

    Token {
      identifier: self.identifier.clone(),
      location:   self.location.clone(),
      caveats:    new_caveats,
      tag:        tag
    }
  }

  pub fn serialize(&self) -> Vec<u8> {
    // TODO: estimate capacity and use Vec::with_capacity
    let mut result: Vec<u8> = Vec::new();

    Token::packetize(&mut result, "location",   &self.location);
    Token::packetize(&mut result, "identifier", &self.identifier);

    for caveat in self.caveats.iter() {
      let Predicate(predicate_bytes) = caveat.predicate.clone();
      Token::packetize(&mut result, "cid", &predicate_bytes);
    }

    let Tag(signature_bytes) = self.tag;
    let mut signature_vec = Vec::new();
    signature_vec.push_all(&signature_bytes);

    Token::packetize(&mut result, "signature", &signature_vec);

    result.to_base64(base64::URL_SAFE).into_bytes()
  }

  fn packetize(result: &mut Vec<u8>, field: &str, value: &Vec<u8>) {
    let field_bytes: Vec<u8> = String::from_str(field).into_bytes();
    let packet_length = PACKET_PREFIX_LENGTH + field_bytes.len() + value.len() + 2;

    if packet_length > MAX_PACKET_LENGTH {
      panic!("packet too large to serialize");
    }

    let mut pkt_line = format!("{:04x}{} ", packet_length, field).into_bytes();
    result.append(&mut pkt_line);
    result.append(&mut value.clone());
    result.push('\n' as u8);
  }
}
<|endoftext|>


<commit_before>//! Primitives to Configure the Execution Environment

use parking_lot::RwLock;
use slog::{DrainExt, Logger};
use conn::{ConnectOpts, ConnectionManager};
use r2d2::{Pool, PooledConnection};
use errors::*;
use super::Result;
use slog_term;

lazy_static! {
    static ref POOL: RwLock<Option<Vec<Pool<ConnectionManager>>>> = RwLock::new(None);

    static ref LOGGER: RwLock<Logger> = RwLock::new(
                    Logger::root(
                        slog_term::streamer().compact().build().fuse(),
                        o!("version" => env!("CARGO_PKG_VERSION"))
                        )
                    );

    static ref CONFIG: RwLock<ConnectOpts> = RwLock::new(ConnectOpts::default());
}

pub struct Client;

pub struct Config;

impl Client {
    pub fn logger() -> &'static RwLock<Logger> {
        &LOGGER
    }

    pub fn pool() -> &'static RwLock<Option<Vec<Pool<ConnectionManager>>>> {
        &POOL
    }

    pub fn config() -> &'static RwLock<ConnectOpts> {
        &CONFIG
    }

    pub fn conn() -> Result<PooledConnection<ConnectionManager>> {
        let logger = Self::logger().read();
        let cfg = Self::config().read();
        trace!(logger, "Calling Client::conn()");
        let pool = Self::pool().read();
        match *pool {
            Some(ref pool) => {
                let mut num_retries = cfg.retries;
                let msg = String::from("Failed to get a connection.");
                let mut last_error = Err(From::from(ConnectionError::Other(msg)));
                while num_retries > 0 {
                    let mut least_connections = 0;
                    let mut least_connected_server = 0;
                    let mut most_idle = 0;
                    let mut most_idle_server = 0;
                    for (i, p) in pool.iter().enumerate() {
                        let state = p.state();
                        if least_connections == 0 || least_connections > state.connections {
                            least_connections = state.connections;
                            least_connected_server = i
                        }
                        if most_idle == 0 || most_idle < state.idle_connections {
                            most_idle = state.idle_connections;
                            most_idle_server = i
                        }
                    }
                    if most_idle > 0 {
                        match pool[most_idle_server].get() {
                            Ok(conn) => return Ok(conn),
                            Err(error) => last_error = Err(From::from(error)),
                        }
                    } else if least_connections > 0 {
                        match pool[least_connected_server].get() {
                            Ok(conn) => return Ok(conn),
                            Err(error) => last_error = Err(From::from(error)),
                        }
                    } else {
                        let msg = String::from("All servers are currently down.");
                        last_error = Err(From::from(ConnectionError::Other(msg)));
                    }
                    num_retries -= 1;
                }
                return last_error;
            }
            None => {
                let msg = String::from("Your connection pool is not initialised. \
                                   Use `r.connection().connect()` to initialise the pool \
                                   before trying to send any connections to the database. \
                                   This is typically done in the `main` function.");
                return Err(From::from(ConnectionError::Other(msg)));
            }
        }
    }

    pub fn set_pool(p: Vec<Pool<ConnectionManager>>) -> Result<()> {
        let mut pool = POOL.write();
        *pool = Some(p);
        Ok(())
    }

    pub fn set_config(c: ConnectOpts) -> Result<()> {
        let mut cfg = CONFIG.write();
        *cfg = c;
        Ok(())
    }
}
<commit_msg>Refactor logger var<commit_after>//! Primitives to Configure the Execution Environment

use parking_lot::RwLock;
use slog::{DrainExt, Logger};
use conn::{ConnectOpts, ConnectionManager};
use r2d2::{Pool, PooledConnection};
use errors::*;
use super::Result;
use slog_term;

lazy_static! {
    static ref POOL: RwLock<Option<Vec<Pool<ConnectionManager>>>> = RwLock::new(None);

    static ref LOGGER: RwLock<Logger> = RwLock::new({
        let drain = slog_term::streamer().compact().build().fuse();
        Logger::root(drain, o!("version" => env!("CARGO_PKG_VERSION")))
    });

    static ref CONFIG: RwLock<ConnectOpts> = RwLock::new(ConnectOpts::default());
}

pub struct Client;

pub struct Config;

impl Client {
    pub fn logger() -> &'static RwLock<Logger> {
        &LOGGER
    }

    pub fn pool() -> &'static RwLock<Option<Vec<Pool<ConnectionManager>>>> {
        &POOL
    }

    pub fn config() -> &'static RwLock<ConnectOpts> {
        &CONFIG
    }

    pub fn conn() -> Result<PooledConnection<ConnectionManager>> {
        let logger = Self::logger().read();
        let cfg = Self::config().read();
        trace!(logger, "Calling Client::conn()");
        let pool = Self::pool().read();
        match *pool {
            Some(ref pool) => {
                let mut num_retries = cfg.retries;
                let msg = String::from("Failed to get a connection.");
                let mut last_error = Err(From::from(ConnectionError::Other(msg)));
                while num_retries > 0 {
                    let mut least_connections = 0;
                    let mut least_connected_server = 0;
                    let mut most_idle = 0;
                    let mut most_idle_server = 0;
                    for (i, p) in pool.iter().enumerate() {
                        let state = p.state();
                        if least_connections == 0 || least_connections > state.connections {
                            least_connections = state.connections;
                            least_connected_server = i
                        }
                        if most_idle == 0 || most_idle < state.idle_connections {
                            most_idle = state.idle_connections;
                            most_idle_server = i
                        }
                    }
                    if most_idle > 0 {
                        match pool[most_idle_server].get() {
                            Ok(conn) => return Ok(conn),
                            Err(error) => last_error = Err(From::from(error)),
                        }
                    } else if least_connections > 0 {
                        match pool[least_connected_server].get() {
                            Ok(conn) => return Ok(conn),
                            Err(error) => last_error = Err(From::from(error)),
                        }
                    } else {
                        let msg = String::from("All servers are currently down.");
                        last_error = Err(From::from(ConnectionError::Other(msg)));
                    }
                    num_retries -= 1;
                }
                return last_error;
            }
            None => {
                let msg = String::from("Your connection pool is not initialised. \
                                   Use `r.connection().connect()` to initialise the pool \
                                   before trying to send any connections to the database. \
                                   This is typically done in the `main` function.");
                return Err(From::from(ConnectionError::Other(msg)));
            }
        }
    }

    pub fn set_pool(p: Vec<Pool<ConnectionManager>>) -> Result<()> {
        let mut pool = POOL.write();
        *pool = Some(p);
        Ok(())
    }

    pub fn set_config(c: ConnectOpts) -> Result<()> {
        let mut cfg = CONFIG.write();
        *cfg = c;
        Ok(())
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

/*!
Syntax extension to create floating point literals from hexadecimal strings

Once loaded, hexfloat!() is called with a string containing the hexadecimal
floating-point literal, and an optional type (f32 or f64).
If the type is omitted, the literal is treated the same as a normal unsuffixed
literal.

# Examples

To load the extension and use it:

```rust,ignore
#[phase(plugin)]
extern crate hexfloat;

fn main() {
    let val = hexfloat!("0x1.ffffb4", f32);
}
```

# References

* [ExploringBinary: hexadecimal floating point constants]
  (http://www.exploringbinary.com/hexadecimal-floating-point-constants/)

*/

#![crate_name = "hexfloat"]
#![experimental]
#![crate_type = "rlib"]
#![crate_type = "dylib"]
#![license = "MIT/ASL2"]
#![doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "http://www.rust-lang.org/favicon.ico",
       html_root_url = "http://doc.rust-lang.org/master/")]
#![feature(plugin_registrar)]

extern crate syntax;
extern crate rustc;

use syntax::ast;
use syntax::codemap::{Span, mk_sp};
use syntax::ext::base;
use syntax::ext::base::{ExtCtxt, MacExpr};
use syntax::ext::build::AstBuilder;
use syntax::parse::token;
use syntax::ptr::P;
use rustc::plugin::Registry;

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("hexfloat", expand_syntax_ext);
}

//Check if the literal is valid (as LLVM expects),
//and return a descriptive error if not.
fn hex_float_lit_err(s: &str) -> Option<(uint, String)> {
    let mut chars = s.chars().peekable();
    let mut i = 0;
    if chars.peek() == Some(&'-') { chars.next(); i+= 1 }
    if chars.next() != Some('0') {
        return Some((i, "Expected '0'".to_string()));
    } i+=1;
    if chars.next() != Some('x') {
        return Some((i, "Expected 'x'".to_string()));
    } i+=1;
    let mut d_len = 0i;
    for _ in chars.take_while(|c| c.is_digit(16)) { chars.next(); i+=1; d_len += 1;}
    if chars.next() != Some('.') {
        return Some((i, "Expected '.'".to_string()));
    } i+=1;
    let mut f_len = 0i;
    for _ in chars.take_while(|c| c.is_digit(16)) { chars.next(); i+=1; f_len += 1;}
    if d_len == 0 && f_len == 0 {
        return Some((i, "Expected digits before or after decimal \
                         point".to_string()));
    }
    if chars.next() != Some('p') {
        return Some((i, "Expected 'p'".to_string()));
    } i+=1;
    if chars.peek() == Some(&'-') { chars.next(); i+= 1 }
    let mut e_len = 0i;
    for _ in chars.take_while(|c| c.is_numeric()) { chars.next(); i+=1; e_len += 1}
    if e_len == 0 {
        return Some((i, "Expected exponent digits".to_string()));
    }
    match chars.next() {
        None => None,
        Some(_) => Some((i, "Expected end of string".to_string()))
    }
}

pub fn expand_syntax_ext(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
                         -> Box<base::MacResult + 'static> {
    let (expr, ty_lit) = parse_tts(cx, tts);

    let ty = match ty_lit {
        None => None,
        Some(Ident{ident, span}) => match token::get_ident(ident).get() {
            "f32" => Some(ast::TyF32),
            "f64" => Some(ast::TyF64),
            _ => {
                cx.span_err(span, "invalid floating point type in hexfloat!");
                None
            }
        }
    };

    let s = match expr.node {
        // expression is a literal
        ast::ExprLit(ref lit) => match lit.node {
            // string literal
            ast::LitStr(ref s, _) => {
                s.clone()
            }
            _ => {
                cx.span_err(expr.span, "unsupported literal in hexfloat!");
                return base::DummyResult::expr(sp);
            }
        },
        _ => {
            cx.span_err(expr.span, "non-literal in hexfloat!");
            return base::DummyResult::expr(sp);
        }
    };

    {
        let err = hex_float_lit_err(s.get());
        match err {
            Some((err_pos, err_str)) => {
                let pos = expr.span.lo + syntax::codemap::Pos::from_uint(err_pos + 1);
                let span = syntax::codemap::mk_sp(pos,pos);
                cx.span_err(span,
                            format!("invalid hex float literal in hexfloat!: \
                                     {}",
                                    err_str).as_slice());
                return base::DummyResult::expr(sp);
            }
            _ => ()
        }
    }

    let lit = match ty {
        None => ast::LitFloatUnsuffixed(s),
        Some (ty) => ast::LitFloat(s, ty)
    };
    MacExpr::new(cx.expr_lit(sp, lit))
}

struct Ident {
    ident: ast::Ident,
    span: Span
}

fn parse_tts(cx: &ExtCtxt,
             tts: &[ast::TokenTree]) -> (P<ast::Expr>, Option<Ident>) {
    let p = &mut cx.new_parser_from_tts(tts);
    let ex = p.parse_expr();
    let id = if p.token == token::Eof {
        None
    } else {
        p.expect(&token::Comma);
        let lo = p.span.lo;
        let ident = p.parse_ident();
        let hi = p.last_span.hi;
        Some(Ident{ident: ident, span: mk_sp(lo, hi)})
    };
    if p.token != token::Eof {
        p.unexpected();
    }
    (ex, id)
}

// FIXME (10872): This is required to prevent an LLVM assert on Windows
#[test]
fn dummy_test() { }
<commit_msg>remove unused license attribute<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

/*!
Syntax extension to create floating point literals from hexadecimal strings

Once loaded, hexfloat!() is called with a string containing the hexadecimal
floating-point literal, and an optional type (f32 or f64).
If the type is omitted, the literal is treated the same as a normal unsuffixed
literal.

# Examples

To load the extension and use it:

```rust,ignore
#[phase(plugin)]
extern crate hexfloat;

fn main() {
    let val = hexfloat!("0x1.ffffb4", f32);
}
```

# References

* [ExploringBinary: hexadecimal floating point constants]
  (http://www.exploringbinary.com/hexadecimal-floating-point-constants/)

*/

#![crate_name = "hexfloat"]
#![experimental]
#![crate_type = "rlib"]
#![crate_type = "dylib"]
#![doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "http://www.rust-lang.org/favicon.ico",
       html_root_url = "http://doc.rust-lang.org/master/")]
#![feature(plugin_registrar)]

extern crate syntax;
extern crate rustc;

use syntax::ast;
use syntax::codemap::{Span, mk_sp};
use syntax::ext::base;
use syntax::ext::base::{ExtCtxt, MacExpr};
use syntax::ext::build::AstBuilder;
use syntax::parse::token;
use syntax::ptr::P;
use rustc::plugin::Registry;

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("hexfloat", expand_syntax_ext);
}

//Check if the literal is valid (as LLVM expects),
//and return a descriptive error if not.
fn hex_float_lit_err(s: &str) -> Option<(uint, String)> {
    let mut chars = s.chars().peekable();
    let mut i = 0;
    if chars.peek() == Some(&'-') { chars.next(); i+= 1 }
    if chars.next() != Some('0') {
        return Some((i, "Expected '0'".to_string()));
    } i+=1;
    if chars.next() != Some('x') {
        return Some((i, "Expected 'x'".to_string()));
    } i+=1;
    let mut d_len = 0i;
    for _ in chars.take_while(|c| c.is_digit(16)) { chars.next(); i+=1; d_len += 1;}
    if chars.next() != Some('.') {
        return Some((i, "Expected '.'".to_string()));
    } i+=1;
    let mut f_len = 0i;
    for _ in chars.take_while(|c| c.is_digit(16)) { chars.next(); i+=1; f_len += 1;}
    if d_len == 0 && f_len == 0 {
        return Some((i, "Expected digits before or after decimal \
                         point".to_string()));
    }
    if chars.next() != Some('p') {
        return Some((i, "Expected 'p'".to_string()));
    } i+=1;
    if chars.peek() == Some(&'-') { chars.next(); i+= 1 }
    let mut e_len = 0i;
    for _ in chars.take_while(|c| c.is_numeric()) { chars.next(); i+=1; e_len += 1}
    if e_len == 0 {
        return Some((i, "Expected exponent digits".to_string()));
    }
    match chars.next() {
        None => None,
        Some(_) => Some((i, "Expected end of string".to_string()))
    }
}

pub fn expand_syntax_ext(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
                         -> Box<base::MacResult + 'static> {
    let (expr, ty_lit) = parse_tts(cx, tts);

    let ty = match ty_lit {
        None => None,
        Some(Ident{ident, span}) => match token::get_ident(ident).get() {
            "f32" => Some(ast::TyF32),
            "f64" => Some(ast::TyF64),
            _ => {
                cx.span_err(span, "invalid floating point type in hexfloat!");
                None
            }
        }
    };

    let s = match expr.node {
        // expression is a literal
        ast::ExprLit(ref lit) => match lit.node {
            // string literal
            ast::LitStr(ref s, _) => {
                s.clone()
            }
            _ => {
                cx.span_err(expr.span, "unsupported literal in hexfloat!");
                return base::DummyResult::expr(sp);
            }
        },
        _ => {
            cx.span_err(expr.span, "non-literal in hexfloat!");
            return base::DummyResult::expr(sp);
        }
    };

    {
        let err = hex_float_lit_err(s.get());
        match err {
            Some((err_pos, err_str)) => {
                let pos = expr.span.lo + syntax::codemap::Pos::from_uint(err_pos + 1);
                let span = syntax::codemap::mk_sp(pos,pos);
                cx.span_err(span,
                            format!("invalid hex float literal in hexfloat!: \
                                     {}",
                                    err_str).as_slice());
                return base::DummyResult::expr(sp);
            }
            _ => ()
        }
    }

    let lit = match ty {
        None => ast::LitFloatUnsuffixed(s),
        Some (ty) => ast::LitFloat(s, ty)
    };
    MacExpr::new(cx.expr_lit(sp, lit))
}

struct Ident {
    ident: ast::Ident,
    span: Span
}

fn parse_tts(cx: &ExtCtxt,
             tts: &[ast::TokenTree]) -> (P<ast::Expr>, Option<Ident>) {
    let p = &mut cx.new_parser_from_tts(tts);
    let ex = p.parse_expr();
    let id = if p.token == token::Eof {
        None
    } else {
        p.expect(&token::Comma);
        let lo = p.span.lo;
        let ident = p.parse_ident();
        let hi = p.last_span.hi;
        Some(Ident{ident: ident, span: mk_sp(lo, hi)})
    };
    if p.token != token::Eof {
        p.unexpected();
    }
    (ex, id)
}

// FIXME (10872): This is required to prevent an LLVM assert on Windows
#[test]
fn dummy_test() { }
<|endoftext|>


<commit_before>// For explanation of lint checks, run `rustc -W help`
// This is adapted from
// https://github.com/maidsafe/QA/blob/master/Documentation/Rust%20Lint%20Checks.md
#![forbid(bad_style, exceeding_bitshifts, mutable_transmutes, no_mangle_const_items,
unknown_crate_types, warnings)]
#![deny(deprecated, drop_with_repr_extern, improper_ctypes, //missing_docs,
non_shorthand_field_patterns, overflowing_literals, plugin_as_library,
private_no_mangle_fns, private_no_mangle_statics, stable_features, unconditional_recursion,
unknown_lints, unsafe_code, unused, unused_allocation, unused_attributes,
unused_comparisons, unused_features, unused_parens, while_true)]
#![warn(trivial_casts, unused_extern_crates, unused_import_braces,
unused_qualifications, unused_results, variant_size_differences)]
#![allow(box_pointers, fat_ptr_transmutes, missing_copy_implementations,
missing_debug_implementations)]

///! # multiaddr
///!
///! Implementation of [multiaddr](https://github.com/jbenet/multiaddr)
///! in Rust.
#[macro_use]
extern crate nom;

extern crate byteorder;

pub use self::protocols::*;
pub mod protocols;

use self::parser::*;
mod parser;

use std::net::{SocketAddr, SocketAddrV4, SocketAddrV6, Ipv4Addr, Ipv6Addr};
use std::io;

/// Representation of a Multiaddr.
#[derive(PartialEq, Eq, Clone, Debug)]
pub struct Multiaddr {
    bytes: Vec<u8>
}

impl ToString for Multiaddr {
    /// Convert a Multiaddr to a string
    ///
    /// # Examples
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udt").unwrap();
    /// assert_eq!(address.to_string(), "/ip4/127.0.0.1/udt");
    /// ```
    ///
    fn to_string(&self) -> String {
        address_from_bytes(&self.bytes[..])
    }
}

impl Multiaddr {
    /// Create a new multiaddr based on a string representation, like
    /// `/ip4/127.0.0.1/udp/1234`.
    ///
    /// # Examples
    ///
    /// Simple construction
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udp/1234").unwrap();
    /// assert_eq!(address.to_bytes(), [
    ///     0, 4, 127, 0, 0, 1,
    ///     0, 17, 4, 210
    /// ]);
    /// ```
    ///
    pub fn new(input: &str) -> io::Result<Multiaddr> {
        let bytes = try!(multiaddr_from_str(input));

        Ok(Multiaddr {
            bytes: bytes,
        })
    }

    /// Return a copy to disallow changing the bytes directly
    pub fn to_bytes(&self) -> Vec<u8> {
        self.bytes.to_owned()
    }

    /// Return a list of protocols
    ///
    /// # Examples
    ///
    /// A single protocol
    ///
    /// ```
    /// use multiaddr::{Multiaddr, Protocols};
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1").unwrap();
    /// assert_eq!(address.protocols(), vec![Protocols::IP4]);
    /// ```
    ///
    pub fn protocols(&self) -> Vec<Protocols> {
        protocols_from_bytes(&self.bytes[..])
    }

    /// Wrap a given Multiaddr and return the combination.
    ///
    /// # Examples
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1").unwrap();
    /// let nested = address.encapsulate("/udt").unwrap();
    /// assert_eq!(nested, Multiaddr::new("/ip4/127.0.0.1/udt").unwrap());
    /// ```
    ///
    pub fn encapsulate(&self, input: &str) -> io::Result<Multiaddr> {
        let mut bytes = self.bytes.clone();
        let new = try!(multiaddr_from_str(input));
        println!("bytes: {:?}, new: {:?}", bytes, new);
        bytes.extend(new);
        println!("res {:?}", bytes);
        Ok(Multiaddr {
            bytes: bytes
        })
    }

    /// Remove the outer most address from itself.
    ///
    /// # Examples
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udt/sctp/5678").unwrap();
    /// let unwrapped = address.decapsulate(Multiaddr::new("/udt").unwrap());
    /// assert_eq!(unwrapped, Multiaddr::new("/ip4/127.0.0.1").unwrap());
    /// ```
    ///
    /// Returns the original if the passed in address is not found
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udt/sctp/5678").unwrap();
    /// let unwrapped = address.decapsulate(Multiaddr::new("/ip4/127.0.1.1").unwrap());
    /// assert_eq!(unwrapped, address);
    /// ```
    ///
    pub fn decapsulate(&self, input: Multiaddr) -> Multiaddr {
        let bytes = self.bytes.clone();
        let input = input.to_bytes();
        let bytes_len = bytes.len();
        let input_length = input.len();

        let mut input_pos = 0;
        let mut matches = false;

        for (i, _) in bytes.iter().enumerate() {
            let next = i + input_length;

            if next > bytes_len {
                continue;
            }

            if &bytes[i..next] == &input[..] {
                matches = true;
                input_pos = i;
                break;
            }
        }

        if !matches {
            return Multiaddr {bytes: bytes}
        }

        let mut bytes = bytes;
        bytes.truncate(input_pos);

        Multiaddr {
            bytes: bytes
        }
    }
}

/// A trait for objects which can be converted to a
/// Multiaddr.
///
/// This trait is implemented by default for
///
/// * `SocketAddr`, `SocketAddrV4` and `SocketAddrV6`, assuming that the
///   the given port is a tcp port.
///
/// * `Ipv4Addr`, `Ipv6Addr`
///
/// * `String` and `&str`, requiring the default string format for a Multiaddr.
///
pub trait ToMultiaddr {
    /// Converts this object to a Multiaddr
    ///
    /// # Errors
    ///
    /// Any errors encountered during parsing will be returned
    /// as an `Err`.
    fn to_multiaddr(&self) -> io::Result<Multiaddr>;
}

impl ToMultiaddr for SocketAddr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        match *self {
            SocketAddr::V4(ref a) => (*a).to_multiaddr(),
            SocketAddr::V6(ref a) => (*a).to_multiaddr(),
        }
    }
}

impl ToMultiaddr for SocketAddrV4 {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&format!("/ip4/{}/tcp/{}", self.ip(), self.port()))
    }
}

impl ToMultiaddr for SocketAddrV6 {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        // TODO: Should how should we handle `flowinfo` and `scope_id`?
        Multiaddr::new(&format!("/ip6/{}/tcp/{}", self.ip(), self.port()))
    }
}

impl ToMultiaddr for Ipv4Addr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&format!("/ip4/{}", &self))
    }
}

impl ToMultiaddr for Ipv6Addr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&format!("/ip6/{}", &self))
    }
}

impl ToMultiaddr for String {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&self)
    }
}

impl<'a> ToMultiaddr for &'a str {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(self)
    }
}
<commit_msg>Integrate ToMultiaddr into encapsulate and decapsulate<commit_after>// For explanation of lint checks, run `rustc -W help`
// This is adapted from
// https://github.com/maidsafe/QA/blob/master/Documentation/Rust%20Lint%20Checks.md
#![forbid(bad_style, exceeding_bitshifts, mutable_transmutes, no_mangle_const_items,
unknown_crate_types, warnings)]
#![deny(deprecated, drop_with_repr_extern, improper_ctypes, //missing_docs,
non_shorthand_field_patterns, overflowing_literals, plugin_as_library,
private_no_mangle_fns, private_no_mangle_statics, stable_features, unconditional_recursion,
unknown_lints, unsafe_code, unused, unused_allocation, unused_attributes,
unused_comparisons, unused_features, unused_parens, while_true)]
#![warn(trivial_casts, unused_extern_crates, unused_import_braces,
unused_qualifications, unused_results, variant_size_differences)]
#![allow(box_pointers, fat_ptr_transmutes, missing_copy_implementations,
missing_debug_implementations)]

///! # multiaddr
///!
///! Implementation of [multiaddr](https://github.com/jbenet/multiaddr)
///! in Rust.
#[macro_use]
extern crate nom;

extern crate byteorder;

pub use self::protocols::*;
pub mod protocols;

use self::parser::*;
mod parser;

use std::net::{SocketAddr, SocketAddrV4, SocketAddrV6, Ipv4Addr, Ipv6Addr};
use std::io;

/// Representation of a Multiaddr.
#[derive(PartialEq, Eq, Clone, Debug)]
pub struct Multiaddr {
    bytes: Vec<u8>
}

impl ToString for Multiaddr {
    /// Convert a Multiaddr to a string
    ///
    /// # Examples
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udt").unwrap();
    /// assert_eq!(address.to_string(), "/ip4/127.0.0.1/udt");
    /// ```
    ///
    fn to_string(&self) -> String {
        address_from_bytes(&self.bytes[..])
    }
}

impl Multiaddr {
    /// Create a new multiaddr based on a string representation, like
    /// `/ip4/127.0.0.1/udp/1234`.
    ///
    /// # Examples
    ///
    /// Simple construction
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udp/1234").unwrap();
    /// assert_eq!(address.to_bytes(), [
    ///     0, 4, 127, 0, 0, 1,
    ///     0, 17, 4, 210
    /// ]);
    /// ```
    ///
    pub fn new(input: &str) -> io::Result<Multiaddr> {
        let bytes = try!(multiaddr_from_str(input));

        Ok(Multiaddr {
            bytes: bytes,
        })
    }

    /// Return a copy to disallow changing the bytes directly
    pub fn to_bytes(&self) -> Vec<u8> {
        self.bytes.to_owned()
    }

    /// Return a list of protocols
    ///
    /// # Examples
    ///
    /// A single protocol
    ///
    /// ```
    /// use multiaddr::{Multiaddr, Protocols};
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1").unwrap();
    /// assert_eq!(address.protocols(), vec![Protocols::IP4]);
    /// ```
    ///
    pub fn protocols(&self) -> Vec<Protocols> {
        protocols_from_bytes(&self.bytes[..])
    }

    /// Wrap a given Multiaddr and return the combination.
    ///
    /// # Examples
    ///
    /// ```
    /// use multiaddr::Multiaddr;
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1").unwrap();
    /// let nested = address.encapsulate("/udt").unwrap();
    /// assert_eq!(nested, Multiaddr::new("/ip4/127.0.0.1/udt").unwrap());
    /// ```
    ///
    pub fn encapsulate<T: ToMultiaddr>(&self, input: T) -> io::Result<Multiaddr> {
        let new = try!(input.to_multiaddr());
        let mut bytes = self.bytes.clone();

        bytes.extend(new.to_bytes());

        Ok(Multiaddr {
            bytes: bytes
        })
    }

    /// Remove the outer most address from itself.
    ///
    /// # Examples
    ///
    /// ```
    /// use multiaddr::{Multiaddr, ToMultiaddr};
    ///
    /// let address = Multiaddr::new("/ip4/127.0.0.1/udt/sctp/5678").unwrap();
    /// let unwrapped = address.decapsulate("/udt").unwrap();
    /// assert_eq!(unwrapped, Multiaddr::new("/ip4/127.0.0.1").unwrap());
    ///
    /// assert_eq!(
    ///     address.decapsulate("/udt").unwrap(),
    ///     "/ip4/127.0.0.1/".to_multiaddr().unwrap()
    /// );
    /// ```
    ///
    /// Returns the original if the passed in address is not found
    ///
    /// ```
    /// use multiaddr::{Multiaddr, ToMultiaddr};
    ///
    /// let address = "/ip4/127.0.0.1/udt/sctp/5678".to_multiaddr().unwrap();
    /// let unwrapped = address.decapsulate("/ip4/127.0.1.1").unwrap();
    /// assert_eq!(unwrapped, address);
    /// ```
    ///
    pub fn decapsulate<T: ToMultiaddr>(&self, input: T) -> io::Result<Multiaddr> {
        let input = try!(input.to_multiaddr());

        let bytes = self.bytes.clone();
        let input = input.to_bytes();
        let bytes_len = bytes.len();
        let input_length = input.len();

        let mut input_pos = 0;
        let mut matches = false;

        for (i, _) in bytes.iter().enumerate() {
            let next = i + input_length;

            if next > bytes_len {
                continue;
            }

            if &bytes[i..next] == &input[..] {
                matches = true;
                input_pos = i;
                break;
            }
        }

        if !matches {
            return Ok(Multiaddr {bytes: bytes})
        }

        let mut bytes = bytes;
        bytes.truncate(input_pos);

        Ok(Multiaddr {
            bytes: bytes
        })
    }
}

/// A trait for objects which can be converted to a
/// Multiaddr.
///
/// This trait is implemented by default for
///
/// * `SocketAddr`, `SocketAddrV4` and `SocketAddrV6`, assuming that the
///   the given port is a tcp port.
///
/// * `Ipv4Addr`, `Ipv6Addr`
///
/// * `String` and `&str`, requiring the default string format for a Multiaddr.
///
pub trait ToMultiaddr {
    /// Converts this object to a Multiaddr
    ///
    /// # Errors
    ///
    /// Any errors encountered during parsing will be returned
    /// as an `Err`.
    fn to_multiaddr(&self) -> io::Result<Multiaddr>;
}

impl ToMultiaddr for SocketAddr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        match *self {
            SocketAddr::V4(ref a) => (*a).to_multiaddr(),
            SocketAddr::V6(ref a) => (*a).to_multiaddr(),
        }
    }
}

impl ToMultiaddr for SocketAddrV4 {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&format!("/ip4/{}/tcp/{}", self.ip(), self.port()))
    }
}

impl ToMultiaddr for SocketAddrV6 {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        // TODO: Should how should we handle `flowinfo` and `scope_id`?
        Multiaddr::new(&format!("/ip6/{}/tcp/{}", self.ip(), self.port()))
    }
}

impl ToMultiaddr for Ipv4Addr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&format!("/ip4/{}", &self))
    }
}

impl ToMultiaddr for Ipv6Addr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&format!("/ip6/{}", &self))
    }
}

impl ToMultiaddr for String {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(&self)
    }
}

impl<'a> ToMultiaddr for &'a str {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Multiaddr::new(self)
    }
}

impl ToMultiaddr for Multiaddr {
    fn to_multiaddr(&self) -> io::Result<Multiaddr> {
        Ok(self.clone())
    }
}
<|endoftext|>


<commit_before>#![feature(collections, unboxed_closures, core)]
extern crate itertools;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::iter;
use std::ops;

use itertools::{GroupBy, Itertools};

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        self.id.hash(state);
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

fn analyze_doc(doc: &str)
-> iter::FlatMap<
    iter::Filter<
        GroupBy<bool, std::str::CharIndices, fn(&(usize, char)) -> bool>,
        fn(&(bool, Vec<(usize, char)>)) -> bool>,
    iter::Map<ops::Range<usize>, Ngrams>,
    fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, HashSet<IndexedDocument>>,
    docs: BTreeMap<String, Arc<Document>>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed  = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, highlights) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&IndexedDocument { doc: doc.clone(), highlighted: highlights });
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        let unique_terms: HashSet<_> = query.split_whitespace().collect();
        let map = unique_terms.into_iter()
            .flat_map(|word| self.index.get(&word.to_lowercase()))
            .flat_map(|doc| doc)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        let mut results: Vec<_> = map.into_iter()
            .map(SearchResult::new_)
            .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

#[derive(Clone, Debug, RustcEncodable)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
    highlighted: String,
    score: f32,
}

impl PartialEq for SearchResult {
    fn eq(&self, other: &SearchResult) -> bool {
        self.doc == other.doc && self.highlights == other.highlights
    }
}

impl Eq for SearchResult {}

impl Hash for SearchResult {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        self.doc.hash(state);
        self.highlights.hash(state);
    }
}

impl SearchResult {
    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted(&self) -> &String {
        &self.highlighted
    }

    pub fn score(&self) -> f32 {
        self.score
    }

    fn new_((doc, highlights): (Arc<Document>, Vec<(usize, usize)>)) -> SearchResult {
        SearchResult::new(doc, highlights)
    }

    fn new(doc: Arc<Document>, mut highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            score: highlights.iter().map(|&(begin, end)| end - begin).sum::<usize>() as f32 / doc.content.len() as f32,
            highlighted: SearchResult::highlighted_content(&doc.content, &mut highlights),
            doc: doc,
            highlights: highlights,
        }
    }

    fn highlighted_content(content: &str, highlights: &mut [(usize, usize)]) -> String {
        highlights.sort();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &mut (begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect::<Vec<_>>());
    assert_eq!("learn to program in rust today", expected[0].highlighted);
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect::<Vec<_>>());
    assert_eq!("learn to program in rust today", expected[0].highlighted);

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect::<Vec<_>>());
    assert_eq!(expected[0].highlighted, "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}

#[test]
fn test_update_doc() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc);
    let doc = Document::new("0", "different");
    index.index(doc);
    let search_results = index.search("å");
    assert!(search_results.is_empty());
    assert_eq!(index.docs.len(), 1);
}

#[test]
fn test_ranking() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "beat");
    index.index(doc.clone());
    let doc2 = Document::new("1", "beast");
    index.index(doc2.clone());
    let search_results = index.search("be");
    assert_eq!(index.docs.len(), 2);
    assert_eq!(&*search_results.into_iter().next().unwrap().doc, &doc);
}

#[test]
fn test_duplicate_term() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "beat");
    index.index(doc.clone());
    let search_results = index.search("be be");
    assert_eq!(search_results.len(), 1);
}

<commit_msg>Chop down a line<commit_after>#![feature(collections, unboxed_closures, core)]
extern crate itertools;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::iter;
use std::ops;

use itertools::{GroupBy, Itertools};

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        self.id.hash(state);
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

fn analyze_doc(doc: &str)
-> iter::FlatMap<
    iter::Filter<
        GroupBy<bool, std::str::CharIndices, fn(&(usize, char)) -> bool>,
        fn(&(bool, Vec<(usize, char)>)) -> bool>,
    iter::Map<ops::Range<usize>, Ngrams>,
    fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, HashSet<IndexedDocument>>,
    docs: BTreeMap<String, Arc<Document>>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed  = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, highlights) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&IndexedDocument { doc: doc.clone(), highlighted: highlights });
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        let unique_terms: HashSet<_> = query.split_whitespace().collect();
        let map = unique_terms.into_iter()
            .flat_map(|word| self.index.get(&word.to_lowercase()))
            .flat_map(|doc| doc)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        let mut results: Vec<_> = map.into_iter()
            .map(SearchResult::new_)
            .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

#[derive(Clone, Debug, RustcEncodable)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
    highlighted: String,
    score: f32,
}

impl PartialEq for SearchResult {
    fn eq(&self, other: &SearchResult) -> bool {
        self.doc == other.doc && self.highlights == other.highlights
    }
}

impl Eq for SearchResult {}

impl Hash for SearchResult {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        self.doc.hash(state);
        self.highlights.hash(state);
    }
}

impl SearchResult {
    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted(&self) -> &String {
        &self.highlighted
    }

    pub fn score(&self) -> f32 {
        self.score
    }

    fn new_((doc, highlights): (Arc<Document>, Vec<(usize, usize)>)) -> SearchResult {
        SearchResult::new(doc, highlights)
    }

    fn new(doc: Arc<Document>, mut highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            score: highlights.iter()
                .map(|&(begin, end)| end - begin)
                .sum::<usize>() as f32 / doc.content.len() as f32,
            highlighted: SearchResult::highlighted_content(&doc.content, &mut highlights),
            doc: doc,
            highlights: highlights,
        }
    }

    fn highlighted_content(content: &str, highlights: &mut [(usize, usize)]) -> String {
        highlights.sort();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &mut (begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect::<Vec<_>>());
    assert_eq!("learn to program in rust today", expected[0].highlighted);
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect::<Vec<_>>());
    assert_eq!("learn to program in rust today", expected[0].highlighted);

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect::<Vec<_>>());
    assert_eq!(expected[0].highlighted, "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}

#[test]
fn test_update_doc() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc);
    let doc = Document::new("0", "different");
    index.index(doc);
    let search_results = index.search("å");
    assert!(search_results.is_empty());
    assert_eq!(index.docs.len(), 1);
}

#[test]
fn test_ranking() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "beat");
    index.index(doc.clone());
    let doc2 = Document::new("1", "beast");
    index.index(doc2.clone());
    let search_results = index.search("be");
    assert_eq!(index.docs.len(), 2);
    assert_eq!(&*search_results.into_iter().next().unwrap().doc, &doc);
}

#[test]
fn test_duplicate_term() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "beat");
    index.index(doc.clone());
    let search_results = index.search("be be");
    assert_eq!(search_results.len(), 1);
}

<|endoftext|>


<commit_before>//! Functional Reactive Programming library for Rust
#![warn(missing_docs)]

pub use maybe_owned;
#[cfg(feature="either")]
pub use either;

#[macro_use] mod helpers;
pub mod types;
mod stream;
mod signal;
pub mod lift;

pub use crate::stream::{Sink, Stream};
pub use crate::signal::Signal;
<commit_msg>don't reexport external crates on lib.rs<commit_after>//! Functional Reactive Programming library for Rust
#![warn(missing_docs)]

#[macro_use] mod helpers;
pub mod types;
mod stream;
mod signal;
pub mod lift;

pub use crate::stream::{Sink, Stream};
pub use crate::signal::Signal;
<|endoftext|>


<commit_before>//! [Algorithm][1] to compute the [discrete Fourier transform][2] and its
//! inverse.
//!
//! [1]: https://en.wikipedia.org/wiki/Fast_Fourier_transform
//! [2]: https://en.wikipedia.org/wiki/Discrete_Fourier_transform

// The implementation is based on:
// http://www.librow.com/articles/article-10

extern crate complex;

use complex::c64;
use std::slice;

/// A means of obtaining a slice of mutable complex numbers.
pub trait AsMutComplex<'l> {
    fn as_mut_complex(self) -> &'l mut [c64];
}

impl<'l> AsMutComplex<'l> for &'l mut [c64] {
    #[inline(always)]
    fn as_mut_complex(self) -> &'l mut [c64] {
        self
    }
}

impl<'l> AsMutComplex<'l> for &'l mut [f64] {
    /// Treat the slice as a collection of pairs of real and imaginary parts and
    /// reinterpret it as a slice of complex numbers.
    ///
    /// ## Panics
    ///
    /// The function panics if the number of elements is not even.
    #[inline]
    fn as_mut_complex(self) -> &'l mut [c64] {
        unsafe {
            let length = self.len();
            assert!(length % 2 == 0, "the number of elements should be even");
            slice::from_raw_parts_mut(self.as_mut_ptr() as *mut _, length / 2)
        }
    }
}

impl<'l> AsMutComplex<'l> for &'l mut Vec<f64> {
    #[inline]
    fn as_mut_complex(self) -> &'l mut [c64] {
        <&mut [f64]>::as_mut_complex(&mut *self)
    }
}

/// Perform the Fourier transform.
///
/// The number of points should be a power of two.
pub fn forward<'l, T: AsMutComplex<'l>>(data: T) {
    let data = data.as_mut_complex();

    let n = data.len();
    if n < 1 || n & (n - 1) != 0 {
        panic!("expected the number of points to be a power of two");
    }

    rearrange(data, n);
    perform(data, n, false);
}

/// Perform the inverse Fourier transform.
///
/// The number of points should be a power of two.
pub fn inverse<'l, T: AsMutComplex<'l>>(data: T, scaling: bool) {
    let data = data.as_mut_complex();

    let n = data.len();
    if n < 1 || n & (n - 1) != 0 {
        panic!("expected the number of points to be a power of two");
    }

    rearrange(data, n);
    perform(data, n, true);
    if scaling {
        scale(data, n);
    }
}

#[inline(always)]
fn rearrange(data: &mut [c64], n: usize) {
    let mut target = 0;
    for position in 0..n {
        if target > position {
            data.swap(position, target);
        }
        let mut mask = n >> 1;
        while target & mask != 0 {
            target &= !mask;
            mask >>= 1;
        }
        target |= mask;
    }
}

#[inline(always)]
fn perform(data: &mut [c64], n: usize, inverse: bool) {
    use std::f64::consts::PI;

    let pi = if inverse { PI } else { -PI };

    let mut step = 1;
    while step < n {
        let jump = step << 1;
        let delta = pi / step as f64;
        let sine = (0.5 * delta).sin();
        let multiplier = c64(-2.0 * sine * sine, delta.sin());
        let mut factor = c64(1.0, 0.0);
        for group in 0..step {
            let mut pair = group;
            while pair < n {
                let match_pair = pair + step;
                let product = factor * data[match_pair];
                data[match_pair] = data[pair] - product;
                data[pair] = data[pair] + product;
                pair += jump;
            }
            factor = multiplier * factor + factor;
        }
        step <<= 1;
    }
}

#[inline(always)]
fn scale(data: &mut [c64], n: usize) {
    let factor = 1.0 / n as f64;
    for position in 0..n {
        data[position] = data[position] * factor;
    }
}
<commit_msg>Take a step to generalize over single and double<commit_after>//! [Algorithm][1] to compute the [discrete Fourier transform][2] and its
//! inverse.
//!
//! [1]: https://en.wikipedia.org/wiki/Fast_Fourier_transform
//! [2]: https://en.wikipedia.org/wiki/Discrete_Fourier_transform

// The implementation is based on:
// http://www.librow.com/articles/article-10

extern crate complex;

use complex::{Complex, c64};
use std::slice;

/// A means of obtaining a slice of mutable complex numbers.
pub trait AsMutComplex<'l> {
    type Complex: Complex + 'l;

    fn as_mut_complex(self) -> &'l mut [Self::Complex];
}

macro_rules! implement(
    ($complex:ty) => (
        impl<'l> AsMutComplex<'l> for &'l mut [$complex] {
            type Complex = $complex;

            #[inline(always)]
            fn as_mut_complex(self) -> &'l mut [Self::Complex] {
                self
            }
        }

        impl<'l> AsMutComplex<'l> for &'l mut Vec<$complex> {
            type Complex = $complex;

            #[inline]
            fn as_mut_complex(self) -> &'l mut [Self::Complex] {
                <&mut [$complex]>::as_mut_complex(&mut *self)
            }
        }
    );
);

implement!(c64);

macro_rules! implement(
    ($complex:ty, $real:ty) => (
        impl<'l> AsMutComplex<'l> for &'l mut [$real] {
            type Complex = $complex;

            /// Treat the slice as a collection of pairs of real and imaginary
            /// parts and reinterpret it as a slice of complex numbers.
            ///
            /// ## Panics
            ///
            /// The function panics if the number of elements is not even.
            #[inline]
            fn as_mut_complex(self) -> &'l mut [Self::Complex] {
                unsafe {
                    let length = self.len();
                    assert!(length % 2 == 0, "the number of elements should be even");
                    slice::from_raw_parts_mut(self.as_mut_ptr() as *mut _, length / 2)
                }
            }
        }

        impl<'l> AsMutComplex<'l> for &'l mut Vec<$real> {
            type Complex = $complex;

            #[inline]
            fn as_mut_complex(self) -> &'l mut [Self::Complex] {
                <&mut [$real]>::as_mut_complex(&mut *self)
            }
        }
    );
);

implement!(c64, f64);

/// Perform the Fourier transform.
///
/// The number of points should be a power of two.
pub fn forward<'l, T>(data: T) where T: AsMutComplex<'l, Complex=c64> {
    let data = data.as_mut_complex();

    let n = data.len();
    if n < 1 || n & (n - 1) != 0 {
        panic!("expected the number of points to be a power of two");
    }

    rearrange(data, n);
    perform(data, n, false);
}

/// Perform the inverse Fourier transform.
///
/// The number of points should be a power of two.
pub fn inverse<'l, T>(data: T, scaling: bool) where T: AsMutComplex<'l, Complex=c64> {
    let data = data.as_mut_complex();

    let n = data.len();
    if n < 1 || n & (n - 1) != 0 {
        panic!("expected the number of points to be a power of two");
    }

    rearrange(data, n);
    perform(data, n, true);
    if scaling {
        scale(data, n);
    }
}

#[inline(always)]
fn rearrange(data: &mut [c64], n: usize) {
    let mut target = 0;
    for position in 0..n {
        if target > position {
            data.swap(position, target);
        }
        let mut mask = n >> 1;
        while target & mask != 0 {
            target &= !mask;
            mask >>= 1;
        }
        target |= mask;
    }
}

#[inline(always)]
fn perform(data: &mut [c64], n: usize, inverse: bool) {
    use std::f64::consts::PI;

    let pi = if inverse { PI } else { -PI };

    let mut step = 1;
    while step < n {
        let jump = step << 1;
        let delta = pi / step as f64;
        let sine = (0.5 * delta).sin();
        let multiplier = c64(-2.0 * sine * sine, delta.sin());
        let mut factor = c64(1.0, 0.0);
        for group in 0..step {
            let mut pair = group;
            while pair < n {
                let match_pair = pair + step;
                let product = factor * data[match_pair];
                data[match_pair] = data[pair] - product;
                data[pair] = data[pair] + product;
                pair += jump;
            }
            factor = multiplier * factor + factor;
        }
        step <<= 1;
    }
}

#[inline(always)]
fn scale(data: &mut [c64], n: usize) {
    let factor = 1.0 / n as f64;
    for position in 0..n {
        data[position] = data[position] * factor;
    }
}
<|endoftext|>