<commit_before>use crate::*;
use bytes::BytesMut;
use network::connect::TcpConnect;
use std::{fmt, net::SocketAddr};
use tokio::{codec::FramedRead, prelude::*};

pub mod session;
use session::*;

pub type PeerId = SocketAddr;

#[derive(Clone, PartialEq)]
pub enum PeerState {
    Disconnected(Option<PeerInfo>), // Potentially last known PeerInfo
    Handshaking(Addr<Session>, SocketAddr),
    Ready(Addr<Session>, PeerInfo),
}

impl fmt::Debug for PeerState {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            PeerState::Disconnected(info) => f.debug_tuple("Disconnected").field(&info).finish(),
            PeerState::Handshaking(_, addr) => f.debug_tuple("Handshaking").field(&addr).finish(),
            PeerState::Ready(_, info) => f.debug_tuple("Ready").field(&info).finish(),
        }
    }
}

#[derive(Clone, Debug, PartialEq)]
pub struct PeerInfo {
    pub id: PeerId,
    pub outbound_addr: Option<SocketAddr>,
    pub peer_addr: SocketAddr,
}

impl PeerInfo {
    #[inline]
    pub fn is_outbound(&self) -> bool {
        self.outbound_addr.is_some()
    }
}

impl fmt::Display for PeerInfo {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_fmt(format_args!("{}", self.id))
    }
}

pub struct Peer<S: 'static, M: 'static + Metrics> {
    net_addr: Addr<Network<S, M>>,
    outbound_addr: Option<SocketAddr>,
    state: PeerState,
}

impl<S: 'static, M: 'static + Metrics> Peer<S, M> {
    pub fn init(conn: TcpConnect, addr: Addr<Network<S, M>>) {
        let stream = conn.0;
        let outbound_addr = conn.1;
        let (peer_addr, conn_type) = if let Some(addr) = outbound_addr {
            (addr, "outbound")
        } else {
            (stream.peer_addr().unwrap(), "inbound")
        };
        debug!("[{}] Accepted {} socket connection", peer_addr, conn_type);
        Peer::create(move |ctx| {
            let peer_tx = ctx.address().recipient();
            Session::create(move |ctx| {
                let (r, w) = stream.split();
                ctx.add_stream(FramedRead::new(r, Codec::new()));
                Session {
                    recipient: peer_tx,
                    writer: actix::io::FramedWrite::new(w, Codec::new(), ctx),
                    peer_addr,
                }
            });

            Peer {
                net_addr: addr,
                outbound_addr,
                state: PeerState::Disconnected(None),
            }
        });
    }

    #[inline]
    pub fn is_outbound(&self) -> bool {
        self.outbound_addr.is_some()
    }
}

impl<S: 'static, M: 'static + Metrics> Actor for Peer<S, M> {
    type Context = Context<Self>;
}

impl<S: 'static, M: 'static + Metrics> Handler<Payload> for Peer<S, M> {
    type Result = ();

    fn handle(&mut self, msg: Payload, _: &mut Self::Context) {
        match &self.state {
            PeerState::Disconnected(info) => {
                let peer = match info {
                    Some(info) => format!("{}", info),
                    None => self
                        .outbound_addr
                        .map_or("Unknown".to_owned(), |v| format!("{}", v)),
                };
                warn!("[{}] Attempted to send message to disconnected peer", peer);
            }
            PeerState::Handshaking(addr, _) | PeerState::Ready(addr, _) => {
                addr.do_send(msg);
            }
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<cmd::Disconnect> for Peer<S, M> {
    type Result = ();

    fn handle(&mut self, _: cmd::Disconnect, ctx: &mut Self::Context) {
        match &self.state {
            PeerState::Disconnected(_) => (),
            PeerState::Handshaking(addr, _) | PeerState::Ready(addr, _) => {
                addr.do_send(session::cmd::Disconnect);
            }
        }
        ctx.stop();
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<SessionMsg> for Peer<S, M> {
    type Result = ();

    fn handle(&mut self, msg: SessionMsg, ctx: &mut Self::Context) {
        match msg {
            SessionMsg::Connected(ses, addr) => match &self.state {
                PeerState::Disconnected(_) => {
                    self.state = PeerState::Handshaking(ses.clone(), addr);
                    if self.is_outbound() {
                        debug!("[{}] Sending outbound handshake", addr);
                        ses.do_send(Payload {
                            id: BytesMut::from(vec![0]),
                            msg: BytesMut::new(),
                        });
                    }
                }
                _ => panic!("Peer state is invalid: {:?}", self.state),
            },
            SessionMsg::Disconnected => match &self.state {
                PeerState::Disconnected(_) => panic!("Peer state is invalid: {:?}", self.state),
                PeerState::Handshaking(_, _) => {
                    self.state = PeerState::Disconnected(None);
                }
                PeerState::Ready(_, info) => {
                    self.net_addr
                        .try_send(msg::Disconnected(info.clone()))
                        .unwrap();
                    self.state = PeerState::Disconnected(Some(info.clone()));
                }
            },
            SessionMsg::Message(msg) => match &self.state {
                PeerState::Disconnected(_) => {
                    panic!("Received message from disconnected peer: {:?}", self.state);
                }
                PeerState::Ready(_, info) => {
                    self.net_addr.try_send(msg::Message(info.id, msg)).unwrap();
                }
                PeerState::Handshaking(ses, peer_addr) => {
                    if !self.is_outbound() {
                        debug!("[{}] Sending inbound handshake", peer_addr);
                        ses.do_send(Payload {
                            id: BytesMut::from(vec![0]),
                            msg: BytesMut::new(),
                        });
                    }
                    let info = PeerInfo {
                        id: *peer_addr,
                        outbound_addr: self.outbound_addr,
                        peer_addr: *peer_addr,
                    };
                    self.state = PeerState::Ready(ses.clone(), info.clone());
                    self.net_addr
                        .try_send(msg::Connected(info, ctx.address()))
                        .unwrap();
                }
            },
        }
    }
}

pub mod msg {
    use super::*;

    #[derive(Message, Debug)]
    pub struct Handshake(pub SocketAddr, pub Payload);

    #[derive(Message)]
    pub struct Connected<S: 'static, M: 'static + Metrics>(pub PeerInfo, pub Addr<Peer<S, M>>);

    #[derive(Message)]
    pub struct Disconnected(pub PeerInfo);

    #[derive(Message)]
    pub struct Message(pub PeerId, pub Payload);
}

pub mod cmd {
    use super::*;

    #[derive(Message)]
    pub struct Disconnect;
}
<commit_msg>Reorder peer state matching<commit_after>use crate::*;
use bytes::BytesMut;
use network::connect::TcpConnect;
use std::{fmt, net::SocketAddr};
use tokio::{codec::FramedRead, prelude::*};

pub mod session;
use session::*;

pub type PeerId = SocketAddr;

#[derive(Clone, PartialEq)]
pub enum PeerState {
    Disconnected(Option<PeerInfo>), // Potentially last known PeerInfo
    Handshaking(Addr<Session>, SocketAddr),
    Ready(Addr<Session>, PeerInfo),
}

impl fmt::Debug for PeerState {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            PeerState::Disconnected(info) => f.debug_tuple("Disconnected").field(&info).finish(),
            PeerState::Handshaking(_, addr) => f.debug_tuple("Handshaking").field(&addr).finish(),
            PeerState::Ready(_, info) => f.debug_tuple("Ready").field(&info).finish(),
        }
    }
}

#[derive(Clone, Debug, PartialEq)]
pub struct PeerInfo {
    pub id: PeerId,
    pub outbound_addr: Option<SocketAddr>,
    pub peer_addr: SocketAddr,
}

impl PeerInfo {
    #[inline]
    pub fn is_outbound(&self) -> bool {
        self.outbound_addr.is_some()
    }
}

impl fmt::Display for PeerInfo {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_fmt(format_args!("{}", self.id))
    }
}

pub struct Peer<S: 'static, M: 'static + Metrics> {
    net_addr: Addr<Network<S, M>>,
    outbound_addr: Option<SocketAddr>,
    state: PeerState,
}

impl<S: 'static, M: 'static + Metrics> Peer<S, M> {
    pub fn init(conn: TcpConnect, addr: Addr<Network<S, M>>) {
        let stream = conn.0;
        let outbound_addr = conn.1;
        let (peer_addr, conn_type) = if let Some(addr) = outbound_addr {
            (addr, "outbound")
        } else {
            (stream.peer_addr().unwrap(), "inbound")
        };
        debug!("[{}] Accepted {} socket connection", peer_addr, conn_type);
        Peer::create(move |ctx| {
            let peer_tx = ctx.address().recipient();
            Session::create(move |ctx| {
                let (r, w) = stream.split();
                ctx.add_stream(FramedRead::new(r, Codec::new()));
                Session {
                    recipient: peer_tx,
                    writer: actix::io::FramedWrite::new(w, Codec::new(), ctx),
                    peer_addr,
                }
            });

            Peer {
                net_addr: addr,
                outbound_addr,
                state: PeerState::Disconnected(None),
            }
        });
    }

    #[inline]
    pub fn is_outbound(&self) -> bool {
        self.outbound_addr.is_some()
    }
}

impl<S: 'static, M: 'static + Metrics> Actor for Peer<S, M> {
    type Context = Context<Self>;
}

impl<S: 'static, M: 'static + Metrics> Handler<Payload> for Peer<S, M> {
    type Result = ();

    fn handle(&mut self, msg: Payload, _: &mut Self::Context) {
        match &self.state {
            PeerState::Disconnected(info) => {
                let peer = match info {
                    Some(info) => format!("{}", info),
                    None => self
                        .outbound_addr
                        .map_or("Unknown".to_owned(), |v| format!("{}", v)),
                };
                warn!("[{}] Attempted to send message to disconnected peer", peer);
            }
            PeerState::Handshaking(addr, _) | PeerState::Ready(addr, _) => {
                addr.do_send(msg);
            }
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<cmd::Disconnect> for Peer<S, M> {
    type Result = ();

    fn handle(&mut self, _: cmd::Disconnect, ctx: &mut Self::Context) {
        match &self.state {
            PeerState::Disconnected(_) => (),
            PeerState::Handshaking(addr, _) | PeerState::Ready(addr, _) => {
                addr.do_send(session::cmd::Disconnect);
            }
        }
        ctx.stop();
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<SessionMsg> for Peer<S, M> {
    type Result = ();

    fn handle(&mut self, msg: SessionMsg, ctx: &mut Self::Context) {
        match msg {
            SessionMsg::Connected(ses, addr) => match &self.state {
                PeerState::Disconnected(_) => {
                    self.state = PeerState::Handshaking(ses.clone(), addr);
                    if self.is_outbound() {
                        debug!("[{}] Sending outbound handshake", addr);
                        ses.do_send(Payload {
                            id: BytesMut::from(vec![0]),
                            msg: BytesMut::new(),
                        });
                    }
                }
                _ => panic!("Peer state is invalid: {:?}", self.state),
            },
            SessionMsg::Disconnected => match &self.state {
                PeerState::Disconnected(_) => panic!("Peer state is invalid: {:?}", self.state),
                PeerState::Handshaking(_, _) => {
                    self.state = PeerState::Disconnected(None);
                }
                PeerState::Ready(_, info) => {
                    self.net_addr
                        .try_send(msg::Disconnected(info.clone()))
                        .unwrap();
                    self.state = PeerState::Disconnected(Some(info.clone()));
                }
            },
            SessionMsg::Message(msg) => match &self.state {
                PeerState::Disconnected(_) => {
                    panic!("Received message from disconnected peer: {:?}", self.state);
                }
                PeerState::Handshaking(ses, peer_addr) => {
                    if !self.is_outbound() {
                        debug!("[{}] Sending inbound handshake", peer_addr);
                        ses.do_send(Payload {
                            id: BytesMut::from(vec![0]),
                            msg: BytesMut::new(),
                        });
                    }
                    let info = PeerInfo {
                        id: *peer_addr,
                        outbound_addr: self.outbound_addr,
                        peer_addr: *peer_addr,
                    };
                    self.state = PeerState::Ready(ses.clone(), info.clone());
                    self.net_addr
                        .try_send(msg::Connected(info, ctx.address()))
                        .unwrap();
                }
                PeerState::Ready(_, info) => {
                    self.net_addr.try_send(msg::Message(info.id, msg)).unwrap();
                }
            },
        }
    }
}

pub mod msg {
    use super::*;

    #[derive(Message, Debug)]
    pub struct Handshake(pub SocketAddr, pub Payload);

    #[derive(Message)]
    pub struct Connected<S: 'static, M: 'static + Metrics>(pub PeerInfo, pub Addr<Peer<S, M>>);

    #[derive(Message)]
    pub struct Disconnected(pub PeerInfo);

    #[derive(Message)]
    pub struct Message(pub PeerId, pub Payload);
}

pub mod cmd {
    use super::*;

    #[derive(Message)]
    pub struct Disconnect;
}
<|endoftext|>


<commit_before>//! Log all requests in a format similar to Heroku's router, but with additional
//! information that we care about like User-Agent

use super::prelude::*;
use crate::middleware::current_user::TrustedUserId;
use crate::util::request_header;
use conduit::{header, Host, RequestExt, Scheme, StatusCode};
use sentry::Level;
use std::fmt::{self, Display, Formatter};
use std::time::Instant;

const SLOW_REQUEST_THRESHOLD_MS: u64 = 1000;

const FILTERED_HEADERS: [&str; 3] = ["Authorization", "Cookie", "X-Real-Ip"];

#[derive(Default)]
pub(super) struct LogRequests();

struct RequestStart(Instant);
struct OriginalPath(String);

impl Middleware for LogRequests {
    fn before(&self, req: &mut dyn RequestExt) -> BeforeResult {
        req.mut_extensions().insert(RequestStart(Instant::now()));
        let path = OriginalPath(req.path().to_string());
        req.mut_extensions().insert(path);
        Ok(())
    }

    fn after(&self, req: &mut dyn RequestExt, res: AfterResult) -> AfterResult {
        // Unwrap shouldn't panic as no other code has access to the private struct to remove it
        let request_start = req.extensions().find::<RequestStart>().unwrap().0;

        let response_time = request_start.elapsed();
        let response_time =
            response_time.as_secs() * 1000 + u64::from(response_time.subsec_nanos()) / 1_000_000;

        println!(
            "{}",
            RequestLine {
                req,
                res: &res,
                response_time,
            }
        );

        report_to_sentry(req, &res, response_time);

        res
    }
}

struct CustomMetadata {
    entries: Vec<(&'static str, String)>,
}

pub fn add_custom_metadata<V: Display>(req: &mut dyn RequestExt, key: &'static str, value: V) {
    if let Some(metadata) = req.mut_extensions().find_mut::<CustomMetadata>() {
        metadata.entries.push((key, value.to_string()));
    } else {
        let mut metadata = CustomMetadata {
            entries: Vec::new(),
        };
        metadata.entries.push((key, value.to_string()));
        req.mut_extensions().insert(metadata);
    }
}

fn report_to_sentry(req: &dyn RequestExt, res: &AfterResult, response_time: u64) {
    let (message, level) = match res {
        Err(e) => (e.to_string(), Level::Error),
        Ok(_) => {
            if response_time <= SLOW_REQUEST_THRESHOLD_MS {
                return;
            }

            let message = format!("Slow Request: {} {}", req.method(), req.path());
            (message, Level::Info)
        }
    };

    let config = |scope: &mut sentry::Scope| {
        let method = Some(req.method().as_str().to_owned());

        let scheme = match req.scheme() {
            Scheme::Http => "http",
            Scheme::Https => "https",
        };

        let host = match req.host() {
            Host::Name(name) => name.to_owned(),
            Host::Socket(addr) => addr.to_string(),
        };

        let path = &req.extensions().find::<OriginalPath>().unwrap().0;

        let url = format!("{}://{}{}", scheme, host, path).parse().ok();

        {
            let id = req
                .extensions()
                .find::<TrustedUserId>()
                .map(|x| x.0.to_string());

            let user = sentry::User {
                id,
                ..Default::default()
            };

            scope.set_user(Some(user));
        }

        {
            let headers = req
                .headers()
                .iter()
                .filter(|(k, _v)| !FILTERED_HEADERS.iter().any(|name| k == name))
                .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default().to_string()))
                .collect();

            let sentry_req = sentry::protocol::Request {
                method,
                url,
                headers,
                ..Default::default()
            };

            scope.add_event_processor(Box::new(move |mut event| {
                if event.request.is_none() {
                    event.request = Some(sentry_req.clone());
                }
                Some(event)
            }));
        }

        if let Some(request_id) = req
            .headers()
            .get("x-request-id")
            .and_then(|value| value.to_str().ok())
        {
            scope.set_tag("request.id", request_id);
        }

        {
            let status = res
                .as_ref()
                .map(|resp| resp.status())
                .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);

            scope.set_tag("response.status", status.as_str());
        }

        scope.set_extra("Response time [ms]", response_time.into());

        if let Some(metadata) = req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                scope.set_extra(key, value.to_string().into());
            }
        }
    };

    sentry::with_scope(config, || sentry::capture_message(&message, level));
}

#[cfg(test)]
pub(crate) fn get_log_message(req: &dyn RequestExt, key: &'static str) -> String {
    // Unwrap shouldn't panic as no other code has access to the private struct to remove it
    for (k, v) in &req.extensions().find::<CustomMetadata>().unwrap().entries {
        if key == *k {
            return v.clone();
        }
    }
    panic!("expected log message for {} not found", key);
}

struct RequestLine<'r> {
    req: &'r dyn RequestExt,
    res: &'r AfterResult,
    response_time: u64,
}

impl Display for RequestLine<'_> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        let mut line = LogLine::new(f);

        let (at, status) = match self.res {
            Ok(resp) => ("info", resp.status()),
            Err(_) => ("error", StatusCode::INTERNAL_SERVER_ERROR),
        };

        line.add_field("at", at)?;
        line.add_field("method", self.req.method())?;
        line.add_quoted_field("path", FullPath(self.req))?;
        line.add_field("request_id", request_header(self.req, "x-request-id"))?;
        line.add_quoted_field("fwd", request_header(self.req, "x-real-ip"))?;
        line.add_field("service", TimeMs(self.response_time))?;
        line.add_field("status", status.as_str())?;
        line.add_quoted_field("user_agent", request_header(self.req, header::USER_AGENT))?;

        if let Some(metadata) = self.req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                line.add_quoted_field(key, value)?;
            }
        }

        if let Err(err) = self.res {
            line.add_quoted_field("error", err)?;
        }

        if self.response_time > SLOW_REQUEST_THRESHOLD_MS {
            line.add_marker("SLOW REQUEST")?;
        }

        Ok(())
    }
}

struct FullPath<'a>(&'a dyn RequestExt);

impl<'a> Display for FullPath<'a> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        // Unwrap shouldn't panic as no other code has access to the private struct to remove it
        write!(
            f,
            "{}",
            self.0.extensions().find::<OriginalPath>().unwrap().0
        )?;
        if let Some(q_string) = self.0.query_string() {
            write!(f, "?{}", q_string)?;
        }
        Ok(())
    }
}

struct TimeMs(u64);

impl Display for TimeMs {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)?;
        f.write_str("ms")?;
        Ok(())
    }
}

struct LogLine<'f, 'g> {
    f: &'f mut Formatter<'g>,
    first: bool,
}

impl<'f, 'g> LogLine<'f, 'g> {
    fn new(f: &'f mut Formatter<'g>) -> Self {
        Self { f, first: true }
    }

    fn add_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=")?;
        value.fmt(self.f)?;

        Ok(())
    }

    fn add_quoted_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=\"")?;
        value.fmt(self.f)?;
        self.f.write_str("\"")?;

        Ok(())
    }

    fn add_marker<M: Display>(&mut self, marker: M) -> fmt::Result {
        self.start_item()?;

        marker.fmt(self.f)?;

        Ok(())
    }

    fn start_item(&mut self) -> fmt::Result {
        if !self.first {
            self.f.write_str(" ")?;
        }
        self.first = false;
        Ok(())
    }
}
<commit_msg>sentry: filter the X-Forwarded-For header since it contains PII<commit_after>//! Log all requests in a format similar to Heroku's router, but with additional
//! information that we care about like User-Agent

use super::prelude::*;
use crate::middleware::current_user::TrustedUserId;
use crate::util::request_header;
use conduit::{header, Host, RequestExt, Scheme, StatusCode};
use sentry::Level;
use std::fmt::{self, Display, Formatter};
use std::time::Instant;

const SLOW_REQUEST_THRESHOLD_MS: u64 = 1000;

const FILTERED_HEADERS: &[&str] = &["Authorization", "Cookie", "X-Real-Ip", "X-Forwarded-For"];

#[derive(Default)]
pub(super) struct LogRequests();

struct RequestStart(Instant);
struct OriginalPath(String);

impl Middleware for LogRequests {
    fn before(&self, req: &mut dyn RequestExt) -> BeforeResult {
        req.mut_extensions().insert(RequestStart(Instant::now()));
        let path = OriginalPath(req.path().to_string());
        req.mut_extensions().insert(path);
        Ok(())
    }

    fn after(&self, req: &mut dyn RequestExt, res: AfterResult) -> AfterResult {
        // Unwrap shouldn't panic as no other code has access to the private struct to remove it
        let request_start = req.extensions().find::<RequestStart>().unwrap().0;

        let response_time = request_start.elapsed();
        let response_time =
            response_time.as_secs() * 1000 + u64::from(response_time.subsec_nanos()) / 1_000_000;

        println!(
            "{}",
            RequestLine {
                req,
                res: &res,
                response_time,
            }
        );

        report_to_sentry(req, &res, response_time);

        res
    }
}

struct CustomMetadata {
    entries: Vec<(&'static str, String)>,
}

pub fn add_custom_metadata<V: Display>(req: &mut dyn RequestExt, key: &'static str, value: V) {
    if let Some(metadata) = req.mut_extensions().find_mut::<CustomMetadata>() {
        metadata.entries.push((key, value.to_string()));
    } else {
        let mut metadata = CustomMetadata {
            entries: Vec::new(),
        };
        metadata.entries.push((key, value.to_string()));
        req.mut_extensions().insert(metadata);
    }
}

fn report_to_sentry(req: &dyn RequestExt, res: &AfterResult, response_time: u64) {
    let (message, level) = match res {
        Err(e) => (e.to_string(), Level::Error),
        Ok(_) => {
            if response_time <= SLOW_REQUEST_THRESHOLD_MS {
                return;
            }

            let message = format!("Slow Request: {} {}", req.method(), req.path());
            (message, Level::Info)
        }
    };

    let config = |scope: &mut sentry::Scope| {
        let method = Some(req.method().as_str().to_owned());

        let scheme = match req.scheme() {
            Scheme::Http => "http",
            Scheme::Https => "https",
        };

        let host = match req.host() {
            Host::Name(name) => name.to_owned(),
            Host::Socket(addr) => addr.to_string(),
        };

        let path = &req.extensions().find::<OriginalPath>().unwrap().0;

        let url = format!("{}://{}{}", scheme, host, path).parse().ok();

        {
            let id = req
                .extensions()
                .find::<TrustedUserId>()
                .map(|x| x.0.to_string());

            let user = sentry::User {
                id,
                ..Default::default()
            };

            scope.set_user(Some(user));
        }

        {
            let headers = req
                .headers()
                .iter()
                .filter(|(k, _v)| !FILTERED_HEADERS.iter().any(|name| k == name))
                .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default().to_string()))
                .collect();

            let sentry_req = sentry::protocol::Request {
                method,
                url,
                headers,
                ..Default::default()
            };

            scope.add_event_processor(Box::new(move |mut event| {
                if event.request.is_none() {
                    event.request = Some(sentry_req.clone());
                }
                Some(event)
            }));
        }

        if let Some(request_id) = req
            .headers()
            .get("x-request-id")
            .and_then(|value| value.to_str().ok())
        {
            scope.set_tag("request.id", request_id);
        }

        {
            let status = res
                .as_ref()
                .map(|resp| resp.status())
                .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);

            scope.set_tag("response.status", status.as_str());
        }

        scope.set_extra("Response time [ms]", response_time.into());

        if let Some(metadata) = req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                scope.set_extra(key, value.to_string().into());
            }
        }
    };

    sentry::with_scope(config, || sentry::capture_message(&message, level));
}

#[cfg(test)]
pub(crate) fn get_log_message(req: &dyn RequestExt, key: &'static str) -> String {
    // Unwrap shouldn't panic as no other code has access to the private struct to remove it
    for (k, v) in &req.extensions().find::<CustomMetadata>().unwrap().entries {
        if key == *k {
            return v.clone();
        }
    }
    panic!("expected log message for {} not found", key);
}

struct RequestLine<'r> {
    req: &'r dyn RequestExt,
    res: &'r AfterResult,
    response_time: u64,
}

impl Display for RequestLine<'_> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        let mut line = LogLine::new(f);

        let (at, status) = match self.res {
            Ok(resp) => ("info", resp.status()),
            Err(_) => ("error", StatusCode::INTERNAL_SERVER_ERROR),
        };

        line.add_field("at", at)?;
        line.add_field("method", self.req.method())?;
        line.add_quoted_field("path", FullPath(self.req))?;
        line.add_field("request_id", request_header(self.req, "x-request-id"))?;
        line.add_quoted_field("fwd", request_header(self.req, "x-real-ip"))?;
        line.add_field("service", TimeMs(self.response_time))?;
        line.add_field("status", status.as_str())?;
        line.add_quoted_field("user_agent", request_header(self.req, header::USER_AGENT))?;

        if let Some(metadata) = self.req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                line.add_quoted_field(key, value)?;
            }
        }

        if let Err(err) = self.res {
            line.add_quoted_field("error", err)?;
        }

        if self.response_time > SLOW_REQUEST_THRESHOLD_MS {
            line.add_marker("SLOW REQUEST")?;
        }

        Ok(())
    }
}

struct FullPath<'a>(&'a dyn RequestExt);

impl<'a> Display for FullPath<'a> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        // Unwrap shouldn't panic as no other code has access to the private struct to remove it
        write!(
            f,
            "{}",
            self.0.extensions().find::<OriginalPath>().unwrap().0
        )?;
        if let Some(q_string) = self.0.query_string() {
            write!(f, "?{}", q_string)?;
        }
        Ok(())
    }
}

struct TimeMs(u64);

impl Display for TimeMs {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)?;
        f.write_str("ms")?;
        Ok(())
    }
}

struct LogLine<'f, 'g> {
    f: &'f mut Formatter<'g>,
    first: bool,
}

impl<'f, 'g> LogLine<'f, 'g> {
    fn new(f: &'f mut Formatter<'g>) -> Self {
        Self { f, first: true }
    }

    fn add_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=")?;
        value.fmt(self.f)?;

        Ok(())
    }

    fn add_quoted_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=\"")?;
        value.fmt(self.f)?;
        self.f.write_str("\"")?;

        Ok(())
    }

    fn add_marker<M: Display>(&mut self, marker: M) -> fmt::Result {
        self.start_item()?;

        marker.fmt(self.f)?;

        Ok(())
    }

    fn start_item(&mut self) -> fmt::Result {
        if !self.first {
            self.f.write_str(" ")?;
        }
        self.first = false;
        Ok(())
    }
}
<|endoftext|>


<commit_before>use super::*;

/// Represents lifetimes and type parameters attached to a declaration
/// of a function, enum, trait, etc.
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct Generics {
    pub lifetimes: Vec<LifetimeDef>,
    pub ty_params: Vec<TyParam>,
    pub where_clause: WhereClause,
}

#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Lifetime {
    pub ident: Ident,
}

/// A lifetime definition, e.g. `'a: 'b+'c+'d`
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct LifetimeDef {
    pub lifetime: Lifetime,
    pub bounds: Vec<Lifetime>,
}

#[derive(Debug, Clone, Eq, PartialEq)]
pub struct TyParam {
    pub ident: Ident,
    pub bounds: Vec<TyParamBound>,
    pub default: Option<Ty>,
}

/// The AST represents all type param bounds as types.
/// `typeck::collect::compute_bounds` matches these against
/// the "special" built-in traits (see `middle::lang_items`) and
/// detects Copy, Send and Sync.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum TyParamBound {
    Trait(PolyTraitRef, TraitBoundModifier),
    Region(Lifetime),
}

/// A modifier on a bound, currently this is only used for `?Sized`, where the
/// modifier is `Maybe`. Negative bounds should also be handled here.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum TraitBoundModifier {
    None,
    Maybe,
}

/// A `where` clause in a definition
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct WhereClause {
    pub predicates: Vec<WherePredicate>,
}

/// A single predicate in a `where` clause
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum WherePredicate {
    /// A type binding, e.g. `for<'c> Foo: Send+Clone+'c`
    BoundPredicate(WhereBoundPredicate),
    /// A lifetime predicate, e.g. `'a: 'b+'c`
    RegionPredicate(WhereRegionPredicate),
}

/// A type bound.
///
/// E.g. `for<'c> Foo: Send+Clone+'c`
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WhereBoundPredicate {
    /// Any lifetimes from a `for` binding
    pub bound_lifetimes: Vec<LifetimeDef>,
    /// The type being bounded
    pub bounded_ty: Ty,
    /// Trait and lifetime bounds (`Clone+Send+'static`)
    pub bounds: Vec<TyParamBound>,
}

/// A lifetime predicate.
///
/// E.g. `'a: 'b+'c`
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WhereRegionPredicate {
    pub lifetime: Lifetime,
    pub bounds: Vec<Lifetime>,
}

#[cfg(feature = "parsing")]
pub mod parsing {
    use super::*;
    use ident::parsing::ident;
    use ty::parsing::{ty, poly_trait_ref};

    named!(pub generics -> Generics, map!(
        alt!(
            do_parse!(
                punct!("<") >>
                lifetimes: separated_list!(punct!(","), lifetime_def) >>
                ty_params: opt_vec!(preceded!(
                    cond!(!lifetimes.is_empty(), punct!(",")),
                    separated_nonempty_list!(punct!(","), ty_param)
                )) >>
                punct!(">") >>
                (lifetimes, ty_params)
            )
            |
            epsilon!() => { |_| (Vec::new(), Vec::new()) }
        ),
        |(lifetimes, ty_params)| Generics {
            lifetimes: lifetimes,
            ty_params: ty_params,
            where_clause: Default::default(),
        }
    ));

    named!(pub lifetime -> Lifetime, preceded!(
        punct!("'"),
        map!(ident, |id| Lifetime {
            ident: format!("'{}", id).into(),
        })
    ));

    named!(pub lifetime_def -> LifetimeDef, do_parse!(
        life: lifetime >>
        bounds: opt_vec!(preceded!(
            punct!(":"),
            separated_nonempty_list!(punct!("+"), lifetime)
        )) >>
        (LifetimeDef {
            lifetime: life,
            bounds: bounds,
        })
    ));

    named!(pub bound_lifetimes -> Vec<LifetimeDef>, opt_vec!(do_parse!(
        keyword!("for") >>
        punct!("<") >>
        lifetimes: separated_list!(punct!(","), lifetime_def) >>
        punct!(">") >>
        (lifetimes)
    )));

    named!(ty_param -> TyParam, do_parse!(
        id: ident >>
        bounds: opt_vec!(preceded!(
            punct!(":"),
            separated_nonempty_list!(punct!("+"), ty_param_bound)
        )) >>
        default: option!(preceded!(
            punct!("="),
            ty
        )) >>
        (TyParam {
            ident: id,
            bounds: bounds,
            default: default,
        })
    ));

    named!(pub ty_param_bound -> TyParamBound, alt!(
        preceded!(punct!("?"), poly_trait_ref) => {
            |poly| TyParamBound::Trait(poly, TraitBoundModifier::Maybe)
        }
        |
        lifetime => { TyParamBound::Region }
        |
        poly_trait_ref => {
            |poly| TyParamBound::Trait(poly, TraitBoundModifier::None)
        }
    ));

    named!(pub where_clause -> WhereClause, alt!(
        do_parse!(
            keyword!("where") >>
            predicates: separated_nonempty_list!(punct!(","), where_predicate) >>
            option!(punct!(",")) >>
            (WhereClause { predicates: predicates })
        )
        |
        epsilon!() => { |_| Default::default() }
    ));

    named!(where_predicate -> WherePredicate, alt!(
        do_parse!(
            ident: lifetime >>
            punct!(":") >>
            bounds: separated_nonempty_list!(punct!("+"), lifetime) >>
            (WherePredicate::RegionPredicate(WhereRegionPredicate {
                lifetime: ident,
                bounds: bounds,
            }))
        )
        |
        do_parse!(
            bound_lifetimes: bound_lifetimes >>
            bounded_ty: ty >>
            punct!(":") >>
            bounds: separated_nonempty_list!(punct!("+"), ty_param_bound) >>
            (WherePredicate::BoundPredicate(WhereBoundPredicate {
                bound_lifetimes: bound_lifetimes,
                bounded_ty: bounded_ty,
                bounds: bounds,
            }))
        )
    ));
}

#[cfg(feature = "printing")]
mod printing {
    use super::*;
    use quote::{Tokens, ToTokens};

    impl ToTokens for Generics {
        fn to_tokens(&self, tokens: &mut Tokens) {
            let has_lifetimes = !self.lifetimes.is_empty();
            let has_ty_params = !self.ty_params.is_empty();
            if has_lifetimes || has_ty_params {
                tokens.append("<");
                tokens.append_separated(&self.lifetimes, ",");
                if has_lifetimes && has_ty_params {
                    tokens.append(",");
                }
                tokens.append_separated(&self.ty_params, ",");
                tokens.append(">");
            }
        }
    }

    impl ToTokens for Lifetime {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.ident.to_tokens(tokens);
        }
    }

    impl ToTokens for LifetimeDef {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.lifetime.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
        }
    }

    impl ToTokens for TyParam {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.ident.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
            if let Some(ref default) = self.default {
                tokens.append("=");
                default.to_tokens(tokens);
            }
        }
    }

    impl ToTokens for TyParamBound {
        fn to_tokens(&self, tokens: &mut Tokens) {
            match *self {
                TyParamBound::Region(ref lifetime) => lifetime.to_tokens(tokens),
                TyParamBound::Trait(ref trait_ref, modifier) => {
                    match modifier {
                        TraitBoundModifier::None => {}
                        TraitBoundModifier::Maybe => tokens.append("?"),
                    }
                    trait_ref.to_tokens(tokens);
                }
            }
        }
    }

    impl ToTokens for WhereClause {
        fn to_tokens(&self, tokens: &mut Tokens) {
            if !self.predicates.is_empty() {
                tokens.append("where");
                tokens.append_separated(&self.predicates, ",");
            }
        }
    }

    impl ToTokens for WherePredicate {
        fn to_tokens(&self, tokens: &mut Tokens) {
            match *self {
                WherePredicate::BoundPredicate(ref predicate) => {
                    predicate.to_tokens(tokens);
                }
                WherePredicate::RegionPredicate(ref predicate) => {
                    predicate.to_tokens(tokens);
                }
            }
        }
    }

    impl ToTokens for WhereBoundPredicate {
        fn to_tokens(&self, tokens: &mut Tokens) {
            if !self.bound_lifetimes.is_empty() {
                tokens.append("for");
                tokens.append("<");
                tokens.append_separated(&self.bound_lifetimes, ",");
                tokens.append(">");
            }
            self.bounded_ty.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
        }
    }

    impl ToTokens for WhereRegionPredicate {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.lifetime.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
        }
    }
}
<commit_msg>Add Lifetime::new<commit_after>use super::*;

/// Represents lifetimes and type parameters attached to a declaration
/// of a function, enum, trait, etc.
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct Generics {
    pub lifetimes: Vec<LifetimeDef>,
    pub ty_params: Vec<TyParam>,
    pub where_clause: WhereClause,
}

#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Lifetime {
    pub ident: Ident,
}

impl Lifetime {
    pub fn new<T: Into<Ident>>(t: T) -> Self {
        Lifetime {
            ident: Ident::new(t),
        }
    }
}

/// A lifetime definition, e.g. `'a: 'b+'c+'d`
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct LifetimeDef {
    pub lifetime: Lifetime,
    pub bounds: Vec<Lifetime>,
}

#[derive(Debug, Clone, Eq, PartialEq)]
pub struct TyParam {
    pub ident: Ident,
    pub bounds: Vec<TyParamBound>,
    pub default: Option<Ty>,
}

/// The AST represents all type param bounds as types.
/// `typeck::collect::compute_bounds` matches these against
/// the "special" built-in traits (see `middle::lang_items`) and
/// detects Copy, Send and Sync.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum TyParamBound {
    Trait(PolyTraitRef, TraitBoundModifier),
    Region(Lifetime),
}

/// A modifier on a bound, currently this is only used for `?Sized`, where the
/// modifier is `Maybe`. Negative bounds should also be handled here.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum TraitBoundModifier {
    None,
    Maybe,
}

/// A `where` clause in a definition
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct WhereClause {
    pub predicates: Vec<WherePredicate>,
}

/// A single predicate in a `where` clause
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum WherePredicate {
    /// A type binding, e.g. `for<'c> Foo: Send+Clone+'c`
    BoundPredicate(WhereBoundPredicate),
    /// A lifetime predicate, e.g. `'a: 'b+'c`
    RegionPredicate(WhereRegionPredicate),
}

/// A type bound.
///
/// E.g. `for<'c> Foo: Send+Clone+'c`
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WhereBoundPredicate {
    /// Any lifetimes from a `for` binding
    pub bound_lifetimes: Vec<LifetimeDef>,
    /// The type being bounded
    pub bounded_ty: Ty,
    /// Trait and lifetime bounds (`Clone+Send+'static`)
    pub bounds: Vec<TyParamBound>,
}

/// A lifetime predicate.
///
/// E.g. `'a: 'b+'c`
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WhereRegionPredicate {
    pub lifetime: Lifetime,
    pub bounds: Vec<Lifetime>,
}

#[cfg(feature = "parsing")]
pub mod parsing {
    use super::*;
    use ident::parsing::ident;
    use ty::parsing::{ty, poly_trait_ref};

    named!(pub generics -> Generics, map!(
        alt!(
            do_parse!(
                punct!("<") >>
                lifetimes: separated_list!(punct!(","), lifetime_def) >>
                ty_params: opt_vec!(preceded!(
                    cond!(!lifetimes.is_empty(), punct!(",")),
                    separated_nonempty_list!(punct!(","), ty_param)
                )) >>
                punct!(">") >>
                (lifetimes, ty_params)
            )
            |
            epsilon!() => { |_| (Vec::new(), Vec::new()) }
        ),
        |(lifetimes, ty_params)| Generics {
            lifetimes: lifetimes,
            ty_params: ty_params,
            where_clause: Default::default(),
        }
    ));

    named!(pub lifetime -> Lifetime, preceded!(
        punct!("'"),
        map!(ident, |id| Lifetime {
            ident: format!("'{}", id).into(),
        })
    ));

    named!(pub lifetime_def -> LifetimeDef, do_parse!(
        life: lifetime >>
        bounds: opt_vec!(preceded!(
            punct!(":"),
            separated_nonempty_list!(punct!("+"), lifetime)
        )) >>
        (LifetimeDef {
            lifetime: life,
            bounds: bounds,
        })
    ));

    named!(pub bound_lifetimes -> Vec<LifetimeDef>, opt_vec!(do_parse!(
        keyword!("for") >>
        punct!("<") >>
        lifetimes: separated_list!(punct!(","), lifetime_def) >>
        punct!(">") >>
        (lifetimes)
    )));

    named!(ty_param -> TyParam, do_parse!(
        id: ident >>
        bounds: opt_vec!(preceded!(
            punct!(":"),
            separated_nonempty_list!(punct!("+"), ty_param_bound)
        )) >>
        default: option!(preceded!(
            punct!("="),
            ty
        )) >>
        (TyParam {
            ident: id,
            bounds: bounds,
            default: default,
        })
    ));

    named!(pub ty_param_bound -> TyParamBound, alt!(
        preceded!(punct!("?"), poly_trait_ref) => {
            |poly| TyParamBound::Trait(poly, TraitBoundModifier::Maybe)
        }
        |
        lifetime => { TyParamBound::Region }
        |
        poly_trait_ref => {
            |poly| TyParamBound::Trait(poly, TraitBoundModifier::None)
        }
    ));

    named!(pub where_clause -> WhereClause, alt!(
        do_parse!(
            keyword!("where") >>
            predicates: separated_nonempty_list!(punct!(","), where_predicate) >>
            option!(punct!(",")) >>
            (WhereClause { predicates: predicates })
        )
        |
        epsilon!() => { |_| Default::default() }
    ));

    named!(where_predicate -> WherePredicate, alt!(
        do_parse!(
            ident: lifetime >>
            punct!(":") >>
            bounds: separated_nonempty_list!(punct!("+"), lifetime) >>
            (WherePredicate::RegionPredicate(WhereRegionPredicate {
                lifetime: ident,
                bounds: bounds,
            }))
        )
        |
        do_parse!(
            bound_lifetimes: bound_lifetimes >>
            bounded_ty: ty >>
            punct!(":") >>
            bounds: separated_nonempty_list!(punct!("+"), ty_param_bound) >>
            (WherePredicate::BoundPredicate(WhereBoundPredicate {
                bound_lifetimes: bound_lifetimes,
                bounded_ty: bounded_ty,
                bounds: bounds,
            }))
        )
    ));
}

#[cfg(feature = "printing")]
mod printing {
    use super::*;
    use quote::{Tokens, ToTokens};

    impl ToTokens for Generics {
        fn to_tokens(&self, tokens: &mut Tokens) {
            let has_lifetimes = !self.lifetimes.is_empty();
            let has_ty_params = !self.ty_params.is_empty();
            if has_lifetimes || has_ty_params {
                tokens.append("<");
                tokens.append_separated(&self.lifetimes, ",");
                if has_lifetimes && has_ty_params {
                    tokens.append(",");
                }
                tokens.append_separated(&self.ty_params, ",");
                tokens.append(">");
            }
        }
    }

    impl ToTokens for Lifetime {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.ident.to_tokens(tokens);
        }
    }

    impl ToTokens for LifetimeDef {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.lifetime.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
        }
    }

    impl ToTokens for TyParam {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.ident.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
            if let Some(ref default) = self.default {
                tokens.append("=");
                default.to_tokens(tokens);
            }
        }
    }

    impl ToTokens for TyParamBound {
        fn to_tokens(&self, tokens: &mut Tokens) {
            match *self {
                TyParamBound::Region(ref lifetime) => lifetime.to_tokens(tokens),
                TyParamBound::Trait(ref trait_ref, modifier) => {
                    match modifier {
                        TraitBoundModifier::None => {}
                        TraitBoundModifier::Maybe => tokens.append("?"),
                    }
                    trait_ref.to_tokens(tokens);
                }
            }
        }
    }

    impl ToTokens for WhereClause {
        fn to_tokens(&self, tokens: &mut Tokens) {
            if !self.predicates.is_empty() {
                tokens.append("where");
                tokens.append_separated(&self.predicates, ",");
            }
        }
    }

    impl ToTokens for WherePredicate {
        fn to_tokens(&self, tokens: &mut Tokens) {
            match *self {
                WherePredicate::BoundPredicate(ref predicate) => {
                    predicate.to_tokens(tokens);
                }
                WherePredicate::RegionPredicate(ref predicate) => {
                    predicate.to_tokens(tokens);
                }
            }
        }
    }

    impl ToTokens for WhereBoundPredicate {
        fn to_tokens(&self, tokens: &mut Tokens) {
            if !self.bound_lifetimes.is_empty() {
                tokens.append("for");
                tokens.append("<");
                tokens.append_separated(&self.bound_lifetimes, ",");
                tokens.append(">");
            }
            self.bounded_ty.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
        }
    }

    impl ToTokens for WhereRegionPredicate {
        fn to_tokens(&self, tokens: &mut Tokens) {
            self.lifetime.to_tokens(tokens);
            if !self.bounds.is_empty() {
                tokens.append(":");
                tokens.append_separated(&self.bounds, "+");
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::{HashSet, HashMap};

use rustc_serialize::json;

use std::f64::consts::PI;

use std::fmt;

const PRECISION : f64 = 10000.0;

/// Point
#[derive(PartialEq, Eq, Clone, Debug, Hash, Copy)]
pub struct Point {
    lat : i64,
    lon : i64,
}

fn deg2rad(deg : f64) -> f64 {
    ((2.0 * PI)/180.0) * deg
}

impl fmt::Display for Point {
    fn fmt(&self, f : &mut fmt::Formatter) -> fmt::Result {
        write!(f, "({}, {})", self.lat(), self.lon())
    }
}

impl Point {
    pub fn new(lat : f64, lon : f64) -> Point {
        Point {
            lat : (lat*PRECISION).floor() as i64,
            lon : (lon*PRECISION).floor() as i64,
        }
    }

    pub fn lat(&self) -> f64 {
        (self.lat as f64)/PRECISION
    }

    pub fn lon(&self) -> f64 {
        (self.lon as f64)/PRECISION
    }

    pub fn geo_distance(&self, p : &Point) -> f64 {
        let R = 6371000.0; // metres
        let lat1 = p.lat();
        let lon1 = p.lon();
        let lat2 = self.lat();
        let lon2 = self.lon();
        let sig1 = deg2rad(lat1);
        let sig2 = deg2rad(lat2);
        let deltasig = deg2rad(lat2-lat1);
        let deltalambda = deg2rad(lon2-lon1);
        let a = (deltasig/2.0).sin() * (deltasig/2.0).sin() +
            sig1.cos() * sig2.cos() *
            (deltalambda/2.0).sin() * (deltalambda/2.0).sin();
        let c = 2.0 * a.sqrt().atan2((1.0-a).sqrt());
        R * c
    }

}

/// Path
pub type Path = Vec<Point>;

/// Maintains a sparse routing graph
pub struct RouteGraph {
    vertices : HashSet<Point>,
    edges : HashMap<Point, Vec<Point>>,
    paths : HashMap<(Point, Point), Path>,
}

/// Convert the TFL lineStrings attribute to a simple flat vectory of paths.
/// lineStrings in TFL data is a JSON array of string values, containing
/// either an array of points or an array of arrays of points, we handle both.
pub fn linestrings_to_paths(line_strings : &Vec<String>) -> Vec<Path> {
    let mut paths : Vec<Path> = Vec::new();
    for line_string in line_strings {
        match json::decode::<Vec<(f64, f64)>>(&line_string) {
            Ok(raw_path) => paths.push(raw_path.iter().map(|&(lon, lat)| Point::new(lat, lon)).collect()),
            Err(err) => {
                match json::decode::<Vec<Vec<(f64, f64)>>>(&line_string) {
                    Ok(raw_paths) => {
                        for raw_path in raw_paths {
                            paths.push(raw_path.iter().map(|&(lon, lat)| Point::new(lat, lon)).collect());
                        }
                    },
                    Err(err2) => println!("Errors decoding line string, single line {}, multi line {}", err, err2),
                }
            },
        }
    }
    paths
}

impl RouteGraph {
    /// New Route Graph
    pub fn new() -> RouteGraph {
        RouteGraph {
            vertices : HashSet::new(),
            edges : HashMap::new(),
            paths : HashMap::new(),
        }
    }

    /// Add many paths
    pub fn add_paths(&mut self, paths : &Vec<Path>) {
        for path in paths {
            for path in paths {
                self.add_path(path);
            }
        }
    }

    /// Add single path to the graph
    pub fn add_path(&mut self, path : &Path) {
        // add points
        let first = path.first().unwrap();
        let last = path.last().unwrap();
        self.vertices.insert(first.clone());
        self.vertices.insert(last.clone());

        // add bidirectional edges
        self.edges.entry(first.clone()).or_insert(vec![last.clone()]).push(last.clone());
        self.edges.entry(last.clone()).or_insert(vec![first.clone()]).push(first.clone());

        // add paths
        self.paths.insert((first.clone(), last.clone()), path.clone());
        self.paths.insert((last.clone(), first.clone()), path.iter().rev().cloned().collect());
        assert!(self.paths.contains_key(&(first.clone(), last.clone())));
        assert!(self.paths.contains_key(&(last.clone(), first.clone())));
    }

    /// Find the closest actual point (vertex) in our graph since they are not
    /// going to be exact matches
    fn closest_point(&self, pt : &Point) -> (Point, f64) {
        let far_pt = Point::new(pt.lat()+90.0, pt.lon()+90.0);
        let far_dist = far_pt.geo_distance(pt);
        let (min_pt, min_dist) = self.edges.keys().fold((&far_pt, far_dist), |(min_pt, min_dist), vert| {
            let vert_dist = vert.geo_distance(pt);
            match vert_dist < min_dist {
                true => (vert, vert_dist),
                false => (min_pt, min_dist),
            }
        });
        (min_pt.clone(), min_dist)
    }

    /// Find a path between two points if one exists
    pub fn path(&self, p0 : Point, p1 : Point) -> Option<Path> {
        let (start, start_dist) = self.closest_point(&p0);
        if start_dist > 2000.0 {
            println!("start point {} to closest {} distance is {} > 2000", p0, start, start_dist);
            return None;
        }
        let (end, end_dist) = self.closest_point(&p1);
        if end_dist > 2000.0 {
            println!("end point {} to closest {} distance is {} > 2000", p1, end, end_dist);
            return None;
        }
        let mut visited : HashSet<Point> = HashSet::new();
        self.find_path(start, end, &visited)
    }

    // Recursively search the path graph from p0 to p1, when a path is found
    // the callstack will inherently build up a single Path containing all the
    // points between the two.
    // Visited is a simple HashSet marking points we've already visited to avoid
    // infinitely recursing in circles.
    fn find_path(&self, p0 : Point, p1 : Point, visited : &HashSet<Point>) -> Option<Path> {
        match self.edges.get(&p0) {
            Some(to_vertices) => match visited.contains(&p0) {
                true => None,
                false => {
                    for next in to_vertices {
                        if next.clone() == p1 {
                            return Some(self.paths.get(&(p0, next.clone())).unwrap().clone());
                        }
                        let mut visited0 = visited.clone();
                        visited0.insert(next.clone());
                        match self.find_path(next.clone(), p1, &visited0) {
                            Some(path) => {
                                return Some(vec![&(self.paths.get(&(p0, next.clone())).unwrap())[..], &path[..]].concat());
                            },
                            None => (),
                        }
                    }
                    None
                },
            },
            None => None,
        }
    }
}
<commit_msg>Explain the use of integers, with TODO<commit_after>use std::collections::{HashSet, HashMap};

use rustc_serialize::json;

use std::f64::consts::PI;

use std::fmt;

const PRECISION : f64 = 10000.0;

/// Point containing latitude and longitude values as integers.
/// Integers are used here due to Rust itself not providing some basic
/// floating point functionality at the moment. All due to the undefined
/// behavior revolving around NaN float values. For example, what is the boolean
/// result of 0.0 > NaN ? Its undefined, so therefore > is not implemented by
/// Rust Proper! Perhaps they'll fix this massive inconvienence in the future.
/// TODO use floats
#[derive(PartialEq, Eq, Clone, Debug, Hash, Copy)]
pub struct Point {
    lat : i64,
    lon : i64,
}

/// Degress to Radians
fn deg2rad(deg : f64) -> f64 {
    ((2.0 * PI)/180.0) * deg
}

impl fmt::Display for Point {
    fn fmt(&self, f : &mut fmt::Formatter) -> fmt::Result {
        write!(f, "({}, {})", self.lat(), self.lon())
    }
}

impl Point {
    /// New integer stored Point from floating point lat/lon coordinates
    pub fn new(lat : f64, lon : f64) -> Point {
        Point {
            lat : (lat*PRECISION).floor() as i64,
            lon : (lon*PRECISION).floor() as i64,
        }
    }

    /// Latitude value
    pub fn lat(&self) -> f64 {
        (self.lat as f64)/PRECISION
    }

    /// Longitude value
    pub fn lon(&self) -> f64 {
        (self.lon as f64)/PRECISION
    }

    /// Spheroid distance calculation given earth coordinates as lat/lon values.
    /// Returns the distance in meters.
    pub fn geo_distance(&self, p : &Point) -> f64 {
        let R = 6371000.0; // metres
        let lat1 = p.lat();
        let lon1 = p.lon();
        let lat2 = self.lat();
        let lon2 = self.lon();
        let sig1 = deg2rad(lat1);
        let sig2 = deg2rad(lat2);
        let deltasig = deg2rad(lat2-lat1);
        let deltalambda = deg2rad(lon2-lon1);
        let a = (deltasig/2.0).sin() * (deltasig/2.0).sin() +
            sig1.cos() * sig2.cos() *
            (deltalambda/2.0).sin() * (deltalambda/2.0).sin();
        let c = 2.0 * a.sqrt().atan2((1.0-a).sqrt());
        R * c
    }

}

/// Path
pub type Path = Vec<Point>;

/// Maintains a sparse routing graph
pub struct RouteGraph {
    vertices : HashSet<Point>,
    edges : HashMap<Point, Vec<Point>>,
    paths : HashMap<(Point, Point), Path>,
}

/// Convert the TFL lineStrings attribute to a simple flat vectory of paths.
/// lineStrings in TFL data is a JSON array of string values, containing
/// either an array of points or an array of arrays of points, we handle both.
pub fn linestrings_to_paths(line_strings : &Vec<String>) -> Vec<Path> {
    let mut paths : Vec<Path> = Vec::new();
    for line_string in line_strings {
        match json::decode::<Vec<(f64, f64)>>(&line_string) {
            Ok(raw_path) => paths.push(raw_path.iter().map(|&(lon, lat)| Point::new(lat, lon)).collect()),
            Err(err) => {
                match json::decode::<Vec<Vec<(f64, f64)>>>(&line_string) {
                    Ok(raw_paths) => {
                        for raw_path in raw_paths {
                            paths.push(raw_path.iter().map(|&(lon, lat)| Point::new(lat, lon)).collect());
                        }
                    },
                    Err(err2) => println!("Errors decoding line string, single line {}, multi line {}", err, err2),
                }
            },
        }
    }
    paths
}

impl RouteGraph {
    /// New Route Graph
    pub fn new() -> RouteGraph {
        RouteGraph {
            vertices : HashSet::new(),
            edges : HashMap::new(),
            paths : HashMap::new(),
        }
    }

    /// Add many paths
    pub fn add_paths(&mut self, paths : &Vec<Path>) {
        for path in paths {
            for path in paths {
                self.add_path(path);
            }
        }
    }

    /// Add single path to the graph
    pub fn add_path(&mut self, path : &Path) {
        // add points
        let first = path.first().unwrap();
        let last = path.last().unwrap();
        self.vertices.insert(first.clone());
        self.vertices.insert(last.clone());

        // add bidirectional edges
        self.edges.entry(first.clone()).or_insert(vec![last.clone()]).push(last.clone());
        self.edges.entry(last.clone()).or_insert(vec![first.clone()]).push(first.clone());

        // add paths
        self.paths.insert((first.clone(), last.clone()), path.clone());
        self.paths.insert((last.clone(), first.clone()), path.iter().rev().cloned().collect());
        assert!(self.paths.contains_key(&(first.clone(), last.clone())));
        assert!(self.paths.contains_key(&(last.clone(), first.clone())));
    }

    /// Find the closest actual point (vertex) in our graph since they are not
    /// going to be exact matches
    fn closest_point(&self, pt : &Point) -> (Point, f64) {
        let far_pt = Point::new(pt.lat()+90.0, pt.lon()+90.0);
        let far_dist = far_pt.geo_distance(pt);
        let (min_pt, min_dist) = self.edges.keys().fold((&far_pt, far_dist), |(min_pt, min_dist), vert| {
            let vert_dist = vert.geo_distance(pt);
            match vert_dist < min_dist {
                true => (vert, vert_dist),
                false => (min_pt, min_dist),
            }
        });
        (min_pt.clone(), min_dist)
    }

    /// Find a path between two points if one exists
    pub fn path(&self, p0 : Point, p1 : Point) -> Option<Path> {
        let (start, start_dist) = self.closest_point(&p0);
        if start_dist > 2000.0 {
            println!("start point {} to closest {} distance is {} > 2000", p0, start, start_dist);
            return None;
        }
        let (end, end_dist) = self.closest_point(&p1);
        if end_dist > 2000.0 {
            println!("end point {} to closest {} distance is {} > 2000", p1, end, end_dist);
            return None;
        }
        let mut visited : HashSet<Point> = HashSet::new();
        self.find_path(start, end, &visited)
    }

    // Recursively search the path graph from p0 to p1, when a path is found
    // the callstack will inherently build up a single Path containing all the
    // points between the two.
    // Visited is a simple HashSet marking points we've already visited to avoid
    // infinitely recursing in circles.
    fn find_path(&self, p0 : Point, p1 : Point, visited : &HashSet<Point>) -> Option<Path> {
        match self.edges.get(&p0) {
            Some(to_vertices) => match visited.contains(&p0) {
                true => None,
                false => {
                    for next in to_vertices {
                        if next.clone() == p1 {
                            return Some(self.paths.get(&(p0, next.clone())).unwrap().clone());
                        }
                        let mut visited0 = visited.clone();
                        visited0.insert(next.clone());
                        match self.find_path(next.clone(), p1, &visited0) {
                            Some(path) => {
                                return Some(vec![&(self.paths.get(&(p0, next.clone())).unwrap())[..], &path[..]].concat());
                            },
                            None => (),
                        }
                    }
                    None
                },
            },
            None => None,
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Local state for known hashes and their external location (blob reference).

use blob;
use crypto;
use db;

use errors::{DieselError, RetryError};

use std::sync::{Arc, Mutex, MutexGuard};
use tags;
use util::UniquePriorityQueue;

pub mod tree;

#[cfg(test)]
mod tests;
#[cfg(all(test, feature = "benchmarks"))]
mod benchmarks;

pub struct HashIndex(InternalHashIndex);

/// A wrapper around Hash digests.
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
pub struct Hash {
    pub bytes: Vec<u8>,
}

impl Hash {
    /// Computes `hash(text)` and stores this digest as the `bytes` field in a new `Hash` structure.
    pub fn new(text: &[u8]) -> Hash {
        Hash { bytes: crypto::authed::hash::new(text) }
    }
}

/// An entry that can be inserted into the hash index.
#[derive(Clone)]
pub struct Entry {
    /// The hash of this entry (unique among all entries in the index).
    pub hash: Hash,

    /// The level in a hash tree that this entry is from. Level `0` represents `leaf`s, i.e. entries
    /// that represent user-data, where levels `1` and up represents `branches` of the tree,
    /// i.e. internal meta-data.
    pub node: blob::NodeType,
    pub leaf: blob::LeafType,

    /// An optional list of child hash ids.
    pub childs: Option<Vec<i64>>,

    /// A reference to a location in the external persistent storage (a chunk reference) that
    /// contains the data for this entry (e.g. an object-name and a byte range).
    pub persistent_ref: Option<blob::ChunkRef>,
}

pub enum ReserveResult {
    HashKnown(i64),
    ReserveOk(i64),
}

type Queue = UniquePriorityQueue<i64, Vec<u8>, db::QueueEntry>;

pub struct InternalHashIndex {
    index: Arc<db::Index>,
    queue: Mutex<Queue>,
}

impl Drop for InternalHashIndex {
    fn drop(&mut self) {
        // Sanity check that we flushed this hash index fully before dropping it.
        // Blob store accumulates chunks for the next blob and needs flushing.
        assert_eq!(0, self.queue.lock().unwrap().len());
    }
}

impl InternalHashIndex {
    fn new(index: Arc<db::Index>) -> Result<InternalHashIndex, DieselError> {
        Ok(InternalHashIndex {
            index: index,
            queue: Mutex::new(UniquePriorityQueue::new()),
        })
    }

    pub fn queue_lock(&self) -> MutexGuard<Queue> {
        self.queue.lock().expect("Hash queue mutex poisoned")
    }

    pub fn lock(&self) -> (MutexGuard<Queue>, db::IndexGuard) {
        let queue = self.queue_lock();
        let index = self.index.lock();
        (queue, index)
    }

    fn locate(&self,
              hash: &Hash,
              queue: &MutexGuard<Queue>,
              index: &mut db::IndexGuard)
              -> Option<db::QueueEntry> {
        let result_opt = queue.find_value_of_key(&hash.bytes).cloned();
        result_opt.or_else(|| index.hash_locate(hash))
    }

    fn reserve(&self,
               hash_entry: &Entry,
               mut queue: &mut MutexGuard<Queue>,
               mut index: &mut db::IndexGuard)
               -> i64 {
        index.maybe_flush();

        let Entry { ref hash, node, leaf, ref childs, ref persistent_ref } = *hash_entry;
        assert!(!hash.bytes.is_empty());

        let my_id = index.hash_next_id();
        let qe = db::QueueEntry {
            id: my_id,
            node: node,
            leaf: leaf,
            childs: childs.clone(),
            tag: None,
            persistent_ref: persistent_ref.clone(),
        };
        index.hash_insert_new(my_id, hash.bytes.clone(), qe.clone());
        assert!(queue.put_value(my_id, hash.bytes.clone(), qe).is_ok());

        my_id
    }

    fn reserved_id(&self, hash: &Hash, queue: &MutexGuard<Queue>) -> Option<i64> {
        queue.find_key(&hash.bytes).cloned()
    }

    fn update_reserved(&self,
                       id: i64,
                       hash_entry: Entry,
                       mut queue: &mut MutexGuard<Queue>) {
        let Entry { hash, node, leaf, childs, persistent_ref } = hash_entry;
        assert!(!hash.bytes.is_empty());

        let old_id = queue.find_key(&hash.bytes).cloned();
        match old_id {
            None => {
                // FIXME(jos): avoid having to set the blob ref with update_reserved.
                return;
            }
            Some(old) => {
                assert_eq!(old, id);
            }
        }
        assert_eq!(old_id, Some(id));

        // If we didn't already commit and pop() the hash, update it:
        queue.update_value(&hash.bytes, |qe| {
            qe.node = node;
            qe.leaf = leaf;
            qe.childs = childs;
            qe.persistent_ref = persistent_ref;
        });
    }

    fn insert_completed_in_order(&self,
                                 mut queue: &mut MutexGuard<Queue>,
                                 mut index: &mut db::IndexGuard) {
        while let Some((id_, _, queue_entry)) = queue.pop_min_if_complete() {
            assert_eq!(id_, queue_entry.id);
            index.hash_set_ready(id_, &queue_entry);
        }
        index.maybe_flush();
    }

    fn commit(&self,
              id: i64,
              entry_opt: Option<Entry>,
              mut queue: &mut MutexGuard<Queue>,
              mut index: &mut db::IndexGuard) {
        entry_opt.map(|e| self.update_reserved(id, e, queue));

        queue.set_ready(&id);
        self.insert_completed_in_order(&mut queue, &mut index);
    }
}

impl HashIndex {
    pub fn new(index: Arc<db::Index>) -> Result<HashIndex, DieselError> {
        index.lock().hash_delete_not_ready();
        Ok(HashIndex(InternalHashIndex::new(index)?))
    }

    /// Locate the local ID of this hash.
    pub fn get_id(&self, hash: &Hash) -> Option<i64> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|entry| entry.id)
    }

    /// Locate hash entry from its ID.
    pub fn get_hash(&self, id: i64) -> Option<db::Entry> {
        self.0.index.lock().hash_locate_by_id(id)
    }

    /// Check whether this `Hash` already exists in the system.
    pub fn hash_exists(&self, hash: &Hash) -> bool {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).is_some()
    }

    /// Locate the local childs of the `Hash`.
    pub fn fetch_childs(&self, hash: &Hash) -> Option<Option<Vec<i64>>> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|queue_entry| queue_entry.childs)
    }

    /// Locate the persistent reference (external blob reference) for this `Hash`.
    pub fn fetch_persistent_ref(&self, hash: &Hash) -> Result<Option<blob::ChunkRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => Ok(Some(queue_entry.persistent_ref.expect("persistent_ref"))),
            None => Ok(None),
        }
    }

    /// Locate the hash reference (including persistent blob reference) for this `Hash~.
    pub fn fetch_hash_ref(&self, hash: &Hash) -> Result<Option<tree::HashRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => {
                Ok(Some(tree::HashRef {
                    hash: hash.clone(),
                    node: queue_entry.node,
                    leaf: queue_entry.leaf,
                    info: None,
                    persistent_ref: queue_entry.persistent_ref.expect("persistent_ref"),
                }))
            }
            None => Ok(None),
        }
    }

    /// Reserve a `Hash` in the index, while sending its content to external storage.
    /// This is used to ensure that each `Hash` is stored only once.
    pub fn reserve(&self, hash_entry: &Entry) -> ReserveResult {
        assert!(!hash_entry.hash.bytes.is_empty());
        // To avoid unused IO, we store entries in-memory until committed to persistent
        // storage. This allows us to continue after a crash without needing to scan
        // through and delete uncommitted entries.
        let (mut queue, mut index) = self.0.lock();
        match self.0.locate(&hash_entry.hash, &queue, &mut index) {
            Some(entry) => ReserveResult::HashKnown(entry.id),
            None => {
                let id = self.0.reserve(hash_entry, &mut queue, &mut index);
                ReserveResult::ReserveOk(id)
            }
        }
    }

    /// Check whether an entry was previously reserved.
    pub fn reserved_id(&self, hash: &Hash) -> Option<i64> {
        let queue = self.0.queue_lock();
        self.0.reserved_id(hash, &queue)
    }

    /// Update the info for a reserved `Hash`. The `Hash` remains reserved. This is used to update
    /// the persistent reference (external blob reference) as soon as it is available (to allow new
    /// references to the `Hash` to be created before it is committed).
    pub fn update_reserved(&self, id: i64, hash_entry: Entry) {
        assert!(!hash_entry.hash.bytes.is_empty());
        let mut queue = self.0.queue_lock();
        self.0.update_reserved(id, hash_entry, &mut queue);
    }

    /// A `Hash` is committed when it has been `finalized` in the external storage. `Commit`
    /// includes the persistent reference that the content is available at.
    pub fn commit(&self, id: i64, entry: Option<Entry>) {
        let (mut queue, mut index) = self.0.lock();
        self.0.commit(id, entry, &mut queue, &mut index);
    }

    /// List all hash entries.
    pub fn list(&self) -> Vec<db::Entry> {
        self.0.index.lock().hash_list()
    }

    /// Permanently delete hash by its ID.
    pub fn delete(&self, id: i64) {
        self.0.index.lock().hash_delete(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_tag(&self, id: i64, tag: tags::Tag) {
        self.0.index.lock().hash_set_tag(Some(id), tag);
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_all_tags(&self, tag: tags::Tag) {
        self.0.index.lock().hash_set_tag(None, tag)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_tag(&self, id: i64) -> Option<tags::Tag> {
        {
            if let Some(ref q) = self.0.queue_lock().find_mut_value_of_priority(&id) {
                return q.tag;
            }
        }
        self.0.index.lock().hash_get_tag(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_ids_by_tag(&self, tag: i64) -> Vec<i64> {
        self.0.index.lock().hash_list_ids_by_tag(tag)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn read_gc_data(&self, hash_id: i64, family_id: i64) -> db::GcData {
        self.0.index.lock().hash_read_gc_data(hash_id, family_id)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_gc_data<F: db::UpdateFn>(&self,
                                           hash_id: i64,
                                           family_id: i64,
                                           update_fn: F)
                                           -> db::GcData {
        self.0.index.lock().hash_update_gc_data(hash_id, family_id, update_fn)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_family_gc_data<F: db::UpdateFn, I: Iterator<Item = F>>(&self,
                                                                         family_id: i64,
                                                                         update_fns: I) {
        self.0.index.lock().hash_update_family_gc_data(family_id, update_fns)
    }

    /// Manual commit. This also disables automatic periodic commit.
    pub fn manual_commit(&self) {
        let mut guard = self.0.index.lock();
        guard.flush();
        guard.set_auto_flush(false);
    }

    /// Flush the hash index to clear internal buffers and commit the underlying database.
    pub fn flush(&self) {
        self.0.index.lock().flush()
    }
}
<commit_msg>Cleanup: we no longer need to call update_reserved after commit.<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Local state for known hashes and their external location (blob reference).

use blob;
use crypto;
use db;

use errors::{DieselError, RetryError};

use std::sync::{Arc, Mutex, MutexGuard};
use tags;
use util::UniquePriorityQueue;

pub mod tree;

#[cfg(test)]
mod tests;
#[cfg(all(test, feature = "benchmarks"))]
mod benchmarks;

pub struct HashIndex(InternalHashIndex);

/// A wrapper around Hash digests.
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
pub struct Hash {
    pub bytes: Vec<u8>,
}

impl Hash {
    /// Computes `hash(text)` and stores this digest as the `bytes` field in a new `Hash` structure.
    pub fn new(text: &[u8]) -> Hash {
        Hash { bytes: crypto::authed::hash::new(text) }
    }
}

/// An entry that can be inserted into the hash index.
#[derive(Clone)]
pub struct Entry {
    /// The hash of this entry (unique among all entries in the index).
    pub hash: Hash,

    /// The level in a hash tree that this entry is from. Level `0` represents `leaf`s, i.e. entries
    /// that represent user-data, where levels `1` and up represents `branches` of the tree,
    /// i.e. internal meta-data.
    pub node: blob::NodeType,
    pub leaf: blob::LeafType,

    /// An optional list of child hash ids.
    pub childs: Option<Vec<i64>>,

    /// A reference to a location in the external persistent storage (a chunk reference) that
    /// contains the data for this entry (e.g. an object-name and a byte range).
    pub persistent_ref: Option<blob::ChunkRef>,
}

pub enum ReserveResult {
    HashKnown(i64),
    ReserveOk(i64),
}

type Queue = UniquePriorityQueue<i64, Vec<u8>, db::QueueEntry>;

pub struct InternalHashIndex {
    index: Arc<db::Index>,
    queue: Mutex<Queue>,
}

impl Drop for InternalHashIndex {
    fn drop(&mut self) {
        // Sanity check that we flushed this hash index fully before dropping it.
        // Blob store accumulates chunks for the next blob and needs flushing.
        assert_eq!(0, self.queue.lock().unwrap().len());
    }
}

impl InternalHashIndex {
    fn new(index: Arc<db::Index>) -> Result<InternalHashIndex, DieselError> {
        Ok(InternalHashIndex {
            index: index,
            queue: Mutex::new(UniquePriorityQueue::new()),
        })
    }

    pub fn queue_lock(&self) -> MutexGuard<Queue> {
        self.queue.lock().expect("Hash queue mutex poisoned")
    }

    pub fn lock(&self) -> (MutexGuard<Queue>, db::IndexGuard) {
        let queue = self.queue_lock();
        let index = self.index.lock();
        (queue, index)
    }

    fn locate(&self,
              hash: &Hash,
              queue: &MutexGuard<Queue>,
              index: &mut db::IndexGuard)
              -> Option<db::QueueEntry> {
        let result_opt = queue.find_value_of_key(&hash.bytes).cloned();
        result_opt.or_else(|| index.hash_locate(hash))
    }

    fn reserve(&self,
               hash_entry: &Entry,
               mut queue: &mut MutexGuard<Queue>,
               mut index: &mut db::IndexGuard)
               -> i64 {
        index.maybe_flush();

        let Entry { ref hash, node, leaf, ref childs, ref persistent_ref } = *hash_entry;
        assert!(!hash.bytes.is_empty());

        let my_id = index.hash_next_id();
        let qe = db::QueueEntry {
            id: my_id,
            node: node,
            leaf: leaf,
            childs: childs.clone(),
            tag: None,
            persistent_ref: persistent_ref.clone(),
        };
        index.hash_insert_new(my_id, hash.bytes.clone(), qe.clone());
        assert!(queue.put_value(my_id, hash.bytes.clone(), qe).is_ok());

        my_id
    }

    fn reserved_id(&self, hash: &Hash, queue: &MutexGuard<Queue>) -> Option<i64> {
        queue.find_key(&hash.bytes).cloned()
    }

    fn update_reserved(&self,
                       id: i64,
                       hash_entry: Entry,
                       mut queue: &mut MutexGuard<Queue>) {
        let Entry { hash, node, leaf, childs, persistent_ref } = hash_entry;
        assert!(!hash.bytes.is_empty());

        if let Some(old_id) = queue.find_key(&hash.bytes) {
            assert_eq!(*old_id, id);
        } else {
            unreachable!("Tried to update unreserved hash.");
        }

        // If we didn't already commit and pop() the hash, update it:
        queue.update_value(&hash.bytes, |qe| {
            qe.node = node;
            qe.leaf = leaf;
            qe.childs = childs;
            qe.persistent_ref = persistent_ref;
        });
    }

    fn insert_completed_in_order(&self,
                                 mut queue: &mut MutexGuard<Queue>,
                                 mut index: &mut db::IndexGuard) {
        while let Some((id_, _, queue_entry)) = queue.pop_min_if_complete() {
            assert_eq!(id_, queue_entry.id);
            index.hash_set_ready(id_, &queue_entry);
        }
        index.maybe_flush();
    }

    fn commit(&self,
              id: i64,
              entry_opt: Option<Entry>,
              mut queue: &mut MutexGuard<Queue>,
              mut index: &mut db::IndexGuard) {
        entry_opt.map(|e| self.update_reserved(id, e, queue));

        queue.set_ready(&id);
        self.insert_completed_in_order(&mut queue, &mut index);
    }
}

impl HashIndex {
    pub fn new(index: Arc<db::Index>) -> Result<HashIndex, DieselError> {
        index.lock().hash_delete_not_ready();
        Ok(HashIndex(InternalHashIndex::new(index)?))
    }

    /// Locate the local ID of this hash.
    pub fn get_id(&self, hash: &Hash) -> Option<i64> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|entry| entry.id)
    }

    /// Locate hash entry from its ID.
    pub fn get_hash(&self, id: i64) -> Option<db::Entry> {
        self.0.index.lock().hash_locate_by_id(id)
    }

    /// Check whether this `Hash` already exists in the system.
    pub fn hash_exists(&self, hash: &Hash) -> bool {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).is_some()
    }

    /// Locate the local childs of the `Hash`.
    pub fn fetch_childs(&self, hash: &Hash) -> Option<Option<Vec<i64>>> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        self.0.locate(hash, &queue, &mut index).map(|queue_entry| queue_entry.childs)
    }

    /// Locate the persistent reference (external blob reference) for this `Hash`.
    pub fn fetch_persistent_ref(&self, hash: &Hash) -> Result<Option<blob::ChunkRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => Ok(Some(queue_entry.persistent_ref.expect("persistent_ref"))),
            None => Ok(None),
        }
    }

    /// Locate the hash reference (including persistent blob reference) for this `Hash~.
    pub fn fetch_hash_ref(&self, hash: &Hash) -> Result<Option<tree::HashRef>, RetryError> {
        assert!(!hash.bytes.is_empty());
        let (queue, mut index) = self.0.lock();
        match self.0.locate(hash, &queue, &mut index) {
            Some(ref queue_entry) if queue_entry.persistent_ref.is_none() => Err(RetryError),
            Some(queue_entry) => {
                Ok(Some(tree::HashRef {
                    hash: hash.clone(),
                    node: queue_entry.node,
                    leaf: queue_entry.leaf,
                    info: None,
                    persistent_ref: queue_entry.persistent_ref.expect("persistent_ref"),
                }))
            }
            None => Ok(None),
        }
    }

    /// Reserve a `Hash` in the index, while sending its content to external storage.
    /// This is used to ensure that each `Hash` is stored only once.
    pub fn reserve(&self, hash_entry: &Entry) -> ReserveResult {
        assert!(!hash_entry.hash.bytes.is_empty());
        // To avoid unused IO, we store entries in-memory until committed to persistent
        // storage. This allows us to continue after a crash without needing to scan
        // through and delete uncommitted entries.
        let (mut queue, mut index) = self.0.lock();
        match self.0.locate(&hash_entry.hash, &queue, &mut index) {
            Some(entry) => ReserveResult::HashKnown(entry.id),
            None => {
                let id = self.0.reserve(hash_entry, &mut queue, &mut index);
                ReserveResult::ReserveOk(id)
            }
        }
    }

    /// Check whether an entry was previously reserved.
    pub fn reserved_id(&self, hash: &Hash) -> Option<i64> {
        let queue = self.0.queue_lock();
        self.0.reserved_id(hash, &queue)
    }

    /// Update the info for a reserved `Hash`. The `Hash` remains reserved. This is used to update
    /// the persistent reference (external blob reference) as soon as it is available (to allow new
    /// references to the `Hash` to be created before it is committed).
    pub fn update_reserved(&self, id: i64, hash_entry: Entry) {
        assert!(!hash_entry.hash.bytes.is_empty());
        let mut queue = self.0.queue_lock();
        self.0.update_reserved(id, hash_entry, &mut queue);
    }

    /// A `Hash` is committed when it has been `finalized` in the external storage. `Commit`
    /// includes the persistent reference that the content is available at.
    pub fn commit(&self, id: i64, entry: Option<Entry>) {
        let (mut queue, mut index) = self.0.lock();
        self.0.commit(id, entry, &mut queue, &mut index);
    }

    /// List all hash entries.
    pub fn list(&self) -> Vec<db::Entry> {
        self.0.index.lock().hash_list()
    }

    /// Permanently delete hash by its ID.
    pub fn delete(&self, id: i64) {
        self.0.index.lock().hash_delete(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_tag(&self, id: i64, tag: tags::Tag) {
        self.0.index.lock().hash_set_tag(Some(id), tag);
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn set_all_tags(&self, tag: tags::Tag) {
        self.0.index.lock().hash_set_tag(None, tag)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_tag(&self, id: i64) -> Option<tags::Tag> {
        {
            if let Some(ref q) = self.0.queue_lock().find_mut_value_of_priority(&id) {
                return q.tag;
            }
        }
        self.0.index.lock().hash_get_tag(id)
    }

    /// API related to tagging, which is useful to indicate state during operation stages.
    /// It operates directly on the underlying IDs.
    pub fn get_ids_by_tag(&self, tag: i64) -> Vec<i64> {
        self.0.index.lock().hash_list_ids_by_tag(tag)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn read_gc_data(&self, hash_id: i64, family_id: i64) -> db::GcData {
        self.0.index.lock().hash_read_gc_data(hash_id, family_id)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_gc_data<F: db::UpdateFn>(&self,
                                           hash_id: i64,
                                           family_id: i64,
                                           update_fn: F)
                                           -> db::GcData {
        self.0.index.lock().hash_update_gc_data(hash_id, family_id, update_fn)
    }

    /// API related to garbage collector metadata tied to (hash id, family id) pairs.
    pub fn update_family_gc_data<F: db::UpdateFn, I: Iterator<Item = F>>(&self,
                                                                         family_id: i64,
                                                                         update_fns: I) {
        self.0.index.lock().hash_update_family_gc_data(family_id, update_fns)
    }

    /// Manual commit. This also disables automatic periodic commit.
    pub fn manual_commit(&self) {
        let mut guard = self.0.index.lock();
        guard.flush();
        guard.set_auto_flush(false);
    }

    /// Flush the hash index to clear internal buffers and commit the underlying database.
    pub fn flush(&self) {
        self.0.index.lock().flush()
    }
}
<|endoftext|>