<commit_before>//! Log all requests in a format similar to Heroku's router, but with additional
//! information that we care about like User-Agent

use super::prelude::*;
use crate::util::request_header;

use conduit::{header, Host, RequestExt, Scheme, StatusCode};
use conduit_cookie::RequestSession;
use sentry::Level;

use std::fmt::{self, Display, Formatter};
use std::time::{SystemTime, UNIX_EPOCH};

const SLOW_REQUEST_THRESHOLD_MS: u64 = 1000;

const FILTERED_HEADERS: &[&str] = &["Authorization", "Cookie", "X-Real-Ip", "X-Forwarded-For"];

#[derive(Default)]
pub(super) struct LogRequests();

struct OriginalPath(String);

impl Middleware for LogRequests {
    fn before(&self, req: &mut dyn RequestExt) -> BeforeResult {
        let path = OriginalPath(req.path().to_string());
        req.mut_extensions().insert(path);
        Ok(())
    }

    fn after(&self, req: &mut dyn RequestExt, res: AfterResult) -> AfterResult {
        let response_time =
            if let Ok(start_ms) = request_header(req, "x-request-start").parse::<u128>() {
                let current_ms = SystemTime::now()
                    .duration_since(UNIX_EPOCH)
                    .expect("Time went way backwards")
                    .as_millis();

                if current_ms > start_ms {
                    // The result cannot be negative
                    current_ms - start_ms
                } else {
                    // Because our nginx proxy and app run on the same dyno in production, we
                    // shouldn't have to worry about clock drift. But if something goes wrong,
                    // calculate the response time based on when the request reached this app.
                    fallback_response_time(req)
                }
            } else {
                // X-Request-Start header couldn't be parsed.
                // We are probably running locally and not behind nginx.
                fallback_response_time(req)
            };

        // This will only trucate for requests lasting > 500 million years
        let response_time = response_time as u64;

        println!(
            "{}",
            RequestLine {
                req,
                res: &res,
                response_time,
            }
        );

        report_to_sentry(req, &res, response_time);

        res
    }
}

/// Calculate the response time based on when the request reached the in-app web server.
///
/// This serves as a fallback in case the `X-Request-Start` header is missing or invalid.
fn fallback_response_time(req: &mut dyn RequestExt) -> u128 {
    req.elapsed().as_millis()
}

struct CustomMetadata {
    entries: Vec<(&'static str, String)>,
}

pub fn add_custom_metadata<V: Display>(req: &mut dyn RequestExt, key: &'static str, value: V) {
    if let Some(metadata) = req.mut_extensions().find_mut::<CustomMetadata>() {
        metadata.entries.push((key, value.to_string()));
    } else {
        let mut metadata = CustomMetadata {
            entries: Vec::new(),
        };
        metadata.entries.push((key, value.to_string()));
        req.mut_extensions().insert(metadata);
    }
}

fn report_to_sentry(req: &dyn RequestExt, res: &AfterResult, response_time: u64) {
    let (message, level) = match res {
        Err(e) => (e.to_string(), Level::Error),
        Ok(_) => return,
    };

    let config = |scope: &mut sentry::Scope| {
        let method = Some(req.method().as_str().to_owned());

        let scheme = match req.scheme() {
            Scheme::Http => "http",
            Scheme::Https => "https",
        };

        let host = match req.host() {
            Host::Name(name) => name.to_owned(),
            Host::Socket(addr) => addr.to_string(),
        };

        let path = &req.extensions().find::<OriginalPath>().unwrap().0;

        let url = format!("{}://{}{}", scheme, host, path).parse().ok();

        {
            let id = req.session().get("user_id").map(|str| str.to_string());

            let user = sentry::User {
                id,
                ..Default::default()
            };

            scope.set_user(Some(user));
        }

        {
            let headers = req
                .headers()
                .iter()
                .filter(|(k, _v)| !FILTERED_HEADERS.iter().any(|name| k == name))
                .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default().to_string()))
                .collect();

            let sentry_req = sentry::protocol::Request {
                url,
                method,
                headers,
                ..Default::default()
            };

            scope.add_event_processor(Box::new(move |mut event| {
                if event.request.is_none() {
                    event.request = Some(sentry_req.clone());
                }
                Some(event)
            }));
        }

        if let Some(request_id) = req
            .headers()
            .get("x-request-id")
            .and_then(|value| value.to_str().ok())
        {
            scope.set_tag("request.id", request_id);
        }

        {
            let status = res
                .as_ref()
                .map(|resp| resp.status())
                .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);

            scope.set_tag("response.status", status.as_str());
        }

        scope.set_extra("Response time [ms]", response_time.into());

        if let Some(metadata) = req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                scope.set_extra(key, value.to_string().into());
            }
        }
    };

    sentry::with_scope(config, || sentry::capture_message(&message, level));
}

#[cfg(test)]
pub(crate) fn get_log_message(req: &dyn RequestExt, key: &'static str) -> String {
    // Unwrap shouldn't panic as no other code has access to the private struct to remove it
    for (k, v) in &req.extensions().find::<CustomMetadata>().unwrap().entries {
        if key == *k {
            return v.clone();
        }
    }
    panic!("expected log message for {} not found", key);
}

struct RequestLine<'r> {
    req: &'r dyn RequestExt,
    res: &'r AfterResult,
    response_time: u64,
}

impl Display for RequestLine<'_> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        let mut line = LogLine::new(f);

        let (at, status) = match self.res {
            Ok(resp) => ("info", resp.status()),
            Err(_) => ("error", StatusCode::INTERNAL_SERVER_ERROR),
        };

        line.add_field("at", at)?;
        line.add_field("method", self.req.method())?;
        line.add_quoted_field("path", FullPath(self.req))?;

        // The request_id is not logged for successful download requests
        if !(self.req.path().ends_with("/download")
            && self
                .res
                .as_ref()
                .ok()
                .map(|ok| ok.status().is_redirection())
                == Some(true))
        {
            line.add_field("request_id", request_header(self.req, "x-request-id"))?;
        }

        line.add_quoted_field("fwd", request_header(self.req, "x-real-ip"))?;
        line.add_field("service", TimeMs(self.response_time))?;
        line.add_field("status", status.as_str())?;
        line.add_quoted_field("user_agent", request_header(self.req, header::USER_AGENT))?;

        if let Some(metadata) = self.req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                line.add_quoted_field(key, value)?;
            }
        }

        if let Err(err) = self.res {
            line.add_quoted_field("error", err)?;
        }

        if self.response_time > SLOW_REQUEST_THRESHOLD_MS {
            line.add_marker("SLOW REQUEST")?;
        }

        Ok(())
    }
}

struct FullPath<'a>(&'a dyn RequestExt);

impl<'a> Display for FullPath<'a> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        // Unwrap shouldn't panic as no other code has access to the private struct to remove it
        write!(
            f,
            "{}",
            self.0.extensions().find::<OriginalPath>().unwrap().0
        )?;
        if let Some(q_string) = self.0.query_string() {
            write!(f, "?{}", q_string)?;
        }
        Ok(())
    }
}

struct TimeMs(u64);

impl Display for TimeMs {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)?;
        f.write_str("ms")?;
        Ok(())
    }
}

struct LogLine<'f, 'g> {
    f: &'f mut Formatter<'g>,
    first: bool,
}

impl<'f, 'g> LogLine<'f, 'g> {
    fn new(f: &'f mut Formatter<'g>) -> Self {
        Self { f, first: true }
    }

    fn add_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=")?;
        value.fmt(self.f)?;

        Ok(())
    }

    fn add_quoted_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=\"")?;
        value.fmt(self.f)?;
        self.f.write_str("\"")?;

        Ok(())
    }

    fn add_marker<M: Display>(&mut self, marker: M) -> fmt::Result {
        self.start_item()?;

        marker.fmt(self.f)?;

        Ok(())
    }

    fn start_item(&mut self) -> fmt::Result {
        if !self.first {
            self.f.write_str(" ")?;
        }
        self.first = false;
        Ok(())
    }
}
<commit_msg>LogRequest: Use `at=error` for all 5xx status code responses<commit_after>//! Log all requests in a format similar to Heroku's router, but with additional
//! information that we care about like User-Agent

use super::prelude::*;
use crate::util::request_header;

use conduit::{header, Host, RequestExt, Scheme, StatusCode};
use conduit_cookie::RequestSession;
use sentry::Level;

use std::fmt::{self, Display, Formatter};
use std::time::{SystemTime, UNIX_EPOCH};

const SLOW_REQUEST_THRESHOLD_MS: u64 = 1000;

const FILTERED_HEADERS: &[&str] = &["Authorization", "Cookie", "X-Real-Ip", "X-Forwarded-For"];

#[derive(Default)]
pub(super) struct LogRequests();

struct OriginalPath(String);

impl Middleware for LogRequests {
    fn before(&self, req: &mut dyn RequestExt) -> BeforeResult {
        let path = OriginalPath(req.path().to_string());
        req.mut_extensions().insert(path);
        Ok(())
    }

    fn after(&self, req: &mut dyn RequestExt, res: AfterResult) -> AfterResult {
        let response_time =
            if let Ok(start_ms) = request_header(req, "x-request-start").parse::<u128>() {
                let current_ms = SystemTime::now()
                    .duration_since(UNIX_EPOCH)
                    .expect("Time went way backwards")
                    .as_millis();

                if current_ms > start_ms {
                    // The result cannot be negative
                    current_ms - start_ms
                } else {
                    // Because our nginx proxy and app run on the same dyno in production, we
                    // shouldn't have to worry about clock drift. But if something goes wrong,
                    // calculate the response time based on when the request reached this app.
                    fallback_response_time(req)
                }
            } else {
                // X-Request-Start header couldn't be parsed.
                // We are probably running locally and not behind nginx.
                fallback_response_time(req)
            };

        // This will only trucate for requests lasting > 500 million years
        let response_time = response_time as u64;

        println!(
            "{}",
            RequestLine {
                req,
                res: &res,
                response_time,
            }
        );

        report_to_sentry(req, &res, response_time);

        res
    }
}

/// Calculate the response time based on when the request reached the in-app web server.
///
/// This serves as a fallback in case the `X-Request-Start` header is missing or invalid.
fn fallback_response_time(req: &mut dyn RequestExt) -> u128 {
    req.elapsed().as_millis()
}

struct CustomMetadata {
    entries: Vec<(&'static str, String)>,
}

pub fn add_custom_metadata<V: Display>(req: &mut dyn RequestExt, key: &'static str, value: V) {
    if let Some(metadata) = req.mut_extensions().find_mut::<CustomMetadata>() {
        metadata.entries.push((key, value.to_string()));
    } else {
        let mut metadata = CustomMetadata {
            entries: Vec::new(),
        };
        metadata.entries.push((key, value.to_string()));
        req.mut_extensions().insert(metadata);
    }
}

fn report_to_sentry(req: &dyn RequestExt, res: &AfterResult, response_time: u64) {
    let (message, level) = match res {
        Err(e) => (e.to_string(), Level::Error),
        Ok(_) => return,
    };

    let config = |scope: &mut sentry::Scope| {
        let method = Some(req.method().as_str().to_owned());

        let scheme = match req.scheme() {
            Scheme::Http => "http",
            Scheme::Https => "https",
        };

        let host = match req.host() {
            Host::Name(name) => name.to_owned(),
            Host::Socket(addr) => addr.to_string(),
        };

        let path = &req.extensions().find::<OriginalPath>().unwrap().0;

        let url = format!("{}://{}{}", scheme, host, path).parse().ok();

        {
            let id = req.session().get("user_id").map(|str| str.to_string());

            let user = sentry::User {
                id,
                ..Default::default()
            };

            scope.set_user(Some(user));
        }

        {
            let headers = req
                .headers()
                .iter()
                .filter(|(k, _v)| !FILTERED_HEADERS.iter().any(|name| k == name))
                .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default().to_string()))
                .collect();

            let sentry_req = sentry::protocol::Request {
                url,
                method,
                headers,
                ..Default::default()
            };

            scope.add_event_processor(Box::new(move |mut event| {
                if event.request.is_none() {
                    event.request = Some(sentry_req.clone());
                }
                Some(event)
            }));
        }

        if let Some(request_id) = req
            .headers()
            .get("x-request-id")
            .and_then(|value| value.to_str().ok())
        {
            scope.set_tag("request.id", request_id);
        }

        {
            let status = res
                .as_ref()
                .map(|resp| resp.status())
                .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);

            scope.set_tag("response.status", status.as_str());
        }

        scope.set_extra("Response time [ms]", response_time.into());

        if let Some(metadata) = req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                scope.set_extra(key, value.to_string().into());
            }
        }
    };

    sentry::with_scope(config, || sentry::capture_message(&message, level));
}

#[cfg(test)]
pub(crate) fn get_log_message(req: &dyn RequestExt, key: &'static str) -> String {
    // Unwrap shouldn't panic as no other code has access to the private struct to remove it
    for (k, v) in &req.extensions().find::<CustomMetadata>().unwrap().entries {
        if key == *k {
            return v.clone();
        }
    }
    panic!("expected log message for {} not found", key);
}

struct RequestLine<'r> {
    req: &'r dyn RequestExt,
    res: &'r AfterResult,
    response_time: u64,
}

impl Display for RequestLine<'_> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        let mut line = LogLine::new(f);

        let status = self.res.as_ref().map(|res| res.status());
        let status = status.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);

        let at = if status.is_server_error() {
            "error"
        } else {
            "info"
        };

        line.add_field("at", at)?;
        line.add_field("method", self.req.method())?;
        line.add_quoted_field("path", FullPath(self.req))?;

        // The request_id is not logged for successful download requests
        if !(self.req.path().ends_with("/download")
            && self
                .res
                .as_ref()
                .ok()
                .map(|ok| ok.status().is_redirection())
                == Some(true))
        {
            line.add_field("request_id", request_header(self.req, "x-request-id"))?;
        }

        line.add_quoted_field("fwd", request_header(self.req, "x-real-ip"))?;
        line.add_field("service", TimeMs(self.response_time))?;
        line.add_field("status", status.as_str())?;
        line.add_quoted_field("user_agent", request_header(self.req, header::USER_AGENT))?;

        if let Some(metadata) = self.req.extensions().find::<CustomMetadata>() {
            for (key, value) in &metadata.entries {
                line.add_quoted_field(key, value)?;
            }
        }

        if let Err(err) = self.res {
            line.add_quoted_field("error", err)?;
        }

        if self.response_time > SLOW_REQUEST_THRESHOLD_MS {
            line.add_marker("SLOW REQUEST")?;
        }

        Ok(())
    }
}

struct FullPath<'a>(&'a dyn RequestExt);

impl<'a> Display for FullPath<'a> {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        // Unwrap shouldn't panic as no other code has access to the private struct to remove it
        write!(
            f,
            "{}",
            self.0.extensions().find::<OriginalPath>().unwrap().0
        )?;
        if let Some(q_string) = self.0.query_string() {
            write!(f, "?{}", q_string)?;
        }
        Ok(())
    }
}

struct TimeMs(u64);

impl Display for TimeMs {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)?;
        f.write_str("ms")?;
        Ok(())
    }
}

struct LogLine<'f, 'g> {
    f: &'f mut Formatter<'g>,
    first: bool,
}

impl<'f, 'g> LogLine<'f, 'g> {
    fn new(f: &'f mut Formatter<'g>) -> Self {
        Self { f, first: true }
    }

    fn add_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=")?;
        value.fmt(self.f)?;

        Ok(())
    }

    fn add_quoted_field<K: Display, V: Display>(&mut self, key: K, value: V) -> fmt::Result {
        self.start_item()?;

        key.fmt(self.f)?;
        self.f.write_str("=\"")?;
        value.fmt(self.f)?;
        self.f.write_str("\"")?;

        Ok(())
    }

    fn add_marker<M: Display>(&mut self, marker: M) -> fmt::Result {
        self.start_item()?;

        marker.fmt(self.f)?;

        Ok(())
    }

    fn start_item(&mut self) -> fmt::Result {
        if !self.first {
            self.f.write_str(" ")?;
        }
        self.first = false;
        Ok(())
    }
}
<|endoftext|>


<commit_before>pub mod markdown {
    use hoedown::renderer::Render;
    use hoedown::Buffer;
    use hoedown::Markdown;

    pub type LinkTitle  = String;
    pub type LinkURL    = String;

    pub struct Link {
        pub title: LinkTitle,
        pub url: LinkURL,
    }

    struct LinkExtractRenderer {
        links : Vec<Link>
    }

    impl LinkExtractRenderer {

        fn new() -> LinkExtractRenderer {
            LinkExtractRenderer {
                links: vec![],
            }
        }

        fn extract(self) -> Vec<Link> {
            self.links
        }

    }

    impl Render for LinkExtractRenderer {

        fn link(&mut self,
                output: &mut Buffer,
                content: &Buffer,
                link: &Buffer,
                title: &Buffer) -> bool {

            let l = String::from(link.to_str().unwrap_or("<<UTF8 Error>>"));
            let t = String::from(title.to_str().unwrap_or("<<UTF8 Error>>"));

            debug!("[Markdown] Push link: '{}' -> '{}'", t, l);
            self.links.push(Link {
                title:  t,
                url:    l,
            });
            true
        }

    }

    pub struct MarkdownParser {
        text: Markdown,
    }

    impl MarkdownParser {

        pub fn new(s: &String) -> MarkdownParser {
            MarkdownParser {
                text: Markdown::new(&s[..])
            }
        }

        pub fn links(&self) -> Vec<Link> {
            let mut renderer = LinkExtractRenderer::new();
            renderer.render(&self.text);
            renderer.extract()
        }

        pub fn to_html(self) -> String {
            use hoedown::renderer::html::Html;
            use hoedown::renderer::html;

            String::from(
                Html::new(html::Flags::empty(), 0)
                    .render(&self.text)
                    .to_str()
                    .unwrap_or("UTF8Error"))
        }

    }

}
<commit_msg>Add helper to put content into  tags<commit_after>pub mod markdown {
    use hoedown::renderer::Render;
    use hoedown::Buffer;
    use hoedown::Markdown;

    pub type LinkTitle  = String;
    pub type LinkURL    = String;

    pub struct Link {
        pub title: LinkTitle,
        pub url: LinkURL,
    }

    struct LinkExtractRenderer {
        links : Vec<Link>
    }

    impl LinkExtractRenderer {

        fn new() -> LinkExtractRenderer {
            LinkExtractRenderer {
                links: vec![],
            }
        }

        fn extract(self) -> Vec<Link> {
            self.links
        }

    }

    impl Render for LinkExtractRenderer {

        fn link(&mut self,
                output: &mut Buffer,
                content: &Buffer,
                link: &Buffer,
                title: &Buffer) -> bool {

            let l = String::from(link.to_str().unwrap_or("<<UTF8 Error>>"));
            let t = String::from(title.to_str().unwrap_or("<<UTF8 Error>>"));

            debug!("[Markdown] Push link: '{}' -> '{}'", t, l);
            self.links.push(Link {
                title:  t,
                url:    l,
            });
            true
        }

    }

    pub struct MarkdownParser {
        text: Markdown,
    }

    impl MarkdownParser {

        pub fn new(s: &String) -> MarkdownParser {
            MarkdownParser {
                text: Markdown::new(&s[..])
            }
        }

        pub fn links(&self) -> Vec<Link> {
            let mut renderer = LinkExtractRenderer::new();
            renderer.render(&self.text);
            renderer.extract()
        }

        pub fn to_html(self) -> String {
            use hoedown::renderer::html::Html;
            use hoedown::renderer::html;

            String::from(
                Html::new(html::Flags::empty(), 0)
                    .render(&self.text)
                    .to_str()
                    .unwrap_or("UTF8Error"))
        }

        pub fn to_html_page(self) -> String {
            let header = "

                    <meta http-equiv='content-type' content='text/html; charset=utf-8'>

            ";
            let content = self.to_html();
            let footer = "";
            format!("{}{}{}", header, content, footer)
        }

    }

}
<|endoftext|>


<commit_before>//! The distrib module provides methods for executing the rendering in a
//! distributed environment across multiple nodes. The worker module provides
//! the Worker which does the actual job of rendering a subsection of the image.
//! The master module provides the Master which instructs the Workers and collects
//! their results.
//!
//! **Note:** At this time I do nothing for distributed fault handling.

use bincode::rustc_serialize::encoded_size;

pub use self::worker::Worker;
pub use self::master::Master;

pub mod worker;
pub mod master;

#[derive(Debug, Clone, RustcEncodable, RustcDecodable)]
struct Instructions {
    pub encoded_size: u64,
    pub scene: String,
    pub frames: (usize, usize),
    pub block_start: usize,
    pub block_count: usize,
}

impl Instructions {
    pub fn new(scene: &String, frames: (usize, usize), block_start: usize,
               block_count: usize) -> Instructions {
        let mut instr = Instructions { encoded_size: 0, scene: scene.clone(), frames: frames,
                       block_start: block_start, block_count: block_count };
        instr.encoded_size = encoded_size(&instr);
        instr
    }
}

<commit_msg>Add some more doc on fault handling issues and lack of work stealing<commit_after>//! The distrib module provides methods for executing the rendering in a
//! distributed environment across multiple nodes. The worker module provides
//! the Worker which does the actual job of rendering a subsection of the image.
//! The master module provides the Master which instructs the Workers and collects
//! their results.
//!
//! **Note:** At this time I do nothing for distributed fault handling or work
//! stealing. If a node crashes during rendering it's results will be lost and the
//! master will hang forever waiting to hear back from the crashed node.

use bincode::rustc_serialize::encoded_size;

pub use self::worker::Worker;
pub use self::master::Master;

pub mod worker;
pub mod master;

#[derive(Debug, Clone, RustcEncodable, RustcDecodable)]
struct Instructions {
    pub encoded_size: u64,
    pub scene: String,
    pub frames: (usize, usize),
    pub block_start: usize,
    pub block_count: usize,
}

impl Instructions {
    pub fn new(scene: &String, frames: (usize, usize), block_start: usize,
               block_count: usize) -> Instructions {
        let mut instr = Instructions { encoded_size: 0, scene: scene.clone(), frames: frames,
                       block_start: block_start, block_count: block_count };
        instr.encoded_size = encoded_size(&instr);
        instr
    }
}

<|endoftext|>


<commit_before>use std::io;
use std::collections::HashMap;
use directory::ReadOnlySource;
use common::BinarySerializable;
use DocId;
use schema::{Field, SchemaBuilder};
use std::path::Path;
use schema::FAST;
use directory::{WritePtr, RAMDirectory, Directory};
use fastfield::FastFieldSerializer;
use fastfield::FastFieldsWriter;
use common::bitpacker::compute_num_bits;
use common::bitpacker::BitUnpacker;
use schema::FieldType;
use common;
use owning_ref::OwningRef;

/// Trait for accessing a fastfield.
///
/// Depending on the field type, a different
/// fast field is required.
pub trait FastFieldReader: Sized {
    /// Type of the value stored in the fastfield.
    type ValueType;

    /// Return the value associated to the given document.
    ///
    /// This accessor should return as fast as possible.
    fn get(&self, doc: DocId) -> Self::ValueType;

    /// Opens a fast field given a source.
    fn open(source: ReadOnlySource) -> Self;

    /// Returns true iff the given field_type makes
    /// it possible to access the field values via a
    /// fastfield.
    fn is_enabled(field_type: &FieldType) -> bool;
}

/// `FastFieldReader` for unsigned 64-bits integers.
pub struct U64FastFieldReader {
    bit_unpacker: BitUnpacker<OwningRef<ReadOnlySource, [u8]>>,
    min_value: u64,
    max_value: u64,
}

impl U64FastFieldReader {
    /// Returns the minimum value for this fast field.
    ///
    /// The min value does not take in account of possible
    /// deleted document, and should be considered as a lower bound
    /// of the actual minimum value.
    pub fn min_value(&self) -> u64 {
        self.min_value
    }

    /// Returns the maximum value for this fast field.
    ///
    /// The max value does not take in account of possible
    /// deleted document, and should be considered as an upper bound
    /// of the actual maximum value.
    pub fn max_value(&self) -> u64 {
        self.max_value
    }
}

impl FastFieldReader for U64FastFieldReader {
    type ValueType = u64;

    fn get(&self, doc: DocId) -> u64 {
        self.min_value + self.bit_unpacker.get(doc as usize)
    }

    fn is_enabled(field_type: &FieldType) -> bool {
        match *field_type {
            FieldType::U64(ref integer_options) => integer_options.is_fast(),
            _ => false,
        }
    }

    /// Opens a new fast field reader given a read only source.
    ///
    /// # Panics
    /// Panics if the data is corrupted.
    fn open(data: ReadOnlySource) -> U64FastFieldReader {
        let min_value: u64;
        let amplitude: u64;
        {
            let mut cursor = data.as_slice();
            min_value = u64::deserialize(&mut cursor)
                .expect("Failed to read the min_value of fast field.");
            amplitude = u64::deserialize(&mut cursor)
                .expect("Failed to read the amplitude of fast field.");

        }
        let max_value = min_value + amplitude;
        let num_bits = compute_num_bits(amplitude);
        let owning_ref = OwningRef::new(data).map(|data| &data[16..]);
        let bit_unpacker = BitUnpacker::new(owning_ref, num_bits as usize);
        U64FastFieldReader {
            min_value: min_value,
            max_value: max_value,
            bit_unpacker: bit_unpacker,
        }
    }
}

impl From<Vec<u64>> for U64FastFieldReader {
    fn from(vals: Vec<u64>) -> U64FastFieldReader {
        let mut schema_builder = SchemaBuilder::default();
        let field = schema_builder.add_u64_field("field", FAST);
        let schema = schema_builder.build();
        let path = Path::new("test");
        let mut directory: RAMDirectory = RAMDirectory::create();
        {
            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
            let mut serializer = FastFieldSerializer::new(write).unwrap();
            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
            for val in vals {
                let mut fast_field_writer = fast_field_writers.get_field_writer(field).unwrap();
                fast_field_writer.add_val(val);
            }
            fast_field_writers.serialize(&mut serializer).unwrap();
            serializer.close().unwrap();
        }
        let source = directory.open_read(path).unwrap();
        let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
        fast_field_readers.open_reader(field).unwrap()
    }
}

/// `FastFieldReader` for signed 64-bits integers.
pub struct I64FastFieldReader {
    underlying: U64FastFieldReader,
}

impl I64FastFieldReader {
    /// Returns the minimum value for this fast field.
    ///
    /// The min value does not take in account of possible
    /// deleted document, and should be considered as a lower bound
    /// of the actual minimum value.
    pub fn min_value(&self) -> i64 {
        common::u64_to_i64(self.underlying.min_value())
    }

    /// Returns the maximum value for this fast field.
    ///
    /// The max value does not take in account of possible
    /// deleted document, and should be considered as an upper bound
    /// of the actual maximum value.
    pub fn max_value(&self) -> i64 {
        common::u64_to_i64(self.underlying.max_value())
    }
}

impl FastFieldReader for I64FastFieldReader {
    type ValueType = i64;

    ///
    ///
    /// # Panics
    ///
    /// May panic or return wrong random result if `doc`
    /// is greater or equal to the segment's `maxdoc`.
    fn get(&self, doc: DocId) -> i64 {
        common::u64_to_i64(self.underlying.get(doc))
    }

    /// Opens a new fast field reader given a read only source.
    ///
    /// # Panics
    /// Panics if the data is corrupted.
    fn open(data: ReadOnlySource) -> I64FastFieldReader {
        I64FastFieldReader { underlying: U64FastFieldReader::open(data) }
    }

    fn is_enabled(field_type: &FieldType) -> bool {
        match *field_type {
            FieldType::I64(ref integer_options) => integer_options.is_fast(),
            _ => false,
        }
    }
}

/// The `FastFieldsReader` is the datastructure containing
/// all of the fast fields' data.
///
/// It contains a mapping that associated these fields to
/// the proper slice in the fastfield reader file.
pub struct FastFieldsReader {
    source: ReadOnlySource,
    field_offsets: HashMap<Field, (u32, u32)>,
}

impl FastFieldsReader {
    /// Opens a `FastFieldsReader`
    ///
    /// When opening the fast field reader, the
    /// the list of the offset is read (as a footer of the
    /// data file).
    pub fn from_source(source: ReadOnlySource) -> io::Result<FastFieldsReader> {
        let header_offset;
        let field_offsets: Vec<(Field, u32)>;
        {
            let buffer = source.as_slice();
            {
                let mut cursor = buffer;
                header_offset = u32::deserialize(&mut cursor)?;
            }
            {
                let mut cursor = &buffer[header_offset as usize..];
                field_offsets = Vec::deserialize(&mut cursor)?;
            }
        }
        let mut end_offsets: Vec<u32> = field_offsets.iter().map(|&(_, offset)| offset).collect();
        end_offsets.push(header_offset);
        let mut field_offsets_map: HashMap<Field, (u32, u32)> = HashMap::new();
        for (field_start_offsets, stop_offset) in
            field_offsets.iter().zip(end_offsets.iter().skip(1)) {
            let (field, start_offset) = *field_start_offsets;
            field_offsets_map.insert(field, (start_offset, *stop_offset));
        }
        Ok(FastFieldsReader {
               field_offsets: field_offsets_map,
               source: source,
           })
    }

    /// Returns the u64 fast value reader if the field
    /// is a u64 field indexed as "fast".
    ///
    /// Return None if the field is not a u64 field
    /// indexed with the fast option.
    ///
    /// # Panics
    /// May panic if the index is corrupted.
    pub fn open_reader<FFReader: FastFieldReader>(&self, field: Field) -> Option<FFReader> {
        self.field_offsets
            .get(&field)
            .map(|&(start, stop)| {
                     let field_source = self.source.slice(start as usize, stop as usize);
                     FFReader::open(field_source)
                 })
    }
}

unsafe impl Send for U64FastFieldReader {}
unsafe impl Sync for U64FastFieldReader {}
unsafe impl Send for I64FastFieldReader {}
unsafe impl Sync for I64FastFieldReader {}
<commit_msg>Send and Sync impl now useless<commit_after>use std::io;
use std::collections::HashMap;
use directory::ReadOnlySource;
use common::BinarySerializable;
use DocId;
use schema::{Field, SchemaBuilder};
use std::path::Path;
use schema::FAST;
use directory::{WritePtr, RAMDirectory, Directory};
use fastfield::FastFieldSerializer;
use fastfield::FastFieldsWriter;
use common::bitpacker::compute_num_bits;
use common::bitpacker::BitUnpacker;
use schema::FieldType;
use common;
use owning_ref::OwningRef;

/// Trait for accessing a fastfield.
///
/// Depending on the field type, a different
/// fast field is required.
pub trait FastFieldReader: Sized {
    /// Type of the value stored in the fastfield.
    type ValueType;

    /// Return the value associated to the given document.
    ///
    /// This accessor should return as fast as possible.
    fn get(&self, doc: DocId) -> Self::ValueType;

    /// Opens a fast field given a source.
    fn open(source: ReadOnlySource) -> Self;

    /// Returns true iff the given field_type makes
    /// it possible to access the field values via a
    /// fastfield.
    fn is_enabled(field_type: &FieldType) -> bool;
}

/// `FastFieldReader` for unsigned 64-bits integers.
pub struct U64FastFieldReader {
    bit_unpacker: BitUnpacker<OwningRef<ReadOnlySource, [u8]>>,
    min_value: u64,
    max_value: u64,
}

impl U64FastFieldReader {
    /// Returns the minimum value for this fast field.
    ///
    /// The min value does not take in account of possible
    /// deleted document, and should be considered as a lower bound
    /// of the actual minimum value.
    pub fn min_value(&self) -> u64 {
        self.min_value
    }

    /// Returns the maximum value for this fast field.
    ///
    /// The max value does not take in account of possible
    /// deleted document, and should be considered as an upper bound
    /// of the actual maximum value.
    pub fn max_value(&self) -> u64 {
        self.max_value
    }
}

impl FastFieldReader for U64FastFieldReader {
    type ValueType = u64;

    fn get(&self, doc: DocId) -> u64 {
        self.min_value + self.bit_unpacker.get(doc as usize)
    }

    fn is_enabled(field_type: &FieldType) -> bool {
        match *field_type {
            FieldType::U64(ref integer_options) => integer_options.is_fast(),
            _ => false,
        }
    }

    /// Opens a new fast field reader given a read only source.
    ///
    /// # Panics
    /// Panics if the data is corrupted.
    fn open(data: ReadOnlySource) -> U64FastFieldReader {
        let min_value: u64;
        let amplitude: u64;
        {
            let mut cursor = data.as_slice();
            min_value = u64::deserialize(&mut cursor)
                .expect("Failed to read the min_value of fast field.");
            amplitude = u64::deserialize(&mut cursor)
                .expect("Failed to read the amplitude of fast field.");

        }
        let max_value = min_value + amplitude;
        let num_bits = compute_num_bits(amplitude);
        let owning_ref = OwningRef::new(data).map(|data| &data[16..]);
        let bit_unpacker = BitUnpacker::new(owning_ref, num_bits as usize);
        U64FastFieldReader {
            min_value: min_value,
            max_value: max_value,
            bit_unpacker: bit_unpacker,
        }
    }
}

impl From<Vec<u64>> for U64FastFieldReader {
    fn from(vals: Vec<u64>) -> U64FastFieldReader {
        let mut schema_builder = SchemaBuilder::default();
        let field = schema_builder.add_u64_field("field", FAST);
        let schema = schema_builder.build();
        let path = Path::new("test");
        let mut directory: RAMDirectory = RAMDirectory::create();
        {
            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
            let mut serializer = FastFieldSerializer::new(write).unwrap();
            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
            for val in vals {
                let mut fast_field_writer = fast_field_writers.get_field_writer(field).unwrap();
                fast_field_writer.add_val(val);
            }
            fast_field_writers.serialize(&mut serializer).unwrap();
            serializer.close().unwrap();
        }
        let source = directory.open_read(path).unwrap();
        let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
        fast_field_readers.open_reader(field).unwrap()
    }
}

/// `FastFieldReader` for signed 64-bits integers.
pub struct I64FastFieldReader {
    underlying: U64FastFieldReader,
}

impl I64FastFieldReader {
    /// Returns the minimum value for this fast field.
    ///
    /// The min value does not take in account of possible
    /// deleted document, and should be considered as a lower bound
    /// of the actual minimum value.
    pub fn min_value(&self) -> i64 {
        common::u64_to_i64(self.underlying.min_value())
    }

    /// Returns the maximum value for this fast field.
    ///
    /// The max value does not take in account of possible
    /// deleted document, and should be considered as an upper bound
    /// of the actual maximum value.
    pub fn max_value(&self) -> i64 {
        common::u64_to_i64(self.underlying.max_value())
    }
}

impl FastFieldReader for I64FastFieldReader {
    type ValueType = i64;

    ///
    ///
    /// # Panics
    ///
    /// May panic or return wrong random result if `doc`
    /// is greater or equal to the segment's `maxdoc`.
    fn get(&self, doc: DocId) -> i64 {
        common::u64_to_i64(self.underlying.get(doc))
    }

    /// Opens a new fast field reader given a read only source.
    ///
    /// # Panics
    /// Panics if the data is corrupted.
    fn open(data: ReadOnlySource) -> I64FastFieldReader {
        I64FastFieldReader { underlying: U64FastFieldReader::open(data) }
    }

    fn is_enabled(field_type: &FieldType) -> bool {
        match *field_type {
            FieldType::I64(ref integer_options) => integer_options.is_fast(),
            _ => false,
        }
    }
}

/// The `FastFieldsReader` is the datastructure containing
/// all of the fast fields' data.
///
/// It contains a mapping that associated these fields to
/// the proper slice in the fastfield reader file.
pub struct FastFieldsReader {
    source: ReadOnlySource,
    field_offsets: HashMap<Field, (u32, u32)>,
}

impl FastFieldsReader {
    /// Opens a `FastFieldsReader`
    ///
    /// When opening the fast field reader, the
    /// the list of the offset is read (as a footer of the
    /// data file).
    pub fn from_source(source: ReadOnlySource) -> io::Result<FastFieldsReader> {
        let header_offset;
        let field_offsets: Vec<(Field, u32)>;
        {
            let buffer = source.as_slice();
            {
                let mut cursor = buffer;
                header_offset = u32::deserialize(&mut cursor)?;
            }
            {
                let mut cursor = &buffer[header_offset as usize..];
                field_offsets = Vec::deserialize(&mut cursor)?;
            }
        }
        let mut end_offsets: Vec<u32> = field_offsets.iter().map(|&(_, offset)| offset).collect();
        end_offsets.push(header_offset);
        let mut field_offsets_map: HashMap<Field, (u32, u32)> = HashMap::new();
        for (field_start_offsets, stop_offset) in
            field_offsets.iter().zip(end_offsets.iter().skip(1)) {
            let (field, start_offset) = *field_start_offsets;
            field_offsets_map.insert(field, (start_offset, *stop_offset));
        }
        Ok(FastFieldsReader {
               field_offsets: field_offsets_map,
               source: source,
           })
    }

    /// Returns the u64 fast value reader if the field
    /// is a u64 field indexed as "fast".
    ///
    /// Return None if the field is not a u64 field
    /// indexed with the fast option.
    ///
    /// # Panics
    /// May panic if the index is corrupted.
    pub fn open_reader<FFReader: FastFieldReader>(&self, field: Field) -> Option<FFReader> {
        self.field_offsets
            .get(&field)
            .map(|&(start, stop)| {
                     let field_source = self.source.slice(start as usize, stop as usize);
                     FFReader::open(field_source)
                 })
    }
}
<|endoftext|>


<commit_before>// TODO: COPYRIGHT, USE & AUTHORS

use super::ga_core::GASolution;

use std::cmp::Ordering;
use std::iter::FromIterator;

// Better name than 'Basis'?
pub enum GAPopulationSortBasis
{
    Raw,
    Scaled,
}

// Isn't a boolean enough?
pub enum GAPopulationSortOrder
{
    LowIsBest,
    HighIsBest,
}

/// Genetic Algorithm Population
// TODO: RUST DOCS!
pub struct GAPopulation<T: GASolution>
{
    population: Vec<T>,

    sort_order: GAPopulationSortOrder,

    // 'population' ordered by Raw score.
    population_order_raw: Vec<usize>,
    // Is 'population_order_raw' sorted?
    is_raw_sorted: bool,

    // 'population' ordered by Scaled score.
    population_order_scaled: Vec<usize>,
    // Is 'population_scaled' sorted?
    is_scaled_sorted: bool,

    // We keep 2 lists of indexes to the population vector.
    // One sorted by Raw Score and one by Scaled Score (Fitness).
}
impl<T: GASolution> GAPopulation<T>
{
    // Need best() and sort() for Selectors.

    // TODO: New should use some parameters, maybe a Config
    pub fn new(p: Vec<T>, order: GAPopulationSortOrder) -> GAPopulation<T>
    {
        let gap = GAPopulation
                  {
                      population: p,
                      sort_order: order,
                      population_order_raw: vec![],
                      is_raw_sorted: false,
                      population_order_scaled: vec![],
                      is_scaled_sorted: false
                  };

        gap
    }

    pub fn population(&mut self) -> &mut Vec<T>
    {
        return &mut self.population
    }

    pub fn evaluate(&mut self)
    {
        for ref mut ind in &mut self.population
        {
            ind.evaluate();
        }
    }

    pub fn size(&self) -> usize
    {
        self.population.len()
    }

    //TODO: this is a temporary implementation
    pub fn select(&self) -> &T
    {
        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn best(&self) -> &T
    {
        // TODO: Call GAPopulation.scale().

        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn worst(&self) -> &T
    {
        self.individual(self.size()-1, GAPopulationSortBasis::Scaled)
    }

    pub fn individual(&self, i : usize, sort_basis : GAPopulationSortBasis) -> &T
    {
        // TODO: Check that i makes sense
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            => { &self.population[self.population_order_raw[i]] },
            GAPopulationSortBasis::Scaled
            => { &self.population[self.population_order_scaled[i]] },
        }
    }

    pub fn sort(&mut self)
    {
        self.sort_int(false, GAPopulationSortBasis::Scaled);
        self.sort_int(false, GAPopulationSortBasis::Raw);
    }

    //TODO: I hate this name
    pub fn sort_int(&mut self, force_sort: bool, sort_basis: GAPopulationSortBasis)
    {
        let mut ordered : Vec<usize> = Vec::from_iter(0..self.size());
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            =>  if (!self.is_raw_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].score()
                                                .partial_cmp(&self.population[*s2].score()).unwrap_or(Ordering::Equal));

                        },
                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].score()
                                                .partial_cmp(&self.population[*s1].score()).unwrap_or(Ordering::Equal));

                        },
                    };
                    self.population_order_raw = ordered;
                    self.is_raw_sorted = true;
                },

            GAPopulationSortBasis::Scaled
            =>  if (!self.is_scaled_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].fitness()
                                                .partial_cmp(&self.population[*s2].fitness()).unwrap_or(Ordering::Equal));
                        },

                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].fitness()
                                                .partial_cmp(&self.population[*s1].fitness()).unwrap_or(Ordering::Equal));
                        }
                    };
                    self.population_order_scaled = ordered;
                    self.is_scaled_sorted = true;
                },
        };
    }
}

#[cfg(test)]
mod test
{
}
<commit_msg>Remove last push's comments.<commit_after>// TODO: COPYRIGHT, USE & AUTHORS

use super::ga_core::GASolution;

use std::cmp::Ordering;
use std::iter::FromIterator;

// Better name than 'Basis'?
pub enum GAPopulationSortBasis
{
    Raw,
    Scaled,
}

// Isn't a boolean enough?
pub enum GAPopulationSortOrder
{
    LowIsBest,
    HighIsBest,
}

/// Genetic Algorithm Population
// TODO: RUST DOCS!
pub struct GAPopulation<T: GASolution>
{
    population: Vec<T>,

    sort_order: GAPopulationSortOrder,

    // 'population' ordered by Raw score.
    population_order_raw: Vec<usize>,
    // Is 'population_order_raw' sorted?
    is_raw_sorted: bool,

    // 'population' ordered by Scaled score.
    population_order_scaled: Vec<usize>,
    // Is 'population_scaled' sorted?
    is_scaled_sorted: bool,

    // We keep 2 lists of indexes to the population vector.
    // One sorted by Raw Score and one by Scaled Score (Fitness).
}
impl<T: GASolution> GAPopulation<T>
{
    // TODO: New should use some parameters, maybe a Config
    pub fn new(p: Vec<T>, order: GAPopulationSortOrder) -> GAPopulation<T>
    {
        let gap = GAPopulation
                  {
                      population: p,
                      sort_order: order,
                      population_order_raw: vec![],
                      is_raw_sorted: false,
                      population_order_scaled: vec![],
                      is_scaled_sorted: false
                  };

        gap
    }

    pub fn population(&mut self) -> &mut Vec<T>
    {
        return &mut self.population
    }

    pub fn evaluate(&mut self)
    {
        for ref mut ind in &mut self.population
        {
            ind.evaluate();
        }
    }

    pub fn size(&self) -> usize
    {
        self.population.len()
    }

    //TODO: this is a temporary implementation
    pub fn select(&self) -> &T
    {
        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn best(&self) -> &T
    {
        // TODO: Call GAPopulation.scale().

        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn worst(&self) -> &T
    {
        self.individual(self.size()-1, GAPopulationSortBasis::Scaled)
    }

    pub fn individual(&self, i : usize, sort_basis : GAPopulationSortBasis) -> &T
    {
        // TODO: Check that i makes sense
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            => { &self.population[self.population_order_raw[i]] },
            GAPopulationSortBasis::Scaled
            => { &self.population[self.population_order_scaled[i]] },
        }
    }

    pub fn sort(&mut self)
    {
        self.sort_int(false, GAPopulationSortBasis::Scaled);
        self.sort_int(false, GAPopulationSortBasis::Raw);
    }

    //TODO: I hate this name
    pub fn sort_int(&mut self, force_sort: bool, sort_basis: GAPopulationSortBasis)
    {
        let mut ordered : Vec<usize> = Vec::from_iter(0..self.size());
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            =>  if (!self.is_raw_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].score()
                                                .partial_cmp(&self.population[*s2].score()).unwrap_or(Ordering::Equal));

                        },
                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].score()
                                                .partial_cmp(&self.population[*s1].score()).unwrap_or(Ordering::Equal));

                        },
                    };
                    self.population_order_raw = ordered;
                    self.is_raw_sorted = true;
                },

            GAPopulationSortBasis::Scaled
            =>  if (!self.is_scaled_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].fitness()
                                                .partial_cmp(&self.population[*s2].fitness()).unwrap_or(Ordering::Equal));
                        },

                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].fitness()
                                                .partial_cmp(&self.population[*s1].fitness()).unwrap_or(Ordering::Equal));
                        }
                    };
                    self.population_order_scaled = ordered;
                    self.is_scaled_sorted = true;
                },
        };
    }
}

#[cfg(test)]
mod test
{
}
<|endoftext|>


<commit_before>// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
// Licensed under the Apache License, Version 2.0 (see LICENSE).

use std::collections::HashMap;
use std::hash;
use std::sync::atomic;

use cpython::{ObjectProtocol, PyErr, PyType, Python, ToPyObject};
use parking_lot::{Mutex, RwLock};

use crate::core::{Fnv, Key, Value};
use crate::externs;

///
/// A struct that encapsulates interning of python `Value`s as comparable `Key`s.
///
/// To minimize the total amount of time spent in python code comparing objects (represented on
/// the rust side of the FFI boundary as `Value` instances) to one another, this API supports
/// memoizing `Value`s as `Key`s.
///
/// Creating a `Key` involves interning a `Value` under a (private) `InternKey` struct which
/// implements `Hash` and `Eq` using the precomputed python `__hash__` for the `Value` and
/// delegating to python's `__eq__`, respectively.
///
/// Currently `Value`s are interned indefinitely as `Key`s, meaning that they can never
/// be collected: it's possible that this can eventually be improved by either:
///
///   1) switching to directly linking-against or embedding python, such that the `Value`
///      type goes away in favor of direct usage of a python object wrapper struct.
///   2) This structure might begin storing weak-references to `Key`s and/or `Value`s, which
///      would allow the associated `Value` handles to be dropped when they were no longer used.
///      The challenge to this approach is that it would make it more difficult to pass
///      `Key`/`Value` instances across the FFI boundary.
///   3) `Value` could implement `Eq`/`Hash` directly via extern calls to python (although we've
///      avoided doing this so far because it would hide a relatively expensive operation behind
///      those usually-inexpensive traits).
///
/// To avoid deadlocks, methods of Interns require that the GIL is held, and then explicitly release
/// it before acquiring inner locks. That way we can guarantee that these locks are always acquired
/// before the GIL (Value equality in particular might re-acquire it).
///
#[derive(Default)]
pub struct Interns {
  forward_keys: Mutex<HashMap<InternKey, Key, Fnv>>,
  reverse_keys: RwLock<HashMap<Key, Value, Fnv>>,
  id_generator: atomic::AtomicU64,
}

impl Interns {
  pub fn new() -> Interns {
    Interns::default()
  }

  pub fn key_insert(&self, py: Python, v: Value) -> Result<Key, PyErr> {
    let (intern_key, type_id) = {
      let obj = v.to_py_object(py).into();
      (InternKey(v.hash(py)?, obj), (&v.get_type(py)).into())
    };

    py.allow_threads(|| {
      let mut forward_keys = self.forward_keys.lock();
      let key = if let Some(key) = forward_keys.get(&intern_key) {
        *key
      } else {
        let id = self.id_generator.fetch_add(1, atomic::Ordering::Relaxed);
        let key = Key::new(id, type_id);
        self.reverse_keys.write().insert(key, v);
        forward_keys.insert(intern_key, key);
        key
      };
      Ok(key)
    })
  }

  pub fn key_get(&self, k: &Key) -> Value {
    // NB: We do not need to acquire+release the GIL before getting a Value for a Key, because
    // neither `Key::eq` nor `Value::clone` acquire the GIL.
    self
      .reverse_keys
      .read()
      .get(&k)
      .cloned()
      .unwrap_or_else(|| {
        panic!(
          "Previously memoized object disappeared for Key {{ id: {}, type_id: {} }}!",
          k.id(),
          k.type_id()
        )
      })
  }
}

struct InternKey(isize, Value);

impl Eq for InternKey {}

impl PartialEq for InternKey {
  fn eq(&self, other: &InternKey) -> bool {
    externs::equals(&self.1, &other.1)
  }
}

impl hash::Hash for InternKey {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}

struct InternType(isize, PyType);

impl Eq for InternType {}

impl PartialEq for InternType {
  fn eq(&self, other: &InternType) -> bool {
    self.1 == other.1
  }
}

impl hash::Hash for InternType {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}
<commit_msg>Explain Interns::key_get Key Debug sensitivity. (#12167)<commit_after>// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
// Licensed under the Apache License, Version 2.0 (see LICENSE).

use std::collections::HashMap;
use std::hash;
use std::sync::atomic;

use cpython::{ObjectProtocol, PyErr, PyType, Python, ToPyObject};
use parking_lot::{Mutex, RwLock};

use crate::core::{Fnv, Key, Value};
use crate::externs;

///
/// A struct that encapsulates interning of python `Value`s as comparable `Key`s.
///
/// To minimize the total amount of time spent in python code comparing objects (represented on
/// the rust side of the FFI boundary as `Value` instances) to one another, this API supports
/// memoizing `Value`s as `Key`s.
///
/// Creating a `Key` involves interning a `Value` under a (private) `InternKey` struct which
/// implements `Hash` and `Eq` using the precomputed python `__hash__` for the `Value` and
/// delegating to python's `__eq__`, respectively.
///
/// Currently `Value`s are interned indefinitely as `Key`s, meaning that they can never
/// be collected: it's possible that this can eventually be improved by either:
///
///   1) switching to directly linking-against or embedding python, such that the `Value`
///      type goes away in favor of direct usage of a python object wrapper struct.
///   2) This structure might begin storing weak-references to `Key`s and/or `Value`s, which
///      would allow the associated `Value` handles to be dropped when they were no longer used.
///      The challenge to this approach is that it would make it more difficult to pass
///      `Key`/`Value` instances across the FFI boundary.
///   3) `Value` could implement `Eq`/`Hash` directly via extern calls to python (although we've
///      avoided doing this so far because it would hide a relatively expensive operation behind
///      those usually-inexpensive traits).
///
/// To avoid deadlocks, methods of Interns require that the GIL is held, and then explicitly release
/// it before acquiring inner locks. That way we can guarantee that these locks are always acquired
/// before the GIL (Value equality in particular might re-acquire it).
///
#[derive(Default)]
pub struct Interns {
  forward_keys: Mutex<HashMap<InternKey, Key, Fnv>>,
  reverse_keys: RwLock<HashMap<Key, Value, Fnv>>,
  id_generator: atomic::AtomicU64,
}

impl Interns {
  pub fn new() -> Interns {
    Interns::default()
  }

  pub fn key_insert(&self, py: Python, v: Value) -> Result<Key, PyErr> {
    let (intern_key, type_id) = {
      let obj = v.to_py_object(py).into();
      (InternKey(v.hash(py)?, obj), (&v.get_type(py)).into())
    };

    py.allow_threads(|| {
      let mut forward_keys = self.forward_keys.lock();
      let key = if let Some(key) = forward_keys.get(&intern_key) {
        *key
      } else {
        let id = self.id_generator.fetch_add(1, atomic::Ordering::Relaxed);
        let key = Key::new(id, type_id);
        self.reverse_keys.write().insert(key, v);
        forward_keys.insert(intern_key, key);
        key
      };
      Ok(key)
    })
  }

  pub fn key_get(&self, k: &Key) -> Value {
    // NB: We do not need to acquire+release the GIL before getting a Value for a Key, because
    // neither `Key::eq` nor `Value::clone` acquire the GIL.
    self
      .reverse_keys
      .read()
      .get(&k)
      .cloned()
      .unwrap_or_else(|| {
        // N.B.: This panic is effectively an assertion that `Key::new` is only ever called above in
        // `key_insert` under an exclusive lock where it is then inserted in `reverse_keys` before
        // exiting the lock and being returned to the caller. This ensures that all `Key`s in the
        // wild _must_ be in `reverse_keys`. As such, the code involved in generating the panic
        // message should be immaterial and never fire. If it does fire though, then the assertion
        // was proven incorrect and the `Key` is not, in fact, in `reverse_keys`. Since the `Debug`
        // impl for `Key` currently uses this very method to render the `Key` we avoid using the
        // debug formatting for `Key` to avoid generating a panic while panicking.
        panic!(
          "Previously memoized object disappeared for Key {{ id: {}, type_id: {} }}!",
          k.id(),
          k.type_id()
        )
      })
  }
}

struct InternKey(isize, Value);

impl Eq for InternKey {}

impl PartialEq for InternKey {
  fn eq(&self, other: &InternKey) -> bool {
    externs::equals(&self.1, &other.1)
  }
}

impl hash::Hash for InternKey {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}

struct InternType(isize, PyType);

impl Eq for InternType {}

impl PartialEq for InternType {
  fn eq(&self, other: &InternType) -> bool {
    self.1 == other.1
  }
}

impl hash::Hash for InternType {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}
<|endoftext|>


<commit_before>use super::common::{replace_whitespace, tags, whitespace, INVALID_TITLE_CHARS};
use nom::bytes::complete::take_while;
use nom::sequence::tuple;
use nom::IResult;
use std::fs::DirEntry;
use std::path::Path;

#[inline]
pub fn closest_match<I, S>(name: S, min_confidence: f32, items: I) -> Option<DirEntry>
where
    I: Iterator<Item = DirEntry>,
    S: Into<String>,
{
    let mut name = name.into();
    name.make_ascii_lowercase();

    crate::closest_match(items, min_confidence, |dir| {
        let mut dir_name = parse_title(dir.file_name())?;
        dir_name.make_ascii_lowercase();

        Some(strsim::jaro(&dir_name, &name) as f32)
    })
    .map(|(_, dir)| dir)
}

#[inline]
pub fn parse_title<S>(dir: S) -> Option<String>
where
    S: AsRef<Path>,
{
    let dir = dir.as_ref();
    let dir_name = dir.file_name()?.to_string_lossy();

    let (_, (_, _, parsed)) = tuple((tags, whitespace, title))(&dir_name).ok()?;
    let parsed = replace_whitespace(parsed);

    Some(parsed)
}

fn title(input: &str) -> IResult<&str, &str> {
    take_while(|ch| !INVALID_TITLE_CHARS.contains(&(ch as u8)))(input)
}
<commit_msg>anime: Add nickname generation function<commit_after>use super::common::{replace_whitespace, tags, whitespace, INVALID_TITLE_CHARS};
use nom::bytes::complete::take_while;
use nom::sequence::tuple;
use smallvec::SmallVec;
use std::fs::DirEntry;
use std::path::Path;

#[inline]
pub fn closest_match<I, S>(name: S, min_confidence: f32, items: I) -> Option<DirEntry>
where
    I: Iterator<Item = DirEntry>,
    S: Into<String>,
{
    let mut name = name.into();
    name.make_ascii_lowercase();

    crate::closest_match(items, min_confidence, |dir| {
        let mut dir_name = parse_title(dir.file_name())?;
        dir_name.make_ascii_lowercase();

        Some(strsim::jaro(&dir_name, &name) as f32)
    })
    .map(|(_, dir)| dir)
}

#[inline]
pub fn parse_title<S>(dir: S) -> Option<String>
where
    S: AsRef<Path>,
{
    let dir = dir.as_ref();
    let dir_name = dir.file_name()?.to_string_lossy();

    let title = take_while(|ch| !INVALID_TITLE_CHARS.contains(&(ch as u8)));
    let (_, (_, _, parsed)) = tuple((tags, whitespace, title))(&dir_name).ok()?;
    let parsed = replace_whitespace(parsed);

    Some(parsed)
}

/// Attempts to generate a short and readable nickname for the given folder title.
///
/// A folder title can be obtained from the `parse_title` function.
pub fn generate_nickname<S>(title: S) -> Option<String>
where
    S: Into<String>,
{
    const SPACER: &str = "_";
    const TITLE_WHITESPACE: [u8; 4] = [b' ', b'_', b'.', b'-'];
    const SKIP_WORDS: [&str; 1] = ["the"];
    const SPECIAL_WORDS: [&str; 4] = ["special", "ova", "ona", "movie"];

    let is_special_word = |word: &str| {
        SPECIAL_WORDS
            .iter()
            .any(|special| word.starts_with(special))
    };

    let title = {
        let mut title = title.into();
        title.make_ascii_lowercase();
        title
    };

    let fragments = title
        .split(|ch| TITLE_WHITESPACE.contains(&(ch as u8)))
        .collect::<SmallVec<[_; 8]>>();

    let mut nickname: SmallVec<[&str; 4]> = SmallVec::new();

    let (fragments, end_fragment) = match fragments.last() {
        Some(last) if is_special_word(last) => (&fragments[..fragments.len() - 1], Some(*last)),
        Some(last) => {
            let end_fragment = parse_season_number(last);

            let fragments = if end_fragment.is_some() {
                &fragments[..fragments.len() - 1]
            } else {
                &fragments
            };

            (fragments, end_fragment)
        }
        None => return None,
    };

    let mut used_frags = 0;

    for fragment in fragments {
        let len = fragment.len();

        if len <= 2 || SKIP_WORDS.contains(fragment) {
            continue;
        }

        nickname.push(fragment);

        if len > 8 {
            break;
        }

        used_frags += 1;

        if used_frags >= 2 {
            break;
        }
    }

    if nickname.is_empty() {
        return None;
    }

    if let Some(end) = end_fragment {
        nickname.push(end);
    }

    Some(nickname.join(SPACER))
}

fn parse_season_number(slice: &str) -> Option<&str> {
    let is_digits = |digits: &[u8]| digits.iter().all(u8::is_ascii_digit);

    let offset = match slice.as_bytes() {
        [b's', b'0', b'1', ..] | [b's', b'1', ..] | [] => None,
        [b's', b'0', ..] => Some(2),
        [b's', rest @ ..] if is_digits(rest) => Some(1),
        rest if is_digits(rest) => Some(0),
        _ => None,
    };

    offset.map(|offset| &slice[offset..])
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn nickname_generation() {
        let titles = vec![
            ("series title", Some("series_title")),
            ("longer series title test", Some("longer_series")),
            ("the series title", Some("series_title")),
            ("title of series", Some("title_series")),
            ("longfirstword of series", Some("longfirstword")),
            ("longfirstword S02", Some("longfirstword_2")),
            ("title longsecondword test", Some("title_longsecondword")),
            ("title test longthirdword", Some("title_test")),
            ("series title 2", Some("series_title_2")),
            ("longer series title 2", Some("longer_series_2")),
            ("longer series title s02", Some("longer_series_2")),
            ("series title s01", Some("series_title")),
            ("Yuragi-sou no Yuuna-san OVA", Some("yuragi_sou_ova")),
            ("Kaguya-sama wa wa Kokurasetai S2", Some("kaguya_sama_2")),
            ("series title OVAs", Some("series_title_ovas")),
            ("test s02", Some("test_2")),
            ("test s2", Some("test_2")),
            ("s.m.o.l S02", None),
            ("s.m.o.l OVA", None),
            ("s2", None),
        ];

        for (title, expected) in titles {
            assert_eq!(
                generate_nickname(title).as_deref(),
                expected,
                "nickname mismatch for title: {}",
                title
            );
        }
    }
}
<|endoftext|>


<commit_before>pub use protocol::{self, Encode, Decode, Type, ThriftTyped};
pub use {Protocol, Transport, Result, Error};

pub use std::collections::{HashSet, HashMap};
pub use std::hash::Hash;

impl ThriftTyped for bool { fn typ() -> Type { Type::Bool } }
impl ThriftTyped for i8  { fn typ() -> Type { Type::Byte } }
impl ThriftTyped for i16 { fn typ() -> Type { Type::I16 } }
impl ThriftTyped for i32 { fn typ() -> Type { Type::I32 } }
impl ThriftTyped for i64 { fn typ() -> Type { Type::I64 } }
impl ThriftTyped for f64 { fn typ() -> Type { Type::Double } }
impl ThriftTyped for () { fn typ() -> Type { Type::Void } }
impl ThriftTyped for String { fn typ() -> Type { Type::String } }
impl<T: ThriftTyped> ThriftTyped for Vec<T> { fn typ() -> Type { Type::List } }
impl<T: ThriftTyped> ThriftTyped for Option<T> { fn typ() -> Type { T::typ() } }
impl<T: ThriftTyped> ThriftTyped for HashSet<T> { fn typ() -> Type { Type::Set } }
impl<K: ThriftTyped, V: ThriftTyped> ThriftTyped for HashMap<K, V> { fn typ() -> Type { Type::Map } }

impl<X: Encode> Encode for Vec<X> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_list_begin(transport, X::typ(), self.len()));

        for el in self {
            try!(el.encode(protocol, transport));
        }

        try!(protocol.write_list_end(transport));

        Ok(())
    }
}

impl<X: Encode + Hash + Eq> Encode for HashSet<X> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_set_begin(transport, X::typ(), self.len()));

        for el in self {
            try!(el.encode(protocol, transport));
        }

        try!(protocol.write_set_end(transport));

        Ok(())
    }
}

impl<K: Encode + Hash + Eq, V: Encode> Encode for HashMap<K, V> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_map_begin(transport, K::typ(), V::typ(), self.len()));

        for (k, v) in self.iter() {
            try!(k.encode(protocol, transport));
            try!(v.encode(protocol, transport));
        }

        try!(protocol.write_map_end(transport));

        Ok(())
    }
}

impl<X: Encode> Encode for Option<X> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        self.as_ref().map(|this| this.encode(protocol, transport)).unwrap_or(Ok(()))
    }
}

impl Encode for String {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_string(transport, self));
        Ok(())
    }
}

impl Encode for () {
    fn encode<P, T>(&self, _: &mut P, _: &mut T) -> Result<()>
    where P: Protocol, T: Transport { Ok(()) }
}

macro_rules! prim_encode {
    ($($T:ty => $method:ident),*) => {
        $(impl Encode for $T {
            fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
            where P: Protocol, T: Transport {
                try!(protocol.$method(transport, *self));
                Ok(())
            }
        })*
    }
}

prim_encode! {
    bool => write_bool, i8 => write_byte, i16 => write_i16,
    i32 => write_i32, i64 => write_i64, f64 => write_double
}

fn decode<D, P, T>(protocol: &mut P, transport: &mut T) -> Result<D>
where D: Decode, P: Protocol, T: Transport {
     let mut elem = D::default();
     try!(elem.decode(protocol, transport));
     Ok(elem)
}

impl<X: Decode> Decode for Vec<X> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let (typ, len) = try!(protocol.read_list_begin(transport));

        if typ == X::typ() {
            self.reserve(len as usize);
            for _ in 0..len { self.push(try!(decode(protocol, transport))); }
            try!(protocol.read_list_end(transport));
            Ok(())
        } else {
            Err(Error::from(protocol::Error::ProtocolViolation))
        }
    }
}

impl<X: Decode + Eq + Hash> Decode for HashSet<X> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let (typ, len) = try!(protocol.read_set_begin(transport));

        if typ == X::typ() {
            self.reserve(len as usize);
            for _ in 0..len { self.insert(try!(decode(protocol, transport))); }
            try!(protocol.read_set_end(transport));
            Ok(())
        } else {
            Err(Error::from(protocol::Error::ProtocolViolation))
        }
    }
}

impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let (ktyp, vtyp, len) = try!(protocol.read_map_begin(transport));

        if ktyp == K::typ() && vtyp == V::typ() {
            self.reserve(len as usize);
            for _ in 0..len {
                let key = try!(decode(protocol, transport));
                let value = try!(decode(protocol, transport));
                self.insert(key, value);
            }

            try!(protocol.read_map_end(transport));
            Ok(())
        } else {
            Err(Error::from(protocol::Error::ProtocolViolation))
        }
    }
}

impl<X: Decode> Decode for Option<X> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let mut this = X::default();
        try!(this.decode(protocol, transport));
        *self = Some(this);
        Ok(())
    }
}

impl Decode for () {
    fn decode<P, T>(&mut self, _: &mut P, _: &mut T) -> Result<()>
    where P: Protocol, T: Transport { Ok(()) }
}

macro_rules! prim_decode {
    ($($T:ty => $method:ident),*) => {
        $(impl Decode for $T {
            fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
            where P: Protocol, T: Transport {
                *self = try!(protocol.$method(transport));
                Ok(())
            }
        })*
    }
}

prim_decode! {
    bool => read_bool, i8 => read_byte, i16 => read_i16,
    i32 => read_i32, i64 => read_i64, f64 => read_double,
    String => read_string
}

<commit_msg>Add support for binary type<commit_after>pub use protocol::{self, Encode, Decode, Type, ThriftTyped};
pub use {Protocol, Transport, Result, Error};

pub use std::collections::{HashSet, HashMap};
pub use std::hash::Hash;

impl ThriftTyped for bool { fn typ() -> Type { Type::Bool } }
impl ThriftTyped for i8  { fn typ() -> Type { Type::Byte } }
impl ThriftTyped for i16 { fn typ() -> Type { Type::I16 } }
impl ThriftTyped for i32 { fn typ() -> Type { Type::I32 } }
impl ThriftTyped for i64 { fn typ() -> Type { Type::I64 } }
impl ThriftTyped for f64 { fn typ() -> Type { Type::Double } }
impl ThriftTyped for () { fn typ() -> Type { Type::Void } }
impl ThriftTyped for String { fn typ() -> Type { Type::String } }
impl ThriftTyped for Vec<u8> { fn typ() -> Type { Type::String } }
impl<T: ThriftTyped> ThriftTyped for Vec<T> { fn typ() -> Type { Type::List } }
impl<T: ThriftTyped> ThriftTyped for Option<T> { fn typ() -> Type { T::typ() } }
impl<T: ThriftTyped> ThriftTyped for HashSet<T> { fn typ() -> Type { Type::Set } }
impl<K: ThriftTyped, V: ThriftTyped> ThriftTyped for HashMap<K, V> { fn typ() -> Type { Type::Map } }

impl<X: Encode> Encode for Vec<X> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_list_begin(transport, X::typ(), self.len()));

        for el in self {
            try!(el.encode(protocol, transport));
        }

        try!(protocol.write_list_end(transport));

        Ok(())
    }
}

impl<X: Encode + Hash + Eq> Encode for HashSet<X> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_set_begin(transport, X::typ(), self.len()));

        for el in self {
            try!(el.encode(protocol, transport));
        }

        try!(protocol.write_set_end(transport));

        Ok(())
    }
}

impl<K: Encode + Hash + Eq, V: Encode> Encode for HashMap<K, V> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_map_begin(transport, K::typ(), V::typ(), self.len()));

        for (k, v) in self.iter() {
            try!(k.encode(protocol, transport));
            try!(v.encode(protocol, transport));
        }

        try!(protocol.write_map_end(transport));

        Ok(())
    }
}

impl<X: Encode> Encode for Option<X> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        self.as_ref().map(|this| this.encode(protocol, transport)).unwrap_or(Ok(()))
    }
}

impl Encode for String {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_string(transport, self));
        Ok(())
    }
}

impl Encode for Vec<u8> {
    fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        try!(protocol.write_binary(transport, self));
        Ok(())
    }
}

impl Encode for () {
    fn encode<P, T>(&self, _: &mut P, _: &mut T) -> Result<()>
    where P: Protocol, T: Transport { Ok(()) }
}

macro_rules! prim_encode {
    ($($T:ty => $method:ident),*) => {
        $(impl Encode for $T {
            fn encode<P, T>(&self, protocol: &mut P, transport: &mut T) -> Result<()>
            where P: Protocol, T: Transport {
                try!(protocol.$method(transport, *self));
                Ok(())
            }
        })*
    }
}

prim_encode! {
    bool => write_bool, i8 => write_byte, i16 => write_i16,
    i32 => write_i32, i64 => write_i64, f64 => write_double
}

fn decode<D, P, T>(protocol: &mut P, transport: &mut T) -> Result<D>
where D: Decode, P: Protocol, T: Transport {
     let mut elem = D::default();
     try!(elem.decode(protocol, transport));
     Ok(elem)
}

impl<X: Decode> Decode for Vec<X> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let (typ, len) = try!(protocol.read_list_begin(transport));

        if typ == X::typ() {
            self.reserve(len as usize);
            for _ in 0..len { self.push(try!(decode(protocol, transport))); }
            try!(protocol.read_list_end(transport));
            Ok(())
        } else {
            Err(Error::from(protocol::Error::ProtocolViolation))
        }
    }
}

impl<X: Decode + Eq + Hash> Decode for HashSet<X> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let (typ, len) = try!(protocol.read_set_begin(transport));

        if typ == X::typ() {
            self.reserve(len as usize);
            for _ in 0..len { self.insert(try!(decode(protocol, transport))); }
            try!(protocol.read_set_end(transport));
            Ok(())
        } else {
            Err(Error::from(protocol::Error::ProtocolViolation))
        }
    }
}

impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let (ktyp, vtyp, len) = try!(protocol.read_map_begin(transport));

        if ktyp == K::typ() && vtyp == V::typ() {
            self.reserve(len as usize);
            for _ in 0..len {
                let key = try!(decode(protocol, transport));
                let value = try!(decode(protocol, transport));
                self.insert(key, value);
            }

            try!(protocol.read_map_end(transport));
            Ok(())
        } else {
            Err(Error::from(protocol::Error::ProtocolViolation))
        }
    }
}

impl<X: Decode> Decode for Option<X> {
    fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
    where P: Protocol, T: Transport {
        let mut this = X::default();
        try!(this.decode(protocol, transport));
        *self = Some(this);
        Ok(())
    }
}

impl Decode for () {
    fn decode<P, T>(&mut self, _: &mut P, _: &mut T) -> Result<()>
    where P: Protocol, T: Transport { Ok(()) }
}

macro_rules! prim_decode {
    ($($T:ty => $method:ident),*) => {
        $(impl Decode for $T {
            fn decode<P, T>(&mut self, protocol: &mut P, transport: &mut T) -> Result<()>
            where P: Protocol, T: Transport {
                *self = try!(protocol.$method(transport));
                Ok(())
            }
        })*
    }
}

prim_decode! {
    bool => read_bool, i8 => read_byte, i16 => read_i16,
    i32 => read_i32, i64 => read_i64, f64 => read_double,
    String => read_string,
    Vec<u8> => read_binary
}

<|endoftext|>


<commit_before>#![feature(test)]

extern crate mime;
extern crate test;

use mime::*;
use test::Bencher;

#[bench]
fn bench_eq_parsed_atom(b: &mut Bencher) {
    let mime = "text/plain; charset=utf-8".parse::<MediaType>().unwrap();
    b.bytes = mime.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime, TEXT_PLAIN_UTF_8);
    })
}

#[bench]
fn bench_eq_parsed_dynamic(b: &mut Bencher) {
    let mime1 = "text/foo; charset=utf-8".parse::<MediaType>().unwrap();
    let mime2 =  mime1.clone();
    b.bytes = mime1.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime1, mime2);
    })
}

#[bench]
fn bench_eq_consts(b: &mut Bencher) {
    let mime = TEXT_PLAIN_UTF_8;
    b.bytes = mime.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime, TEXT_PLAIN_UTF_8);
    });
}

#[cfg(feature = "macro")]
#[bench]
fn bench_eq_proc_macro(b: &mut Bencher) {
    let mime = media_type!("text/plain; charset=utf-8");
    b.bytes = mime.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime, TEXT_PLAIN_UTF_8);
    });
}

#[bench]
fn bench_ne_consts(b: &mut Bencher) {
    let one = TEXT_XML;
    let two = TEXT_CSS;
    b.bytes = one.as_ref().len() as u64;
    b.iter(|| {
        assert_ne!(one, two);
    });
}

#[bench]
fn bench_eq_type_(b: &mut Bencher) {
    let mime = TEXT_PLAIN_UTF_8;
    let name = TEXT;
    b.bytes = name.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime.type_(), name);
    });
}
<commit_msg>fix inference error in cmp benchmarks<commit_after>#![feature(test)]

extern crate mime;
extern crate test;

use mime::*;
use test::Bencher;

#[bench]
fn bench_eq_parsed_atom(b: &mut Bencher) {
    let mime = "text/plain; charset=utf-8".parse::<MediaType>().unwrap();
    b.bytes = mime.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime, TEXT_PLAIN_UTF_8);
    })
}

#[bench]
fn bench_eq_parsed_dynamic(b: &mut Bencher) {
    let mime1 = "text/foo; charset=utf-8".parse::<MediaType>().unwrap();
    let mime2 =  mime1.clone();
    b.bytes = mime1.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime1, mime2);
    })
}

#[bench]
fn bench_eq_consts(b: &mut Bencher) {
    let mime = TEXT_PLAIN_UTF_8;
    b.bytes = mime.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime, TEXT_PLAIN_UTF_8);
    });
}

#[cfg(feature = "macro")]
#[bench]
fn bench_eq_proc_macro(b: &mut Bencher) {
    let mime = media_type!("text/plain; charset=utf-8");
    b.bytes = mime.as_ref().len() as u64;
    b.iter(|| {
        assert_eq!(mime, TEXT_PLAIN_UTF_8);
    });
}

#[bench]
fn bench_ne_consts(b: &mut Bencher) {
    let one = TEXT_XML;
    let two = TEXT_CSS;
    b.bytes = one.as_ref().len() as u64;
    b.iter(|| {
        assert_ne!(one, two);
    });
}

#[bench]
fn bench_eq_type_(b: &mut Bencher) {
    let mime = TEXT_PLAIN_UTF_8;
    let name = TEXT;
    b.bytes = name.len() as u64;
    b.iter(|| {
        assert_eq!(mime.type_(), name);
    });
}
<|endoftext|>


<commit_before>//! Opening files with DirectIO and managing DirectIO buffers
extern crate std;
extern crate libc;

use libc::{c_void};

use std::num::{Int, SignedInt};
use std::path::Path;
use std::io::{FileMode, FileAccess};
use std::io::{IoResult, IoError, Open, Append, Truncate, Read, Write, ReadWrite};
use std::os::unix::{AsRawFd, Fd};

use super::FD;
use aligned::AlignedBuf;

pub struct DirectFile(FD, uint);

const O_DIRECT: i32 = 0x4000;   // Linux

#[inline]
fn retry<T: SignedInt> (f: || -> T) -> T {
    let one: T = Int::one();
    loop {
        let n = f();
        if n == -one && std::os::errno() == libc::EINTR as uint { }
        else { return n }
    }
}

impl DirectFile {
    // XXX auto-query directio alignment
    pub fn open(path: &Path, fm: FileMode, fa: FileAccess, alignment: uint) -> IoResult<DirectFile> {
        let flags = O_DIRECT | match fm {
            Open => 0,
            Append => libc::O_APPEND,
            Truncate => libc::O_TRUNC,
        };
        // Opening with a write permission must silently create the file.
        let (flags, mode) = match fa {
            Read => (flags | libc::O_RDONLY, 0),
            Write => (flags | libc::O_WRONLY | libc::O_CREAT,
                      libc::S_IRUSR | libc::S_IWUSR),
            ReadWrite => (flags | libc::O_RDWR | libc::O_CREAT,
                          libc::S_IRUSR | libc::S_IWUSR),
        };

        let path = path.to_c_str();
        match retry(|| unsafe { libc::open(path.as_ptr(), flags, mode) }) {
            -1 => Err(IoError::last_error()),
            fd => Ok(DirectFile(FD(fd), alignment)),
        }
    }

    pub fn alignment(&self) -> uint { let DirectFile(_, a) = *self; a }

    pub fn pread(&self, buf: &mut AlignedBuf, off: u64) -> IoResult<uint> {
        let DirectFile(fd, _) = *self;
        let r = unsafe { ::libc::pread(fd.as_raw_fd(), buf.as_mut_ptr() as *mut c_void, buf.len() as u64, off as i64) };

        if r < 0 {
            Err(IoError::last_error())
        } else {
            Ok(r as uint)
        }
    }

    pub fn pwrite(&self, buf: &AlignedBuf, off: u64) -> IoResult<uint> {
        let DirectFile(fd, _) = *self;
        let r = unsafe { ::libc::pwrite(fd.as_raw_fd(), buf.as_slice().as_ptr() as *const c_void, buf.len() as u64, off as i64) };

        if r < 0 {
            Err(IoError::last_error())
        } else {
            Ok(r as uint)
        }
    }
}

impl AsRawFd for DirectFile {
    fn as_raw_fd(&self) -> Fd {
        let DirectFile(fd, _) = *self;
        fd.as_raw_fd()
    }
}

#[cfg(test)]
mod test {
    use std::io::{TempDir, Truncate, ReadWrite};
    use std::path::Path;
    use super::DirectFile;
    use aligned::AlignedBuf;

    fn tmpfile(name: &str) -> DirectFile {
        let tmp = TempDir::new_in(&Path::new("."), "test").unwrap();
        let mut path = tmp.path().clone();

        path.push(name);
        match DirectFile::open(&path, Truncate, ReadWrite, 4096) {
            Err(e) => panic!("open failed {}", e),
            Ok(f) => f
        }
    }

    #[test]
    fn simple() {
        let file = tmpfile("direct");
        let data = match AlignedBuf::from_slice(['x' as u8, ..4096].as_slice(), 4096) {
            None => panic!("buf alloc"),
            Some(b) => b
        };

        match file.pwrite(&data, 0) {
            Ok(_) => (),
            Err(e) => panic!("write error {}", e)
        }
    }
}
<commit_msg>fix directfile<commit_after>//! Opening files with DirectIO and managing DirectIO buffers
extern crate std;
extern crate libc;

use libc::{c_void};

use std::num::{Int, SignedInt};
use std::path::Path;
use std::io::{FileMode, FileAccess};
use std::io::{IoResult, IoError, Open, Append, Truncate, Read, Write, ReadWrite};
use std::os::unix::{AsRawFd, Fd};

use super::FD;
use aligned::AlignedBuf;

pub struct DirectFile {
    fd: FD,
    alignment: uint,
}

const O_DIRECT: i32 = 0x4000;   // Linux

#[inline]
fn retry<T: SignedInt> (f: || -> T) -> T {
    let one: T = Int::one();
    loop {
        let n = f();
        if n == -one && std::os::errno() == libc::EINTR as uint { }
        else { return n }
    }
}

impl DirectFile {
    // XXX auto-query directio alignment
    pub fn open(path: &Path, fm: FileMode, fa: FileAccess, alignment: uint) -> IoResult<DirectFile> {
        let flags = O_DIRECT | match fm {
            Open => 0,
            Append => libc::O_APPEND,
            Truncate => libc::O_TRUNC,
        };
        // Opening with a write permission must silently create the file.
        let (flags, mode) = match fa {
            Read => (flags | libc::O_RDONLY, 0),
            Write => (flags | libc::O_WRONLY | libc::O_CREAT,
                      libc::S_IRUSR | libc::S_IWUSR),
            ReadWrite => (flags | libc::O_RDWR | libc::O_CREAT,
                          libc::S_IRUSR | libc::S_IWUSR),
        };

        let path = path.to_c_str();
        match retry(|| unsafe { libc::open(path.as_ptr(), flags, mode) }) {
            -1 => Err(IoError::last_error()),
            fd => Ok(DirectFile { fd: FD(fd), alignment: alignment }),
        }
    }

    pub fn alignment(&self) -> uint { self.alignment }

    pub fn pread(&self, buf: &mut AlignedBuf, off: u64) -> IoResult<uint> {
        let r = unsafe { ::libc::pread(self.fd.as_raw_fd(), buf.as_mut_ptr() as *mut c_void, buf.len() as u64, off as i64) };

        if r < 0 {
            Err(IoError::last_error())
        } else {
            Ok(r as uint)
        }
    }

    pub fn pwrite(&self, buf: &AlignedBuf, off: u64) -> IoResult<uint> {
        let r = unsafe { ::libc::pwrite(self.fd.as_raw_fd(), buf.as_slice().as_ptr() as *const c_void, buf.len() as u64, off as i64) };

        if r < 0 {
            Err(IoError::last_error())
        } else {
            Ok(r as uint)
        }
    }
}

impl AsRawFd for DirectFile {
    fn as_raw_fd(&self) -> Fd { self.fd.as_raw_fd() }
}

#[cfg(test)]
mod test {
    use std::io::{TempDir, Truncate, ReadWrite};
    use std::path::Path;
    use super::DirectFile;
    use aligned::AlignedBuf;

    fn tmpfile(name: &str) -> DirectFile {
        let tmp = TempDir::new_in(&Path::new("."), "test").unwrap();
        let mut path = tmp.path().clone();

        path.push(name);
        match DirectFile::open(&path, Truncate, ReadWrite, 4096) {
            Err(e) => panic!("open failed {}", e),
            Ok(f) => f
        }
    }

    #[test]
    fn simple() {
        let file = tmpfile("direct");
        let data = match AlignedBuf::from_slice(['x' as u8, ..4096].as_slice(), 4096) {
            None => panic!("buf alloc"),
            Some(b) => b
        };

        match file.pwrite(&data, 0) {
            Ok(_) => (),
            Err(e) => panic!("write error {}", e)
        }
    }
}
<|endoftext|>