<commit_before>//! # Span
//!
//! `Span`s represent a single operation within a trace. `Span`s can be nested to form a trace
//! tree. Each trace contains a root span, which typically describes the end-to-end latency and,
//! optionally, one or more sub-spans for its sub-operations.
//!
//! The `Span`'s start and end timestamps reflect the elapsed real time of the operation. A `Span`'s
//! start time is set to the current time on span creation. After the `Span` is created, it
//! is possible to change its name, set its `Attributes`, and add `Links` and `Events`.
//! These cannot be changed after the `Span`'s end time has been set.
use crate::trace::{Event, SpanContext, SpanId, StatusCode, TraceId, TraceState};
use crate::{exporter, sdk, KeyValue};
use std::sync::{Arc, Mutex};
use std::time::SystemTime;

/// Single operation within a trace.
#[derive(Clone, Debug)]
pub struct Span {
    id: SpanId,
    inner: Arc<SpanInner>,
}

/// Inner data, processed and exported on drop
#[derive(Debug)]
struct SpanInner {
    data: Option<Mutex<Option<exporter::trace::SpanData>>>,
    tracer: sdk::trace::Tracer,
}

impl Span {
    pub(crate) fn new(
        id: SpanId,
        data: Option<exporter::trace::SpanData>,
        tracer: sdk::trace::Tracer,
    ) -> Self {
        Span {
            id,
            inner: Arc::new(SpanInner {
                data: data.map(|data| Mutex::new(Some(data))),
                tracer,
            }),
        }
    }

    /// Operate on reference to span inner
    fn with_data<T, F>(&self, f: F) -> Option<T>
    where
        F: FnOnce(&exporter::trace::SpanData) -> T,
    {
        self.inner.data.as_ref().and_then(|inner| {
            inner
                .lock()
                .ok()
                .and_then(|span_data| span_data.as_ref().map(f))
        })
    }

    /// Operate on mutable reference to span inner
    fn with_data_mut<T, F>(&self, f: F) -> Option<T>
    where
        F: FnOnce(&mut exporter::trace::SpanData) -> T,
    {
        self.inner.data.as_ref().and_then(|inner| {
            inner
                .lock()
                .ok()
                .and_then(|mut span_data| span_data.as_mut().map(f))
        })
    }
}

impl crate::trace::Span for Span {
    /// Records events at a specific time in the context of a given `Span`.
    ///
    /// Note that the OpenTelemetry project documents certain ["standard event names and
    /// keys"](https://github.com/open-telemetry/opentelemetry-specification/tree/v0.5.0/specification/trace/semantic_conventions/README.md)
    /// which have prescribed semantic meanings.
    fn add_event_with_timestamp(
        &self,
        name: String,
        timestamp: SystemTime,
        attributes: Vec<KeyValue>,
    ) {
        self.with_data_mut(|data| {
            data.message_events
                .push_back(Event::new(name, timestamp, attributes))
        });
    }

    /// Returns the `SpanContext` for the given `Span`.
    fn span_context(&self) -> SpanContext {
        self.with_data(|data| data.span_context.clone())
            .unwrap_or_else(|| {
                SpanContext::new(
                    TraceId::invalid(),
                    SpanId::invalid(),
                    0,
                    false,
                    TraceState::default(),
                )
            })
    }

    /// Returns true if this `Span` is recording information like events with the `add_event`
    /// operation, attributes using `set_attributes`, status with `set_status`, etc.
    fn is_recording(&self) -> bool {
        self.inner.data.is_some()
    }

    /// Sets a single `Attribute` where the attribute properties are passed as arguments.
    ///
    /// Note that the OpenTelemetry project documents certain ["standard
    /// attributes"](https://github.com/open-telemetry/opentelemetry-specification/tree/v0.5.0/specification/trace/semantic_conventions/README.md)
    /// that have prescribed semantic meanings.
    fn set_attribute(&self, attribute: KeyValue) {
        self.with_data_mut(|data| {
            data.attributes.insert(attribute);
        });
    }

    /// Sets the status of the `Span`. If used, this will override the default `Span`
    /// status, which is `OK`.
    fn set_status(&self, code: StatusCode, message: String) {
        self.with_data_mut(|data| {
            data.status_code = code;
            data.status_message = message
        });
    }

    /// Updates the `Span`'s name.
    fn update_name(&self, new_name: String) {
        self.with_data_mut(|data| {
            data.name = new_name;
        });
    }

    /// Finishes the span with given timestamp.
    fn end_with_timestamp(&self, timestamp: SystemTime) {
        self.with_data_mut(|data| {
            data.end_time = timestamp;
        });
    }
}

impl Drop for SpanInner {
    /// Report span on inner drop
    fn drop(&mut self) {
        if let Some(data) = self.data.take() {
            if let Ok(mut span_data) = data.lock().map(|mut data| data.take()) {
                if let Some(provider) = self.tracer.provider() {
                    // Set end time if unset or invalid
                    if let Some(data) = span_data.as_mut() {
                        if data.end_time <= data.start_time {
                            data.end_time = SystemTime::now();
                        }
                    }
                    let mut processors = provider.span_processors().iter().peekable();
                    while let Some(processor) = processors.next() {
                        let span_data = if processors.peek().is_none() {
                            // last loop or single processor/exporter, move data
                            span_data.take()
                        } else {
                            // clone so each exporter gets owned data
                            span_data.clone()
                        };

                        if let Some(span_data) = span_data {
                            processor.on_end(span_data);
                        }
                    }
                }
            }
        }
    }
}
<commit_msg>Add basic tests for SDK Span (#309)<commit_after>//! # Span
//!
//! `Span`s represent a single operation within a trace. `Span`s can be nested to form a trace
//! tree. Each trace contains a root span, which typically describes the end-to-end latency and,
//! optionally, one or more sub-spans for its sub-operations.
//!
//! The `Span`'s start and end timestamps reflect the elapsed real time of the operation. A `Span`'s
//! start time is set to the current time on span creation. After the `Span` is created, it
//! is possible to change its name, set its `Attributes`, and add `Links` and `Events`.
//! These cannot be changed after the `Span`'s end time has been set.
use crate::trace::{Event, SpanContext, SpanId, StatusCode, TraceId, TraceState};
use crate::{exporter::trace::SpanData, sdk, KeyValue};
use std::sync::{Arc, Mutex};
use std::time::SystemTime;

/// Single operation within a trace.
#[derive(Clone, Debug)]
pub struct Span {
    id: SpanId,
    inner: Arc<SpanInner>,
}

/// Inner data, processed and exported on drop
#[derive(Debug)]
struct SpanInner {
    data: Option<Mutex<Option<SpanData>>>,
    tracer: sdk::trace::Tracer,
}

impl Span {
    pub(crate) fn new(id: SpanId, data: Option<SpanData>, tracer: sdk::trace::Tracer) -> Self {
        Span {
            id,
            inner: Arc::new(SpanInner {
                data: data.map(|data| Mutex::new(Some(data))),
                tracer,
            }),
        }
    }

    /// Operate on reference to span inner
    fn with_data<T, F>(&self, f: F) -> Option<T>
    where
        F: FnOnce(&SpanData) -> T,
    {
        self.inner.data.as_ref().and_then(|inner| {
            inner
                .lock()
                .ok()
                .and_then(|span_data| span_data.as_ref().map(f))
        })
    }

    /// Operate on mutable reference to span inner
    fn with_data_mut<T, F>(&self, f: F) -> Option<T>
    where
        F: FnOnce(&mut SpanData) -> T,
    {
        self.inner.data.as_ref().and_then(|inner| {
            inner
                .lock()
                .ok()
                .and_then(|mut span_data| span_data.as_mut().map(f))
        })
    }
}

impl crate::trace::Span for Span {
    /// Records events at a specific time in the context of a given `Span`.
    ///
    /// Note that the OpenTelemetry project documents certain ["standard event names and
    /// keys"](https://github.com/open-telemetry/opentelemetry-specification/tree/v0.5.0/specification/trace/semantic_conventions/README.md)
    /// which have prescribed semantic meanings.
    fn add_event_with_timestamp(
        &self,
        name: String,
        timestamp: SystemTime,
        attributes: Vec<KeyValue>,
    ) {
        self.with_data_mut(|data| {
            data.message_events
                .push_back(Event::new(name, timestamp, attributes))
        });
    }

    /// Returns the `SpanContext` for the given `Span`.
    fn span_context(&self) -> SpanContext {
        self.with_data(|data| data.span_context.clone())
            .unwrap_or_else(|| {
                SpanContext::new(
                    TraceId::invalid(),
                    SpanId::invalid(),
                    0,
                    false,
                    TraceState::default(),
                )
            })
    }

    /// Returns true if this `Span` is recording information like events with the `add_event`
    /// operation, attributes using `set_attributes`, status with `set_status`, etc.
    fn is_recording(&self) -> bool {
        self.inner.data.is_some()
    }

    /// Sets a single `Attribute` where the attribute properties are passed as arguments.
    ///
    /// Note that the OpenTelemetry project documents certain ["standard
    /// attributes"](https://github.com/open-telemetry/opentelemetry-specification/tree/v0.5.0/specification/trace/semantic_conventions/README.md)
    /// that have prescribed semantic meanings.
    fn set_attribute(&self, attribute: KeyValue) {
        self.with_data_mut(|data| {
            data.attributes.insert(attribute);
        });
    }

    /// Sets the status of the `Span`. If used, this will override the default `Span`
    /// status, which is `OK`.
    fn set_status(&self, code: StatusCode, message: String) {
        self.with_data_mut(|data| {
            data.status_code = code;
            data.status_message = message
        });
    }

    /// Updates the `Span`'s name.
    fn update_name(&self, new_name: String) {
        self.with_data_mut(|data| {
            data.name = new_name;
        });
    }

    /// Finishes the span with given timestamp.
    fn end_with_timestamp(&self, timestamp: SystemTime) {
        self.with_data_mut(|data| {
            data.end_time = timestamp;
        });
    }
}

impl Drop for SpanInner {
    /// Report span on inner drop
    fn drop(&mut self) {
        if let Some(data) = self.data.take() {
            if let Ok(mut span_data) = data.lock().map(|mut data| data.take()) {
                if let Some(provider) = self.tracer.provider() {
                    // Set end time if unset or invalid
                    if let Some(data) = span_data.as_mut() {
                        if data.end_time <= data.start_time {
                            data.end_time = SystemTime::now();
                        }
                    }
                    let mut processors = provider.span_processors().iter().peekable();
                    while let Some(processor) = processors.next() {
                        let span_data = if processors.peek().is_none() {
                            // last loop or single processor/exporter, move data
                            span_data.take()
                        } else {
                            // clone so each exporter gets owned data
                            span_data.clone()
                        };

                        if let Some(span_data) = span_data {
                            processor.on_end(span_data);
                        }
                    }
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{api, api::core::KeyValue, api::trace::Span as _, api::trace::TracerProvider};
    use std::time::Duration;

    fn init() -> (sdk::trace::Tracer, SpanData) {
        let provider = sdk::trace::TracerProvider::default();
        let config = provider.config();
        let tracer = provider.get_tracer("opentelemetry", Some(env!("CARGO_PKG_VERSION")));
        let data = SpanData {
            span_context: SpanContext::new(
                TraceId::from_u128(0),
                SpanId::from_u64(0),
                api::trace::TRACE_FLAG_NOT_SAMPLED,
                false,
                TraceState::default(),
            ),
            parent_span_id: SpanId::from_u64(0),
            span_kind: api::trace::SpanKind::Internal,
            name: "opentelemetry".to_string(),
            start_time: SystemTime::now(),
            end_time: SystemTime::now(),
            attributes: sdk::trace::EvictedHashMap::new(config.max_attributes_per_span),
            message_events: sdk::trace::EvictedQueue::new(config.max_events_per_span),
            links: sdk::trace::EvictedQueue::new(config.max_links_per_span),
            status_code: StatusCode::OK,
            status_message: "".to_string(),
            resource: config.resource.clone(),
            instrumentation_lib: *tracer.instrumentation_library(),
        };
        (tracer, data)
    }

    fn create_span() -> Span {
        let (tracer, data) = init();
        Span::new(SpanId::from_u64(0), Some(data), tracer)
    }

    #[test]
    fn create_span_without_data() {
        let (tracer, _) = init();
        let span = Span::new(SpanId::from_u64(0), None, tracer);
        span.with_data(|_data| panic!("there are data"));
    }

    #[test]
    fn create_span_with_data() {
        let (tracer, data) = init();
        let span = Span::new(SpanId::from_u64(0), Some(data.clone()), tracer);
        span.with_data(|d| assert_eq!(*d, data));
    }

    #[test]
    fn add_event() {
        let span = create_span();
        let name = "some_event".to_string();
        let attributes = vec![KeyValue::new("k", "v")];
        span.add_event(name.clone(), attributes.clone());
        span.with_data(|data| {
            if let Some(event) = data.message_events.iter().next() {
                assert_eq!(event.name, name);
                assert_eq!(event.attributes, attributes);
            } else {
                panic!("no event");
            }
        });
    }

    #[test]
    fn add_event_with_timestamp() {
        let span = create_span();
        let name = "some_event".to_string();
        let attributes = vec![KeyValue::new("k", "v")];
        let timestamp = SystemTime::now();
        span.add_event_with_timestamp(name.clone(), timestamp, attributes.clone());
        span.with_data(|data| {
            if let Some(event) = data.message_events.iter().next() {
                assert_eq!(event.timestamp, timestamp);
                assert_eq!(event.name, name);
                assert_eq!(event.attributes, attributes);
            } else {
                panic!("no event");
            }
        });
    }

    #[test]
    fn record_exception() {
        let span = create_span();
        let err = std::io::Error::from(std::io::ErrorKind::Other);
        span.record_exception(&err);
        span.with_data(|data| {
            if let Some(event) = data.message_events.iter().next() {
                assert_eq!(event.name, "exception");
                assert_eq!(
                    event.attributes,
                    vec![KeyValue::new("exception.message", err.to_string())]
                );
            } else {
                panic!("no event");
            }
        });
    }

    #[test]
    fn record_exception_with_stacktrace() {
        let span = create_span();
        let err = std::io::Error::from(std::io::ErrorKind::Other);
        let stacktrace = "stacktrace...".to_string();
        span.record_exception_with_stacktrace(&err, stacktrace.clone());
        span.with_data(|data| {
            if let Some(event) = data.message_events.iter().next() {
                assert_eq!(event.name, "exception");
                assert_eq!(
                    event.attributes,
                    vec![
                        KeyValue::new("exception.message", err.to_string()),
                        KeyValue::new("exception.stacktrace", stacktrace),
                    ]
                );
            } else {
                panic!("no event");
            }
        });
    }

    #[test]
    fn set_attribute() {
        let span = create_span();
        let attributes = KeyValue::new("k", "v");
        span.set_attribute(attributes.clone());
        span.with_data(|data| {
            if let Some(val) = data.attributes.get(&attributes.key) {
                assert_eq!(*val, attributes.value);
            } else {
                panic!("no attribute");
            }
        });
    }

    #[test]
    fn set_status() {
        let span = create_span();
        let status = StatusCode::Unknown;
        let message = "UNKNOWN".to_string();
        span.set_status(status.clone(), message.clone());
        span.with_data(|data| {
            assert_eq!(data.status_code, status);
            assert_eq!(data.status_message, message);
        });
    }

    #[test]
    fn update_name() {
        let span = create_span();
        let name = "new_name".to_string();
        span.update_name(name.clone());
        span.with_data(|data| {
            assert_eq!(data.name, name);
        });
    }

    #[test]
    fn end() {
        let span = create_span();
        span.end();
    }

    #[test]
    fn end_with_timestamp() {
        let span = create_span();
        let timestamp = SystemTime::now();
        span.end_with_timestamp(timestamp);
        span.with_data(|data| assert_eq!(data.end_time, timestamp));
    }

    #[test]
    #[ignore = "not yet implemented"]
    fn end_only_once() {
        let span = create_span();
        let timestamp = SystemTime::now();
        span.end_with_timestamp(timestamp);
        span.end_with_timestamp(timestamp.checked_add(Duration::from_secs(10)).unwrap());
        span.with_data(|data| assert_eq!(data.end_time, timestamp));
    }

    #[test]
    #[ignore = "not yet implemented"]
    fn noop_after_end() {
        let span = create_span();
        let initial = span.with_data(|data| data.clone()).unwrap();
        span.end();
        span.add_event("some_event".to_string(), vec![KeyValue::new("k", "v")]);
        span.add_event_with_timestamp(
            "some_event".to_string(),
            SystemTime::now(),
            vec![KeyValue::new("k", "v")],
        );
        let err = std::io::Error::from(std::io::ErrorKind::Other);
        span.record_exception(&err);
        span.record_exception_with_stacktrace(&err, "stacktrace...".to_string());
        span.set_attribute(KeyValue::new("k", "v"));
        span.set_status(StatusCode::Unknown, "UNKNOWN".to_string());
        span.update_name("new_name".to_string());
        span.with_data(|data| {
            assert_eq!(data.message_events, initial.message_events);
            assert_eq!(data.attributes, initial.attributes);
            assert_eq!(data.status_code, initial.status_code);
            assert_eq!(data.status_message, initial.status_message);
            assert_eq!(data.name, initial.name);
        });
    }

    #[test]
    fn is_recording_true_when_not_ended() {
        let span = create_span();
        assert!(span.is_recording());
    }

    #[test]
    #[ignore = "not yet implemented"]
    fn is_recording_false_after_end() {
        let span = create_span();
        span.end();
        assert!(!span.is_recording());
    }
}
<|endoftext|>


<commit_before>use percy_dom::{html, IterableNodes, View, VirtualNode};
use time::OffsetDateTime;

use crate::site::{SiteSection, SubmissionCounts};
use crate::template::{site_footer::SiteFooter, site_header::SiteHeader};

pub mod chip;
pub mod console_submission_list;
pub mod dmg_submission_list;
pub mod markdown;
pub mod markdown_page;
pub mod raw_html;
pub mod site_footer;
pub mod site_header;

pub fn page(
    title: &str,
    section: SiteSection,
    content: VirtualNode,
    counts: &SubmissionCounts,
) -> String {
    let today = OffsetDateTime::now_local()
        .unwrap_or_else(|_| OffsetDateTime::now_utc())
        .date();
    let cartridge_submission_count = counts.cartridges;
    let console_submission_count = counts.consoles.values().sum();

    let content = html! {

        <meta charSet="utf-8" />
        <meta name="viewport" content="width=device-width, initial-scale=1" />
        <title>{title}</title>
        <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:400,700" />
        <link rel="stylesheet" href="/static/gbhwdb.css" />
        <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
        <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png" />
        <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png" />
        <link rel="manifest" href="/site.webmanifest" />
        <link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5" />
        <meta name="msapplication-TileColor" content="#2b5797" />
        <meta name="theme-color" content="#ffffff" />

        <SiteHeader section={section} />

{content}

        { SiteFooter { today, console_submission_count, cartridge_submission_count }.render() }

    }
    .to_string();
    format!("<!DOCTYPE html>{content}")
}
<commit_msg>Fix page titles<commit_after>use percy_dom::{html, IterableNodes, View, VirtualNode};
use time::OffsetDateTime;

use crate::site::{SiteSection, SubmissionCounts};
use crate::template::{site_footer::SiteFooter, site_header::SiteHeader};

pub mod chip;
pub mod console_submission_list;
pub mod dmg_submission_list;
pub mod markdown;
pub mod markdown_page;
pub mod raw_html;
pub mod site_footer;
pub mod site_header;

pub fn page(
    title: &str,
    section: SiteSection,
    content: VirtualNode,
    counts: &SubmissionCounts,
) -> String {
    let today = OffsetDateTime::now_local()
        .unwrap_or_else(|_| OffsetDateTime::now_utc())
        .date();
    let cartridge_submission_count = counts.cartridges;
    let console_submission_count = counts.consoles.values().sum();

    let content = html! {

        <meta charSet="utf-8" />
        <meta name="viewport" content="width=device-width, initial-scale=1" />
        <title>{format!("{title} - Game Boy hardware database")}</title>
        <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:400,700" />
        <link rel="stylesheet" href="/static/gbhwdb.css" />
        <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
        <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png" />
        <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png" />
        <link rel="manifest" href="/site.webmanifest" />
        <link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5" />
        <meta name="msapplication-TileColor" content="#2b5797" />
        <meta name="theme-color" content="#ffffff" />

        <SiteHeader section={section} />

{content}

        { SiteFooter { today, console_submission_count, cartridge_submission_count }.render() }

    }
    .to_string();
    format!("<!DOCTYPE html>{content}")
}
<|endoftext|>


<commit_before>use std::{fmt, fs, io};
use std::io::Read;
use syntax::{ast, codemap, ptr};
use syntax::parse::token;
use toml;

/// Get the configuration file from arguments.
pub fn conf_file(args: &[ptr::P<ast::MetaItem>]) -> Result<Option<token::InternedString>, (&'static str, codemap::Span)> {
    for arg in args {
        match arg.node {
            ast::MetaItemKind::Word(ref name) |
            ast::MetaItemKind::List(ref name, _) => {
                if name == &"conf_file" {
                    return Err(("`conf_file` must be a named value", arg.span));
                }
            }
            ast::MetaItemKind::NameValue(ref name, ref value) => {
                if name == &"conf_file" {
                    return if let ast::LitKind::Str(ref file, _) = value.node {
                        Ok(Some(file.clone()))
                    } else {
                        Err(("`conf_file` value must be a string", value.span))
                    };
                }
            }
        }
    }

    Ok(None)
}

/// Error from reading a configuration file.
#[derive(Debug)]
pub enum ConfError {
    IoError(io::Error),
    TomlError(Vec<toml::ParserError>),
    TypeError(&'static str, &'static str, &'static str),
    UnknownKey(String),
}

impl fmt::Display for ConfError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            ConfError::IoError(ref err) => err.fmt(f),
            ConfError::TomlError(ref errs) => {
                let mut first = true;
                for err in errs {
                    if !first {
                        try!(", ".fmt(f));
                        first = false;
                    }

                    try!(err.fmt(f));
                }

                Ok(())
            }
            ConfError::TypeError(ref key, ref expected, ref got) => {
                write!(f, "`{}` is expected to be a `{}` but is a `{}`", key, expected, got)
            }
            ConfError::UnknownKey(ref key) => write!(f, "unknown key `{}`", key),
        }
    }
}

impl From<io::Error> for ConfError {
    fn from(e: io::Error) -> Self {
        ConfError::IoError(e)
    }
}

macro_rules! define_Conf {
    ($(#[$doc: meta] ($toml_name: tt, $rust_name: ident, $default: expr => $($ty: tt)+),)+) => {
        /// Type used to store lint configuration.
        pub struct Conf {
            $(#[$doc] pub $rust_name: define_Conf!(TY $($ty)+),)+
        }

        impl Default for Conf {
            fn default() -> Conf {
                Conf {
                    $($rust_name: define_Conf!(DEFAULT $($ty)+, $default),)+
                }
            }
        }

        impl Conf {
            /// Set the property `name` (which must be the `toml` name) to the given value
            #[allow(cast_sign_loss)]
            fn set(&mut self, name: String, value: toml::Value) -> Result<(), ConfError> {
                match name.as_str() {
                    $(
                        define_Conf!(PAT $toml_name) => {
                            if let Some(value) = define_Conf!(CONV $($ty)+, value) {
                                self.$rust_name = value;
                            }
                            else {
                                return Err(ConfError::TypeError(define_Conf!(EXPR $toml_name),
                                                                stringify!($($ty)+),
                                                                value.type_str()));
                            }
                        },
                    )+
                    "third-party" => {
                        // for external tools such as clippy-service
                        return Ok(());
                    }
                    _ => {
                        return Err(ConfError::UnknownKey(name));
                    }
                }

                Ok(())
            }
        }
    };

    // hack to convert tts
    (PAT $pat: pat) => { $pat };
    (EXPR $e: expr) => { $e };
    (TY $ty: ty) => { $ty };

    // how to read the value?
    (CONV i64, $value: expr) => { $value.as_integer() };
    (CONV u64, $value: expr) => { $value.as_integer().iter().filter_map(|&i| if i >= 0 { Some(i as u64) } else { None }).next() };
    (CONV String, $value: expr) => { $value.as_str().map(Into::into) };
    (CONV Vec<String>, $value: expr) => {{
        let slice = $value.as_slice();

        if let Some(slice) = slice {
            if slice.iter().any(|v| v.as_str().is_none()) {
                None
            }
            else {
                Some(slice.iter().map(|v| v.as_str().unwrap_or_else(|| unreachable!()).to_owned()).collect())
            }
        }
        else {
            None
        }
    }};

    // provide a nicer syntax to declare the default value of `Vec<String>` variables
    (DEFAULT Vec<String>, $e: expr) => { $e.iter().map(|&e| e.to_owned()).collect() };
    (DEFAULT $ty: ty, $e: expr) => { $e };
}

define_Conf! {
    /// Lint: BLACKLISTED_NAME. The list of blacklisted names to lint about
    ("blacklisted-names", blacklisted_names, ["foo", "bar", "baz"] => Vec<String>),
    /// Lint: CYCLOMATIC_COMPLEXITY. The maximum cyclomatic complexity a function can have
    ("cyclomatic-complexity-threshold", cyclomatic_complexity_threshold, 25 => u64),
    /// Lint: DOC_MARKDOWN. The list of words this lint should not consider as identifiers needing ticks
    ("doc-valid-idents", doc_valid_idents, ["MiB", "GiB", "TiB", "PiB", "EiB", "GitHub", "NaN"] => Vec<String>),
    /// Lint: TOO_MANY_ARGUMENTS. The maximum number of argument a function or method can have
    ("too-many-arguments-threshold", too_many_arguments_threshold, 7 => u64),
    /// Lint: TYPE_COMPLEXITY. The maximum complexity a type can have
    ("type-complexity-threshold", type_complexity_threshold, 250 => u64),
    /// Lint: MANY_SINGLE_CHAR_NAMES. The maximum number of single char bindings a scope may have
    ("single-char-binding-names-threshold", max_single_char_names, 5 => u64),
}

/// Read the `toml` configuration file. The function will ignore “File not found” errors iif
/// `!must_exist`, in which case, it will return the default configuration.
/// In case of error, the function tries to continue as much as possible.
pub fn read_conf(path: &str, must_exist: bool) -> (Conf, Vec<ConfError>) {
    let mut conf = Conf::default();
    let mut errors = Vec::new();

    let file = match fs::File::open(path) {
        Ok(mut file) => {
            let mut buf = String::new();

            if let Err(err) = file.read_to_string(&mut buf) {
                errors.push(err.into());
                return (conf, errors);
            }

            buf
        }
        Err(ref err) if !must_exist && err.kind() == io::ErrorKind::NotFound => {
            return (conf, errors);
        }
        Err(err) => {
            errors.push(err.into());
            return (conf, errors);
        }
    };

    let mut parser = toml::Parser::new(&file);
    let toml = if let Some(toml) = parser.parse() {
        toml
    } else {
        errors.push(ConfError::TomlError(parser.errors));
        return (conf, errors);
    };

    for (key, value) in toml {
        if let Err(err) = conf.set(key, value) {
            errors.push(err);
        }
    }

    (conf, errors)
}
<commit_msg>added GPLv{2,3} to doc-valid-idents<commit_after>use std::{fmt, fs, io};
use std::io::Read;
use syntax::{ast, codemap, ptr};
use syntax::parse::token;
use toml;

/// Get the configuration file from arguments.
pub fn conf_file(args: &[ptr::P<ast::MetaItem>]) -> Result<Option<token::InternedString>, (&'static str, codemap::Span)> {
    for arg in args {
        match arg.node {
            ast::MetaItemKind::Word(ref name) |
            ast::MetaItemKind::List(ref name, _) => {
                if name == &"conf_file" {
                    return Err(("`conf_file` must be a named value", arg.span));
                }
            }
            ast::MetaItemKind::NameValue(ref name, ref value) => {
                if name == &"conf_file" {
                    return if let ast::LitKind::Str(ref file, _) = value.node {
                        Ok(Some(file.clone()))
                    } else {
                        Err(("`conf_file` value must be a string", value.span))
                    };
                }
            }
        }
    }

    Ok(None)
}

/// Error from reading a configuration file.
#[derive(Debug)]
pub enum ConfError {
    IoError(io::Error),
    TomlError(Vec<toml::ParserError>),
    TypeError(&'static str, &'static str, &'static str),
    UnknownKey(String),
}

impl fmt::Display for ConfError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            ConfError::IoError(ref err) => err.fmt(f),
            ConfError::TomlError(ref errs) => {
                let mut first = true;
                for err in errs {
                    if !first {
                        try!(", ".fmt(f));
                        first = false;
                    }

                    try!(err.fmt(f));
                }

                Ok(())
            }
            ConfError::TypeError(ref key, ref expected, ref got) => {
                write!(f, "`{}` is expected to be a `{}` but is a `{}`", key, expected, got)
            }
            ConfError::UnknownKey(ref key) => write!(f, "unknown key `{}`", key),
        }
    }
}

impl From<io::Error> for ConfError {
    fn from(e: io::Error) -> Self {
        ConfError::IoError(e)
    }
}

macro_rules! define_Conf {
    ($(#[$doc: meta] ($toml_name: tt, $rust_name: ident, $default: expr => $($ty: tt)+),)+) => {
        /// Type used to store lint configuration.
        pub struct Conf {
            $(#[$doc] pub $rust_name: define_Conf!(TY $($ty)+),)+
        }

        impl Default for Conf {
            fn default() -> Conf {
                Conf {
                    $($rust_name: define_Conf!(DEFAULT $($ty)+, $default),)+
                }
            }
        }

        impl Conf {
            /// Set the property `name` (which must be the `toml` name) to the given value
            #[allow(cast_sign_loss)]
            fn set(&mut self, name: String, value: toml::Value) -> Result<(), ConfError> {
                match name.as_str() {
                    $(
                        define_Conf!(PAT $toml_name) => {
                            if let Some(value) = define_Conf!(CONV $($ty)+, value) {
                                self.$rust_name = value;
                            }
                            else {
                                return Err(ConfError::TypeError(define_Conf!(EXPR $toml_name),
                                                                stringify!($($ty)+),
                                                                value.type_str()));
                            }
                        },
                    )+
                    "third-party" => {
                        // for external tools such as clippy-service
                        return Ok(());
                    }
                    _ => {
                        return Err(ConfError::UnknownKey(name));
                    }
                }

                Ok(())
            }
        }
    };

    // hack to convert tts
    (PAT $pat: pat) => { $pat };
    (EXPR $e: expr) => { $e };
    (TY $ty: ty) => { $ty };

    // how to read the value?
    (CONV i64, $value: expr) => { $value.as_integer() };
    (CONV u64, $value: expr) => { $value.as_integer().iter().filter_map(|&i| if i >= 0 { Some(i as u64) } else { None }).next() };
    (CONV String, $value: expr) => { $value.as_str().map(Into::into) };
    (CONV Vec<String>, $value: expr) => {{
        let slice = $value.as_slice();

        if let Some(slice) = slice {
            if slice.iter().any(|v| v.as_str().is_none()) {
                None
            }
            else {
                Some(slice.iter().map(|v| v.as_str().unwrap_or_else(|| unreachable!()).to_owned()).collect())
            }
        }
        else {
            None
        }
    }};

    // provide a nicer syntax to declare the default value of `Vec<String>` variables
    (DEFAULT Vec<String>, $e: expr) => { $e.iter().map(|&e| e.to_owned()).collect() };
    (DEFAULT $ty: ty, $e: expr) => { $e };
}

define_Conf! {
    /// Lint: BLACKLISTED_NAME. The list of blacklisted names to lint about
    ("blacklisted-names", blacklisted_names, ["foo", "bar", "baz"] => Vec<String>),
    /// Lint: CYCLOMATIC_COMPLEXITY. The maximum cyclomatic complexity a function can have
    ("cyclomatic-complexity-threshold", cyclomatic_complexity_threshold, 25 => u64),
    /// Lint: DOC_MARKDOWN. The list of words this lint should not consider as identifiers needing ticks
    ("doc-valid-idents", doc_valid_idents, ["MiB", "GiB", "TiB", "PiB", "EiB", "GitHub", "NaN", "GPLv2", "GPLv3"] => Vec<String>),
    /// Lint: TOO_MANY_ARGUMENTS. The maximum number of argument a function or method can have
    ("too-many-arguments-threshold", too_many_arguments_threshold, 7 => u64),
    /// Lint: TYPE_COMPLEXITY. The maximum complexity a type can have
    ("type-complexity-threshold", type_complexity_threshold, 250 => u64),
    /// Lint: MANY_SINGLE_CHAR_NAMES. The maximum number of single char bindings a scope may have
    ("single-char-binding-names-threshold", max_single_char_names, 5 => u64),
}

/// Read the `toml` configuration file. The function will ignore “File not found” errors iif
/// `!must_exist`, in which case, it will return the default configuration.
/// In case of error, the function tries to continue as much as possible.
pub fn read_conf(path: &str, must_exist: bool) -> (Conf, Vec<ConfError>) {
    let mut conf = Conf::default();
    let mut errors = Vec::new();

    let file = match fs::File::open(path) {
        Ok(mut file) => {
            let mut buf = String::new();

            if let Err(err) = file.read_to_string(&mut buf) {
                errors.push(err.into());
                return (conf, errors);
            }

            buf
        }
        Err(ref err) if !must_exist && err.kind() == io::ErrorKind::NotFound => {
            return (conf, errors);
        }
        Err(err) => {
            errors.push(err.into());
            return (conf, errors);
        }
    };

    let mut parser = toml::Parser::new(&file);
    let toml = if let Some(toml) = parser.parse() {
        toml
    } else {
        errors.push(ConfError::TomlError(parser.errors));
        return (conf, errors);
    };

    for (key, value) in toml {
        if let Err(err) = conf.set(key, value) {
            errors.push(err);
        }
    }

    (conf, errors)
}
<|endoftext|>


<commit_before>fn main() {
    println!("Hello, world!");
}
<commit_msg>Example for async in rust<commit_after>use {
    hyper::{
    Body, Client, Request, Response, Server, Uri,
    service::service_fn,
    rt::run,
},
futures::{
    compat::Future01CompatExt,
    future::{FutureExt, TryFutureExt},
},
std::net::SocketAddr,
};

async fn serve_req(_req: Request<Body>) -> Result<Response<Body>, hyper::Error> {
    Ok(Response::new(Body::from("hello, wworld.")))
}

async fn run_server(addr: SocketAddr) {
    println!("Listening on http://{}", addr);
    let serve_future = Server::bind(&addr).serve(|| service_fn(|req| serve_req(req).boxed().compat()));
    if let Err(e) = serve_future.compat().await {
        eprintln!("server error: {}", e);
    }
}
fn main() {
    let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
    let futures_03_future = run_server(addr);
    let futures_01_future = futures_03_future.unit_error().boxed().compat();
    run(futures_01_future);
}
<|endoftext|>


<commit_before>// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0.

//! Storage configuration.

use crate::config::BLOCK_CACHE_RATE;
use crate::server::ttl::TTLCheckerTask;
use crate::server::CONFIG_ROCKSDB_GAUGE;
use engine_rocks::raw::{Cache, LRUCacheOptions, MemoryAllocator};
use engine_rocks::RocksEngine;
use engine_traits::{CFOptionsExt, ColumnFamilyOptions, CF_DEFAULT};
use file_system::{get_io_rate_limiter, IOPriority, IORateLimitMode, IORateLimiter, IOType};
use libc::c_int;
use online_config::{ConfigChange, ConfigManager, ConfigValue, OnlineConfig, Result as CfgResult};
use std::error::Error;
use tikv_util::config::{self, OptionReadableSize, ReadableDuration, ReadableSize};
use tikv_util::sys::SysQuota;
use tikv_util::worker::Scheduler;

pub const DEFAULT_DATA_DIR: &str = "./";
const DEFAULT_GC_RATIO_THRESHOLD: f64 = 1.1;
const DEFAULT_MAX_KEY_SIZE: usize = 4 * 1024;
const DEFAULT_SCHED_CONCURRENCY: usize = 1024 * 512;
const MAX_SCHED_CONCURRENCY: usize = 2 * 1024 * 1024;

// According to "Little's law", assuming you can write 100MB per
// second, and it takes about 100ms to process the write requests
// on average, in that situation the writing bytes estimated 10MB,
// here we use 100MB as default value for tolerate 1s latency.
const DEFAULT_SCHED_PENDING_WRITE_MB: u64 = 100;

const DEFAULT_RESERVED_SPACE_GB: u64 = 5;

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, OnlineConfig)]
#[serde(default)]
#[serde(rename_all = "kebab-case")]
pub struct Config {
    #[online_config(skip)]
    pub data_dir: String,
    // Replaced by `GcConfig.ratio_threshold`. Keep it for backward compatibility.
    #[online_config(skip)]
    pub gc_ratio_threshold: f64,
    #[online_config(skip)]
    pub max_key_size: usize,
    #[online_config(skip)]
    pub scheduler_concurrency: usize,
    #[online_config(skip)]
    pub scheduler_worker_pool_size: usize,
    #[online_config(skip)]
    pub scheduler_pending_write_threshold: ReadableSize,
    #[online_config(skip)]
    // Reserve disk space to make tikv would have enough space to compact when disk is full.
    pub reserve_space: ReadableSize,
    #[online_config(skip)]
    pub enable_async_apply_prewrite: bool,
    #[online_config(skip)]
    pub enable_ttl: bool,
    /// Interval to check TTL for all SSTs,
    pub ttl_check_poll_interval: ReadableDuration,
    #[online_config(submodule)]
    pub block_cache: BlockCacheConfig,
    #[online_config(submodule)]
    pub io_rate_limit: IORateLimitConfig,
}

impl Default for Config {
    fn default() -> Config {
        let cpu_num = SysQuota::cpu_cores_quota();
        Config {
            data_dir: DEFAULT_DATA_DIR.to_owned(),
            gc_ratio_threshold: DEFAULT_GC_RATIO_THRESHOLD,
            max_key_size: DEFAULT_MAX_KEY_SIZE,
            scheduler_concurrency: DEFAULT_SCHED_CONCURRENCY,
            scheduler_worker_pool_size: if cpu_num >= 16.0 { 8 } else { 4 },
            scheduler_pending_write_threshold: ReadableSize::mb(DEFAULT_SCHED_PENDING_WRITE_MB),
            reserve_space: ReadableSize::gb(DEFAULT_RESERVED_SPACE_GB),
            enable_async_apply_prewrite: false,
            enable_ttl: false,
            ttl_check_poll_interval: ReadableDuration::hours(12),
            block_cache: BlockCacheConfig::default(),
            io_rate_limit: IORateLimitConfig::default(),
        }
    }
}

impl Config {
    pub fn validate(&mut self) -> Result<(), Box<dyn Error>> {
        if self.data_dir != DEFAULT_DATA_DIR {
            self.data_dir = config::canonicalize_path(&self.data_dir)?
        }
        if self.scheduler_concurrency > MAX_SCHED_CONCURRENCY {
            warn!(
                "TiKV has optimized latch since v4.0, so it is not necessary to set large schedule \
                concurrency. To save memory, change it from {:?} to {:?}",
                self.scheduler_concurrency, MAX_SCHED_CONCURRENCY
            );
            self.scheduler_concurrency = MAX_SCHED_CONCURRENCY;
        }
        self.io_rate_limit.validate()
    }
}

pub struct StorageConfigManger {
    kvdb: RocksEngine,
    shared_block_cache: bool,
    ttl_checker_scheduler: Scheduler<TTLCheckerTask>,
}

impl StorageConfigManger {
    pub fn new(
        kvdb: RocksEngine,
        shared_block_cache: bool,
        ttl_checker_scheduler: Scheduler<TTLCheckerTask>,
    ) -> StorageConfigManger {
        StorageConfigManger {
            kvdb,
            shared_block_cache,
            ttl_checker_scheduler,
        }
    }
}

impl ConfigManager for StorageConfigManger {
    fn dispatch(&mut self, mut change: ConfigChange) -> CfgResult<()> {
        if let Some(ConfigValue::Module(mut block_cache)) = change.remove("block_cache") {
            if !self.shared_block_cache {
                return Err("shared block cache is disabled".into());
            }
            if let Some(size) = block_cache.remove("capacity") {
                let s: OptionReadableSize = size.into();
                if let Some(size) = s.0 {
                    // Hack: since all CFs in both kvdb and raftdb share a block cache, we can change
                    // the size through any of them. Here we change it through default CF in kvdb.
                    // A better way to do it is to hold the cache reference somewhere, and use it to
                    // change cache size.
                    let opt = self.kvdb.get_options_cf(CF_DEFAULT).unwrap(); // FIXME unwrap
                    opt.set_block_cache_capacity(size.0)?;
                    // Write config to metric
                    CONFIG_ROCKSDB_GAUGE
                        .with_label_values(&[CF_DEFAULT, "block_cache_size"])
                        .set(size.0 as f64);
                }
            }
        } else if let Some(v) = change.remove("ttl_check_poll_interval") {
            let interval: ReadableDuration = v.into();
            self.ttl_checker_scheduler
                .schedule(TTLCheckerTask::UpdatePollInterval(interval.into()))
                .unwrap();
        }
        if let Some(ConfigValue::Module(mut io_rate_limit)) = change.remove("io_rate_limit") {
            let limiter = match get_io_rate_limiter() {
                None => return Err("IO rate limiter is not present".into()),
                Some(limiter) => limiter,
            };
            if let Some(limit) = io_rate_limit.remove("max_bytes_per_sec") {
                let limit: ReadableSize = limit.into();
                limiter.set_io_rate_limit(limit.0 as usize);
            }
        }
        Ok(())
    }
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, OnlineConfig)]
#[serde(default)]
#[serde(rename_all = "kebab-case")]
pub struct BlockCacheConfig {
    #[online_config(skip)]
    pub shared: bool,
    pub capacity: OptionReadableSize,
    #[online_config(skip)]
    pub num_shard_bits: i32,
    #[online_config(skip)]
    pub strict_capacity_limit: bool,
    #[online_config(skip)]
    pub high_pri_pool_ratio: f64,
    #[online_config(skip)]
    pub memory_allocator: Option<String>,
}

impl Default for BlockCacheConfig {
    fn default() -> BlockCacheConfig {
        BlockCacheConfig {
            shared: true,
            capacity: OptionReadableSize(None),
            num_shard_bits: 6,
            strict_capacity_limit: false,
            high_pri_pool_ratio: 0.8,
            memory_allocator: Some(String::from("nodump")),
        }
    }
}

impl BlockCacheConfig {
    pub fn build_shared_cache(&self) -> Option<Cache> {
        if !self.shared {
            return None;
        }
        let capacity = match self.capacity.0 {
            None => {
                let total_mem = SysQuota::memory_limit_in_bytes();
                ((total_mem as f64) * BLOCK_CACHE_RATE) as usize
            }
            Some(c) => c.0 as usize,
        };
        let mut cache_opts = LRUCacheOptions::new();
        cache_opts.set_capacity(capacity);
        cache_opts.set_num_shard_bits(self.num_shard_bits as c_int);
        cache_opts.set_strict_capacity_limit(self.strict_capacity_limit);
        cache_opts.set_high_pri_pool_ratio(self.high_pri_pool_ratio);
        if let Some(allocator) = self.new_memory_allocator() {
            cache_opts.set_memory_allocator(allocator);
        }
        Some(Cache::new_lru_cache(cache_opts))
    }

    fn new_memory_allocator(&self) -> Option<MemoryAllocator> {
        if let Some(ref alloc) = self.memory_allocator {
            match alloc.as_str() {
                #[cfg(feature = "jemalloc")]
                "nodump" => match MemoryAllocator::new_jemalloc_memory_allocator() {
                    Ok(allocator) => {
                        return Some(allocator);
                    }
                    Err(e) => {
                        warn!(
                            "Create jemalloc nodump allocator for block cache failed: {}, continue with default allocator",
                            e
                        );
                    }
                },
                "" => {}
                other => {
                    warn!(
                        "Memory allocator {} is not supported, continue with default allocator",
                        other
                    );
                }
            }
        };
        None
    }
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, OnlineConfig)]
#[serde(default)]
#[serde(rename_all = "kebab-case")]
pub struct IORateLimitConfig {
    pub max_bytes_per_sec: ReadableSize,
    #[online_config(skip)]
    pub mode: IORateLimitMode,
    /// When this flag is off, high-priority IOs are counted but not limited. Default
    /// set to false because the optimal throughput target provided by user might not be
    /// the maximum available bandwidth. For multi-tenancy use case, this flag should be
    /// turned on.
    #[online_config(skip)]
    pub strict: bool,
    #[online_config(skip)]
    pub foreground_read_priority: IOPriority,
    #[online_config(skip)]
    pub foreground_write_priority: IOPriority,
    #[online_config(skip)]
    pub flush_priority: IOPriority,
    #[online_config(skip)]
    pub level_zero_compaction_priority: IOPriority,
    #[online_config(skip)]
    pub compaction_priority: IOPriority,
    #[online_config(skip)]
    pub replication_priority: IOPriority,
    #[online_config(skip)]
    pub load_balance_priority: IOPriority,
    #[online_config(skip)]
    pub gc_priority: IOPriority,
    #[online_config(skip)]
    pub import_priority: IOPriority,
    #[online_config(skip)]
    pub export_priority: IOPriority,
    #[online_config(skip)]
    pub other_priority: IOPriority,
}

impl Default for IORateLimitConfig {
    fn default() -> IORateLimitConfig {
        IORateLimitConfig {
            max_bytes_per_sec: ReadableSize::mb(0),
            mode: IORateLimitMode::WriteOnly,
            strict: false,
            foreground_read_priority: IOPriority::High,
            foreground_write_priority: IOPriority::High,
            flush_priority: IOPriority::High,
            level_zero_compaction_priority: IOPriority::Medium,
            compaction_priority: IOPriority::Low,
            replication_priority: IOPriority::High,
            load_balance_priority: IOPriority::High,
            gc_priority: IOPriority::High,
            import_priority: IOPriority::Low,
            export_priority: IOPriority::Low,
            other_priority: IOPriority::High,
        }
    }
}

impl IORateLimitConfig {
    pub fn build(&self, enable_statistics: bool) -> IORateLimiter {
        let mut limiter = IORateLimiter::new(self.mode, self.strict, enable_statistics);
        limiter.set_io_rate_limit(self.max_bytes_per_sec.0 as usize);
        limiter.set_io_priority(IOType::ForegroundRead, self.foreground_read_priority);
        limiter.set_io_priority(IOType::ForegroundWrite, self.foreground_write_priority);
        limiter.set_io_priority(IOType::Flush, self.flush_priority);
        limiter.set_io_priority(
            IOType::LevelZeroCompaction,
            self.level_zero_compaction_priority,
        );
        limiter.set_io_priority(IOType::Compaction, self.compaction_priority);
        limiter.set_io_priority(IOType::Replication, self.replication_priority);
        limiter.set_io_priority(IOType::LoadBalance, self.load_balance_priority);
        limiter.set_io_priority(IOType::Gc, self.gc_priority);
        limiter.set_io_priority(IOType::Import, self.import_priority);
        limiter.set_io_priority(IOType::Export, self.export_priority);
        limiter.set_io_priority(IOType::Other, self.other_priority);
        limiter
    }

    fn validate(&mut self) -> Result<(), Box<dyn Error>> {
        if self.other_priority != IOPriority::High {
            warn!(
                "Occasionally some critical IO operations are tagged as IOType::Other, \
                  e.g. IOs are fired from unmanaged threads, thread-local type storage exceeds \
                  capacity. To be on the safe side, change priority for IOType::Other from \
                  {:?} to {:?}",
                self.other_priority,
                IOPriority::High
            );
            self.other_priority = IOPriority::High;
        }
        if self.gc_priority != self.foreground_write_priority {
            warn!(
                "GC writes are merged with foreground writes. To avoid priority inversion, change \
                  priority for IOType::Gc from {:?} to {:?}",
                self.gc_priority, self.foreground_write_priority,
            );
            self.gc_priority = self.foreground_write_priority;
        }
        if self.mode != IORateLimitMode::WriteOnly {
            return Err(
                "storage.io-rate-limit.mode other than write-only is not supported.".into(),
            );
        }
        Ok(())
    }
}
<commit_msg>Escalate import/export priority (#10390)<commit_after>// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0.

//! Storage configuration.

use crate::config::BLOCK_CACHE_RATE;
use crate::server::ttl::TTLCheckerTask;
use crate::server::CONFIG_ROCKSDB_GAUGE;
use engine_rocks::raw::{Cache, LRUCacheOptions, MemoryAllocator};
use engine_rocks::RocksEngine;
use engine_traits::{CFOptionsExt, ColumnFamilyOptions, CF_DEFAULT};
use file_system::{get_io_rate_limiter, IOPriority, IORateLimitMode, IORateLimiter, IOType};
use libc::c_int;
use online_config::{ConfigChange, ConfigManager, ConfigValue, OnlineConfig, Result as CfgResult};
use std::error::Error;
use tikv_util::config::{self, OptionReadableSize, ReadableDuration, ReadableSize};
use tikv_util::sys::SysQuota;
use tikv_util::worker::Scheduler;

pub const DEFAULT_DATA_DIR: &str = "./";
const DEFAULT_GC_RATIO_THRESHOLD: f64 = 1.1;
const DEFAULT_MAX_KEY_SIZE: usize = 4 * 1024;
const DEFAULT_SCHED_CONCURRENCY: usize = 1024 * 512;
const MAX_SCHED_CONCURRENCY: usize = 2 * 1024 * 1024;

// According to "Little's law", assuming you can write 100MB per
// second, and it takes about 100ms to process the write requests
// on average, in that situation the writing bytes estimated 10MB,
// here we use 100MB as default value for tolerate 1s latency.
const DEFAULT_SCHED_PENDING_WRITE_MB: u64 = 100;

const DEFAULT_RESERVED_SPACE_GB: u64 = 5;

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, OnlineConfig)]
#[serde(default)]
#[serde(rename_all = "kebab-case")]
pub struct Config {
    #[online_config(skip)]
    pub data_dir: String,
    // Replaced by `GcConfig.ratio_threshold`. Keep it for backward compatibility.
    #[online_config(skip)]
    pub gc_ratio_threshold: f64,
    #[online_config(skip)]
    pub max_key_size: usize,
    #[online_config(skip)]
    pub scheduler_concurrency: usize,
    #[online_config(skip)]
    pub scheduler_worker_pool_size: usize,
    #[online_config(skip)]
    pub scheduler_pending_write_threshold: ReadableSize,
    #[online_config(skip)]
    // Reserve disk space to make tikv would have enough space to compact when disk is full.
    pub reserve_space: ReadableSize,
    #[online_config(skip)]
    pub enable_async_apply_prewrite: bool,
    #[online_config(skip)]
    pub enable_ttl: bool,
    /// Interval to check TTL for all SSTs,
    pub ttl_check_poll_interval: ReadableDuration,
    #[online_config(submodule)]
    pub block_cache: BlockCacheConfig,
    #[online_config(submodule)]
    pub io_rate_limit: IORateLimitConfig,
}

impl Default for Config {
    fn default() -> Config {
        let cpu_num = SysQuota::cpu_cores_quota();
        Config {
            data_dir: DEFAULT_DATA_DIR.to_owned(),
            gc_ratio_threshold: DEFAULT_GC_RATIO_THRESHOLD,
            max_key_size: DEFAULT_MAX_KEY_SIZE,
            scheduler_concurrency: DEFAULT_SCHED_CONCURRENCY,
            scheduler_worker_pool_size: if cpu_num >= 16.0 { 8 } else { 4 },
            scheduler_pending_write_threshold: ReadableSize::mb(DEFAULT_SCHED_PENDING_WRITE_MB),
            reserve_space: ReadableSize::gb(DEFAULT_RESERVED_SPACE_GB),
            enable_async_apply_prewrite: false,
            enable_ttl: false,
            ttl_check_poll_interval: ReadableDuration::hours(12),
            block_cache: BlockCacheConfig::default(),
            io_rate_limit: IORateLimitConfig::default(),
        }
    }
}

impl Config {
    pub fn validate(&mut self) -> Result<(), Box<dyn Error>> {
        if self.data_dir != DEFAULT_DATA_DIR {
            self.data_dir = config::canonicalize_path(&self.data_dir)?
        }
        if self.scheduler_concurrency > MAX_SCHED_CONCURRENCY {
            warn!(
                "TiKV has optimized latch since v4.0, so it is not necessary to set large schedule \
                concurrency. To save memory, change it from {:?} to {:?}",
                self.scheduler_concurrency, MAX_SCHED_CONCURRENCY
            );
            self.scheduler_concurrency = MAX_SCHED_CONCURRENCY;
        }
        self.io_rate_limit.validate()
    }
}

pub struct StorageConfigManger {
    kvdb: RocksEngine,
    shared_block_cache: bool,
    ttl_checker_scheduler: Scheduler<TTLCheckerTask>,
}

impl StorageConfigManger {
    pub fn new(
        kvdb: RocksEngine,
        shared_block_cache: bool,
        ttl_checker_scheduler: Scheduler<TTLCheckerTask>,
    ) -> StorageConfigManger {
        StorageConfigManger {
            kvdb,
            shared_block_cache,
            ttl_checker_scheduler,
        }
    }
}

impl ConfigManager for StorageConfigManger {
    fn dispatch(&mut self, mut change: ConfigChange) -> CfgResult<()> {
        if let Some(ConfigValue::Module(mut block_cache)) = change.remove("block_cache") {
            if !self.shared_block_cache {
                return Err("shared block cache is disabled".into());
            }
            if let Some(size) = block_cache.remove("capacity") {
                let s: OptionReadableSize = size.into();
                if let Some(size) = s.0 {
                    // Hack: since all CFs in both kvdb and raftdb share a block cache, we can change
                    // the size through any of them. Here we change it through default CF in kvdb.
                    // A better way to do it is to hold the cache reference somewhere, and use it to
                    // change cache size.
                    let opt = self.kvdb.get_options_cf(CF_DEFAULT).unwrap(); // FIXME unwrap
                    opt.set_block_cache_capacity(size.0)?;
                    // Write config to metric
                    CONFIG_ROCKSDB_GAUGE
                        .with_label_values(&[CF_DEFAULT, "block_cache_size"])
                        .set(size.0 as f64);
                }
            }
        } else if let Some(v) = change.remove("ttl_check_poll_interval") {
            let interval: ReadableDuration = v.into();
            self.ttl_checker_scheduler
                .schedule(TTLCheckerTask::UpdatePollInterval(interval.into()))
                .unwrap();
        }
        if let Some(ConfigValue::Module(mut io_rate_limit)) = change.remove("io_rate_limit") {
            let limiter = match get_io_rate_limiter() {
                None => return Err("IO rate limiter is not present".into()),
                Some(limiter) => limiter,
            };
            if let Some(limit) = io_rate_limit.remove("max_bytes_per_sec") {
                let limit: ReadableSize = limit.into();
                limiter.set_io_rate_limit(limit.0 as usize);
            }
        }
        Ok(())
    }
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, OnlineConfig)]
#[serde(default)]
#[serde(rename_all = "kebab-case")]
pub struct BlockCacheConfig {
    #[online_config(skip)]
    pub shared: bool,
    pub capacity: OptionReadableSize,
    #[online_config(skip)]
    pub num_shard_bits: i32,
    #[online_config(skip)]
    pub strict_capacity_limit: bool,
    #[online_config(skip)]
    pub high_pri_pool_ratio: f64,
    #[online_config(skip)]
    pub memory_allocator: Option<String>,
}

impl Default for BlockCacheConfig {
    fn default() -> BlockCacheConfig {
        BlockCacheConfig {
            shared: true,
            capacity: OptionReadableSize(None),
            num_shard_bits: 6,
            strict_capacity_limit: false,
            high_pri_pool_ratio: 0.8,
            memory_allocator: Some(String::from("nodump")),
        }
    }
}

impl BlockCacheConfig {
    pub fn build_shared_cache(&self) -> Option<Cache> {
        if !self.shared {
            return None;
        }
        let capacity = match self.capacity.0 {
            None => {
                let total_mem = SysQuota::memory_limit_in_bytes();
                ((total_mem as f64) * BLOCK_CACHE_RATE) as usize
            }
            Some(c) => c.0 as usize,
        };
        let mut cache_opts = LRUCacheOptions::new();
        cache_opts.set_capacity(capacity);
        cache_opts.set_num_shard_bits(self.num_shard_bits as c_int);
        cache_opts.set_strict_capacity_limit(self.strict_capacity_limit);
        cache_opts.set_high_pri_pool_ratio(self.high_pri_pool_ratio);
        if let Some(allocator) = self.new_memory_allocator() {
            cache_opts.set_memory_allocator(allocator);
        }
        Some(Cache::new_lru_cache(cache_opts))
    }

    fn new_memory_allocator(&self) -> Option<MemoryAllocator> {
        if let Some(ref alloc) = self.memory_allocator {
            match alloc.as_str() {
                #[cfg(feature = "jemalloc")]
                "nodump" => match MemoryAllocator::new_jemalloc_memory_allocator() {
                    Ok(allocator) => {
                        return Some(allocator);
                    }
                    Err(e) => {
                        warn!(
                            "Create jemalloc nodump allocator for block cache failed: {}, continue with default allocator",
                            e
                        );
                    }
                },
                "" => {}
                other => {
                    warn!(
                        "Memory allocator {} is not supported, continue with default allocator",
                        other
                    );
                }
            }
        };
        None
    }
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, OnlineConfig)]
#[serde(default)]
#[serde(rename_all = "kebab-case")]
pub struct IORateLimitConfig {
    pub max_bytes_per_sec: ReadableSize,
    #[online_config(skip)]
    pub mode: IORateLimitMode,
    /// When this flag is off, high-priority IOs are counted but not limited. Default
    /// set to false because the optimal throughput target provided by user might not be
    /// the maximum available bandwidth. For multi-tenancy use case, this flag should be
    /// turned on.
    #[online_config(skip)]
    pub strict: bool,
    #[online_config(skip)]
    pub foreground_read_priority: IOPriority,
    #[online_config(skip)]
    pub foreground_write_priority: IOPriority,
    #[online_config(skip)]
    pub flush_priority: IOPriority,
    #[online_config(skip)]
    pub level_zero_compaction_priority: IOPriority,
    #[online_config(skip)]
    pub compaction_priority: IOPriority,
    #[online_config(skip)]
    pub replication_priority: IOPriority,
    #[online_config(skip)]
    pub load_balance_priority: IOPriority,
    #[online_config(skip)]
    pub gc_priority: IOPriority,
    #[online_config(skip)]
    pub import_priority: IOPriority,
    #[online_config(skip)]
    pub export_priority: IOPriority,
    #[online_config(skip)]
    pub other_priority: IOPriority,
}

impl Default for IORateLimitConfig {
    fn default() -> IORateLimitConfig {
        IORateLimitConfig {
            max_bytes_per_sec: ReadableSize::mb(0),
            mode: IORateLimitMode::WriteOnly,
            strict: false,
            foreground_read_priority: IOPriority::High,
            foreground_write_priority: IOPriority::High,
            flush_priority: IOPriority::High,
            level_zero_compaction_priority: IOPriority::Medium,
            compaction_priority: IOPriority::Low,
            replication_priority: IOPriority::High,
            load_balance_priority: IOPriority::High,
            gc_priority: IOPriority::High,
            import_priority: IOPriority::Medium,
            export_priority: IOPriority::Medium,
            other_priority: IOPriority::High,
        }
    }
}

impl IORateLimitConfig {
    pub fn build(&self, enable_statistics: bool) -> IORateLimiter {
        let mut limiter = IORateLimiter::new(self.mode, self.strict, enable_statistics);
        limiter.set_io_rate_limit(self.max_bytes_per_sec.0 as usize);
        limiter.set_io_priority(IOType::ForegroundRead, self.foreground_read_priority);
        limiter.set_io_priority(IOType::ForegroundWrite, self.foreground_write_priority);
        limiter.set_io_priority(IOType::Flush, self.flush_priority);
        limiter.set_io_priority(
            IOType::LevelZeroCompaction,
            self.level_zero_compaction_priority,
        );
        limiter.set_io_priority(IOType::Compaction, self.compaction_priority);
        limiter.set_io_priority(IOType::Replication, self.replication_priority);
        limiter.set_io_priority(IOType::LoadBalance, self.load_balance_priority);
        limiter.set_io_priority(IOType::Gc, self.gc_priority);
        limiter.set_io_priority(IOType::Import, self.import_priority);
        limiter.set_io_priority(IOType::Export, self.export_priority);
        limiter.set_io_priority(IOType::Other, self.other_priority);
        limiter
    }

    fn validate(&mut self) -> Result<(), Box<dyn Error>> {
        if self.other_priority != IOPriority::High {
            warn!(
                "Occasionally some critical IO operations are tagged as IOType::Other, \
                  e.g. IOs are fired from unmanaged threads, thread-local type storage exceeds \
                  capacity. To be on the safe side, change priority for IOType::Other from \
                  {:?} to {:?}",
                self.other_priority,
                IOPriority::High
            );
            self.other_priority = IOPriority::High;
        }
        if self.gc_priority != self.foreground_write_priority {
            warn!(
                "GC writes are merged with foreground writes. To avoid priority inversion, change \
                  priority for IOType::Gc from {:?} to {:?}",
                self.gc_priority, self.foreground_write_priority,
            );
            self.gc_priority = self.foreground_write_priority;
        }
        if self.mode != IORateLimitMode::WriteOnly {
            return Err(
                "storage.io-rate-limit.mode other than write-only is not supported.".into(),
            );
        }
        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::str;

use num::bigint::BigUint;
use num::bigint::ToBigUint;

static BASE58_ALPHABET: &'static str = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";

pub fn to_base58(data: &[u8]) -> ~str
{
    let mut s = str::from_utf8(data).unwrap();
    let origlen = s.len();
    s = s.trim_left_chars(&'\0');
    let newlen = s.len();

    let mut num = BigUint::new(Vec::new());
    let mut p = 1u.to_biguint().expect("wtf");
    for c in s.bytes_rev()
    {
        let mut n = c.to_biguint().expect("wtf");
        n = n * p;
        num = num + n;
        p = p << 8;
    }

    let mut result = StrBuf::new();
    while num >= 58u.to_biguint().expect("wtf")
    {
        let d = num / 58u.to_biguint().expect("wtf");
        let r = num % 58u.to_biguint().expect("wtf");

        num = d;
        result.push_char(BASE58_ALPHABET[r.to_uint().expect("wtf")] as char);
    }

    result.push_char(BASE58_ALPHABET[num.to_uint().expect("wtf")] as char);

    let fill = Vec::from_elem(origlen - newlen, BASE58_ALPHABET[0] as char);
    result.push_str(str::from_chars(fill.as_slice()));

    fn str_reverse(s: &str) -> ~str
    {
        s.chars_rev().collect()
    }
    str_reverse(result.as_slice())
}

#[cfg(test)]
mod test
{
    use super::to_base58;

    #[test]
    fn test_simple()
    {
        let s = "hello world";
        let r = to_base58(s.as_bytes());

        assert_eq!(r, ~"StV1DL6CwTryKyV");
    }

    #[test]
    fn test_padding()
    {
        let s = "\0\0\0hello world";
        let r = to_base58(s.as_bytes());

        assert_eq!(r, ~"111StV1DL6CwTryKyV");
    }
}

#[cfg(test)]
mod bench
{
    use super::to_base58;
    use test::test::Bencher;

    #[bench]
    fn bench_simple(bh: &mut Bencher)
    {
        let s = "hello world";

        bh.iter(|| {
            to_base58(s.as_bytes());
        });
        bh.bytes = s.len() as u64;
    }
}
<commit_msg>Improve base58 mod.<commit_after>use num::bigint::ToBigUint;
use num::Integer;

static BASE58_ALPHABET: &'static str = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";

pub fn to_base58(data: &[u8]) -> ~str
{
    let mut n = 0u.to_biguint().unwrap();
    for (i, c) in data.rev_iter().enumerate() {
        let c = c.to_biguint().unwrap();

        n =n + (c << (i * 8));
    }

    let mut result = StrBuf::new();
    let limit = 58u.to_biguint().unwrap();
    while n >= limit {
        let (d, r) = n.div_rem(&limit);
        let r = r.to_uint().unwrap();

        n = d;
        result.push_char(BASE58_ALPHABET[r] as char);
    }
    let r = n.to_uint().unwrap();

    if r > 0 {
        result.push_char(BASE58_ALPHABET[r] as char);
    }

    for c in data.iter() {
        if *c == 0 {
            result.push_char(BASE58_ALPHABET[0] as char);
        } else {
            break;
        }
    }

    result.as_slice().chars_rev().collect()
}

#[cfg(test)]
mod test
{
    use super::to_base58;

    #[test]
    fn test_simple()
    {
        let tests = ~[
            ("hello world".as_bytes(), "StV1DL6CwTryKyV"),
            ("Hello World".as_bytes(), "JxF12TrwUP45BMd"),
        ];

        for &(s, n) in tests.iter() {
            let r = to_base58(s);

            assert_eq!(r.as_slice(), n);
        }
    }
}

#[cfg(test)]
mod bench
{
    use super::to_base58;
    use test::test::Bencher;

    #[bench]
    fn bench_simple(bh: &mut Bencher)
    {
        let s = "hello world";

        bh.iter(|| {
            to_base58(s.as_bytes());
        });
        bh.bytes = s.len() as u64;
    }
}
<|endoftext|>


<commit_before>use super::{pointer::Pointer, Heap};
use crate::{
    builtin_functions::BuiltinFunction,
    compiler::{
        hir_to_lir::HirToLir,
        lir::{Instruction, Lir},
    },
    database::Database,
    module::Module,
};
use itertools::Itertools;
use num_bigint::BigInt;
use std::{
    collections::{hash_map::DefaultHasher, HashMap},
    hash::{Hash, Hasher},
    ops::Deref,
};

#[derive(Clone)]
pub struct Object {
    pub reference_count: usize,
    pub data: Data,
}
#[derive(Clone)]
pub enum Data {
    Int(Int),
    Text(Text),
    Symbol(Symbol),
    Struct(Struct),
    Closure(Closure),
    Builtin(Builtin),
}

#[derive(Clone)]
pub struct Int {
    pub value: BigInt,
}

#[derive(Clone)]
pub struct Text {
    pub value: String,
}

#[derive(Clone)]
pub struct Symbol {
    // TODO: Choose a more efficient representation.
    pub value: String,
}

#[derive(Default, Clone)]
pub struct Struct {
    // TODO: Choose a more efficient representation.
    pub fields: Vec<(u64, Pointer, Pointer)>, // hash, key, and value
}

#[derive(Clone)]
pub struct Closure {
    pub captured: Vec<Pointer>,
    pub num_args: usize,
    pub body: Vec<Instruction>,
}

#[derive(Clone)]
pub struct Builtin {
    pub function: BuiltinFunction,
}

impl Struct {
    pub fn from_fields(heap: &Heap, fields: HashMap<Pointer, Pointer>) -> Self {
        let mut s = Self::default();
        for (key, value) in fields {
            s.insert(heap, key, value);
        }
        s
    }
    fn insert(&mut self, heap: &Heap, key: Pointer, value: Pointer) {
        let hash = key.hash(heap);
        for (existing_hash, existing_key, existing_value) in &mut self.fields {
            if hash != *existing_hash {
                continue;
            }
            if existing_key.equals(heap, key) {
                *existing_value = value;
                return;
            }
        }
        let entry = (hash, key, value);
        if let Some(index) = self
            .fields
            .iter()
            .position(|(the_hash, _, _)| *the_hash > hash)
        {
            self.fields.insert(index, entry);
        } else {
            self.fields.push(entry);
        }
    }
    pub fn get(&self, heap: &Heap, key: Pointer) -> Option<Pointer> {
        let hash = key.hash(heap);
        Some(
            self.fields
                .iter()
                .find(|(field_hash, field_key, _)| {
                    hash == *field_hash && key.equals(heap, *field_key)
                })?
                .2,
        )
    }
    fn len(&self) -> usize {
        self.fields.len()
    }
    pub fn iter(&self) -> impl Iterator<Item = (Pointer, Pointer)> {
        self.fields
            .clone()
            .into_iter()
            .map(|(_, key, value)| (key, value))
    }
    fn equals(&self, heap: &Heap, other: &Struct) -> bool {
        if self.len() != other.len() {
            return false;
        }
        for (key_a, value_a) in self.iter() {
            for (key_b, value_b) in other.iter() {
                if key_a.equals(heap, key_b) && !value_a.equals(heap, value_b) {
                    return false;
                }
            }
        }
        true
    }
}

impl Closure {
    pub fn of_lir(module: Module, lir: Lir) -> Self {
        Closure {
            captured: vec![],
            num_args: 0,
            body: vec![
                Instruction::TraceModuleStarts { module },
                Instruction::CreateClosure {
                    captured: vec![],
                    num_args: 0,
                    body: lir.instructions,
                },
                Instruction::Call { num_args: 0 },
                Instruction::TraceModuleEnds,
                Instruction::Return,
            ],
        }
    }
    pub fn of_module(db: &Database, module: Module) -> Option<Self> {
        let lir = db.lir(module.clone())?;
        Some(Self::of_lir(module, (*lir).clone()))
    }
}

impl Data {
    fn hash(&self, heap: &Heap) -> u64 {
        let mut state = DefaultHasher::new();
        self.hash_with_state(heap, &mut state);
        state.finish()
    }

    fn hash_with_state<H: Hasher>(&self, heap: &Heap, state: &mut H) {
        match self {
            Data::Int(int) => int.value.hash(state),
            Data::Text(text) => text.value.hash(state),
            Data::Symbol(symbol) => symbol.value.hash(state),
            Data::Struct(struct_) => {
                let mut s = 0;
                for (key, value) in struct_.iter() {
                    s ^= key.hash(heap);
                    s ^= value.hash(heap);
                }
                s.hash(state)
            }
            Data::Closure(closure) => {
                for captured in &closure.captured {
                    captured.hash_with_state(heap, state);
                }
                closure.num_args.hash(state);
                closure.body.hash(state);
            }
            Data::Builtin(builtin) => builtin.function.hash(state),
        }
    }

    pub fn equals(&self, heap: &Heap, other: &Self) -> bool {
        match (self, other) {
            (Data::Int(a), Data::Int(b)) => a.value == b.value,
            (Data::Text(a), Data::Text(b)) => a.value == b.value,
            (Data::Symbol(a), Data::Symbol(b)) => a.value == b.value,
            (Data::Struct(a), Data::Struct(b)) => a.equals(heap, b),
            (Data::Closure(_), Data::Closure(_)) => false,
            (Data::Builtin(a), Data::Builtin(b)) => a.function == b.function,
            _ => false,
        }
    }

    pub fn format(&self, heap: &Heap) -> String {
        match self {
            Data::Int(int) => format!("{}", int.value),
            Data::Text(text) => format!("\"{}\"", text.value),
            Data::Symbol(symbol) => symbol.value.to_string(),
            Data::Struct(struct_) => format!(
                "[{}]",
                struct_
                    .fields
                    .iter()
                    .map(|(_, key, value)| (key.format(heap), value.format(heap)))
                    .sorted_by(|(key_a, _), (key_b, _)| key_a.cmp(key_b))
                    .map(|(key, value)| format!("{}: {}", key, value))
                    .join(", ")
            ),
            Data::Closure(_) => "{...}".to_string(),
            Data::Builtin(builtin) => format!("builtin{:?}", builtin.function),
        }
    }

    pub fn children(&self) -> Vec<Pointer> {
        match self {
            Data::Int(_) | Data::Text(_) | Data::Symbol(_) | Data::Builtin(_) => vec![],
            Data::Struct(struct_) => struct_
                .iter()
                .flat_map(|(a, b)| vec![a, b].into_iter())
                .collect_vec(),
            Data::Closure(closure) => closure.captured.clone(),
        }
    }
}

impl Deref for Object {
    type Target = Data;

    fn deref(&self) -> &Self::Target {
        &self.data
    }
}

impl Pointer {
    fn hash(&self, heap: &Heap) -> u64 {
        heap.get(*self).hash(heap)
    }
    fn hash_with_state<H: Hasher>(&self, heap: &Heap, state: &mut H) {
        heap.get(*self).hash_with_state(heap, state)
    }
    pub fn equals(&self, heap: &Heap, other: Self) -> bool {
        if *self == other {
            return true;
        }
        heap.get(*self).equals(heap, heap.get(other))
    }
    pub fn format(&self, heap: &Heap) -> String {
        heap.get(*self).format(heap)
    }
}
<commit_msg>Use correct ellipsis<commit_after>use super::{pointer::Pointer, Heap};
use crate::{
    builtin_functions::BuiltinFunction,
    compiler::{
        hir_to_lir::HirToLir,
        lir::{Instruction, Lir},
    },
    database::Database,
    module::Module,
};
use itertools::Itertools;
use num_bigint::BigInt;
use std::{
    collections::{hash_map::DefaultHasher, HashMap},
    hash::{Hash, Hasher},
    ops::Deref,
};

#[derive(Clone)]
pub struct Object {
    pub reference_count: usize,
    pub data: Data,
}
#[derive(Clone)]
pub enum Data {
    Int(Int),
    Text(Text),
    Symbol(Symbol),
    Struct(Struct),
    Closure(Closure),
    Builtin(Builtin),
}

#[derive(Clone)]
pub struct Int {
    pub value: BigInt,
}

#[derive(Clone)]
pub struct Text {
    pub value: String,
}

#[derive(Clone)]
pub struct Symbol {
    // TODO: Choose a more efficient representation.
    pub value: String,
}

#[derive(Default, Clone)]
pub struct Struct {
    // TODO: Choose a more efficient representation.
    pub fields: Vec<(u64, Pointer, Pointer)>, // hash, key, and value
}

#[derive(Clone)]
pub struct Closure {
    pub captured: Vec<Pointer>,
    pub num_args: usize,
    pub body: Vec<Instruction>,
}

#[derive(Clone)]
pub struct Builtin {
    pub function: BuiltinFunction,
}

impl Struct {
    pub fn from_fields(heap: &Heap, fields: HashMap<Pointer, Pointer>) -> Self {
        let mut s = Self::default();
        for (key, value) in fields {
            s.insert(heap, key, value);
        }
        s
    }
    fn insert(&mut self, heap: &Heap, key: Pointer, value: Pointer) {
        let hash = key.hash(heap);
        for (existing_hash, existing_key, existing_value) in &mut self.fields {
            if hash != *existing_hash {
                continue;
            }
            if existing_key.equals(heap, key) {
                *existing_value = value;
                return;
            }
        }
        let entry = (hash, key, value);
        if let Some(index) = self
            .fields
            .iter()
            .position(|(the_hash, _, _)| *the_hash > hash)
        {
            self.fields.insert(index, entry);
        } else {
            self.fields.push(entry);
        }
    }
    pub fn get(&self, heap: &Heap, key: Pointer) -> Option<Pointer> {
        let hash = key.hash(heap);
        Some(
            self.fields
                .iter()
                .find(|(field_hash, field_key, _)| {
                    hash == *field_hash && key.equals(heap, *field_key)
                })?
                .2,
        )
    }
    fn len(&self) -> usize {
        self.fields.len()
    }
    pub fn iter(&self) -> impl Iterator<Item = (Pointer, Pointer)> {
        self.fields
            .clone()
            .into_iter()
            .map(|(_, key, value)| (key, value))
    }
    fn equals(&self, heap: &Heap, other: &Struct) -> bool {
        if self.len() != other.len() {
            return false;
        }
        for (key_a, value_a) in self.iter() {
            for (key_b, value_b) in other.iter() {
                if key_a.equals(heap, key_b) && !value_a.equals(heap, value_b) {
                    return false;
                }
            }
        }
        true
    }
}

impl Closure {
    pub fn of_lir(module: Module, lir: Lir) -> Self {
        Closure {
            captured: vec![],
            num_args: 0,
            body: vec![
                Instruction::TraceModuleStarts { module },
                Instruction::CreateClosure {
                    captured: vec![],
                    num_args: 0,
                    body: lir.instructions,
                },
                Instruction::Call { num_args: 0 },
                Instruction::TraceModuleEnds,
                Instruction::Return,
            ],
        }
    }
    pub fn of_module(db: &Database, module: Module) -> Option<Self> {
        let lir = db.lir(module.clone())?;
        Some(Self::of_lir(module, (*lir).clone()))
    }
}

impl Data {
    fn hash(&self, heap: &Heap) -> u64 {
        let mut state = DefaultHasher::new();
        self.hash_with_state(heap, &mut state);
        state.finish()
    }

    fn hash_with_state<H: Hasher>(&self, heap: &Heap, state: &mut H) {
        match self {
            Data::Int(int) => int.value.hash(state),
            Data::Text(text) => text.value.hash(state),
            Data::Symbol(symbol) => symbol.value.hash(state),
            Data::Struct(struct_) => {
                let mut s = 0;
                for (key, value) in struct_.iter() {
                    s ^= key.hash(heap);
                    s ^= value.hash(heap);
                }
                s.hash(state)
            }
            Data::Closure(closure) => {
                for captured in &closure.captured {
                    captured.hash_with_state(heap, state);
                }
                closure.num_args.hash(state);
                closure.body.hash(state);
            }
            Data::Builtin(builtin) => builtin.function.hash(state),
        }
    }

    pub fn equals(&self, heap: &Heap, other: &Self) -> bool {
        match (self, other) {
            (Data::Int(a), Data::Int(b)) => a.value == b.value,
            (Data::Text(a), Data::Text(b)) => a.value == b.value,
            (Data::Symbol(a), Data::Symbol(b)) => a.value == b.value,
            (Data::Struct(a), Data::Struct(b)) => a.equals(heap, b),
            (Data::Closure(_), Data::Closure(_)) => false,
            (Data::Builtin(a), Data::Builtin(b)) => a.function == b.function,
            _ => false,
        }
    }

    pub fn format(&self, heap: &Heap) -> String {
        match self {
            Data::Int(int) => format!("{}", int.value),
            Data::Text(text) => format!("\"{}\"", text.value),
            Data::Symbol(symbol) => symbol.value.to_string(),
            Data::Struct(struct_) => format!(
                "[{}]",
                struct_
                    .fields
                    .iter()
                    .map(|(_, key, value)| (key.format(heap), value.format(heap)))
                    .sorted_by(|(key_a, _), (key_b, _)| key_a.cmp(key_b))
                    .map(|(key, value)| format!("{}: {}", key, value))
                    .join(", ")
            ),
            Data::Closure(_) => "{…}".to_string(),
            Data::Builtin(builtin) => format!("builtin{:?}", builtin.function),
        }
    }

    pub fn children(&self) -> Vec<Pointer> {
        match self {
            Data::Int(_) | Data::Text(_) | Data::Symbol(_) | Data::Builtin(_) => vec![],
            Data::Struct(struct_) => struct_
                .iter()
                .flat_map(|(a, b)| vec![a, b].into_iter())
                .collect_vec(),
            Data::Closure(closure) => closure.captured.clone(),
        }
    }
}

impl Deref for Object {
    type Target = Data;

    fn deref(&self) -> &Self::Target {
        &self.data
    }
}

impl Pointer {
    fn hash(&self, heap: &Heap) -> u64 {
        heap.get(*self).hash(heap)
    }
    fn hash_with_state<H: Hasher>(&self, heap: &Heap, state: &mut H) {
        heap.get(*self).hash_with_state(heap, state)
    }
    pub fn equals(&self, heap: &Heap, other: Self) -> bool {
        if *self == other {
            return true;
        }
        heap.get(*self).equals(heap, heap.get(other))
    }
    pub fn format(&self, heap: &Heap) -> String {
        heap.get(*self).format(heap)
    }
}
<|endoftext|>