<commit_before>//! High-level, futures-enabled Kafka producer.
//!
//! See the [`FutureProducer`] for details.
// TODO: extend docs

use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};

use futures::channel::oneshot;
use futures::FutureExt;

use crate::client::{Client, ClientContext, DefaultClientContext};
use crate::config::{ClientConfig, FromClientConfig, FromClientConfigAndContext, RDKafkaLogLevel};
use crate::error::{KafkaError, KafkaResult, RDKafkaError};
use crate::message::{Message, OwnedHeaders, OwnedMessage, Timestamp, ToBytes};
use crate::producer::{BaseRecord, DeliveryResult, ProducerContext, ThreadedProducer};
use crate::statistics::Statistics;
#[cfg(feature = "tokio")]
use crate::util::TokioRuntime;
use crate::util::{AsyncRuntime, IntoOpaque, Timeout};

//
// ********** FUTURE PRODUCER **********
//

/// A record for the future producer.
///
/// Like [`BaseRecord`], but specific to the [`FutureProducer`]. The only
/// difference is that the [FutureRecord] doesn't provide custom delivery opaque
/// object.
#[derive(Debug)]
pub struct FutureRecord<'a, K: ToBytes + ?Sized + 'a, P: ToBytes + ?Sized + 'a> {
    /// Required destination topic.
    pub topic: &'a str,
    /// Optional destination partition.
    pub partition: Option<i32>,
    /// Optional payload.
    pub payload: Option<&'a P>,
    /// Optional key.
    pub key: Option<&'a K>,
    /// Optional timestamp.
    pub timestamp: Option<i64>,
    /// Optional message headers.
    pub headers: Option<OwnedHeaders>,
}

impl<'a, K: ToBytes + ?Sized, P: ToBytes + ?Sized> FutureRecord<'a, K, P> {
    /// Creates a new record with the specified topic name.
    pub fn to(topic: &'a str) -> FutureRecord<'a, K, P> {
        FutureRecord {
            topic,
            partition: None,
            payload: None,
            key: None,
            timestamp: None,
            headers: None,
        }
    }

    fn from_base_record<D: IntoOpaque>(
        base_record: BaseRecord<'a, K, P, D>,
    ) -> FutureRecord<'a, K, P> {
        FutureRecord {
            topic: base_record.topic,
            partition: base_record.partition,
            key: base_record.key,
            payload: base_record.payload,
            timestamp: base_record.timestamp,
            headers: base_record.headers,
        }
    }

    /// Sets the destination partition of the record.
    pub fn partition(mut self, partition: i32) -> FutureRecord<'a, K, P> {
        self.partition = Some(partition);
        self
    }

    /// Sets the destination payload of the record.
    pub fn payload(mut self, payload: &'a P) -> FutureRecord<'a, K, P> {
        self.payload = Some(payload);
        self
    }

    /// Sets the destination key of the record.
    pub fn key(mut self, key: &'a K) -> FutureRecord<'a, K, P> {
        self.key = Some(key);
        self
    }

    /// Sets the destination timestamp of the record.
    pub fn timestamp(mut self, timestamp: i64) -> FutureRecord<'a, K, P> {
        self.timestamp = Some(timestamp);
        self
    }

    /// Sets the headers of the record.
    pub fn headers(mut self, headers: OwnedHeaders) -> FutureRecord<'a, K, P> {
        self.headers = Some(headers);
        self
    }

    fn into_base_record<D: IntoOpaque>(self, delivery_opaque: D) -> BaseRecord<'a, K, P, D> {
        BaseRecord {
            topic: self.topic,
            partition: self.partition,
            key: self.key,
            payload: self.payload,
            timestamp: self.timestamp,
            headers: self.headers,
            delivery_opaque,
        }
    }
}

/// The [`ProducerContext`] used by the [`FutureProducer`].
///
/// This context will use a [`Future`] as its `DeliveryOpaque` and will complete
/// the future when the message is delivered (or failed to).
#[derive(Clone)]
pub struct FutureProducerContext<C: ClientContext + 'static> {
    wrapped_context: C,
}

/// Represents the result of message production as performed from the
/// `FutureProducer`.
///
/// If message delivery was successful, `OwnedDeliveryResult` will return the
/// partition and offset of the message. If the message failed to be delivered
/// an error will be returned, together with an owned copy of the original
/// message.
type OwnedDeliveryResult = Result<(i32, i64), (KafkaError, OwnedMessage)>;

// Delegates all the methods calls to the wrapped context.
impl<C: ClientContext + 'static> ClientContext for FutureProducerContext<C> {
    fn log(&self, level: RDKafkaLogLevel, fac: &str, log_message: &str) {
        self.wrapped_context.log(level, fac, log_message);
    }

    fn stats(&self, statistics: Statistics) {
        self.wrapped_context.stats(statistics);
    }

    fn error(&self, error: KafkaError, reason: &str) {
        self.wrapped_context.error(error, reason);
    }
}

impl<C: ClientContext + 'static> ProducerContext for FutureProducerContext<C> {
    type DeliveryOpaque = Box<oneshot::Sender<OwnedDeliveryResult>>;

    fn delivery(
        &self,
        delivery_result: &DeliveryResult,
        tx: Box<oneshot::Sender<OwnedDeliveryResult>>,
    ) {
        let owned_delivery_result = match *delivery_result {
            Ok(ref message) => Ok((message.partition(), message.offset())),
            Err((ref error, ref message)) => Err((error.clone(), message.detach())),
        };
        let _ = tx.send(owned_delivery_result); // TODO: handle error
    }
}

/// A producer that returns a [`Future`] for every message being produced.
///
/// Since message production in rdkafka is asynchronous, the caller cannot
/// immediately know if the delivery of the message was successful or not. The
/// FutureProducer provides this information in a [`Future`], which will be
/// completed once the information becomes available.
///
/// This producer has an internal polling thread and as such it doesn't need to
/// be polled. It can be cheaply cloned to get a reference to the same
/// underlying producer. The internal polling thread will be terminated when the
/// `FutureProducer` goes out of scope.
#[must_use = "Producer polling thread will stop immediately if unused"]
pub struct FutureProducer<C: ClientContext + 'static = DefaultClientContext> {
    producer: Arc<ThreadedProducer<FutureProducerContext<C>>>,
}

impl<C: ClientContext + 'static> Clone for FutureProducer<C> {
    fn clone(&self) -> FutureProducer<C> {
        FutureProducer {
            producer: self.producer.clone(),
        }
    }
}

impl FromClientConfig for FutureProducer {
    fn from_config(config: &ClientConfig) -> KafkaResult<FutureProducer> {
        FutureProducer::from_config_and_context(config, DefaultClientContext)
    }
}

impl<C: ClientContext + 'static> FromClientConfigAndContext<C> for FutureProducer<C> {
    fn from_config_and_context(
        config: &ClientConfig,
        context: C,
    ) -> KafkaResult<FutureProducer<C>> {
        let future_context = FutureProducerContext {
            wrapped_context: context,
        };
        let threaded_producer = ThreadedProducer::from_config_and_context(config, future_context)?;
        Ok(FutureProducer {
            producer: Arc::new(threaded_producer),
        })
    }
}

/// A [`Future`] wrapping the result of the message production.
///
/// Once completed, the future will contain an `OwnedDeliveryResult` with
/// information on the delivery status of the message. If the producer is
/// dropped before the delivery status is received, the future will instead
/// resolve with [`oneshot::Canceled`].
pub struct DeliveryFuture {
    rx: oneshot::Receiver<OwnedDeliveryResult>,
}

impl Future for DeliveryFuture {
    type Output = Result<OwnedDeliveryResult, oneshot::Canceled>;

    fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
        self.rx.poll_unpin(cx)
    }
}

impl<C: ClientContext + 'static> FutureProducer<C> {
    /// Sends a message to Kafka, returning the result of the send.
    ///
    /// The `queue_timeout` parameter controls how long to retry for if the
    /// librdkafka producer queue is full. Set it to `Timeout::Never` to retry
    /// forever or `Timeout::After(0)` to never block. If the timeout is reached
    /// and the queue is still full, an [`RDKafkaError::QueueFull`] error will
    /// be reported in the [`OwnedDeliveryResult`].
    ///
    /// Keep in mind that `queue_timeout` only applies to the first phase of the
    /// send operation. Once the message is queued, the underlying librdkafka
    /// client has separate timeout parameters that apply, like
    /// `delivery.timeout.ms`.
    ///
    /// See also the [`FutureProducer::send_result`] method, which will not
    /// retry the queue operation if the queue is full.
    ///
    /// **Note:** this method must be called from within the context of a Tokio
    /// runtime.
    #[cfg(feature = "tokio")]
    #[cfg_attr(docsrs, doc(cfg(feature = "tokio")))]
    pub async fn send<K, P, T>(
        &self,
        record: FutureRecord<'_, K, P>,
        queue_timeout: T,
    ) -> OwnedDeliveryResult
    where
        K: ToBytes + ?Sized,
        P: ToBytes + ?Sized,
        T: Into<Timeout>,
    {
        self.send_with_runtime::<TokioRuntime, _, _, _>(record, queue_timeout)
            .await
    }

    /// Like [`FutureProducer::send`], but with a customizable asynchronous
    /// runtime.
    ///
    /// See the [`AsyncRuntime`] trait for the details on the interface the
    /// runtime must satisfy.
    pub async fn send_with_runtime<R, K, P, T>(
        &self,
        record: FutureRecord<'_, K, P>,
        queue_timeout: T,
    ) -> OwnedDeliveryResult
    where
        R: AsyncRuntime,
        K: ToBytes + ?Sized,
        P: ToBytes + ?Sized,
        T: Into<Timeout>,
    {
        let start_time = Instant::now();
        let queue_timeout = queue_timeout.into();
        let can_retry = || match queue_timeout.into() {
            Timeout::Never => true,
            Timeout::After(t) if start_time.elapsed() < t => true,
            _ => false,
        };

        let (tx, rx) = oneshot::channel();
        let mut base_record = record.into_base_record(Box::new(tx));

        loop {
            match self.producer.send(base_record) {
                Err((e, record))
                    if e == KafkaError::MessageProduction(RDKafkaError::QueueFull)
                        && can_retry() =>
                {
                    base_record = record;
                    R::delay_for(Duration::from_millis(100)).await;
                }
                Ok(_) => {
                    // We hold a reference to the producer, so it should not be
                    // possible for the producer to vanish and cancel the
                    // oneshot.
                    break rx.await.expect("producer unexpectedly dropped");
                }
                Err((e, record)) => {
                    let owned_message = OwnedMessage::new(
                        record.payload.map(|p| p.to_bytes().to_vec()),
                        record.key.map(|k| k.to_bytes().to_vec()),
                        record.topic.to_owned(),
                        record
                            .timestamp
                            .map_or(Timestamp::NotAvailable, Timestamp::CreateTime),
                        record.partition.unwrap_or(-1),
                        0,
                        record.headers,
                    );
                    break Err((e, owned_message));
                }
            }
        }
    }

    /// Like [`FutureProducer::send`], but if enqueuing fails, an error will be
    /// returned immediately, alongside the [`FutureRecord`] provided.
    pub fn send_result<'a, K, P>(
        &self,
        record: FutureRecord<'a, K, P>,
    ) -> Result<DeliveryFuture, (KafkaError, FutureRecord<'a, K, P>)>
    where
        K: ToBytes + ?Sized,
        P: ToBytes + ?Sized,
    {
        let (tx, rx) = oneshot::channel();
        let base_record = record.into_base_record(Box::new(tx));
        self.producer
            .send(base_record)
            .map(|()| DeliveryFuture { rx })
            .map_err(|(e, record)| (e, FutureRecord::from_base_record(record)))
    }

    /// Polls the internal producer.
    ///
    /// This is not normally required since the `FutureProducer` had a thread
    /// dedicated to calling `poll` regularly.
    pub fn poll<T: Into<Timeout>>(&self, timeout: T) {
        self.producer.poll(timeout);
    }

    /// Flushes any pending messages.
    ///
    /// This method should be called before termination to ensure delivery of
    /// all enqueued messages.
    pub fn flush<T: Into<Timeout>>(&self, timeout: T) {
        self.producer.flush(timeout);
    }

    /// Returns the number of messages that are either waiting to be sent or are
    /// sent but are waiting to be acknowledged.
    pub fn in_flight_count(&self) -> i32 {
        self.producer.in_flight_count()
    }

    /// Returns the [`Client`] underlying this producer.
    pub fn client(&self) -> &Client<FutureProducerContext<C>> {
        self.producer.client()
    }
}

#[cfg(test)]
mod tests {
    // Just test that there are no panics, and that each struct implements the expected
    // traits (Clone, Send, Sync etc.). Behavior is tested in the integrations tests.
    use super::*;
    use crate::config::ClientConfig;

    struct TestContext;

    impl ClientContext for TestContext {}
    impl ProducerContext for TestContext {
        type DeliveryOpaque = Box<i32>;

        fn delivery(&self, _: &DeliveryResult, _: Self::DeliveryOpaque) {
            unimplemented!()
        }
    }

    // Verify that the future producer is clone, according to documentation.
    #[test]
    fn test_future_producer_clone() {
        let producer = ClientConfig::new().create::<FutureProducer>().unwrap();
        let _producer_clone = producer.clone();
    }

    // Test that the future producer can be cloned even if the context is not Clone.
    #[test]
    fn test_base_future_topic_send_sync() {
        let test_context = TestContext;
        let producer = ClientConfig::new()
            .create_with_context::<_, FutureProducer<TestContext>>(test_context)
            .unwrap();
        let _producer_clone = producer.clone();
    }
}
<commit_msg>Mark OwnedDeliveryResult as public<commit_after>//! High-level, futures-enabled Kafka producer.
//!
//! See the [`FutureProducer`] for details.
// TODO: extend docs

use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};

use futures::channel::oneshot;
use futures::FutureExt;

use crate::client::{Client, ClientContext, DefaultClientContext};
use crate::config::{ClientConfig, FromClientConfig, FromClientConfigAndContext, RDKafkaLogLevel};
use crate::error::{KafkaError, KafkaResult, RDKafkaError};
use crate::message::{Message, OwnedHeaders, OwnedMessage, Timestamp, ToBytes};
use crate::producer::{BaseRecord, DeliveryResult, ProducerContext, ThreadedProducer};
use crate::statistics::Statistics;
#[cfg(feature = "tokio")]
use crate::util::TokioRuntime;
use crate::util::{AsyncRuntime, IntoOpaque, Timeout};

//
// ********** FUTURE PRODUCER **********
//

/// A record for the future producer.
///
/// Like [`BaseRecord`], but specific to the [`FutureProducer`]. The only
/// difference is that the [FutureRecord] doesn't provide custom delivery opaque
/// object.
#[derive(Debug)]
pub struct FutureRecord<'a, K: ToBytes + ?Sized + 'a, P: ToBytes + ?Sized + 'a> {
    /// Required destination topic.
    pub topic: &'a str,
    /// Optional destination partition.
    pub partition: Option<i32>,
    /// Optional payload.
    pub payload: Option<&'a P>,
    /// Optional key.
    pub key: Option<&'a K>,
    /// Optional timestamp.
    pub timestamp: Option<i64>,
    /// Optional message headers.
    pub headers: Option<OwnedHeaders>,
}

impl<'a, K: ToBytes + ?Sized, P: ToBytes + ?Sized> FutureRecord<'a, K, P> {
    /// Creates a new record with the specified topic name.
    pub fn to(topic: &'a str) -> FutureRecord<'a, K, P> {
        FutureRecord {
            topic,
            partition: None,
            payload: None,
            key: None,
            timestamp: None,
            headers: None,
        }
    }

    fn from_base_record<D: IntoOpaque>(
        base_record: BaseRecord<'a, K, P, D>,
    ) -> FutureRecord<'a, K, P> {
        FutureRecord {
            topic: base_record.topic,
            partition: base_record.partition,
            key: base_record.key,
            payload: base_record.payload,
            timestamp: base_record.timestamp,
            headers: base_record.headers,
        }
    }

    /// Sets the destination partition of the record.
    pub fn partition(mut self, partition: i32) -> FutureRecord<'a, K, P> {
        self.partition = Some(partition);
        self
    }

    /// Sets the destination payload of the record.
    pub fn payload(mut self, payload: &'a P) -> FutureRecord<'a, K, P> {
        self.payload = Some(payload);
        self
    }

    /// Sets the destination key of the record.
    pub fn key(mut self, key: &'a K) -> FutureRecord<'a, K, P> {
        self.key = Some(key);
        self
    }

    /// Sets the destination timestamp of the record.
    pub fn timestamp(mut self, timestamp: i64) -> FutureRecord<'a, K, P> {
        self.timestamp = Some(timestamp);
        self
    }

    /// Sets the headers of the record.
    pub fn headers(mut self, headers: OwnedHeaders) -> FutureRecord<'a, K, P> {
        self.headers = Some(headers);
        self
    }

    fn into_base_record<D: IntoOpaque>(self, delivery_opaque: D) -> BaseRecord<'a, K, P, D> {
        BaseRecord {
            topic: self.topic,
            partition: self.partition,
            key: self.key,
            payload: self.payload,
            timestamp: self.timestamp,
            headers: self.headers,
            delivery_opaque,
        }
    }
}

/// The [`ProducerContext`] used by the [`FutureProducer`].
///
/// This context will use a [`Future`] as its `DeliveryOpaque` and will complete
/// the future when the message is delivered (or failed to).
#[derive(Clone)]
pub struct FutureProducerContext<C: ClientContext + 'static> {
    wrapped_context: C,
}

/// Represents the result of message production as performed from the
/// `FutureProducer`.
///
/// If message delivery was successful, `OwnedDeliveryResult` will return the
/// partition and offset of the message. If the message failed to be delivered
/// an error will be returned, together with an owned copy of the original
/// message.
pub type OwnedDeliveryResult = Result<(i32, i64), (KafkaError, OwnedMessage)>;

// Delegates all the methods calls to the wrapped context.
impl<C: ClientContext + 'static> ClientContext for FutureProducerContext<C> {
    fn log(&self, level: RDKafkaLogLevel, fac: &str, log_message: &str) {
        self.wrapped_context.log(level, fac, log_message);
    }

    fn stats(&self, statistics: Statistics) {
        self.wrapped_context.stats(statistics);
    }

    fn error(&self, error: KafkaError, reason: &str) {
        self.wrapped_context.error(error, reason);
    }
}

impl<C: ClientContext + 'static> ProducerContext for FutureProducerContext<C> {
    type DeliveryOpaque = Box<oneshot::Sender<OwnedDeliveryResult>>;

    fn delivery(
        &self,
        delivery_result: &DeliveryResult,
        tx: Box<oneshot::Sender<OwnedDeliveryResult>>,
    ) {
        let owned_delivery_result = match *delivery_result {
            Ok(ref message) => Ok((message.partition(), message.offset())),
            Err((ref error, ref message)) => Err((error.clone(), message.detach())),
        };
        let _ = tx.send(owned_delivery_result); // TODO: handle error
    }
}

/// A producer that returns a [`Future`] for every message being produced.
///
/// Since message production in rdkafka is asynchronous, the caller cannot
/// immediately know if the delivery of the message was successful or not. The
/// FutureProducer provides this information in a [`Future`], which will be
/// completed once the information becomes available.
///
/// This producer has an internal polling thread and as such it doesn't need to
/// be polled. It can be cheaply cloned to get a reference to the same
/// underlying producer. The internal polling thread will be terminated when the
/// `FutureProducer` goes out of scope.
#[must_use = "Producer polling thread will stop immediately if unused"]
pub struct FutureProducer<C: ClientContext + 'static = DefaultClientContext> {
    producer: Arc<ThreadedProducer<FutureProducerContext<C>>>,
}

impl<C: ClientContext + 'static> Clone for FutureProducer<C> {
    fn clone(&self) -> FutureProducer<C> {
        FutureProducer {
            producer: self.producer.clone(),
        }
    }
}

impl FromClientConfig for FutureProducer {
    fn from_config(config: &ClientConfig) -> KafkaResult<FutureProducer> {
        FutureProducer::from_config_and_context(config, DefaultClientContext)
    }
}

impl<C: ClientContext + 'static> FromClientConfigAndContext<C> for FutureProducer<C> {
    fn from_config_and_context(
        config: &ClientConfig,
        context: C,
    ) -> KafkaResult<FutureProducer<C>> {
        let future_context = FutureProducerContext {
            wrapped_context: context,
        };
        let threaded_producer = ThreadedProducer::from_config_and_context(config, future_context)?;
        Ok(FutureProducer {
            producer: Arc::new(threaded_producer),
        })
    }
}

/// A [`Future`] wrapping the result of the message production.
///
/// Once completed, the future will contain an `OwnedDeliveryResult` with
/// information on the delivery status of the message. If the producer is
/// dropped before the delivery status is received, the future will instead
/// resolve with [`oneshot::Canceled`].
pub struct DeliveryFuture {
    rx: oneshot::Receiver<OwnedDeliveryResult>,
}

impl Future for DeliveryFuture {
    type Output = Result<OwnedDeliveryResult, oneshot::Canceled>;

    fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
        self.rx.poll_unpin(cx)
    }
}

impl<C: ClientContext + 'static> FutureProducer<C> {
    /// Sends a message to Kafka, returning the result of the send.
    ///
    /// The `queue_timeout` parameter controls how long to retry for if the
    /// librdkafka producer queue is full. Set it to `Timeout::Never` to retry
    /// forever or `Timeout::After(0)` to never block. If the timeout is reached
    /// and the queue is still full, an [`RDKafkaError::QueueFull`] error will
    /// be reported in the [`OwnedDeliveryResult`].
    ///
    /// Keep in mind that `queue_timeout` only applies to the first phase of the
    /// send operation. Once the message is queued, the underlying librdkafka
    /// client has separate timeout parameters that apply, like
    /// `delivery.timeout.ms`.
    ///
    /// See also the [`FutureProducer::send_result`] method, which will not
    /// retry the queue operation if the queue is full.
    ///
    /// **Note:** this method must be called from within the context of a Tokio
    /// runtime.
    #[cfg(feature = "tokio")]
    #[cfg_attr(docsrs, doc(cfg(feature = "tokio")))]
    pub async fn send<K, P, T>(
        &self,
        record: FutureRecord<'_, K, P>,
        queue_timeout: T,
    ) -> OwnedDeliveryResult
    where
        K: ToBytes + ?Sized,
        P: ToBytes + ?Sized,
        T: Into<Timeout>,
    {
        self.send_with_runtime::<TokioRuntime, _, _, _>(record, queue_timeout)
            .await
    }

    /// Like [`FutureProducer::send`], but with a customizable asynchronous
    /// runtime.
    ///
    /// See the [`AsyncRuntime`] trait for the details on the interface the
    /// runtime must satisfy.
    pub async fn send_with_runtime<R, K, P, T>(
        &self,
        record: FutureRecord<'_, K, P>,
        queue_timeout: T,
    ) -> OwnedDeliveryResult
    where
        R: AsyncRuntime,
        K: ToBytes + ?Sized,
        P: ToBytes + ?Sized,
        T: Into<Timeout>,
    {
        let start_time = Instant::now();
        let queue_timeout = queue_timeout.into();
        let can_retry = || match queue_timeout.into() {
            Timeout::Never => true,
            Timeout::After(t) if start_time.elapsed() < t => true,
            _ => false,
        };

        let (tx, rx) = oneshot::channel();
        let mut base_record = record.into_base_record(Box::new(tx));

        loop {
            match self.producer.send(base_record) {
                Err((e, record))
                    if e == KafkaError::MessageProduction(RDKafkaError::QueueFull)
                        && can_retry() =>
                {
                    base_record = record;
                    R::delay_for(Duration::from_millis(100)).await;
                }
                Ok(_) => {
                    // We hold a reference to the producer, so it should not be
                    // possible for the producer to vanish and cancel the
                    // oneshot.
                    break rx.await.expect("producer unexpectedly dropped");
                }
                Err((e, record)) => {
                    let owned_message = OwnedMessage::new(
                        record.payload.map(|p| p.to_bytes().to_vec()),
                        record.key.map(|k| k.to_bytes().to_vec()),
                        record.topic.to_owned(),
                        record
                            .timestamp
                            .map_or(Timestamp::NotAvailable, Timestamp::CreateTime),
                        record.partition.unwrap_or(-1),
                        0,
                        record.headers,
                    );
                    break Err((e, owned_message));
                }
            }
        }
    }

    /// Like [`FutureProducer::send`], but if enqueuing fails, an error will be
    /// returned immediately, alongside the [`FutureRecord`] provided.
    pub fn send_result<'a, K, P>(
        &self,
        record: FutureRecord<'a, K, P>,
    ) -> Result<DeliveryFuture, (KafkaError, FutureRecord<'a, K, P>)>
    where
        K: ToBytes + ?Sized,
        P: ToBytes + ?Sized,
    {
        let (tx, rx) = oneshot::channel();
        let base_record = record.into_base_record(Box::new(tx));
        self.producer
            .send(base_record)
            .map(|()| DeliveryFuture { rx })
            .map_err(|(e, record)| (e, FutureRecord::from_base_record(record)))
    }

    /// Polls the internal producer.
    ///
    /// This is not normally required since the `FutureProducer` had a thread
    /// dedicated to calling `poll` regularly.
    pub fn poll<T: Into<Timeout>>(&self, timeout: T) {
        self.producer.poll(timeout);
    }

    /// Flushes any pending messages.
    ///
    /// This method should be called before termination to ensure delivery of
    /// all enqueued messages.
    pub fn flush<T: Into<Timeout>>(&self, timeout: T) {
        self.producer.flush(timeout);
    }

    /// Returns the number of messages that are either waiting to be sent or are
    /// sent but are waiting to be acknowledged.
    pub fn in_flight_count(&self) -> i32 {
        self.producer.in_flight_count()
    }

    /// Returns the [`Client`] underlying this producer.
    pub fn client(&self) -> &Client<FutureProducerContext<C>> {
        self.producer.client()
    }
}

#[cfg(test)]
mod tests {
    // Just test that there are no panics, and that each struct implements the expected
    // traits (Clone, Send, Sync etc.). Behavior is tested in the integrations tests.
    use super::*;
    use crate::config::ClientConfig;

    struct TestContext;

    impl ClientContext for TestContext {}
    impl ProducerContext for TestContext {
        type DeliveryOpaque = Box<i32>;

        fn delivery(&self, _: &DeliveryResult, _: Self::DeliveryOpaque) {
            unimplemented!()
        }
    }

    // Verify that the future producer is clone, according to documentation.
    #[test]
    fn test_future_producer_clone() {
        let producer = ClientConfig::new().create::<FutureProducer>().unwrap();
        let _producer_clone = producer.clone();
    }

    // Test that the future producer can be cloned even if the context is not Clone.
    #[test]
    fn test_base_future_topic_send_sync() {
        let test_context = TestContext;
        let producer = ClientConfig::new()
            .create_with_context::<_, FutureProducer<TestContext>>(test_context)
            .unwrap();
        let _producer_clone = producer.clone();
    }
}
<|endoftext|>


<commit_before>/// This is a test for needless `fn main()` in doctests.
///
/// # Examples
///
/// This should lint
/// ```
/// fn main() {
///     unimplemented!();
/// }
/// ```
///
/// This should, too.
///
/// ```rust
/// fn main() {
///     unimplemented!();
/// }
/// ```
///
/// This one too.
///
/// ```no_run
/// fn main() {
///     unimplemented!();
/// }
/// ```
fn bad_doctests() {}

/// # Examples
///
/// This shouldn't lint, because the `main` is empty:
/// ```
/// fn main(){}
/// ```
///
/// This shouldn't lint either, because there's a `static`:
/// ```
/// static ANSWER: i32 = 42;
///
/// fn main() {
///     assert_eq!(42, ANSWER);
/// }
/// ```
///
/// Neither should this lint because of `extern crate`:
/// ```
/// #![feature(test)]
/// extern crate test;
/// fn main() {
///     assert_eq(1u8, test::black_box(1));
/// }
/// ```
///
/// We should not lint non-rust examples:
///
/// ```rust,ignore
/// fn main() {
///     unimplemented!();
/// }
/// ```
///
/// Or even non-rust examples:
///
/// ```text
/// fn main() {
///     is what starts the program
/// }
/// ```
fn no_false_positives() {}

fn main() {
    bad_doctests();
    no_false_positives();
}
<commit_msg>Update tests/ui/needless_doc_main.rs<commit_after>/// This is a test for needless `fn main()` in doctests.
///
/// # Examples
///
/// This should lint
/// ```
/// fn main() {
///     unimplemented!();
/// }
/// ```
///
/// This should, too.
///
/// ```rust
/// fn main() {
///     unimplemented!();
/// }
/// ```
///
/// This one too.
///
/// ```no_run
/// fn main() {
///     unimplemented!();
/// }
/// ```
fn bad_doctests() {}

/// # Examples
///
/// This shouldn't lint, because the `main` is empty:
/// ```
/// fn main(){}
/// ```
///
/// This shouldn't lint either, because there's a `static`:
/// ```
/// static ANSWER: i32 = 42;
///
/// fn main() {
///     assert_eq!(42, ANSWER);
/// }
/// ```
///
/// Neither should this lint because of `extern crate`:
/// ```
/// #![feature(test)]
/// extern crate test;
/// fn main() {
///     assert_eq(1u8, test::black_box(1));
/// }
/// ```
///
/// We should not lint ignored examples:
///
/// ```rust,ignore
/// fn main() {
///     unimplemented!();
/// }
/// ```
///
/// Or even non-rust examples:
///
/// ```text
/// fn main() {
///     is what starts the program
/// }
/// ```
fn no_false_positives() {}

fn main() {
    bad_doctests();
    no_false_positives();
}
<|endoftext|>


<commit_before>use std::any::Any;
use std::fmt::Debug;

use frontend::radeco_containers::{RadecoFunction, RadecoModule};

/// This trait provides access to extra informations generated during the analysis pass.
pub trait AnalyzerResult : Any + Debug { }

/// Kind of `Analyzer`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AnalyzerKind { }

/// Basic trait for all analyzers.
pub trait Analyzer : Any + Debug {
    /// Return the name of this `Analyzer`.
    fn name(&self) -> String;

    /// Return the kind of this `Analyzer`.
    fn kind(&self) -> AnalyzerKind;
}

/// An `Analyzer` that takes a function.
pub trait FuncAnalyzer : Analyzer {
    fn analyze(&mut self, func: &mut RadecoFunction) -> Option<Box<AnalyzerResult>>;
}

/// An `Analyzer` that takes a module.
pub trait ModuleAnalyzer : Analyzer {
    fn analyze(&mut self, module: &mut RadecoModule) -> Option<Box<AnalyzerResult>>;
}

/// Get all the available `FuncAnalyzer`s
pub fn all_func_analysis() -> Vec<AnalyzerKind> {
    vec![]
}

/// Get all the available `ModuleAnalyzer`s
pub fn all_module_analysis() -> Vec<AnalyzerKind> {
    vec![]
}
<commit_msg>Add `requires` api<commit_after>use std::any::Any;
use std::fmt::Debug;

use frontend::radeco_containers::{RadecoFunction, RadecoModule};

/// This trait provides access to extra informations generated during the analysis pass.
pub trait AnalyzerResult : Any + Debug { }

/// Kind of `Analyzer`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AnalyzerKind { }

/// Basic trait for all analyzers.
pub trait Analyzer : Any + Debug {
    /// Return the name of this `Analyzer`.
    fn name(&self) -> String;

    /// Return the kind of this `Analyzer`.
    fn kind(&self) -> AnalyzerKind;

    /// Return a list of `Analyzer`s to run before this one.
    fn requires(&self) -> Vec<AnalyzerKind>;
}

/// An `Analyzer` that takes a function.
pub trait FuncAnalyzer : Analyzer {
    fn analyze(&mut self, func: &mut RadecoFunction) -> Option<Box<AnalyzerResult>>;
}

/// An `Analyzer` that takes a module.
pub trait ModuleAnalyzer : Analyzer {
    fn analyze(&mut self, module: &mut RadecoModule) -> Option<Box<AnalyzerResult>>;
}

/// Get all the available `FuncAnalyzer`s
pub fn all_func_analysis() -> Vec<AnalyzerKind> {
    vec![]
}

/// Get all the available `ModuleAnalyzer`s
pub fn all_module_analysis() -> Vec<AnalyzerKind> {
    vec![]
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;
extern crate rls_analysis;
extern crate rls_data as data;
extern crate serde;
#[macro_use]
extern crate serde_json;

use data::GlobalCrateId;
use rls_analysis::{AnalysisHost, AnalysisLoader};
use std::collections::HashMap;
use std::fs::{self, File};
use std::path::{Path, PathBuf};

/// A global definition id in a crate.
///
/// FIXME(emilio): This key is kind of slow, because GlobalCrateId contains a
/// String. There's a "disambiguator" field which may be more than enough for
/// our purposes.
#[derive(Clone, Hash, Debug, Eq, PartialEq)]
pub struct DefId(GlobalCrateId, u32);

/// A map from global definition ids to the actual definition.
pub struct Defs {
    map: HashMap<DefId, data::Def>,
}

impl Defs {
    fn new() -> Self {
        Self { map: HashMap::new() }
    }

    fn insert(&mut self, analysis: &data::Analysis, def: &data::Def) {
        let crate_id = analysis.prelude.as_ref().unwrap().crate_id.clone();
        let mut definition = def.clone();
        let crate_independent_qualname =
            format!("{}{}", crate_id.name, def.qualname);
        definition.qualname = crate_independent_qualname;

        let index = definition.id.index;
        let previous = self.map.insert(DefId(crate_id, index), definition);
        assert!(
            previous.is_none(),
            "Found a definition with the same ID twice? {:?}",
            def,
        );
    }

    /// Getter for a given local id, which takes care of converting to a global
    /// ID and returning the definition if present.
    fn get(&self, analysis: &data::Analysis, id: data::Id) -> Option<data::Def> {
        let prelude = analysis.prelude.as_ref().unwrap();
        let krate_id = if id.krate == 0 {
            prelude.crate_id.clone()
        } else {
            // TODO(emilio): This escales with the number of crates in this
            // particular crate, but it's probably not too bad, since it should
            // be a pretty fast linear search.
            let krate = prelude.external_crates.iter().find(|krate| {
                krate.num == id.krate
            });

            let krate = match krate {
                Some(k) => k,
                None => {
                    eprintln!("Crate not found: {:?}", id);
                    return None;
                }
            };

            krate.id.clone()
        };

        let id = DefId(krate_id, id.index);
        self.map.get(&id).cloned()
    }
}

#[derive(Clone)]
pub struct Loader {
    deps_dir: PathBuf,
}

impl Loader {
    pub fn new(deps_dir: PathBuf) -> Self {
        Self { deps_dir }
    }
}

impl AnalysisLoader for Loader {
    fn needs_hard_reload(&self, _: &Path) -> bool {
        true
    }

    fn fresh_host(&self) -> AnalysisHost<Self> {
        AnalysisHost::new_with_loader(self.clone())
    }

    fn set_path_prefix(&mut self, _: &Path) {}

    fn abs_path_prefix(&self) -> Option<PathBuf> {
        None
    }
    fn search_directories(&self) -> Vec<PathBuf> {
        vec![self.deps_dir.clone()]
    }
}

// Searchfox uses 1-indexed lines, 0-indexed columns.
fn span_to_string(span: &data::SpanData) -> String {
    // Rust spans are multi-line... So we just set the start column if it spans
    // multiple rows, searchfox has fallback code to handle this.
    if span.line_start != span.line_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    if span.column_start == span.column_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    format!("{}:{}-{}", span.line_start.0, span.column_start.0 - 1, span.column_end.0 - 1)
}

fn visit(
    file: &mut File,
    kind: &'static str,
    location: &data::SpanData,
    qualname: &str,
    context: Option<&str>,
) {
    use serde_json::map::Map;
    use serde_json::value::Value;
    use std::io::Write;

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("target".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));
    if let Some(context) = context {
        out.insert("context".into(), Value::String(context.into()));
        out.insert("contextsym".into(), Value::String(context.into()));
    }

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("source".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();
}

fn analyze_file(
    file_name: &PathBuf,
    defs: &Defs,
    file_analysis: &data::Analysis,
    src_dir: &Path,
    output_dir: &Path,
) {
    let file = match file_name.strip_prefix(src_dir) {
        Ok(f) => f,
        Err(..) => {
            eprintln!("File not in the source directory: {}", file_name.display());
            return;
        }
    };

    let output_file = output_dir.join(file);
    let mut output_dir = output_file.clone();
    output_dir.pop();
    if let Err(err) = fs::create_dir_all(output_dir) {
        eprintln!(
            "Couldn't create dir for: {}, {:?}",
            output_file.display(),
            err
        );
        return;
    }
    let mut file = match File::create(&output_file) {
        Ok(f) => f,
        Err(err) => {
            eprintln!(
                "Couldn't open output file: {}, {:?}",
                output_file.display(),
                err
            );
            return;
        }
    };

    for import in &file_analysis.imports {
        let id = match import.ref_id {
            Some(id) => id,
            None => {
                eprintln!("Dropping import {}: {}, no ref", import.name, import.value);
                continue;
            }
        };

        let def = match defs.get(file_analysis, id) {
            Some(def) => def,
            None => {
                eprintln!("Dropping import {}: {}, no def for ref {:?}", import.name, import.value, id);
                continue;
            }
        };

        visit(&mut file, "use", &import.span, &def.qualname, None)
    }

    for def in &file_analysis.defs {
        let parent = def.parent
            .and_then(|parent_id| defs.get(file_analysis, parent_id).map(|d| d.qualname));

        let crate_name = &file_analysis.prelude.as_ref().unwrap().crate_id.name;
        let qualname = format!("{}{}", crate_name, def.qualname);
        visit(&mut file, "def", &def.span, &qualname, parent.as_ref().map(|p| &**p))
    }

    for ref_ in &file_analysis.refs {
        let def = match defs.get(file_analysis, ref_.ref_id) {
            Some(d) => d,
            None => {
                eprintln!("Dropping ref {:?}, kind {:?}, no def", ref_.ref_id, ref_.kind);
                continue;
            }
        };
        visit(
            &mut file,
            "use",
            &ref_.span,
            &def.qualname,
            /* context = */ None, // TODO
        )
    }
}

fn analyze_crate(
    analysis: &data::Analysis,
    defs: &Defs,
    src_dir: &Path,
    output_dir: &Path,
) {
    let mut per_file = HashMap::new();

    println!("Analyzing crate: {:?}", analysis.prelude);

    macro_rules! flat_map_per_file {
        ($field:ident) => {
            for item in &analysis.$field {
                let mut file_analysis =
                    per_file.entry(item.span.file_name.clone())
                        .or_insert_with(|| {
                            let prelude = analysis.prelude.clone();
                            let mut analysis = data::Analysis::new(analysis.config.clone());
                            analysis.prelude = prelude;
                            analysis
                        });
                file_analysis.$field.push(item.clone());
            }
        }
    }

    flat_map_per_file!(imports);
    flat_map_per_file!(defs);
    flat_map_per_file!(impls);
    flat_map_per_file!(refs);
    flat_map_per_file!(macro_refs);
    flat_map_per_file!(relations);

    for (mut name, analysis) in per_file.drain() {
        if name.is_relative() {
            name = src_dir.join(name);
        }
        analyze_file(&name, defs, &analysis, src_dir, output_dir);
    }
}

fn main() {
    let matches = app_from_crate!()
        .args_from_usage(
            "<src>    'Points to the source root'
             <input>  'Points to the deps/save-analysis directory'
             <output> 'Points to the directory where searchfox metadata should go'",
        )
        .get_matches();

    let src_dir = Path::new(matches.value_of("src").unwrap());
    let input_dir = Path::new(matches.value_of("input").unwrap());
    let output_dir = Path::new(matches.value_of("output").unwrap());

    let loader = Loader::new(PathBuf::from(input_dir));

    let crates = rls_analysis::read_analysis_from_files(&loader, Default::default(), &[]);

    let mut defs = Defs::new();
    for krate in &crates {
        for def in &krate.analysis.defs {
            println!("Indexing def: {:?}", def);
            defs.insert(&krate.analysis, def);
        }
    }

    for krate in crates {
        analyze_crate(&krate.analysis, &defs, &src_dir, &output_dir);
    }
}
<commit_msg>rust-indexer: Don't panic when finding a dupe definition.<commit_after>#[macro_use]
extern crate clap;
extern crate rls_analysis;
extern crate rls_data as data;
extern crate serde;
#[macro_use]
extern crate serde_json;

use data::GlobalCrateId;
use rls_analysis::{AnalysisHost, AnalysisLoader};
use std::collections::HashMap;
use std::fs::{self, File};
use std::path::{Path, PathBuf};

/// A global definition id in a crate.
///
/// FIXME(emilio): This key is kind of slow, because GlobalCrateId contains a
/// String. There's a "disambiguator" field which may be more than enough for
/// our purposes.
#[derive(Clone, Hash, Debug, Eq, PartialEq)]
pub struct DefId(GlobalCrateId, u32);

/// A map from global definition ids to the actual definition.
pub struct Defs {
    map: HashMap<DefId, data::Def>,
}

impl Defs {
    fn new() -> Self {
        Self { map: HashMap::new() }
    }

    fn insert(&mut self, analysis: &data::Analysis, def: &data::Def) {
        let crate_id = analysis.prelude.as_ref().unwrap().crate_id.clone();
        let mut definition = def.clone();
        let crate_independent_qualname =
            format!("{}{}", crate_id.name, def.qualname);
        definition.qualname = crate_independent_qualname;

        let index = definition.id.index;
        let previous = self.map.insert(DefId(crate_id, index), definition);
        if previous.is_none() {
            // This shouldn't happen, but as of right now it can happen with
            // some builtin definitions when highly generic types are involved.
            // This is probably a rust bug, just ignore it for now.
            eprintln!(
                "Found a definition with the same ID twice? {:?}, {:?}",
                previous,
                def,
            );
        }
    }

    /// Getter for a given local id, which takes care of converting to a global
    /// ID and returning the definition if present.
    fn get(&self, analysis: &data::Analysis, id: data::Id) -> Option<data::Def> {
        let prelude = analysis.prelude.as_ref().unwrap();
        let krate_id = if id.krate == 0 {
            prelude.crate_id.clone()
        } else {
            // TODO(emilio): This escales with the number of crates in this
            // particular crate, but it's probably not too bad, since it should
            // be a pretty fast linear search.
            let krate = prelude.external_crates.iter().find(|krate| {
                krate.num == id.krate
            });

            let krate = match krate {
                Some(k) => k,
                None => {
                    eprintln!("Crate not found: {:?}", id);
                    return None;
                }
            };

            krate.id.clone()
        };

        let id = DefId(krate_id, id.index);
        self.map.get(&id).cloned()
    }
}

#[derive(Clone)]
pub struct Loader {
    deps_dir: PathBuf,
}

impl Loader {
    pub fn new(deps_dir: PathBuf) -> Self {
        Self { deps_dir }
    }
}

impl AnalysisLoader for Loader {
    fn needs_hard_reload(&self, _: &Path) -> bool {
        true
    }

    fn fresh_host(&self) -> AnalysisHost<Self> {
        AnalysisHost::new_with_loader(self.clone())
    }

    fn set_path_prefix(&mut self, _: &Path) {}

    fn abs_path_prefix(&self) -> Option<PathBuf> {
        None
    }
    fn search_directories(&self) -> Vec<PathBuf> {
        vec![self.deps_dir.clone()]
    }
}

// Searchfox uses 1-indexed lines, 0-indexed columns.
fn span_to_string(span: &data::SpanData) -> String {
    // Rust spans are multi-line... So we just set the start column if it spans
    // multiple rows, searchfox has fallback code to handle this.
    if span.line_start != span.line_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    if span.column_start == span.column_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    format!("{}:{}-{}", span.line_start.0, span.column_start.0 - 1, span.column_end.0 - 1)
}

fn visit(
    file: &mut File,
    kind: &'static str,
    location: &data::SpanData,
    qualname: &str,
    context: Option<&str>,
) {
    use serde_json::map::Map;
    use serde_json::value::Value;
    use std::io::Write;

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("target".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));
    if let Some(context) = context {
        out.insert("context".into(), Value::String(context.into()));
        out.insert("contextsym".into(), Value::String(context.into()));
    }

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("source".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();
}

fn analyze_file(
    file_name: &PathBuf,
    defs: &Defs,
    file_analysis: &data::Analysis,
    src_dir: &Path,
    output_dir: &Path,
) {
    let file = match file_name.strip_prefix(src_dir) {
        Ok(f) => f,
        Err(..) => {
            eprintln!("File not in the source directory: {}", file_name.display());
            return;
        }
    };

    let output_file = output_dir.join(file);
    let mut output_dir = output_file.clone();
    output_dir.pop();
    if let Err(err) = fs::create_dir_all(output_dir) {
        eprintln!(
            "Couldn't create dir for: {}, {:?}",
            output_file.display(),
            err
        );
        return;
    }
    let mut file = match File::create(&output_file) {
        Ok(f) => f,
        Err(err) => {
            eprintln!(
                "Couldn't open output file: {}, {:?}",
                output_file.display(),
                err
            );
            return;
        }
    };

    for import in &file_analysis.imports {
        let id = match import.ref_id {
            Some(id) => id,
            None => {
                eprintln!("Dropping import {}: {}, no ref", import.name, import.value);
                continue;
            }
        };

        let def = match defs.get(file_analysis, id) {
            Some(def) => def,
            None => {
                eprintln!("Dropping import {}: {}, no def for ref {:?}", import.name, import.value, id);
                continue;
            }
        };

        visit(&mut file, "use", &import.span, &def.qualname, None)
    }

    for def in &file_analysis.defs {
        let parent = def.parent
            .and_then(|parent_id| defs.get(file_analysis, parent_id).map(|d| d.qualname));

        let crate_name = &file_analysis.prelude.as_ref().unwrap().crate_id.name;
        let qualname = format!("{}{}", crate_name, def.qualname);
        visit(&mut file, "def", &def.span, &qualname, parent.as_ref().map(|p| &**p))
    }

    for ref_ in &file_analysis.refs {
        let def = match defs.get(file_analysis, ref_.ref_id) {
            Some(d) => d,
            None => {
                eprintln!("Dropping ref {:?}, kind {:?}, no def", ref_.ref_id, ref_.kind);
                continue;
            }
        };
        visit(
            &mut file,
            "use",
            &ref_.span,
            &def.qualname,
            /* context = */ None, // TODO
        )
    }
}

fn analyze_crate(
    analysis: &data::Analysis,
    defs: &Defs,
    src_dir: &Path,
    output_dir: &Path,
) {
    let mut per_file = HashMap::new();

    println!("Analyzing crate: {:?}", analysis.prelude);

    macro_rules! flat_map_per_file {
        ($field:ident) => {
            for item in &analysis.$field {
                let mut file_analysis =
                    per_file.entry(item.span.file_name.clone())
                        .or_insert_with(|| {
                            let prelude = analysis.prelude.clone();
                            let mut analysis = data::Analysis::new(analysis.config.clone());
                            analysis.prelude = prelude;
                            analysis
                        });
                file_analysis.$field.push(item.clone());
            }
        }
    }

    flat_map_per_file!(imports);
    flat_map_per_file!(defs);
    flat_map_per_file!(impls);
    flat_map_per_file!(refs);
    flat_map_per_file!(macro_refs);
    flat_map_per_file!(relations);

    for (mut name, analysis) in per_file.drain() {
        if name.is_relative() {
            name = src_dir.join(name);
        }
        analyze_file(&name, defs, &analysis, src_dir, output_dir);
    }
}

fn main() {
    let matches = app_from_crate!()
        .args_from_usage(
            "<src>    'Points to the source root'
             <input>  'Points to the deps/save-analysis directory'
             <output> 'Points to the directory where searchfox metadata should go'",
        )
        .get_matches();

    let src_dir = Path::new(matches.value_of("src").unwrap());
    let input_dir = Path::new(matches.value_of("input").unwrap());
    let output_dir = Path::new(matches.value_of("output").unwrap());

    let loader = Loader::new(PathBuf::from(input_dir));

    let crates = rls_analysis::read_analysis_from_files(&loader, Default::default(), &[]);

    let mut defs = Defs::new();
    for krate in &crates {
        for def in &krate.analysis.defs {
            println!("Indexing def: {:?}", def);
            defs.insert(&krate.analysis, def);
        }
    }

    for krate in crates {
        analyze_crate(&krate.analysis, &defs, &src_dir, &output_dir);
    }
}
<|endoftext|>


<commit_before>extern crate llvm_tools;
extern crate target_build_utils;
extern crate walkdir;

use std::env;
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
#[cfg(feature = "mem")]
use std::process;
use std::process::Command;
use target_build_utils::TargetInfo;
use walkdir::{DirEntry, WalkDir};

fn build_hermit(src_dir: &Path, target_dir_opt: Option<&Path>) {
	let target = TargetInfo::new().expect("Could not get target info");
	let profile = env::var("PROFILE").expect("PROFILE was not set");
	let mut cmd = Command::new("cargo");

	if target.target_arch() == "x86_64" {
		cmd.current_dir(src_dir)
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("x86_64-unknown-hermit-kernel");
	} else if target.target_arch() == "aarch64" {
		cmd.current_dir(src_dir)
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("aarch64-unknown-hermit");
	} else {
		panic!("Try to build for an unsupported platform");
	}

	if let Some(target_dir) = target_dir_opt {
		cmd.arg("--target-dir").arg(target_dir);
	}
	let target_dir = match target_dir_opt {
		Some(target_dir) => target_dir.to_path_buf(),
		None => src_dir.join("target"),
	};

	if profile == "release" {
		cmd.arg("--release");
	}

	// disable all default features
	cmd.arg("--no-default-features");

	// do we have to enable PCI support?
	#[cfg(feature = "pci")]
	{
		cmd.arg("--features");
		cmd.arg("pci");
	}

	// do we have to enable acpi support?
	#[cfg(feature = "acpi")]
	{
		cmd.arg("--features");
		cmd.arg("acpi");
	}

	// do we have to enable FSGSBASE support?
	#[cfg(feature = "fsgs_base")]
	{
		cmd.arg("--features");
		cmd.arg("fsgs_base");
	}

	// do we support multi-processor systems?
	#[cfg(feature = "smp")]
	{
		cmd.arg("--features");
		cmd.arg("smp");
	}

	// do we have to enable VGA support
	#[cfg(feature = "vga")]
	{
		cmd.arg("--features");
		cmd.arg("vga");
	}

	#[cfg(feature = "instrument")]
	{
		cmd.env("RUSTFLAGS", "-Z instrument-mcount");
		// if instrument is not set, ensure that instrument is not in environment variables!
		cmd.env(
			"RUSTFLAGS",
			env::var("RUSTFLAGS")
				.unwrap_or_else(|_| "".into())
				.replace("-Z instrument-mcount", ""),
		);
	}

	let output = cmd.output().expect("Unable to build kernel");
	let stdout = std::string::String::from_utf8(output.stdout);
	let stderr = std::string::String::from_utf8(output.stderr);

	println!("Build libhermit-rs output-status: {}", output.status);
	println!("Build libhermit-rs output-stdout: {}", stdout.unwrap());
	println!("Build libhermit-rs output-stderr: {}", stderr.unwrap());
	assert!(output.status.success());

	let lib_location = if target.target_arch() == "x86_64" {
		target_dir
			.join("x86_64-unknown-hermit-kernel")
			.join(&profile)
			.canonicalize()
			.unwrap() // Must exist after building
	} else if target.target_arch() == "aarch64" {
		target_dir
			.join("aarch64-unknown-hermit")
			.join(&profile)
			.canonicalize()
			.unwrap() // Must exist after building
	} else {
		panic!("Try to build for an unsupported platform");
	};
	println!("Lib location: {}", lib_location.display());

	#[cfg(feature = "mem")]
	{
		// Kernel and user space has its own versions of memcpy, memset, etc,
		// Consequently, we rename the functions in the libos to avoid collisions.
		// In addition, it provides us the offer to create a optimized version of memcpy
		// in user space.

		// get access to llvm tools shipped in the llvm-tools-preview rustup component
		let llvm_tools = match llvm_tools::LlvmTools::new() {
			Ok(tools) => tools,
			Err(llvm_tools::Error::NotFound) => {
				eprintln!("Error: llvm-tools not found");
				eprintln!("Maybe the rustup component `llvm-tools-preview` is missing?");
				eprintln!("  Install it through: `rustup component add llvm-tools-preview`");
				process::exit(1);
			}
			Err(err) => {
				eprintln!("Failed to retrieve llvm-tools component: {:?}", err);
				process::exit(1);
			}
		};

		let lib = lib_location.join("libhermit.a");

		let symbols: [&str; 5] = ["memcpy", "memmove", "memset", "memcmp", "bcmp"];
		for symbol in symbols.iter() {
			// determine llvm_objcopy
			let llvm_objcopy = llvm_tools
				.tool(&llvm_tools::exe("llvm-objcopy"))
				.expect("llvm_objcopy not found in llvm-tools");

			// rename symbols
			let mut cmd = Command::new(llvm_objcopy);
			cmd.arg("--redefine-sym")
				.arg(String::from(*symbol) + &String::from("=kernel") + &String::from(*symbol))
				.arg(lib.display().to_string());

			println!("cmd {:?}", cmd);
			let output = cmd.output().expect("Unable to rename symbols");
			let stdout = std::string::String::from_utf8(output.stdout);
			let stderr = std::string::String::from_utf8(output.stderr);
			println!("Rename symbols output-status: {}", output.status);
			println!("Rename symbols output-stdout: {}", stdout.unwrap());
			println!("Rename symbols output-stderr: {}", stderr.unwrap());
		}
	}

	println!("cargo:rustc-link-search=native={}", lib_location.display());
	println!("cargo:rustc-link-lib=static=hermit");

	//HERMIT_LOG_LEVEL_FILTER sets the log level filter at compile time
	// Doesn't actually rebuild atm - see: https://github.com/rust-lang/cargo/issues/8306
	println!("cargo:rerun-if-env-changed=HERMIT_LOG_LEVEL_FILTER");
}

#[cfg(all(not(feature = "rustc-dep-of-std"), not(feature = "with_submodule")))]
fn build() {
	#[cfg(windows)]
	let out_dir = env::temp_dir();
	#[cfg(not(windows))]
	let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
	let src_dir = out_dir.join("rusty-hermit");

	let _output = Command::new("cargo")
		.current_dir(out_dir)
		.arg("download")
		.arg("--output")
		.arg(src_dir.clone().into_os_string())
		.arg("--extract")
		.arg("rusty-hermit")
		.output()
		.expect("Unable to download rusty-hermit. Please install `cargo-download`.");

	build_hermit(src_dir.as_ref(), None);
}

#[cfg(all(not(feature = "rustc-dep-of-std"), feature = "with_submodule"))]
fn build() {
	let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
	let target_dir = out_dir.clone().join("target");
	let src_dir = env::current_dir()
		.unwrap()
		.parent()
		.unwrap()
		.join("libhermit-rs");

	build_hermit(src_dir.as_ref(), Some(target_dir.as_ref()));
	configure_cargo_rerun_if_changed(src_dir.as_ref());
}

#[cfg(all(not(feature = "rustc-dep-of-std"), feature = "with_submodule"))]
fn configure_cargo_rerun_if_changed(src_dir: &Path) {
	fn is_not_ignored(entry: &DirEntry) -> bool {
		// Ignore .git .vscode and target directories, but not .cargo or .github
		if entry.depth() == 1
			&& entry.path().is_dir()
			&& (entry.path().ends_with("target")
				|| entry.path().ends_with(".git")
				|| entry.path().ends_with(".vscode"))
		{
			return false;
		}
		true
	}

	WalkDir::new(src_dir)
		.into_iter()
		.filter_entry(|e| is_not_ignored(e))
		.filter_map(|v| v.ok())
		.filter_map(|v| v.path().canonicalize().ok())
		.for_each(|x| println!("cargo:rerun-if-changed={}", x.display()));
}

#[cfg(not(feature = "rustc-dep-of-std"))]
fn create_constants() {
	let out_dir = env::var("OUT_DIR").expect("No out dir");
	let dest_path = Path::new(&out_dir).join("constants.rs");
	let mut f = File::create(&dest_path).expect("Could not create file");

	let ip = option_env!("HERMIT_IP");
	let ip = ip.map_or("10.0.5.3", |v| v);

	let gateway = option_env!("HERMIT_GATEWAY");
	let gateway = gateway.map_or("10.0.5.1", |v| v);

	let mask = option_env!("HERMIT_MASK");
	let mask = mask.map_or("255.255.255.0", |v| v);

	writeln!(&mut f, "const HERMIT_IP: &str = \"{}\";", ip).expect("Could not write file");
	println!("cargo:rerun-if-env-changed=HERMIT_IP");

	writeln!(&mut f, "const HERMIT_GATEWAY: &str = \"{}\";", gateway)
		.expect("Could not write file");
	println!("cargo:rerun-if-env-changed=HERMIT_GATEWAY");

	writeln!(&mut f, "const HERMIT_MASK: &str = \"{}\";", mask).expect("Could not write file");
	println!("cargo:rerun-if-env-changed=HERMIT_MASK");
}

fn main() {
	#[cfg(not(feature = "rustc-dep-of-std"))]
	create_constants();
	#[cfg(not(feature = "rustc-dep-of-std"))]
	build();
}
<commit_msg>rename target to build libos<commit_after>extern crate llvm_tools;
extern crate target_build_utils;
extern crate walkdir;

use std::env;
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
#[cfg(feature = "mem")]
use std::process;
use std::process::Command;
use target_build_utils::TargetInfo;
use walkdir::{DirEntry, WalkDir};

fn build_hermit(src_dir: &Path, target_dir_opt: Option<&Path>) {
	let target = TargetInfo::new().expect("Could not get target info");
	let profile = env::var("PROFILE").expect("PROFILE was not set");
	let mut cmd = Command::new("cargo");

	if target.target_arch() == "x86_64" {
		cmd.current_dir(src_dir)
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("x86_64-unknown-none-hermitkernel");
	} else if target.target_arch() == "aarch64" {
		cmd.current_dir(src_dir)
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("aarch64-unknown-hermit");
	} else {
		panic!("Try to build for an unsupported platform");
	}

	if let Some(target_dir) = target_dir_opt {
		cmd.arg("--target-dir").arg(target_dir);
	}
	let target_dir = match target_dir_opt {
		Some(target_dir) => target_dir.to_path_buf(),
		None => src_dir.join("target"),
	};

	if profile == "release" {
		cmd.arg("--release");
	}

	// disable all default features
	cmd.arg("--no-default-features");

	// do we have to enable PCI support?
	#[cfg(feature = "pci")]
	{
		cmd.arg("--features");
		cmd.arg("pci");
	}

	// do we have to enable acpi support?
	#[cfg(feature = "acpi")]
	{
		cmd.arg("--features");
		cmd.arg("acpi");
	}

	// do we have to enable FSGSBASE support?
	#[cfg(feature = "fsgs_base")]
	{
		cmd.arg("--features");
		cmd.arg("fsgs_base");
	}

	// do we support multi-processor systems?
	#[cfg(feature = "smp")]
	{
		cmd.arg("--features");
		cmd.arg("smp");
	}

	// do we have to enable VGA support
	#[cfg(feature = "vga")]
	{
		cmd.arg("--features");
		cmd.arg("vga");
	}

	#[cfg(feature = "instrument")]
	{
		cmd.env("RUSTFLAGS", "-Z instrument-mcount");
		// if instrument is not set, ensure that instrument is not in environment variables!
		cmd.env(
			"RUSTFLAGS",
			env::var("RUSTFLAGS")
				.unwrap_or_else(|_| "".into())
				.replace("-Z instrument-mcount", ""),
		);
	}

	let output = cmd.output().expect("Unable to build kernel");
	let stdout = std::string::String::from_utf8(output.stdout);
	let stderr = std::string::String::from_utf8(output.stderr);

	println!("Build libhermit-rs output-status: {}", output.status);
	println!("Build libhermit-rs output-stdout: {}", stdout.unwrap());
	println!("Build libhermit-rs output-stderr: {}", stderr.unwrap());
	assert!(output.status.success());

	let lib_location = if target.target_arch() == "x86_64" {
		target_dir
			.join("x86_64-unknown-none-hermitkernel")
			.join(&profile)
			.canonicalize()
			.unwrap() // Must exist after building
	} else if target.target_arch() == "aarch64" {
		target_dir
			.join("aarch64-unknown-hermit")
			.join(&profile)
			.canonicalize()
			.unwrap() // Must exist after building
	} else {
		panic!("Try to build for an unsupported platform");
	};
	println!("Lib location: {}", lib_location.display());

	#[cfg(feature = "mem")]
	{
		// Kernel and user space has its own versions of memcpy, memset, etc,
		// Consequently, we rename the functions in the libos to avoid collisions.
		// In addition, it provides us the offer to create a optimized version of memcpy
		// in user space.

		// get access to llvm tools shipped in the llvm-tools-preview rustup component
		let llvm_tools = match llvm_tools::LlvmTools::new() {
			Ok(tools) => tools,
			Err(llvm_tools::Error::NotFound) => {
				eprintln!("Error: llvm-tools not found");
				eprintln!("Maybe the rustup component `llvm-tools-preview` is missing?");
				eprintln!("  Install it through: `rustup component add llvm-tools-preview`");
				process::exit(1);
			}
			Err(err) => {
				eprintln!("Failed to retrieve llvm-tools component: {:?}", err);
				process::exit(1);
			}
		};

		let lib = lib_location.join("libhermit.a");

		let symbols: [&str; 5] = ["memcpy", "memmove", "memset", "memcmp", "bcmp"];
		for symbol in symbols.iter() {
			// determine llvm_objcopy
			let llvm_objcopy = llvm_tools
				.tool(&llvm_tools::exe("llvm-objcopy"))
				.expect("llvm_objcopy not found in llvm-tools");

			// rename symbols
			let mut cmd = Command::new(llvm_objcopy);
			cmd.arg("--redefine-sym")
				.arg(String::from(*symbol) + &String::from("=kernel") + &String::from(*symbol))
				.arg(lib.display().to_string());

			println!("cmd {:?}", cmd);
			let output = cmd.output().expect("Unable to rename symbols");
			let stdout = std::string::String::from_utf8(output.stdout);
			let stderr = std::string::String::from_utf8(output.stderr);
			println!("Rename symbols output-status: {}", output.status);
			println!("Rename symbols output-stdout: {}", stdout.unwrap());
			println!("Rename symbols output-stderr: {}", stderr.unwrap());
		}
	}

	println!("cargo:rustc-link-search=native={}", lib_location.display());
	println!("cargo:rustc-link-lib=static=hermit");

	//HERMIT_LOG_LEVEL_FILTER sets the log level filter at compile time
	// Doesn't actually rebuild atm - see: https://github.com/rust-lang/cargo/issues/8306
	println!("cargo:rerun-if-env-changed=HERMIT_LOG_LEVEL_FILTER");
}

#[cfg(all(not(feature = "rustc-dep-of-std"), not(feature = "with_submodule")))]
fn build() {
	#[cfg(windows)]
	let out_dir = env::temp_dir();
	#[cfg(not(windows))]
	let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
	let src_dir = out_dir.join("rusty-hermit");

	let _output = Command::new("cargo")
		.current_dir(out_dir)
		.arg("download")
		.arg("--output")
		.arg(src_dir.clone().into_os_string())
		.arg("--extract")
		.arg("rusty-hermit")
		.output()
		.expect("Unable to download rusty-hermit. Please install `cargo-download`.");

	build_hermit(src_dir.as_ref(), None);
}

#[cfg(all(not(feature = "rustc-dep-of-std"), feature = "with_submodule"))]
fn build() {
	let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
	let target_dir = out_dir.clone().join("target");
	let src_dir = env::current_dir()
		.unwrap()
		.parent()
		.unwrap()
		.join("libhermit-rs");

	build_hermit(src_dir.as_ref(), Some(target_dir.as_ref()));
	configure_cargo_rerun_if_changed(src_dir.as_ref());
}

#[cfg(all(not(feature = "rustc-dep-of-std"), feature = "with_submodule"))]
fn configure_cargo_rerun_if_changed(src_dir: &Path) {
	fn is_not_ignored(entry: &DirEntry) -> bool {
		// Ignore .git .vscode and target directories, but not .cargo or .github
		if entry.depth() == 1
			&& entry.path().is_dir()
			&& (entry.path().ends_with("target")
				|| entry.path().ends_with(".git")
				|| entry.path().ends_with(".vscode"))
		{
			return false;
		}
		true
	}

	WalkDir::new(src_dir)
		.into_iter()
		.filter_entry(|e| is_not_ignored(e))
		.filter_map(|v| v.ok())
		.filter_map(|v| v.path().canonicalize().ok())
		.for_each(|x| println!("cargo:rerun-if-changed={}", x.display()));
}

#[cfg(not(feature = "rustc-dep-of-std"))]
fn create_constants() {
	let out_dir = env::var("OUT_DIR").expect("No out dir");
	let dest_path = Path::new(&out_dir).join("constants.rs");
	let mut f = File::create(&dest_path).expect("Could not create file");

	let ip = option_env!("HERMIT_IP");
	let ip = ip.map_or("10.0.5.3", |v| v);

	let gateway = option_env!("HERMIT_GATEWAY");
	let gateway = gateway.map_or("10.0.5.1", |v| v);

	let mask = option_env!("HERMIT_MASK");
	let mask = mask.map_or("255.255.255.0", |v| v);

	writeln!(&mut f, "const HERMIT_IP: &str = \"{}\";", ip).expect("Could not write file");
	println!("cargo:rerun-if-env-changed=HERMIT_IP");

	writeln!(&mut f, "const HERMIT_GATEWAY: &str = \"{}\";", gateway)
		.expect("Could not write file");
	println!("cargo:rerun-if-env-changed=HERMIT_GATEWAY");

	writeln!(&mut f, "const HERMIT_MASK: &str = \"{}\";", mask).expect("Could not write file");
	println!("cargo:rerun-if-env-changed=HERMIT_MASK");
}

fn main() {
	#[cfg(not(feature = "rustc-dep-of-std"))]
	create_constants();
	#[cfg(not(feature = "rustc-dep-of-std"))]
	build();
}
<|endoftext|>


<commit_before>use bitstream::byteread::*;

pub trait BitRead {
    fn peek_val(&mut self, n:usize) -> u64;
    fn merge_val(msp:u64, lsp:u64, msb:usize, lsb:usize) -> u64;
    fn skip_rem(&mut self, n:usize) -> ();
    fn fill32(&self) -> u64;
    fn fill64(&self) -> u64;

    fn get_val(&mut self, n:usize) -> u64;
    fn refill32(&mut self) -> ();
    fn refill64(&mut self) -> ();

    fn consumed(&self) -> usize;
    fn available(&self) -> usize;
    fn can_refill(&self) -> bool;

    fn get_bit(&mut self) -> bool;
    fn get_bits_64(&mut self, n: usize) -> u64;
    fn get_bits_32(&mut self, n: usize) -> u32;

    fn peek_bits_32(&mut self, n: usize) -> u32;
    fn peek_bits_64(&mut self, n: usize) -> u64;

    fn skip_bits(&mut self, size : usize) -> ();
    fn align_bits(&mut self) -> ();
}

#[derive(Debug, Clone, Copy)]
pub struct BitReadLE <'a> {
    buffer : &'a[u8], /// read buffer, 8-bytes padded
    index : usize,
    cache : u64,
    left : usize,
}

impl <'a> BitRead for BitReadLE<'a> {
    #[inline]
    fn peek_val(&mut self, n:usize) -> u64 {
        self.cache & ((1u64 << n) - 1)
    }
    #[inline]
    fn skip_rem(&mut self, n:usize) -> () {
        self.cache = self.cache >> n;
        self.left -= n;
    }
    #[inline]
    fn merge_val(msp:u64, lsp:u64, msb:usize, lsb:usize) -> u64 {
        msp << msb | lsp
    }
    #[inline(always)]
    fn fill32(&self) -> u64 {
        get_u32l(&self.buffer[self.index..]) as u64
    }
    #[inline(always)]
    fn fill64(&self) -> u64 {
        get_u64l(&self.buffer[self.index..])
    }

    #[inline]
    fn get_val(&mut self, n:usize) -> u64 {
        let ret = self.peek_val(n);

        self.skip_rem(n);

        return ret;
    }
    #[inline]
    fn refill32(&mut self) -> () {
        if !self.can_refill() {
            return;
        }
        let val = self.fill32();

        self.cache  = Self::merge_val(val, self.cache,
                                      self.left, 32 - self.left);
        self.index += 4;
        self.left  += 32;
    }
    #[inline]
    fn refill64(&mut self) -> () {
        if !self.can_refill() {
            return;
        }

        self.cache  = self.fill64();
        self.index += 8;
        self.left   = 64;
    }

    #[inline]
    fn consumed(&self) -> usize {
        self.index * 8 - self.left
    }

    #[inline]
    fn available(&self) -> usize {
        (self.buffer.len() - self.index) * 8 + self.left
    }

    #[inline]
    fn can_refill(&self) -> bool {
        self.index < self.buffer.len() - 8
    }

    #[inline]
    fn get_bit(&mut self) -> bool {
        if self.left <= 0 {
            self.refill64();
        }

        self.get_val(1) != 0
    }

    #[inline]
    fn get_bits_64(&mut self, mut n:usize) -> u64 {
        let mut left = 0;
        let mut ret = 0;

        if n == 0 {
            return 0;
        }

        if self.left < n {
            n   -= left;
            left = self.left;
            ret  = self.get_val(left);
            self.refill64();
        }

        Self::merge_val(self.get_val(n), ret, left, n)
    }

    #[inline]
    fn peek_bits_64(&mut self, n:usize) -> u64 {
        let mut tmp = self.clone();

        tmp.get_bits_64(n)
    }

    #[inline]
    fn get_bits_32(&mut self, n:usize) -> u32 {
        if n == 0 {
            return 0;
        }

        if self.left <= n {
            self.refill32();
        }

        return self.get_val(n) as u32;
    }

    #[inline]
    fn peek_bits_32(&mut self, n:usize) -> u32 {
        if n == 0 {
            return 0;
        }

        if self.left <= n {
            self.refill32();
        }

        return self.peek_val(n) as u32;
    }

    #[inline]
    fn skip_bits(&mut self, mut n:usize) -> () {
        if self.left < n {
            n -= self.left;
            self.skip_rem(n);
            if n > 64 {
                let skip = n / 8;

                n -= skip * 8;
                self.index += skip;
            }
            self.refill64();
        }

        self.skip_rem(n);
    }

    #[inline]
    fn align_bits(&mut self) -> () {
        let left = self.left;

        self.skip_bits(left);
    }
}

#[cfg(test)]
mod test {
    use super::*;

    const CHECKBOARD0101: [u8; 128] = [0b01010101; 128];
    const CHECKBOARD0011: [u8; 128] = [0b00110011; 128];

#[test]
    fn get_bit() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bit());
        assert!(!reader.get_bit());
    }

#[test]
    fn get_bits_64() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bits_64(1) == 1);
        assert!(reader.get_bits_64(2) == 2);
        assert!(reader.get_bits_64(4) == 10);
        assert!(reader.get_bits_64(1) == 0);
        assert!(reader.get_bits_64(8) == 85);
    }

#[test]
    fn peek_bits_64() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.peek_bits_64(1) == 1);
        assert!(reader.peek_bits_64(1) == 1);
        assert!(reader.peek_bits_64(2) == 1);
        assert!(reader.peek_bits_64(2) == 1);
    }
#[test]
    fn get_bits_32() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bits_64(1) == 1);
        assert!(reader.get_bits_64(2) == 2);
        assert!(reader.get_bits_64(4) == 10);
        assert!(reader.get_bits_64(1) == 0);
        assert!(reader.get_bits_64(8) == 85);
    }

#[test]
    fn peek_bits_32() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.peek_bits_32(1) == 1);
        assert!(reader.peek_bits_32(1) == 1);
        assert!(reader.peek_bits_32(2) == 1);
        assert!(reader.peek_bits_32(2) == 1);
    }
#[test]
    fn skip_bits() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        reader.skip_bits(0);
        assert!(reader.peek_bits_32(1) == 1);
        reader.skip_bits(2);
        assert!(reader.peek_bits_32(1) == 1);
        reader.skip_bits(2);
        assert!(reader.peek_bits_32(1) == 1);
    }
#[test]
    fn align_bits() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0011,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bits_64(3) == 3);
        reader.align_bits();
        assert!(reader.get_bits_64(4) == 3);
        reader.skip_bits(1);
        reader.align_bits();
        assert!(reader.get_bits_64(4) == 3);
    }
}
<commit_msg>bitstream: Split the BitRead trait in public and private<commit_after>use bitstream::byteread::*;

trait BitReadEndian {
    fn peek_val(&mut self, n:usize) -> u64;
    fn merge_val(msp:u64, lsp:u64, msb:usize, lsb:usize) -> u64;
    fn skip_rem(&mut self, n:usize) -> ();
    fn fill32(&self) -> u64;
    fn fill64(&self) -> u64;
}

trait BitReadInternal {
    fn get_val(&mut self, n:usize) -> u64;
    fn refill32(&mut self) -> ();
    fn refill64(&mut self) -> ();
}

pub trait BitRead {
    fn consumed(&self) -> usize;
    fn available(&self) -> usize;
    fn can_refill(&self) -> bool;

    fn get_bit(&mut self) -> bool;
    fn get_bits_64(&mut self, n: usize) -> u64;
    fn get_bits_32(&mut self, n: usize) -> u32;

    fn peek_bits_32(&mut self, n: usize) -> u32;
    fn peek_bits_64(&mut self, n: usize) -> u64;

    fn skip_bits(&mut self, size : usize) -> ();
    fn align_bits(&mut self) -> ();
}

#[derive(Debug, Clone, Copy)]
pub struct BitReadLE <'a> {
    buffer : &'a[u8], /// read buffer, 8-bytes padded
    index : usize,
    cache : u64,
    left : usize,
}

impl <'a> BitReadEndian for BitReadLE<'a> {
    #[inline]
    fn peek_val(&mut self, n:usize) -> u64 {
        self.cache & ((1u64 << n) - 1)
    }
    #[inline]
    fn skip_rem(&mut self, n:usize) -> () {
        self.cache = self.cache >> n;
        self.left -= n;
    }
    #[inline]
    fn merge_val(msp:u64, lsp:u64, msb:usize, lsb:usize) -> u64 {
        msp << msb | lsp
    }
    #[inline(always)]
    fn fill32(&self) -> u64 {
        get_u32l(&self.buffer[self.index..]) as u64
    }
    #[inline(always)]
    fn fill64(&self) -> u64 {
        get_u64l(&self.buffer[self.index..])
    }
}

impl <'a> BitReadInternal for BitReadLE<'a> {
    #[inline]
    fn get_val(&mut self, n:usize) -> u64 {
        let ret = self.peek_val(n);

        self.skip_rem(n);

        return ret;
    }
    #[inline]
    fn refill32(&mut self) -> () {
        if !self.can_refill() {
            return;
        }
        let val = self.fill32();

        self.cache  = Self::merge_val(val, self.cache,
                                      self.left, 32 - self.left);
        self.index += 4;
        self.left  += 32;
    }
    #[inline]
    fn refill64(&mut self) -> () {
        if !self.can_refill() {
            return;
        }

        self.cache  = self.fill64();
        self.index += 8;
        self.left   = 64;
    }
}

impl <'a> BitRead for BitReadLE<'a> {
    #[inline]
    fn consumed(&self) -> usize {
        self.index * 8 - self.left
    }

    #[inline]
    fn available(&self) -> usize {
        (self.buffer.len() - self.index) * 8 + self.left
    }

    #[inline]
    fn can_refill(&self) -> bool {
        self.index < self.buffer.len() - 8
    }

    #[inline]
    fn get_bit(&mut self) -> bool {
        if self.left <= 0 {
            self.refill64();
        }

        self.get_val(1) != 0
    }

    #[inline]
    fn get_bits_64(&mut self, mut n:usize) -> u64 {
        let mut left = 0;
        let mut ret = 0;

        if n == 0 {
            return 0;
        }

        if self.left < n {
            n   -= left;
            left = self.left;
            ret  = self.get_val(left);
            self.refill64();
        }

        Self::merge_val(self.get_val(n), ret, left, n)
    }

    #[inline]
    fn peek_bits_64(&mut self, n:usize) -> u64 {
        let mut tmp = self.clone();

        tmp.get_bits_64(n)
    }

    #[inline]
    fn get_bits_32(&mut self, n:usize) -> u32 {
        if n == 0 {
            return 0;
        }

        if self.left <= n {
            self.refill32();
        }

        return self.get_val(n) as u32;
    }

    #[inline]
    fn peek_bits_32(&mut self, n:usize) -> u32 {
        if n == 0 {
            return 0;
        }

        if self.left <= n {
            self.refill32();
        }

        return self.peek_val(n) as u32;
    }

    #[inline]
    fn skip_bits(&mut self, mut n:usize) -> () {
        if self.left < n {
            n -= self.left;
            self.skip_rem(n);
            if n > 64 {
                let skip = n / 8;

                n -= skip * 8;
                self.index += skip;
            }
            self.refill64();
        }

        self.skip_rem(n);
    }

    #[inline]
    fn align_bits(&mut self) -> () {
        let left = self.left;

        self.skip_bits(left);
    }
}

#[cfg(test)]
mod test {
    use super::*;

    const CHECKBOARD0101: [u8; 128] = [0b01010101; 128];
    const CHECKBOARD0011: [u8; 128] = [0b00110011; 128];

#[test]
    fn get_bit() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bit());
        assert!(!reader.get_bit());
    }

#[test]
    fn get_bits_64() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bits_64(1) == 1);
        assert!(reader.get_bits_64(2) == 2);
        assert!(reader.get_bits_64(4) == 10);
        assert!(reader.get_bits_64(1) == 0);
        assert!(reader.get_bits_64(8) == 85);
    }

#[test]
    fn peek_bits_64() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.peek_bits_64(1) == 1);
        assert!(reader.peek_bits_64(1) == 1);
        assert!(reader.peek_bits_64(2) == 1);
        assert!(reader.peek_bits_64(2) == 1);
    }
#[test]
    fn get_bits_32() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bits_64(1) == 1);
        assert!(reader.get_bits_64(2) == 2);
        assert!(reader.get_bits_64(4) == 10);
        assert!(reader.get_bits_64(1) == 0);
        assert!(reader.get_bits_64(8) == 85);
    }

#[test]
    fn peek_bits_32() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.peek_bits_32(1) == 1);
        assert!(reader.peek_bits_32(1) == 1);
        assert!(reader.peek_bits_32(2) == 1);
        assert!(reader.peek_bits_32(2) == 1);
    }
#[test]
    fn skip_bits() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0101,
            index: 0,
            cache: 0,
            left: 0
        };

        reader.skip_bits(0);
        assert!(reader.peek_bits_32(1) == 1);
        reader.skip_bits(2);
        assert!(reader.peek_bits_32(1) == 1);
        reader.skip_bits(2);
        assert!(reader.peek_bits_32(1) == 1);
    }
#[test]
    fn align_bits() {
        let mut reader = BitReadLE {
            buffer: &CHECKBOARD0011,
            index: 0,
            cache: 0,
            left: 0
        };

        assert!(reader.get_bits_64(3) == 3);
        reader.align_bits();
        assert!(reader.get_bits_64(4) == 3);
        reader.skip_bits(1);
        reader.align_bits();
        assert!(reader.get_bits_64(4) == 3);
    }
}
<|endoftext|>


<commit_before>use std::io::File;

pub struct Program {
    content: Vec<char>,
    pub pointer: uint
}

impl Program {
    pub fn new(file_path: &Path) -> Program {
        let content = Program::read_program(file_path);
        Program {
            content: content,
            pointer: 0
        }
    }

    pub fn len(&mut self) -> uint {
        self.content.len()
    }

    pub fn forward(&mut self) {
        self.pointer += 1;
    }

    pub fn back(&mut self) {
        self.pointer -= 1;
    }

    pub fn command(&mut self) -> char {
        self.content[self.pointer]
    }

    pub fn fast_forward(&mut self, count: uint) {
        if count != 0 {
            self.forward();

            match self.command() {
                ']' => self.fast_forward(count - 1),
                '[' => self.fast_forward(count + 1),
                 _  => self.fast_forward(count)
            }
        }
    }

    pub fn rewind(&mut self, count: uint) {
        if count != 0 {
            self.back();

            match self.command() {
                '[' => self.rewind(count - 1),
                ']' => self.rewind(count + 1),
                 _  => self.rewind(count)
            }
        }
    }

    fn read_program(path: &Path) -> Vec<char> {
        let file = match File::open(path) {
            Ok(mut f) => f.read_to_string().ok().unwrap(),
            Err(e) => fail!("Could not open file: {}", e)
        };

        file.as_slice().chars().collect()
    }
}
<commit_msg>Only use `&mut self` when mutability is required<commit_after>use std::io::File;

pub struct Program {
    content: Vec<char>,
    pub pointer: uint
}

impl Program {
    pub fn new(file_path: &Path) -> Program {
        let content = Program::read_program(file_path);
        Program {
            content: content,
            pointer: 0
        }
    }

    pub fn len(&self) -> uint {
        self.content.len()
    }

    pub fn forward(&mut self) {
        self.pointer += 1;
    }

    pub fn back(&mut self) {
        self.pointer -= 1;
    }

    pub fn command(&self) -> char {
        self.content[self.pointer]
    }

    pub fn fast_forward(&mut self, count: uint) {
        if count != 0 {
            self.forward();

            match self.command() {
                ']' => self.fast_forward(count - 1),
                '[' => self.fast_forward(count + 1),
                 _  => self.fast_forward(count)
            }
        }
    }

    pub fn rewind(&mut self, count: uint) {
        if count != 0 {
            self.back();

            match self.command() {
                '[' => self.rewind(count - 1),
                ']' => self.rewind(count + 1),
                 _  => self.rewind(count)
            }
        }
    }

    fn read_program(path: &Path) -> Vec<char> {
        let file = match File::open(path) {
            Ok(mut f) => f.read_to_string().ok().unwrap(),
            Err(e) => fail!("Could not open file: {}", e)
        };

        file.as_slice().chars().collect()
    }
}
<|endoftext|>


<commit_before>//! Module providing the building blocks to create macros and expand them.
use std::sync::RwLock;
use std::collections::HashMap;
use std::error::Error as StdError;

use ast;
use ast::MutVisitor;
use types::TcIdent;
use error::Errors;

pub type Error = Box<StdError>;

/// A trait which abstracts over macros.
///
/// A macro is similiar to a function call but is run at compile time instead of at runtime.
pub trait Macro<Env>: ::mopa::Any + Send + Sync {
    fn expand(&self,
              env: &Env,
              arguments: &mut [ast::LExpr<TcIdent>])
              -> Result<ast::LExpr<TcIdent>, Error>;
    fn clone(&self) -> Box<Macro<Env>>;
}
mopafy!(Macro<Env>);

impl<F: ::mopa::Any + Clone + Send + Sync, Env> Macro<Env> for F
    where F: Fn(&Env, &mut [ast::LExpr<TcIdent>]) -> Result<ast::LExpr<TcIdent>, Error>
{
    fn expand(&self,
              env: &Env,
              arguments: &mut [ast::LExpr<TcIdent>])
              -> Result<ast::LExpr<TcIdent>, Error> {
        self(env, arguments)
    }
    fn clone(&self) -> Box<Macro<Env>> {
        Box::new(Clone::clone(self))
    }
}

/// Type containing macros bound to symbols which can be applied on an AST expression to transform
/// it.
pub struct MacroEnv<Env> {
    macros: RwLock<HashMap<String, Box<Macro<Env>>>>,
}

impl<Env> MacroEnv<Env> {
    /// Creates a new `MacroEnv`
    pub fn new() -> MacroEnv<Env> {
        MacroEnv { macros: RwLock::new(HashMap::new()) }
    }

    /// Inserts a `Macro` which acts on any occurance of `symbol` when applied to an expression.
    pub fn insert<M>(&self, name: String, mac: M)
        where M: Macro<Env> + 'static
    {
        self.macros.write().unwrap().insert(name, Box::new(mac));
    }

    /// Retrieves the macro bound to `symbol`
    pub fn get(&self, name: &str) -> Option<Box<Macro<Env>>> {
        self.macros.read().unwrap().get(name).map(|x| (**x).clone())
    }

    /// Runs the macros in this `MacroEnv` on `expr` using `env` as the context of the expansion
    pub fn run(&self, env: &Env, expr: &mut ast::LExpr<TcIdent>) -> Result<(), Errors<Error>> {
        let mut expander = MacroExpander {
            env: env,
            macros: self,
            errors: Errors::new(),
        };
        expander.visit_expr(expr);
        if expander.errors.has_errors() {
            Err(expander.errors)
        } else {
            Ok(())
        }
    }
}

struct MacroExpander<'a, Env: 'a> {
    env: &'a Env,
    macros: &'a MacroEnv<Env>,
    errors: Errors<Error>,
}

impl<'a, Env> MutVisitor for MacroExpander<'a, Env> {
    type T = TcIdent;

    fn visit_expr(&mut self, expr: &mut ast::LExpr<TcIdent>) {
        let replacement = match expr.value {
            ast::Expr::Call(ref mut id, ref mut args) => {
                match ***id {
                    ast::Expr::Identifier(ref id) => {
                        match self.macros.get(id.name.as_ref()) {
                            Some(m) => {
                                match m.expand(self.env, args) {
                                    Ok(e) => Some(e),
                                    Err(err) => {
                                        self.errors.error(err);
                                        None
                                    }
                                }
                            }
                            None => None,
                        }
                    }
                    _ => None,
                }
            }
            _ => None,
        };
        if let Some(mut e) = replacement {
            e.location = expr.location;
            *expr = e;
        }
        ast::walk_mut_expr(self, expr);
    }
}
<commit_msg>fix(macro): Require Send and Sync for Macro errors<commit_after>//! Module providing the building blocks to create macros and expand them.
use std::sync::RwLock;
use std::collections::HashMap;
use std::error::Error as StdError;

use ast;
use ast::MutVisitor;
use types::TcIdent;
use error::Errors;

pub type Error = Box<StdError + Send + Sync>;

/// A trait which abstracts over macros.
///
/// A macro is similiar to a function call but is run at compile time instead of at runtime.
pub trait Macro<Env>: ::mopa::Any + Send + Sync {
    fn expand(&self,
              env: &Env,
              arguments: &mut [ast::LExpr<TcIdent>])
              -> Result<ast::LExpr<TcIdent>, Error>;
    fn clone(&self) -> Box<Macro<Env>>;
}
mopafy!(Macro<Env>);

impl<F: ::mopa::Any + Clone + Send + Sync, Env> Macro<Env> for F
    where F: Fn(&Env, &mut [ast::LExpr<TcIdent>]) -> Result<ast::LExpr<TcIdent>, Error>
{
    fn expand(&self,
              env: &Env,
              arguments: &mut [ast::LExpr<TcIdent>])
              -> Result<ast::LExpr<TcIdent>, Error> {
        self(env, arguments)
    }
    fn clone(&self) -> Box<Macro<Env>> {
        Box::new(Clone::clone(self))
    }
}

/// Type containing macros bound to symbols which can be applied on an AST expression to transform
/// it.
pub struct MacroEnv<Env> {
    macros: RwLock<HashMap<String, Box<Macro<Env>>>>,
}

impl<Env> MacroEnv<Env> {
    /// Creates a new `MacroEnv`
    pub fn new() -> MacroEnv<Env> {
        MacroEnv { macros: RwLock::new(HashMap::new()) }
    }

    /// Inserts a `Macro` which acts on any occurance of `symbol` when applied to an expression.
    pub fn insert<M>(&self, name: String, mac: M)
        where M: Macro<Env> + 'static
    {
        self.macros.write().unwrap().insert(name, Box::new(mac));
    }

    /// Retrieves the macro bound to `symbol`
    pub fn get(&self, name: &str) -> Option<Box<Macro<Env>>> {
        self.macros.read().unwrap().get(name).map(|x| (**x).clone())
    }

    /// Runs the macros in this `MacroEnv` on `expr` using `env` as the context of the expansion
    pub fn run(&self, env: &Env, expr: &mut ast::LExpr<TcIdent>) -> Result<(), Errors<Error>> {
        let mut expander = MacroExpander {
            env: env,
            macros: self,
            errors: Errors::new(),
        };
        expander.visit_expr(expr);
        if expander.errors.has_errors() {
            Err(expander.errors)
        } else {
            Ok(())
        }
    }
}

struct MacroExpander<'a, Env: 'a> {
    env: &'a Env,
    macros: &'a MacroEnv<Env>,
    errors: Errors<Error>,
}

impl<'a, Env> MutVisitor for MacroExpander<'a, Env> {
    type T = TcIdent;

    fn visit_expr(&mut self, expr: &mut ast::LExpr<TcIdent>) {
        let replacement = match expr.value {
            ast::Expr::Call(ref mut id, ref mut args) => {
                match ***id {
                    ast::Expr::Identifier(ref id) => {
                        match self.macros.get(id.name.as_ref()) {
                            Some(m) => {
                                match m.expand(self.env, args) {
                                    Ok(e) => Some(e),
                                    Err(err) => {
                                        self.errors.error(err);
                                        None
                                    }
                                }
                            }
                            None => None,
                        }
                    }
                    _ => None,
                }
            }
            _ => None,
        };
        if let Some(mut e) = replacement {
            e.location = expr.location;
            *expr = e;
        }
        ast::walk_mut_expr(self, expr);
    }
}
<|endoftext|>