<commit_before>// Copyright 2015-2018 Benjamin Fry <benjaminfry@me.com>
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use std::io;
use std::marker::PhantomData;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::pin::Pin;
use std::task::{Context, Poll};

use async_trait::async_trait;
use futures::channel::mpsc::{unbounded, UnboundedReceiver};
use futures::stream::{Fuse, Peekable, Stream, StreamExt};
use futures::{ready, Future, FutureExt, TryFutureExt};
use log::debug;
use rand;
use rand::distributions::{uniform::Uniform, Distribution};

use crate::xfer::{BufStreamHandle, SerialMessage};

/// Trait for UdpSocket
#[async_trait]
pub trait UdpSocket
where
    Self: Sized + Unpin,
{
    /// UdpSocket
    async fn bind(addr: &SocketAddr) -> io::Result<Self>;
    /// Receive data from the socket and returns the number of bytes read and the address from
    /// where the data came on success.
    async fn recv_from(&mut self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)>;
    /// Send data to the given address.
    async fn send_to(&mut self, buf: &[u8], target: &SocketAddr) -> io::Result<usize>;
}

/// A UDP stream of DNS binary packets
#[must_use = "futures do nothing unless polled"]
pub struct UdpStream<S: Send> {
    socket: S,
    outbound_messages: Peekable<Fuse<UnboundedReceiver<SerialMessage>>>,
}

impl<S: UdpSocket + Send + 'static> UdpStream<S> {
    /// This method is intended for client connections, see `with_bound` for a method better for
    ///  straight listening. It is expected that the resolver wrapper will be responsible for
    ///  creating and managing new UdpStreams such that each new client would have a random port
    ///  (reduce chance of cache poisoning). This will return a randomly assigned local port.
    ///
    /// # Arguments
    ///
    /// * `name_server` - socket address for the remote server (used to determine IPv4 or IPv6)
    ///
    /// # Return
    ///
    /// a tuple of a Future Stream which will handle sending and receiving messages, and a
    ///  handle which can be used to send messages into the stream.
    #[allow(clippy::type_complexity)]
    pub fn new(
        name_server: SocketAddr,
    ) -> (
        Box<dyn Future<Output = Result<UdpStream<S>, io::Error>> + Send + Unpin>,
        BufStreamHandle,
    ) {
        let (message_sender, outbound_messages) = unbounded();
        let message_sender = BufStreamHandle::new(message_sender);

        // TODO: allow the bind address to be specified...
        // constructs a future for getting the next randomly bound port to a UdpSocket
        let next_socket = NextRandomUdpSocket::new(&name_server);

        // This set of futures collapses the next udp socket into a stream which can be used for
        //  sending and receiving udp packets.
        let stream = Box::new(next_socket.map_ok(move |socket| UdpStream {
            socket,
            outbound_messages: outbound_messages.fuse().peekable(),
        }));

        (stream, message_sender)
    }

    /// Initialize the Stream with an already bound socket. Generally this should be only used for
    ///  server listening sockets. See `new` for a client oriented socket. Specifically, this there
    ///  is already a bound socket in this context, whereas `new` makes sure to randomize ports
    ///  for additional cache poison prevention.
    ///
    /// # Arguments
    ///
    /// * `socket` - an already bound UDP socket
    ///
    /// # Return
    ///
    /// a tuple of a Future Stream which will handle sending and receiving messsages, and a
    ///  handle which can be used to send messages into the stream.
    pub fn with_bound(socket: S) -> (Self, BufStreamHandle) {
        let (message_sender, outbound_messages) = unbounded();
        let message_sender = BufStreamHandle::new(message_sender);

        let stream = UdpStream {
            socket,
            outbound_messages: outbound_messages.fuse().peekable(),
        };

        (stream, message_sender)
    }

    #[allow(unused)]
    pub(crate) fn from_parts(
        socket: S,
        outbound_messages: UnboundedReceiver<SerialMessage>,
    ) -> Self {
        UdpStream {
            socket,
            outbound_messages: outbound_messages.fuse().peekable(),
        }
    }
}

impl<S: Send> UdpStream<S> {
    #[allow(clippy::type_complexity)]
    fn pollable_split(
        &mut self,
    ) -> (
        &mut S,
        &mut Peekable<Fuse<UnboundedReceiver<SerialMessage>>>,
    ) {
        (&mut self.socket, &mut self.outbound_messages)
    }
}

impl<S: UdpSocket + Send + 'static> Stream for UdpStream<S> {
    type Item = Result<SerialMessage, io::Error>;

    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
        let (socket, outbound_messages) = self.pollable_split();
        let mut socket = Pin::new(socket);
        let mut outbound_messages = Pin::new(outbound_messages);

        // this will not accept incoming data while there is data to send
        //  makes this self throttling.
        while let Poll::Ready(Some(message)) = outbound_messages.as_mut().poll_peek(cx) {
            // first try to send
            let addr = &message.addr();

            // this wiil return if not ready,
            //   meaning that sending will be prefered over receiving...

            // TODO: shouldn't this return the error to send to the sender?
            ready!(socket.send_to(message.bytes(), addr).poll_unpin(cx))?;

            // message sent, need to pop the message
            assert!(outbound_messages.as_mut().poll_next(cx).is_ready());
        }

        // For QoS, this will only accept one message and output that
        // receive all inbound messages

        // TODO: this should match edns settings
        let mut buf = [0u8; 2048];
        let (len, src) = ready!(socket.recv_from(&mut buf).poll_unpin(cx))?;

        Poll::Ready(Some(Ok(SerialMessage::new(
            buf.iter().take(len).cloned().collect(),
            src,
        ))))
    }
}

#[must_use = "futures do nothing unless polled"]
pub(crate) struct NextRandomUdpSocket<S> {
    bind_address: IpAddr,
    marker: PhantomData<S>,
}

impl<S: UdpSocket> NextRandomUdpSocket<S> {
    /// Creates a future for randomly binding to a local socket address for client connections.
    pub(crate) fn new(name_server: &SocketAddr) -> NextRandomUdpSocket<S> {
        let zero_addr: IpAddr = match *name_server {
            SocketAddr::V4(..) => IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
            SocketAddr::V6(..) => IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)),
        };

        NextRandomUdpSocket {
            bind_address: zero_addr,
            marker: PhantomData,
        }
    }

    async fn bind(zero_addr: SocketAddr) -> Result<S, io::Error> {
        S::bind(&zero_addr).await
    }
}

impl<S: UdpSocket> Future for NextRandomUdpSocket<S> {
    type Output = Result<S, io::Error>;

    /// polls until there is an available next random UDP port.
    ///
    /// if there is no port available after 10 attempts, returns NotReady
    fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
        let rand_port_range = Uniform::new_inclusive(1025_u16, u16::max_value());
        let mut rand = rand::thread_rng();

        for attempt in 0..10 {
            let port = rand_port_range.sample(&mut rand); // the range is [0 ... u16::max]
            let zero_addr = SocketAddr::new(self.bind_address, port);

            // TODO: allow TTL to be adjusted...
            // TODO: this immediate poll might be wrong in some cases...
            match Box::pin(Self::bind(zero_addr)).as_mut().poll(cx) {
                Poll::Ready(Ok(socket)) => {
                    debug!("created socket successfully");
                    return Poll::Ready(Ok(socket));
                }
                Poll::Ready(Err(err)) => {
                    debug!("unable to bind port, attempt: {}: {}", attempt, err)
                }
                Poll::Pending => debug!("unable to bind port, attempt: {}", attempt),
            }
        }

        debug!("could not get next random port, delaying");

        // TODO: because no interest is registered anywhere, we must awake.
        cx.waker().wake_by_ref();

        // returning NotReady here, perhaps the next poll there will be some more socket available.
        Poll::Pending
    }
}

#[cfg(feature = "tokio-runtime")]
#[async_trait]
impl UdpSocket for tokio::net::UdpSocket {
    async fn bind(addr: &SocketAddr) -> io::Result<Self> {
        tokio::net::UdpSocket::bind(addr).await
    }

    async fn recv_from(&mut self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> {
        self.recv_from(buf).await
    }

    async fn send_to(&mut self, buf: &[u8], target: &SocketAddr) -> io::Result<usize> {
        self.send_to(buf, target).await
    }
}

#[cfg(test)]
#[cfg(feature = "tokio-runtime")]
mod tests {
    #[cfg(not(target_os = "linux"))] // ignored until Travis-CI fixes IPv6
    use std::net::Ipv6Addr;
    use std::net::{IpAddr, Ipv4Addr};
    use tokio::{net::UdpSocket as TokioUdpSocket, runtime::Runtime};

    #[test]
    fn test_next_random_socket() {
        use crate::tests::next_random_socket_test;
        let io_loop = Runtime::new().expect("failed to create tokio runtime");
        next_random_socket_test::<TokioUdpSocket, Runtime>(io_loop)
    }

    #[test]
    fn test_udp_stream_ipv4() {
        use crate::tests::udp_stream_test;
        let io_loop = Runtime::new().expect("failed to create tokio runtime");
        udp_stream_test::<TokioUdpSocket, Runtime>(
            IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
            io_loop,
        );
    }

    #[test]
    #[cfg(not(target_os = "linux"))] // ignored until Travis-CI fixes IPv6
    fn test_udp_stream_ipv6() {
        use crate::tests::udp_stream_test;
        let io_loop = Runtime::new().expect("failed to create tokio runtime");
        udp_stream_test::<TokioUdpSocket, Runtime>(
            IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)),
            io_loop,
        );
    }
}
<commit_msg>Increased the UDP buffer size to 4096 as recommended by RFC6891 when using DNSSEC.<commit_after>// Copyright 2015-2018 Benjamin Fry <benjaminfry@me.com>
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use std::io;
use std::marker::PhantomData;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::pin::Pin;
use std::task::{Context, Poll};

use async_trait::async_trait;
use futures::channel::mpsc::{unbounded, UnboundedReceiver};
use futures::stream::{Fuse, Peekable, Stream, StreamExt};
use futures::{ready, Future, FutureExt, TryFutureExt};
use log::debug;
use rand;
use rand::distributions::{uniform::Uniform, Distribution};

use crate::xfer::{BufStreamHandle, SerialMessage};

/// Trait for UdpSocket
#[async_trait]
pub trait UdpSocket
where
    Self: Sized + Unpin,
{
    /// UdpSocket
    async fn bind(addr: &SocketAddr) -> io::Result<Self>;
    /// Receive data from the socket and returns the number of bytes read and the address from
    /// where the data came on success.
    async fn recv_from(&mut self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)>;
    /// Send data to the given address.
    async fn send_to(&mut self, buf: &[u8], target: &SocketAddr) -> io::Result<usize>;
}

/// A UDP stream of DNS binary packets
#[must_use = "futures do nothing unless polled"]
pub struct UdpStream<S: Send> {
    socket: S,
    outbound_messages: Peekable<Fuse<UnboundedReceiver<SerialMessage>>>,
}

impl<S: UdpSocket + Send + 'static> UdpStream<S> {
    /// This method is intended for client connections, see `with_bound` for a method better for
    ///  straight listening. It is expected that the resolver wrapper will be responsible for
    ///  creating and managing new UdpStreams such that each new client would have a random port
    ///  (reduce chance of cache poisoning). This will return a randomly assigned local port.
    ///
    /// # Arguments
    ///
    /// * `name_server` - socket address for the remote server (used to determine IPv4 or IPv6)
    ///
    /// # Return
    ///
    /// a tuple of a Future Stream which will handle sending and receiving messages, and a
    ///  handle which can be used to send messages into the stream.
    #[allow(clippy::type_complexity)]
    pub fn new(
        name_server: SocketAddr,
    ) -> (
        Box<dyn Future<Output = Result<UdpStream<S>, io::Error>> + Send + Unpin>,
        BufStreamHandle,
    ) {
        let (message_sender, outbound_messages) = unbounded();
        let message_sender = BufStreamHandle::new(message_sender);

        // TODO: allow the bind address to be specified...
        // constructs a future for getting the next randomly bound port to a UdpSocket
        let next_socket = NextRandomUdpSocket::new(&name_server);

        // This set of futures collapses the next udp socket into a stream which can be used for
        //  sending and receiving udp packets.
        let stream = Box::new(next_socket.map_ok(move |socket| UdpStream {
            socket,
            outbound_messages: outbound_messages.fuse().peekable(),
        }));

        (stream, message_sender)
    }

    /// Initialize the Stream with an already bound socket. Generally this should be only used for
    ///  server listening sockets. See `new` for a client oriented socket. Specifically, this there
    ///  is already a bound socket in this context, whereas `new` makes sure to randomize ports
    ///  for additional cache poison prevention.
    ///
    /// # Arguments
    ///
    /// * `socket` - an already bound UDP socket
    ///
    /// # Return
    ///
    /// a tuple of a Future Stream which will handle sending and receiving messsages, and a
    ///  handle which can be used to send messages into the stream.
    pub fn with_bound(socket: S) -> (Self, BufStreamHandle) {
        let (message_sender, outbound_messages) = unbounded();
        let message_sender = BufStreamHandle::new(message_sender);

        let stream = UdpStream {
            socket,
            outbound_messages: outbound_messages.fuse().peekable(),
        };

        (stream, message_sender)
    }

    #[allow(unused)]
    pub(crate) fn from_parts(
        socket: S,
        outbound_messages: UnboundedReceiver<SerialMessage>,
    ) -> Self {
        UdpStream {
            socket,
            outbound_messages: outbound_messages.fuse().peekable(),
        }
    }
}

impl<S: Send> UdpStream<S> {
    #[allow(clippy::type_complexity)]
    fn pollable_split(
        &mut self,
    ) -> (
        &mut S,
        &mut Peekable<Fuse<UnboundedReceiver<SerialMessage>>>,
    ) {
        (&mut self.socket, &mut self.outbound_messages)
    }
}

impl<S: UdpSocket + Send + 'static> Stream for UdpStream<S> {
    type Item = Result<SerialMessage, io::Error>;

    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
        let (socket, outbound_messages) = self.pollable_split();
        let mut socket = Pin::new(socket);
        let mut outbound_messages = Pin::new(outbound_messages);

        // this will not accept incoming data while there is data to send
        //  makes this self throttling.
        while let Poll::Ready(Some(message)) = outbound_messages.as_mut().poll_peek(cx) {
            // first try to send
            let addr = &message.addr();

            // this wiil return if not ready,
            //   meaning that sending will be prefered over receiving...

            // TODO: shouldn't this return the error to send to the sender?
            ready!(socket.send_to(message.bytes(), addr).poll_unpin(cx))?;

            // message sent, need to pop the message
            assert!(outbound_messages.as_mut().poll_next(cx).is_ready());
        }

        // For QoS, this will only accept one message and output that
        // receive all inbound messages

        // TODO: this should match edns settings
        let mut buf = [0u8; 4096];
        let (len, src) = ready!(socket.recv_from(&mut buf).poll_unpin(cx))?;

        Poll::Ready(Some(Ok(SerialMessage::new(
            buf.iter().take(len).cloned().collect(),
            src,
        ))))
    }
}

#[must_use = "futures do nothing unless polled"]
pub(crate) struct NextRandomUdpSocket<S> {
    bind_address: IpAddr,
    marker: PhantomData<S>,
}

impl<S: UdpSocket> NextRandomUdpSocket<S> {
    /// Creates a future for randomly binding to a local socket address for client connections.
    pub(crate) fn new(name_server: &SocketAddr) -> NextRandomUdpSocket<S> {
        let zero_addr: IpAddr = match *name_server {
            SocketAddr::V4(..) => IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
            SocketAddr::V6(..) => IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)),
        };

        NextRandomUdpSocket {
            bind_address: zero_addr,
            marker: PhantomData,
        }
    }

    async fn bind(zero_addr: SocketAddr) -> Result<S, io::Error> {
        S::bind(&zero_addr).await
    }
}

impl<S: UdpSocket> Future for NextRandomUdpSocket<S> {
    type Output = Result<S, io::Error>;

    /// polls until there is an available next random UDP port.
    ///
    /// if there is no port available after 10 attempts, returns NotReady
    fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
        let rand_port_range = Uniform::new_inclusive(1025_u16, u16::max_value());
        let mut rand = rand::thread_rng();

        for attempt in 0..10 {
            let port = rand_port_range.sample(&mut rand); // the range is [0 ... u16::max]
            let zero_addr = SocketAddr::new(self.bind_address, port);

            // TODO: allow TTL to be adjusted...
            // TODO: this immediate poll might be wrong in some cases...
            match Box::pin(Self::bind(zero_addr)).as_mut().poll(cx) {
                Poll::Ready(Ok(socket)) => {
                    debug!("created socket successfully");
                    return Poll::Ready(Ok(socket));
                }
                Poll::Ready(Err(err)) => {
                    debug!("unable to bind port, attempt: {}: {}", attempt, err)
                }
                Poll::Pending => debug!("unable to bind port, attempt: {}", attempt),
            }
        }

        debug!("could not get next random port, delaying");

        // TODO: because no interest is registered anywhere, we must awake.
        cx.waker().wake_by_ref();

        // returning NotReady here, perhaps the next poll there will be some more socket available.
        Poll::Pending
    }
}

#[cfg(feature = "tokio-runtime")]
#[async_trait]
impl UdpSocket for tokio::net::UdpSocket {
    async fn bind(addr: &SocketAddr) -> io::Result<Self> {
        tokio::net::UdpSocket::bind(addr).await
    }

    async fn recv_from(&mut self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> {
        self.recv_from(buf).await
    }

    async fn send_to(&mut self, buf: &[u8], target: &SocketAddr) -> io::Result<usize> {
        self.send_to(buf, target).await
    }
}

#[cfg(test)]
#[cfg(feature = "tokio-runtime")]
mod tests {
    #[cfg(not(target_os = "linux"))] // ignored until Travis-CI fixes IPv6
    use std::net::Ipv6Addr;
    use std::net::{IpAddr, Ipv4Addr};
    use tokio::{net::UdpSocket as TokioUdpSocket, runtime::Runtime};

    #[test]
    fn test_next_random_socket() {
        use crate::tests::next_random_socket_test;
        let io_loop = Runtime::new().expect("failed to create tokio runtime");
        next_random_socket_test::<TokioUdpSocket, Runtime>(io_loop)
    }

    #[test]
    fn test_udp_stream_ipv4() {
        use crate::tests::udp_stream_test;
        let io_loop = Runtime::new().expect("failed to create tokio runtime");
        udp_stream_test::<TokioUdpSocket, Runtime>(
            IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
            io_loop,
        );
    }

    #[test]
    #[cfg(not(target_os = "linux"))] // ignored until Travis-CI fixes IPv6
    fn test_udp_stream_ipv6() {
        use crate::tests::udp_stream_test;
        let io_loop = Runtime::new().expect("failed to create tokio runtime");
        udp_stream_test::<TokioUdpSocket, Runtime>(
            IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)),
            io_loop,
        );
    }
}
<|endoftext|>


<commit_before>//! FIXME: write short doc here

mod cargo_workspace;
mod json_project;
mod sysroot;

use std::{
    error::Error,
    fs::File,
    io::BufReader,
    path::{Path, PathBuf},
    process::Command,
};

use ra_cfg::CfgOptions;
use ra_db::{CrateGraph, CrateId, Edition, Env, FileId};
use rustc_hash::FxHashMap;
use serde_json::from_reader;

pub use crate::{
    cargo_workspace::{CargoFeatures, CargoWorkspace, Package, Target, TargetKind},
    json_project::JsonProject,
    sysroot::Sysroot,
};

pub type Result<T> = ::std::result::Result<T, Box<dyn Error + Send + Sync>>;

#[derive(Clone, PartialEq, Eq, Hash, Debug)]
pub struct CargoTomlNotFoundError(pub PathBuf);

impl std::fmt::Display for CargoTomlNotFoundError {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(fmt, "can't find Cargo.toml at {}", self.0.display())
    }
}

impl Error for CargoTomlNotFoundError {}

#[derive(Debug, Clone)]
pub enum ProjectWorkspace {
    /// Project workspace was discovered by running `cargo metadata` and `rustc --print sysroot`.
    Cargo { cargo: CargoWorkspace, sysroot: Sysroot },
    /// Project workspace was manually specified using a `rust-project.json` file.
    Json { project: JsonProject },
}

/// `PackageRoot` describes a package root folder.
/// Which may be an external dependency, or a member of
/// the current workspace.
#[derive(Clone)]
pub struct PackageRoot {
    /// Path to the root folder
    path: PathBuf,
    /// Is a member of the current workspace
    is_member: bool,
}

impl PackageRoot {
    pub fn new(path: PathBuf, is_member: bool) -> PackageRoot {
        PackageRoot { path, is_member }
    }

    pub fn path(&self) -> &PathBuf {
        &self.path
    }

    pub fn is_member(&self) -> bool {
        self.is_member
    }
}

impl ProjectWorkspace {
    pub fn discover(path: &Path, cargo_features: &CargoFeatures) -> Result<ProjectWorkspace> {
        ProjectWorkspace::discover_with_sysroot(path, true, cargo_features)
    }

    pub fn discover_with_sysroot(
        path: &Path,
        with_sysroot: bool,
        cargo_features: &CargoFeatures,
    ) -> Result<ProjectWorkspace> {
        match find_rust_project_json(path) {
            Some(json_path) => {
                let file = File::open(json_path)?;
                let reader = BufReader::new(file);
                Ok(ProjectWorkspace::Json { project: from_reader(reader)? })
            }
            None => {
                let cargo_toml = find_cargo_toml(path)?;
                let cargo = CargoWorkspace::from_cargo_metadata(&cargo_toml, cargo_features)?;
                let sysroot =
                    if with_sysroot { Sysroot::discover(&cargo_toml)? } else { Sysroot::default() };
                Ok(ProjectWorkspace::Cargo { cargo, sysroot })
            }
        }
    }

    /// Returns the roots for the current `ProjectWorkspace`
    /// The return type contains the path and whether or not
    /// the root is a member of the current workspace
    pub fn to_roots(&self) -> Vec<PackageRoot> {
        match self {
            ProjectWorkspace::Json { project } => {
                let mut roots = Vec::with_capacity(project.roots.len());
                for root in &project.roots {
                    roots.push(PackageRoot::new(root.path.clone(), true));
                }
                roots
            }
            ProjectWorkspace::Cargo { cargo, sysroot } => {
                let mut roots = Vec::with_capacity(cargo.packages().len() + sysroot.crates().len());
                for pkg in cargo.packages() {
                    let root = pkg.root(&cargo).to_path_buf();
                    let member = pkg.is_member(&cargo);
                    roots.push(PackageRoot::new(root, member));
                }
                for krate in sysroot.crates() {
                    roots.push(PackageRoot::new(krate.root_dir(&sysroot).to_path_buf(), false))
                }
                roots
            }
        }
    }

    pub fn n_packages(&self) -> usize {
        match self {
            ProjectWorkspace::Json { project } => project.crates.len(),
            ProjectWorkspace::Cargo { cargo, sysroot } => {
                cargo.packages().len() + sysroot.crates().len()
            }
        }
    }

    pub fn to_crate_graph(
        &self,
        default_cfg_options: &CfgOptions,
        load: &mut dyn FnMut(&Path) -> Option<FileId>,
    ) -> (CrateGraph, FxHashMap<CrateId, String>) {
        let mut crate_graph = CrateGraph::default();
        let mut names = FxHashMap::default();
        match self {
            ProjectWorkspace::Json { project } => {
                let mut crates = FxHashMap::default();
                for (id, krate) in project.crates.iter().enumerate() {
                    let crate_id = json_project::CrateId(id);
                    if let Some(file_id) = load(&krate.root_module) {
                        let edition = match krate.edition {
                            json_project::Edition::Edition2015 => Edition::Edition2015,
                            json_project::Edition::Edition2018 => Edition::Edition2018,
                        };
                        let cfg_options = {
                            let mut opts = default_cfg_options.clone();
                            for name in &krate.atom_cfgs {
                                opts.insert_atom(name.into());
                            }
                            for (key, value) in &krate.key_value_cfgs {
                                opts.insert_key_value(key.into(), value.into());
                            }
                            opts
                        };
                        crates.insert(
                            crate_id,
                            crate_graph.add_crate_root(
                                file_id,
                                edition,
                                cfg_options,
                                Env::default(),
                            ),
                        );
                    }
                }

                for (id, krate) in project.crates.iter().enumerate() {
                    for dep in &krate.deps {
                        let from_crate_id = json_project::CrateId(id);
                        let to_crate_id = dep.krate;
                        if let (Some(&from), Some(&to)) =
                            (crates.get(&from_crate_id), crates.get(&to_crate_id))
                        {
                            if let Err(_) = crate_graph.add_dep(from, dep.name.clone().into(), to) {
                                log::error!(
                                    "cyclic dependency {:?} -> {:?}",
                                    from_crate_id,
                                    to_crate_id
                                );
                            }
                        }
                    }
                }
            }
            ProjectWorkspace::Cargo { cargo, sysroot } => {
                let mut sysroot_crates = FxHashMap::default();
                for krate in sysroot.crates() {
                    if let Some(file_id) = load(krate.root(&sysroot)) {
                        // Crates from sysroot have `cfg(test)` disabled
                        let cfg_options = {
                            let mut opts = default_cfg_options.clone();
                            opts.remove_atom("test");
                            opts
                        };

                        let crate_id = crate_graph.add_crate_root(
                            file_id,
                            Edition::Edition2018,
                            cfg_options,
                            Env::default(),
                        );
                        sysroot_crates.insert(krate, crate_id);
                        names.insert(crate_id, krate.name(&sysroot).to_string());
                    }
                }
                for from in sysroot.crates() {
                    for to in from.deps(&sysroot) {
                        let name = to.name(&sysroot);
                        if let (Some(&from), Some(&to)) =
                            (sysroot_crates.get(&from), sysroot_crates.get(&to))
                        {
                            if let Err(_) = crate_graph.add_dep(from, name.into(), to) {
                                log::error!("cyclic dependency between sysroot crates")
                            }
                        }
                    }
                }

                let libcore = sysroot.core().and_then(|it| sysroot_crates.get(&it).copied());
                let liballoc = sysroot.alloc().and_then(|it| sysroot_crates.get(&it).copied());
                let libstd = sysroot.std().and_then(|it| sysroot_crates.get(&it).copied());
                let libproc_macro =
                    sysroot.proc_macro().and_then(|it| sysroot_crates.get(&it).copied());

                let mut pkg_to_lib_crate = FxHashMap::default();
                let mut pkg_crates = FxHashMap::default();
                // Next, create crates for each package, target pair
                for pkg in cargo.packages() {
                    let mut lib_tgt = None;
                    for tgt in pkg.targets(&cargo) {
                        let root = tgt.root(&cargo);
                        if let Some(file_id) = load(root) {
                            let edition = pkg.edition(&cargo);
                            let cfg_options = {
                                let mut opts = default_cfg_options.clone();
                                opts.insert_features(pkg.features(&cargo).iter().map(Into::into));
                                opts
                            };
                            let crate_id = crate_graph.add_crate_root(
                                file_id,
                                edition,
                                cfg_options,
                                Env::default(),
                            );
                            names.insert(crate_id, pkg.name(&cargo).to_string());
                            if tgt.kind(&cargo) == TargetKind::Lib {
                                lib_tgt = Some(crate_id);
                                pkg_to_lib_crate.insert(pkg, crate_id);
                            }
                            if tgt.is_proc_macro(&cargo) {
                                if let Some(proc_macro) = libproc_macro {
                                    if let Err(_) = crate_graph.add_dep(
                                        crate_id,
                                        "proc_macro".into(),
                                        proc_macro,
                                    ) {
                                        log::error!(
                                            "cyclic dependency on proc_macro for {}",
                                            pkg.name(&cargo)
                                        )
                                    }
                                }
                            }

                            pkg_crates.entry(pkg).or_insert_with(Vec::new).push(crate_id);
                        }
                    }

                    // Set deps to the core, std and to the lib target of the current package
                    for &from in pkg_crates.get(&pkg).into_iter().flatten() {
                        if let Some(to) = lib_tgt {
                            if to != from {
                                if let Err(_) =
                                    crate_graph.add_dep(from, pkg.name(&cargo).into(), to)
                                {
                                    log::error!(
                                        "cyclic dependency between targets of {}",
                                        pkg.name(&cargo)
                                    )
                                }
                            }
                        }
                        // core is added as a dependency before std in order to
                        // mimic rustcs dependency order
                        if let Some(core) = libcore {
                            if let Err(_) = crate_graph.add_dep(from, "core".into(), core) {
                                log::error!("cyclic dependency on core for {}", pkg.name(&cargo))
                            }
                        }
                        if let Some(alloc) = liballoc {
                            if let Err(_) = crate_graph.add_dep(from, "alloc".into(), alloc) {
                                log::error!("cyclic dependency on alloc for {}", pkg.name(&cargo))
                            }
                        }
                        if let Some(std) = libstd {
                            if let Err(_) = crate_graph.add_dep(from, "std".into(), std) {
                                log::error!("cyclic dependency on std for {}", pkg.name(&cargo))
                            }
                        }
                    }
                }

                // Now add a dep edge from all targets of upstream to the lib
                // target of downstream.
                for pkg in cargo.packages() {
                    for dep in pkg.dependencies(&cargo) {
                        if let Some(&to) = pkg_to_lib_crate.get(&dep.pkg) {
                            for &from in pkg_crates.get(&pkg).into_iter().flatten() {
                                if let Err(_) =
                                    crate_graph.add_dep(from, dep.name.clone().into(), to)
                                {
                                    log::error!(
                                        "cyclic dependency {} -> {}",
                                        pkg.name(&cargo),
                                        dep.pkg.name(&cargo)
                                    )
                                }
                            }
                        }
                    }
                }
            }
        }
        (crate_graph, names)
    }

    pub fn workspace_root_for(&self, path: &Path) -> Option<&Path> {
        match self {
            ProjectWorkspace::Cargo { cargo, .. } => {
                Some(cargo.workspace_root.as_ref()).filter(|root| path.starts_with(root))
            }
            ProjectWorkspace::Json { project: JsonProject { roots, .. } } => roots
                .iter()
                .find(|root| path.starts_with(&root.path))
                .map(|root| root.path.as_ref()),
        }
    }
}

fn find_rust_project_json(path: &Path) -> Option<PathBuf> {
    if path.ends_with("rust-project.json") {
        return Some(path.to_path_buf());
    }

    let mut curr = Some(path);
    while let Some(path) = curr {
        let candidate = path.join("rust-project.json");
        if candidate.exists() {
            return Some(candidate);
        }
        curr = path.parent();
    }

    None
}

fn find_cargo_toml(path: &Path) -> Result<PathBuf> {
    if path.ends_with("Cargo.toml") {
        return Ok(path.to_path_buf());
    }
    let mut curr = Some(path);
    while let Some(path) = curr {
        let candidate = path.join("Cargo.toml");
        if candidate.exists() {
            return Ok(candidate);
        }
        curr = path.parent();
    }
    Err(Box::new(CargoTomlNotFoundError(path.to_path_buf())))
}

pub fn get_rustc_cfg_options() -> CfgOptions {
    let mut cfg_options = CfgOptions::default();

    // Some nightly-only cfgs, which are required for stdlib
    {
        cfg_options.insert_atom("target_thread_local".into());
        for &target_has_atomic in ["16", "32", "64", "8", "cas", "ptr"].iter() {
            cfg_options.insert_key_value("target_has_atomic".into(), target_has_atomic.into())
        }
    }

    match (|| -> Result<_> {
        // `cfg(test)` and `cfg(debug_assertion)` are handled outside, so we suppress them here.
        let output = Command::new("rustc").args(&["--print", "cfg", "-O"]).output()?;
        if !output.status.success() {
            Err("failed to get rustc cfgs")?;
        }
        Ok(String::from_utf8(output.stdout)?)
    })() {
        Ok(rustc_cfgs) => {
            for line in rustc_cfgs.lines() {
                match line.find('=') {
                    None => cfg_options.insert_atom(line.into()),
                    Some(pos) => {
                        let key = &line[..pos];
                        let value = line[pos + 1..].trim_matches('"');
                        cfg_options.insert_key_value(key.into(), value.into());
                    }
                }
            }
        }
        Err(e) => log::error!("failed to get rustc cfgs: {}", e),
    }

    cfg_options
}
<commit_msg>Remove unneeded Box::new<commit_after>//! FIXME: write short doc here

mod cargo_workspace;
mod json_project;
mod sysroot;

use std::{
    error::Error,
    fs::File,
    io::BufReader,
    path::{Path, PathBuf},
    process::Command,
};

use ra_cfg::CfgOptions;
use ra_db::{CrateGraph, CrateId, Edition, Env, FileId};
use rustc_hash::FxHashMap;
use serde_json::from_reader;

pub use crate::{
    cargo_workspace::{CargoFeatures, CargoWorkspace, Package, Target, TargetKind},
    json_project::JsonProject,
    sysroot::Sysroot,
};

pub type Result<T> = ::std::result::Result<T, Box<dyn Error + Send + Sync>>;

#[derive(Clone, PartialEq, Eq, Hash, Debug)]
pub struct CargoTomlNotFoundError(pub PathBuf);

impl std::fmt::Display for CargoTomlNotFoundError {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(fmt, "can't find Cargo.toml at {}", self.0.display())
    }
}

impl Error for CargoTomlNotFoundError {}

#[derive(Debug, Clone)]
pub enum ProjectWorkspace {
    /// Project workspace was discovered by running `cargo metadata` and `rustc --print sysroot`.
    Cargo { cargo: CargoWorkspace, sysroot: Sysroot },
    /// Project workspace was manually specified using a `rust-project.json` file.
    Json { project: JsonProject },
}

/// `PackageRoot` describes a package root folder.
/// Which may be an external dependency, or a member of
/// the current workspace.
#[derive(Clone)]
pub struct PackageRoot {
    /// Path to the root folder
    path: PathBuf,
    /// Is a member of the current workspace
    is_member: bool,
}

impl PackageRoot {
    pub fn new(path: PathBuf, is_member: bool) -> PackageRoot {
        PackageRoot { path, is_member }
    }

    pub fn path(&self) -> &PathBuf {
        &self.path
    }

    pub fn is_member(&self) -> bool {
        self.is_member
    }
}

impl ProjectWorkspace {
    pub fn discover(path: &Path, cargo_features: &CargoFeatures) -> Result<ProjectWorkspace> {
        ProjectWorkspace::discover_with_sysroot(path, true, cargo_features)
    }

    pub fn discover_with_sysroot(
        path: &Path,
        with_sysroot: bool,
        cargo_features: &CargoFeatures,
    ) -> Result<ProjectWorkspace> {
        match find_rust_project_json(path) {
            Some(json_path) => {
                let file = File::open(json_path)?;
                let reader = BufReader::new(file);
                Ok(ProjectWorkspace::Json { project: from_reader(reader)? })
            }
            None => {
                let cargo_toml = find_cargo_toml(path)?;
                let cargo = CargoWorkspace::from_cargo_metadata(&cargo_toml, cargo_features)?;
                let sysroot =
                    if with_sysroot { Sysroot::discover(&cargo_toml)? } else { Sysroot::default() };
                Ok(ProjectWorkspace::Cargo { cargo, sysroot })
            }
        }
    }

    /// Returns the roots for the current `ProjectWorkspace`
    /// The return type contains the path and whether or not
    /// the root is a member of the current workspace
    pub fn to_roots(&self) -> Vec<PackageRoot> {
        match self {
            ProjectWorkspace::Json { project } => {
                let mut roots = Vec::with_capacity(project.roots.len());
                for root in &project.roots {
                    roots.push(PackageRoot::new(root.path.clone(), true));
                }
                roots
            }
            ProjectWorkspace::Cargo { cargo, sysroot } => {
                let mut roots = Vec::with_capacity(cargo.packages().len() + sysroot.crates().len());
                for pkg in cargo.packages() {
                    let root = pkg.root(&cargo).to_path_buf();
                    let member = pkg.is_member(&cargo);
                    roots.push(PackageRoot::new(root, member));
                }
                for krate in sysroot.crates() {
                    roots.push(PackageRoot::new(krate.root_dir(&sysroot).to_path_buf(), false))
                }
                roots
            }
        }
    }

    pub fn n_packages(&self) -> usize {
        match self {
            ProjectWorkspace::Json { project } => project.crates.len(),
            ProjectWorkspace::Cargo { cargo, sysroot } => {
                cargo.packages().len() + sysroot.crates().len()
            }
        }
    }

    pub fn to_crate_graph(
        &self,
        default_cfg_options: &CfgOptions,
        load: &mut dyn FnMut(&Path) -> Option<FileId>,
    ) -> (CrateGraph, FxHashMap<CrateId, String>) {
        let mut crate_graph = CrateGraph::default();
        let mut names = FxHashMap::default();
        match self {
            ProjectWorkspace::Json { project } => {
                let mut crates = FxHashMap::default();
                for (id, krate) in project.crates.iter().enumerate() {
                    let crate_id = json_project::CrateId(id);
                    if let Some(file_id) = load(&krate.root_module) {
                        let edition = match krate.edition {
                            json_project::Edition::Edition2015 => Edition::Edition2015,
                            json_project::Edition::Edition2018 => Edition::Edition2018,
                        };
                        let cfg_options = {
                            let mut opts = default_cfg_options.clone();
                            for name in &krate.atom_cfgs {
                                opts.insert_atom(name.into());
                            }
                            for (key, value) in &krate.key_value_cfgs {
                                opts.insert_key_value(key.into(), value.into());
                            }
                            opts
                        };
                        crates.insert(
                            crate_id,
                            crate_graph.add_crate_root(
                                file_id,
                                edition,
                                cfg_options,
                                Env::default(),
                            ),
                        );
                    }
                }

                for (id, krate) in project.crates.iter().enumerate() {
                    for dep in &krate.deps {
                        let from_crate_id = json_project::CrateId(id);
                        let to_crate_id = dep.krate;
                        if let (Some(&from), Some(&to)) =
                            (crates.get(&from_crate_id), crates.get(&to_crate_id))
                        {
                            if let Err(_) = crate_graph.add_dep(from, dep.name.clone().into(), to) {
                                log::error!(
                                    "cyclic dependency {:?} -> {:?}",
                                    from_crate_id,
                                    to_crate_id
                                );
                            }
                        }
                    }
                }
            }
            ProjectWorkspace::Cargo { cargo, sysroot } => {
                let mut sysroot_crates = FxHashMap::default();
                for krate in sysroot.crates() {
                    if let Some(file_id) = load(krate.root(&sysroot)) {
                        // Crates from sysroot have `cfg(test)` disabled
                        let cfg_options = {
                            let mut opts = default_cfg_options.clone();
                            opts.remove_atom("test");
                            opts
                        };

                        let crate_id = crate_graph.add_crate_root(
                            file_id,
                            Edition::Edition2018,
                            cfg_options,
                            Env::default(),
                        );
                        sysroot_crates.insert(krate, crate_id);
                        names.insert(crate_id, krate.name(&sysroot).to_string());
                    }
                }
                for from in sysroot.crates() {
                    for to in from.deps(&sysroot) {
                        let name = to.name(&sysroot);
                        if let (Some(&from), Some(&to)) =
                            (sysroot_crates.get(&from), sysroot_crates.get(&to))
                        {
                            if let Err(_) = crate_graph.add_dep(from, name.into(), to) {
                                log::error!("cyclic dependency between sysroot crates")
                            }
                        }
                    }
                }

                let libcore = sysroot.core().and_then(|it| sysroot_crates.get(&it).copied());
                let liballoc = sysroot.alloc().and_then(|it| sysroot_crates.get(&it).copied());
                let libstd = sysroot.std().and_then(|it| sysroot_crates.get(&it).copied());
                let libproc_macro =
                    sysroot.proc_macro().and_then(|it| sysroot_crates.get(&it).copied());

                let mut pkg_to_lib_crate = FxHashMap::default();
                let mut pkg_crates = FxHashMap::default();
                // Next, create crates for each package, target pair
                for pkg in cargo.packages() {
                    let mut lib_tgt = None;
                    for tgt in pkg.targets(&cargo) {
                        let root = tgt.root(&cargo);
                        if let Some(file_id) = load(root) {
                            let edition = pkg.edition(&cargo);
                            let cfg_options = {
                                let mut opts = default_cfg_options.clone();
                                opts.insert_features(pkg.features(&cargo).iter().map(Into::into));
                                opts
                            };
                            let crate_id = crate_graph.add_crate_root(
                                file_id,
                                edition,
                                cfg_options,
                                Env::default(),
                            );
                            names.insert(crate_id, pkg.name(&cargo).to_string());
                            if tgt.kind(&cargo) == TargetKind::Lib {
                                lib_tgt = Some(crate_id);
                                pkg_to_lib_crate.insert(pkg, crate_id);
                            }
                            if tgt.is_proc_macro(&cargo) {
                                if let Some(proc_macro) = libproc_macro {
                                    if let Err(_) = crate_graph.add_dep(
                                        crate_id,
                                        "proc_macro".into(),
                                        proc_macro,
                                    ) {
                                        log::error!(
                                            "cyclic dependency on proc_macro for {}",
                                            pkg.name(&cargo)
                                        )
                                    }
                                }
                            }

                            pkg_crates.entry(pkg).or_insert_with(Vec::new).push(crate_id);
                        }
                    }

                    // Set deps to the core, std and to the lib target of the current package
                    for &from in pkg_crates.get(&pkg).into_iter().flatten() {
                        if let Some(to) = lib_tgt {
                            if to != from {
                                if let Err(_) =
                                    crate_graph.add_dep(from, pkg.name(&cargo).into(), to)
                                {
                                    log::error!(
                                        "cyclic dependency between targets of {}",
                                        pkg.name(&cargo)
                                    )
                                }
                            }
                        }
                        // core is added as a dependency before std in order to
                        // mimic rustcs dependency order
                        if let Some(core) = libcore {
                            if let Err(_) = crate_graph.add_dep(from, "core".into(), core) {
                                log::error!("cyclic dependency on core for {}", pkg.name(&cargo))
                            }
                        }
                        if let Some(alloc) = liballoc {
                            if let Err(_) = crate_graph.add_dep(from, "alloc".into(), alloc) {
                                log::error!("cyclic dependency on alloc for {}", pkg.name(&cargo))
                            }
                        }
                        if let Some(std) = libstd {
                            if let Err(_) = crate_graph.add_dep(from, "std".into(), std) {
                                log::error!("cyclic dependency on std for {}", pkg.name(&cargo))
                            }
                        }
                    }
                }

                // Now add a dep edge from all targets of upstream to the lib
                // target of downstream.
                for pkg in cargo.packages() {
                    for dep in pkg.dependencies(&cargo) {
                        if let Some(&to) = pkg_to_lib_crate.get(&dep.pkg) {
                            for &from in pkg_crates.get(&pkg).into_iter().flatten() {
                                if let Err(_) =
                                    crate_graph.add_dep(from, dep.name.clone().into(), to)
                                {
                                    log::error!(
                                        "cyclic dependency {} -> {}",
                                        pkg.name(&cargo),
                                        dep.pkg.name(&cargo)
                                    )
                                }
                            }
                        }
                    }
                }
            }
        }
        (crate_graph, names)
    }

    pub fn workspace_root_for(&self, path: &Path) -> Option<&Path> {
        match self {
            ProjectWorkspace::Cargo { cargo, .. } => {
                Some(cargo.workspace_root.as_ref()).filter(|root| path.starts_with(root))
            }
            ProjectWorkspace::Json { project: JsonProject { roots, .. } } => roots
                .iter()
                .find(|root| path.starts_with(&root.path))
                .map(|root| root.path.as_ref()),
        }
    }
}

fn find_rust_project_json(path: &Path) -> Option<PathBuf> {
    if path.ends_with("rust-project.json") {
        return Some(path.to_path_buf());
    }

    let mut curr = Some(path);
    while let Some(path) = curr {
        let candidate = path.join("rust-project.json");
        if candidate.exists() {
            return Some(candidate);
        }
        curr = path.parent();
    }

    None
}

fn find_cargo_toml(path: &Path) -> Result<PathBuf> {
    if path.ends_with("Cargo.toml") {
        return Ok(path.to_path_buf());
    }
    let mut curr = Some(path);
    while let Some(path) = curr {
        let candidate = path.join("Cargo.toml");
        if candidate.exists() {
            return Ok(candidate);
        }
        curr = path.parent();
    }
    Err(CargoTomlNotFoundError(path.to_path_buf()))?
}

pub fn get_rustc_cfg_options() -> CfgOptions {
    let mut cfg_options = CfgOptions::default();

    // Some nightly-only cfgs, which are required for stdlib
    {
        cfg_options.insert_atom("target_thread_local".into());
        for &target_has_atomic in ["16", "32", "64", "8", "cas", "ptr"].iter() {
            cfg_options.insert_key_value("target_has_atomic".into(), target_has_atomic.into())
        }
    }

    match (|| -> Result<_> {
        // `cfg(test)` and `cfg(debug_assertion)` are handled outside, so we suppress them here.
        let output = Command::new("rustc").args(&["--print", "cfg", "-O"]).output()?;
        if !output.status.success() {
            Err("failed to get rustc cfgs")?;
        }
        Ok(String::from_utf8(output.stdout)?)
    })() {
        Ok(rustc_cfgs) => {
            for line in rustc_cfgs.lines() {
                match line.find('=') {
                    None => cfg_options.insert_atom(line.into()),
                    Some(pos) => {
                        let key = &line[..pos];
                        let value = line[pos + 1..].trim_matches('"');
                        cfg_options.insert_key_value(key.into(), value.into());
                    }
                }
            }
        }
        Err(e) => log::error!("failed to get rustc cfgs: {}", e),
    }

    cfg_options
}
<|endoftext|>


<commit_before>//! Operations related to UTF-8 validation.

use crate::mem;

use super::Utf8Error;

/// Returns the initial codepoint accumulator for the first byte.
/// The first byte is special, only want bottom 5 bits for width 2, 4 bits
/// for width 3, and 3 bits for width 4.
#[inline]
fn utf8_first_byte(byte: u8, width: u32) -> u32 {
    (byte & (0x7F >> width)) as u32
}

/// Returns the value of `ch` updated with continuation byte `byte`.
#[inline]
fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 {
    (ch << 6) | (byte & CONT_MASK) as u32
}

/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the
/// bits `10`).
#[inline]
pub(super) fn utf8_is_cont_byte(byte: u8) -> bool {
    (byte as i8) < -64
}

#[inline]
fn unwrap_or_0(opt: Option<&u8>) -> u8 {
    match opt {
        Some(&byte) => byte,
        None => 0,
    }
}

/// Reads the next code point out of a byte iterator (assuming a
/// UTF-8-like encoding).
#[unstable(feature = "str_internals", issue = "none")]
#[inline]
pub fn next_code_point<'a, I: Iterator<Item = &'a u8>>(bytes: &mut I) -> Option<u32> {
    // Decode UTF-8
    let x = *bytes.next()?;
    if x < 128 {
        return Some(x as u32);
    }

    // Multibyte case follows
    // Decode from a byte combination out of: [[[x y] z] w]
    // NOTE: Performance is sensitive to the exact formulation here
    let init = utf8_first_byte(x, 2);
    let y = unwrap_or_0(bytes.next());
    let mut ch = utf8_acc_cont_byte(init, y);
    if x >= 0xE0 {
        // [[x y z] w] case
        // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid
        let z = unwrap_or_0(bytes.next());
        let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z);
        ch = init << 12 | y_z;
        if x >= 0xF0 {
            // [x y z w] case
            // use only the lower 3 bits of `init`
            let w = unwrap_or_0(bytes.next());
            ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w);
        }
    }

    Some(ch)
}

/// Reads the last code point out of a byte iterator (assuming a
/// UTF-8-like encoding).
#[inline]
pub(super) fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option<u32>
where
    I: DoubleEndedIterator<Item = &'a u8>,
{
    // Decode UTF-8
    let w = match *bytes.next_back()? {
        next_byte if next_byte < 128 => return Some(next_byte as u32),
        back_byte => back_byte,
    };

    // Multibyte case follows
    // Decode from a byte combination out of: [x [y [z w]]]
    let mut ch;
    let z = unwrap_or_0(bytes.next_back());
    ch = utf8_first_byte(z, 2);
    if utf8_is_cont_byte(z) {
        let y = unwrap_or_0(bytes.next_back());
        ch = utf8_first_byte(y, 3);
        if utf8_is_cont_byte(y) {
            let x = unwrap_or_0(bytes.next_back());
            ch = utf8_first_byte(x, 4);
            ch = utf8_acc_cont_byte(ch, y);
        }
        ch = utf8_acc_cont_byte(ch, z);
    }
    ch = utf8_acc_cont_byte(ch, w);

    Some(ch)
}

// use truncation to fit u64 into usize
const NONASCII_MASK: usize = 0x80808080_80808080u64 as usize;

/// Returns `true` if any byte in the word `x` is nonascii (>= 128).
#[inline]
fn contains_nonascii(x: usize) -> bool {
    (x & NONASCII_MASK) != 0
}

/// Walks through `v` checking that it's a valid UTF-8 sequence,
/// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`.
#[inline(always)]
pub(super) fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> {
    let mut index = 0;
    let len = v.len();

    let usize_bytes = mem::size_of::<usize>();
    let ascii_block_size = 2 * usize_bytes;
    let blocks_end = if len >= ascii_block_size { len - ascii_block_size + 1 } else { 0 };
    let align = v.as_ptr().align_offset(usize_bytes);

    while index < len {
        let old_offset = index;
        macro_rules! err {
            ($error_len: expr) => {
                return Err(Utf8Error { valid_up_to: old_offset, error_len: $error_len })
            };
        }

        macro_rules! next {
            () => {{
                index += 1;
                // we needed data, but there was none: error!
                if index >= len {
                    err!(None)
                }
                v[index]
            }};
        }

        let first = v[index];
        if first >= 128 {
            let w = UTF8_CHAR_WIDTH[first as usize];
            // 2-byte encoding is for codepoints  \u{0080} to  \u{07ff}
            //        first  C2 80        last DF BF
            // 3-byte encoding is for codepoints  \u{0800} to  \u{ffff}
            //        first  E0 A0 80     last EF BF BF
            //   excluding surrogates codepoints  \u{d800} to  \u{dfff}
            //               ED A0 80 to       ED BF BF
            // 4-byte encoding is for codepoints \u{1000}0 to \u{10ff}ff
            //        first  F0 90 80 80  last F4 8F BF BF
            //
            // Use the UTF-8 syntax from the RFC
            //
            // https://tools.ietf.org/html/rfc3629
            // UTF8-1      = %x00-7F
            // UTF8-2      = %xC2-DF UTF8-tail
            // UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
            //               %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
            // UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
            //               %xF4 %x80-8F 2( UTF8-tail )
            match w {
                2 => {
                    if !utf8_is_cont_byte(next!()) {
                        err!(Some(1))
                    }
                }
                3 => {
                    match (first, next!()) {
                        (0xE0, 0xA0..=0xBF)
                        | (0xE1..=0xEC, 0x80..=0xBF)
                        | (0xED, 0x80..=0x9F)
                        | (0xEE..=0xEF, 0x80..=0xBF) => {}
                        _ => err!(Some(1)),
                    }
                    if !utf8_is_cont_byte(next!()) {
                        err!(Some(2))
                    }
                }
                4 => {
                    match (first, next!()) {
                        (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {}
                        _ => err!(Some(1)),
                    }
                    if !utf8_is_cont_byte(next!()) {
                        err!(Some(2))
                    }
                    if !utf8_is_cont_byte(next!()) {
                        err!(Some(3))
                    }
                }
                _ => err!(Some(1)),
            }
            index += 1;
        } else {
            // Ascii case, try to skip forward quickly.
            // When the pointer is aligned, read 2 words of data per iteration
            // until we find a word containing a non-ascii byte.
            if align != usize::MAX && align.wrapping_sub(index) % usize_bytes == 0 {
                let ptr = v.as_ptr();
                while index < blocks_end {
                    // SAFETY: since `align - index` and `ascii_block_size` are
                    // multiples of `usize_bytes`, `block = ptr.add(index)` is
                    // always aligned with a `usize` so it's safe to dereference
                    // both `block` and `block.offset(1)`.
                    unsafe {
                        let block = ptr.add(index) as *const usize;
                        // break if there is a nonascii byte
                        let zu = contains_nonascii(*block);
                        let zv = contains_nonascii(*block.offset(1));
                        if zu | zv {
                            break;
                        }
                    }
                    index += ascii_block_size;
                }
                // step from the point where the wordwise loop stopped
                while index < len && v[index] < 128 {
                    index += 1;
                }
            } else {
                index += 1;
            }
        }
    }

    Ok(())
}

// https://tools.ietf.org/html/rfc3629
static UTF8_CHAR_WIDTH: [u8; 256] = [
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x1F
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x3F
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x5F
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x7F
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, // 0x9F
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, // 0xBF
    0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    2, // 0xDF
    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xEF
    4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xFF
];

/// Given a first byte, determines how many bytes are in this UTF-8 character.
#[unstable(feature = "str_internals", issue = "none")]
#[inline]
pub fn utf8_char_width(b: u8) -> usize {
    UTF8_CHAR_WIDTH[b as usize] as usize
}

/// Mask of the value bits of a continuation byte.
const CONT_MASK: u8 = 0b0011_1111;

// truncate `&str` to length at most equal to `max`
// return `true` if it were truncated, and the new str.
pub(super) fn truncate_to_char_boundary(s: &str, mut max: usize) -> (bool, &str) {
    if max >= s.len() {
        (false, s)
    } else {
        while !s.is_char_boundary(max) {
            max -= 1;
        }
        (true, &s[..max])
    }
}
<commit_msg>manually inline function<commit_after>//! Operations related to UTF-8 validation.

use crate::mem;

use super::Utf8Error;

/// Returns the initial codepoint accumulator for the first byte.
/// The first byte is special, only want bottom 5 bits for width 2, 4 bits
/// for width 3, and 3 bits for width 4.
#[inline]
fn utf8_first_byte(byte: u8, width: u32) -> u32 {
    (byte & (0x7F >> width)) as u32
}

/// Returns the value of `ch` updated with continuation byte `byte`.
#[inline]
fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 {
    (ch << 6) | (byte & CONT_MASK) as u32
}

/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the
/// bits `10`).
#[inline]
pub(super) fn utf8_is_cont_byte(byte: u8) -> bool {
    (byte as i8) < -64
}

#[inline]
fn unwrap_or_0(opt: Option<&u8>) -> u8 {
    match opt {
        Some(&byte) => byte,
        None => 0,
    }
}

/// Reads the next code point out of a byte iterator (assuming a
/// UTF-8-like encoding).
#[unstable(feature = "str_internals", issue = "none")]
#[inline]
pub fn next_code_point<'a, I: Iterator<Item = &'a u8>>(bytes: &mut I) -> Option<u32> {
    // Decode UTF-8
    let x = *bytes.next()?;
    if x < 128 {
        return Some(x as u32);
    }

    // Multibyte case follows
    // Decode from a byte combination out of: [[[x y] z] w]
    // NOTE: Performance is sensitive to the exact formulation here
    let init = utf8_first_byte(x, 2);
    let y = unwrap_or_0(bytes.next());
    let mut ch = utf8_acc_cont_byte(init, y);
    if x >= 0xE0 {
        // [[x y z] w] case
        // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid
        let z = unwrap_or_0(bytes.next());
        let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z);
        ch = init << 12 | y_z;
        if x >= 0xF0 {
            // [x y z w] case
            // use only the lower 3 bits of `init`
            let w = unwrap_or_0(bytes.next());
            ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w);
        }
    }

    Some(ch)
}

/// Reads the last code point out of a byte iterator (assuming a
/// UTF-8-like encoding).
#[inline]
pub(super) fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option<u32>
where
    I: DoubleEndedIterator<Item = &'a u8>,
{
    // Decode UTF-8
    let w = match *bytes.next_back()? {
        next_byte if next_byte < 128 => return Some(next_byte as u32),
        back_byte => back_byte,
    };

    // Multibyte case follows
    // Decode from a byte combination out of: [x [y [z w]]]
    let mut ch;
    let z = unwrap_or_0(bytes.next_back());
    ch = utf8_first_byte(z, 2);
    if utf8_is_cont_byte(z) {
        let y = unwrap_or_0(bytes.next_back());
        ch = utf8_first_byte(y, 3);
        if utf8_is_cont_byte(y) {
            let x = unwrap_or_0(bytes.next_back());
            ch = utf8_first_byte(x, 4);
            ch = utf8_acc_cont_byte(ch, y);
        }
        ch = utf8_acc_cont_byte(ch, z);
    }
    ch = utf8_acc_cont_byte(ch, w);

    Some(ch)
}

// use truncation to fit u64 into usize
const NONASCII_MASK: usize = 0x80808080_80808080u64 as usize;

/// Returns `true` if any byte in the word `x` is nonascii (>= 128).
#[inline]
fn contains_nonascii(x: usize) -> bool {
    (x & NONASCII_MASK) != 0
}

/// Walks through `v` checking that it's a valid UTF-8 sequence,
/// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`.
#[inline(always)]
pub(super) fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> {
    let mut index = 0;
    let len = v.len();

    let usize_bytes = mem::size_of::<usize>();
    let ascii_block_size = 2 * usize_bytes;
    let blocks_end = if len >= ascii_block_size { len - ascii_block_size + 1 } else { 0 };
    let align = v.as_ptr().align_offset(usize_bytes);

    while index < len {
        let old_offset = index;
        macro_rules! err {
            ($error_len: expr) => {
                return Err(Utf8Error { valid_up_to: old_offset, error_len: $error_len })
            };
        }

        macro_rules! next {
            () => {{
                index += 1;
                // we needed data, but there was none: error!
                if index >= len {
                    err!(None)
                }
                v[index]
            }};
        }

        let first = v[index];
        if first >= 128 {
            let w = UTF8_CHAR_WIDTH[first as usize];
            // 2-byte encoding is for codepoints  \u{0080} to  \u{07ff}
            //        first  C2 80        last DF BF
            // 3-byte encoding is for codepoints  \u{0800} to  \u{ffff}
            //        first  E0 A0 80     last EF BF BF
            //   excluding surrogates codepoints  \u{d800} to  \u{dfff}
            //               ED A0 80 to       ED BF BF
            // 4-byte encoding is for codepoints \u{1000}0 to \u{10ff}ff
            //        first  F0 90 80 80  last F4 8F BF BF
            //
            // Use the UTF-8 syntax from the RFC
            //
            // https://tools.ietf.org/html/rfc3629
            // UTF8-1      = %x00-7F
            // UTF8-2      = %xC2-DF UTF8-tail
            // UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
            //               %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
            // UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
            //               %xF4 %x80-8F 2( UTF8-tail )
            match w {
                2 => {
                    if next!() as i8 >= -64 {
                        err!(Some(1))
                    }
                }
                3 => {
                    match (first, next!()) {
                        (0xE0, 0xA0..=0xBF)
                        | (0xE1..=0xEC, 0x80..=0xBF)
                        | (0xED, 0x80..=0x9F)
                        | (0xEE..=0xEF, 0x80..=0xBF) => {}
                        _ => err!(Some(1)),
                    }
                    if next!() as i8 >= -64 {
                        err!(Some(2))
                    }
                }
                4 => {
                    match (first, next!()) {
                        (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {}
                        _ => err!(Some(1)),
                    }
                    if next!() as i8 >= -64 {
                        err!(Some(2))
                    }
                    if next!() as i8 >= -64 {
                        err!(Some(3))
                    }
                }
                _ => err!(Some(1)),
            }
            index += 1;
        } else {
            // Ascii case, try to skip forward quickly.
            // When the pointer is aligned, read 2 words of data per iteration
            // until we find a word containing a non-ascii byte.
            if align != usize::MAX && align.wrapping_sub(index) % usize_bytes == 0 {
                let ptr = v.as_ptr();
                while index < blocks_end {
                    // SAFETY: since `align - index` and `ascii_block_size` are
                    // multiples of `usize_bytes`, `block = ptr.add(index)` is
                    // always aligned with a `usize` so it's safe to dereference
                    // both `block` and `block.offset(1)`.
                    unsafe {
                        let block = ptr.add(index) as *const usize;
                        // break if there is a nonascii byte
                        let zu = contains_nonascii(*block);
                        let zv = contains_nonascii(*block.offset(1));
                        if zu | zv {
                            break;
                        }
                    }
                    index += ascii_block_size;
                }
                // step from the point where the wordwise loop stopped
                while index < len && v[index] < 128 {
                    index += 1;
                }
            } else {
                index += 1;
            }
        }
    }

    Ok(())
}

// https://tools.ietf.org/html/rfc3629
static UTF8_CHAR_WIDTH: [u8; 256] = [
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x1F
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x3F
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x5F
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, // 0x7F
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, // 0x9F
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, // 0xBF
    0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    2, // 0xDF
    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xEF
    4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xFF
];

/// Given a first byte, determines how many bytes are in this UTF-8 character.
#[unstable(feature = "str_internals", issue = "none")]
#[inline]
pub fn utf8_char_width(b: u8) -> usize {
    UTF8_CHAR_WIDTH[b as usize] as usize
}

/// Mask of the value bits of a continuation byte.
const CONT_MASK: u8 = 0b0011_1111;

// truncate `&str` to length at most equal to `max`
// return `true` if it were truncated, and the new str.
pub(super) fn truncate_to_char_boundary(s: &str, mut max: usize) -> (bool, &str) {
    if max >= s.len() {
        (false, s)
    } else {
        while !s.is_char_boundary(max) {
            max -= 1;
        }
        (true, &s[..max])
    }
}
<|endoftext|>


<commit_before>//! RA Proc Macro Server
//!
//! This library is able to call compiled Rust custom derive dynamic libraries on arbitrary code.
//! The general idea here is based on https://github.com/fedochet/rust-proc-macro-expander.
//!
//! But we adapt it to better fit RA needs:
//!
//! * We use `ra_tt` for proc-macro `TokenStream` server, it is easy to manipulate and interact with
//!   RA than `proc-macro2` token stream.
//! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable`
//!   rustc rather than `unstable`. (Although in gerenal ABI compatibility is still an issue)

#[allow(dead_code)]
#[doc(hidden)]
mod proc_macro;

#[doc(hidden)]
mod rustc_server;

mod dylib;

use proc_macro::bridge::client::TokenStream;
use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask};
use std::path::Path;

pub(crate) fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
    let expander = create_expander(&task.lib);

    match expander.expand(&task.macro_name, &task.macro_body, task.attributes.as_ref()) {
        Ok(expansion) => Ok(ExpansionResult { expansion }),
        Err(msg) => {
            Err(format!("Cannot perform expansion for {}: error {:?}", &task.macro_name, msg))
        }
    }
}

pub(crate) fn list_macros(task: &ListMacrosTask) -> ListMacrosResult {
    let expander = create_expander(&task.lib);

    ListMacrosResult { macros: expander.list_macros() }
}

fn create_expander(lib: &Path) -> dylib::Expander {
    dylib::Expander::new(lib)
        .unwrap_or_else(|err| panic!("Cannot create expander for {:?}: {:?}", lib, err))
}

pub mod cli;

#[cfg(test)]
mod tests;
<commit_msg>Display path not debug it<commit_after>//! RA Proc Macro Server
//!
//! This library is able to call compiled Rust custom derive dynamic libraries on arbitrary code.
//! The general idea here is based on https://github.com/fedochet/rust-proc-macro-expander.
//!
//! But we adapt it to better fit RA needs:
//!
//! * We use `ra_tt` for proc-macro `TokenStream` server, it is easy to manipulate and interact with
//!   RA than `proc-macro2` token stream.
//! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable`
//!   rustc rather than `unstable`. (Although in gerenal ABI compatibility is still an issue)

#[allow(dead_code)]
#[doc(hidden)]
mod proc_macro;

#[doc(hidden)]
mod rustc_server;

mod dylib;

use proc_macro::bridge::client::TokenStream;
use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask};
use std::path::Path;

pub(crate) fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
    let expander = create_expander(&task.lib);

    match expander.expand(&task.macro_name, &task.macro_body, task.attributes.as_ref()) {
        Ok(expansion) => Ok(ExpansionResult { expansion }),
        Err(msg) => {
            Err(format!("Cannot perform expansion for {}: error {:?}", &task.macro_name, msg))
        }
    }
}

pub(crate) fn list_macros(task: &ListMacrosTask) -> ListMacrosResult {
    let expander = create_expander(&task.lib);

    ListMacrosResult { macros: expander.list_macros() }
}

fn create_expander(lib: &Path) -> dylib::Expander {
    dylib::Expander::new(lib)
        .unwrap_or_else(|err| panic!("Cannot create expander for {}: {}", lib.display(), err))
}

pub mod cli;

#[cfg(test)]
mod tests;
<|endoftext|>


<commit_before>//! Verification of MIPS MSA intrinsics
#![feature(try_trait)]
#![allow(bad_style, unused)]

// This file is obtained from
// https://gcc.gnu.org/onlinedocs//gcc/MIPS-SIMD-Architecture-Built-in-Functions.html
static HEADER: &str = include_str!("../mips-msa.h");

stdsimd_verify::mips_functions!(static FUNCTIONS);

struct Function {
    name: &'static str,
    arguments: &'static [&'static Type],
    ret: Option<&'static Type>,
    target_feature: Option<&'static str>,
    instrs: &'static [&'static str],
    file: &'static str,
    required_const: &'static [usize],
}

static F16: Type = Type::PrimFloat(16);
static F32: Type = Type::PrimFloat(32);
static F64: Type = Type::PrimFloat(64);
static I8: Type = Type::PrimSigned(8);
static I16: Type = Type::PrimSigned(16);
static I32: Type = Type::PrimSigned(32);
static I64: Type = Type::PrimSigned(64);
static U8: Type = Type::PrimUnsigned(8);
static U16: Type = Type::PrimUnsigned(16);
static U32: Type = Type::PrimUnsigned(32);
static U64: Type = Type::PrimUnsigned(64);
static NEVER: Type = Type::Never;
static TUPLE: Type = Type::Tuple;
static v16i8: Type = Type::I(8, 16, 1);
static v8i16: Type = Type::I(16, 8, 1);
static v4i32: Type = Type::I(32, 4, 1);
static v2i64: Type = Type::I(64, 2, 1);
static v16u8: Type = Type::U(8, 16, 1);
static v8u16: Type = Type::U(16, 8, 1);
static v4u32: Type = Type::U(32, 4, 1);
static v2u64: Type = Type::U(64, 2, 1);
static v8f16: Type = Type::F(16, 8, 1);
static v4f32: Type = Type::F(32, 4, 1);
static v2f64: Type = Type::F(64, 2, 1);

#[derive(Debug, Copy, Clone, PartialEq)]
enum Type {
    PrimFloat(u8),
    PrimSigned(u8),
    PrimUnsigned(u8),
    PrimPoly(u8),
    MutPtr(&'static Type),
    ConstPtr(&'static Type),
    Ptr(&'static Type),
    Tuple,
    I(u8, u8, u8),
    U(u8, u8, u8),
    P(u8, u8, u8),
    F(u8, u8, u8),
    Never,
}

#[derive(Copy, Clone, Debug, PartialEq)]
#[allow(non_camel_case_types)]
enum MsaTy {
    v16i8,
    v8i16,
    v4i32,
    v2i64,
    v16u8,
    v8u16,
    v4u32,
    v2u64,
    v8f16,
    v4f32,
    v2f64,
    imm0_1,
    imm0_3,
    imm0_7,
    imm0_15,
    imm0_31,
    imm0_63,
    imm0_255,
    imm_n16_15,
    imm_n512_511,
    imm_n1024_1022,
    imm_n2048_2044,
    imm_n4096_4088,
    i32,
    u32,
    i64,
    u64,
    Void,
    VoidPtr,
}

impl<'a> From<&'a str> for MsaTy {
    fn from(s: &'a str) -> MsaTy {
        match s {
            "v16i8" => MsaTy::v16i8,
            "v8i16" => MsaTy::v8i16,
            "v4i32" => MsaTy::v4i32,
            "v2i64" => MsaTy::v2i64,
            "v16u8" => MsaTy::v16u8,
            "v8u16" => MsaTy::v8u16,
            "v4u32" => MsaTy::v4u32,
            "v2u64" => MsaTy::v2u64,
            "v8f16" => MsaTy::v8f16,
            "v4f32" => MsaTy::v4f32,
            "v2f64" => MsaTy::v2f64,
            "imm0_1" => MsaTy::imm0_1,
            "imm0_3" => MsaTy::imm0_3,
            "imm0_7" => MsaTy::imm0_7,
            "imm0_15" => MsaTy::imm0_15,
            "imm0_31" => MsaTy::imm0_31,
            "imm0_63" => MsaTy::imm0_63,
            "imm0_255" => MsaTy::imm0_255,
            "imm_n16_15" => MsaTy::imm_n16_15,
            "imm_n512_511" => MsaTy::imm_n512_511,
            "imm_n1024_1022" => MsaTy::imm_n1024_1022,
            "imm_n2048_2044" => MsaTy::imm_n2048_2044,
            "imm_n4096_4088" => MsaTy::imm_n4096_4088,
            "i32" => MsaTy::i32,
            "u32" => MsaTy::u32,
            "i64" => MsaTy::i64,
            "u64" => MsaTy::u64,
            "void" => MsaTy::Void,
            "void *" => MsaTy::VoidPtr,
            v => panic!("unknown ty: \"{}\"", v),
        }
    }
}

#[derive(Debug, Clone)]
struct MsaIntrinsic {
    id: String,
    arg_tys: Vec<MsaTy>,
    ret_ty: MsaTy,
    instruction: String,
}

impl std::convert::TryFrom<&'static str> for MsaIntrinsic {
    // The intrinsics are just C function declarations of the form:
    // $ret_ty __builtin_${fn_id}($($arg_ty),*);
    type Error = std::option::NoneError;
    fn try_from(line: &'static str) -> Result<Self, Self::Error> {
        let first_whitespace = line.find(char::is_whitespace)?;
        let ret_ty = &line[0..first_whitespace];
        let ret_ty = MsaTy::from(ret_ty);

        let first_parentheses = line.find('(')?;
        assert!(first_parentheses > first_whitespace);
        let id = &line[first_whitespace + 1..first_parentheses].trim();
        assert!(id.starts_with("__builtin"));
        let mut id_str = "_".to_string();
        id_str += &id[9..];
        let id = id_str;

        let mut arg_tys = Vec::new();

        let last_parentheses = line.find(')')?;
        for arg in (&line[first_parentheses + 1..last_parentheses]).split(',') {
            let arg = arg.trim();
            arg_tys.push(MsaTy::from(arg));
        }

        // The instruction is the intrinsic name without the __msa_ prefix.
        let instruction = &id[6..];
        let mut instruction = instruction.to_string();
        // With all underscores but the first one replaced with a `.`
        if let Some(first_underscore) = instruction.find('_') {
            let postfix = instruction[first_underscore + 1..].replace('_', ".");
            instruction = instruction[0..=first_underscore].to_string();
            instruction += &postfix;
        }

        Ok(MsaIntrinsic {
            id,
            ret_ty,
            arg_tys,
            instruction,
        })
    }
}

#[test]
fn verify_all_signatures() {
    // Parse the C intrinsic header file:
    let mut intrinsics = std::collections::HashMap::<String, MsaIntrinsic>::new();
    for line in HEADER.lines() {
        if line.is_empty() {
            continue;
        }

        use std::convert::TryFrom;
        let intrinsic: MsaIntrinsic =
            TryFrom::try_from(line).expect(&format!("failed to parse line: \"{}\"", line));
        assert!(!intrinsics.contains_key(&intrinsic.id));
        intrinsics.insert(intrinsic.id.clone(), intrinsic);
    }

    let mut all_valid = true;
    for rust in FUNCTIONS {
        // Skip some intrinsics that aren't part of MSA
        match rust.name {
            "break_" => continue,
            _ => {}
        }
        let mips = match intrinsics.get(rust.name) {
            Some(i) => i,
            None => {
                eprintln!(
                    "missing mips definition for {:?} in {}",
                    rust.name, rust.file
                );
                all_valid = false;
                continue;
            }
        };

        if let Err(e) = matches(rust, mips) {
            println!("failed to verify `{}`", rust.name);
            println!("  * {}", e);
            all_valid = false;
        }
    }
    assert!(all_valid);
}

fn matches(rust: &Function, mips: &MsaIntrinsic) -> Result<(), String> {
    macro_rules! bail {
        ($($t:tt)*) => (return Err(format!($($t)*)))
    }

    if rust.ret.is_none() && mips.ret_ty != MsaTy::Void {
        bail!("mismatched return value")
    }

    if rust.arguments.len() != mips.arg_tys.len() {
        bail!("mismatched argument lengths");
    }

    let mut nconst = 0;
    for (i, (rust_arg, mips_arg)) in rust.arguments.iter().zip(mips.arg_tys.iter()).enumerate() {
        match mips_arg {
            MsaTy::v16i8 if **rust_arg == v16i8 => (),
            MsaTy::v8i16 if **rust_arg == v8i16 => (),
            MsaTy::v4i32 if **rust_arg == v4i32 => (),
            MsaTy::v2i64 if **rust_arg == v2i64 => (),
            MsaTy::v16u8 if **rust_arg == v16u8 => (),
            MsaTy::v8u16 if **rust_arg == v8u16 => (),
            MsaTy::v4u32 if **rust_arg == v4u32 => (),
            MsaTy::v2u64 if **rust_arg == v2u64 => (),
            MsaTy::v4f32 if **rust_arg == v4f32 => (),
            MsaTy::v2f64 if **rust_arg == v2f64 => (),
            MsaTy::imm0_1
            | MsaTy::imm0_3
            | MsaTy::imm0_7
            | MsaTy::imm0_15
            | MsaTy::imm0_31
            | MsaTy::imm0_63
            | MsaTy::imm0_255
            | MsaTy::imm_n16_15
            | MsaTy::imm_n512_511
            | MsaTy::imm_n1024_1022
            | MsaTy::imm_n2048_2044
            | MsaTy::imm_n4096_4088
                if **rust_arg == I32 =>
            {
                ()
            }
            MsaTy::i32 if **rust_arg == I32 => (),
            MsaTy::i64 if **rust_arg == I64 => (),
            MsaTy::u32 if **rust_arg == U32 => (),
            MsaTy::u64 if **rust_arg == U64 => (),
            MsaTy::VoidPtr if **rust_arg == Type::Ptr(&U8) => (),
            m => bail!(
                "mismatched argument \"{}\"= \"{:?}\" != \"{:?}\"",
                i,
                m,
                *rust_arg
            ),
        }

        let is_const = match mips_arg {
            MsaTy::imm0_1
            | MsaTy::imm0_3
            | MsaTy::imm0_7
            | MsaTy::imm0_15
            | MsaTy::imm0_31
            | MsaTy::imm0_63
            | MsaTy::imm0_255
            | MsaTy::imm_n16_15
            | MsaTy::imm_n512_511
            | MsaTy::imm_n1024_1022
            | MsaTy::imm_n2048_2044
            | MsaTy::imm_n4096_4088 => true,
            _ => false,
        };
        if is_const {
            nconst += 1;
            if !rust.required_const.contains(&i) {
                bail!("argument const mismatch");
            }
        }
    }

    if nconst != rust.required_const.len() {
        bail!("wrong number of const arguments");
    }

    if rust.target_feature != Some("msa") {
        bail!("wrong target_feature");
    }

    if !rust.instrs.is_empty() {
        // Normalize slightly to get rid of assembler differences
        let actual = rust.instrs[0].replace(".", "_");
        let expected = mips.instruction.replace(".", "_");
        if actual != expected {
            bail!("wrong instruction: \"{}\" != \"{}\"", rust.instrs[0], mips.instruction);
        }
    } else {
        bail!(
            "missing assert_instr for \"{}\" (should be \"{}\")",
            mips.id, mips.instruction);
    }

    Ok(())
}
<commit_msg>Formatting<commit_after>//! Verification of MIPS MSA intrinsics
#![feature(try_trait)]
#![allow(bad_style, unused)]

// This file is obtained from
// https://gcc.gnu.org/onlinedocs//gcc/MIPS-SIMD-Architecture-Built-in-Functions.html
static HEADER: &str = include_str!("../mips-msa.h");

stdsimd_verify::mips_functions!(static FUNCTIONS);

struct Function {
    name: &'static str,
    arguments: &'static [&'static Type],
    ret: Option<&'static Type>,
    target_feature: Option<&'static str>,
    instrs: &'static [&'static str],
    file: &'static str,
    required_const: &'static [usize],
}

static F16: Type = Type::PrimFloat(16);
static F32: Type = Type::PrimFloat(32);
static F64: Type = Type::PrimFloat(64);
static I8: Type = Type::PrimSigned(8);
static I16: Type = Type::PrimSigned(16);
static I32: Type = Type::PrimSigned(32);
static I64: Type = Type::PrimSigned(64);
static U8: Type = Type::PrimUnsigned(8);
static U16: Type = Type::PrimUnsigned(16);
static U32: Type = Type::PrimUnsigned(32);
static U64: Type = Type::PrimUnsigned(64);
static NEVER: Type = Type::Never;
static TUPLE: Type = Type::Tuple;
static v16i8: Type = Type::I(8, 16, 1);
static v8i16: Type = Type::I(16, 8, 1);
static v4i32: Type = Type::I(32, 4, 1);
static v2i64: Type = Type::I(64, 2, 1);
static v16u8: Type = Type::U(8, 16, 1);
static v8u16: Type = Type::U(16, 8, 1);
static v4u32: Type = Type::U(32, 4, 1);
static v2u64: Type = Type::U(64, 2, 1);
static v8f16: Type = Type::F(16, 8, 1);
static v4f32: Type = Type::F(32, 4, 1);
static v2f64: Type = Type::F(64, 2, 1);

#[derive(Debug, Copy, Clone, PartialEq)]
enum Type {
    PrimFloat(u8),
    PrimSigned(u8),
    PrimUnsigned(u8),
    PrimPoly(u8),
    MutPtr(&'static Type),
    ConstPtr(&'static Type),
    Ptr(&'static Type),
    Tuple,
    I(u8, u8, u8),
    U(u8, u8, u8),
    P(u8, u8, u8),
    F(u8, u8, u8),
    Never,
}

#[derive(Copy, Clone, Debug, PartialEq)]
#[allow(non_camel_case_types)]
enum MsaTy {
    v16i8,
    v8i16,
    v4i32,
    v2i64,
    v16u8,
    v8u16,
    v4u32,
    v2u64,
    v8f16,
    v4f32,
    v2f64,
    imm0_1,
    imm0_3,
    imm0_7,
    imm0_15,
    imm0_31,
    imm0_63,
    imm0_255,
    imm_n16_15,
    imm_n512_511,
    imm_n1024_1022,
    imm_n2048_2044,
    imm_n4096_4088,
    i32,
    u32,
    i64,
    u64,
    Void,
    VoidPtr,
}

impl<'a> From<&'a str> for MsaTy {
    fn from(s: &'a str) -> MsaTy {
        match s {
            "v16i8" => MsaTy::v16i8,
            "v8i16" => MsaTy::v8i16,
            "v4i32" => MsaTy::v4i32,
            "v2i64" => MsaTy::v2i64,
            "v16u8" => MsaTy::v16u8,
            "v8u16" => MsaTy::v8u16,
            "v4u32" => MsaTy::v4u32,
            "v2u64" => MsaTy::v2u64,
            "v8f16" => MsaTy::v8f16,
            "v4f32" => MsaTy::v4f32,
            "v2f64" => MsaTy::v2f64,
            "imm0_1" => MsaTy::imm0_1,
            "imm0_3" => MsaTy::imm0_3,
            "imm0_7" => MsaTy::imm0_7,
            "imm0_15" => MsaTy::imm0_15,
            "imm0_31" => MsaTy::imm0_31,
            "imm0_63" => MsaTy::imm0_63,
            "imm0_255" => MsaTy::imm0_255,
            "imm_n16_15" => MsaTy::imm_n16_15,
            "imm_n512_511" => MsaTy::imm_n512_511,
            "imm_n1024_1022" => MsaTy::imm_n1024_1022,
            "imm_n2048_2044" => MsaTy::imm_n2048_2044,
            "imm_n4096_4088" => MsaTy::imm_n4096_4088,
            "i32" => MsaTy::i32,
            "u32" => MsaTy::u32,
            "i64" => MsaTy::i64,
            "u64" => MsaTy::u64,
            "void" => MsaTy::Void,
            "void *" => MsaTy::VoidPtr,
            v => panic!("unknown ty: \"{}\"", v),
        }
    }
}

#[derive(Debug, Clone)]
struct MsaIntrinsic {
    id: String,
    arg_tys: Vec<MsaTy>,
    ret_ty: MsaTy,
    instruction: String,
}

impl std::convert::TryFrom<&'static str> for MsaIntrinsic {
    // The intrinsics are just C function declarations of the form:
    // $ret_ty __builtin_${fn_id}($($arg_ty),*);
    type Error = std::option::NoneError;
    fn try_from(line: &'static str) -> Result<Self, Self::Error> {
        let first_whitespace = line.find(char::is_whitespace)?;
        let ret_ty = &line[0..first_whitespace];
        let ret_ty = MsaTy::from(ret_ty);

        let first_parentheses = line.find('(')?;
        assert!(first_parentheses > first_whitespace);
        let id = &line[first_whitespace + 1..first_parentheses].trim();
        assert!(id.starts_with("__builtin"));
        let mut id_str = "_".to_string();
        id_str += &id[9..];
        let id = id_str;

        let mut arg_tys = Vec::new();

        let last_parentheses = line.find(')')?;
        for arg in (&line[first_parentheses + 1..last_parentheses]).split(',') {
            let arg = arg.trim();
            arg_tys.push(MsaTy::from(arg));
        }

        // The instruction is the intrinsic name without the __msa_ prefix.
        let instruction = &id[6..];
        let mut instruction = instruction.to_string();
        // With all underscores but the first one replaced with a `.`
        if let Some(first_underscore) = instruction.find('_') {
            let postfix = instruction[first_underscore + 1..].replace('_', ".");
            instruction = instruction[0..=first_underscore].to_string();
            instruction += &postfix;
        }

        Ok(MsaIntrinsic {
            id,
            ret_ty,
            arg_tys,
            instruction,
        })
    }
}

#[test]
fn verify_all_signatures() {
    // Parse the C intrinsic header file:
    let mut intrinsics = std::collections::HashMap::<String, MsaIntrinsic>::new();
    for line in HEADER.lines() {
        if line.is_empty() {
            continue;
        }

        use std::convert::TryFrom;
        let intrinsic: MsaIntrinsic =
            TryFrom::try_from(line).expect(&format!("failed to parse line: \"{}\"", line));
        assert!(!intrinsics.contains_key(&intrinsic.id));
        intrinsics.insert(intrinsic.id.clone(), intrinsic);
    }

    let mut all_valid = true;
    for rust in FUNCTIONS {
        // Skip some intrinsics that aren't part of MSA
        match rust.name {
            "break_" => continue,
            _ => {}
        }
        let mips = match intrinsics.get(rust.name) {
            Some(i) => i,
            None => {
                eprintln!(
                    "missing mips definition for {:?} in {}",
                    rust.name, rust.file
                );
                all_valid = false;
                continue;
            }
        };

        if let Err(e) = matches(rust, mips) {
            println!("failed to verify `{}`", rust.name);
            println!("  * {}", e);
            all_valid = false;
        }
    }
    assert!(all_valid);
}

fn matches(rust: &Function, mips: &MsaIntrinsic) -> Result<(), String> {
    macro_rules! bail {
        ($($t:tt)*) => (return Err(format!($($t)*)))
    }

    if rust.ret.is_none() && mips.ret_ty != MsaTy::Void {
        bail!("mismatched return value")
    }

    if rust.arguments.len() != mips.arg_tys.len() {
        bail!("mismatched argument lengths");
    }

    let mut nconst = 0;
    for (i, (rust_arg, mips_arg)) in rust.arguments.iter().zip(mips.arg_tys.iter()).enumerate() {
        match mips_arg {
            MsaTy::v16i8 if **rust_arg == v16i8 => (),
            MsaTy::v8i16 if **rust_arg == v8i16 => (),
            MsaTy::v4i32 if **rust_arg == v4i32 => (),
            MsaTy::v2i64 if **rust_arg == v2i64 => (),
            MsaTy::v16u8 if **rust_arg == v16u8 => (),
            MsaTy::v8u16 if **rust_arg == v8u16 => (),
            MsaTy::v4u32 if **rust_arg == v4u32 => (),
            MsaTy::v2u64 if **rust_arg == v2u64 => (),
            MsaTy::v4f32 if **rust_arg == v4f32 => (),
            MsaTy::v2f64 if **rust_arg == v2f64 => (),
            MsaTy::imm0_1
            | MsaTy::imm0_3
            | MsaTy::imm0_7
            | MsaTy::imm0_15
            | MsaTy::imm0_31
            | MsaTy::imm0_63
            | MsaTy::imm0_255
            | MsaTy::imm_n16_15
            | MsaTy::imm_n512_511
            | MsaTy::imm_n1024_1022
            | MsaTy::imm_n2048_2044
            | MsaTy::imm_n4096_4088
                if **rust_arg == I32 =>
            {
                ()
            }
            MsaTy::i32 if **rust_arg == I32 => (),
            MsaTy::i64 if **rust_arg == I64 => (),
            MsaTy::u32 if **rust_arg == U32 => (),
            MsaTy::u64 if **rust_arg == U64 => (),
            MsaTy::VoidPtr if **rust_arg == Type::Ptr(&U8) => (),
            m => bail!(
                "mismatched argument \"{}\"= \"{:?}\" != \"{:?}\"",
                i,
                m,
                *rust_arg
            ),
        }

        let is_const = match mips_arg {
            MsaTy::imm0_1
            | MsaTy::imm0_3
            | MsaTy::imm0_7
            | MsaTy::imm0_15
            | MsaTy::imm0_31
            | MsaTy::imm0_63
            | MsaTy::imm0_255
            | MsaTy::imm_n16_15
            | MsaTy::imm_n512_511
            | MsaTy::imm_n1024_1022
            | MsaTy::imm_n2048_2044
            | MsaTy::imm_n4096_4088 => true,
            _ => false,
        };
        if is_const {
            nconst += 1;
            if !rust.required_const.contains(&i) {
                bail!("argument const mismatch");
            }
        }
    }

    if nconst != rust.required_const.len() {
        bail!("wrong number of const arguments");
    }

    if rust.target_feature != Some("msa") {
        bail!("wrong target_feature");
    }

    if !rust.instrs.is_empty() {
        // Normalize slightly to get rid of assembler differences
        let actual = rust.instrs[0].replace(".", "_");
        let expected = mips.instruction.replace(".", "_");
        if actual != expected {
            bail!(
                "wrong instruction: \"{}\" != \"{}\"",
                rust.instrs[0],
                mips.instruction
            );
        }
    } else {
        bail!(
            "missing assert_instr for \"{}\" (should be \"{}\")",
            mips.id,
            mips.instruction
        );
    }

    Ok(())
}
<|endoftext|>


<commit_before>extern crate rand;

pub struct Repeatedly<F> {
    pub f: F,
}

impl<T, F: FnMut() -> T> Iterator for Repeatedly<F> {
    type Item = T;

    fn next(&mut self) -> Option<T> {
        let mut f = &mut self.f;
        Some(f())
    }
}

/// takes a function `f` of no args, presumably with side effects, and
/// returns an iterator that endlessly calls `f` and yields the return value
pub fn repeatedly<T, F: FnMut() -> T>(f: F) -> Repeatedly<F> {
    Repeatedly { f: f }
}

#[test]
fn test_repeatedly() {
    use self::rand::{StdRng, SeedableRng};
    use helpers::random01;

    assert_eq!(vec![3, 3, 3, 3],
               repeatedly(|| 3).take(4).collect::<Vec<usize>>());

    let mut i = 0;
    assert_eq!(vec![1, 2, 3, 4],
               repeatedly(move || {
                   i += 1;
                   i
               })
                   .take(4)
                   .collect::<Vec<usize>>());

    let seed: &[_] = &[1, 2, 3, 4];
    let mut rng: StdRng = SeedableRng::from_seed(seed);
    assert_eq!(vec![0.5162139860908154, 0.13628294371987984, 0.21635575241586105, 0.10006169673911681],
               repeatedly(move || random01(&mut rng))
                   .take(4)
                   .collect::<Vec<f64>>());
}
<commit_msg>remove unnecessary `move` from closures<commit_after>extern crate rand;

pub struct Repeatedly<F> {
    pub f: F,
}

impl<T, F: FnMut() -> T> Iterator for Repeatedly<F> {
    type Item = T;

    fn next(&mut self) -> Option<T> {
        let mut f = &mut self.f;
        Some(f())
    }
}

/// takes a function `f` of no args, presumably with side effects, and
/// returns an iterator that endlessly calls `f` and yields the return value
pub fn repeatedly<T, F: FnMut() -> T>(f: F) -> Repeatedly<F> {
    Repeatedly { f: f }
}

#[test]
fn test_repeatedly() {
    use self::rand::{StdRng, SeedableRng};
    use helpers::random01;

    assert_eq!(vec![3, 3, 3, 3],
               repeatedly(|| 3).take(4).collect::<Vec<usize>>());

    let mut i = 0;
    assert_eq!(vec![1, 2, 3, 4],
               repeatedly(|| {
                   i += 1;
                   i
               })
                   .take(4)
                   .collect::<Vec<usize>>());
    assert_eq!(i, 4);

    let seed: &[_] = &[1, 2, 3, 4];
    let mut rng: StdRng = SeedableRng::from_seed(seed);
    assert_eq!(vec![0.5162139860908154, 0.13628294371987984, 0.21635575241586105, 0.10006169673911681],
               repeatedly(|| random01(&mut rng))
                   .take(4)
                   .collect::<Vec<f64>>());
}
<|endoftext|>