<commit_before>extern crate dns_parser;
extern crate resolv_conf;

use std::time::{Duration, Instant};
use std::io::{self, Read};
use std::fs::File;
use std::net::{IpAddr, SocketAddr, UdpSocket};
use std::collections::HashMap;

pub struct Resolver {
    server: SocketAddr,
    cache: HashMap<String, CacheEntry>,
    queries: HashMap<u16, Query>,
    responses: HashMap<String, Vec<usize>>,
    buf: Vec<u8>,
    qnum: u16,
}

struct Query {
    resps: u8,
    domain: String,
    deadline: Instant,
}

struct CacheEntry {
    ip: IpAddr,
    deadline: Instant,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Response {
    pub id: usize,
    pub result: Result<IpAddr, Error>,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Error {
    NotFound,
    Timeout,
}

impl Resolver {
    pub fn new(server: SocketAddr) -> Resolver {
        let mut buf = Vec::with_capacity(512);
        unsafe {
            buf.set_len(512);
        }
        Resolver {
            server,
            queries: HashMap::new(),
            responses: HashMap::new(),
            cache: HashMap::new(),
            buf,
            qnum: 0,
        }
    }

    pub fn purge(&mut self) {
        self.cache.clear();
    }

    pub fn from_resolv() -> io::Result<Resolver> {
        let mut buf = Vec::with_capacity(512);
        unsafe {
            buf.set_len(512);
        }

        let mut conf = Vec::with_capacity(4096);
        let mut f = File::open("/etc/resolv.conf")?;
        f.read_to_end(&mut conf)?;
        let cfg = resolv_conf::Config::parse(&conf).map_err(|e| {
            io::Error::new(io::ErrorKind::Other, format!("invalid resolv.conf: {}", e))
        })?;
        let server = match cfg.nameservers.first().cloned() {
            Some(resolv_conf::ScopedIp::V4(ip)) => SocketAddr::new(IpAddr::V4(ip), 53),
            Some(resolv_conf::ScopedIp::V6(ip, _)) => SocketAddr::new(IpAddr::V6(ip), 53),
            None => {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    "No nameservers found in resolv.conf!",
                ))
            }
        };

        Ok(Resolver {
            server,
            queries: HashMap::new(),
            responses: HashMap::new(),
            cache: HashMap::new(),
            buf,
            qnum: 0,
        })
    }

    pub fn query(
        &mut self,
        sock: &mut UdpSocket,
        id: usize,
        domain: &str,
    ) -> io::Result<Option<IpAddr>> {
        if let Some(entry) = self.cache.get(domain) {
            return Ok(Some(entry.ip));
        }
        if let Ok(entry) = domain.parse() {
            return Ok(Some(entry));
        }
        if self.responses.get(domain).is_none() {
            let qn = self.qnum;
            self.qnum.wrapping_add(1);
            let mut query = dns_parser::Builder::new_query(qn, true);
            query.add_question(domain, dns_parser::QueryType::A, dns_parser::QueryClass::IN);
            let packet = query.build().unwrap_or_else(|d| d);
            sock.send_to(&packet, self.server)?;

            let mut query = dns_parser::Builder::new_query(qn, true);
            query.add_question(
                domain,
                dns_parser::QueryType::AAAA,
                dns_parser::QueryClass::IN,
            );
            let packet = query.build().unwrap_or_else(|d| d);
            sock.send_to(&packet, self.server)?;

            self.responses.insert(domain.to_string(), vec![]);
            self.queries.insert(
                qn,
                Query {
                    resps: 0,
                    domain: domain.to_string(),
                    deadline: Instant::now() + Duration::from_secs(3),
                },
            );
        }
        self.responses.get_mut(domain).unwrap().push(id);
        Ok(None)
    }

    pub fn read<F: FnMut(Response)>(&mut self, sock: &mut UdpSocket, mut f: F) -> io::Result<()> {
        'process: loop {
            match sock.recv_from(&mut self.buf) {
                Ok((amnt, _)) => {
                    match dns_parser::Packet::parse(&self.buf[..amnt]) {
                        Ok(packet) => {
                            let qn = packet.header.id;
                            let (domain, mut resps, deadline) = match self.queries.remove(&qn) {
                                Some(q) => (q.domain, q.resps, q.deadline),
                                // This could happen if timeout is exceeeded but we eventually get
                                // a response, ignore.
                                None => continue,
                            };
                            let now = Instant::now();
                            for answer in packet.answers {
                                match answer.data {
                                    dns_parser::RRData::A(addr) => {
                                        for id in self.responses.remove(&domain).unwrap() {
                                            f(Response {
                                                id,
                                                result: Ok(addr.into()),
                                            });
                                        }
                                        self.cache.insert(
                                            domain,
                                            CacheEntry {
                                                ip: addr.into(),
                                                deadline: now
                                                    + Duration::from_secs(answer.ttl.into()),
                                            },
                                        );
                                        continue 'process;
                                    }
                                    dns_parser::RRData::AAAA(addr) => {
                                        for id in self.responses.remove(&domain).unwrap() {
                                            f(Response {
                                                id,
                                                result: Ok(addr.into()),
                                            });
                                        }
                                        self.cache.insert(
                                            domain,
                                            CacheEntry {
                                                ip: addr.into(),
                                                deadline: now
                                                    + Duration::from_secs(answer.ttl.into()),
                                            },
                                        );
                                        continue 'process;
                                    }
                                    _ => continue,
                                }
                            }
                            if resps == 0 {
                                self.queries.insert(
                                    qn,
                                    Query {
                                        domain,
                                        resps: resps + 1,
                                        deadline,
                                    },
                                );
                            } else {
                                for id in self.responses.remove(&domain).unwrap() {
                                    f(Response {
                                        id,
                                        result: Err(Error::NotFound),
                                    });
                                }
                            }
                        }
                        Err(e) => {
                            return Err(io::Error::new(
                                io::ErrorKind::Other,
                                format!("malformed dns packet received: {}", e),
                            ));
                        }
                    }
                }
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(()),
                Err(e) => return Err(e),
            }
        }
    }

    pub fn tick<F: FnMut(Response)>(&mut self, mut f: F) {
        let now = Instant::now();
        let responses = &mut self.responses;
        self.queries.retain(|_, query| {
            if now > query.deadline {
                for id in responses.remove(&query.domain).unwrap() {
                    f(Response {
                        id,
                        result: Err(Error::Timeout),
                    });
                }
                false
            } else {
                true
            }
        });
        self.cache.retain(|_, entry| now < entry.deadline);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_google() {
        let mut resolver = Resolver::new("8.8.8.8:53".parse().unwrap());
        let mut sock = UdpSocket::bind("0.0.0.0:0").unwrap();
        sock.set_nonblocking(true).unwrap();

        assert_eq!(resolver.query(&mut sock, 0, "google.com").unwrap(), None);
        assert_eq!(resolver.query(&mut sock, 1, "google.com").unwrap(), None);
        assert_eq!(resolver.responses.get("google.com").unwrap().len(), 2);
        std::thread::sleep(Duration::from_millis(100));
        resolver.tick(|_| panic!("timeout should not have occured yet!"));
        let mut count = 0;
        resolver
            .read(&mut sock, |resp| {
                count += 1;
                assert!(resp.result.is_ok());
            })
            .unwrap();
        assert_eq!(count, 2);

        assert!(
            resolver
                .query(&mut sock, 0, "google.com")
                .unwrap()
                .is_some()
        );

        resolver
            .query(&mut sock, 0, "thiswebsiteshouldexit12589t69.com")
            .unwrap();
        std::thread::sleep(Duration::from_millis(200));
        let mut processed = false;
        resolver
            .read(&mut sock, |resp| {
                processed = true;
                assert_eq!(resp.result, Err(Error::NotFound))
            })
            .unwrap();
        #[cfg(not(target_os = "macos"))]
        assert!(processed);
    }
}
<commit_msg>Ensure DNS query IDs increment properly<commit_after>extern crate dns_parser;
extern crate resolv_conf;

use std::time::{Duration, Instant};
use std::io::{self, Read};
use std::fs::File;
use std::net::{IpAddr, SocketAddr, UdpSocket};
use std::collections::HashMap;

pub struct Resolver {
    server: SocketAddr,
    cache: HashMap<String, CacheEntry>,
    queries: HashMap<u16, Query>,
    responses: HashMap<String, Vec<usize>>,
    buf: Vec<u8>,
    qnum: u16,
}

struct Query {
    resps: u8,
    domain: String,
    deadline: Instant,
}

struct CacheEntry {
    ip: IpAddr,
    deadline: Instant,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Response {
    pub id: usize,
    pub result: Result<IpAddr, Error>,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Error {
    NotFound,
    Timeout,
}

impl Resolver {
    pub fn new(server: SocketAddr) -> Resolver {
        let mut buf = Vec::with_capacity(512);
        unsafe {
            buf.set_len(512);
        }
        Resolver {
            server,
            queries: HashMap::new(),
            responses: HashMap::new(),
            cache: HashMap::new(),
            buf,
            qnum: 0,
        }
    }

    pub fn purge(&mut self) {
        self.cache.clear();
    }

    pub fn from_resolv() -> io::Result<Resolver> {
        let mut buf = Vec::with_capacity(512);
        unsafe {
            buf.set_len(512);
        }

        let mut conf = Vec::with_capacity(4096);
        let mut f = File::open("/etc/resolv.conf")?;
        f.read_to_end(&mut conf)?;
        let cfg = resolv_conf::Config::parse(&conf).map_err(|e| {
            io::Error::new(io::ErrorKind::Other, format!("invalid resolv.conf: {}", e))
        })?;
        let server = match cfg.nameservers.first().cloned() {
            Some(resolv_conf::ScopedIp::V4(ip)) => SocketAddr::new(IpAddr::V4(ip), 53),
            Some(resolv_conf::ScopedIp::V6(ip, _)) => SocketAddr::new(IpAddr::V6(ip), 53),
            None => {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    "No nameservers found in resolv.conf!",
                ))
            }
        };

        Ok(Resolver {
            server,
            queries: HashMap::new(),
            responses: HashMap::new(),
            cache: HashMap::new(),
            buf,
            qnum: 0,
        })
    }

    pub fn query(
        &mut self,
        sock: &mut UdpSocket,
        id: usize,
        domain: &str,
    ) -> io::Result<Option<IpAddr>> {
        if let Some(entry) = self.cache.get(domain) {
            return Ok(Some(entry.ip));
        }
        if let Ok(entry) = domain.parse() {
            return Ok(Some(entry));
        }
        if self.responses.get(domain).is_none() {
            let qn = self.qnum;
            self.qnum = self.qnum.wrapping_add(1);
            let mut query = dns_parser::Builder::new_query(qn, true);
            query.add_question(domain, dns_parser::QueryType::A, dns_parser::QueryClass::IN);
            let packet = query.build().unwrap_or_else(|d| d);
            sock.send_to(&packet, self.server)?;

            let mut query = dns_parser::Builder::new_query(qn, true);
            query.add_question(
                domain,
                dns_parser::QueryType::AAAA,
                dns_parser::QueryClass::IN,
            );
            let packet = query.build().unwrap_or_else(|d| d);
            sock.send_to(&packet, self.server)?;

            self.responses.insert(domain.to_string(), vec![]);
            self.queries.insert(
                qn,
                Query {
                    resps: 0,
                    domain: domain.to_string(),
                    deadline: Instant::now() + Duration::from_secs(3),
                },
            );
        }
        self.responses.get_mut(domain).unwrap().push(id);
        Ok(None)
    }

    pub fn read<F: FnMut(Response)>(&mut self, sock: &mut UdpSocket, mut f: F) -> io::Result<()> {
        'process: loop {
            match sock.recv_from(&mut self.buf) {
                Ok((amnt, _)) => {
                    match dns_parser::Packet::parse(&self.buf[..amnt]) {
                        Ok(packet) => {
                            let qn = packet.header.id;
                            let (domain, mut resps, deadline) = match self.queries.remove(&qn) {
                                Some(q) => (q.domain, q.resps, q.deadline),
                                // This could happen if timeout is exceeeded but we eventually get
                                // a response, ignore.
                                None => continue,
                            };
                            let now = Instant::now();
                            for answer in packet.answers {
                                match answer.data {
                                    dns_parser::RRData::A(addr) => {
                                        for id in self.responses.remove(&domain).unwrap() {
                                            f(Response {
                                                id,
                                                result: Ok(addr.into()),
                                            });
                                        }
                                        self.cache.insert(
                                            domain,
                                            CacheEntry {
                                                ip: addr.into(),
                                                deadline: now
                                                    + Duration::from_secs(answer.ttl.into()),
                                            },
                                        );
                                        continue 'process;
                                    }
                                    dns_parser::RRData::AAAA(addr) => {
                                        for id in self.responses.remove(&domain).unwrap() {
                                            f(Response {
                                                id,
                                                result: Ok(addr.into()),
                                            });
                                        }
                                        self.cache.insert(
                                            domain,
                                            CacheEntry {
                                                ip: addr.into(),
                                                deadline: now
                                                    + Duration::from_secs(answer.ttl.into()),
                                            },
                                        );
                                        continue 'process;
                                    }
                                    _ => continue,
                                }
                            }
                            if resps == 0 {
                                self.queries.insert(
                                    qn,
                                    Query {
                                        domain,
                                        resps: resps + 1,
                                        deadline,
                                    },
                                );
                            } else {
                                for id in self.responses.remove(&domain).unwrap() {
                                    f(Response {
                                        id,
                                        result: Err(Error::NotFound),
                                    });
                                }
                            }
                        }
                        Err(e) => {
                            return Err(io::Error::new(
                                io::ErrorKind::Other,
                                format!("malformed dns packet received: {}", e),
                            ));
                        }
                    }
                }
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(()),
                Err(e) => return Err(e),
            }
        }
    }

    pub fn tick<F: FnMut(Response)>(&mut self, mut f: F) {
        let now = Instant::now();
        let responses = &mut self.responses;
        self.queries.retain(|_, query| {
            if now > query.deadline {
                for id in responses.remove(&query.domain).unwrap() {
                    f(Response {
                        id,
                        result: Err(Error::Timeout),
                    });
                }
                false
            } else {
                true
            }
        });
        self.cache.retain(|_, entry| now < entry.deadline);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_google() {
        let mut resolver = Resolver::new("8.8.8.8:53".parse().unwrap());
        let mut sock = UdpSocket::bind("0.0.0.0:0").unwrap();
        sock.set_nonblocking(true).unwrap();

        assert_eq!(resolver.query(&mut sock, 0, "google.com").unwrap(), None);
        assert_eq!(resolver.query(&mut sock, 1, "google.com").unwrap(), None);
        assert_eq!(resolver.responses.get("google.com").unwrap().len(), 2);
        std::thread::sleep(Duration::from_millis(100));
        resolver.tick(|_| panic!("timeout should not have occured yet!"));
        let mut count = 0;
        resolver
            .read(&mut sock, |resp| {
                count += 1;
                assert!(resp.result.is_ok());
            })
            .unwrap();
        assert_eq!(count, 2);

        assert!(
            resolver
                .query(&mut sock, 0, "google.com")
                .unwrap()
                .is_some()
        );

        resolver
            .query(&mut sock, 0, "thiswebsiteshouldexit12589t69.com")
            .unwrap();
        std::thread::sleep(Duration::from_millis(200));
        let mut processed = false;
        resolver
            .read(&mut sock, |resp| {
                processed = true;
                assert_eq!(resp.result, Err(Error::NotFound))
            })
            .unwrap();
        #[cfg(not(target_os = "macos"))]
        assert!(processed);
    }
}
<|endoftext|>


<commit_before>// Copyright 2019 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#![no_main]

use sys_util::{GuestAddress, GuestMemory, SharedMemory};

use std::fs::File;
use std::io::Write;
use std::panic;
use std::process;
use std::slice;

fn make_elf_bin(elf_bytes: &[u8]) -> File {
    let mut shm = SharedMemory::new(None).expect("failed to create shared memory");
    shm.set_size(elf_bytes.len() as u64)
        .expect("failed to set shared memory size");
    shm.write_all(elf_bytes)
        .expect("failed to write elf to shared memoy");
    shm.into()
}

#[export_name = "LLVMFuzzerTestOneInput"]
pub fn test_one_input(data: *const u8, size: usize) -> i32 {
    // We cannot unwind past ffi boundaries.
    panic::catch_unwind(|| {
        // Safe because the libfuzzer runtime will guarantee that `data` is at least
        // `size` bytes long and that it will be valid for the lifetime of this
        // function.
        let bytes = unsafe { slice::from_raw_parts(data, size) };
        let mut kimage = make_elf_bin(bytes);
        let mem = GuestMemory::new(&[(GuestAddress(0), bytes.len() as u64 + 0x1000)]).unwrap();
        let _ = kernel_loader::load_kernel(&mem, GuestAddress(0), &mut kimage);
    })
    .err()
    .map(|_| process::abort());

    0
}
<commit_msg>fuzz: Comply with new guest memory requirement for page alignment<commit_after>// Copyright 2019 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#![no_main]

use sys_util::{round_up_to_page_size, GuestAddress, GuestMemory, SharedMemory};

use std::fs::File;
use std::io::Write;
use std::panic;
use std::process;
use std::slice;

fn make_elf_bin(elf_bytes: &[u8]) -> File {
    let mut shm = SharedMemory::new(None).expect("failed to create shared memory");
    shm.set_size(elf_bytes.len() as u64)
        .expect("failed to set shared memory size");
    shm.write_all(elf_bytes)
        .expect("failed to write elf to shared memoy");
    shm.into()
}

#[export_name = "LLVMFuzzerTestOneInput"]
pub fn test_one_input(data: *const u8, size: usize) -> i32 {
    // We cannot unwind past ffi boundaries.
    panic::catch_unwind(|| {
        // Safe because the libfuzzer runtime will guarantee that `data` is at least
        // `size` bytes long and that it will be valid for the lifetime of this
        // function.
        let bytes = unsafe { slice::from_raw_parts(data, size) };
        let mut kimage = make_elf_bin(bytes);
        // `GuestMemory` only accepts page aligned segments.
        let len = round_up_to_page_size(bytes.len()) as u64;
        let mem = GuestMemory::new(&[(GuestAddress(0), len)]).unwrap();
        let _ = kernel_loader::load_kernel(&mem, GuestAddress(0), &mut kimage);
    })
    .err()
    .map(|_| process::abort());

    0
}
<|endoftext|>


<commit_before>//! An example of how to build multi-key transactions on top of
//! sled's single-key atomicity guarantees.
//!
//! The basic idea is to hash all of the keys, and acquire locks
//! for each one in a big array of RwLock's. If we fail to acquire
//! a lock in a non-blocking way, we try to acquire a global serializing
//! mutex, then take out blocking locks on the required keys. The
//! multiple locks are to avoid deadlock. Maybe it's just really buggy!
//! The global mutex does not prevent other thgets from claiming sharded
//! mutexes in a non-blocking way, so it shouldn't get hit all that often.
//! But this isn't measured, so maybe it happens all the time!
//! Computers, right?

#![allow(dead_code)]

extern crate sled;

mod crc16;

use std::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};

use sled::{Config, DbResult, Error, Tree};

macro_rules! rep_no_copy {
    ($e:expr; $n:expr) => {
        {
            let mut v = Vec::with_capacity($n);
            for _ in 0..$n {
                v.push($e);
            }
            v
        }
    };
}

type TxId = u64;
type Key = Vec<u8>;
type Value = Vec<u8>;
type PredicateFn = fn(&Key, &Option<Value>) -> bool;

#[derive(Debug, PartialEq)]
enum TxRet {
    Committed(Vec<(Key, Option<Value>)>),
    PredicateFailure,
}

unsafe impl Send for TxRet {}

struct TxDb {
    locks: Vec<RwLock<()>>,
    tree: Tree,
    blocker: Mutex<()>,
}

unsafe impl Send for TxDb {}
unsafe impl Sync for TxDb {}

struct Tx<'a> {
    predicates: Vec<Predicate>,
    sets: Vec<Write>,
    gets: Vec<Read>,
    deletes: Vec<Delete>,
    db: &'a TxDb,
}

unsafe impl<'a> Send for Tx<'a> {}

struct Predicate(Key, Box<PredicateFn>);
struct Write(Key, Value);
struct Read(Key);
struct Delete(Key);

fn split_txid_from_v(v: Value) -> DbResult<(TxId, Value), ()> {
    if v.len() < 8 {
        return Err(Error::ReportableBug("get bad txid".to_owned()));
    }
    let mut first_8 = [0; 8];
    first_8.copy_from_slice(&*v);
    let txid = unsafe { std::mem::transmute(first_8) };
    Ok((txid, v.into_iter().skip(8).collect()))
}

impl TxDb {
    fn start(config: Config) -> DbResult<TxDb, ()> {
        Ok(TxDb {
            // pluggable
            tree: Tree::start(config)?,
            blocker: Mutex::new(()),
            // 65k locks! webscale!!!
            locks: rep_no_copy!(RwLock::new(()); 1 << 16),
        })
    }

    fn tx<'a>(&'a self) -> Tx<'a> {
        Tx {
            predicates: vec![],
            gets: vec![],
            sets: vec![],
            deletes: vec![],
            db: &self,
        }
    }

    fn lock_keys(
        &self,
        get_hashes: &[u16],
        set_hashes: &[u16],
    ) -> (Vec<RwLockReadGuard<()>>, Vec<RwLockWriteGuard<()>>) {
        let mut get_locks = vec![];
        for get_idx in get_hashes {
            if let Ok(get_guard) = self.locks[*get_idx as usize].try_read() {
                get_locks.push(get_guard);
            } else {
                drop(get_locks);
                return self.pessimistic_lock_keys(get_hashes, set_hashes);
            }
        }

        let mut set_locks = vec![];
        for set_idx in set_hashes {
            if let Ok(set_guard) = self.locks[*set_idx as usize].try_write() {
                set_locks.push(set_guard);
            } else {
                drop(set_locks);
                let (_r, w) = self.pessimistic_lock_keys(&[], set_hashes);
                set_locks = w;
                break;
            }
        }

        (get_locks, set_locks)
    }

    fn pessimistic_lock_keys(
        &self,
        get_hashes: &[u16],
        set_hashes: &[u16],
    ) -> (Vec<RwLockReadGuard<()>>, Vec<RwLockWriteGuard<()>>) {
        let _global_mu = self.blocker.lock().unwrap();

        let mut get_locks = vec![];
        for get_idx in get_hashes {
            let get_guard = self.locks[*get_idx as usize].read().unwrap();
            get_locks.push(get_guard);
        }

        let mut set_locks = vec![];
        for set_idx in set_hashes {
            let set_guard = self.locks[*set_idx as usize].write().unwrap();
            set_locks.push(set_guard);
        }

        (get_locks, set_locks)
    }
}

impl<'a> Tx<'a> {
    fn predicate(&mut self, k: Key, p: PredicateFn) {
        self.predicates.push(Predicate(k, Box::new(p)));
    }

    fn get(&mut self, k: Key) {
        self.gets.push(Read(k));
    }

    fn set(&mut self, k: Key, v: Value) {
        self.sets.push(Write(k, v));
    }

    fn del(&mut self, k: Key) {
        self.deletes.push(Delete(k));
    }

    fn execute(self) -> DbResult<TxRet, ()> {
        // claim locks
        let mut get_keys = vec![];
        let mut set_keys = vec![];

        for &Predicate(ref k, _) in &self.predicates {
            get_keys.push(k);
        }

        for &Read(ref k) in &self.gets {
            get_keys.push(k);
        }

        for &Write(ref k, _) in &self.sets {
            set_keys.push(k);
        }

        for &Delete(ref k) in &self.deletes {
            set_keys.push(k);
        }

        let mut set_hashes: Vec<u16> =
            set_keys.into_iter().map(|k| crc16::crc16(k)).collect();
        set_hashes.sort();
        set_hashes.dedup();

        let mut get_hashes: Vec<u16> = get_keys
            .into_iter()
            .map(|k| crc16::crc16(k))
            .filter(|hash| !set_hashes.contains(hash))
            .collect();
        get_hashes.sort();
        get_hashes.dedup();

        let _locks = self.db.lock_keys(&*get_hashes, &*set_hashes);

        // perform predicate matches
        for &Predicate(ref k, ref p) in &self.predicates {
            let current = self.db.tree.get(k)?;
            if !p(&k, &current) {
                return Ok(TxRet::PredicateFailure);
            }
        }

        // perform sets
        for &Write(ref k, ref v) in &self.sets {
            self.db.tree.set(k.clone(), v.clone())?;
        }

        // perform deletes
        for &Delete(ref k) in &self.deletes {
            self.db.tree.del(k)?;
        }

        // perform gets
        let mut ret_gets = vec![];
        for &Read(ref k) in &self.gets {
            ret_gets.push((k.clone(), self.db.tree.get(k)?));
        }

        Ok(TxRet::Committed(ret_gets))
    }
}

#[test]
fn it_works() {
    let conf = sled::ConfigBuilder::new().temporary(true).build();
    let txdb = TxDb::start(conf).unwrap();

    let mut tx = txdb.tx();
    tx.set(b"cats".to_vec(), b"meow".to_vec());
    tx.set(b"dogs".to_vec(), b"woof".to_vec());
    assert_eq!(tx.execute(), Ok(TxRet::Committed(vec![])));

    let mut tx = txdb.tx();
    tx.predicate(b"cats".to_vec(), |_k, v| *v == Some(b"meow".to_vec()));
    tx.predicate(b"dogs".to_vec(), |_k, v| *v == Some(b"woof".to_vec()));
    tx.set(b"cats".to_vec(), b"woof".to_vec());
    tx.set(b"dogs".to_vec(), b"meow".to_vec());
    tx.get(b"dogs".to_vec());
    assert_eq!(
        tx.execute(),
        Ok(TxRet::Committed(vec![(b"dogs".to_vec(), Some(b"meow".to_vec()))]))
    );

    let mut tx = txdb.tx();
    tx.predicate(b"cats".to_vec(), |_k, v| *v == Some(b"meow".to_vec()));
    assert_eq!(tx.execute(), Ok(TxRet::PredicateFailure));
}
<commit_msg>add small safety notice about incorrectness of implementation<commit_after>//! An example of how to build multi-key transactions on top of
//! sled's single-key atomicity guarantees.
//!
//! The basic idea is to hash all of the keys, and acquire locks
//! for each one in a big array of RwLock's. If we fail to acquire
//! a lock in a non-blocking way, we try to acquire a global serializing
//! mutex, then take out blocking locks on the required keys. The
//! multiple locks are to avoid deadlock. Maybe it's just really buggy!
//! The global mutex does not prevent other thgets from claiming sharded
//! mutexes in a non-blocking way, so it shouldn't get hit all that often.
//! But this isn't measured, so maybe it happens all the time!
//! Computers, right?
//!
//! This is half-sketched-out and doesn't currently provide atomicity
//! during crashes LOLOLOLLOLOLOLOL

#![allow(dead_code)]

extern crate sled;

mod crc16;

use std::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};

use sled::{Config, DbResult, Error, Tree};

macro_rules! rep_no_copy {
    ($e:expr; $n:expr) => {
        {
            let mut v = Vec::with_capacity($n);
            for _ in 0..$n {
                v.push($e);
            }
            v
        }
    };
}

type TxId = u64;
type Key = Vec<u8>;
type Value = Vec<u8>;
type PredicateFn = fn(&Key, &Option<Value>) -> bool;

#[derive(Debug, PartialEq)]
enum TxRet {
    Committed(Vec<(Key, Option<Value>)>),
    PredicateFailure,
}

unsafe impl Send for TxRet {}

struct TxDb {
    locks: Vec<RwLock<()>>,
    tree: Tree,
    blocker: Mutex<()>,
}

unsafe impl Send for TxDb {}
unsafe impl Sync for TxDb {}

struct Tx<'a> {
    predicates: Vec<Predicate>,
    sets: Vec<Write>,
    gets: Vec<Read>,
    deletes: Vec<Delete>,
    db: &'a TxDb,
}

unsafe impl<'a> Send for Tx<'a> {}

struct Predicate(Key, Box<PredicateFn>);
struct Write(Key, Value);
struct Read(Key);
struct Delete(Key);

fn split_txid_from_v(v: Value) -> DbResult<(TxId, Value), ()> {
    if v.len() < 8 {
        return Err(Error::ReportableBug("get bad txid".to_owned()));
    }
    let mut first_8 = [0; 8];
    first_8.copy_from_slice(&*v);
    let txid = unsafe { std::mem::transmute(first_8) };
    Ok((txid, v.into_iter().skip(8).collect()))
}

impl TxDb {
    fn start(config: Config) -> DbResult<TxDb, ()> {
        Ok(TxDb {
            // pluggable
            tree: Tree::start(config)?,
            blocker: Mutex::new(()),
            // 65k locks! webscale!!!
            locks: rep_no_copy!(RwLock::new(()); 1 << 16),
        })
    }

    fn tx<'a>(&'a self) -> Tx<'a> {
        Tx {
            predicates: vec![],
            gets: vec![],
            sets: vec![],
            deletes: vec![],
            db: &self,
        }
    }

    fn lock_keys(
        &self,
        get_hashes: &[u16],
        set_hashes: &[u16],
    ) -> (Vec<RwLockReadGuard<()>>, Vec<RwLockWriteGuard<()>>) {
        let mut get_locks = vec![];
        for get_idx in get_hashes {
            if let Ok(get_guard) = self.locks[*get_idx as usize].try_read() {
                get_locks.push(get_guard);
            } else {
                drop(get_locks);
                return self.pessimistic_lock_keys(get_hashes, set_hashes);
            }
        }

        let mut set_locks = vec![];
        for set_idx in set_hashes {
            if let Ok(set_guard) = self.locks[*set_idx as usize].try_write() {
                set_locks.push(set_guard);
            } else {
                drop(set_locks);
                let (_r, w) = self.pessimistic_lock_keys(&[], set_hashes);
                set_locks = w;
                break;
            }
        }

        (get_locks, set_locks)
    }

    fn pessimistic_lock_keys(
        &self,
        get_hashes: &[u16],
        set_hashes: &[u16],
    ) -> (Vec<RwLockReadGuard<()>>, Vec<RwLockWriteGuard<()>>) {
        let _global_mu = self.blocker.lock().unwrap();

        let mut get_locks = vec![];
        for get_idx in get_hashes {
            let get_guard = self.locks[*get_idx as usize].read().unwrap();
            get_locks.push(get_guard);
        }

        let mut set_locks = vec![];
        for set_idx in set_hashes {
            let set_guard = self.locks[*set_idx as usize].write().unwrap();
            set_locks.push(set_guard);
        }

        (get_locks, set_locks)
    }
}

impl<'a> Tx<'a> {
    fn predicate(&mut self, k: Key, p: PredicateFn) {
        self.predicates.push(Predicate(k, Box::new(p)));
    }

    fn get(&mut self, k: Key) {
        self.gets.push(Read(k));
    }

    fn set(&mut self, k: Key, v: Value) {
        self.sets.push(Write(k, v));
    }

    fn del(&mut self, k: Key) {
        self.deletes.push(Delete(k));
    }

    fn execute(self) -> DbResult<TxRet, ()> {
        // claim locks
        let mut get_keys = vec![];
        let mut set_keys = vec![];

        for &Predicate(ref k, _) in &self.predicates {
            get_keys.push(k);
        }

        for &Read(ref k) in &self.gets {
            get_keys.push(k);
        }

        for &Write(ref k, _) in &self.sets {
            set_keys.push(k);
        }

        for &Delete(ref k) in &self.deletes {
            set_keys.push(k);
        }

        let mut set_hashes: Vec<u16> =
            set_keys.into_iter().map(|k| crc16::crc16(k)).collect();
        set_hashes.sort();
        set_hashes.dedup();

        let mut get_hashes: Vec<u16> = get_keys
            .into_iter()
            .map(|k| crc16::crc16(k))
            .filter(|hash| !set_hashes.contains(hash))
            .collect();
        get_hashes.sort();
        get_hashes.dedup();

        let _locks = self.db.lock_keys(&*get_hashes, &*set_hashes);

        // perform predicate matches
        for &Predicate(ref k, ref p) in &self.predicates {
            let current = self.db.tree.get(k)?;
            if !p(&k, &current) {
                return Ok(TxRet::PredicateFailure);
            }
        }

        // perform sets
        for &Write(ref k, ref v) in &self.sets {
            self.db.tree.set(k.clone(), v.clone())?;
        }

        // perform deletes
        for &Delete(ref k) in &self.deletes {
            self.db.tree.del(k)?;
        }

        // perform gets
        let mut ret_gets = vec![];
        for &Read(ref k) in &self.gets {
            ret_gets.push((k.clone(), self.db.tree.get(k)?));
        }

        Ok(TxRet::Committed(ret_gets))
    }
}

#[test]
fn it_works() {
    let conf = sled::ConfigBuilder::new().temporary(true).build();
    let txdb = TxDb::start(conf).unwrap();

    let mut tx = txdb.tx();
    tx.set(b"cats".to_vec(), b"meow".to_vec());
    tx.set(b"dogs".to_vec(), b"woof".to_vec());
    assert_eq!(tx.execute(), Ok(TxRet::Committed(vec![])));

    let mut tx = txdb.tx();
    tx.predicate(b"cats".to_vec(), |_k, v| *v == Some(b"meow".to_vec()));
    tx.predicate(b"dogs".to_vec(), |_k, v| *v == Some(b"woof".to_vec()));
    tx.set(b"cats".to_vec(), b"woof".to_vec());
    tx.set(b"dogs".to_vec(), b"meow".to_vec());
    tx.get(b"dogs".to_vec());
    assert_eq!(
        tx.execute(),
        Ok(TxRet::Committed(vec![(b"dogs".to_vec(), Some(b"meow".to_vec()))]))
    );

    let mut tx = txdb.tx();
    tx.predicate(b"cats".to_vec(), |_k, v| *v == Some(b"meow".to_vec()));
    assert_eq!(tx.execute(), Ok(TxRet::PredicateFailure));
}
<|endoftext|>


<commit_before>/// Minimal example for getting sdl2 and wgpu working together with raw-window-handle.
extern crate pollster;
extern crate sdl2;
extern crate wgpu;

use std::borrow::Cow;
use wgpu::{SurfaceError, SurfaceTexture};

use sdl2::event::{Event, WindowEvent};
use sdl2::keyboard::Keycode;

fn main() -> Result<(), String> {
    // Show logs from wgpu
    env_logger::init();

    let sdl_context = sdl2::init()?;
    let video_subsystem = sdl_context.video()?;
    let window = video_subsystem
        .window("Raw Window Handle Example", 800, 600)
        .position_centered()
        .resizable()
        .build()
        .map_err(|e| e.to_string())?;
    let (width, height) = window.size();

    let instance = wgpu::Instance::new(wgpu::Backends::PRIMARY);
    let surface = unsafe { instance.create_surface(&window) };
    let adapter_opt = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
        power_preference: wgpu::PowerPreference::HighPerformance,
        force_fallback_adapter: false,
        compatible_surface: Some(&surface),
    }));
    let adapter = match adapter_opt {
        Some(a) => a,
        None => return Err(String::from("No adapter found")),
    };

    let (device, queue) = match pollster::block_on(adapter.request_device(
        &wgpu::DeviceDescriptor {
            limits: wgpu::Limits::default(),
            label: Some("device"),
            features: wgpu::Features::empty(),
        },
        None,
    )) {
        Ok(a) => a,
        Err(e) => return Err(e.to_string()),
    };

    let shader = device.create_shader_module(&wgpu::ShaderModuleDescriptor {
        label: Some("shader"),
        source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(include_str!("shader.wgsl"))),
    });

    let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
        entries: &[],
        label: Some("bind_group_layout"),
    });
    let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
        layout: &bind_group_layout,
        entries: &[],
        label: Some("bind_group"),
    });
    let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
        bind_group_layouts: &[&bind_group_layout],
        label: None,
        push_constant_ranges: &[],
    });

    let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
        layout: Some(&pipeline_layout),
        vertex: wgpu::VertexState {
            buffers: &[],
            module: &shader,
            entry_point: "vs_main",
        },
        fragment: Some(wgpu::FragmentState {
            targets: &[wgpu::ColorTargetState {
                format: wgpu::TextureFormat::Bgra8UnormSrgb,
                blend: None,
                write_mask: wgpu::ColorWrites::ALL,
            }],
            module: &shader,
            entry_point: "fs_main",
        }),
        primitive: wgpu::PrimitiveState {
            topology: wgpu::PrimitiveTopology::TriangleList,
            strip_index_format: None,
            front_face: wgpu::FrontFace::Ccw,
            cull_mode: Some(wgpu::Face::Front),
            unclipped_depth: false,
            polygon_mode: wgpu::PolygonMode::Fill,
            conservative: false,
        },
        depth_stencil: None,
        label: None,
        multisample: wgpu::MultisampleState {
            count: 1,
            mask: !0,
            alpha_to_coverage_enabled: false,
        },
        multiview: None
    });

    let mut config = wgpu::SurfaceConfiguration {
        usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
        format: surface.get_preferred_format(&adapter).unwrap(),
        width,
        height,
        present_mode: wgpu::PresentMode::Mailbox,
    };
    surface.configure(&device, &config);

    let mut event_pump = sdl_context.event_pump()?;
    'running: loop {
        for event in event_pump.poll_iter() {
            match event {
                Event::Window {
                    window_id,
                    win_event: WindowEvent::SizeChanged(width, height),
                    ..
                } if window_id == window.id() => {
                    config.width = width as u32;
                    config.height = height as u32;
                    surface.configure(&device, &config);
                }
                Event::Quit { .. }
                | Event::KeyDown {
                    keycode: Some(Keycode::Escape),
                    ..
                } => {
                    break 'running;
                }
                e => {
                    dbg!(e);
                }
            }
        }

        let mut frame = match surface.get_current_texture() {
                Ok(frame) => {frame},
                Err(err) => {
                    let reason = match (err) {
                        SurfaceError::Timeout => { "Timeout" }
                        SurfaceError::Outdated => { "Outdated" }
                        SurfaceError::Lost => { "Lost" }
                        SurfaceError::OutOfMemory => { "OutOfMemory" }
                    };
                    panic!("Failed to get current surface texture! Reason: {}", reason)
                }
            };

        let output = frame
            .texture
            .create_view(&wgpu::TextureViewDescriptor::default());
        let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
            label: Some("command_encoder"),
        });

        {
            let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
                color_attachments: &[wgpu::RenderPassColorAttachment {
                    view: &output,
                    resolve_target: None,
                    ops: wgpu::Operations {
                        load: wgpu::LoadOp::Clear(wgpu::Color::GREEN),
                        store: true,
                    },
                }],
                depth_stencil_attachment: None,
                label: None,
            });
            rpass.set_pipeline(&render_pipeline);
            rpass.set_bind_group(0, &bind_group, &[]);
            rpass.draw(0..3, 0..1);
        }
        queue.submit([encoder.finish()]);
        frame.present();
    }

    Ok(())
}
<commit_msg>cargo fmt<commit_after>/// Minimal example for getting sdl2 and wgpu working together with raw-window-handle.
extern crate pollster;
extern crate sdl2;
extern crate wgpu;

use std::borrow::Cow;
use wgpu::{SurfaceError, SurfaceTexture};

use sdl2::event::{Event, WindowEvent};
use sdl2::keyboard::Keycode;

fn main() -> Result<(), String> {
    // Show logs from wgpu
    env_logger::init();

    let sdl_context = sdl2::init()?;
    let video_subsystem = sdl_context.video()?;
    let window = video_subsystem
        .window("Raw Window Handle Example", 800, 600)
        .position_centered()
        .resizable()
        .build()
        .map_err(|e| e.to_string())?;
    let (width, height) = window.size();

    let instance = wgpu::Instance::new(wgpu::Backends::PRIMARY);
    let surface = unsafe { instance.create_surface(&window) };
    let adapter_opt = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
        power_preference: wgpu::PowerPreference::HighPerformance,
        force_fallback_adapter: false,
        compatible_surface: Some(&surface),
    }));
    let adapter = match adapter_opt {
        Some(a) => a,
        None => return Err(String::from("No adapter found")),
    };

    let (device, queue) = match pollster::block_on(adapter.request_device(
        &wgpu::DeviceDescriptor {
            limits: wgpu::Limits::default(),
            label: Some("device"),
            features: wgpu::Features::empty(),
        },
        None,
    )) {
        Ok(a) => a,
        Err(e) => return Err(e.to_string()),
    };

    let shader = device.create_shader_module(&wgpu::ShaderModuleDescriptor {
        label: Some("shader"),
        source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(include_str!("shader.wgsl"))),
    });

    let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
        entries: &[],
        label: Some("bind_group_layout"),
    });
    let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
        layout: &bind_group_layout,
        entries: &[],
        label: Some("bind_group"),
    });
    let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
        bind_group_layouts: &[&bind_group_layout],
        label: None,
        push_constant_ranges: &[],
    });

    let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
        layout: Some(&pipeline_layout),
        vertex: wgpu::VertexState {
            buffers: &[],
            module: &shader,
            entry_point: "vs_main",
        },
        fragment: Some(wgpu::FragmentState {
            targets: &[wgpu::ColorTargetState {
                format: wgpu::TextureFormat::Bgra8UnormSrgb,
                blend: None,
                write_mask: wgpu::ColorWrites::ALL,
            }],
            module: &shader,
            entry_point: "fs_main",
        }),
        primitive: wgpu::PrimitiveState {
            topology: wgpu::PrimitiveTopology::TriangleList,
            strip_index_format: None,
            front_face: wgpu::FrontFace::Ccw,
            cull_mode: Some(wgpu::Face::Front),
            unclipped_depth: false,
            polygon_mode: wgpu::PolygonMode::Fill,
            conservative: false,
        },
        depth_stencil: None,
        label: None,
        multisample: wgpu::MultisampleState {
            count: 1,
            mask: !0,
            alpha_to_coverage_enabled: false,
        },
        multiview: None,
    });

    let mut config = wgpu::SurfaceConfiguration {
        usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
        format: surface.get_preferred_format(&adapter).unwrap(),
        width,
        height,
        present_mode: wgpu::PresentMode::Mailbox,
    };
    surface.configure(&device, &config);

    let mut event_pump = sdl_context.event_pump()?;
    'running: loop {
        for event in event_pump.poll_iter() {
            match event {
                Event::Window {
                    window_id,
                    win_event: WindowEvent::SizeChanged(width, height),
                    ..
                } if window_id == window.id() => {
                    config.width = width as u32;
                    config.height = height as u32;
                    surface.configure(&device, &config);
                }
                Event::Quit { .. }
                | Event::KeyDown {
                    keycode: Some(Keycode::Escape),
                    ..
                } => {
                    break 'running;
                }
                e => {
                    dbg!(e);
                }
            }
        }

        let mut frame = match surface.get_current_texture() {
            Ok(frame) => frame,
            Err(err) => {
                let reason = match (err) {
                    SurfaceError::Timeout => "Timeout",
                    SurfaceError::Outdated => "Outdated",
                    SurfaceError::Lost => "Lost",
                    SurfaceError::OutOfMemory => "OutOfMemory",
                };
                panic!("Failed to get current surface texture! Reason: {}", reason)
            }
        };

        let output = frame
            .texture
            .create_view(&wgpu::TextureViewDescriptor::default());
        let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
            label: Some("command_encoder"),
        });

        {
            let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
                color_attachments: &[wgpu::RenderPassColorAttachment {
                    view: &output,
                    resolve_target: None,
                    ops: wgpu::Operations {
                        load: wgpu::LoadOp::Clear(wgpu::Color::GREEN),
                        store: true,
                    },
                }],
                depth_stencil_attachment: None,
                label: None,
            });
            rpass.set_pipeline(&render_pipeline);
            rpass.set_bind_group(0, &bind_group, &[]);
            rpass.draw(0..3, 0..1);
        }
        queue.submit([encoder.finish()]);
        frame.present();
    }

    Ok(())
}
<|endoftext|>


<commit_before>use super::item::ParseItem;
use crate::grammars::{Production, SyntaxGrammar};
use std::collections::HashMap;

pub(crate) struct InlinedProductionMap {
    pub inlined_productions: Vec<Production>,
    item_map: HashMap<ParseItem, Vec<u32>>,
}

impl InlinedProductionMap {
    pub fn new(grammar: &SyntaxGrammar) -> Self {
        let mut result = Self {
            inlined_productions: Vec::new(),
            item_map: HashMap::new(),
        };

        let mut items_to_process = Vec::new();
        for (variable_index, variable) in grammar.variables.iter().enumerate() {
            for production_index in 0..variable.productions.len() {
                items_to_process.push(ParseItem::Normal {
                    variable_index: variable_index as u32,
                    production_index: production_index as u32,
                    step_index: 0,
                });
                while !items_to_process.is_empty() {
                    let mut i = 0;
                    while i < items_to_process.len() {
                        let item = &items_to_process[i];
                        if let Some(step) = item.step(grammar, &result) {
                            if grammar.variables_to_inline.contains(&step.symbol) {
                                let inlined_items = result
                                    .inline(*item, grammar)
                                    .into_iter()
                                    .map(|production_index| ParseItem::Inlined {
                                        variable_index: item.variable_index(),
                                        production_index: *production_index,
                                        step_index: item.step_index() as u32,
                                    })
                                    .collect::<Vec<_>>();
                                items_to_process.splice(i..i + 1, inlined_items);
                            } else {
                                items_to_process[i] = item.successor();
                                i += 1;
                            }
                        } else {
                            items_to_process.remove(i);
                        }
                    }
                }
            }
        }

        result
    }

    pub fn inlined_items<'a>(
        &'a self,
        item: ParseItem,
    ) -> Option<impl Iterator<Item = ParseItem> + 'a> {
        self.item_map.get(&item).map(|production_indices| {
            production_indices
                .iter()
                .cloned()
                .map(move |production_index| ParseItem::Inlined {
                    variable_index: item.variable_index(),
                    production_index,
                    step_index: item.step_index() as u32,
                })
        })
    }

    fn inline(&mut self, item: ParseItem, grammar: &SyntaxGrammar) -> &Vec<u32> {
        let step_index = item.step_index();
        let mut productions_to_add = grammar.variables
            [item.step(grammar, self).unwrap().symbol.index]
            .productions
            .clone();

        let mut i = 0;
        while i < productions_to_add.len() {
            if let Some(first_symbol) = productions_to_add[i].first_symbol() {
                if grammar.variables_to_inline.contains(&first_symbol) {
                    // Remove the production from the vector, replacing it with a placeholder.
                    let production = productions_to_add
                        .splice(i..i + 1, [Production::default()].iter().cloned())
                        .next()
                        .unwrap();

                    // Replace the placeholder with the inlined productions.
                    productions_to_add.splice(
                        i..i + 1,
                        grammar.variables[first_symbol.index]
                            .productions
                            .iter()
                            .map(|p| {
                                let mut p = p.clone();
                                p.steps.extend(production.steps[1..].iter().cloned());
                                p
                            }),
                    );
                    continue;
                }
            }
            i += 1;
        }

        let result = productions_to_add
            .into_iter()
            .map(|production_to_add| {
                let mut inlined_production = item.production(grammar, &self).clone();
                inlined_production.steps.splice(
                    step_index..step_index + 1,
                    production_to_add.steps.iter().cloned(),
                );
                self.inlined_productions
                    .iter()
                    .position(|p| *p == inlined_production)
                    .unwrap_or({
                        self.inlined_productions.push(inlined_production);
                        self.inlined_productions.len() - 1
                    }) as u32
            })
            .collect();

        self.item_map.entry(item).or_insert(result)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::grammars::{ProductionStep, SyntaxVariable, VariableType};
    use crate::rules::Symbol;

    #[test]
    fn test_basic_inlining() {
        let grammar = SyntaxGrammar {
            expected_conflicts: Vec::new(),
            extra_tokens: Vec::new(),
            external_tokens: Vec::new(),
            word_token: None,
            variables_to_inline: vec![Symbol::non_terminal(1)],
            variables: vec![
                SyntaxVariable {
                    name: "var0".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![
                            ProductionStep::new(Symbol::terminal(10)),
                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
                            ProductionStep::new(Symbol::terminal(11)),
                        ],
                    }],
                },
                SyntaxVariable {
                    name: "var1".to_string(),
                    kind: VariableType::Named,
                    productions: vec![
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![
                                ProductionStep::new(Symbol::terminal(12)),
                                ProductionStep::new(Symbol::terminal(13)),
                            ],
                        },
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![ProductionStep::new(Symbol::terminal(14))],
                        },
                    ],
                },
            ],
        };

        let inline_map = InlinedProductionMap::new(&grammar);

        // Nothing to inline at step 0.
        assert_eq!(
            display_items(
                inline_map.inlined_items(ParseItem::Normal {
                    variable_index: 0,
                    production_index: 0,
                    step_index: 0
                }),
                &grammar,
                &inline_map
            ),
            None
        );

        // Inlining variable 1 yields two productions.
        assert_eq!(
            display_items(
                inline_map.inlined_items(ParseItem::Normal {
                    variable_index: 0,
                    production_index: 0,
                    step_index: 1
                }),
                &grammar,
                &inline_map
            ),
            Some(vec![
                "terminal-10 • terminal-12 terminal-13 terminal-11".to_string(),
                "terminal-10 • terminal-14 terminal-11".to_string(),
            ])
        );
    }

    #[test]
    fn test_nested_inlining() {
        let grammar = SyntaxGrammar {
            variables: vec![
                SyntaxVariable {
                    name: "var0".to_string(),
                    kind: VariableType::Named,
                    productions: vec![
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![
                                ProductionStep::new(Symbol::terminal(10)),
                                ProductionStep::new(Symbol::non_terminal(1)), // inlined
                                ProductionStep::new(Symbol::terminal(11)),
                                ProductionStep::new(Symbol::non_terminal(2)), // inlined
                                ProductionStep::new(Symbol::terminal(12)),
                            ],
                        },
                    ],
                },
                SyntaxVariable {
                    name: "var1".to_string(),
                    kind: VariableType::Named,
                    productions: vec![
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![ProductionStep::new(Symbol::terminal(13))],
                        },
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![
                                ProductionStep::new(Symbol::non_terminal(3)), // inlined
                                ProductionStep::new(Symbol::terminal(14)),
                            ],
                        },
                    ],
                },
                SyntaxVariable {
                    name: "var2".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![ProductionStep::new(Symbol::terminal(15))],
                    }],
                },
                SyntaxVariable {
                    name: "var3".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![ProductionStep::new(Symbol::terminal(16))],
                    }],
                },
            ],
            variables_to_inline: vec![
                Symbol::non_terminal(1),
                Symbol::non_terminal(2),
                Symbol::non_terminal(3),
            ],
            expected_conflicts: Vec::new(),
            extra_tokens: Vec::new(),
            external_tokens: Vec::new(),
            word_token: None,
        };

        let inline_map = InlinedProductionMap::new(&grammar);

        let items = inline_map.inlined_items(ParseItem::Normal {
            variable_index: 0,
            production_index: 0,
            step_index: 1
        }).unwrap().collect::<Vec<_>>();

        assert_eq!(
            display_items(Some(items.iter().cloned()), &grammar, &inline_map),
            Some(vec![
                "terminal-10 • terminal-13 terminal-11 non-terminal-2 terminal-12".to_string(),
                "terminal-10 • terminal-16 terminal-14 terminal-11 non-terminal-2 terminal-12".to_string()
            ])
        );

        let item = items[0].successor().successor();
        assert_eq!(
            display_items(Some([item].iter().cloned()), &grammar, &inline_map),
            Some(vec![
                "terminal-10 terminal-13 terminal-11 • non-terminal-2 terminal-12".to_string(),
            ])
        );

        assert_eq!(
            display_items(inline_map.inlined_items(item), &grammar, &inline_map),
            Some(vec![
                "terminal-10 terminal-13 terminal-11 • terminal-15 terminal-12".to_string(),
            ])
        );
    }

    fn display_items(
        items: Option<impl Iterator<Item = ParseItem>>,
        grammar: &SyntaxGrammar,
        inline_map: &InlinedProductionMap,
    ) -> Option<Vec<String>> {
        items.map(|items| {
            items
                .map(|item| format!("{}", item.with(grammar, inline_map)))
                .collect()
        })
    }
}
<commit_msg>Handle precedence and aliases properly when inlining variables<commit_after>use super::item::ParseItem;
use crate::grammars::{Production, SyntaxGrammar};
use std::collections::HashMap;

pub(crate) struct InlinedProductionMap {
    pub inlined_productions: Vec<Production>,
    item_map: HashMap<ParseItem, Vec<u32>>,
}

impl InlinedProductionMap {
    pub fn new(grammar: &SyntaxGrammar) -> Self {
        let mut result = Self {
            inlined_productions: Vec::new(),
            item_map: HashMap::new(),
        };

        let mut items_to_process = Vec::new();
        for (variable_index, variable) in grammar.variables.iter().enumerate() {
            for production_index in 0..variable.productions.len() {
                items_to_process.push(ParseItem::Normal {
                    variable_index: variable_index as u32,
                    production_index: production_index as u32,
                    step_index: 0,
                });
                while !items_to_process.is_empty() {
                    let mut i = 0;
                    while i < items_to_process.len() {
                        let item = &items_to_process[i];
                        if let Some(step) = item.step(grammar, &result) {
                            if grammar.variables_to_inline.contains(&step.symbol) {
                                let inlined_items = result
                                    .inline(*item, grammar)
                                    .into_iter()
                                    .map(|production_index| ParseItem::Inlined {
                                        variable_index: item.variable_index(),
                                        production_index: *production_index,
                                        step_index: item.step_index() as u32,
                                    })
                                    .collect::<Vec<_>>();
                                items_to_process.splice(i..i + 1, inlined_items);
                            } else {
                                items_to_process[i] = item.successor();
                                i += 1;
                            }
                        } else {
                            items_to_process.remove(i);
                        }
                    }
                }
            }
        }

        result
    }

    pub fn inlined_items<'a>(
        &'a self,
        item: ParseItem,
    ) -> Option<impl Iterator<Item = ParseItem> + 'a> {
        self.item_map.get(&item).map(|production_indices| {
            production_indices
                .iter()
                .cloned()
                .map(move |production_index| ParseItem::Inlined {
                    variable_index: item.variable_index(),
                    production_index,
                    step_index: item.step_index() as u32,
                })
        })
    }

    fn inline(&mut self, item: ParseItem, grammar: &SyntaxGrammar) -> &Vec<u32> {
        let step_index = item.step_index();
        let mut productions_to_add = grammar.variables
            [item.step(grammar, self).unwrap().symbol.index]
            .productions
            .clone();

        let mut i = 0;
        while i < productions_to_add.len() {
            if let Some(first_symbol) = productions_to_add[i].first_symbol() {
                if grammar.variables_to_inline.contains(&first_symbol) {
                    // Remove the production from the vector, replacing it with a placeholder.
                    let production = productions_to_add
                        .splice(i..i + 1, [Production::default()].iter().cloned())
                        .next()
                        .unwrap();

                    // Replace the placeholder with the inlined productions.
                    productions_to_add.splice(
                        i..i + 1,
                        grammar.variables[first_symbol.index]
                            .productions
                            .iter()
                            .map(|p| {
                                let mut p = p.clone();
                                p.steps.extend(production.steps[1..].iter().cloned());
                                p
                            }),
                    );
                    continue;
                }
            }
            i += 1;
        }

        let result = productions_to_add
            .into_iter()
            .map(|production_to_add| {
                let mut inlined_production = item.production(grammar, &self).clone();
                let removed_step = inlined_production
                    .steps
                    .splice(
                        step_index..step_index + 1,
                        production_to_add.steps.iter().cloned(),
                    )
                    .next()
                    .unwrap();
                let inserted_steps = &mut inlined_production.steps
                    [step_index..step_index + production_to_add.steps.len()];
                if let Some(alias) = removed_step.alias {
                    for inserted_step in inserted_steps.iter_mut() {
                        inserted_step.alias = Some(alias.clone());
                    }
                }
                if let Some(last_inserted_step) = inserted_steps.last_mut() {
                    last_inserted_step.precedence = removed_step.precedence;
                    last_inserted_step.associativity = removed_step.associativity;
                }
                self.inlined_productions
                    .iter()
                    .position(|p| *p == inlined_production)
                    .unwrap_or({
                        self.inlined_productions.push(inlined_production);
                        self.inlined_productions.len() - 1
                    }) as u32
            })
            .collect();

        self.item_map.entry(item).or_insert(result)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::grammars::{LexicalGrammar, ProductionStep, SyntaxVariable, VariableType};
    use crate::rules::{Alias, Associativity, Symbol};
    use std::borrow::Borrow;

    #[test]
    fn test_basic_inlining() {
        let grammar = SyntaxGrammar {
            expected_conflicts: Vec::new(),
            extra_tokens: Vec::new(),
            external_tokens: Vec::new(),
            word_token: None,
            variables_to_inline: vec![Symbol::non_terminal(1)],
            variables: vec![
                SyntaxVariable {
                    name: "non-terminal-0".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![
                            ProductionStep::new(Symbol::terminal(10)),
                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
                            ProductionStep::new(Symbol::terminal(11)),
                        ],
                    }],
                },
                SyntaxVariable {
                    name: "non-terminal-1".to_string(),
                    kind: VariableType::Named,
                    productions: vec![
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![
                                ProductionStep::new(Symbol::terminal(12)),
                                ProductionStep::new(Symbol::terminal(13)),
                            ],
                        },
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![ProductionStep::new(Symbol::terminal(14))],
                        },
                    ],
                },
            ],
        };

        let inline_map = InlinedProductionMap::new(&grammar);

        // Nothing to inline at step 0.
        assert!(inline_map
            .inlined_items(ParseItem::Normal {
                variable_index: 0,
                production_index: 0,
                step_index: 0
            })
            .is_none());

        // Inlining variable 1 yields two productions.
        assert_eq!(
            display_items(
                inline_map
                    .inlined_items(ParseItem::Normal {
                        variable_index: 0,
                        production_index: 0,
                        step_index: 1
                    })
                    .unwrap(),
                &grammar,
                &inline_map
            ),
            vec![
                "non-terminal-0 → terminal-10 • terminal-12 terminal-13 terminal-11"
                    .to_string(),
                "non-terminal-0 → terminal-10 • terminal-14 terminal-11".to_string(),
            ]
        );
    }

    #[test]
    fn test_nested_inlining() {
        let grammar = SyntaxGrammar {
            variables: vec![
                SyntaxVariable {
                    name: "non-terminal-0".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![
                            ProductionStep::new(Symbol::terminal(10)),
                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
                            ProductionStep::new(Symbol::terminal(11)),
                            ProductionStep::new(Symbol::non_terminal(2)), // inlined
                            ProductionStep::new(Symbol::terminal(12)),
                        ],
                    }],
                },
                SyntaxVariable {
                    name: "non-terminal-1".to_string(),
                    kind: VariableType::Named,
                    productions: vec![
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![ProductionStep::new(Symbol::terminal(13))],
                        },
                        Production {
                            dynamic_precedence: 0,
                            steps: vec![
                                ProductionStep::new(Symbol::non_terminal(3)), // inlined
                                ProductionStep::new(Symbol::terminal(14)),
                            ],
                        },
                    ],
                },
                SyntaxVariable {
                    name: "non-terminal-2".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![ProductionStep::new(Symbol::terminal(15))],
                    }],
                },
                SyntaxVariable {
                    name: "non-terminal-3".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![ProductionStep::new(Symbol::terminal(16))],
                    }],
                },
            ],
            variables_to_inline: vec![
                Symbol::non_terminal(1),
                Symbol::non_terminal(2),
                Symbol::non_terminal(3),
            ],
            expected_conflicts: Vec::new(),
            extra_tokens: Vec::new(),
            external_tokens: Vec::new(),
            word_token: None,
        };

        let inline_map = InlinedProductionMap::new(&grammar);

        let items = inline_map
            .inlined_items(ParseItem::Normal {
                variable_index: 0,
                production_index: 0,
                step_index: 1,
            })
            .unwrap()
            .collect::<Vec<_>>();

        assert_eq!(
            display_items(&items, &grammar, &inline_map),
            vec![
                "non-terminal-0 → terminal-10 • terminal-13 terminal-11 non-terminal-2 terminal-12".to_string(),
                "non-terminal-0 → terminal-10 • terminal-16 terminal-14 terminal-11 non-terminal-2 terminal-12".to_string()
            ]
        );

        let item = items[0].successor().successor();
        assert_eq!(
            display_items(&[item], &grammar, &inline_map),
            vec![
                "non-terminal-0 → terminal-10 terminal-13 terminal-11 • non-terminal-2 terminal-12".to_string(),
            ]
        );

        assert_eq!(
            display_items(inline_map.inlined_items(item).unwrap(), &grammar, &inline_map),
            vec![
                "non-terminal-0 → terminal-10 terminal-13 terminal-11 • terminal-15 terminal-12".to_string(),
            ]
        );
    }

    #[test]
    fn test_inlining_with_precedence_and_alias() {
        let grammar = SyntaxGrammar {
            variables_to_inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
            variables: vec![
                SyntaxVariable {
                    name: "non-terminal-0".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![
                            ProductionStep::new(Symbol::non_terminal(1)) // inlined
                                .with_prec(1, Some(Associativity::Left)),
                            ProductionStep::new(Symbol::terminal(10)),
                            ProductionStep::new(Symbol::non_terminal(2)), // inlined
                        ],
                    }],
                },
                SyntaxVariable {
                    name: "non-terminal-1".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![
                            ProductionStep::new(Symbol::terminal(11))
                                .with_prec(2, None)
                                .with_alias("inner_alias", true),
                            ProductionStep::new(Symbol::terminal(12)).with_prec(3, None),
                        ],
                    }],
                },
                SyntaxVariable {
                    name: "non-terminal-2".to_string(),
                    kind: VariableType::Named,
                    productions: vec![Production {
                        dynamic_precedence: 0,
                        steps: vec![ProductionStep::new(Symbol::terminal(13))
                            .with_alias("outer_alias", true)],
                    }],
                },
            ],
            expected_conflicts: Vec::new(),
            extra_tokens: Vec::new(),
            external_tokens: Vec::new(),
            word_token: None,
        };

        let inline_map = InlinedProductionMap::new(&grammar);

        let items = inline_map
            .inlined_items(ParseItem::Normal {
                variable_index: 0,
                production_index: 0,
                step_index: 0,
            })
            .unwrap()
            .collect::<Vec<_>>();
        assert_eq!(
            display_items(&items, &grammar, &inline_map)[0],
            "non-terminal-0 → • terminal-11 terminal-12 terminal-10 non-terminal-2".to_string(),
        );

        // The first step in the inlined production retains its precedence and alias.
        let item = items[0].successor();
        assert_eq!(
            display_items(&[item], &grammar, &inline_map)[0],
            "non-terminal-0 → terminal-11 • terminal-12 terminal-10 non-terminal-2".to_string(),
        );
        assert_eq!(item.precedence(&grammar, &inline_map), 2);
        assert_eq!(
            items[0].step(&grammar, &inline_map).unwrap().alias,
            Some(Alias {
                value: "inner_alias".to_string(),
                is_named: true,
            })
        );

        // The final terminal of the inlined production inherits the precedence of
        // the inlined step.
        let item = item.successor();
        assert_eq!(
            display_items(&[item], &grammar, &inline_map)[0],
            "non-terminal-0 → terminal-11 terminal-12 • terminal-10 non-terminal-2".to_string(),
        );
        assert_eq!(item.precedence(&grammar, &inline_map), 1);

        let item = item.successor();
        assert_eq!(
            display_items(&[item], &grammar, &inline_map)[0],
            "non-terminal-0 → terminal-11 terminal-12 terminal-10 • non-terminal-2".to_string(),
        );

        // All steps of the inlined production inherit their alias from the
        // inlined step.
        let items = inline_map.inlined_items(item).unwrap().collect::<Vec<_>>();
        assert_eq!(
            display_items(&items, &grammar, &inline_map)[0],
            "non-terminal-0 → terminal-11 terminal-12 terminal-10 • terminal-13".to_string(),
        );
        assert_eq!(
            items[0].step(&grammar, &inline_map).unwrap().alias,
            Some(Alias {
                value: "outer_alias".to_string(),
                is_named: true,
            })
        )
    }

    fn display_items(
        items: impl IntoIterator<Item = impl Borrow<ParseItem>>,
        grammar: &SyntaxGrammar,
        inline_map: &InlinedProductionMap,
    ) -> Vec<String> {
        let lex = LexicalGrammar::default();
        items
            .into_iter()
            .map(|item| format!("{}", item.borrow().display_with(grammar, &lex, inline_map)))
            .collect()
    }
}
<|endoftext|>


<commit_before>use std::collections::{HashMap, HashSet};
use std::fs::{self, File};
use std::io::prelude::*;
use std::io;
use std::path::{Path, PathBuf};

use core::{Package, Manifest, SourceId, PackageId};
use util::{self, CargoResult, human, Config, ChainError};
use util::important_paths::find_project_manifest_exact;
use util::toml::{Layout, project_layout};

pub fn read_manifest(contents: &[u8], layout: Layout, source_id: &SourceId,
                     config: &Config)
                     -> CargoResult<(Manifest, Vec<PathBuf>)> {
    let root = layout.root.clone();
    util::toml::to_manifest(contents, source_id, layout, config).chain_error(|| {
        human(format!("failed to parse manifest at `{}`",
                      root.join("Cargo.toml").display()))
    })
}

pub fn read_package(path: &Path, source_id: &SourceId, config: &Config)
                    -> CargoResult<(Package, Vec<PathBuf>)> {
    trace!("read_package; path={}; source-id={}", path.display(), source_id);
    let mut file = try!(File::open(path));
    let mut data = Vec::new();
    try!(file.read_to_end(&mut data));

    let layout = project_layout(path.parent().unwrap());
    let (manifest, nested) =
        try!(read_manifest(&data, layout, source_id, config));

    Ok((Package::new(manifest, path), nested))
}

pub fn read_packages(path: &Path, source_id: &SourceId, config: &Config)
                     -> CargoResult<Vec<Package>> {
    let mut all_packages = HashMap::new();
    let mut visited = HashSet::<PathBuf>::new();

    trace!("looking for root package: {}, source_id={}", path.display(), source_id);

    try!(walk(path, &mut |dir| {
        trace!("looking for child package: {}", dir.display());

        // Don't recurse into git databases
        if dir.file_name().and_then(|s| s.to_str()) == Some(".git") {
            return Ok(false)
        }

        // Don't automatically discover packages across git submodules
        if dir != path && fs::metadata(&dir.join(".git")).is_ok() {
            return Ok(false)
        }

        // Don't ever look at target directories
        if dir.file_name().and_then(|s| s.to_str()) == Some("target") &&
           has_manifest(dir.parent().unwrap()) {
            return Ok(false)
        }

        if has_manifest(dir) {
            try!(read_nested_packages(dir, &mut all_packages, source_id, config,
                                      &mut visited));
        }
        Ok(true)
    }));

    if all_packages.is_empty() {
        Err(human(format!("Could not find Cargo.toml in `{}`", path.display())))
    } else {
        Ok(all_packages.into_iter().map(|(_, v)| v).collect())
    }
}

fn walk(path: &Path, callback: &mut FnMut(&Path) -> CargoResult<bool>)
        -> CargoResult<()> {
    if !try!(callback(path)) {
        trace!("not processing {}", path.display());
        return Ok(())
    }

    // Ignore any permission denied errors because temporary directories
    // can often have some weird permissions on them.
    let dirs = match fs::read_dir(path) {
        Ok(dirs) => dirs,
        Err(ref e) if e.kind() == io::ErrorKind::PermissionDenied => {
            return Ok(())
        }
        Err(e) => return Err(From::from(e)),
    };
    for dir in dirs {
        let dir = try!(dir);
        if try!(dir.file_type()).is_dir() {
            try!(walk(&dir.path(), callback));
        }
    }
    Ok(())
}

fn has_manifest(path: &Path) -> bool {
    find_project_manifest_exact(path, "Cargo.toml").is_ok()
}

fn read_nested_packages(path: &Path,
                        all_packages: &mut HashMap<PackageId, Package>,
                        source_id: &SourceId,
                        config: &Config,
                        visited: &mut HashSet<PathBuf>) -> CargoResult<()> {
    if !visited.insert(path.to_path_buf()) { return Ok(()) }

    let manifest = try!(find_project_manifest_exact(path, "Cargo.toml"));

    let (pkg, nested) = try!(read_package(&manifest, source_id, config));
    let pkg_id = pkg.package_id().clone();
    if !all_packages.contains_key(&pkg_id) {
        all_packages.insert(pkg_id, pkg);
    } else {
        warn!("skipping nested package `{}` found at `{}`",
              &pkg.name(), &path.to_string_lossy());
    }

    // Registry sources are not allowed to have `path=` dependencies because
    // they're all translated to actual registry dependencies.
    //
    // We normalize the path here ensure that we don't infinitely walk around
    // looking for crates. By normalizing we ensure that we visit this crate at
    // most once.
    //
    // TODO: filesystem/symlink implications?
    if !source_id.is_registry() {
        for p in nested.iter() {
            let path = util::normalize_path(&path.join(p));
            try!(read_nested_packages(&path, all_packages, source_id,
                                      config, visited));
        }
    }

    Ok(())
}
<commit_msg>Use info! instead of warn! for skipping message<commit_after>use std::collections::{HashMap, HashSet};
use std::fs::{self, File};
use std::io::prelude::*;
use std::io;
use std::path::{Path, PathBuf};

use core::{Package, Manifest, SourceId, PackageId};
use util::{self, CargoResult, human, Config, ChainError};
use util::important_paths::find_project_manifest_exact;
use util::toml::{Layout, project_layout};

pub fn read_manifest(contents: &[u8], layout: Layout, source_id: &SourceId,
                     config: &Config)
                     -> CargoResult<(Manifest, Vec<PathBuf>)> {
    let root = layout.root.clone();
    util::toml::to_manifest(contents, source_id, layout, config).chain_error(|| {
        human(format!("failed to parse manifest at `{}`",
                      root.join("Cargo.toml").display()))
    })
}

pub fn read_package(path: &Path, source_id: &SourceId, config: &Config)
                    -> CargoResult<(Package, Vec<PathBuf>)> {
    trace!("read_package; path={}; source-id={}", path.display(), source_id);
    let mut file = try!(File::open(path));
    let mut data = Vec::new();
    try!(file.read_to_end(&mut data));

    let layout = project_layout(path.parent().unwrap());
    let (manifest, nested) =
        try!(read_manifest(&data, layout, source_id, config));

    Ok((Package::new(manifest, path), nested))
}

pub fn read_packages(path: &Path, source_id: &SourceId, config: &Config)
                     -> CargoResult<Vec<Package>> {
    let mut all_packages = HashMap::new();
    let mut visited = HashSet::<PathBuf>::new();

    trace!("looking for root package: {}, source_id={}", path.display(), source_id);

    try!(walk(path, &mut |dir| {
        trace!("looking for child package: {}", dir.display());

        // Don't recurse into git databases
        if dir.file_name().and_then(|s| s.to_str()) == Some(".git") {
            return Ok(false)
        }

        // Don't automatically discover packages across git submodules
        if dir != path && fs::metadata(&dir.join(".git")).is_ok() {
            return Ok(false)
        }

        // Don't ever look at target directories
        if dir.file_name().and_then(|s| s.to_str()) == Some("target") &&
           has_manifest(dir.parent().unwrap()) {
            return Ok(false)
        }

        if has_manifest(dir) {
            try!(read_nested_packages(dir, &mut all_packages, source_id, config,
                                      &mut visited));
        }
        Ok(true)
    }));

    if all_packages.is_empty() {
        Err(human(format!("Could not find Cargo.toml in `{}`", path.display())))
    } else {
        Ok(all_packages.into_iter().map(|(_, v)| v).collect())
    }
}

fn walk(path: &Path, callback: &mut FnMut(&Path) -> CargoResult<bool>)
        -> CargoResult<()> {
    if !try!(callback(path)) {
        trace!("not processing {}", path.display());
        return Ok(())
    }

    // Ignore any permission denied errors because temporary directories
    // can often have some weird permissions on them.
    let dirs = match fs::read_dir(path) {
        Ok(dirs) => dirs,
        Err(ref e) if e.kind() == io::ErrorKind::PermissionDenied => {
            return Ok(())
        }
        Err(e) => return Err(From::from(e)),
    };
    for dir in dirs {
        let dir = try!(dir);
        if try!(dir.file_type()).is_dir() {
            try!(walk(&dir.path(), callback));
        }
    }
    Ok(())
}

fn has_manifest(path: &Path) -> bool {
    find_project_manifest_exact(path, "Cargo.toml").is_ok()
}

fn read_nested_packages(path: &Path,
                        all_packages: &mut HashMap<PackageId, Package>,
                        source_id: &SourceId,
                        config: &Config,
                        visited: &mut HashSet<PathBuf>) -> CargoResult<()> {
    if !visited.insert(path.to_path_buf()) { return Ok(()) }

    let manifest = try!(find_project_manifest_exact(path, "Cargo.toml"));

    let (pkg, nested) = try!(read_package(&manifest, source_id, config));
    let pkg_id = pkg.package_id().clone();
    if !all_packages.contains_key(&pkg_id) {
        all_packages.insert(pkg_id, pkg);
    } else {
        info!("skipping nested package `{}` found at `{}`",
              pkg.name(), path.to_string_lossy());
    }

    // Registry sources are not allowed to have `path=` dependencies because
    // they're all translated to actual registry dependencies.
    //
    // We normalize the path here ensure that we don't infinitely walk around
    // looking for crates. By normalizing we ensure that we visit this crate at
    // most once.
    //
    // TODO: filesystem/symlink implications?
    if !source_id.is_registry() {
        for p in nested.iter() {
            let path = util::normalize_path(&path.join(p));
            try!(read_nested_packages(&path, all_packages, source_id,
                                      config, visited));
        }
    }

    Ok(())
}
<|endoftext|>