<commit_before>use atomicwrites;
use common::make_io_err;
use directory::Directory;
use directory::error::{OpenWriteError, FileError, OpenDirectoryError};
use directory::ReadOnlySource;
use directory::shared_vec_slice::SharedVecSlice;
use directory::WritePtr;
use fst::raw::MmapReadOnly;
use memmap::{Mmap, Protection};
use std::collections::hash_map::Entry as HashMapEntry;
use std::collections::HashMap;
use std::mem;
use std::convert::From;
use std::fmt;
use std::fs;
use std::fs::File;
use std::fs::OpenOptions;
use std::io;
use std::io::{Seek, SeekFrom};
use std::io::BufWriter;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::result;
use std::sync::Arc;
use std::sync::RwLock;
use std::sync::Weak;
use tempdir::TempDir;

fn open_mmap(full_path: &PathBuf) -> result::Result<Option<Arc<Mmap>>, FileError> {
    let convert_file_error = |err: io::Error| {
        if err.kind() == io::ErrorKind::NotFound {
            FileError::FileDoesNotExist(full_path.clone())
        }
        else {
            FileError::IOError(err)
        }
    };
    let file = File::open(&full_path).map_err(convert_file_error)?;
    if try!(file.metadata()).len() == 0 {
        // if the file size is 0, it will not be possible
        // to mmap the file, so we return an anonymous mmap_cache
        // instead.
        return Ok(None)
    }
    Ok(Some(Arc::new(Mmap::open(&file, Protection::Read)?)))
}

#[derive(Default,Clone,Debug,RustcDecodable,RustcEncodable)]
pub struct CacheCounters {
    hit: usize,
    miss_empty: usize,
    miss_weak: usize,
}

#[derive(Clone,Debug,RustcDecodable,RustcEncodable)]
pub struct CacheInfo {
    pub counters: CacheCounters,
    pub mmapped: Vec<PathBuf>,
}

#[derive(Default)]
struct MmapCache {
    counters: CacheCounters,
    cache: HashMap<PathBuf, Weak<Mmap>>,
}

impl MmapCache {

   fn cleanup(&mut self) {
        let mut new_cache = HashMap::new();
        mem::swap(&mut new_cache, &mut self.cache);
        self.cache = new_cache
            .into_iter()
            .filter(|&(_, ref weak_ref)| weak_ref.upgrade().is_some())
            .collect();
    }

    fn get_info(&mut self) -> CacheInfo {
        self.cleanup();
        let paths: Vec<PathBuf> = self.cache.keys()
            .cloned()
            .collect();
        CacheInfo {
            counters: self.counters.clone(),
            mmapped: paths,
        }
    }

    fn get_mmap(&mut self, full_path: PathBuf) -> Result<Option<Arc<Mmap>>, FileError> {
        if self.cache.len() > 100 {
            self.cleanup();
        }
        Ok(match self.cache.entry(full_path.clone()) {
            HashMapEntry::Occupied(mut occupied_entry) => {
                if let Some(mmap_arc) = occupied_entry.get().upgrade() {
                    self.counters.hit += 1;
                    Some(mmap_arc.clone())
                }
                else {
                    // The entry exists but the weak ref has been destroyed.
                    self.counters.miss_weak += 1;
                    if let Some(mmap_arc) = open_mmap(&full_path)? {
                        occupied_entry.insert(Arc::downgrade(&mmap_arc));
                        Some(mmap_arc)
                    }
                    else {
                        None
                    }
                }
            }
            HashMapEntry::Vacant(vacant_entry) => {
                self.counters.miss_empty += 1;
                if let Some(mmap_arc) = open_mmap(&full_path)? {
                    vacant_entry.insert(Arc::downgrade(&mmap_arc));
                    Some(mmap_arc)
                }
                else {
                    None
                }
            }
        })
    }
}

/// Directory storing data in files, read via mmap.
///
/// The Mmap object are cached to limit the
/// system calls.
#[derive(Clone)]
pub struct MmapDirectory {
    root_path: PathBuf,
    mmap_cache: Arc<RwLock<MmapCache>>,
    _temp_directory: Arc<Option<TempDir>>,

}

impl fmt::Debug for MmapDirectory {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
       write!(f, "MmapDirectory({:?})", self.root_path)
   }
}

impl MmapDirectory {

    /// Creates a new MmapDirectory in a temporary directory.
    ///
    /// This is mostly useful to test the MmapDirectory itself.
    /// For your unit tests, prefer the RAMDirectory.
    pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
        let tempdir = try!(TempDir::new("index"));
        let tempdir_path = PathBuf::from(tempdir.path());
        let directory = MmapDirectory {
            root_path: PathBuf::from(tempdir_path),
            mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
            _temp_directory: Arc::new(Some(tempdir))
        };
        Ok(directory)
    }

    /// Opens a MmapDirectory in a directory.
    ///
    /// Returns an error if the `directory_path` does not
    /// exist or if it is not a directory.
    pub fn open(directory_path: &Path) -> Result<MmapDirectory, OpenDirectoryError> {
        if !directory_path.exists() {
            Err(OpenDirectoryError::DoesNotExist(PathBuf::from(directory_path)))
        }
        else if !directory_path.is_dir() {
            Err(OpenDirectoryError::NotADirectory(PathBuf::from(directory_path)))
        }
        else {
            Ok(MmapDirectory {
                root_path: PathBuf::from(directory_path),
                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
                _temp_directory: Arc::new(None)
            })
        }
    }

    /// Joins a relative_path to the directory `root_path`
    /// to create a proper complete `filepath`.
    fn resolve_path(&self, relative_path: &Path) -> PathBuf {
        self.root_path.join(relative_path)
    }

    /// Sync the root directory.
    /// In certain FS, this is required to persistently create
    /// a file.
    fn sync_directory(&self,) -> Result<(), io::Error> {
        let fd = try!(File::open(&self.root_path));
        try!(fd.sync_all());
        Ok(())
    }

    pub fn get_cache_info(&mut self) -> CacheInfo {
        self.mmap_cache
            .write()
            .expect("Mmap cache lock is poisoned.")
            .get_info()
    }

}

/// This Write wraps a File, but has the specificity of
/// call `sync_all` on flush.
struct SafeFileWriter(File);

impl SafeFileWriter {
    fn new(file: File) -> SafeFileWriter {
        SafeFileWriter(file)
    }
}

impl Write for SafeFileWriter {

    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.0.write(buf)
    }

    fn flush(&mut self) -> io::Result<()> {
        try!(self.0.flush());
        self.0.sync_all()
    }
}

impl Seek for SafeFileWriter {
    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
        self.0.seek(pos)
    }
}

impl Directory for MmapDirectory {

    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, FileError> {
        debug!("Open Read {:?}", path);
        let full_path = self.resolve_path(path);

        let mut mmap_cache = self.mmap_cache
            .write()
            .map_err(|_| {
                make_io_err(format!("Failed to acquired write lock on mmap cache while reading {:?}", path))
            })?;

        Ok(mmap_cache.get_mmap(full_path)?
            .map(MmapReadOnly::from)
            .map(ReadOnlySource::Mmap)
            .unwrap_or(ReadOnlySource::Anonymous(SharedVecSlice::empty()))
        )
    }

    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
        debug!("Open Write {:?}", path);
        let full_path = self.resolve_path(path);

        let open_res = OpenOptions::new()
            .write(true)
            .create_new(true)
            .open(full_path);

        let mut file = try!(
            open_res.map_err(|err| {
                if err.kind() == io::ErrorKind::AlreadyExists {
                    OpenWriteError::FileAlreadyExists(PathBuf::from(path))
                }
                else {
                    OpenWriteError::IOError(err)
                }
            })
        );

        // making sure the file is created.
        try!(file.flush());

        // Apparetntly, on some filesystem syncing the parent
        // directory is required.
        try!(self.sync_directory());

        let writer = SafeFileWriter::new(file);
        Ok(BufWriter::new(Box::new(writer)))
    }

    fn delete(&self, path: &Path) -> result::Result<(), FileError> {
        debug!("Delete {:?}", path);
        let full_path = self.resolve_path(path);
        let mut mmap_cache = try!(self.mmap_cache
            .write()
            .map_err(|_| {
                make_io_err(format!("Failed to acquired write lock on mmap cache while deleting {:?}", path))
            })
        );
        // Removing the entry in the MMap cache.
        // The munmap will appear on Drop,
        // when the last reference is gone.
        mmap_cache.cache.remove(&full_path);
        try!(fs::remove_file(&full_path));
        try!(self.sync_directory());
        Ok(())
    }

    fn exists(&self, path: &Path) -> bool {
        let full_path = self.resolve_path(path);
        full_path.exists()
    }

    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
        debug!("Atomic Write {:?}", path);
        let full_path = self.resolve_path(path);
        let meta_file = atomicwrites::AtomicFile::new(full_path, atomicwrites::AllowOverwrite);
        try!(meta_file.write(|f| {
            f.write_all(data)
        }));
        Ok(())
    }

    fn box_clone(&self,) -> Box<Directory> {
        Box::new(self.clone())
    }

}

#[cfg(test)]
mod tests {

    // There are more tests in directory/mod.rs
    // The following tests are specific to the MmapDirectory

    use super::*;

    #[test]
    fn test_open_empty() {
        // empty file is actually an edge case because those
        // cannot be mmapped.
        //
        // In that case the directory returns a SharedVecSlice.
        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
        let path = PathBuf::from("test");
        {
            let mut w = mmap_directory.open_write(&path).unwrap();
            w.flush().unwrap();
        }
        let readonlymap = mmap_directory.open_read(&path).unwrap();
        assert_eq!(readonlymap.len(), 0);
    }

    #[test]
    fn test_cache() {

        let content = "abc".as_bytes();

        // here we test if the cache releases
        // mmaps correctly.
        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
        let paths: Vec<PathBuf> = (0..10)
            .map(|i| PathBuf::from(&*format!("file_{}", i)))
            .collect();
        {
            for path in &paths {
                let mut w = mmap_directory.open_write(path).unwrap();
                w.write(content).unwrap();
                w.flush().unwrap();
            }
        }
        {
            for path in &paths {
                {
                    let _r = mmap_directory.open_read(path).unwrap();
                    assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
                }
                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
            }
        }
        assert_eq!(mmap_directory.get_cache_info().counters.miss_empty, 10);

        {
            // test weak miss
            // the first pass create the weak refs.
            for path in &paths {
                let _r = mmap_directory.open_read(path).unwrap();
            }
            // ... the second hits the weak refs.
            for path in &paths {
                let _r = mmap_directory.open_read(path).unwrap();
            }
            let cache_info = mmap_directory.get_cache_info();
            assert_eq!(cache_info.counters.miss_empty, 20);
            assert_eq!(cache_info.counters.miss_weak, 10);
        }

        {
            let mut saved_readmmaps = vec!();
            // Keeps reference alive
            for (i, path) in paths.iter().enumerate() {
                let r = mmap_directory.open_read(path).unwrap();
                saved_readmmaps.push(r);
                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
            }
            let cache_info = mmap_directory.get_cache_info();
            println!("{:?}", cache_info);
            assert_eq!(cache_info.counters.miss_empty, 30);
            assert_eq!(cache_info.counters.miss_weak, 10);
            assert_eq!(cache_info.mmapped.len(), 10);

            for saved_readmmap in saved_readmmaps {
                assert_eq!(saved_readmmap.as_slice(), content);
            }
        }

        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);

    }

}
<commit_msg>NOBUG made the cleanup limit adaptive in MmapCache<commit_after>use atomicwrites;
use common::make_io_err;
use directory::Directory;
use directory::error::{OpenWriteError, FileError, OpenDirectoryError};
use directory::ReadOnlySource;
use directory::shared_vec_slice::SharedVecSlice;
use directory::WritePtr;
use fst::raw::MmapReadOnly;
use memmap::{Mmap, Protection};
use std::collections::hash_map::Entry as HashMapEntry;
use std::collections::HashMap;
use std::mem;
use std::convert::From;
use std::fmt;
use std::fs;
use std::fs::File;
use std::fs::OpenOptions;
use std::io;
use std::io::{Seek, SeekFrom};
use std::io::BufWriter;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::result;
use std::sync::Arc;
use std::sync::RwLock;
use std::sync::Weak;
use tempdir::TempDir;

fn open_mmap(full_path: &PathBuf) -> result::Result<Option<Arc<Mmap>>, FileError> {
    let convert_file_error = |err: io::Error| {
        if err.kind() == io::ErrorKind::NotFound {
            FileError::FileDoesNotExist(full_path.clone())
        }
        else {
            FileError::IOError(err)
        }
    };
    let file = File::open(&full_path).map_err(convert_file_error)?;
    if try!(file.metadata()).len() == 0 {
        // if the file size is 0, it will not be possible
        // to mmap the file, so we return an anonymous mmap_cache
        // instead.
        return Ok(None)
    }
    Ok(Some(Arc::new(Mmap::open(&file, Protection::Read)?)))
}

#[derive(Default,Clone,Debug,RustcDecodable,RustcEncodable)]
pub struct CacheCounters {
    hit: usize,
    miss_empty: usize,
    miss_weak: usize,
}

#[derive(Clone,Debug,RustcDecodable,RustcEncodable)]
pub struct CacheInfo {
    pub counters: CacheCounters,
    pub mmapped: Vec<PathBuf>,
}

struct MmapCache {
    counters: CacheCounters,
    cache: HashMap<PathBuf, Weak<Mmap>>,
    purge_weak_limit: usize,
}

const STARTING_PURGE_WEAK_LIMIT: usize = 1_000;

impl Default for MmapCache {
    fn default() -> MmapCache {
        MmapCache {
            counters: CacheCounters::default(),
            cache: HashMap::new(),
            purge_weak_limit: STARTING_PURGE_WEAK_LIMIT,
        }
    }
}

impl MmapCache {

   fn cleanup(&mut self) {
        let previous_cache_size = self.cache.len();
        let mut new_cache = HashMap::new();
        mem::swap(&mut new_cache, &mut self.cache);
        self.cache = new_cache
            .into_iter()
            .filter(|&(_, ref weak_ref)| weak_ref.upgrade().is_some())
            .collect();
        if self.cache.len() == previous_cache_size {
            self.purge_weak_limit *= 2;
        }
    }

    fn get_info(&mut self) -> CacheInfo {
        self.cleanup();
        let paths: Vec<PathBuf> = self.cache.keys()
            .cloned()
            .collect();
        CacheInfo {
            counters: self.counters.clone(),
            mmapped: paths,
        }
    }

    fn get_mmap(&mut self, full_path: PathBuf) -> Result<Option<Arc<Mmap>>, FileError> {
        // if we exceed this limit, then we go through the weak
        // and remove those that are obsolete.
        if self.cache.len() > self.purge_weak_limit {
            self.cleanup();
        }
        Ok(match self.cache.entry(full_path.clone()) {
            HashMapEntry::Occupied(mut occupied_entry) => {
                if let Some(mmap_arc) = occupied_entry.get().upgrade() {
                    self.counters.hit += 1;
                    Some(mmap_arc.clone())
                }
                else {
                    // The entry exists but the weak ref has been destroyed.
                    self.counters.miss_weak += 1;
                    if let Some(mmap_arc) = open_mmap(&full_path)? {
                        occupied_entry.insert(Arc::downgrade(&mmap_arc));
                        Some(mmap_arc)
                    }
                    else {
                        None
                    }
                }
            }
            HashMapEntry::Vacant(vacant_entry) => {
                self.counters.miss_empty += 1;
                if let Some(mmap_arc) = open_mmap(&full_path)? {
                    vacant_entry.insert(Arc::downgrade(&mmap_arc));
                    Some(mmap_arc)
                }
                else {
                    None
                }
            }
        })
    }
}

/// Directory storing data in files, read via mmap.
///
/// The Mmap object are cached to limit the
/// system calls.
#[derive(Clone)]
pub struct MmapDirectory {
    root_path: PathBuf,
    mmap_cache: Arc<RwLock<MmapCache>>,
    _temp_directory: Arc<Option<TempDir>>,

}

impl fmt::Debug for MmapDirectory {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
       write!(f, "MmapDirectory({:?})", self.root_path)
   }
}

impl MmapDirectory {

    /// Creates a new MmapDirectory in a temporary directory.
    ///
    /// This is mostly useful to test the MmapDirectory itself.
    /// For your unit tests, prefer the RAMDirectory.
    pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
        let tempdir = try!(TempDir::new("index"));
        let tempdir_path = PathBuf::from(tempdir.path());
        let directory = MmapDirectory {
            root_path: PathBuf::from(tempdir_path),
            mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
            _temp_directory: Arc::new(Some(tempdir))
        };
        Ok(directory)
    }

    /// Opens a MmapDirectory in a directory.
    ///
    /// Returns an error if the `directory_path` does not
    /// exist or if it is not a directory.
    pub fn open(directory_path: &Path) -> Result<MmapDirectory, OpenDirectoryError> {
        if !directory_path.exists() {
            Err(OpenDirectoryError::DoesNotExist(PathBuf::from(directory_path)))
        }
        else if !directory_path.is_dir() {
            Err(OpenDirectoryError::NotADirectory(PathBuf::from(directory_path)))
        }
        else {
            Ok(MmapDirectory {
                root_path: PathBuf::from(directory_path),
                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
                _temp_directory: Arc::new(None)
            })
        }
    }

    /// Joins a relative_path to the directory `root_path`
    /// to create a proper complete `filepath`.
    fn resolve_path(&self, relative_path: &Path) -> PathBuf {
        self.root_path.join(relative_path)
    }

    /// Sync the root directory.
    /// In certain FS, this is required to persistently create
    /// a file.
    fn sync_directory(&self,) -> Result<(), io::Error> {
        let fd = try!(File::open(&self.root_path));
        try!(fd.sync_all());
        Ok(())
    }

    pub fn get_cache_info(&mut self) -> CacheInfo {
        self.mmap_cache
            .write()
            .expect("Mmap cache lock is poisoned.")
            .get_info()
    }

}

/// This Write wraps a File, but has the specificity of
/// call `sync_all` on flush.
struct SafeFileWriter(File);

impl SafeFileWriter {
    fn new(file: File) -> SafeFileWriter {
        SafeFileWriter(file)
    }
}

impl Write for SafeFileWriter {

    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.0.write(buf)
    }

    fn flush(&mut self) -> io::Result<()> {
        try!(self.0.flush());
        self.0.sync_all()
    }
}

impl Seek for SafeFileWriter {
    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
        self.0.seek(pos)
    }
}

impl Directory for MmapDirectory {

    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, FileError> {
        debug!("Open Read {:?}", path);
        let full_path = self.resolve_path(path);

        let mut mmap_cache = self.mmap_cache
            .write()
            .map_err(|_| {
                make_io_err(format!("Failed to acquired write lock on mmap cache while reading {:?}", path))
            })?;

        Ok(mmap_cache.get_mmap(full_path)?
            .map(MmapReadOnly::from)
            .map(ReadOnlySource::Mmap)
            .unwrap_or(ReadOnlySource::Anonymous(SharedVecSlice::empty()))
        )
    }

    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
        debug!("Open Write {:?}", path);
        let full_path = self.resolve_path(path);

        let open_res = OpenOptions::new()
            .write(true)
            .create_new(true)
            .open(full_path);

        let mut file = try!(
            open_res.map_err(|err| {
                if err.kind() == io::ErrorKind::AlreadyExists {
                    OpenWriteError::FileAlreadyExists(PathBuf::from(path))
                }
                else {
                    OpenWriteError::IOError(err)
                }
            })
        );

        // making sure the file is created.
        try!(file.flush());

        // Apparetntly, on some filesystem syncing the parent
        // directory is required.
        try!(self.sync_directory());

        let writer = SafeFileWriter::new(file);
        Ok(BufWriter::new(Box::new(writer)))
    }

    fn delete(&self, path: &Path) -> result::Result<(), FileError> {
        debug!("Delete {:?}", path);
        let full_path = self.resolve_path(path);
        let mut mmap_cache = try!(self.mmap_cache
            .write()
            .map_err(|_| {
                make_io_err(format!("Failed to acquired write lock on mmap cache while deleting {:?}", path))
            })
        );
        // Removing the entry in the MMap cache.
        // The munmap will appear on Drop,
        // when the last reference is gone.
        mmap_cache.cache.remove(&full_path);
        try!(fs::remove_file(&full_path));
        try!(self.sync_directory());
        Ok(())
    }

    fn exists(&self, path: &Path) -> bool {
        let full_path = self.resolve_path(path);
        full_path.exists()
    }

    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
        debug!("Atomic Write {:?}", path);
        let full_path = self.resolve_path(path);
        let meta_file = atomicwrites::AtomicFile::new(full_path, atomicwrites::AllowOverwrite);
        try!(meta_file.write(|f| {
            f.write_all(data)
        }));
        Ok(())
    }

    fn box_clone(&self,) -> Box<Directory> {
        Box::new(self.clone())
    }

}

#[cfg(test)]
mod tests {

    // There are more tests in directory/mod.rs
    // The following tests are specific to the MmapDirectory

    use super::*;

    #[test]
    fn test_open_empty() {
        // empty file is actually an edge case because those
        // cannot be mmapped.
        //
        // In that case the directory returns a SharedVecSlice.
        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
        let path = PathBuf::from("test");
        {
            let mut w = mmap_directory.open_write(&path).unwrap();
            w.flush().unwrap();
        }
        let readonlymap = mmap_directory.open_read(&path).unwrap();
        assert_eq!(readonlymap.len(), 0);
    }

    #[test]
    fn test_cache() {
        let content = "abc".as_bytes();

        // here we test if the cache releases
        // mmaps correctly.
        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
        let paths: Vec<PathBuf> = (0..10)
            .map(|i| PathBuf::from(&*format!("file_{}", i)))
            .collect();
        {
            for path in &paths {
                let mut w = mmap_directory.open_write(path).unwrap();
                w.write(content).unwrap();
                w.flush().unwrap();
            }
        }
        {
            for path in &paths {
                {
                    let _r = mmap_directory.open_read(path).unwrap();
                    assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
                }
                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
            }
        }
        assert_eq!(mmap_directory.get_cache_info().counters.miss_empty, 10);

        {
            // test weak miss
            // the first pass create the weak refs.
            for path in &paths {
                let _r = mmap_directory.open_read(path).unwrap();
            }
            // ... the second hits the weak refs.
            for path in &paths {
                let _r = mmap_directory.open_read(path).unwrap();
            }
            let cache_info = mmap_directory.get_cache_info();
            assert_eq!(cache_info.counters.miss_empty, 20);
            assert_eq!(cache_info.counters.miss_weak, 10);
        }

        {
            let mut saved_readmmaps = vec!();
            // Keeps reference alive
            for (i, path) in paths.iter().enumerate() {
                let r = mmap_directory.open_read(path).unwrap();
                saved_readmmaps.push(r);
                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
            }
            let cache_info = mmap_directory.get_cache_info();
            println!("{:?}", cache_info);
            assert_eq!(cache_info.counters.miss_empty, 30);
            assert_eq!(cache_info.counters.miss_weak, 10);
            assert_eq!(cache_info.mmapped.len(), 10);

            for saved_readmmap in saved_readmmaps {
                assert_eq!(saved_readmmap.as_slice(), content);
            }
        }

        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);

    }

}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

/// Utilities to support Stratis.

use std::path::Path;
use std::process::Command;

use uuid::Uuid;

use super::super::errors::{EngineError, EngineResult, ErrorEnum};

/// Common function to call a command line utility, returning an Result with an error message which
/// also includes stdout & stderr if it fails.
pub fn execute_cmd(cmd: &mut Command, error_msg: &str) -> EngineResult<()> {
    let result = cmd.output()?;
    if result.status.success() {
        Ok(())
    } else {
        let std_out_txt = String::from_utf8_lossy(&result.stdout);
        let std_err_txt = String::from_utf8_lossy(&result.stderr);
        let err_msg = format!("{} stdout: {} stderr: {}",
                              error_msg,
                              std_out_txt,
                              std_err_txt);
        Err(EngineError::Engine(ErrorEnum::Error, err_msg))
    }
}

/// Create a filesystem on devnode.
pub fn create_fs(devnode: &Path, uuid: Uuid) -> EngineResult<()> {
    return execute_cmd(Command::new("mkfs.xfs")
                           .arg("-f")
                           .arg("-q")
                           .arg(&devnode)
                           .arg("-m")
                           .arg(format!("uuid={}", uuid)),
                       &format!("Failed to create new filesystem at {:?}", devnode));
}

/// Use the xfs_growfs command to expand a filesystem mounted at the given
/// mount point.
pub fn xfs_growfs(mount_point: &Path) -> EngineResult<()> {
    return execute_cmd(Command::new("xfs_growfs").arg(mount_point).arg("-d"),
                       &format!("Failed to expand filesystem {:?}", mount_point));
}

/// Set a new UUID for filesystem on the devnode.
pub fn set_uuid(devnode: &Path, uuid: Uuid) -> EngineResult<()> {
    return execute_cmd(Command::new("xfs_admin")
                           .arg("-U")
                           .arg(format!("{}", uuid))
                           .arg(&devnode),
                       &format!("Failed to set UUID for filesystem {:?}", devnode));
}
<commit_msg>Fix clippy error<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

/// Utilities to support Stratis.

use std::path::Path;
use std::process::Command;

use uuid::Uuid;

use super::super::errors::{EngineError, EngineResult, ErrorEnum};

/// Common function to call a command line utility, returning an Result with an error message which
/// also includes stdout & stderr if it fails.
pub fn execute_cmd(cmd: &mut Command, error_msg: &str) -> EngineResult<()> {
    let result = cmd.output()?;
    if result.status.success() {
        Ok(())
    } else {
        let std_out_txt = String::from_utf8_lossy(&result.stdout);
        let std_err_txt = String::from_utf8_lossy(&result.stderr);
        let err_msg = format!("{} stdout: {} stderr: {}",
                              error_msg,
                              std_out_txt,
                              std_err_txt);
        Err(EngineError::Engine(ErrorEnum::Error, err_msg))
    }
}

/// Create a filesystem on devnode.
pub fn create_fs(devnode: &Path, uuid: Uuid) -> EngineResult<()> {
    execute_cmd(Command::new("mkfs.xfs")
                    .arg("-f")
                    .arg("-q")
                    .arg(&devnode)
                    .arg("-m")
                    .arg(format!("uuid={}", uuid)),
                &format!("Failed to create new filesystem at {:?}", devnode))
}

/// Use the xfs_growfs command to expand a filesystem mounted at the given
/// mount point.
pub fn xfs_growfs(mount_point: &Path) -> EngineResult<()> {
    execute_cmd(Command::new("xfs_growfs").arg(mount_point).arg("-d"),
                &format!("Failed to expand filesystem {:?}", mount_point))
}

/// Set a new UUID for filesystem on the devnode.
pub fn set_uuid(devnode: &Path, uuid: Uuid) -> EngineResult<()> {
    execute_cmd(Command::new("xfs_admin")
                    .arg("-U")
                    .arg(format!("{}", uuid))
                    .arg(&devnode),
                &format!("Failed to set UUID for filesystem {:?}", devnode))
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(clippy)]

#![warn(while_let_loop, empty_loop, while_let_on_iterator)]
#![allow(dead_code, never_loop, unused, cyclomatic_complexity)]

fn main() {
    let y = Some(true);
    loop {
        if let Some(_x) = y {
            let _v = 1;
        } else {
            break
        }
    }
    loop { // no error, break is not in else clause
        if let Some(_x) = y {
            let _v = 1;
        }
        break;
    }
    loop {
        match y {
            Some(_x) => true,
            None => break
        };
    }
    loop {
        let x = match y {
            Some(x) => x,
            None => break
        };
        let _x = x;
        let _str = "foo";
    }
    loop {
        let x = match y {
            Some(x) => x,
            None => break,
        };
        { let _a = "bar"; };
        { let _b = "foobar"; }
    }
    loop { // no error, else branch does something other than break
        match y {
            Some(_x) => true,
            _ => {
                let _z = 1;
                break;
            }
        };
    }
    while let Some(x) = y { // no error, obviously
        println!("{}", x);
    }

    // #675, this used to have a wrong suggestion
    loop {
        let (e, l) = match "".split_whitespace().next() {
            Some(word) => (word.is_empty(), word.len()),
            None => break
        };

        let _ = (e, l);
    }

    let mut iter = 1..20;
    while let Option::Some(x) = iter.next() {
        println!("{}", x);
    }

    let mut iter = 1..20;
    while let Some(x) = iter.next() {
        println!("{}", x);
    }

    let mut iter = 1..20;
    while let Some(_) = iter.next() {}

    let mut iter = 1..20;
    while let None = iter.next() {} // this is fine (if nonsensical)

    let mut iter = 1..20;
    if let Some(x) = iter.next() { // also fine
        println!("{}", x)
    }

    // the following shouldn't warn because it can't be written with a for loop
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {
        println!("next: {:?}", iter.next())
    }

    // neither can this
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {
        println!("next: {:?}", iter.next());
    }

    // or this
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {break;}
    println!("Remaining iter {:?}", iter);

    // or this
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {
        iter = 1..20;
    }
}

// regression test (#360)
// this should not panic
// it's okay if further iterations of the lint
// cause this function to trigger it
fn no_panic<T>(slice: &[T]) {
    let mut iter = slice.iter();
    loop {
        let _ = match iter.next() {
            Some(ele) => ele,
            None => break
        };
        loop {}
    }
}

fn issue1017() {
    let r: Result<u32, u32> = Ok(42);
    let mut len = 1337;

    loop {
        match r {
            Err(_) => len = 0,
            Ok(length) => {
                len = length;
                break
            }
        }
    }
}

// Issue #1188
fn refutable() {
    let a = [42, 1337];
    let mut b = a.iter();

    // consume all the 42s
    while let Some(&42) = b.next() {
    }

    let a = [(1, 2, 3)];
    let mut b = a.iter();

    while let Some(&(1, 2, 3)) = b.next() {
    }

    let a = [Some(42)];
    let mut b = a.iter();

    while let Some(&None) = b.next() {
    }

    /* This gives “refutable pattern in `for` loop binding: `&_` not covered”
    for &42 in b {}
    for &(1, 2, 3) in b {}
    for &Option::None in b.next() {}
    // */

    let x = a.iter();
    loop { // x is reused, so don't lint here
        while let Some(v) = x.next() {
        }
    }

    let y = a.iter();
    for _ in 0..2 {
        while let Some(v) = x.next() {
        }
    }
}
<commit_msg>fix tests<commit_after>#![feature(plugin)]
#![plugin(clippy)]

#![warn(while_let_loop, empty_loop, while_let_on_iterator)]
#![allow(dead_code, never_loop, unused, cyclomatic_complexity)]

fn main() {
    let y = Some(true);
    loop {
        if let Some(_x) = y {
            let _v = 1;
        } else {
            break
        }
    }
    loop { // no error, break is not in else clause
        if let Some(_x) = y {
            let _v = 1;
        }
        break;
    }
    loop {
        match y {
            Some(_x) => true,
            None => break
        };
    }
    loop {
        let x = match y {
            Some(x) => x,
            None => break
        };
        let _x = x;
        let _str = "foo";
    }
    loop {
        let x = match y {
            Some(x) => x,
            None => break,
        };
        { let _a = "bar"; };
        { let _b = "foobar"; }
    }
    loop { // no error, else branch does something other than break
        match y {
            Some(_x) => true,
            _ => {
                let _z = 1;
                break;
            }
        };
    }
    while let Some(x) = y { // no error, obviously
        println!("{}", x);
    }

    // #675, this used to have a wrong suggestion
    loop {
        let (e, l) = match "".split_whitespace().next() {
            Some(word) => (word.is_empty(), word.len()),
            None => break
        };

        let _ = (e, l);
    }

    let mut iter = 1..20;
    while let Option::Some(x) = iter.next() {
        println!("{}", x);
    }

    let mut iter = 1..20;
    while let Some(x) = iter.next() {
        println!("{}", x);
    }

    let mut iter = 1..20;
    while let Some(_) = iter.next() {}

    let mut iter = 1..20;
    while let None = iter.next() {} // this is fine (if nonsensical)

    let mut iter = 1..20;
    if let Some(x) = iter.next() { // also fine
        println!("{}", x)
    }

    // the following shouldn't warn because it can't be written with a for loop
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {
        println!("next: {:?}", iter.next())
    }

    // neither can this
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {
        println!("next: {:?}", iter.next());
    }

    // or this
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {break;}
    println!("Remaining iter {:?}", iter);

    // or this
    let mut iter = 1u32..20;
    while let Some(x) = iter.next() {
        iter = 1..20;
    }
}

// regression test (#360)
// this should not panic
// it's okay if further iterations of the lint
// cause this function to trigger it
fn no_panic<T>(slice: &[T]) {
    let mut iter = slice.iter();
    loop {
        let _ = match iter.next() {
            Some(ele) => ele,
            None => break
        };
        loop {}
    }
}

fn issue1017() {
    let r: Result<u32, u32> = Ok(42);
    let mut len = 1337;

    loop {
        match r {
            Err(_) => len = 0,
            Ok(length) => {
                len = length;
                break
            }
        }
    }
}

// Issue #1188
fn refutable() {
    let a = [42, 1337];
    let mut b = a.iter();

    // consume all the 42s
    while let Some(&42) = b.next() {
    }

    let a = [(1, 2, 3)];
    let mut b = a.iter();

    while let Some(&(1, 2, 3)) = b.next() {
    }

    let a = [Some(42)];
    let mut b = a.iter();

    while let Some(&None) = b.next() {
    }

    /* This gives “refutable pattern in `for` loop binding: `&_` not covered”
    for &42 in b {}
    for &(1, 2, 3) in b {}
    for &Option::None in b.next() {}
    // */

    let mut y = a.iter();
    loop { // x is reused, so don't lint here
        while let Some(v) = y.next() {
        }
    }

    let mut y = a.iter();
    for _ in 0..2 {
        while let Some(v) = y.next() {
        }
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::bindings::cell::DOMRefCell;
use dom::bindings::codegen::Bindings::EventBinding::EventMethods;
use dom::bindings::codegen::Bindings::CloseEventBinding;
use dom::bindings::codegen::Bindings::CloseEventBinding::CloseEventMethods;
use dom::bindings::codegen::InheritTypes::EventCast;
use dom::bindings::error::Fallible;
use dom::bindings::global::GlobalRef;
use dom::bindings::js::{JSRef,Temporary, Rootable};
use dom::bindings::utils::reflect_dom_object;
use dom::event::{Event, EventTypeId, EventBubbles, EventCancelable};
use script_task::ScriptChan;
use std::borrow::ToOwned;
use std::cell::Cell;
use util::str::DOMString;

#[dom_struct]
pub struct CloseEvent{
    event: Event,
    wasClean: Cell<bool>,
    code: Cell<u16>,
    reason: DOMRefCell<DOMString>
}

#[derive(PartialEq)]
pub enum Clean {
    Clean,
    NotClean
}

impl CloseEvent{
    pub fn new_inherited(type_id: EventTypeId) -> CloseEvent{
        CloseEvent{
            event: Event::new_inherited(type_id),
            wasClean: Cell::new(false),
            code: Cell::new(0),
            reason: DOMRefCell::new("".to_owned())
        }
    }

    pub fn new(global: GlobalRef,
               type_: DOMString,
               bubbles: EventBubbles,
               cancelable: EventCancelable,
               wasClean: bool,
               code: u16,
               reason: DOMString) -> Temporary<CloseEvent> {
        let ev = reflect_dom_object(box CloseEvent::new_inherited(EventTypeId::CloseEvent),
                                    global,
                                    CloseEventBinding::Wrap);
		let ev = ev.root();
        let event: JSRef<Event> = EventCast::from_ref(ev.r());
        event.InitEvent(type_,
                        bubbles == EventBubbles::Bubbles,
                        cancelable == EventCancelable::Cancelable);
        let ev = ev.r();
        ev.wasClean.set(wasClean);
        ev.code.set(code);
        *ev.reason.borrow_mut() = reason;
        Temporary::from_rooted(ev)
    }

    pub fn Constructor(global: GlobalRef,
                       type_: DOMString,
                       init: &CloseEventBinding::CloseEventInit) -> Fallible<Temporary<CloseEvent>> {
        let clean_status = init.wasClean.unwrap();
        let cd = init.code.unwrap_or(0);
        let rsn = match init.reason.as_ref() {
            Some(reason) => reason.clone(),
            None => "".to_owned(),
        };
        let bubbles = if init.parent.bubbles { EventBubbles::Bubbles } else { EventBubbles::DoesNotBubble };
        let cancelable = if init.parent.cancelable { EventCancelable::Cancelable } else { EventCancelable::NotCancelable };
        Ok(CloseEvent::new(global, type_, bubbles, cancelable, clean_status, cd, rsn))
    }
}

impl<'a> CloseEventMethods for JSRef<'a, CloseEvent>{
    fn WasClean(self) -> bool {
        self.wasClean.get()
    }

    fn Code(self) -> u16 {
        self.code.get()
    }

    fn Reason(self) -> DOMString {
        let reason = self.reason.borrow();
        reason.clone()
    }
}
<commit_msg>closeevent.rs - unwrap_or in place of unwrap<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::bindings::cell::DOMRefCell;
use dom::bindings::codegen::Bindings::EventBinding::EventMethods;
use dom::bindings::codegen::Bindings::CloseEventBinding;
use dom::bindings::codegen::Bindings::CloseEventBinding::CloseEventMethods;
use dom::bindings::codegen::InheritTypes::EventCast;
use dom::bindings::error::Fallible;
use dom::bindings::global::GlobalRef;
use dom::bindings::js::{JSRef,Temporary, Rootable};
use dom::bindings::utils::reflect_dom_object;
use dom::event::{Event, EventTypeId, EventBubbles, EventCancelable};
use script_task::ScriptChan;
use std::borrow::ToOwned;
use std::cell::Cell;
use util::str::DOMString;

#[dom_struct]
pub struct CloseEvent{
    event: Event,
    wasClean: Cell<bool>,
    code: Cell<u16>,
    reason: DOMRefCell<DOMString>
}

#[derive(PartialEq)]
pub enum Clean {
    Clean,
    NotClean
}

impl CloseEvent{
    pub fn new_inherited(type_id: EventTypeId) -> CloseEvent{
        CloseEvent{
            event: Event::new_inherited(type_id),
            wasClean: Cell::new(false),
            code: Cell::new(0),
            reason: DOMRefCell::new("".to_owned())
        }
    }

    pub fn new(global: GlobalRef,
               type_: DOMString,
               bubbles: EventBubbles,
               cancelable: EventCancelable,
               wasClean: bool,
               code: u16,
               reason: DOMString) -> Temporary<CloseEvent> {
        let ev = reflect_dom_object(box CloseEvent::new_inherited(EventTypeId::CloseEvent),
                                    global,
                                    CloseEventBinding::Wrap);
		let ev = ev.root();
        let event: JSRef<Event> = EventCast::from_ref(ev.r());
        event.InitEvent(type_,
                        bubbles == EventBubbles::Bubbles,
                        cancelable == EventCancelable::Cancelable);
        let ev = ev.r();
        ev.wasClean.set(wasClean);
        ev.code.set(code);
        *ev.reason.borrow_mut() = reason;
        Temporary::from_rooted(ev)
    }

    pub fn Constructor(global: GlobalRef,
                       type_: DOMString,
                       init: &CloseEventBinding::CloseEventInit) -> Fallible<Temporary<CloseEvent>> {
        let clean_status = init.wasClean.unwrap_or();
        let cd = init.code.unwrap_or(0);
        let rsn = match init.reason.as_ref() {
            Some(reason) => reason.clone(),
            None => "".to_owned(),
        };
        let bubbles = if init.parent.bubbles { EventBubbles::Bubbles } else { EventBubbles::DoesNotBubble };
        let cancelable = if init.parent.cancelable { EventCancelable::Cancelable } else { EventCancelable::NotCancelable };
        Ok(CloseEvent::new(global, type_, bubbles, cancelable, clean_status, cd, rsn))
    }
}

impl<'a> CloseEventMethods for JSRef<'a, CloseEvent>{
    fn WasClean(self) -> bool {
        self.wasClean.get()
    }

    fn Code(self) -> u16 {
        self.code.get()
    }

    fn Reason(self) -> DOMString {
        let reason = self.reason.borrow();
        reason.clone()
    }
}
<|endoftext|>


<commit_before>#![recursion_limit = "1000"]

#[macro_use]
extern crate error_chain;
extern crate lalrpop_util;
extern crate reproto_core;
extern crate num;
extern crate linked_hash_map;

pub mod ast;
pub mod errors;
pub mod lexer;
#[allow(unused)]
mod parser;
mod token;
mod utils;

use lalrpop_util::ParseError;
use self::errors::*;
use std::fs;
use std::io::Read;
use std::path::Path;
use std::rc::Rc;

pub fn read_file(path: &Path) -> Result<String> {
    let mut f = fs::File::open(path)
        .map_err(|e| ErrorKind::File(format!("could not open file: {}", e), path.to_owned()))?;

    let mut content = String::new();

    f.read_to_string(&mut content)
        .map_err(|e| ErrorKind::File(format!("could not read file: {}", e), path.to_owned()))?;

    Ok(content)
}

pub fn parse_file<'input>(path: &'input Path, input: &'input str) -> Result<ast::File<'input>> {
    use self::ErrorKind::*;

    let path = Rc::new(path.to_owned());
    let lexer = lexer::lex(input);

    match parser::parse_File(&path, lexer) {
        Ok(file) => Ok(file),
        Err(e) => {
            match e {
                ParseError::InvalidToken { location } => {
                    let pos = (path.clone(), location, location);
                    Err(Syntax(Some(pos.into()), vec![]).into())
                }
                ParseError::UnrecognizedToken { token, expected } => {
                    let pos = token.map(|(start, _, end)| (path.clone(), start, end));
                    Err(Syntax(pos.map(Into::into), expected).into())
                }
                ParseError::User { error } => {
                    match error {
                        token::Error::UnterminatedString { start } => {
                            let pos = (path.clone(), start, start);
                            return Err(Parse("unterminated string", pos.into()).into());
                        }
                        token::Error::UnterminatedEscape { start } => {
                            let pos = (path.clone(), start, start);
                            return Err(Parse("unterminated escape sequence", pos.into()).into());
                        }
                        token::Error::InvalidEscape { pos, message } => {
                            let pos = (path.clone(), pos, pos);
                            return Err(Parse(message, pos.into()).into());
                        }
                        token::Error::UnterminatedCodeBlock { start } => {
                            let pos = (path.clone(), start, start);
                            return Err(Parse("unterminated code block", pos.into()).into());
                        }
                        token::Error::InvalidNumber { pos, message } => {
                            let pos = (path.clone(), pos, pos);
                            return Err(Parse(message, pos.into()).into());
                        }
                        token::Error::Unexpected { pos } => {
                            let pos = (path.clone(), pos, pos);
                            return Err(Parse("unexpected input", pos.into()).into());
                        }
                        token::Error::InvalidVersion { start, end } => {
                            let pos = (path.clone(), start, end);
                            return Err(Parse("invalid version", pos.into()).into());
                        }
                        token::Error::InvalidVersionReq { start, end } => {
                            let pos = (path.clone(), start, end);
                            return Err(Parse("invalid version requirement", pos.into()).into());
                        }
                    }
                }
                _ => Err("parse error".into()),
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use std::path::PathBuf;
    use super::*;
    use super::ast::*;

    /// Check that a parsed value equals expected.
    macro_rules! assert_value_eq {
        ($expected:expr, $input:expr) => {{
            let path = Rc::new(PathBuf::from(""));
            let v = parser::parse_Value(&path, parse($input)).unwrap();
            assert_eq!($expected, v);
        }}
    }

    macro_rules! assert_type_spec_eq {
        ($expected:expr, $input:expr) => {{
            let path = Rc::new(PathBuf::from(""));
            let v = parser::parse_TypeSpec(&path, parse($input)).unwrap();
            assert_eq!($expected, v);
        }}
    }

    const FILE1: &[u8] = include_bytes!("tests/file1.reproto");
    const INTERFACE1: &[u8] = include_bytes!("tests/interface1.reproto");

    fn parse(input: &'static str)
             -> Box<Iterator<Item = token::Result<(usize, token::Token<'static>, usize)>>> {
        Box::new(lexer::lex(input))
    }

    fn parse_file(input: &'static str) -> File {
        let path = Rc::new(PathBuf::from(""));
        parser::parse_File(&path, parse(input)).unwrap()
    }

    fn parse_member(input: &'static str) -> Member {
        let path = Rc::new(PathBuf::from(""));
        parser::parse_Member(&path, parse(input)).unwrap()
    }

    fn parse_type_spec(input: &'static str) -> RpType {
        let path = Rc::new(PathBuf::from(""));
        parser::parse_TypeSpec(&path, parse(input)).unwrap()
    }

    #[test]
    fn test_file1() {
        let input = ::std::str::from_utf8(FILE1).unwrap();
        let _ = parse_file(input);
    }

    #[test]
    fn test_array() {
        let ty = parse_type_spec("[string]");

        if let RpType::Array { inner } = ty {
            if let RpType::String = *inner {
                return;
            }
        }

        panic!("Expected Type::Array(Type::String)");
    }

    #[test]
    fn test_map() {
        let ty = parse_type_spec("{string: unsigned/123}");

        // TODO: use #![feature(box_patterns)]:
        // if let Type::Map(box Type::String, box Type::Unsigned(size)) = ty {
        // }
        if let RpType::Map { key, value } = ty {
            if let RpType::String = *key {
                if let RpType::Unsigned { size } = *value {
                    assert_eq!(Some(123usize), size);
                    return;
                }
            }
        }

        panic!("Expected Type::Array(Type::String)");
    }

    #[test]
    fn test_block_comment() {
        parse("/* hello \n world */");
    }

    #[test]
    fn test_line_comment() {
        parse("// hello world\n");
    }

    #[test]
    fn test_code() {
        parse_member("java{{\na { b { c } d } e\n}}");
    }

    #[test]
    fn test_interface() {
        let input = ::std::str::from_utf8(INTERFACE1).unwrap();
        let file = parse_file(input);
        assert_eq!(1, file.decls.len());
    }

    #[test]
    fn test_instance() {
        let path = Rc::new(PathBuf::from(""));

        let c = RpName {
            prefix: None,
            parts: vec!["Foo".to_owned(), "Bar".to_owned()],
        };

        let field = FieldInit {
            name: Loc::new("hello", (path.clone(), 8, 13)),
            value: Loc::new(Value::Number(12.into()), (path.clone(), 15, 17)),
        };

        let field = Loc::new(field, (path.clone(), 8, 17));

        let instance = Instance {
            name: c,
            arguments: Loc::new(vec![field], (path.clone(), 8, 17)),
        };

        assert_value_eq!(Value::Instance(Loc::new(instance, (path.clone(), 0, 18))),
                         "Foo.Bar(hello: 12)");
    }

    #[test]
    fn test_strings() {
        assert_value_eq!(Value::String("foo\nbar".to_owned()), "\"foo\\nbar\"");
    }

    #[test]
    fn test_numbers() {
        assert_value_eq!(Value::Number(1.into()), "1");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: 125.into(),
                             decimal: 2,
                         }),
                         "1.25");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: 12500.into(),
                             decimal: 0,
                         }),
                         "1.25e4");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: (-12500).into(),
                             decimal: 0,
                         }),
                         "-1.25e4");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: (1234).into(),
                             decimal: 8,
                         }),
                         "0.00001234");
    }

    #[test]
    fn test_type_spec() {
        let c = RpName {
            prefix: None,
            parts: vec!["Hello".to_owned(), "World".to_owned()],
        };

        assert_type_spec_eq!(RpType::String, "string");
        assert_type_spec_eq!(RpType::Name { name: c }, "Hello.World");
    }

    #[test]
    fn test_option_decl() {
        let member = parse_member("foo_bar_baz true, foo, \"bar\", 12;");

        if let Member::Option(option) = member {
            assert_eq!("foo_bar_baz", option.name);
            assert_eq!(4, option.values.len());

            assert_eq!(Value::Boolean(true), *option.values[0].as_ref());
            assert_eq!(Value::Identifier("foo".to_owned()),
                       *option.values[1].as_ref());
            assert_eq!(Value::String("bar".to_owned()), *option.values[2].as_ref());
            assert_eq!(Value::Number(12u32.into()), *option.values[3].as_ref());
            return;
        }

        panic!("option did not match");
    }
}
<commit_msg>parser: fix test error<commit_after>#![recursion_limit = "1000"]

#[macro_use]
extern crate error_chain;
extern crate lalrpop_util;
extern crate reproto_core;
extern crate num;
extern crate linked_hash_map;

pub mod ast;
pub mod errors;
pub mod lexer;
#[allow(unused)]
mod parser;
mod token;
mod utils;

use lalrpop_util::ParseError;
use self::errors::*;
use std::fs;
use std::io::Read;
use std::path::Path;
use std::rc::Rc;

pub fn read_file(path: &Path) -> Result<String> {
    let mut f = fs::File::open(path)
        .map_err(|e| ErrorKind::File(format!("could not open file: {}", e), path.to_owned()))?;

    let mut content = String::new();

    f.read_to_string(&mut content)
        .map_err(|e| ErrorKind::File(format!("could not read file: {}", e), path.to_owned()))?;

    Ok(content)
}

pub fn parse_file<'input>(path: &'input Path, input: &'input str) -> Result<ast::File<'input>> {
    use self::ErrorKind::*;

    let path = Rc::new(path.to_owned());
    let lexer = lexer::lex(input);

    match parser::parse_File(&path, lexer) {
        Ok(file) => Ok(file),
        Err(e) => {
            match e {
                ParseError::InvalidToken { location } => {
                    let pos = (path.clone(), location, location);
                    Err(Syntax(Some(pos.into()), vec![]).into())
                }
                ParseError::UnrecognizedToken { token, expected } => {
                    let pos = token.map(|(start, _, end)| (path.clone(), start, end));
                    Err(Syntax(pos.map(Into::into), expected).into())
                }
                ParseError::User { error } => {
                    match error {
                        token::Error::UnterminatedString { start } => {
                            let pos = (path.clone(), start, start);
                            return Err(Parse("unterminated string", pos.into()).into());
                        }
                        token::Error::UnterminatedEscape { start } => {
                            let pos = (path.clone(), start, start);
                            return Err(Parse("unterminated escape sequence", pos.into()).into());
                        }
                        token::Error::InvalidEscape { pos, message } => {
                            let pos = (path.clone(), pos, pos);
                            return Err(Parse(message, pos.into()).into());
                        }
                        token::Error::UnterminatedCodeBlock { start } => {
                            let pos = (path.clone(), start, start);
                            return Err(Parse("unterminated code block", pos.into()).into());
                        }
                        token::Error::InvalidNumber { pos, message } => {
                            let pos = (path.clone(), pos, pos);
                            return Err(Parse(message, pos.into()).into());
                        }
                        token::Error::Unexpected { pos } => {
                            let pos = (path.clone(), pos, pos);
                            return Err(Parse("unexpected input", pos.into()).into());
                        }
                        token::Error::InvalidVersion { start, end } => {
                            let pos = (path.clone(), start, end);
                            return Err(Parse("invalid version", pos.into()).into());
                        }
                        token::Error::InvalidVersionReq { start, end } => {
                            let pos = (path.clone(), start, end);
                            return Err(Parse("invalid version requirement", pos.into()).into());
                        }
                    }
                }
                _ => Err("parse error".into()),
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use std::path::PathBuf;
    use super::*;
    use super::ast::*;

    /// Check that a parsed value equals expected.
    macro_rules! assert_value_eq {
        ($expected:expr, $input:expr) => {{
            let path = Rc::new(PathBuf::from(""));
            let v = parser::parse_Value(&path, parse($input)).unwrap();
            assert_eq!($expected, v);
        }}
    }

    macro_rules! assert_type_spec_eq {
        ($expected:expr, $input:expr) => {{
            let path = Rc::new(PathBuf::from(""));
            let v = parser::parse_TypeSpec(&path, parse($input)).unwrap();
            assert_eq!($expected, v);
        }}
    }

    const FILE1: &[u8] = include_bytes!("tests/file1.reproto");
    const INTERFACE1: &[u8] = include_bytes!("tests/interface1.reproto");

    fn parse(input: &'static str)
             -> Box<Iterator<Item = token::Result<(usize, token::Token<'static>, usize)>>> {
        Box::new(lexer::lex(input))
    }

    fn parse_file(input: &'static str) -> File {
        let path = Rc::new(PathBuf::from(""));
        parser::parse_File(&path, parse(input)).unwrap()
    }

    fn parse_member(input: &'static str) -> Member {
        let path = Rc::new(PathBuf::from(""));
        parser::parse_Member(&path, parse(input)).unwrap()
    }

    fn parse_type_spec(input: &'static str) -> RpType {
        let path = Rc::new(PathBuf::from(""));
        parser::parse_TypeSpec(&path, parse(input)).unwrap()
    }

    #[test]
    fn test_file1() {
        let input = ::std::str::from_utf8(FILE1).unwrap();
        let _ = parse_file(input);
    }

    #[test]
    fn test_array() {
        let ty = parse_type_spec("[string]");

        if let RpType::Array { inner } = ty {
            if let RpType::String = *inner {
                return;
            }
        }

        panic!("Expected Type::Array(Type::String)");
    }

    #[test]
    fn test_map() {
        let ty = parse_type_spec("{string: unsigned/123}");

        // TODO: use #![feature(box_patterns)]:
        // if let Type::Map(box Type::String, box Type::Unsigned(size)) = ty {
        // }
        if let RpType::Map { key, value } = ty {
            if let RpType::String = *key {
                if let RpType::Unsigned { size } = *value {
                    assert_eq!(Some(123usize), size);
                    return;
                }
            }
        }

        panic!("Expected Type::Array(Type::String)");
    }

    #[test]
    fn test_block_comment() {
        parse("/* hello \n world */");
    }

    #[test]
    fn test_line_comment() {
        parse("// hello world\n");
    }

    #[test]
    fn test_code() {
        parse_member("java{{\na { b { c } d } e\n}}");
    }

    #[test]
    fn test_interface() {
        let input = ::std::str::from_utf8(INTERFACE1).unwrap();
        let file = parse_file(input);
        assert_eq!(1, file.decls.len());
    }

    #[test]
    fn test_instance() {
        let path = Rc::new(PathBuf::from(""));

        let c = RpName {
            prefix: None,
            parts: vec!["Foo".to_owned(), "Bar".to_owned()],
        };

        let field = FieldInit {
            name: Loc::new("hello", (path.clone(), 8, 13)),
            value: Loc::new(Value::Number(12.into()), (path.clone(), 15, 17)),
        };

        let field = Loc::new(field, (path.clone(), 8, 17));

        let instance = Instance {
            name: c,
            arguments: Loc::new(vec![field], (path.clone(), 8, 17)),
        };

        let instance = Loc::new(instance, (path.clone(), 0, 18));
        let object = Loc::new(Object::Instance(instance), (path.clone(), 0, 18));

        assert_value_eq!(Value::Object(object), "Foo.Bar(hello: 12)");
    }

    #[test]
    fn test_strings() {
        assert_value_eq!(Value::String("foo\nbar".to_owned()), "\"foo\\nbar\"");
    }

    #[test]
    fn test_numbers() {
        assert_value_eq!(Value::Number(1.into()), "1");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: 125.into(),
                             decimal: 2,
                         }),
                         "1.25");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: 12500.into(),
                             decimal: 0,
                         }),
                         "1.25e4");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: (-12500).into(),
                             decimal: 0,
                         }),
                         "-1.25e4");

        assert_value_eq!(Value::Number(RpNumber {
                             digits: (1234).into(),
                             decimal: 8,
                         }),
                         "0.00001234");
    }

    #[test]
    fn test_type_spec() {
        let c = RpName {
            prefix: None,
            parts: vec!["Hello".to_owned(), "World".to_owned()],
        };

        assert_type_spec_eq!(RpType::String, "string");
        assert_type_spec_eq!(RpType::Name { name: c }, "Hello.World");
    }

    #[test]
    fn test_option_decl() {
        let member = parse_member("foo_bar_baz true, foo, \"bar\", 12;");

        if let Member::Option(option) = member {
            assert_eq!("foo_bar_baz", option.name);
            assert_eq!(4, option.values.len());

            assert_eq!(Value::Boolean(true), *option.values[0].as_ref());
            assert_eq!(Value::Identifier("foo"), *option.values[1].as_ref());
            assert_eq!(Value::String("bar".to_owned()), *option.values[2].as_ref());
            assert_eq!(Value::Number(12u32.into()), *option.values[3].as_ref());
            return;
        }

        panic!("option did not match");
    }
}
<|endoftext|>


<commit_before>//! The parser is a bit more complex than it needs to be as it needs to be fully specialized to
//! avoid a recompilation every time a later part of the compiler is changed. Due to this the
//! string interner and therefore also garbage collector needs to compiled before the parser.

#[macro_use]
extern crate log;
#[macro_use]
extern crate quick_error;
extern crate gluon_base as base;
extern crate combine;
extern crate combine_language;
extern crate lalrpop_util;

use std::error::Error as StdError;
use std::fmt;

use base::ast::{Expr, IdentEnv, SpannedExpr};
use base::error::Errors;
use base::pos::{self, BytePos, Spanned};
use base::symbol::Symbol;
use base::types::ArcType;

use combine::primitives::Error as CombineError;

use infix::{OpTable, Reparser};
use lexer::{Lexer, Token};

mod grammar;
mod infix;
mod lexer;

type LalrpopError<'input> = lalrpop_util::ParseError<BytePos,
                                                     Token<&'input str>,
                                                     CombineError<String, String>>;

/// Shrink hidden spans to fit the visible expressions and flatten singleton blocks.
fn shrink_hidden_spans<Id>(mut expr: SpannedExpr<Id>) -> SpannedExpr<Id> {
    match expr.value {
        Expr::IfElse(_, _, ref last) |
        Expr::LetBindings(_, ref last) |
        Expr::TypeBindings(_, ref last) => expr.span.end = last.span.end,
        Expr::Lambda(ref lambda) => expr.span.end = lambda.body.span.end,
        Expr::Block(ref mut exprs) => {
            match exprs.len() {
                0 => (),
                1 => return exprs.pop().unwrap(),
                _ => expr.span.end = exprs.last().unwrap().span.end,
            }
        }
        Expr::Match(_, ref alts) => {
            if let Some(last_alt) = alts.last() {
                let end = last_alt.expr.span.end;
                expr.span.end = end;
            }
        }
        Expr::App(_, _) |
        Expr::Ident(_) |
        Expr::Literal(_) |
        Expr::Projection(_, _, _) |
        Expr::Infix(_, _, _) |
        Expr::Array(_) |
        Expr::Record { .. } |
        Expr::Tuple(_) => (),
    }
    expr
}

#[derive(Debug, PartialEq)]
pub struct ParseError {
    pub errors: Vec<CombineError<String, String>>,
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        CombineError::fmt_errors(&self.errors, f)
    }
}

impl StdError for ParseError {
    fn description(&self) -> &str {
        "Parse error"
    }
}

fn transform_lalrpop_error(err: LalrpopError) -> Spanned<CombineError<String, String>, BytePos> {
    use lalrpop_util::ParseError::*;
    match err {
        InvalidToken { location } => {
            pos::spanned2(location,
                          location,
                          CombineError::Message("Invalid token".into()))
        }
        UnrecognizedToken { token, .. } => {
            match token {
                Some(token) => {
                    pos::spanned2(token.0,
                                  token.2,
                                  CombineError::Unexpected(format!("{}", token.1).into()))
                }
                None => {
                    pos::spanned2(0.into(),
                                  0.into(),
                                  CombineError::Unexpected("Unrecognized token".into()))
                }
            }
        }
        ExtraToken { token } => {
            pos::spanned2(token.0,
                          token.2,
                          CombineError::Message(format!("Found an extra token `{}`", token.1)
                              .into()))
        }
        User { error } => pos::spanned2(0.into(), 0.into(), error),
    }
}

fn transform_errors<'a, Iter>(errors: Iter) -> Errors<Spanned<Error, BytePos>>
    where Iter: IntoIterator<Item = LalrpopError<'a>>,
{
    errors.into_iter()
        .map(|err| {
            let Spanned { span, value: err } = transform_lalrpop_error(err);
            let err = ParseError { errors: vec![err] };
            pos::spanned(span, Error::Parser(err))
        })
        .collect()
}

quick_error! {
    #[derive(Debug, PartialEq)]
    pub enum Error {
        Parser(err: ParseError) {
            description(err.description())
            display("{}", err)
            from()
        }
        Infix(err: infix::ReparseError) {
            description(err.description())
            display("{}", err)
            from()
        }
    }
}

pub enum FieldPattern<Id> {
    Type(Id, Option<Id>),
    Value(Id, Option<Id>),
}

pub enum FieldExpr<Id> {
    Type(Id, Option<ArcType<Id>>),
    Value(Id, Option<SpannedExpr<Id>>),
}

// Hack around LALRPOP's limited type syntax
type MutIdentEnv<'env, Id> = &'env mut IdentEnv<Ident = Id>;
type ErrorEnv<'err, 'input> = &'err mut Errors<lalrpop_util::ParseError<BytePos,
                                                                        Token<&'input str>,
                                                                        CombineError<String,
                                                                                String>>>;
pub type ParseErrors = Errors<Spanned<Error, BytePos>>;

pub fn parse_partial_expr<Id>(symbols: &mut IdentEnv<Ident = Id>,
                              input: &str)
                              -> Result<SpannedExpr<Id>, (Option<SpannedExpr<Id>>, ParseErrors)>
    where Id: Clone,
{
    let lexer = Lexer::new(input);
    let mut parse_errors = Errors::new();

    match grammar::parse_TopExpr(input, symbols, &mut parse_errors, lexer) {
        Ok(mut expr) => {
            let mut errors = transform_errors(parse_errors);
            let mut reparser = Reparser::new(OpTable::default(), symbols);
            if let Err(reparse_errors) = reparser.reparse(&mut expr) {
                errors.extend(reparse_errors.into_iter()
                        .map(|err| {
                            pos::spanned2(BytePos::from(0), BytePos::from(0), Error::Infix(err))
                        }));
            }
            if errors.has_errors() {
                Err((Some(expr), errors))
            } else {
                Ok(expr)
            }
        }
        Err(err) => {
            parse_errors.push(err);
            Err((None, transform_errors(parse_errors)))
        }
    }
}

pub fn parse_expr(symbols: &mut IdentEnv<Ident = Symbol>,
                  input: &str)
                  -> Result<SpannedExpr<Symbol>, ParseErrors> {
    parse_partial_expr(symbols, input).map_err(|t| t.1)
}

#[cfg(feature = "test")]
pub fn parse_string<'env, 'input>
    (symbols: &'env mut IdentEnv<Ident = String>,
     input: &'input str)
     -> Result<SpannedExpr<String>, (Option<SpannedExpr<String>>, ParseErrors)> {
    parse_partial_expr(symbols, input)
}
<commit_msg>Flatten some patterns<commit_after>//! The parser is a bit more complex than it needs to be as it needs to be fully specialized to
//! avoid a recompilation every time a later part of the compiler is changed. Due to this the
//! string interner and therefore also garbage collector needs to compiled before the parser.

#[macro_use]
extern crate log;
#[macro_use]
extern crate quick_error;
extern crate gluon_base as base;
extern crate combine;
extern crate combine_language;
extern crate lalrpop_util;

use std::error::Error as StdError;
use std::fmt;

use base::ast::{Expr, IdentEnv, SpannedExpr};
use base::error::Errors;
use base::pos::{self, BytePos, Spanned};
use base::symbol::Symbol;
use base::types::ArcType;

use combine::primitives::Error as CombineError;

use infix::{OpTable, Reparser};
use lexer::{Lexer, Token};

mod grammar;
mod infix;
mod lexer;

type LalrpopError<'input> = lalrpop_util::ParseError<BytePos,
                                                     Token<&'input str>,
                                                     CombineError<String, String>>;

/// Shrink hidden spans to fit the visible expressions and flatten singleton blocks.
fn shrink_hidden_spans<Id>(mut expr: SpannedExpr<Id>) -> SpannedExpr<Id> {
    match expr.value {
        Expr::IfElse(_, _, ref last) |
        Expr::LetBindings(_, ref last) |
        Expr::TypeBindings(_, ref last) => expr.span.end = last.span.end,
        Expr::Lambda(ref lambda) => expr.span.end = lambda.body.span.end,
        Expr::Block(ref mut exprs) => {
            match exprs.len() {
                0 => (),
                1 => return exprs.pop().unwrap(),
                _ => expr.span.end = exprs.last().unwrap().span.end,
            }
        }
        Expr::Match(_, ref alts) => {
            if let Some(last_alt) = alts.last() {
                let end = last_alt.expr.span.end;
                expr.span.end = end;
            }
        }
        Expr::App(_, _) |
        Expr::Ident(_) |
        Expr::Literal(_) |
        Expr::Projection(_, _, _) |
        Expr::Infix(_, _, _) |
        Expr::Array(_) |
        Expr::Record { .. } |
        Expr::Tuple(_) => (),
    }
    expr
}

#[derive(Debug, PartialEq)]
pub struct ParseError {
    pub errors: Vec<CombineError<String, String>>,
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        CombineError::fmt_errors(&self.errors, f)
    }
}

impl StdError for ParseError {
    fn description(&self) -> &str {
        "Parse error"
    }
}

fn transform_lalrpop_error(err: LalrpopError) -> Spanned<CombineError<String, String>, BytePos> {
    use lalrpop_util::ParseError::*;

    match err {
        InvalidToken { location } => {
            pos::spanned2(location,
                          location,
                          CombineError::Message("Invalid token".into()))
        }
        UnrecognizedToken { token: Some((lpos, token, rpos)), .. } => {
            pos::spanned2(lpos,
                          rpos,
                          CombineError::Unexpected(format!("{}", token).into()))
        }
        UnrecognizedToken { token: None, .. } => {
            pos::spanned2(0.into(),
                          0.into(),
                          CombineError::Unexpected("Unrecognized token".into()))
        }
        ExtraToken { token: (lpos, token, rpos) } => {
            pos::spanned2(lpos,
                          rpos,
                          CombineError::Message(format!("Found an extra token `{}`", token).into()))
        }
        User { error } => pos::spanned2(0.into(), 0.into(), error),
    }
}

fn transform_errors<'a, Iter>(errors: Iter) -> Errors<Spanned<Error, BytePos>>
    where Iter: IntoIterator<Item = LalrpopError<'a>>,
{
    errors.into_iter()
        .map(|err| {
            let Spanned { span, value: err } = transform_lalrpop_error(err);
            let err = ParseError { errors: vec![err] };
            pos::spanned(span, Error::Parser(err))
        })
        .collect()
}

quick_error! {
    #[derive(Debug, PartialEq)]
    pub enum Error {
        Parser(err: ParseError) {
            description(err.description())
            display("{}", err)
            from()
        }
        Infix(err: infix::ReparseError) {
            description(err.description())
            display("{}", err)
            from()
        }
    }
}

pub enum FieldPattern<Id> {
    Type(Id, Option<Id>),
    Value(Id, Option<Id>),
}

pub enum FieldExpr<Id> {
    Type(Id, Option<ArcType<Id>>),
    Value(Id, Option<SpannedExpr<Id>>),
}

// Hack around LALRPOP's limited type syntax
type MutIdentEnv<'env, Id> = &'env mut IdentEnv<Ident = Id>;
type ErrorEnv<'err, 'input> = &'err mut Errors<lalrpop_util::ParseError<BytePos,
                                                                        Token<&'input str>,
                                                                        CombineError<String,
                                                                                String>>>;
pub type ParseErrors = Errors<Spanned<Error, BytePos>>;

pub fn parse_partial_expr<Id>(symbols: &mut IdentEnv<Ident = Id>,
                              input: &str)
                              -> Result<SpannedExpr<Id>, (Option<SpannedExpr<Id>>, ParseErrors)>
    where Id: Clone,
{
    let lexer = Lexer::new(input);
    let mut parse_errors = Errors::new();

    match grammar::parse_TopExpr(input, symbols, &mut parse_errors, lexer) {
        Ok(mut expr) => {
            let mut errors = transform_errors(parse_errors);
            let mut reparser = Reparser::new(OpTable::default(), symbols);
            if let Err(reparse_errors) = reparser.reparse(&mut expr) {
                errors.extend(reparse_errors.into_iter()
                        .map(|err| {
                            pos::spanned2(BytePos::from(0), BytePos::from(0), Error::Infix(err))
                        }));
            }
            if errors.has_errors() {
                Err((Some(expr), errors))
            } else {
                Ok(expr)
            }
        }
        Err(err) => {
            parse_errors.push(err);
            Err((None, transform_errors(parse_errors)))
        }
    }
}

pub fn parse_expr(symbols: &mut IdentEnv<Ident = Symbol>,
                  input: &str)
                  -> Result<SpannedExpr<Symbol>, ParseErrors> {
    parse_partial_expr(symbols, input).map_err(|t| t.1)
}

#[cfg(feature = "test")]
pub fn parse_string<'env, 'input>
    (symbols: &'env mut IdentEnv<Ident = String>,
     input: &'input str)
     -> Result<SpannedExpr<String>, (Option<SpannedExpr<String>>, ParseErrors)> {
    parse_partial_expr(symbols, input)
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate nom;

pub mod tree;
pub mod parser;

use nom::multispace;

use parser::escape_c_string;

named!(pub parse_include<String>, preceded!(
    tag!("/include/"),
    preceded!( multispace,
        delimited!(
            char!('"'),
            escape_c_string,
            char!('"')
        ))
));
<commit_msg>Implement bounds tracking in device_tree_source<commit_after>#[macro_use]
extern crate nom;

pub mod tree;
pub mod parser;

use std::str::{self, FromStr};
use std::path::{PathBuf, Path};
use std::fs::File;
use std::io::Read;
use std::cmp::Ordering;
use std::borrow::Borrow;

use nom::{IResult, ErrorKind, Needed, FindSubstring, digit, space, multispace, line_ending};

use parser::escape_c_string;

named!(pub parse_include<String>, preceded!(
    tag!("/include/"),
    preceded!( multispace,
        delimited!(
            char!('"'),
            escape_c_string,
            char!('"')
        ))
));

named!(pub find_include<(&[u8], String)>, do_parse!(
    pre: take_until!("/include/") >>
    path: parse_include >>
    (pre, path)
));

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IncludeBounds {
    pub path: PathBuf,
    pub global_start: usize,
    pub child_start: usize,
    pub len: usize,
    pub method: IncludeMethod,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IncludeMethod {
    DTS,
    CPP,
}

impl PartialOrd for IncludeBounds {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for IncludeBounds {
    fn cmp(&self, other: &Self) -> Ordering {
        self.global_start.cmp(&other.global_start)
    }
}

impl IncludeBounds {
    pub fn global_end(&self) -> usize {
        self.global_start + self.len
    }

    pub fn split_bounds(bounds: &mut Vec<IncludeBounds>, start: usize, end: usize, offset: usize) {
        let mut remainders: Vec<IncludeBounds> = Vec::new();

        // println!("split s: {} e: {} off: {}", start, end, offset);

        for b in bounds.iter_mut() {
            // println!("g_start: {} g_end: {}", b.global_start, b.global_end());
            if b.global_start < start && b.global_end() >= start {
                // global_start -- start -- global_end
                let remainder = b.global_end() - start;

                // println!("remainder: {}", remainder);

                remainders.push(IncludeBounds {
                    path: b.path.clone(),
                    global_start: end,
                    child_start: b.child_start + start - b.global_start + offset,
                    len: remainder, // - offset,
                    method: b.method.clone(),
                });

                b.len = start - b.global_start;
            } else if b.global_start == start {
                // split is at begining of the bound
                // offset the start
                {
                    let offset = end - start;
                    b.global_start += offset;
                }
                // shrink the len by the offset
                b.len -= offset;
            }
        }

        bounds.extend_from_slice(&remainders);
        bounds.sort();
    }
}

#[derive(Debug, PartialEq)]
pub struct Linemarker {
    child_line: usize,
    path: PathBuf,
    flag: Option<LinemarkerFlag>,
}

#[derive(Debug, PartialEq)]
pub enum LinemarkerFlag {
    Start,
    Return,
    System,
    Extern,
}

named!(pub parse_linemarker<Linemarker>,
    complete!(do_parse!(
        tag!("#") >>
        opt!(tag!("line")) >>
        space >>
        line: map_res!(map_res!(digit, str::from_utf8), usize::from_str) >>
        space >>
        path: delimited!(
            char!('"'),
            map!(escape_c_string, PathBuf::from),
            char!('"')
        ) >>
        flag: opt!(preceded!(space, map_res!(map_res!(digit, str::from_utf8), u64::from_str))) >>
        line_ending >>
        (Linemarker {
            child_line: line,
            path: path,
            flag: flag.map(|f| match f {
                1 => LinemarkerFlag::Start,
                2 => LinemarkerFlag::Return,
                3 => LinemarkerFlag::System,
                4 => LinemarkerFlag::Extern,
                _ => unreachable!(),
            }),
        })
    ))
);

fn find_linemarker_start(input: &[u8]) -> IResult<&[u8], &[u8]> {
    if "# ".len() > input.len() {
        IResult::Incomplete(Needed::Size("# ".len()))
    } else {
        match input.find_substring("# ").iter().chain(input.find_substring("#line ").iter()).min() {
            None => {
                IResult::Error(error_position!(ErrorKind::TakeUntil, input))
            },
            Some(index) => {
                IResult::Done(&input[*index..], &input[0..*index])
            },
        }
    }
}

named!(find_linemarker<(&[u8], Linemarker)>, do_parse!(
    pre: find_linemarker_start >>
    marker: parse_linemarker >>
    (pre, marker)
));

fn parse_linemarkers(buf: &[u8], bounds: &mut Vec<IncludeBounds>, global_offset: usize) {
    let end_offset = global_offset + buf.len();
    // println!("{}", str::from_utf8(buf).unwrap());
    let mut buf = buf;
    println!("parsing linemarkers");
    loop {
        // look for linemarker
        if let IResult::Done(rem, (pre, marker)) = find_linemarker(buf) {
            // println!("{}", str::from_utf8(line).unwrap());
            // println!("{:?}", marker);
            // println!("pre.len() {}", pre.len());

            // double check that last bound was from a linemarker
            match bounds.last() {
                Some(&IncludeBounds { method: IncludeMethod::CPP, .. }) => {}
                _ => {
                    println!("{:#?}", bounds);
                    panic!("Linemarker found within DTS include")
                }
            }

            // end last
            bounds.last_mut().unwrap().len = pre.len();

            // start at new line
            let new_bound = IncludeBounds {
                path: marker.path.clone(),
                global_start: end_offset - rem.len(),
                child_start: File::open(&marker.path)
                                .map(|f| f.bytes().filter_map(|e| e.ok()))
                                .map(|b| line_to_byte_offset(b, marker.child_line).unwrap()) //TODO: unwraping is bad, SOK?
                                .unwrap_or(0),
                len: rem.len(),
                method: IncludeMethod::CPP,
            };

            bounds.push(new_bound);

            buf = rem;
        } else {
            return;
        }
    }
}

pub fn include_files(path: &Path,
                     main_offset: usize)
                     -> Result<(Vec<u8>, Vec<IncludeBounds>), String> {
    // TODO: check from parent directory of root file
    let mut file = File::open(path).unwrap();
    let mut buffer: Vec<u8> = Vec::new();
    let mut bounds: Vec<IncludeBounds> = Vec::new();

    let mut string_buffer = String::new();
    file.read_to_string(&mut string_buffer).map_err(|_| "IO Error".to_string())?;

    let mut buf = string_buffer.as_bytes();

    named!(first_linemarker<(&[u8], Linemarker)>,
        do_parse!(
            marker: peek!(parse_linemarker) >>
            line: recognize!(parse_linemarker) >>
            (line, marker)
        )
    );

    let start_bound = if let IResult::Done(rem, (line, marker)) = first_linemarker(buf) {
        let bound = IncludeBounds {
            path: marker.path.clone(),
            global_start: buf.len() - rem.len(),
            // TODO: check from parent directory of root file
            child_start: File::open(&marker.path)
                             .map(|f| f.bytes().filter_map(|e| e.ok()))
                             .map(|b| line_to_byte_offset(b, marker.child_line).unwrap()) //TODO: unwraping is bad, SOK?
                             .unwrap_or(0),
            len: File::open(&marker.path).unwrap().bytes().count(),
            method: IncludeMethod::CPP,
        };

        buffer.extend_from_slice(line);
        buf = rem;

        bound
    } else {
        // println!("main_offset {}", main_offset);
        IncludeBounds {
            path: path.to_path_buf(),
            global_start: main_offset,
            child_start: 0,
            // TODO: check from parent directory of root file
            len: File::open(path).unwrap().bytes().count(),
            method: IncludeMethod::DTS,
        }
    };
    bounds.push(start_bound);

    loop {
        // go until /include/
        if let IResult::Done(rem, (pre, file)) = find_include(&buf[..]) {
            parse_linemarkers(pre, &mut bounds, buffer.len());
            buffer.extend_from_slice(pre);

            let offset = pre.len();
            // println!("{}", file);
            // println!("Offset: {}", offset);
            // println!("{}", include_tree);

            let included_path = Path::new(&file);
            let total_len = buffer.len() + main_offset; // - 1;
            let (sub_buf, sub_bounds) = include_files(included_path, total_len)?;
            buffer.extend(sub_buf);

            let inc_start = sub_bounds.first()
                                      .map(|b| b.global_start)
                                      .expect(&format!("No bounds returned: {}",
                                                      included_path.to_string_lossy()));
            let inc_end = sub_bounds.last()
                                    .map(|b| b.global_end())
                                    .expect(&format!("No bounds returned: {}",
                                                    included_path.to_string_lossy()));
            let eaten_len = (buf.len() - offset) - rem.len();
            //include_tree.offset_after_location(inc_start,
            //                                   inc_end as isize - inc_start as isize -
            //                                   eaten_len as isize);
            // println!("After offset");
            // println!("{}", include_tree);
            IncludeBounds::split_bounds(&mut bounds, inc_start, inc_end, eaten_len);
            bounds.extend_from_slice(&sub_bounds);
            bounds.sort();

            // println!("After split");
            // println!("{}", include_tree);

            buf = rem;
        } else {
            parse_linemarkers(buf, &mut bounds, buffer.len());
            // no more includes, just add the rest and return
            buffer.extend(buf);
            return Ok((buffer, bounds));
        }
    }
}

pub fn line_to_byte_offset<K, I>(bytes: I, line: usize) -> Result<usize, String>
    where K: Borrow<u8> + Eq,
          I: Iterator<Item = K>
{
    if line == 1 {
        Ok(0)
    } else {
        bytes.enumerate()
            .filter(|&(_, ref byte)| byte.borrow() == &b'\n')
            .nth(line - 2)
            .map(|(offset, _)| offset)
            .ok_or_else(|| "Failed converting from line to byte offset".to_string())
    }
}

pub fn byte_offset_to_line_col<K, I>(bytes: I, offset: usize) -> (usize, usize)
    where K: Borrow<u8> + Eq,
          I: Iterator<Item = K>
{
    let opt = bytes.enumerate()
        .filter(|&(off, ref byte)| off <= offset && byte.borrow() == &b'\n')
        .map(|(start, _)| start)
        .enumerate()
        .last();

    match opt {
        Some((line, start)) => (line + 2, offset - start),
        None => (1, offset + 1),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use nom::IResult;

    #[test]
    fn lines_to_bytes() {
        let string = "Howdy\nHow goes it\nI'm doing fine\n";
        assert_eq!(line_to_byte_offset(string.as_bytes().iter(), 1).unwrap(),
                   0);
        assert_eq!(line_to_byte_offset(string.as_bytes().iter(), 2).unwrap(),
                   5);
        assert_eq!(line_to_byte_offset(string.as_bytes().iter(), 3).unwrap(),
                   17);
    }

    #[test]
    fn bytes_to_lines() {
        let string = "Howdy\nHow goes it\nI'm doing fine\n";
        assert_eq!(byte_offset_to_line_col(string.as_bytes().iter(), 0),
                   (1, 1));
        assert_eq!(byte_offset_to_line_col(string.as_bytes().iter(), 8),
                   (2, 3));
        assert_eq!(byte_offset_to_line_col(string.as_bytes().iter(), 20),
                   (3, 3));
    }

    #[test]
    fn linemarker_no_flag() {
        let input = b"# 1 \"<built-in>\"\n";
        assert_eq!(
            parse_linemarker(input),
            IResult::Done(
                &b""[..],
                Linemarker {
                    child_line: 1,
                    path: PathBuf::from("<built-in>"),
                    flag: None,
                }
            )
        );
    }

    #[test]
    fn linemarker_flag() {
        let input = b"# 12 \"am33xx.dtsi\" 2\n";
        assert_eq!(
            parse_linemarker(input),
            IResult::Done(
                &b""[..],
                Linemarker {
                    child_line: 12,
                    path: PathBuf::from("am33xx.dtsi"),
                    flag: Some(LinemarkerFlag::Return),
                }
            )
        );
    }
}
<|endoftext|>


<commit_before>use std::fmt::Display;

/// Common result type
pub type CargoResult<T> = anyhow::Result<T>;

/// Common error type
pub type Error = anyhow::Error;

pub use anyhow::Context;

/// CLI-specific result
pub type CliResult = Result<(), CliError>;

#[derive(Debug)]
/// The CLI error is the error type used at Cargo's CLI-layer.
///
/// All errors from the lib side of Cargo will get wrapped with this error.
/// Other errors (such as command-line argument validation) will create this
/// directly.
pub struct CliError {
    /// The error to display. This can be `None` in rare cases to exit with a
    /// code without displaying a message. For example `cargo run -q` where
    /// the resulting process exits with a nonzero code (on Windows), or an
    /// external subcommand that exits nonzero (we assume it printed its own
    /// message).
    pub error: Option<anyhow::Error>,
    /// The process exit code.
    pub exit_code: i32,
}

impl CliError {
    /// Attach an error code to an error
    pub fn new(error: anyhow::Error, code: i32) -> CliError {
        CliError {
            error: Some(error),
            exit_code: code,
        }
    }

    /// Silent error
    pub fn code(code: i32) -> CliError {
        CliError {
            error: None,
            exit_code: code,
        }
    }
}

impl From<anyhow::Error> for CliError {
    fn from(err: anyhow::Error) -> CliError {
        CliError::new(err, 101)
    }
}

impl From<clap::Error> for CliError {
    fn from(err: clap::Error) -> CliError {
        let code = if err.use_stderr() { 1 } else { 0 };
        CliError::new(err.into(), code)
    }
}

impl From<std::io::Error> for CliError {
    fn from(err: std::io::Error) -> CliError {
        CliError::new(err.into(), 1)
    }
}

pub(crate) fn no_crate_err(name: impl Display) -> Error {
    anyhow::format_err!("The crate `{}` could not be found in registry index.", name)
}

pub(crate) fn non_existent_table_err(table: impl Display) -> Error {
    anyhow::format_err!("The table `{}` could not be found.", table)
}

pub(crate) fn non_existent_dependency_err(name: impl Display, table: impl Display) -> Error {
    anyhow::format_err!(
        "The dependency `{}` could not be found in `{}`.",
        name,
        table,
    )
}

pub(crate) fn invalid_cargo_config() -> Error {
    anyhow::format_err!("Invalid cargo config")
}

pub(crate) fn unsupported_version_req(req: impl Display) -> Error {
    anyhow::format_err!("Support for modifying {} is currently unsupported", req)
}

pub(crate) fn invalid_release_level(actual: impl Display, version: impl Display) -> Error {
    anyhow::format_err!("Cannot increment the {} field for {}", actual, version)
}

pub(crate) fn parse_version_err(version: impl Display, dep: impl Display) -> Error {
    anyhow::format_err!(
        "The version `{}` for the dependency `{}` couldn't be parsed",
        version,
        dep
    )
}
<commit_msg>fix: Don't mix up version parse order<commit_after>use std::fmt::Display;

/// Common result type
pub type CargoResult<T> = anyhow::Result<T>;

/// Common error type
pub type Error = anyhow::Error;

pub use anyhow::Context;

/// CLI-specific result
pub type CliResult = Result<(), CliError>;

#[derive(Debug)]
/// The CLI error is the error type used at Cargo's CLI-layer.
///
/// All errors from the lib side of Cargo will get wrapped with this error.
/// Other errors (such as command-line argument validation) will create this
/// directly.
pub struct CliError {
    /// The error to display. This can be `None` in rare cases to exit with a
    /// code without displaying a message. For example `cargo run -q` where
    /// the resulting process exits with a nonzero code (on Windows), or an
    /// external subcommand that exits nonzero (we assume it printed its own
    /// message).
    pub error: Option<anyhow::Error>,
    /// The process exit code.
    pub exit_code: i32,
}

impl CliError {
    /// Attach an error code to an error
    pub fn new(error: anyhow::Error, code: i32) -> CliError {
        CliError {
            error: Some(error),
            exit_code: code,
        }
    }

    /// Silent error
    pub fn code(code: i32) -> CliError {
        CliError {
            error: None,
            exit_code: code,
        }
    }
}

impl From<anyhow::Error> for CliError {
    fn from(err: anyhow::Error) -> CliError {
        CliError::new(err, 101)
    }
}

impl From<clap::Error> for CliError {
    fn from(err: clap::Error) -> CliError {
        let code = if err.use_stderr() { 1 } else { 0 };
        CliError::new(err.into(), code)
    }
}

impl From<std::io::Error> for CliError {
    fn from(err: std::io::Error) -> CliError {
        CliError::new(err.into(), 1)
    }
}

pub(crate) fn no_crate_err(name: impl Display) -> Error {
    anyhow::format_err!("The crate `{}` could not be found in registry index.", name)
}

pub(crate) fn non_existent_table_err(table: impl Display) -> Error {
    anyhow::format_err!("The table `{}` could not be found.", table)
}

pub(crate) fn non_existent_dependency_err(name: impl Display, table: impl Display) -> Error {
    anyhow::format_err!(
        "The dependency `{}` could not be found in `{}`.",
        name,
        table,
    )
}

pub(crate) fn invalid_cargo_config() -> Error {
    anyhow::format_err!("Invalid cargo config")
}

pub(crate) fn unsupported_version_req(req: impl Display) -> Error {
    anyhow::format_err!("Support for modifying {} is currently unsupported", req)
}

pub(crate) fn invalid_release_level(actual: impl Display, version: impl Display) -> Error {
    anyhow::format_err!("Cannot increment the {} field for {}", actual, version)
}

pub(crate) fn parse_version_err(dep: impl Display, version: impl Display) -> Error {
    anyhow::format_err!(
        "The version `{}` for the dependency `{}` couldn't be parsed",
        version,
        dep
    )
}
<|endoftext|>