<commit_before>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! `x86_64` execution contexts.
//!
//! This is inteded to be general-purpose and composable, so that the same
//! code can be reused for interrupts and for multithreading.

use core::mem;
use core::fmt;
use super::flags::Flags;
use super::segment;

/// Registers pushed to the stack when handling an interrupt or context switch.
#[repr(C, packed)]
#[derive(Copy, Clone)]
pub struct Registers { pub rsi: u64
                     , pub rdi: u64
                     , pub r11: u64
                     , pub r10: u64
                     , pub r9:  u64
                     , pub r8:  u64
                     , pub rdx: u64
                     , pub rcx: u64
                     , pub rax: u64
                     }

 impl Registers {
     /// Transform this struct into an array of `u64`s
     /// (if you would ever want to do this)
     pub unsafe fn to_array(&self) -> [u64; 9] {
        //  [ self.rsi, self.rdi, self.r11
        //  , self.r10, self.r9, self.r8
        //  , self.rdx, self.rcx, self.rax
        //  ]
        // using transmute is probably faster and we're already unsafe...
        mem::transmute(*self)
     }

     /// Create a new empty set of Registers
     pub const fn empty() -> Self {
         Registers { rsi: 0, rdi: 0, r11: 0
                   , r10: 0, r9:  0, r8:  0
                   , rdx: 0, rcx: 0, rax: 0
                   }
     }

     /// Push the caller-saved registers to the stack
     /// (such as when handling a context switch or interrupt).
     ///
     /// THIS FUNCTION IS NAKED. DO NOT CALL IT NORMALLY.
     #[naked]
     #[inline(always)]
     pub unsafe fn push() {
         asm!( "push rax
                push rcx
                push rdx
                push r8
                push r9
                push r10
                push r11
                push rdi
                push rsi"
             :::: "intel"
                , "volatile");
     }

     /// Push the caller-saved registers off the stack
     /// (such as when handling a context switch or interrupt).
     ///
     /// THIS FUNCTION IS NAKED. DO NOT CALL IT NORMALLY.
     #[naked]
     #[inline(always)]
     pub unsafe fn pop() {
         asm!( "pop rsi
                pop rdi
                pop r11
                pop r10
                pop r9
                pop r8
                pop rdx
                pop rcx
                pop rax"
             :::: "intel"
                , "volatile");
     }
 }

impl fmt::Debug for Registers {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!( f
               , "    RSI: {:#018x} RDI: {:#018x} R11: {:#018x}\n    \
                      R10: {:#018x} R9:  {:#018x} R8:  {:#018x}\n    \
                      RDX: {:#018x} RCX: {:#018x} RAX: {:#018x}"
              , self.rsi, self.rdi, self.r11
              , self.r10, self.r9,  self.r8
              , self.rdx, self.rcx, self.rax)
     }
}

#[repr(C)]
pub struct InterruptFrame {
    //  this is the actual value of the interrupt stack frame context,
    //  not the old one (which is wrong). note that the old one seems to cause
    //  stack misalignment.
    //          -- eliza, october 4th, 2016
    /// Value of the instruction pointer (`$rip`) register
    pub rip: u64
  // , __pad_1: u32
  // , __pad_2: u16
  , /// Value of the code segment (`$cs`) register
    pub cs: u64
  , /// Value of the CPU flags (`$rflags`) register
    pub rflags: u64
  , /// Value of the stack pointer (`$rsp`) register
    //  TODO: should this actually be a pointer?
    pub rsp: u64
  // , __pad_3: u32
  // , __pad_4: u16
  , /// Value of the stack segment (`$ss`) register
    pub ss: u64
}

impl fmt::Debug for InterruptFrame {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!( f
              , "Interrupt Frame: \
                \n   instruction pointer: {:#018x} \
                \n   code segment selector: {:#018x} \
                \n   rflags: {:#018x} \
                \n   stack pointer: {:#018x} \
                \n   stack segment selector: {:#018x}"
             , self.rip, self.cs, self.rflags, self.rsp, self.ss)
    }
}

/// Thread execution context
#[repr(C, packed)]
pub struct Context { /// Value of the stack pointer (`rsp`) register
                     pub rsp: *mut u8
                   , /// Value of the caller-saved registers
                     pub registers: Registers
                   , /// Value of the instruction pointer (`rip`) register
                     pub rip: *mut u8
                 //, pub stack: [u8] // TODO: should be box
                   }

impl Context {
    pub fn empty() -> Self {
        unsafe {
            Context { rsp: mem::transmute(0u64)
                    , registers: Registers::empty()
                    , rip: mem::transmute(0u64)
                  //, stack: [0u8; 8]
                    }
        }
    }
}
<commit_msg>[x86_64] fix struct layout issues in InterruptFrame<commit_after>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! `x86_64` execution contexts.
//!
//! This is inteded to be general-purpose and composable, so that the same
//! code can be reused for interrupts and for multithreading.

use core::mem;
use core::fmt;
use super::flags::Flags;
use super::segment;

/// Registers pushed to the stack when handling an interrupt or context switch.
#[repr(C, packed)]
#[derive(Copy, Clone)]
pub struct Registers { pub rsi: u64
                     , pub rdi: u64
                     , pub r11: u64
                     , pub r10: u64
                     , pub r9:  u64
                     , pub r8:  u64
                     , pub rdx: u64
                     , pub rcx: u64
                     , pub rax: u64
                     }

 impl Registers {
     /// Transform this struct into an array of `u64`s
     /// (if you would ever want to do this)
     pub unsafe fn to_array(&self) -> [u64; 9] {
        //  [ self.rsi, self.rdi, self.r11
        //  , self.r10, self.r9, self.r8
        //  , self.rdx, self.rcx, self.rax
        //  ]
        // using transmute is probably faster and we're already unsafe...
        mem::transmute(*self)
     }

     /// Create a new empty set of Registers
     pub const fn empty() -> Self {
         Registers { rsi: 0, rdi: 0, r11: 0
                   , r10: 0, r9:  0, r8:  0
                   , rdx: 0, rcx: 0, rax: 0
                   }
     }

     /// Push the caller-saved registers to the stack
     /// (such as when handling a context switch or interrupt).
     ///
     /// THIS FUNCTION IS NAKED. DO NOT CALL IT NORMALLY.
     #[naked]
     #[inline(always)]
     pub unsafe fn push() {
         asm!( "push rax
                push rcx
                push rdx
                push r8
                push r9
                push r10
                push r11
                push rdi
                push rsi"
             :::: "intel"
                , "volatile");
     }

     /// Push the caller-saved registers off the stack
     /// (such as when handling a context switch or interrupt).
     ///
     /// THIS FUNCTION IS NAKED. DO NOT CALL IT NORMALLY.
     #[naked]
     #[inline(always)]
     pub unsafe fn pop() {
         asm!( "pop rsi
                pop rdi
                pop r11
                pop r10
                pop r9
                pop r8
                pop rdx
                pop rcx
                pop rax"
             :::: "intel"
                , "volatile");
     }
 }

impl fmt::Debug for Registers {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!( f
               , "    RSI: {:#018x} RDI: {:#018x} R11: {:#018x}\n    \
                      R10: {:#018x} R9:  {:#018x} R8:  {:#018x}\n    \
                      RDX: {:#018x} RCX: {:#018x} RAX: {:#018x}"
              , self.rsi, self.rdi, self.r11
              , self.r10, self.r9,  self.r8
              , self.rdx, self.rcx, self.rax)
     }
}

#[repr(C, packed)]
pub struct InterruptFrame {
    //  this is the actual value of the interrupt stack frame context,
    //  not the old one (which is wrong). note that the old one seems to cause
    //  stack misalignment.
    //          -- eliza, october 4th, 2016
    /// Value of the instruction pointer (`$rip`) register
    pub rip: u64
  , /// Value of the code segment (`$cs`) register
    pub cs: segment::Selector
  , __pad_1: u32
  , __pad_2: u16
  , /// Value of the CPU flags (`$rflags`) register
    pub rflags: u64
  , /// Value of the stack pointer (`$rsp`) register
    //  TODO: should this actually be a pointer?
    pub rsp: u64
  , /// Value of the stack segment (`$ss`) register
    pub ss: segment::Selector
  , __pad_3: u32
  , __pad_4: u16
}

#[cfg(test)]
mod test {
    #[test]
    fn test_interrupt_frame_correct_size() {
        use core::mem::size_of;
        use super::InterruptFrame;

        assert_eq!(size_of::<InterruptFrame>(), 32);
    }
}

impl fmt::Debug for InterruptFrame {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!( f
              , "Interrupt Frame: \
                \n   instruction pointer: {:#018x} \
                \n   code segment: {} \
                \n   rflags: {:#018x} \
                \n   stack pointer: {:#018x} \
                \n   stack segment: {}"
             , self.rip
            //  , self.__pad_1, self.__pad_2
             , self.cs
             , self.rflags
             , self.rsp
            //  , self.__pad_3, self.__pad_4
             , self.ss)
    }
}

/// Thread execution context
#[repr(C, packed)]
pub struct Context { /// Value of the stack pointer (`rsp`) register
                     pub rsp: *mut u8
                   , /// Value of the caller-saved registers
                     pub registers: Registers
                   , /// Value of the instruction pointer (`rip`) register
                     pub rip: *mut u8
                 //, pub stack: [u8] // TODO: should be box
                   }

impl Context {
    pub fn empty() -> Self {
        unsafe {
            Context { rsp: mem::transmute(0u64)
                    , registers: Registers::empty()
                    , rip: mem::transmute(0u64)
                  //, stack: [0u8; 8]
                    }
        }
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Handles invoking external binaries.
// This module assumes that, for a given machine, there is only one place
// where the desired executable might be installed. It expects the engine
// to identify that place at its initialization by invoking verify_binaries(),
// and to exit immediately if verify_binaries() return an error. If this
// protocol is followed then when any command is executed the unique absolute
// path of the binary for this machine will already have been identified.
// However stratisd may run for a while and it is possible for the binary
// to be caused to be uninstalled while stratisd is being run. Therefore,
// the existence of the file is checked before the command is invoked, and
// an explicit error is returned if the executable can not be found.

use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::process::Command;

use uuid::Uuid;

use stratis::{StratisError, StratisResult};

/// Find the binary with the given name by looking in likely locations.
/// Return None if no binary was found.
/// Search an explicit list of directories rather than the user's PATH
/// environment variable. stratisd may be running when there is no PATH
/// variable set.
fn find_binary(name: &str) -> Option<PathBuf> {
    ["/usr/sbin", "/sbin", "/usr/bin", "/bin"]
        .iter()
        .map(|pre| [pre, name].iter().collect::<PathBuf>())
        .find(|path| path.exists())
}

// These are the external binaries that stratisd relies on.
// Any change in this list requires a corresponding change to BINARIES,
// and vice-versa.
const MKFS_XFS: &str = "mkfs.xfs";
const THIN_CHECK: &str = "thin_check";
const THIN_REPAIR: &str = "thin_repair";
const XFS_ADMIN: &str = "xfs_admin";
const XFS_GROWFS: &str = "xfs_growfs";

lazy_static! {
    static ref BINARIES: HashMap<String, Option<PathBuf>> = [
        (MKFS_XFS.to_string(), find_binary(MKFS_XFS)),
        (THIN_CHECK.to_string(), find_binary(THIN_CHECK)),
        (THIN_REPAIR.to_string(), find_binary(THIN_REPAIR)),
        (XFS_ADMIN.to_string(), find_binary(XFS_ADMIN)),
        (XFS_GROWFS.to_string(), find_binary(XFS_GROWFS))
    ].iter()
        .cloned()
        .collect();
}

/// Verify that all binaries that the engine might invoke are available at some
/// path. Return an error if any are missing. Required to be called on engine
/// initialization.
pub fn verify_binaries() -> StratisResult<()> {
    match BINARIES.iter().find(|(_, ref path)| path.is_none()) {
        None => Ok(()),
        Some((ref name, _)) => Err(StratisError::Error(format!(
            "Unable to find absolute path for \"{}\"",
            name
        ))),
    }
}

/// Invoke the specified command. Return an error if invoking the command
/// fails or if the command itself fails.
fn execute_cmd(cmd: &mut Command) -> StratisResult<()> {
    match cmd.output() {
        Err(err) => Err(StratisError::Error(format!(
            "Failed to execute command {:?}, err: {:?}",
            cmd, err
        ))),
        Ok(result) => {
            if result.status.success() {
                Ok(())
            } else {
                let std_out_txt = String::from_utf8_lossy(&result.stdout);
                let std_err_txt = String::from_utf8_lossy(&result.stderr);
                let err_msg = format!(
                    "Command failed: cmd: {:?}, stdout: {} stderr: {}",
                    cmd, std_out_txt, std_err_txt
                );
                Err(StratisError::Error(err_msg))
            }
        }
    }
}

/// Get an absolute path for the executable with the given name.
/// Precondition: verify_binaries() has already been invoked.
fn get_executable(name: &str) -> StratisResult<PathBuf> {
    Ok(BINARIES
        .get(name)
        .expect("name arguments are all constants defined with BINARIES, lookup can not fail")
        .as_ref()
        .expect("verify_binaries() was previously called and returned no error")
        .to_path_buf())
}

/// Create a filesystem on devnode.
pub fn create_fs(devnode: &Path, uuid: Uuid) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(MKFS_XFS)?.as_os_str())
            .arg("-f")
            .arg("-q")
            .arg(&devnode)
            .arg("-m")
            .arg(format!("uuid={}", uuid)),
    )
}

/// Use the xfs_growfs command to expand a filesystem mounted at the given
/// mount point.
pub fn xfs_growfs(mount_point: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(XFS_GROWFS)?.as_os_str())
            .arg(mount_point)
            .arg("-d"),
    )
}

/// Set a new UUID for filesystem on the devnode.
pub fn set_uuid(devnode: &Path, uuid: Uuid) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(XFS_ADMIN)?.as_os_str())
            .arg("-U")
            .arg(format!("{}", uuid))
            .arg(&devnode),
    )
}

/// Call thin_check on a thinpool
pub fn thin_check(devnode: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(THIN_CHECK)?.as_os_str())
            .arg("-q")
            .arg(devnode),
    )
}

/// Call thin_repair on a thinpool
pub fn thin_repair(meta_dev: &Path, new_meta_dev: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(THIN_REPAIR)?.as_os_str())
            .arg("-i")
            .arg(meta_dev)
            .arg("-o")
            .arg(new_meta_dev),
    )
}

/// Call udevadm settle
#[cfg(test)]
pub fn udev_settle() -> StratisResult<()> {
    execute_cmd(
        Command::new("udevadm").arg("settle"),
        &format!("udevadm settle failed"),
    )
}

#[cfg(test)]
pub fn create_ext3_fs(devnode: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new("mkfs.ext3").arg(&devnode),
        &format!("Failed to create new ext3 filesystem at {:?}", devnode),
    )
}
<commit_msg>fixup for change to execute_cmd args<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Handles invoking external binaries.
// This module assumes that, for a given machine, there is only one place
// where the desired executable might be installed. It expects the engine
// to identify that place at its initialization by invoking verify_binaries(),
// and to exit immediately if verify_binaries() return an error. If this
// protocol is followed then when any command is executed the unique absolute
// path of the binary for this machine will already have been identified.
// However stratisd may run for a while and it is possible for the binary
// to be caused to be uninstalled while stratisd is being run. Therefore,
// the existence of the file is checked before the command is invoked, and
// an explicit error is returned if the executable can not be found.

use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::process::Command;

use uuid::Uuid;

use stratis::{StratisError, StratisResult};

/// Find the binary with the given name by looking in likely locations.
/// Return None if no binary was found.
/// Search an explicit list of directories rather than the user's PATH
/// environment variable. stratisd may be running when there is no PATH
/// variable set.
fn find_binary(name: &str) -> Option<PathBuf> {
    ["/usr/sbin", "/sbin", "/usr/bin", "/bin"]
        .iter()
        .map(|pre| [pre, name].iter().collect::<PathBuf>())
        .find(|path| path.exists())
}

// These are the external binaries that stratisd relies on.
// Any change in this list requires a corresponding change to BINARIES,
// and vice-versa.
const MKFS_XFS: &str = "mkfs.xfs";
const THIN_CHECK: &str = "thin_check";
const THIN_REPAIR: &str = "thin_repair";
const XFS_ADMIN: &str = "xfs_admin";
const XFS_GROWFS: &str = "xfs_growfs";

lazy_static! {
    static ref BINARIES: HashMap<String, Option<PathBuf>> = [
        (MKFS_XFS.to_string(), find_binary(MKFS_XFS)),
        (THIN_CHECK.to_string(), find_binary(THIN_CHECK)),
        (THIN_REPAIR.to_string(), find_binary(THIN_REPAIR)),
        (XFS_ADMIN.to_string(), find_binary(XFS_ADMIN)),
        (XFS_GROWFS.to_string(), find_binary(XFS_GROWFS)),
    ].iter()
        .cloned()
        .collect();
}

/// Verify that all binaries that the engine might invoke are available at some
/// path. Return an error if any are missing. Required to be called on engine
/// initialization.
pub fn verify_binaries() -> StratisResult<()> {
    match BINARIES.iter().find(|(_, ref path)| path.is_none()) {
        None => Ok(()),
        Some((ref name, _)) => Err(StratisError::Error(format!(
            "Unable to find absolute path for \"{}\"",
            name
        ))),
    }
}

/// Invoke the specified command. Return an error if invoking the command
/// fails or if the command itself fails.
fn execute_cmd(cmd: &mut Command) -> StratisResult<()> {
    match cmd.output() {
        Err(err) => Err(StratisError::Error(format!(
            "Failed to execute command {:?}, err: {:?}",
            cmd, err
        ))),
        Ok(result) => {
            if result.status.success() {
                Ok(())
            } else {
                let std_out_txt = String::from_utf8_lossy(&result.stdout);
                let std_err_txt = String::from_utf8_lossy(&result.stderr);
                let err_msg = format!(
                    "Command failed: cmd: {:?}, stdout: {} stderr: {}",
                    cmd, std_out_txt, std_err_txt
                );
                Err(StratisError::Error(err_msg))
            }
        }
    }
}

/// Get an absolute path for the executable with the given name.
/// Precondition: verify_binaries() has already been invoked.
fn get_executable(name: &str) -> StratisResult<PathBuf> {
    Ok(BINARIES
        .get(name)
        .expect("name arguments are all constants defined with BINARIES, lookup can not fail")
        .as_ref()
        .expect("verify_binaries() was previously called and returned no error")
        .to_path_buf())
}

/// Create a filesystem on devnode.
pub fn create_fs(devnode: &Path, uuid: Uuid) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(MKFS_XFS)?.as_os_str())
            .arg("-f")
            .arg("-q")
            .arg(&devnode)
            .arg("-m")
            .arg(format!("uuid={}", uuid)),
    )
}

/// Use the xfs_growfs command to expand a filesystem mounted at the given
/// mount point.
pub fn xfs_growfs(mount_point: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(XFS_GROWFS)?.as_os_str())
            .arg(mount_point)
            .arg("-d"),
    )
}

/// Set a new UUID for filesystem on the devnode.
pub fn set_uuid(devnode: &Path, uuid: Uuid) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(XFS_ADMIN)?.as_os_str())
            .arg("-U")
            .arg(format!("{}", uuid))
            .arg(&devnode),
    )
}

/// Call thin_check on a thinpool
pub fn thin_check(devnode: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(THIN_CHECK)?.as_os_str())
            .arg("-q")
            .arg(devnode),
    )
}

/// Call thin_repair on a thinpool
pub fn thin_repair(meta_dev: &Path, new_meta_dev: &Path) -> StratisResult<()> {
    execute_cmd(
        Command::new(get_executable(THIN_REPAIR)?.as_os_str())
            .arg("-i")
            .arg(meta_dev)
            .arg("-o")
            .arg(new_meta_dev),
    )
}

/// Call udevadm settle
#[cfg(test)]
pub fn udev_settle() -> StratisResult<()> {
    execute_cmd(Command::new("udevadm").arg("settle"))
}

#[cfg(test)]
pub fn create_ext3_fs(devnode: &Path) -> StratisResult<()> {
    execute_cmd(Command::new("mkfs.ext3").arg(&devnode))
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Manage the linear volume that stores metadata on pool levels 5-7.

use std::convert::From;
use std::error::Error;
use std::fs::{create_dir, OpenOptions, read_dir, remove_file, rename};
use std::io::ErrorKind;
use std::io::prelude::*;
use std::os::unix::io::AsRawFd;
use std::path::PathBuf;

use nix::mount::{MsFlags, mount, umount};
use nix::unistd::fsync;
use serde_json;

use devicemapper::{DM, LinearDev, Segment};

use super::super::engine::HasUuid;
use super::super::errors::EngineResult;
use super::super::types::{FilesystemUuid, PoolUuid};

use super::filesystem::{create_fs, StratFilesystem};
use super::serde_structs::{FilesystemSave, Recordable};

// TODO: Monitor fs size and extend linear and fs if needed
// TODO: Document format of stuff on MDV in SWDD (currently ad-hoc)

const DEV_PATH: &'static str = "/dev/stratis";

const FILESYSTEM_DIR: &'static str = "filesystems";

#[derive(Debug)]
pub struct MetadataVol {
    dev: LinearDev,
    mount_pt: PathBuf,
}

impl MetadataVol {
    /// Initialize a new Metadata Volume.
    pub fn initialize(pool_uuid: &PoolUuid, dev: LinearDev) -> EngineResult<MetadataVol> {
        try!(create_fs(try!(dev.devnode()).as_path()));
        MetadataVol::setup(pool_uuid, dev)
    }

    /// Set up an existing Metadata Volume.
    pub fn setup(pool_uuid: &PoolUuid, dev: LinearDev) -> EngineResult<MetadataVol> {
        if let Err(err) = create_dir(DEV_PATH) {
            if err.kind() != ErrorKind::AlreadyExists {
                return Err(From::from(err));
            }
        }

        let filename = format!(".mdv-{}", pool_uuid.simple());
        let mount_pt: PathBuf = vec![DEV_PATH, &filename].iter().collect();

        if let Err(err) = create_dir(&mount_pt) {
            if err.kind() != ErrorKind::AlreadyExists {
                return Err(From::from(err));
            }
        }

        try!(mount(Some(&try!(dev.devnode())),
                   &mount_pt,
                   Some("xfs"),
                   MsFlags::empty(),
                   None as Option<&str>));

        if let Err(err) = create_dir(&mount_pt.join(FILESYSTEM_DIR)) {
            if err.kind() != ErrorKind::AlreadyExists {
                return Err(From::from(err));
            }
        }

        Ok(MetadataVol { dev, mount_pt })
    }

    /// Save info on a new filesystem to persistent storage, or update
    /// the existing info on a filesystem.
    // Write to a temp file and then rename to actual filename, to
    // ensure file contents are not truncated if operation is
    // interrupted.
    pub fn save_fs(&self, fs: &StratFilesystem) -> EngineResult<()> {
        let data = try!(serde_json::to_string(&try!(fs.record())));
        let path = self.mount_pt
            .join(FILESYSTEM_DIR)
            .join(fs.uuid().simple().to_string())
            .with_extension("json");

        let temp_path = path.clone().with_extension("temp");

        // Braces to ensure f is closed before renaming
        {
            let mut f = try!(OpenOptions::new()
                                 .write(true)
                                 .create(true)
                                 .open(&temp_path));
            try!(f.write_all(data.as_bytes()));

            // Try really hard to make sure it goes to disk
            try!(f.flush());
            try!(fsync(f.as_raw_fd()));
        }

        try!(rename(temp_path, path));

        Ok(())
    }

    /// Remove info on a filesystem from persistent storage.
    pub fn rm_fs(&self, fs_uuid: &FilesystemUuid) -> EngineResult<()> {
        let fs_path = self.mount_pt
            .join(FILESYSTEM_DIR)
            .join(fs_uuid.simple().to_string())
            .with_extension("json");
        if let Err(err) = remove_file(fs_path) {
            if err.kind() != ErrorKind::NotFound {
                return Err(From::from(err));
            }
        }

        Ok(())
    }

    /// Check the current state of the MDV.
    pub fn check(&self) -> EngineResult<()> {
        for dir_e in try!(read_dir(self.mount_pt.join(FILESYSTEM_DIR))) {
            let dir_e = try!(dir_e);

            // Clean up any lingering .temp files. These should only
            // exist if there was a crash during save_fs().
            if dir_e.path().ends_with(".temp") {
                match remove_file(dir_e.path()) {
                    Err(err) => {
                        debug!("could not remove file {:?}: {}",
                               dir_e.path(),
                               err.description())
                    }
                    Ok(_) => debug!("Cleaning up temp file {:?}", dir_e.path()),
                }
            }
        }

        Ok(())
    }

    /// Get list of filesystems stored on the MDV.
    pub fn filesystems(&self) -> EngineResult<Vec<FilesystemSave>> {
        let mut filesystems = Vec::new();

        for dir_e in try!(read_dir(self.mount_pt.join(FILESYSTEM_DIR))) {
            let dir_e = try!(dir_e);

            if dir_e.path().ends_with(".temp") {
                continue;
            }

            let mut f = try!(OpenOptions::new().read(true).open(&dir_e.path()));
            let mut data = Vec::new();
            try!(f.read_to_end(&mut data));

            filesystems.push(try!(serde_json::from_slice(&data)));
        }

        Ok(filesystems)
    }

    /// Return the segments used.
    pub fn segments(&self) -> &[Segment] {
        self.dev.segments()
    }

    /// Tear down a Metadata Volume.
    pub fn teardown(self, dm: &DM) -> EngineResult<()> {
        try!(umount(&self.mount_pt));
        try!(self.dev.teardown(dm));

        Ok(())
    }
}
<commit_msg>Avoid failing if the metadata volume is already mounted<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Manage the linear volume that stores metadata on pool levels 5-7.

use std::convert::From;
use std::error::Error;
use std::fs::{create_dir, OpenOptions, read_dir, remove_file, rename};
use std::io::ErrorKind;
use std::io::prelude::*;
use std::os::unix::io::AsRawFd;
use std::path::PathBuf;

use nix;
use nix::mount::{MsFlags, mount, umount};
use nix::unistd::fsync;
use serde_json;

use devicemapper::{DM, LinearDev, Segment};

use super::super::engine::HasUuid;
use super::super::errors::EngineResult;
use super::super::types::{FilesystemUuid, PoolUuid};

use super::filesystem::{create_fs, StratFilesystem};
use super::serde_structs::{FilesystemSave, Recordable};

// TODO: Monitor fs size and extend linear and fs if needed
// TODO: Document format of stuff on MDV in SWDD (currently ad-hoc)

const DEV_PATH: &'static str = "/dev/stratis";

const FILESYSTEM_DIR: &'static str = "filesystems";

#[derive(Debug)]
pub struct MetadataVol {
    dev: LinearDev,
    mount_pt: PathBuf,
}

impl MetadataVol {
    /// Initialize a new Metadata Volume.
    pub fn initialize(pool_uuid: &PoolUuid, dev: LinearDev) -> EngineResult<MetadataVol> {
        try!(create_fs(try!(dev.devnode()).as_path()));
        MetadataVol::setup(pool_uuid, dev)
    }

    /// Set up an existing Metadata Volume.
    pub fn setup(pool_uuid: &PoolUuid, dev: LinearDev) -> EngineResult<MetadataVol> {
        if let Err(err) = create_dir(DEV_PATH) {
            if err.kind() != ErrorKind::AlreadyExists {
                return Err(From::from(err));
            }
        }

        let filename = format!(".mdv-{}", pool_uuid.simple());
        let mount_pt: PathBuf = vec![DEV_PATH, &filename].iter().collect();

        if let Err(err) = create_dir(&mount_pt) {
            if err.kind() != ErrorKind::AlreadyExists {
                return Err(From::from(err));
            }
        }

        if let Err(err) = mount(Some(&try!(dev.devnode())),
                                &mount_pt,
                                Some("xfs"),
                                MsFlags::empty(),
                                None as Option<&str>) {
            match err {
                nix::Error::Sys(nix::Errno::EBUSY) => {
                    // EBUSY is the error returned in this context when the
                    // device is already mounted at the specified mountpoint.
                }
                _ => return Err(From::from(err)),
            }
        }

        if let Err(err) = create_dir(&mount_pt.join(FILESYSTEM_DIR)) {
            if err.kind() != ErrorKind::AlreadyExists {
                return Err(From::from(err));
            }
        }

        Ok(MetadataVol { dev, mount_pt })
    }

    /// Save info on a new filesystem to persistent storage, or update
    /// the existing info on a filesystem.
    // Write to a temp file and then rename to actual filename, to
    // ensure file contents are not truncated if operation is
    // interrupted.
    pub fn save_fs(&self, fs: &StratFilesystem) -> EngineResult<()> {
        let data = try!(serde_json::to_string(&try!(fs.record())));
        let path = self.mount_pt
            .join(FILESYSTEM_DIR)
            .join(fs.uuid().simple().to_string())
            .with_extension("json");

        let temp_path = path.clone().with_extension("temp");

        // Braces to ensure f is closed before renaming
        {
            let mut f = try!(OpenOptions::new()
                                 .write(true)
                                 .create(true)
                                 .open(&temp_path));
            try!(f.write_all(data.as_bytes()));

            // Try really hard to make sure it goes to disk
            try!(f.flush());
            try!(fsync(f.as_raw_fd()));
        }

        try!(rename(temp_path, path));

        Ok(())
    }

    /// Remove info on a filesystem from persistent storage.
    pub fn rm_fs(&self, fs_uuid: &FilesystemUuid) -> EngineResult<()> {
        let fs_path = self.mount_pt
            .join(FILESYSTEM_DIR)
            .join(fs_uuid.simple().to_string())
            .with_extension("json");
        if let Err(err) = remove_file(fs_path) {
            if err.kind() != ErrorKind::NotFound {
                return Err(From::from(err));
            }
        }

        Ok(())
    }

    /// Check the current state of the MDV.
    pub fn check(&self) -> EngineResult<()> {
        for dir_e in try!(read_dir(self.mount_pt.join(FILESYSTEM_DIR))) {
            let dir_e = try!(dir_e);

            // Clean up any lingering .temp files. These should only
            // exist if there was a crash during save_fs().
            if dir_e.path().ends_with(".temp") {
                match remove_file(dir_e.path()) {
                    Err(err) => {
                        debug!("could not remove file {:?}: {}",
                               dir_e.path(),
                               err.description())
                    }
                    Ok(_) => debug!("Cleaning up temp file {:?}", dir_e.path()),
                }
            }
        }

        Ok(())
    }

    /// Get list of filesystems stored on the MDV.
    pub fn filesystems(&self) -> EngineResult<Vec<FilesystemSave>> {
        let mut filesystems = Vec::new();

        for dir_e in try!(read_dir(self.mount_pt.join(FILESYSTEM_DIR))) {
            let dir_e = try!(dir_e);

            if dir_e.path().ends_with(".temp") {
                continue;
            }

            let mut f = try!(OpenOptions::new().read(true).open(&dir_e.path()));
            let mut data = Vec::new();
            try!(f.read_to_end(&mut data));

            filesystems.push(try!(serde_json::from_slice(&data)));
        }

        Ok(filesystems)
    }

    /// Return the segments used.
    pub fn segments(&self) -> &[Segment] {
        self.dev.segments()
    }

    /// Tear down a Metadata Volume.
    pub fn teardown(self, dm: &DM) -> EngineResult<()> {
        try!(umount(&self.mount_pt));
        try!(self.dev.teardown(dm));

        Ok(())
    }
}
<|endoftext|>


<commit_before>//! cargo_check provides the functionality needed to run `cargo check` or
//! another compatible command (f.x. clippy) in a background thread and provide
//! LSP diagnostics based on the output of the command.
use cargo_metadata::Message;
use crossbeam_channel::{never, select, unbounded, Receiver, RecvError, Sender};
use lsp_types::{
    Diagnostic, Url, WorkDoneProgress, WorkDoneProgressBegin, WorkDoneProgressEnd,
    WorkDoneProgressReport,
};
use std::{
    collections::HashMap,
    io::BufReader,
    path::PathBuf,
    process::{Command, Stdio},
    sync::Arc,
    thread::JoinHandle,
    time::Instant,
};

mod conv;

use crate::conv::{map_rust_diagnostic_to_lsp, MappedRustDiagnostic, SuggestedFix};

pub use crate::conv::url_from_path_with_drive_lowercasing;

#[derive(Clone, Debug)]
pub struct CheckOptions {
    pub enable: bool,
    pub args: Vec<String>,
    pub command: String,
    pub all_targets: bool,
}

/// CheckWatcher wraps the shared state and communication machinery used for
/// running `cargo check` (or other compatible command) and providing
/// diagnostics based on the output.
/// The spawned thread is shut down when this struct is dropped.
#[derive(Debug)]
pub struct CheckWatcher {
    pub task_recv: Receiver<CheckTask>,
    pub state: Arc<CheckState>,
    cmd_send: Option<Sender<CheckCommand>>,
    handle: Option<JoinHandle<()>>,
}

impl CheckWatcher {
    pub fn new(options: &CheckOptions, workspace_root: PathBuf) -> CheckWatcher {
        let options = options.clone();
        let state = Arc::new(CheckState::new());

        let (task_send, task_recv) = unbounded::<CheckTask>();
        let (cmd_send, cmd_recv) = unbounded::<CheckCommand>();
        let handle = std::thread::spawn(move || {
            let mut check = CheckWatcherThread::new(options, workspace_root);
            check.run(&task_send, &cmd_recv);
        });
        CheckWatcher { task_recv, cmd_send: Some(cmd_send), handle: Some(handle), state }
    }

    /// Returns a CheckWatcher that doesn't actually do anything
    pub fn dummy() -> CheckWatcher {
        let state = Arc::new(CheckState::new());
        CheckWatcher { task_recv: never(), cmd_send: None, handle: None, state }
    }

    /// Schedule a re-start of the cargo check worker.
    pub fn update(&self) {
        if let Some(cmd_send) = &self.cmd_send {
            cmd_send.send(CheckCommand::Update).unwrap();
        }
    }
}

impl std::ops::Drop for CheckWatcher {
    fn drop(&mut self) {
        if let Some(handle) = self.handle.take() {
            // Take the sender out of the option
            let recv = self.cmd_send.take();

            // Dropping the sender finishes the thread loop
            drop(recv);

            // Join the thread, it should finish shortly. We don't really care
            // whether it panicked, so it is safe to ignore the result
            let _ = handle.join();
        }
    }
}

#[derive(Clone, Debug)]
pub struct CheckState {
    diagnostic_collection: HashMap<Url, Vec<Diagnostic>>,
    suggested_fix_collection: HashMap<Url, Vec<SuggestedFix>>,
}

impl CheckState {
    fn new() -> CheckState {
        CheckState {
            diagnostic_collection: HashMap::new(),
            suggested_fix_collection: HashMap::new(),
        }
    }

    /// Clear the cached diagnostics, and schedule updating diagnostics by the
    /// server, to clear stale results.
    pub fn clear(&mut self) -> Vec<Url> {
        let cleared_files: Vec<Url> = self.diagnostic_collection.keys().cloned().collect();
        self.diagnostic_collection.clear();
        self.suggested_fix_collection.clear();
        cleared_files
    }

    pub fn diagnostics_for(&self, uri: &Url) -> Option<&[Diagnostic]> {
        self.diagnostic_collection.get(uri).map(|d| d.as_slice())
    }

    pub fn fixes_for(&self, uri: &Url) -> Option<&[SuggestedFix]> {
        self.suggested_fix_collection.get(uri).map(|d| d.as_slice())
    }

    pub fn add_diagnostic_with_fixes(&mut self, file_uri: Url, diagnostic: DiagnosticWithFixes) {
        for fix in diagnostic.suggested_fixes {
            self.add_suggested_fix_for_diagnostic(fix, &diagnostic.diagnostic);
        }
        self.add_diagnostic(file_uri, diagnostic.diagnostic);
    }

    fn add_diagnostic(&mut self, file_uri: Url, diagnostic: Diagnostic) {
        let diagnostics = self.diagnostic_collection.entry(file_uri).or_default();

        // If we're building multiple targets it's possible we've already seen this diagnostic
        let is_duplicate = diagnostics.iter().any(|d| are_diagnostics_equal(d, &diagnostic));
        if is_duplicate {
            return;
        }

        diagnostics.push(diagnostic);
    }

    fn add_suggested_fix_for_diagnostic(
        &mut self,
        mut suggested_fix: SuggestedFix,
        diagnostic: &Diagnostic,
    ) {
        let file_uri = suggested_fix.location.uri.clone();
        let file_suggestions = self.suggested_fix_collection.entry(file_uri).or_default();

        let existing_suggestion: Option<&mut SuggestedFix> =
            file_suggestions.iter_mut().find(|s| s == &&suggested_fix);
        if let Some(existing_suggestion) = existing_suggestion {
            // The existing suggestion also applies to this new diagnostic
            existing_suggestion.diagnostics.push(diagnostic.clone());
        } else {
            // We haven't seen this suggestion before
            suggested_fix.diagnostics.push(diagnostic.clone());
            file_suggestions.push(suggested_fix);
        }
    }
}

#[derive(Debug)]
pub enum CheckTask {
    /// Request a clearing of all cached diagnostics from the check watcher
    ClearDiagnostics,

    /// Request adding a diagnostic with fixes included to a file
    AddDiagnostic(Url, DiagnosticWithFixes),

    /// Request check progress notification to client
    Status(WorkDoneProgress),
}

pub enum CheckCommand {
    /// Request re-start of check thread
    Update,
}

struct CheckWatcherThread {
    options: CheckOptions,
    workspace_root: PathBuf,
    watcher: WatchThread,
    last_update_req: Option<Instant>,
}

impl CheckWatcherThread {
    fn new(options: CheckOptions, workspace_root: PathBuf) -> CheckWatcherThread {
        CheckWatcherThread {
            options,
            workspace_root,
            watcher: WatchThread::dummy(),
            last_update_req: None,
        }
    }

    fn run(&mut self, task_send: &Sender<CheckTask>, cmd_recv: &Receiver<CheckCommand>) {
        loop {
            select! {
                recv(&cmd_recv) -> cmd => match cmd {
                    Ok(cmd) => self.handle_command(cmd),
                    Err(RecvError) => {
                        // Command channel has closed, so shut down
                        break;
                    },
                },
                recv(self.watcher.message_recv) -> msg => match msg {
                    Ok(msg) => self.handle_message(msg, task_send),
                    Err(RecvError) => {
                        // Watcher finished, replace it with a never channel to
                        // avoid busy-waiting.
                        std::mem::replace(&mut self.watcher.message_recv, never());
                    },
                }
            };

            if self.should_recheck() {
                self.last_update_req.take();
                task_send.send(CheckTask::ClearDiagnostics).unwrap();

                // By replacing the watcher, we drop the previous one which
                // causes it to shut down automatically.
                self.watcher = WatchThread::new(&self.options, &self.workspace_root);
            }
        }
    }

    fn should_recheck(&mut self) -> bool {
        if let Some(_last_update_req) = &self.last_update_req {
            // We currently only request an update on save, as we need up to
            // date source on disk for cargo check to do it's magic, so we
            // don't really need to debounce the requests at this point.
            return true;
        }
        false
    }

    fn handle_command(&mut self, cmd: CheckCommand) {
        match cmd {
            CheckCommand::Update => self.last_update_req = Some(Instant::now()),
        }
    }

    fn handle_message(&self, msg: CheckEvent, task_send: &Sender<CheckTask>) {
        match msg {
            CheckEvent::Begin => {
                task_send
                    .send(CheckTask::Status(WorkDoneProgress::Begin(WorkDoneProgressBegin {
                        title: "Running 'cargo check'".to_string(),
                        cancellable: Some(false),
                        message: None,
                        percentage: None,
                    })))
                    .unwrap();
            }

            CheckEvent::End => {
                task_send
                    .send(CheckTask::Status(WorkDoneProgress::End(WorkDoneProgressEnd {
                        message: None,
                    })))
                    .unwrap();
            }

            CheckEvent::Msg(Message::CompilerArtifact(msg)) => {
                task_send
                    .send(CheckTask::Status(WorkDoneProgress::Report(WorkDoneProgressReport {
                        cancellable: Some(false),
                        message: Some(msg.target.name),
                        percentage: None,
                    })))
                    .unwrap();
            }

            CheckEvent::Msg(Message::CompilerMessage(msg)) => {
                let map_result =
                    match map_rust_diagnostic_to_lsp(&msg.message, &self.workspace_root) {
                        Some(map_result) => map_result,
                        None => return,
                    };

                let MappedRustDiagnostic { location, diagnostic, suggested_fixes } = map_result;

                let diagnostic = DiagnosticWithFixes { diagnostic, suggested_fixes };
                task_send.send(CheckTask::AddDiagnostic(location.uri, diagnostic)).unwrap();
            }

            CheckEvent::Msg(Message::BuildScriptExecuted(_msg)) => {}
            CheckEvent::Msg(Message::Unknown) => {}
        }
    }
}

#[derive(Debug)]
pub struct DiagnosticWithFixes {
    diagnostic: Diagnostic,
    suggested_fixes: Vec<SuggestedFix>,
}

/// WatchThread exists to wrap around the communication needed to be able to
/// run `cargo check` without blocking. Currently the Rust standard library
/// doesn't provide a way to read sub-process output without blocking, so we
/// have to wrap sub-processes output handling in a thread and pass messages
/// back over a channel.
/// The correct way to dispose of the thread is to drop it, on which the
/// sub-process will be killed, and the thread will be joined.
struct WatchThread {
    handle: Option<JoinHandle<()>>,
    message_recv: Receiver<CheckEvent>,
}

enum CheckEvent {
    Begin,
    Msg(cargo_metadata::Message),
    End,
}

impl WatchThread {
    fn dummy() -> WatchThread {
        WatchThread { handle: None, message_recv: never() }
    }

    fn new(options: &CheckOptions, workspace_root: &PathBuf) -> WatchThread {
        let mut args: Vec<String> = vec![
            options.command.clone(),
            "--message-format=json".to_string(),
            "--manifest-path".to_string(),
            format!("{}/Cargo.toml", workspace_root.to_string_lossy()),
        ];
        if options.all_targets {
            args.push("--all-targets".to_string());
        }
        args.extend(options.args.iter().cloned());

        let (message_send, message_recv) = unbounded();
        let enabled = options.enable;
        let handle = std::thread::spawn(move || {
            if !enabled {
                return;
            }

            let mut command = Command::new("cargo")
                .args(&args)
                .stdout(Stdio::piped())
                .stderr(Stdio::null())
                .spawn()
                .expect("couldn't launch cargo");

            // If we trigger an error here, we will do so in the loop instead,
            // which will break out of the loop, and continue the shutdown
            let _ = message_send.send(CheckEvent::Begin);

            for message in
                cargo_metadata::parse_messages(BufReader::new(command.stdout.take().unwrap()))
            {
                let message = match message {
                    Ok(message) => message,
                    Err(err) => {
                        log::error!("Invalid json from cargo check, ignoring: {}", err);
                        continue;
                    }
                };

                match message_send.send(CheckEvent::Msg(message)) {
                    Ok(()) => {}
                    Err(_err) => {
                        // The send channel was closed, so we want to shutdown
                        break;
                    }
                }
            }

            // We can ignore any error here, as we are already in the progress
            // of shutting down.
            let _ = message_send.send(CheckEvent::End);

            // It is okay to ignore the result, as it only errors if the process is already dead
            let _ = command.kill();

            // Again, we don't care about the exit status so just ignore the result
            let _ = command.wait();
        });
        WatchThread { handle: Some(handle), message_recv }
    }
}

impl std::ops::Drop for WatchThread {
    fn drop(&mut self) {
        if let Some(handle) = self.handle.take() {
            // Replace our reciever with dummy one, so we can drop and close the
            // one actually communicating with the thread
            let recv = std::mem::replace(&mut self.message_recv, never());

            // Dropping the original reciever initiates thread sub-process shutdown
            drop(recv);

            // Join the thread, it should finish shortly. We don't really care
            // whether it panicked, so it is safe to ignore the result
            let _ = handle.join();
        }
    }
}

fn are_diagnostics_equal(left: &Diagnostic, right: &Diagnostic) -> bool {
    left.source == right.source
        && left.severity == right.severity
        && left.range == right.range
        && left.message == right.message
}
<commit_msg>Don't do check progress update for fresh crates<commit_after>//! cargo_check provides the functionality needed to run `cargo check` or
//! another compatible command (f.x. clippy) in a background thread and provide
//! LSP diagnostics based on the output of the command.
use cargo_metadata::Message;
use crossbeam_channel::{never, select, unbounded, Receiver, RecvError, Sender};
use lsp_types::{
    Diagnostic, Url, WorkDoneProgress, WorkDoneProgressBegin, WorkDoneProgressEnd,
    WorkDoneProgressReport,
};
use std::{
    collections::HashMap,
    io::BufReader,
    path::PathBuf,
    process::{Command, Stdio},
    sync::Arc,
    thread::JoinHandle,
    time::Instant,
};

mod conv;

use crate::conv::{map_rust_diagnostic_to_lsp, MappedRustDiagnostic, SuggestedFix};

pub use crate::conv::url_from_path_with_drive_lowercasing;

#[derive(Clone, Debug)]
pub struct CheckOptions {
    pub enable: bool,
    pub args: Vec<String>,
    pub command: String,
    pub all_targets: bool,
}

/// CheckWatcher wraps the shared state and communication machinery used for
/// running `cargo check` (or other compatible command) and providing
/// diagnostics based on the output.
/// The spawned thread is shut down when this struct is dropped.
#[derive(Debug)]
pub struct CheckWatcher {
    pub task_recv: Receiver<CheckTask>,
    pub state: Arc<CheckState>,
    cmd_send: Option<Sender<CheckCommand>>,
    handle: Option<JoinHandle<()>>,
}

impl CheckWatcher {
    pub fn new(options: &CheckOptions, workspace_root: PathBuf) -> CheckWatcher {
        let options = options.clone();
        let state = Arc::new(CheckState::new());

        let (task_send, task_recv) = unbounded::<CheckTask>();
        let (cmd_send, cmd_recv) = unbounded::<CheckCommand>();
        let handle = std::thread::spawn(move || {
            let mut check = CheckWatcherThread::new(options, workspace_root);
            check.run(&task_send, &cmd_recv);
        });
        CheckWatcher { task_recv, cmd_send: Some(cmd_send), handle: Some(handle), state }
    }

    /// Returns a CheckWatcher that doesn't actually do anything
    pub fn dummy() -> CheckWatcher {
        let state = Arc::new(CheckState::new());
        CheckWatcher { task_recv: never(), cmd_send: None, handle: None, state }
    }

    /// Schedule a re-start of the cargo check worker.
    pub fn update(&self) {
        if let Some(cmd_send) = &self.cmd_send {
            cmd_send.send(CheckCommand::Update).unwrap();
        }
    }
}

impl std::ops::Drop for CheckWatcher {
    fn drop(&mut self) {
        if let Some(handle) = self.handle.take() {
            // Take the sender out of the option
            let recv = self.cmd_send.take();

            // Dropping the sender finishes the thread loop
            drop(recv);

            // Join the thread, it should finish shortly. We don't really care
            // whether it panicked, so it is safe to ignore the result
            let _ = handle.join();
        }
    }
}

#[derive(Clone, Debug)]
pub struct CheckState {
    diagnostic_collection: HashMap<Url, Vec<Diagnostic>>,
    suggested_fix_collection: HashMap<Url, Vec<SuggestedFix>>,
}

impl CheckState {
    fn new() -> CheckState {
        CheckState {
            diagnostic_collection: HashMap::new(),
            suggested_fix_collection: HashMap::new(),
        }
    }

    /// Clear the cached diagnostics, and schedule updating diagnostics by the
    /// server, to clear stale results.
    pub fn clear(&mut self) -> Vec<Url> {
        let cleared_files: Vec<Url> = self.diagnostic_collection.keys().cloned().collect();
        self.diagnostic_collection.clear();
        self.suggested_fix_collection.clear();
        cleared_files
    }

    pub fn diagnostics_for(&self, uri: &Url) -> Option<&[Diagnostic]> {
        self.diagnostic_collection.get(uri).map(|d| d.as_slice())
    }

    pub fn fixes_for(&self, uri: &Url) -> Option<&[SuggestedFix]> {
        self.suggested_fix_collection.get(uri).map(|d| d.as_slice())
    }

    pub fn add_diagnostic_with_fixes(&mut self, file_uri: Url, diagnostic: DiagnosticWithFixes) {
        for fix in diagnostic.suggested_fixes {
            self.add_suggested_fix_for_diagnostic(fix, &diagnostic.diagnostic);
        }
        self.add_diagnostic(file_uri, diagnostic.diagnostic);
    }

    fn add_diagnostic(&mut self, file_uri: Url, diagnostic: Diagnostic) {
        let diagnostics = self.diagnostic_collection.entry(file_uri).or_default();

        // If we're building multiple targets it's possible we've already seen this diagnostic
        let is_duplicate = diagnostics.iter().any(|d| are_diagnostics_equal(d, &diagnostic));
        if is_duplicate {
            return;
        }

        diagnostics.push(diagnostic);
    }

    fn add_suggested_fix_for_diagnostic(
        &mut self,
        mut suggested_fix: SuggestedFix,
        diagnostic: &Diagnostic,
    ) {
        let file_uri = suggested_fix.location.uri.clone();
        let file_suggestions = self.suggested_fix_collection.entry(file_uri).or_default();

        let existing_suggestion: Option<&mut SuggestedFix> =
            file_suggestions.iter_mut().find(|s| s == &&suggested_fix);
        if let Some(existing_suggestion) = existing_suggestion {
            // The existing suggestion also applies to this new diagnostic
            existing_suggestion.diagnostics.push(diagnostic.clone());
        } else {
            // We haven't seen this suggestion before
            suggested_fix.diagnostics.push(diagnostic.clone());
            file_suggestions.push(suggested_fix);
        }
    }
}

#[derive(Debug)]
pub enum CheckTask {
    /// Request a clearing of all cached diagnostics from the check watcher
    ClearDiagnostics,

    /// Request adding a diagnostic with fixes included to a file
    AddDiagnostic(Url, DiagnosticWithFixes),

    /// Request check progress notification to client
    Status(WorkDoneProgress),
}

pub enum CheckCommand {
    /// Request re-start of check thread
    Update,
}

struct CheckWatcherThread {
    options: CheckOptions,
    workspace_root: PathBuf,
    watcher: WatchThread,
    last_update_req: Option<Instant>,
}

impl CheckWatcherThread {
    fn new(options: CheckOptions, workspace_root: PathBuf) -> CheckWatcherThread {
        CheckWatcherThread {
            options,
            workspace_root,
            watcher: WatchThread::dummy(),
            last_update_req: None,
        }
    }

    fn run(&mut self, task_send: &Sender<CheckTask>, cmd_recv: &Receiver<CheckCommand>) {
        loop {
            select! {
                recv(&cmd_recv) -> cmd => match cmd {
                    Ok(cmd) => self.handle_command(cmd),
                    Err(RecvError) => {
                        // Command channel has closed, so shut down
                        break;
                    },
                },
                recv(self.watcher.message_recv) -> msg => match msg {
                    Ok(msg) => self.handle_message(msg, task_send),
                    Err(RecvError) => {
                        // Watcher finished, replace it with a never channel to
                        // avoid busy-waiting.
                        std::mem::replace(&mut self.watcher.message_recv, never());
                    },
                }
            };

            if self.should_recheck() {
                self.last_update_req.take();
                task_send.send(CheckTask::ClearDiagnostics).unwrap();

                // By replacing the watcher, we drop the previous one which
                // causes it to shut down automatically.
                self.watcher = WatchThread::new(&self.options, &self.workspace_root);
            }
        }
    }

    fn should_recheck(&mut self) -> bool {
        if let Some(_last_update_req) = &self.last_update_req {
            // We currently only request an update on save, as we need up to
            // date source on disk for cargo check to do it's magic, so we
            // don't really need to debounce the requests at this point.
            return true;
        }
        false
    }

    fn handle_command(&mut self, cmd: CheckCommand) {
        match cmd {
            CheckCommand::Update => self.last_update_req = Some(Instant::now()),
        }
    }

    fn handle_message(&self, msg: CheckEvent, task_send: &Sender<CheckTask>) {
        match msg {
            CheckEvent::Begin => {
                task_send
                    .send(CheckTask::Status(WorkDoneProgress::Begin(WorkDoneProgressBegin {
                        title: "Running 'cargo check'".to_string(),
                        cancellable: Some(false),
                        message: None,
                        percentage: None,
                    })))
                    .unwrap();
            }

            CheckEvent::End => {
                task_send
                    .send(CheckTask::Status(WorkDoneProgress::End(WorkDoneProgressEnd {
                        message: None,
                    })))
                    .unwrap();
            }

            CheckEvent::Msg(Message::CompilerArtifact(msg)) => {
                task_send
                    .send(CheckTask::Status(WorkDoneProgress::Report(WorkDoneProgressReport {
                        cancellable: Some(false),
                        message: Some(msg.target.name),
                        percentage: None,
                    })))
                    .unwrap();
            }

            CheckEvent::Msg(Message::CompilerMessage(msg)) => {
                let map_result =
                    match map_rust_diagnostic_to_lsp(&msg.message, &self.workspace_root) {
                        Some(map_result) => map_result,
                        None => return,
                    };

                let MappedRustDiagnostic { location, diagnostic, suggested_fixes } = map_result;

                let diagnostic = DiagnosticWithFixes { diagnostic, suggested_fixes };
                task_send.send(CheckTask::AddDiagnostic(location.uri, diagnostic)).unwrap();
            }

            CheckEvent::Msg(Message::BuildScriptExecuted(_msg)) => {}
            CheckEvent::Msg(Message::Unknown) => {}
        }
    }
}

#[derive(Debug)]
pub struct DiagnosticWithFixes {
    diagnostic: Diagnostic,
    suggested_fixes: Vec<SuggestedFix>,
}

/// WatchThread exists to wrap around the communication needed to be able to
/// run `cargo check` without blocking. Currently the Rust standard library
/// doesn't provide a way to read sub-process output without blocking, so we
/// have to wrap sub-processes output handling in a thread and pass messages
/// back over a channel.
/// The correct way to dispose of the thread is to drop it, on which the
/// sub-process will be killed, and the thread will be joined.
struct WatchThread {
    handle: Option<JoinHandle<()>>,
    message_recv: Receiver<CheckEvent>,
}

enum CheckEvent {
    Begin,
    Msg(cargo_metadata::Message),
    End,
}

impl WatchThread {
    fn dummy() -> WatchThread {
        WatchThread { handle: None, message_recv: never() }
    }

    fn new(options: &CheckOptions, workspace_root: &PathBuf) -> WatchThread {
        let mut args: Vec<String> = vec![
            options.command.clone(),
            "--message-format=json".to_string(),
            "--manifest-path".to_string(),
            format!("{}/Cargo.toml", workspace_root.to_string_lossy()),
        ];
        if options.all_targets {
            args.push("--all-targets".to_string());
        }
        args.extend(options.args.iter().cloned());

        let (message_send, message_recv) = unbounded();
        let enabled = options.enable;
        let handle = std::thread::spawn(move || {
            if !enabled {
                return;
            }

            let mut command = Command::new("cargo")
                .args(&args)
                .stdout(Stdio::piped())
                .stderr(Stdio::null())
                .spawn()
                .expect("couldn't launch cargo");

            // If we trigger an error here, we will do so in the loop instead,
            // which will break out of the loop, and continue the shutdown
            let _ = message_send.send(CheckEvent::Begin);

            for message in
                cargo_metadata::parse_messages(BufReader::new(command.stdout.take().unwrap()))
            {
                let message = match message {
                    Ok(message) => message,
                    Err(err) => {
                        log::error!("Invalid json from cargo check, ignoring: {}", err);
                        continue;
                    }
                };

                // Skip certain kinds of messages to only spend time on what's useful
                match &message {
                    Message::CompilerArtifact(artifact) if artifact.fresh => continue,
                    Message::BuildScriptExecuted(_) => continue,
                    Message::Unknown => continue,
                    _ => {}
                }

                match message_send.send(CheckEvent::Msg(message)) {
                    Ok(()) => {}
                    Err(_err) => {
                        // The send channel was closed, so we want to shutdown
                        break;
                    }
                }
            }

            // We can ignore any error here, as we are already in the progress
            // of shutting down.
            let _ = message_send.send(CheckEvent::End);

            // It is okay to ignore the result, as it only errors if the process is already dead
            let _ = command.kill();

            // Again, we don't care about the exit status so just ignore the result
            let _ = command.wait();
        });
        WatchThread { handle: Some(handle), message_recv }
    }
}

impl std::ops::Drop for WatchThread {
    fn drop(&mut self) {
        if let Some(handle) = self.handle.take() {
            // Replace our reciever with dummy one, so we can drop and close the
            // one actually communicating with the thread
            let recv = std::mem::replace(&mut self.message_recv, never());

            // Dropping the original reciever initiates thread sub-process shutdown
            drop(recv);

            // Join the thread, it should finish shortly. We don't really care
            // whether it panicked, so it is safe to ignore the result
            let _ = handle.join();
        }
    }
}

fn are_diagnostics_equal(left: &Diagnostic, right: &Diagnostic) -> bool {
    left.source == right.source
        && left.severity == right.severity
        && left.range == right.range
        && left.message == right.message
}
<|endoftext|>


<commit_before>
extern crate itertools;
#[macro_use]
extern crate nom;
extern crate memchr;
#[macro_use]
extern crate quick_error;

pub mod errors;
pub mod items;
pub mod parser;

#[cfg(test)]
mod parser_test;
#[cfg(test)]
mod items_test;

pub fn parse_string(input: &str) -> Result<items::SystemdUnit, errors::ParserError> {

    // FIXME: this should be inside `parse_unit` but then, the lifetime would be wrong
    let input = String::from(input).replace("\\\n", "");
    let units = try!(parser::parse_unit(&input));
    let systemd_unit = try!(items::SystemdUnit::new(&units));
    Ok(systemd_unit)
}

<commit_msg> Parser: ooops<commit_after>
extern crate itertools;
#[macro_use]
extern crate nom;
#[macro_use]
extern crate quick_error;

pub mod errors;
pub mod items;
pub mod parser;

#[cfg(test)]
mod parser_test;
#[cfg(test)]
mod items_test;

pub fn parse_string(input: &str) -> Result<items::SystemdUnit, errors::ParserError> {

    // FIXME: this should be inside `parse_unit` but then, the lifetime would be wrong
    let input = String::from(input).replace("\\\n", "");
    let units = try!(parser::parse_unit(&input));
    let systemd_unit = try!(items::SystemdUnit::new(&units));
    Ok(systemd_unit)
}

<|endoftext|>


<commit_before>//! Generate the site content.

// Standard library
use std::borrow::Cow::{Borrowed, Owned};
use std::fs::File;
use std::io::prelude::*;
use std::path::{Path, PathBuf};

// Third party
use glob::{glob, Paths};
use pulldown_cmark as cmark;
use pulldown_cmark::{Event, Parser, Tag};
use syntect::highlighting::ThemeSet;

// First party
use crate::config::{Config, SyntaxOption};

/// Load the `Paths` for all markdown files in the specified content directory.
///
/// If the specified configuration values are wrong, we return a vector of all
/// the errors in the config as far as this is concerned.
///
/// TODO: move that validation elsewhere? Should probably extract a more general
/// site configuration validator.
fn load_markdown_paths(site_directory: &PathBuf, config: &Config) -> Result<Vec<PathBuf>, String> {
    let site = site_directory
        .to_str()
        .ok_or(format!("bad `site`: {:?}", site_directory));

    let directories = config.directories.content.to_str().ok_or(format!(
        "bad content directory: {:?}",
        config.directories.content
    ));

    let (site, directories) = match (site, directories) {
        (Err(s), Err(d)) => Err(s + "\n" + &d),
        (Err(s), Ok(_)) => Err(s),
        (Ok(_), Err(d)) => Err(d),
        (Ok(s), Ok(d)) => Ok((s, d)),
    }?;

    let content_glob = format!("{}/{}/**/*.md", site, directories);

    let paths = glob(&content_glob).map_err(|err| format!("{:?}", err))?;

    let (valid, errors): (Vec<Result<PathBuf, _>>, Vec<Result<_, String>>) = paths
        .map(|glob_result| glob_result.map_err(|err| format!("{:?}", err)))
        .partition(|path| path.is_ok());

    let errors: Vec<String> = errors.into_iter().map(|e| e.unwrap_err()).collect();
    if errors.len() == 0 {
        Ok(valid.into_iter().map(|v| v.unwrap()).collect())
    } else {
        Err(errors.join(",\n"))
    }
}

// TODO: this is an obvious candidate for a `HashMap`, so use that instead.
type LoadTuple<'a> = (&'a PathBuf, Result<String, String>);

// Instead of using `.collect` at the end of these, I *should* be able to use
// something like `Result<impl Iterator<HashMap<PathBuf, String>, String>`, even
// if the type isn't *quite* taht.
fn load_content<'p>(paths: &'p [PathBuf]) -> Result<Vec<(&'p PathBuf, String)>, String> {
    let (contents, errs): (Vec<LoadTuple>, Vec<LoadTuple>) = paths
        .into_iter()
        .map(|path| {
            let content = std::fs::read_to_string(path).map_err(|e| format!("{:?}", e));
            (path, content)
        })
        .partition(|(_, result)| result.is_ok());

    if errs.len() == 0 {
        Ok(contents
            .into_iter()
            .map(|(path, contents)| (path, contents.unwrap()))
            .collect())
    } else {
        Err(errs
            .into_iter()
            .map(|(_path, err)| err.unwrap_err())
            .collect::<Vec<String>>()
            .join(",\n"))
    }
}

/// Load the templates associated with each taxonomy.
fn load_templates(_site_directory: &PathBuf, _config: &Config) -> Result<Paths, String> {
    unimplemented!()
}

// TODO: put this somewhere else when extracting all the parser logic below into
// its own module.
#[derive(Debug)]
enum ParseState<'s> {
    CodeBlock(&'s syntect::parsing::SyntaxReference),
    PlainTextBlock,
    NonCode,
}

impl<'s> std::default::Default for ParseState<'s> {
    fn default() -> ParseState<'s> {
        ParseState::NonCode
    }
}

enum Syntax {
    Highlight(syntect::highlighting::Theme),
    TagOnly,
    Off,
}

impl Syntax {
    fn try_from(config_syntax: &SyntaxOption) -> Result<Syntax, String> {
        match config_syntax {
            SyntaxOption::Highlight(theme) => {
                let theme_file = PathBuf::from("data").join(theme);
                let theme = ThemeSet::get_theme(theme_file).map_err(|err| format!("{:?}", err))?;
                Ok(Syntax::Highlight(theme))
            }
            SyntaxOption::TagOnly => Ok(Syntax::TagOnly),
            SyntaxOption::Off => Ok(Syntax::Off),
        }
    }
}

/// Generate content from a configuration.
pub fn build(site_directory: PathBuf) -> Result<(), String> {
    // NOTE: this is almost certainly not what we *ultimately* want here, but
    // it'll do as a starting point. There's a lot of `into_iter` and `collect`
    // happening here, which forces us to eagerly materialize a lot of these
    // collections rather than being able to deal with them in a more
    // "streaming" fashion.
    let config = Config::from_file(&PathBuf::from(&site_directory))?;
    let markdown_paths = load_markdown_paths(&site_directory, &config)?;
    let contents = load_content(&markdown_paths)?;

    let configured_syntax = Syntax::try_from(&config.options.syntax)?;

    // TODO: generate these! Best move: on build, generate a `.packdump`, since
    // that seems to be what Syntect knows how to deal with.
    let syntax_dir = PathBuf::from("data/syntaxes");
    let syntax_set =
        syntect::parsing::SyntaxSet::load_from_folder(syntax_dir).expect("can't load syntaxes");

    let parsed_content: Vec<(&PathBuf, String)> = contents
        .into_iter()
        .map(|(path, content)| {
            let parser = Parser::new_ext(&content, pulldown_cmark::Options::all()).scan(
                ParseState::default(),
                |state, event| match event {
                    Event::Start(Tag::CodeBlock(ref language)) => match state {
                        ParseState::NonCode => {
                            *state = match syntax_set.find_syntax_by_name(&language) {
                                Some(syntax) => ParseState::CodeBlock(syntax),
                                None => ParseState::PlainTextBlock,
                            };

                            // We start every code block with `<pre><code class="...">`
                            // so that we always have semantically correct HTML. However, we
                            // don't add the language class if the language isn't set.
                            let content = if language.len() > 0 {
                                Owned(format!(r#"
"#, language))
                            } else {
                                Borrowed("
")
                            };

                            Some(Event::Html(content))
                        }
                        ParseState::CodeBlock(_) | ParseState::PlainTextBlock => {
                            unreachable!("Bad event/state: {:?} with {:?}", event, state);
                        }
                    },

                    Event::End(Tag::CodeBlock(_)) => match state {
                        ParseState::CodeBlock(_) | ParseState::PlainTextBlock => {
                            *state = ParseState::NonCode;
                            Some(Event::Html(Borrowed("
")))
                        }
                        ParseState::NonCode => {
                            unreachable!("Bad event/state: {:?} with {:?}", event, state);
                        }
                    },

                    // When we are in a text block, we *may* be in a code block,
                    // so we may also need to do whatever highlighting is
                    // specified.
                    Event::Text(s) => {
                        match state {
                            ParseState::CodeBlock(syntax) => match &configured_syntax {
                                Syntax::Highlight(theme) => {
                                    let highlighted = syntect::html::highlighted_html_for_string(
                                        &s,
                                        &syntax_set,
                                        &syntax,
                                        theme,
                                    );

                                    Some(Event::Text(Owned(highlighted)))
                                }
                                Syntax::TagOnly => {
                                    // TODO: implement tag-only parsing with syntect
                                    unimplemented!()
                                }
                                Syntax::Off => Some(Event::Text(s)),
                            },
                            ParseState::PlainTextBlock => Some(Event::Text(s)),
                            ParseState::NonCode => Some(Event::Text(s)),
                        }
                    }

                    // TODO: accumulate footnote references and always put them
                    // at the end. Also, generate back-links for them, unless or
                    // until pulldown-cmark does so.
                    Event::FootnoteReference(s) => Some(Event::FootnoteReference(s)),

                    Event::Start(s) => Some(Event::Start(s)),
                    Event::End(e) => Some(Event::End(e)),
                    Event::InlineHtml(s) => Some(Event::InlineHtml(s)),
                    Event::Html(h) => Some(Event::Html(h)),
                    Event::SoftBreak => Some(Event::SoftBreak),
                    Event::HardBreak => Some(Event::HardBreak),
                },
            );

            // These numbers are derived from a brief survey of content in my
            // own website historically, including both code-heavy and
            // prose-heavy posts. It's probably worth exploring other corpora
            // to see if this is basically right.
            let length_bound_estimate = match &config.options.syntax {
                SyntaxOption::Off | SyntaxOption::TagOnly => 2,
                SyntaxOption::Highlight(_) => 8, // TODO: check other pages
            };

            let mut html = String::with_capacity(content.len() * length_bound_estimate);
            cmark::html::push_html(&mut html, parser);

            (path, html)
        })
        .collect();

    // TODO: extract this as part of the writing it out process.
    let output_dir = site_directory.join(&config.directories.output);

    if output_dir.exists() && !output_dir.is_dir() {
        return Err(format!(
            "A file with the same name as the output directory {:?} already exists!",
            output_dir
        ));
    }

    if !output_dir.exists() {
        std::fs::create_dir(&output_dir).expect(&format!(
            "could not create output directory {:?}",
            &output_dir
        ));
    }

    for (path, content) in parsed_content {
        let file_path = path
            .file_name()
            .ok_or(format!("invalid file: {:?}", path))?;

        let dest = output_dir.join(file_path).with_extension("html");

        let mut fd = match File::create(&dest) {
            Ok(file) => file,
            Err(reason) => {
                return Err(format!(
                    "Could not open {} for write: {}",
                    dest.to_string_lossy(),
                    reason
                ));
            }
        };

        let result = write!(fd, "{}", content);
        if let Err(reason) = result {
            return Err(format!("{:?}", reason.kind()));
        }
    }

    Ok(())
}
<commit_msg>style: results of `cargo fix --edition-idioms`.<commit_after>//! Generate the site content.

// Standard library
use std::borrow::Cow::{Borrowed, Owned};
use std::fs::File;
use std::io::prelude::*;
use std::path::{Path, PathBuf};

// Third party
use glob::{glob, Paths};
use pulldown_cmark as cmark;
use pulldown_cmark::{Event, Parser, Tag};
use syntect::highlighting::ThemeSet;

// First party
use crate::config::{Config, SyntaxOption};

/// Load the `Paths` for all markdown files in the specified content directory.
///
/// If the specified configuration values are wrong, we return a vector of all
/// the errors in the config as far as this is concerned.
///
/// TODO: move that validation elsewhere? Should probably extract a more general
/// site configuration validator.
fn load_markdown_paths(site_directory: &PathBuf, config: &Config) -> Result<Vec<PathBuf>, String> {
    let site = site_directory
        .to_str()
        .ok_or(format!("bad `site`: {:?}", site_directory));

    let directories = config.directories.content.to_str().ok_or(format!(
        "bad content directory: {:?}",
        config.directories.content
    ));

    let (site, directories) = match (site, directories) {
        (Err(s), Err(d)) => Err(s + "\n" + &d),
        (Err(s), Ok(_)) => Err(s),
        (Ok(_), Err(d)) => Err(d),
        (Ok(s), Ok(d)) => Ok((s, d)),
    }?;

    let content_glob = format!("{}/{}/**/*.md", site, directories);

    let paths = glob(&content_glob).map_err(|err| format!("{:?}", err))?;

    let (valid, errors): (Vec<Result<PathBuf, _>>, Vec<Result<_, String>>) = paths
        .map(|glob_result| glob_result.map_err(|err| format!("{:?}", err)))
        .partition(|path| path.is_ok());

    let errors: Vec<String> = errors.into_iter().map(|e| e.unwrap_err()).collect();
    if errors.len() == 0 {
        Ok(valid.into_iter().map(|v| v.unwrap()).collect())
    } else {
        Err(errors.join(",\n"))
    }
}

// TODO: this is an obvious candidate for a `HashMap`, so use that instead.
type LoadTuple<'a> = (&'a PathBuf, Result<String, String>);

// Instead of using `.collect` at the end of these, I *should* be able to use
// something like `Result<impl Iterator<HashMap<PathBuf, String>, String>`, even
// if the type isn't *quite* taht.
fn load_content<'p>(paths: &'p [PathBuf]) -> Result<Vec<(&'p PathBuf, String)>, String> {
    let (contents, errs): (Vec<LoadTuple<'_>>, Vec<LoadTuple<'_>>) = paths
        .into_iter()
        .map(|path| {
            let content = std::fs::read_to_string(path).map_err(|e| format!("{:?}", e));
            (path, content)
        })
        .partition(|(_, result)| result.is_ok());

    if errs.len() == 0 {
        Ok(contents
            .into_iter()
            .map(|(path, contents)| (path, contents.unwrap()))
            .collect())
    } else {
        Err(errs
            .into_iter()
            .map(|(_path, err)| err.unwrap_err())
            .collect::<Vec<String>>()
            .join(",\n"))
    }
}

/// Load the templates associated with each taxonomy.
fn load_templates(_site_directory: &PathBuf, _config: &Config) -> Result<Paths, String> {
    unimplemented!()
}

// TODO: put this somewhere else when extracting all the parser logic below into
// its own module.
#[derive(Debug)]
enum ParseState<'s> {
    CodeBlock(&'s syntect::parsing::SyntaxReference),
    PlainTextBlock,
    NonCode,
}

impl<'s> std::default::Default for ParseState<'s> {
    fn default() -> ParseState<'s> {
        ParseState::NonCode
    }
}

enum Syntax {
    Highlight(syntect::highlighting::Theme),
    TagOnly,
    Off,
}

impl Syntax {
    fn try_from(config_syntax: &SyntaxOption) -> Result<Syntax, String> {
        match config_syntax {
            SyntaxOption::Highlight(theme) => {
                let theme_file = PathBuf::from("data").join(theme);
                let theme = ThemeSet::get_theme(theme_file).map_err(|err| format!("{:?}", err))?;
                Ok(Syntax::Highlight(theme))
            }
            SyntaxOption::TagOnly => Ok(Syntax::TagOnly),
            SyntaxOption::Off => Ok(Syntax::Off),
        }
    }
}

/// Generate content from a configuration.
pub fn build(site_directory: PathBuf) -> Result<(), String> {
    // NOTE: this is almost certainly not what we *ultimately* want here, but
    // it'll do as a starting point. There's a lot of `into_iter` and `collect`
    // happening here, which forces us to eagerly materialize a lot of these
    // collections rather than being able to deal with them in a more
    // "streaming" fashion.
    let config = Config::from_file(&PathBuf::from(&site_directory))?;
    let markdown_paths = load_markdown_paths(&site_directory, &config)?;
    let contents = load_content(&markdown_paths)?;

    let configured_syntax = Syntax::try_from(&config.options.syntax)?;

    // TODO: generate these! Best move: on build, generate a `.packdump`, since
    // that seems to be what Syntect knows how to deal with.
    let syntax_dir = PathBuf::from("data/syntaxes");
    let syntax_set =
        syntect::parsing::SyntaxSet::load_from_folder(syntax_dir).expect("can't load syntaxes");

    let parsed_content: Vec<(&PathBuf, String)> = contents
        .into_iter()
        .map(|(path, content)| {
            let parser = Parser::new_ext(&content, pulldown_cmark::Options::all()).scan(
                ParseState::default(),
                |state, event| match event {
                    Event::Start(Tag::CodeBlock(ref language)) => match state {
                        ParseState::NonCode => {
                            *state = match syntax_set.find_syntax_by_name(&language) {
                                Some(syntax) => ParseState::CodeBlock(syntax),
                                None => ParseState::PlainTextBlock,
                            };

                            // We start every code block with `<pre><code class="...">`
                            // so that we always have semantically correct HTML. However, we
                            // don't add the language class if the language isn't set.
                            let content = if language.len() > 0 {
                                Owned(format!(r#"
"#, language))
                            } else {
                                Borrowed("
")
                            };

                            Some(Event::Html(content))
                        }
                        ParseState::CodeBlock(_) | ParseState::PlainTextBlock => {
                            unreachable!("Bad event/state: {:?} with {:?}", event, state);
                        }
                    },

                    Event::End(Tag::CodeBlock(_)) => match state {
                        ParseState::CodeBlock(_) | ParseState::PlainTextBlock => {
                            *state = ParseState::NonCode;
                            Some(Event::Html(Borrowed("
")))
                        }
                        ParseState::NonCode => {
                            unreachable!("Bad event/state: {:?} with {:?}", event, state);
                        }
                    },

                    // When we are in a text block, we *may* be in a code block,
                    // so we may also need to do whatever highlighting is
                    // specified.
                    Event::Text(s) => {
                        match state {
                            ParseState::CodeBlock(syntax) => match &configured_syntax {
                                Syntax::Highlight(theme) => {
                                    let highlighted = syntect::html::highlighted_html_for_string(
                                        &s,
                                        &syntax_set,
                                        &syntax,
                                        theme,
                                    );

                                    Some(Event::Text(Owned(highlighted)))
                                }
                                Syntax::TagOnly => {
                                    // TODO: implement tag-only parsing with syntect
                                    unimplemented!()
                                }
                                Syntax::Off => Some(Event::Text(s)),
                            },
                            ParseState::PlainTextBlock => Some(Event::Text(s)),
                            ParseState::NonCode => Some(Event::Text(s)),
                        }
                    }

                    // TODO: accumulate footnote references and always put them
                    // at the end. Also, generate back-links for them, unless or
                    // until pulldown-cmark does so.
                    Event::FootnoteReference(s) => Some(Event::FootnoteReference(s)),

                    Event::Start(s) => Some(Event::Start(s)),
                    Event::End(e) => Some(Event::End(e)),
                    Event::InlineHtml(s) => Some(Event::InlineHtml(s)),
                    Event::Html(h) => Some(Event::Html(h)),
                    Event::SoftBreak => Some(Event::SoftBreak),
                    Event::HardBreak => Some(Event::HardBreak),
                },
            );

            // These numbers are derived from a brief survey of content in my
            // own website historically, including both code-heavy and
            // prose-heavy posts. It's probably worth exploring other corpora
            // to see if this is basically right.
            let length_bound_estimate = match &config.options.syntax {
                SyntaxOption::Off | SyntaxOption::TagOnly => 2,
                SyntaxOption::Highlight(_) => 8, // TODO: check other pages
            };

            let mut html = String::with_capacity(content.len() * length_bound_estimate);
            cmark::html::push_html(&mut html, parser);

            (path, html)
        })
        .collect();

    // TODO: extract this as part of the writing it out process.
    let output_dir = site_directory.join(&config.directories.output);

    if output_dir.exists() && !output_dir.is_dir() {
        return Err(format!(
            "A file with the same name as the output directory {:?} already exists!",
            output_dir
        ));
    }

    if !output_dir.exists() {
        std::fs::create_dir(&output_dir).expect(&format!(
            "could not create output directory {:?}",
            &output_dir
        ));
    }

    for (path, content) in parsed_content {
        let file_path = path
            .file_name()
            .ok_or(format!("invalid file: {:?}", path))?;

        let dest = output_dir.join(file_path).with_extension("html");

        let mut fd = match File::create(&dest) {
            Ok(file) => file,
            Err(reason) => {
                return Err(format!(
                    "Could not open {} for write: {}",
                    dest.to_string_lossy(),
                    reason
                ));
            }
        };

        let result = write!(fd, "{}", content);
        if let Err(reason) = result {
            return Err(format!("{:?}", reason.kind()));
        }
    }

    Ok(())
}
<|endoftext|>


<commit_before>use std::io::{IoResult, IoError, EndOfFile};
use std::comm::{Sender, Receiver};

use framing;
use framing::{ContentHeaderFrame, Frame};
use table::Table;
use protocol;
use protocol::{MethodFrame, channel, basic};

pub struct Channel {
	chan: (Sender<Frame>, Receiver<Frame>),
	pub id: u16
}

impl Channel {
	pub fn new(id: u16, chan: (Sender<Frame>, Receiver<Frame>)) -> Channel {
		Channel{id: id, chan: chan}
	}

	pub fn open(&self) -> IoResult<protocol::channel::OpenOk> {
		let meth = protocol::channel::Open {out_of_band: "".to_string()};
        self.rpc(&meth, "channel.open-ok")
	}
	pub fn close(&self, reply_code: u16, reply_text: &str) {
		let close = &channel::Close {reply_code: reply_code, reply_text: reply_text.to_string(), class_id: 0, method_id: 0};
		let reply: channel::CloseOk = self.rpc(close, "channel.close-ok").unwrap();
	}

	pub fn read(&self) -> Frame {
		self.chan.ref1().recv()
	}

	fn write(&self, frame: Frame) {
		self.chan.ref0().send(frame)
	}

	pub fn send_method_frame(&self, method: &protocol::Method) {
        debug!("Sending method {} to channel {}", method.name(), self.id);
        self.write(Frame {frame_type: framing::METHOD, channel: self.id, payload: MethodFrame::encode_method(method) })
    }

    pub fn rpc<T: protocol::Method>(&self, method: &protocol::Method, expected_reply: &str) -> IoResult<T> {
        let method_frame = self.raw_rpc(method);
        match method_frame.method_name() {
            m_name if m_name == expected_reply => protocol::Method::decode(method_frame),
            m_name => fail!("Unexpected method frame: {}, expected: {}", m_name, expected_reply)
        }
    }

    pub fn raw_rpc(&self, method: &protocol::Method) -> MethodFrame {
        self.send_method_frame(method);
        MethodFrame::decode(self.read())
    }

	pub fn read_headers(&self) -> IoResult<ContentHeaderFrame> {
		ContentHeaderFrame::decode(self.read())
	}

	pub fn read_body(&self, size: u64) -> IoResult<Vec<u8>> {
    	let mut body = Vec::with_capacity(size as uint);
		while body.len() < size as uint {
    		body = body.append(self.read().payload.as_slice())
    	}
    	Ok(body)
	}

	pub fn basic_publish(&self, ticket: u16, exchange: &str, routing_key: &str, mandatory: bool, immediate: bool,
						 properties: basic::BasicProperties, content: Vec<u8>) {
		let publish = &protocol::basic::Publish {
			ticket: ticket, exchange: exchange.to_string(),
			routing_key: routing_key.to_string(), mandatory: mandatory, immediate: immediate};
		let properties_flags = properties.flags();
		let content_header = ContentHeaderFrame { content_class: 60, weight: 0, body_size: content.len() as u64,
			properties_flags: properties_flags, properties: properties.encode() };
		let content_header_frame = framing::Frame {frame_type: framing::HEADERS, channel: self.id,
			payload: content_header.encode() };

		self.send_method_frame(publish);
		self.write(content_header_frame);
		self.write(framing::Frame { frame_type: framing::BODY, channel: self.id, payload: content});
	}

	pub fn basic_get(&self, ticket: u16, queue: &str, no_ack: bool) -> IoResult<(protocol::basic::BasicProperties, Vec<u8>)> {
  		let get = &basic::Get{ ticket: ticket, queue: queue.to_string(), no_ack: no_ack };
  		let method_frame = self.raw_rpc(get);
  		match method_frame.method_name() {
  			"basic.get-ok" => {
			    let reply: basic::GetOk = try!(protocol::Method::decode(method_frame));
			    let headers = try!(self.read_headers());
			    let properties = try!(basic::BasicProperties::decode(headers.clone()));
			    let body = try!(self.read_body(headers.body_size));
			    Ok((properties, body))
  			}
  			"basic.get-empty" => return Err(IoError{kind: EndOfFile, desc: "The queue is empty", detail: None}),
  			method => fail!(format!("Not expected method: {}", method))
  		}
	}

	pub fn exchange_declare(&self, ticket: u16, exchange: &str, _type: &str, passive: bool, durable: bool,
		auto_delete: bool, internal: bool, nowait: bool, arguments: Table) -> IoResult<protocol::exchange::DeclareOk> {
		let declare = protocol::exchange::Declare {
			ticket: ticket, exchange: exchange.to_string(), _type: _type.to_string(), passive: passive, durable: durable,
			auto_delete: auto_delete, internal: internal, nowait: nowait, arguments: arguments
		};
		self.rpc(&declare,"exchange.declare-ok")
    }

	pub fn queue_declare(&self, ticket: u16, queue: &str, passive: bool, durable: bool, exclusive: bool,
		auto_delete: bool, nowait: bool, arguments: Table) -> IoResult<protocol::queue::DeclareOk> {
		let declare = protocol::queue::Declare {
			ticket: ticket, queue: queue.to_string(), passive: passive, durable: durable, exclusive: exclusive,
			auto_delete: auto_delete, nowait: nowait, arguments: arguments
		};
		self.rpc(&declare, "queue.declare-ok")
	}

	pub fn queue_bind(&self, ticket: u16, queue: &str, exchange: &str, routing_key: &str, nowait: bool, arguments: Table) -> IoResult<protocol::queue::BindOk> {
		let bind = protocol::queue::Bind {
			ticket: ticket, queue: queue.to_string(), exchange: exchange.to_string(),
			routing_key: routing_key.to_string(), nowait: nowait, arguments: arguments
		};
		self.rpc(&bind, "queue.bind-ok")
	}
}
<commit_msg>Fix deprecation warnings. (also covert tabs to spaces).<commit_after>use std::io::{IoResult, IoError, EndOfFile};
use std::comm::{Sender, Receiver};

use framing;
use framing::{ContentHeaderFrame, Frame};
use table::Table;
use protocol;
use protocol::{MethodFrame, channel, basic};

pub struct Channel {
    chan: (Sender<Frame>, Receiver<Frame>),
    pub id: u16
}

impl Channel {
    pub fn new(id: u16, chan: (Sender<Frame>, Receiver<Frame>)) -> Channel {
        Channel{id: id, chan: chan}
    }

    pub fn open(&self) -> IoResult<protocol::channel::OpenOk> {
        let meth = protocol::channel::Open {out_of_band: "".to_string()};
        self.rpc(&meth, "channel.open-ok")
    }
    pub fn close(&self, reply_code: u16, reply_text: &str) {
        let close = &channel::Close {reply_code: reply_code, reply_text: reply_text.to_string(), class_id: 0, method_id: 0};
        let reply: channel::CloseOk = self.rpc(close, "channel.close-ok").unwrap();
    }

    pub fn read(&self) -> Frame {
        self.chan.ref1().recv()
    }

    fn write(&self, frame: Frame) {
        self.chan.ref0().send(frame)
    }

    pub fn send_method_frame(&self, method: &protocol::Method) {
        debug!("Sending method {} to channel {}", method.name(), self.id);
        self.write(Frame {frame_type: framing::METHOD, channel: self.id, payload: MethodFrame::encode_method(method) })
    }

    pub fn rpc<T: protocol::Method>(&self, method: &protocol::Method, expected_reply: &str) -> IoResult<T> {
        let method_frame = self.raw_rpc(method);
        match method_frame.method_name() {
            m_name if m_name == expected_reply => protocol::Method::decode(method_frame),
            m_name => fail!("Unexpected method frame: {}, expected: {}", m_name, expected_reply)
        }
    }

    pub fn raw_rpc(&self, method: &protocol::Method) -> MethodFrame {
        self.send_method_frame(method);
        MethodFrame::decode(self.read())
    }

    pub fn read_headers(&self) -> IoResult<ContentHeaderFrame> {
        ContentHeaderFrame::decode(self.read())
    }

    pub fn read_body(&self, size: u64) -> IoResult<Vec<u8>> {
        let mut body = Vec::with_capacity(size as uint);
        while body.len() < size as uint {
            body.extend(self.read().payload.into_iter())
        }
        Ok(body)
    }

    pub fn basic_publish(&self, ticket: u16, exchange: &str, routing_key: &str, mandatory: bool, immediate: bool,
                         properties: basic::BasicProperties, content: Vec<u8>) {
        let publish = &protocol::basic::Publish {
            ticket: ticket, exchange: exchange.to_string(),
            routing_key: routing_key.to_string(), mandatory: mandatory, immediate: immediate};
        let properties_flags = properties.flags();
        let content_header = ContentHeaderFrame { content_class: 60, weight: 0, body_size: content.len() as u64,
            properties_flags: properties_flags, properties: properties.encode() };
        let content_header_frame = framing::Frame {frame_type: framing::HEADERS, channel: self.id,
            payload: content_header.encode() };

        self.send_method_frame(publish);
        self.write(content_header_frame);
        self.write(framing::Frame { frame_type: framing::BODY, channel: self.id, payload: content});
    }

    pub fn basic_get(&self, ticket: u16, queue: &str, no_ack: bool) -> IoResult<(protocol::basic::BasicProperties, Vec<u8>)> {
        let get = &basic::Get{ ticket: ticket, queue: queue.to_string(), no_ack: no_ack };
        let method_frame = self.raw_rpc(get);
        match method_frame.method_name() {
            "basic.get-ok" => {
                let reply: basic::GetOk = try!(protocol::Method::decode(method_frame));
                let headers = try!(self.read_headers());
                let properties = try!(basic::BasicProperties::decode(headers.clone()));
                let body = try!(self.read_body(headers.body_size));
                Ok((properties, body))
            }
            "basic.get-empty" => return Err(IoError{kind: EndOfFile, desc: "The queue is empty", detail: None}),
            method => fail!(format!("Not expected method: {}", method))
        }
    }

    pub fn exchange_declare(&self, ticket: u16, exchange: &str, _type: &str, passive: bool, durable: bool,
        auto_delete: bool, internal: bool, nowait: bool, arguments: Table) -> IoResult<protocol::exchange::DeclareOk> {
        let declare = protocol::exchange::Declare {
            ticket: ticket, exchange: exchange.to_string(), _type: _type.to_string(), passive: passive, durable: durable,
            auto_delete: auto_delete, internal: internal, nowait: nowait, arguments: arguments
        };
        self.rpc(&declare,"exchange.declare-ok")
    }

    pub fn queue_declare(&self, ticket: u16, queue: &str, passive: bool, durable: bool, exclusive: bool,
        auto_delete: bool, nowait: bool, arguments: Table) -> IoResult<protocol::queue::DeclareOk> {
        let declare = protocol::queue::Declare {
            ticket: ticket, queue: queue.to_string(), passive: passive, durable: durable, exclusive: exclusive,
            auto_delete: auto_delete, nowait: nowait, arguments: arguments
        };
        self.rpc(&declare, "queue.declare-ok")
    }

    pub fn queue_bind(&self, ticket: u16, queue: &str, exchange: &str, routing_key: &str, nowait: bool, arguments: Table) -> IoResult<protocol::queue::BindOk> {
        let bind = protocol::queue::Bind {
            ticket: ticket, queue: queue.to_string(), exchange: exchange.to_string(),
            routing_key: routing_key.to_string(), nowait: nowait, arguments: arguments
        };
        self.rpc(&bind, "queue.bind-ok")
    }
}
<|endoftext|>