<commit_before>//! An implementation of `loader::Handle`, based on `walkdir` and `notify`.
//!
//! The file watching bits here are untested and quite probably buggy. For this
//! reason, by default we don't watch files and rely on editor's file watching
//! capabilities.
//!
//! Hopefully, one day a reliable file watching/walking crate appears on
//! crates.io, and we can reduce this to trivial glue code.
use std::convert::TryFrom;

use crossbeam_channel::{never, select, unbounded, Receiver, Sender};
use notify::{RecommendedWatcher, RecursiveMode, Watcher};
use paths::{AbsPath, AbsPathBuf};
use vfs::loader;
use walkdir::WalkDir;

#[derive(Debug)]
pub struct NotifyHandle {
    // Relative order of fields below is significant.
    sender: Sender<Message>,
    thread: jod_thread::JoinHandle,
}

#[derive(Debug)]
enum Message {
    Config(loader::Config),
    Invalidate(AbsPathBuf),
}

impl loader::Handle for NotifyHandle {
    fn spawn(sender: loader::Sender) -> NotifyHandle {
        let actor = NotifyActor::new(sender);
        let (sender, receiver) = unbounded::<Message>();
        let thread = jod_thread::spawn(move || actor.run(receiver));
        NotifyHandle { sender, thread }
    }
    fn set_config(&mut self, config: loader::Config) {
        self.sender.send(Message::Config(config)).unwrap()
    }
    fn invalidate(&mut self, path: AbsPathBuf) {
        self.sender.send(Message::Invalidate(path)).unwrap();
    }
    fn load_sync(&mut self, path: &AbsPath) -> Option<Vec<u8>> {
        read(path)
    }
}

type NotifyEvent = notify::Result<notify::Event>;

struct NotifyActor {
    sender: loader::Sender,
    watched_entries: Vec<loader::Entry>,
    // Drop order is significant.
    watcher: Option<(RecommendedWatcher, Receiver<NotifyEvent>)>,
}

#[derive(Debug)]
enum Event {
    Message(Message),
    NotifyEvent(NotifyEvent),
}

impl NotifyActor {
    fn new(sender: loader::Sender) -> NotifyActor {
        NotifyActor { sender, watched_entries: Vec::new(), watcher: None }
    }
    fn next_event(&self, receiver: &Receiver<Message>) -> Option<Event> {
        let watcher_receiver = self.watcher.as_ref().map(|(_, receiver)| receiver);
        select! {
            recv(receiver) -> it => it.ok().map(Event::Message),
            recv(watcher_receiver.unwrap_or(&never())) -> it => Some(Event::NotifyEvent(it.unwrap())),
        }
    }
    fn run(mut self, inbox: Receiver<Message>) {
        while let Some(event) = self.next_event(&inbox) {
            log::debug!("vfs-notify event: {:?}", event);
            match event {
                Event::Message(msg) => match msg {
                    Message::Config(config) => {
                        self.watcher = None;
                        if !config.watch.is_empty() {
                            let (watcher_sender, watcher_receiver) = unbounded();
                            let watcher = log_notify_error(Watcher::new_immediate(move |event| {
                                watcher_sender.send(event).unwrap()
                            }));
                            self.watcher = watcher.map(|it| (it, watcher_receiver));
                        }

                        let n_total = config.load.len();
                        self.send(loader::Message::Progress { n_total, n_done: 0 });

                        self.watched_entries.clear();

                        for (i, entry) in config.load.into_iter().enumerate() {
                            let watch = config.watch.contains(&i);
                            if watch {
                                self.watched_entries.push(entry.clone())
                            }
                            let files = self.load_entry(entry, watch);
                            self.send(loader::Message::Loaded { files });
                            self.send(loader::Message::Progress { n_total, n_done: i + 1 });
                        }
                    }
                    Message::Invalidate(path) => {
                        let contents = read(path.as_path());
                        let files = vec![(path, contents)];
                        self.send(loader::Message::Loaded { files });
                    }
                },
                Event::NotifyEvent(event) => {
                    if let Some(event) = log_notify_error(event) {
                        let files = event
                            .paths
                            .into_iter()
                            .map(|path| AbsPathBuf::try_from(path).unwrap())
                            .filter_map(|path| {
                                if path.is_dir()
                                    && self
                                        .watched_entries
                                        .iter()
                                        .any(|entry| entry.contains_dir(&path))
                                {
                                    self.watch(path);
                                    return None;
                                }

                                if !path.is_file() {
                                    return None;
                                }
                                if !self
                                    .watched_entries
                                    .iter()
                                    .any(|entry| entry.contains_file(&path))
                                {
                                    return None;
                                }

                                let contents = read(&path);
                                Some((path, contents))
                            })
                            .collect();
                        self.send(loader::Message::Loaded { files })
                    }
                }
            }
        }
    }
    fn load_entry(
        &mut self,
        entry: loader::Entry,
        watch: bool,
    ) -> Vec<(AbsPathBuf, Option<Vec<u8>>)> {
        match entry {
            loader::Entry::Files(files) => files
                .into_iter()
                .map(|file| {
                    if watch {
                        self.watch(file.clone())
                    }
                    let contents = read(file.as_path());
                    (file, contents)
                })
                .collect::<Vec<_>>(),
            loader::Entry::Directories(dirs) => {
                let mut res = Vec::new();

                for root in dirs.include.iter() {
                    let walkdir = WalkDir::new(root).into_iter().filter_entry(|entry| {
                        if !entry.file_type().is_dir() {
                            return true;
                        }
                        let path = AbsPath::assert(entry.path());
                        root == path
                            || dirs.exclude.iter().chain(&dirs.include).all(|it| it != path)
                    });

                    let files = walkdir.filter_map(|it| it.ok()).filter_map(|entry| {
                        let is_dir = entry.file_type().is_dir();
                        let is_file = entry.file_type().is_file();
                        let abs_path = AbsPathBuf::assert(entry.into_path());
                        if is_dir && watch {
                            self.watch(abs_path.clone());
                        }
                        if !is_file {
                            return None;
                        }
                        let ext = abs_path.extension().unwrap_or_default();
                        if dirs.extensions.iter().all(|it| it.as_str() != ext) {
                            return None;
                        }
                        Some(abs_path)
                    });

                    res.extend(files.map(|file| {
                        let contents = read(file.as_path());
                        (file, contents)
                    }));
                }
                res
            }
        }
    }

    fn watch(&mut self, path: AbsPathBuf) {
        if let Some((watcher, _)) = &mut self.watcher {
            log_notify_error(watcher.watch(&path, RecursiveMode::NonRecursive));
        }
    }
    fn send(&mut self, msg: loader::Message) {
        (self.sender)(msg)
    }
}

fn read(path: &AbsPath) -> Option<Vec<u8>> {
    std::fs::read(path).ok()
}

fn log_notify_error<T>(res: notify::Result<T>) -> Option<T> {
    res.map_err(|err| log::warn!("notify error: {}", err)).ok()
}
<commit_msg>Follow symlinks when walking project trees<commit_after>//! An implementation of `loader::Handle`, based on `walkdir` and `notify`.
//!
//! The file watching bits here are untested and quite probably buggy. For this
//! reason, by default we don't watch files and rely on editor's file watching
//! capabilities.
//!
//! Hopefully, one day a reliable file watching/walking crate appears on
//! crates.io, and we can reduce this to trivial glue code.
use std::convert::TryFrom;

use crossbeam_channel::{never, select, unbounded, Receiver, Sender};
use notify::{RecommendedWatcher, RecursiveMode, Watcher};
use paths::{AbsPath, AbsPathBuf};
use vfs::loader;
use walkdir::WalkDir;

#[derive(Debug)]
pub struct NotifyHandle {
    // Relative order of fields below is significant.
    sender: Sender<Message>,
    thread: jod_thread::JoinHandle,
}

#[derive(Debug)]
enum Message {
    Config(loader::Config),
    Invalidate(AbsPathBuf),
}

impl loader::Handle for NotifyHandle {
    fn spawn(sender: loader::Sender) -> NotifyHandle {
        let actor = NotifyActor::new(sender);
        let (sender, receiver) = unbounded::<Message>();
        let thread = jod_thread::spawn(move || actor.run(receiver));
        NotifyHandle { sender, thread }
    }
    fn set_config(&mut self, config: loader::Config) {
        self.sender.send(Message::Config(config)).unwrap()
    }
    fn invalidate(&mut self, path: AbsPathBuf) {
        self.sender.send(Message::Invalidate(path)).unwrap();
    }
    fn load_sync(&mut self, path: &AbsPath) -> Option<Vec<u8>> {
        read(path)
    }
}

type NotifyEvent = notify::Result<notify::Event>;

struct NotifyActor {
    sender: loader::Sender,
    watched_entries: Vec<loader::Entry>,
    // Drop order is significant.
    watcher: Option<(RecommendedWatcher, Receiver<NotifyEvent>)>,
}

#[derive(Debug)]
enum Event {
    Message(Message),
    NotifyEvent(NotifyEvent),
}

impl NotifyActor {
    fn new(sender: loader::Sender) -> NotifyActor {
        NotifyActor { sender, watched_entries: Vec::new(), watcher: None }
    }
    fn next_event(&self, receiver: &Receiver<Message>) -> Option<Event> {
        let watcher_receiver = self.watcher.as_ref().map(|(_, receiver)| receiver);
        select! {
            recv(receiver) -> it => it.ok().map(Event::Message),
            recv(watcher_receiver.unwrap_or(&never())) -> it => Some(Event::NotifyEvent(it.unwrap())),
        }
    }
    fn run(mut self, inbox: Receiver<Message>) {
        while let Some(event) = self.next_event(&inbox) {
            log::debug!("vfs-notify event: {:?}", event);
            match event {
                Event::Message(msg) => match msg {
                    Message::Config(config) => {
                        self.watcher = None;
                        if !config.watch.is_empty() {
                            let (watcher_sender, watcher_receiver) = unbounded();
                            let watcher = log_notify_error(Watcher::new_immediate(move |event| {
                                watcher_sender.send(event).unwrap()
                            }));
                            self.watcher = watcher.map(|it| (it, watcher_receiver));
                        }

                        let n_total = config.load.len();
                        self.send(loader::Message::Progress { n_total, n_done: 0 });

                        self.watched_entries.clear();

                        for (i, entry) in config.load.into_iter().enumerate() {
                            let watch = config.watch.contains(&i);
                            if watch {
                                self.watched_entries.push(entry.clone())
                            }
                            let files = self.load_entry(entry, watch);
                            self.send(loader::Message::Loaded { files });
                            self.send(loader::Message::Progress { n_total, n_done: i + 1 });
                        }
                    }
                    Message::Invalidate(path) => {
                        let contents = read(path.as_path());
                        let files = vec![(path, contents)];
                        self.send(loader::Message::Loaded { files });
                    }
                },
                Event::NotifyEvent(event) => {
                    if let Some(event) = log_notify_error(event) {
                        let files = event
                            .paths
                            .into_iter()
                            .map(|path| AbsPathBuf::try_from(path).unwrap())
                            .filter_map(|path| {
                                if path.is_dir()
                                    && self
                                        .watched_entries
                                        .iter()
                                        .any(|entry| entry.contains_dir(&path))
                                {
                                    self.watch(path);
                                    return None;
                                }

                                if !path.is_file() {
                                    return None;
                                }
                                if !self
                                    .watched_entries
                                    .iter()
                                    .any(|entry| entry.contains_file(&path))
                                {
                                    return None;
                                }

                                let contents = read(&path);
                                Some((path, contents))
                            })
                            .collect();
                        self.send(loader::Message::Loaded { files })
                    }
                }
            }
        }
    }
    fn load_entry(
        &mut self,
        entry: loader::Entry,
        watch: bool,
    ) -> Vec<(AbsPathBuf, Option<Vec<u8>>)> {
        match entry {
            loader::Entry::Files(files) => files
                .into_iter()
                .map(|file| {
                    if watch {
                        self.watch(file.clone())
                    }
                    let contents = read(file.as_path());
                    (file, contents)
                })
                .collect::<Vec<_>>(),
            loader::Entry::Directories(dirs) => {
                let mut res = Vec::new();

                for root in dirs.include.iter() {
                    let walkdir =
                        WalkDir::new(root).follow_links(true).into_iter().filter_entry(|entry| {
                            if !entry.file_type().is_dir() {
                                return true;
                            }
                            let path = AbsPath::assert(entry.path());
                            root == path
                                || dirs.exclude.iter().chain(&dirs.include).all(|it| it != path)
                        });

                    let files = walkdir.filter_map(|it| it.ok()).filter_map(|entry| {
                        let is_dir = entry.file_type().is_dir();
                        let is_file = entry.file_type().is_file();
                        let abs_path = AbsPathBuf::assert(entry.into_path());
                        if is_dir && watch {
                            self.watch(abs_path.clone());
                        }
                        if !is_file {
                            return None;
                        }
                        let ext = abs_path.extension().unwrap_or_default();
                        if dirs.extensions.iter().all(|it| it.as_str() != ext) {
                            return None;
                        }
                        Some(abs_path)
                    });

                    res.extend(files.map(|file| {
                        let contents = read(file.as_path());
                        (file, contents)
                    }));
                }
                res
            }
        }
    }

    fn watch(&mut self, path: AbsPathBuf) {
        if let Some((watcher, _)) = &mut self.watcher {
            log_notify_error(watcher.watch(&path, RecursiveMode::NonRecursive));
        }
    }
    fn send(&mut self, msg: loader::Message) {
        (self.sender)(msg)
    }
}

fn read(path: &AbsPath) -> Option<Vec<u8>> {
    std::fs::read(path).ok()
}

fn log_notify_error<T>(res: notify::Result<T>) -> Option<T> {
    res.map_err(|err| log::warn!("notify error: {}", err)).ok()
}
<|endoftext|>


<commit_before>use std::{cmp, mem};

use orbclient::{Color, Event, EventOption, Window};
use orbclient::event;

const BLACK: Color = Color::rgb(0, 0, 0);
const RED: Color = Color::rgb(194, 54, 33);
const GREEN: Color = Color::rgb(37, 188, 36);
const YELLOW: Color = Color::rgb(173, 173, 39);
const BLUE: Color = Color::rgb(73, 46, 225);
const MAGENTA: Color = Color::rgb(211, 56, 211);
const CYAN: Color = Color::rgb(51, 187, 200);
const WHITE: Color = Color::rgb(203, 204, 205);

pub struct Console {
    pub window: Box<Window>,
    pub point_x: i32,
    pub point_y: i32,
    pub foreground: Color,
    pub background: Color,
    pub redraw: bool,
    pub command: String,
    pub escape: bool,
    pub escape_sequence: bool,
    pub sequence: Vec<String>,
    pub raw_mode: bool,
}

impl Console {
    pub fn new() -> Console {
        let mut window = Window::new(-1, -1, 640, 480, "Terminal").unwrap();
        window.sync();
        Console {
            window: window,
            point_x: 0,
            point_y: 0,
            foreground: WHITE,
            background: BLACK,
            redraw: true,
            command: String::new(),
            escape: false,
            escape_sequence: false,
            sequence: Vec::new(),
            raw_mode: false,
        }
    }

    pub fn code(&mut self, c: char) {
        if self.escape_sequence {
            match c {
                '0' ... '9' => {
                    // Add a number to the sequence list
                    if let Some(mut value) = self.sequence.last_mut() {
                        value.push(c);
                    }
                },
                ';' => {
                    // Split sequence into list
                    self.sequence.push(String::new());
                },
                'm' => {
                    // Display attributes
                    for value in self.sequence.iter() {
                        match value.as_str() {
                            "0" => {
                                self.foreground = WHITE;
                                self.background = BLACK;
                            },
                            "30" => self.foreground = BLACK,
                            "31" => self.foreground = RED,
                            "32" => self.foreground = GREEN,
                            "33" => self.foreground = YELLOW,
                            "34" => self.foreground = BLUE,
                            "35" => self.foreground = MAGENTA,
                            "36" => self.foreground = CYAN,
                            "37" => self.foreground = WHITE,
                            "40" => self.background = BLACK,
                            "41" => self.background = RED,
                            "42" => self.background = GREEN,
                            "43" => self.background = YELLOW,
                            "44" => self.background = BLUE,
                            "45" => self.background = MAGENTA,
                            "46" => self.background = CYAN,
                            "47" => self.background = WHITE,
                            _ => {},
                        }
                    }
                    self.escape_sequence = false;
                },
                'J' => {
                    match self.sequence.get(0).map_or("", |p| &p).parse::<usize>().unwrap_or(0) {
                        0 => {
                            //TODO: Erase down
                        },
                        1 => {
                            //TODO: Erase up
                        },
                        2 => {
                            // Erase all
                            self.point_x = 0;
                            self.point_y = 0;
                            self.window.set(self.background);
                            self.redraw = true;
                        },
                        _ => {}
                    }

                    self.escape_sequence = false;
                },
                'H' | 'f' => {
                    self.window.rect(self.point_x, self.point_y, 8, 16, self.background);

                    let row = self.sequence.get(0).map_or("", |p| &p).parse::<isize>().unwrap_or(1);
                    self.point_y = cmp::max(0, row - 1) as i32 * 16;

                    let col = self.sequence.get(1).map_or("", |p| &p).parse::<isize>().unwrap_or(1);
                    self.point_x = cmp::max(0, col - 1) as i32 * 8;

                    self.window.rect(self.point_x, self.point_y, 8, 16, self.foreground);
                    self.redraw = true;

                    self.escape_sequence = false;
                },
/*
@MANSTART{terminal-raw-mode}
INTRODUCTION
    Since Redox has no ioctl syscall, it uses escape codes for switching to raw mode.

ENTERING AND EXITING RAW MODE
    Entering raw mode is done using CSI-r (^[r). Unsetting raw mode is done by CSI-R (^[R).

RAW MODE
    Raw mode means that the stdin must be handled solely by the program itself. It will not automatically be printed nor will it be modified in any way (modulo escape codes).

    This means that:
        - stdin is not printed.
        - newlines are interpreted as carriage returns in stdin.
        - stdin is not buffered, meaning that the stream of bytes goes directly to the program, without the user having to press enter.
@MANEND
*/
                'r' => {
                    self.raw_mode = true;
                    self.escape_sequence = false;
                },
                'R' => {
                    self.raw_mode = false;
                    self.escape_sequence = false;
                },
                _ => self.escape_sequence = false,

            }

            if !self.escape_sequence {
                self.sequence.clear();
                self.escape = false;
            }
        } else {
            match c {
                '[' => {
                    // Control sequence initiator

                    self.escape_sequence = true;
                    self.sequence.push(String::new());
                },
                'c' => {
                    // Reset
                    self.point_x = 0;
                    self.point_y = 0;
                    self.raw_mode = false;
                    self.foreground = WHITE;
                    self.background = BLACK;
                    self.window.set(self.background);
                    self.redraw = true;

                    self.escape = false;
                }
                _ => self.escape = false,
            }
        }
    }

    pub fn scroll(&mut self, rows: usize) {
        if rows > 0 && rows < self.window.height() as usize {
            let offset = rows * self.window.width() as usize;
            let data = self.window.data_mut();
            for i in 0..data.len() - offset {
                let color = data[i + offset];
                data[i] = color;
            }
            for i in data.len() - offset..data.len() {
                data[i] = self.background;
            }
        }
    }

    pub fn character(&mut self, c: char) {
        self.window.rect(self.point_x, self.point_y, 8, 16, self.background);

        match c {
            '\0' => {},
            '\x1B' => self.escape = true,
            '\n' => {
                self.point_x = 0;
                self.point_y += 16;
            },
            '\t' => self.point_x = ((self.point_x / 64) + 1) * 64,
            '\r' => self.point_x = 0,
            '\x08' => {
                self.point_x -= 8;
                if self.point_x < 0 {
                    self.point_x = 0
                }
                self.window.rect(self.point_x, self.point_y, 8, 16, self.background);
            },
            _ => {
                self.window.char(self.point_x, self.point_y, c, self.foreground);
                self.point_x += 8;
            }
        }

        if self.point_x >= self.window.width() as i32 {
            self.point_x = 0;
            self.point_y += 16;
        }

        while self.point_y + 16 > self.window.height() as i32 {
            self.scroll(16);
            self.point_y -= 16;
        }
        self.window.rect(self.point_x, self.point_y, 8, 16, self.foreground);
        self.redraw = true;
    }

    pub fn event(&mut self, event: Event) -> Option<String> {
        match event.to_option() {
            EventOption::Key(key_event) => {
                if key_event.pressed {
                    if self.raw_mode {
                        match key_event.scancode {
                            event::K_BKSP => self.command.push_str("\x08"),
                            event::K_UP => self.command.push_str("\x1B[A"),
                            event::K_DOWN => self.command.push_str("\x1B[B"),
                            event::K_RIGHT => self.command.push_str("\x1B[C"),
                            event::K_LEFT => self.command.push_str("\x1B[D"),
                            _ => match key_event.character {
                                '\0' => {},
                                c => {
                                    self.command.push(c);
                                }
                            },
                        }

                        if ! self.command.is_empty() {
                            let mut command = String::new();
                            mem::swap(&mut self.command, &mut command);
                            return Some(command);
                        }
                    } else {
                        match key_event.scancode {
                            event::K_BKSP => if ! self.command.is_empty() {
                                self.write(&[8]);
                                self.command.pop();
                            },
                            _ => match key_event.character {
                                '\0' => (),
                                c => {
                                    self.write(&[c as u8]);
                                    self.command.push(c);

                                    if c == '\n' {
                                        let mut command = String::new();
                                        mem::swap(&mut self.command, &mut command);
                                        return Some(command);
                                    }
                                }
                            },
                        }
                    }
                }
            },
            _ => (),
        }

        None
    }

    pub fn write(&mut self, bytes: &[u8]) {
        for byte in bytes.iter() {
            let c = *byte as char;

            if self.escape {
                self.code(c);
            } else {
                self.character(c);
            }
        }

        if self.redraw {
            self.redraw = false;
            self.window.sync();
        }
    }
}
<commit_msg>A null parameter will also reset colors<commit_after>use std::{cmp, mem};

use orbclient::{Color, Event, EventOption, Window};
use orbclient::event;

const BLACK: Color = Color::rgb(0, 0, 0);
const RED: Color = Color::rgb(194, 54, 33);
const GREEN: Color = Color::rgb(37, 188, 36);
const YELLOW: Color = Color::rgb(173, 173, 39);
const BLUE: Color = Color::rgb(73, 46, 225);
const MAGENTA: Color = Color::rgb(211, 56, 211);
const CYAN: Color = Color::rgb(51, 187, 200);
const WHITE: Color = Color::rgb(203, 204, 205);

pub struct Console {
    pub window: Box<Window>,
    pub point_x: i32,
    pub point_y: i32,
    pub foreground: Color,
    pub background: Color,
    pub redraw: bool,
    pub command: String,
    pub escape: bool,
    pub escape_sequence: bool,
    pub sequence: Vec<String>,
    pub raw_mode: bool,
}

impl Console {
    pub fn new() -> Console {
        let mut window = Window::new(-1, -1, 640, 480, "Terminal").unwrap();
        window.sync();
        Console {
            window: window,
            point_x: 0,
            point_y: 0,
            foreground: WHITE,
            background: BLACK,
            redraw: true,
            command: String::new(),
            escape: false,
            escape_sequence: false,
            sequence: Vec::new(),
            raw_mode: false,
        }
    }

    pub fn code(&mut self, c: char) {
        if self.escape_sequence {
            match c {
                '0' ... '9' => {
                    // Add a number to the sequence list
                    if let Some(mut value) = self.sequence.last_mut() {
                        value.push(c);
                    }
                },
                ';' => {
                    // Split sequence into list
                    self.sequence.push(String::new());
                },
                'm' => {
                    // Display attributes
                    for value in self.sequence.iter() {
                        match value.as_str() {
                            "" | "0" => {
                                self.foreground = WHITE;
                                self.background = BLACK;
                            },
                            "30" => self.foreground = BLACK,
                            "31" => self.foreground = RED,
                            "32" => self.foreground = GREEN,
                            "33" => self.foreground = YELLOW,
                            "34" => self.foreground = BLUE,
                            "35" => self.foreground = MAGENTA,
                            "36" => self.foreground = CYAN,
                            "37" => self.foreground = WHITE,
                            "40" => self.background = BLACK,
                            "41" => self.background = RED,
                            "42" => self.background = GREEN,
                            "43" => self.background = YELLOW,
                            "44" => self.background = BLUE,
                            "45" => self.background = MAGENTA,
                            "46" => self.background = CYAN,
                            "47" => self.background = WHITE,
                            _ => {},
                        }
                    }
                    self.escape_sequence = false;
                },
                'J' => {
                    match self.sequence.get(0).map_or("", |p| &p).parse::<usize>().unwrap_or(0) {
                        0 => {
                            //TODO: Erase down
                        },
                        1 => {
                            //TODO: Erase up
                        },
                        2 => {
                            // Erase all
                            self.point_x = 0;
                            self.point_y = 0;
                            self.window.set(self.background);
                            self.redraw = true;
                        },
                        _ => {}
                    }

                    self.escape_sequence = false;
                },
                'H' | 'f' => {
                    self.window.rect(self.point_x, self.point_y, 8, 16, self.background);

                    let row = self.sequence.get(0).map_or("", |p| &p).parse::<isize>().unwrap_or(1);
                    self.point_y = cmp::max(0, row - 1) as i32 * 16;

                    let col = self.sequence.get(1).map_or("", |p| &p).parse::<isize>().unwrap_or(1);
                    self.point_x = cmp::max(0, col - 1) as i32 * 8;

                    self.window.rect(self.point_x, self.point_y, 8, 16, self.foreground);
                    self.redraw = true;

                    self.escape_sequence = false;
                },
/*
@MANSTART{terminal-raw-mode}
INTRODUCTION
    Since Redox has no ioctl syscall, it uses escape codes for switching to raw mode.

ENTERING AND EXITING RAW MODE
    Entering raw mode is done using CSI-r (^[r). Unsetting raw mode is done by CSI-R (^[R).

RAW MODE
    Raw mode means that the stdin must be handled solely by the program itself. It will not automatically be printed nor will it be modified in any way (modulo escape codes).

    This means that:
        - stdin is not printed.
        - newlines are interpreted as carriage returns in stdin.
        - stdin is not buffered, meaning that the stream of bytes goes directly to the program, without the user having to press enter.
@MANEND
*/
                'r' => {
                    self.raw_mode = true;
                    self.escape_sequence = false;
                },
                'R' => {
                    self.raw_mode = false;
                    self.escape_sequence = false;
                },
                _ => self.escape_sequence = false,

            }

            if !self.escape_sequence {
                self.sequence.clear();
                self.escape = false;
            }
        } else {
            match c {
                '[' => {
                    // Control sequence initiator

                    self.escape_sequence = true;
                    self.sequence.push(String::new());
                },
                'c' => {
                    // Reset
                    self.point_x = 0;
                    self.point_y = 0;
                    self.raw_mode = false;
                    self.foreground = WHITE;
                    self.background = BLACK;
                    self.window.set(self.background);
                    self.redraw = true;

                    self.escape = false;
                }
                _ => self.escape = false,
            }
        }
    }

    pub fn scroll(&mut self, rows: usize) {
        if rows > 0 && rows < self.window.height() as usize {
            let offset = rows * self.window.width() as usize;
            let data = self.window.data_mut();
            for i in 0..data.len() - offset {
                let color = data[i + offset];
                data[i] = color;
            }
            for i in data.len() - offset..data.len() {
                data[i] = self.background;
            }
        }
    }

    pub fn character(&mut self, c: char) {
        self.window.rect(self.point_x, self.point_y, 8, 16, self.background);

        match c {
            '\0' => {},
            '\x1B' => self.escape = true,
            '\n' => {
                self.point_x = 0;
                self.point_y += 16;
            },
            '\t' => self.point_x = ((self.point_x / 64) + 1) * 64,
            '\r' => self.point_x = 0,
            '\x08' => {
                self.point_x -= 8;
                if self.point_x < 0 {
                    self.point_x = 0
                }
                self.window.rect(self.point_x, self.point_y, 8, 16, self.background);
            },
            _ => {
                self.window.char(self.point_x, self.point_y, c, self.foreground);
                self.point_x += 8;
            }
        }

        if self.point_x >= self.window.width() as i32 {
            self.point_x = 0;
            self.point_y += 16;
        }

        while self.point_y + 16 > self.window.height() as i32 {
            self.scroll(16);
            self.point_y -= 16;
        }
        self.window.rect(self.point_x, self.point_y, 8, 16, self.foreground);
        self.redraw = true;
    }

    pub fn event(&mut self, event: Event) -> Option<String> {
        match event.to_option() {
            EventOption::Key(key_event) => {
                if key_event.pressed {
                    if self.raw_mode {
                        match key_event.scancode {
                            event::K_BKSP => self.command.push_str("\x08"),
                            event::K_UP => self.command.push_str("\x1B[A"),
                            event::K_DOWN => self.command.push_str("\x1B[B"),
                            event::K_RIGHT => self.command.push_str("\x1B[C"),
                            event::K_LEFT => self.command.push_str("\x1B[D"),
                            _ => match key_event.character {
                                '\0' => {},
                                c => {
                                    self.command.push(c);
                                }
                            },
                        }

                        if ! self.command.is_empty() {
                            let mut command = String::new();
                            mem::swap(&mut self.command, &mut command);
                            return Some(command);
                        }
                    } else {
                        match key_event.scancode {
                            event::K_BKSP => if ! self.command.is_empty() {
                                self.write(&[8]);
                                self.command.pop();
                            },
                            _ => match key_event.character {
                                '\0' => (),
                                c => {
                                    self.write(&[c as u8]);
                                    self.command.push(c);

                                    if c == '\n' {
                                        let mut command = String::new();
                                        mem::swap(&mut self.command, &mut command);
                                        return Some(command);
                                    }
                                }
                            },
                        }
                    }
                }
            },
            _ => (),
        }

        None
    }

    pub fn write(&mut self, bytes: &[u8]) {
        for byte in bytes.iter() {
            let c = *byte as char;

            if self.escape {
                self.code(c);
            } else {
                self.character(c);
            }
        }

        if self.redraw {
            self.redraw = false;
            self.window.sync();
        }
    }
}
<|endoftext|>


<commit_before>use futures::{Async, Future, Poll, Stream};
use std::io;
use std::ops::Deref;
use std::process::ExitStatus;

/// An interface for waiting on a process to exit.
pub trait Wait {
    /// Try waiting for a process to exit in a non-blocking manner.
    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>>;
}

/// An interface for killing a running process.
pub trait Kill {
    /// Forcefully kill the process.
    fn kill(&mut self) -> io::Result<()>;
}

#[derive(Debug, PartialEq)]
enum WaitResult {
    Exited(ExitStatus),
    Reaped,
}

/// An interface for safely reaping a child process.
trait Reap {
    /// Try to reap the child process if ready.
    fn try_reap(&mut self) -> Poll<WaitResult, io::Error>;
}

#[derive(Debug)]
struct Reaper<W> {
    reaped: bool,
    proc: W,
}

impl<W> Reaper<W> {
    fn new(proc: W) -> Self {
        Self {
            reaped: false,
            proc,
        }
    }

    fn reaped(&self) -> bool {
        self.reaped
    }
}

impl<W> Deref for Reaper<W> {
    type Target = W;

    fn deref(&self) -> &Self::Target {
        &self.proc
    }
}

impl<W: Wait> Reap for Reaper<W> {
    fn try_reap(&mut self) -> Poll<WaitResult, io::Error> {
        if self.reaped {
            return Ok(Async::Ready(WaitResult::Reaped));
        }

        match self.proc.try_wait()? {
            Some(exit) => {
                self.reaped = true;
                Ok(Async::Ready(WaitResult::Exited(exit)))
            },
            None => Ok(Async::NotReady),
        }
    }
}

impl<W: Kill> Kill for Reaper<W> {
    fn kill(&mut self) -> io::Result<()> {
        // NB: ensure we don't issue a kill after we've reaped the child
        // since its process identifier could have been reused.
        if self.reaped {
            Ok(())
        } else {
            self.proc.kill()
        }
    }
}

/// Orchestrates between registering interest for receiving signals when a
/// child process has exited, and attempting to poll for process completion.
#[derive(Debug)]
pub struct EventedReaper<W, S> {
    inner: Reaper<W>,
    signal: S,
}

impl<W, S> Deref for EventedReaper<W, S> {
    type Target = W;

    fn deref(&self) -> &Self::Target {
        &*self.inner
    }
}

impl<W, S> EventedReaper<W, S> {
    pub fn new(inner: W, signal: S) -> Self {
        Self {
            inner: Reaper::new(inner),
            signal,
        }
    }
}

impl<W, S> Future for EventedReaper<W, S>
    where W: Wait,
          S: Stream<Error = io::Error>,
{
    type Item = ExitStatus;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        loop {
            // Ensure we don't register for additional notifications
            // if the child has already finished.
            if self.inner.reaped() {
                return Ok(Async::NotReady);
            }

            // If the child hasn't exited yet, then it's our responsibility to
            // ensure the current task gets notified when it might be able to
            // make progress.
            //
            // As described in `spawn` above, we just indicate that we can
            // next make progress once a SIGCHLD is received.
            //
            // However, we will register for a notification on the next signal
            // BEFORE we poll the child. Otherwise it is possible that the child
            // can exit and the signal can arrive after we last polled the child,
            // but before we've registered for a notification on the next signal
            // (this can cause a deadlock if there are no more spawned children
            // which can generate a different signal for us). A side effect of
            // pre-registering for signal notifications is that when the child
            // exits, we will have already registered for an additional
            // notification we don't need to consume. If another signal arrives,
            // this future's task will be notified/woken up again. Since the
            // futures model allows for spurious wake ups this extra wakeup
            // should not cause significant issues with parent futures.
            let registered_interest = self.signal.poll()?.is_not_ready();

            if let Async::Ready(WaitResult::Exited(status)) = self.inner.try_reap()? {
                return Ok(Async::Ready(status));
            }

            // If our attempt to poll for the next signal was not ready, then
            // we've arranged for our task to get notified and we can bail out.
            if registered_interest {
                return Ok(Async::NotReady);
            } else {
                // Otherwise, if the signal stream delivered a signal to us, we
                // won't get notified at the next signal, so we'll loop and try
                // again.
                continue;
            }
        }
    }
}

impl<W, S> Kill for EventedReaper<W, S>
    where W: Kill,
{
    fn kill(&mut self) -> io::Result<()> {
        self.inner.kill()
    }
}

#[cfg(test)]
mod test {
    use futures::{Async, Poll, Stream};
    use std::process::ExitStatus;
    use std::os::unix::process::ExitStatusExt;
    use super::*;

    struct MockWait {
        total_kills: usize,
        total_waits: usize,
        num_wait_until_status: usize,
        status: ExitStatus,
    }

    impl MockWait {
        fn new(status: ExitStatus, num_wait_until_status: usize) -> Self {
            Self {
                total_kills: 0,
                total_waits: 0,
                num_wait_until_status,
                status
            }
        }
    }

    impl Wait for MockWait {
        fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
            let ret = if self.num_wait_until_status == self.total_waits {
                Some(self.status.clone())
            } else {
                None
            };

            self.total_waits += 1;
            Ok(ret)
        }
    }

    impl Kill for MockWait {
        fn kill(&mut self) -> io::Result<()> {
            self.total_kills += 1;
            Ok(())
        }
    }

    struct MockStream {
        total_polls: usize,
        values: Vec<Option<()>>,
    }

    impl MockStream {
        fn new(values: Vec<Option<()>>) -> Self {
            Self {
                total_polls: 0,
                values
            }
        }
    }

    impl Stream for MockStream {
        type Item = ();
        type Error = io::Error;

        fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
            self.total_polls += 1;
            match self.values.remove(0) {
                Some(()) => Ok(Async::Ready(Some(()))),
                None => Ok(Async::NotReady),
            }
        }
    }

    #[test]
    fn reaper() {
        let exit = ExitStatus::from_raw(0);
        let mock = MockWait::new(exit.clone(), 1);
        let mut grim = Reaper::new(mock);

        // Not yet exited
        assert_eq!(Async::NotReady, grim.try_reap().expect("failed to wait"));
        assert_eq!(1, grim.total_waits);

        // Exited
        assert_eq!(Async::Ready(WaitResult::Exited(exit)), grim.try_reap().expect("failed to wait"));
        assert_eq!(2, grim.total_waits);

        // Cannot call wait another time
        assert_eq!(Async::Ready(WaitResult::Reaped), grim.try_reap().expect("failed to wait"));
        assert_eq!(2, grim.total_waits);
    }

    #[test]
    fn evented_reaper() {
        let exit = ExitStatus::from_raw(0);
        let mock = MockWait::new(exit.clone(), 3);
        let mut grim = EventedReaper::new(mock, MockStream::new(vec!(
            None,
            Some(()),
            None,
            None,
            None,
        )));

        // Not yet exited, interest registered
        assert_eq!(Async::NotReady, grim.poll().expect("failed to wait"));
        assert_eq!(1, grim.signal.total_polls);
        assert_eq!(1, grim.total_waits);

        // Not yet exited, couldn't register interest the first time
        // but managed to register interest the second time around
        assert_eq!(Async::NotReady, grim.poll().expect("failed to wait"));
        assert_eq!(3, grim.signal.total_polls);
        assert_eq!(3, grim.total_waits);

        // Exited
        assert_eq!(Async::Ready(exit), grim.poll().expect("failed to wait"));
        assert_eq!(4, grim.signal.total_polls);
        assert_eq!(4, grim.total_waits);

        // Already reaped, no further calls
        assert_eq!(Async::NotReady, grim.poll().expect("failed to poll"));
        assert_eq!(4, grim.signal.total_polls);
        assert_eq!(4, grim.total_waits);
    }

    #[test]
    fn kill() {
        let exit = ExitStatus::from_raw(0);
        let mut grim = EventedReaper::new(
            MockWait::new(exit, 0),
            MockStream::new(vec!(None))
        );

        grim.kill().unwrap();
        assert_eq!(1, grim.total_kills);

        // Do not kill after reaping
        assert_eq!(Async::Ready(exit), grim.poll().expect("failed to poll"));
        grim.kill().unwrap();
        assert_eq!(1, grim.total_kills);
    }
}
<commit_msg>process: Optimize out the "reaped" flag<commit_after>use futures::{Async, Future, Poll, Stream};
use std::io;
use std::ops::Deref;
use std::process::ExitStatus;

/// An interface for waiting on a process to exit.
pub trait Wait {
    /// Try waiting for a process to exit in a non-blocking manner.
    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>>;
}

/// An interface for killing a running process.
pub trait Kill {
    /// Forcefully kill the process.
    fn kill(&mut self) -> io::Result<()>;
}

/// Orchestrates between registering interest for receiving signals when a
/// child process has exited, and attempting to poll for process completion.
#[derive(Debug)]
pub struct EventedReaper<W, S> {
    inner: W,
    signal: S,
}

impl<W, S> Deref for EventedReaper<W, S> {
    type Target = W;

    fn deref(&self) -> &Self::Target {
        &self.inner
    }
}

impl<W, S> EventedReaper<W, S> {
    pub fn new(inner: W, signal: S) -> Self {
        Self {
            inner,
            signal,
        }
    }
}

impl<W, S> Future for EventedReaper<W, S>
    where W: Wait,
          S: Stream<Error = io::Error>,
{
    type Item = ExitStatus;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        loop {
            // If the child hasn't exited yet, then it's our responsibility to
            // ensure the current task gets notified when it might be able to
            // make progress.
            //
            // As described in `spawn` above, we just indicate that we can
            // next make progress once a SIGCHLD is received.
            //
            // However, we will register for a notification on the next signal
            // BEFORE we poll the child. Otherwise it is possible that the child
            // can exit and the signal can arrive after we last polled the child,
            // but before we've registered for a notification on the next signal
            // (this can cause a deadlock if there are no more spawned children
            // which can generate a different signal for us). A side effect of
            // pre-registering for signal notifications is that when the child
            // exits, we will have already registered for an additional
            // notification we don't need to consume. If another signal arrives,
            // this future's task will be notified/woken up again. Since the
            // futures model allows for spurious wake ups this extra wakeup
            // should not cause significant issues with parent futures.
            let registered_interest = self.signal.poll()?.is_not_ready();

            if let Some(status) = self.inner.try_wait()? {
                return Ok(Async::Ready(status));
            }

            // If our attempt to poll for the next signal was not ready, then
            // we've arranged for our task to get notified and we can bail out.
            if registered_interest {
                return Ok(Async::NotReady);
            } else {
                // Otherwise, if the signal stream delivered a signal to us, we
                // won't get notified at the next signal, so we'll loop and try
                // again.
                continue;
            }
        }
    }
}

impl<W, S> Kill for EventedReaper<W, S>
    where W: Kill,
{
    fn kill(&mut self) -> io::Result<()> {
        self.inner.kill()
    }
}

#[cfg(test)]
mod test {
    use futures::{Async, Poll, Stream};
    use std::process::ExitStatus;
    use std::os::unix::process::ExitStatusExt;
    use super::*;

    struct MockWait {
        total_kills: usize,
        total_waits: usize,
        num_wait_until_status: usize,
        status: ExitStatus,
    }

    impl MockWait {
        fn new(status: ExitStatus, num_wait_until_status: usize) -> Self {
            Self {
                total_kills: 0,
                total_waits: 0,
                num_wait_until_status,
                status
            }
        }
    }

    impl Wait for MockWait {
        fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
            let ret = if self.num_wait_until_status == self.total_waits {
                Some(self.status.clone())
            } else {
                None
            };

            self.total_waits += 1;
            Ok(ret)
        }
    }

    impl Kill for MockWait {
        fn kill(&mut self) -> io::Result<()> {
            self.total_kills += 1;
            Ok(())
        }
    }

    struct MockStream {
        total_polls: usize,
        values: Vec<Option<()>>,
    }

    impl MockStream {
        fn new(values: Vec<Option<()>>) -> Self {
            Self {
                total_polls: 0,
                values
            }
        }
    }

    impl Stream for MockStream {
        type Item = ();
        type Error = io::Error;

        fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
            self.total_polls += 1;
            match self.values.remove(0) {
                Some(()) => Ok(Async::Ready(Some(()))),
                None => Ok(Async::NotReady),
            }
        }
    }

    #[test]
    fn evented_reaper() {
        let exit = ExitStatus::from_raw(0);
        let mock = MockWait::new(exit.clone(), 3);
        let mut grim = EventedReaper::new(mock, MockStream::new(vec!(
            None,
            Some(()),
            None,
            None,
            None,
        )));

        // Not yet exited, interest registered
        assert_eq!(Async::NotReady, grim.poll().expect("failed to wait"));
        assert_eq!(1, grim.signal.total_polls);
        assert_eq!(1, grim.total_waits);

        // Not yet exited, couldn't register interest the first time
        // but managed to register interest the second time around
        assert_eq!(Async::NotReady, grim.poll().expect("failed to wait"));
        assert_eq!(3, grim.signal.total_polls);
        assert_eq!(3, grim.total_waits);

        // Exited
        assert_eq!(Async::Ready(exit), grim.poll().expect("failed to wait"));
        assert_eq!(4, grim.signal.total_polls);
        assert_eq!(4, grim.total_waits);
    }

    #[test]
    fn kill() {
        let exit = ExitStatus::from_raw(0);
        let mut grim = EventedReaper::new(
            MockWait::new(exit, 0),
            MockStream::new(vec!(None))
        );

        grim.kill().unwrap();
        assert_eq!(1, grim.total_kills);
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::VecDeque;
use std::error;
use std::fmt::{self, Display, Formatter};
use std::sync::Arc;
use std::time::Instant;

use rocksdb::DB;
use storage::CF_WRITE;
use util::escape;
use util::rocksdb;
use util::rocksdb::compact_range;
use util::rocksdb::stats::get_range_entries_and_versions;
use util::worker::Runnable;

use super::metrics::COMPACT_RANGE_CF;

type Key = Vec<u8>;

pub enum Task {
    Compact {
        cf_name: String,
        start_key: Option<Key>, // None means smallest key
        end_key: Option<Key>,   // None means largest key
    },

    CheckAndCompact {
        cf_names: Vec<String>,         // Column families need to compact
        ranges: Vec<Key>,              // Ranges need to check
        tombstones_num_threshold: u64, // The minimum RocksDB tombstones a range that need compacting has
        tombstones_percent_threshold: u64,
    },
}

impl Display for Task {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        match *self {
            Task::Compact {
                ref cf_name,
                ref start_key,
                ref end_key,
            } => f
                .debug_struct("Compact")
                .field("cf_name", cf_name)
                .field("start_key", &start_key.as_ref().map(|k| escape(k)))
                .field("end_key", &end_key.as_ref().map(|k| escape(k)))
                .finish(),
            Task::CheckAndCompact {
                ref cf_names,
                ref ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => f
                .debug_struct("CheckAndCompact")
                .field("cf_names", cf_names)
                .field(
                    "ranges",
                    &(
                        ranges.first().as_ref().map(|k| escape(k)),
                        ranges.last().as_ref().map(|k| escape(k)),
                    ),
                )
                .field("tombstones_num_threshold", &tombstones_num_threshold)
                .field(
                    "tombstones_percent_threshold",
                    &tombstones_percent_threshold,
                )
                .finish(),
        }
    }
}

quick_error! {
    #[derive(Debug)]
    pub enum Error {
        Other(err: Box<error::Error + Sync + Send>) {
            from()
            cause(err.as_ref())
            description(err.description())
            display("compact failed {:?}", err)
        }
    }
}

pub struct Runner {
    engine: Arc<DB>,
}

impl Runner {
    pub fn new(engine: Arc<DB>) -> Runner {
        Runner { engine }
    }

    pub fn compact_range_cf(
        &mut self,
        cf_name: String,
        start_key: Option<Vec<u8>>,
        end_key: Option<Vec<u8>>,
    ) -> Result<(), Error> {
        let handle = box_try!(rocksdb::get_cf_handle(&self.engine, &cf_name));
        let timer = Instant::now();
        let compact_range_timer = COMPACT_RANGE_CF
            .with_label_values(&[&cf_name])
            .start_coarse_timer();
        let start = start_key.as_ref().map(Vec::as_slice);
        let end = end_key.as_ref().map(Vec::as_slice);
        compact_range(
            &self.engine,
            handle,
            start,
            end,
            false,
            1, /* threads */
        );
        compact_range_timer.observe_duration();
        info!(
            "compact range ({:?}, {:?}) for cf {:?} finished, takes: {:?}",
            start,
            end,
            cf_name,
            timer.elapsed()
        );
        Ok(())
    }
}

impl Runnable<Task> for Runner {
    fn run(&mut self, task: Task) {
        match task {
            Task::Compact {
                cf_name,
                start_key,
                end_key,
            } => {
                let cf = cf_name.clone();
                if let Err(e) = self.compact_range_cf(cf_name, start_key, end_key) {
                    error!("execute compact range for cf {} failed, err {}", &cf, e);
                }
            }
            Task::CheckAndCompact {
                cf_names,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => match collect_ranges_need_compact(
                &self.engine,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                Ok(mut ranges) => for (start, end) in ranges.drain(..) {
                    for cf in &cf_names {
                        if let Err(e) = self.compact_range_cf(
                            cf.clone(),
                            Some(start.clone()),
                            Some(end.clone()),
                        ) {
                            error!(
                                "compact range ({:?}, {:?}) for cf {:?} failed, error {:?}",
                                start, end, cf, e
                            );
                        }
                    }
                },
                Err(e) => warn!("check ranges need reclaim failed, err: {:?}", e),
            },
        }
    }
}

fn need_compact(
    num_entires: u64,
    num_versions: u64,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> bool {
    if num_entires <= num_versions {
        return false;
    }

    // When the number of tombstones exceed threshold and ratio, this range need compacting.
    let estimate_num_del = num_entires - num_versions;
    estimate_num_del >= tombstones_num_threshold
        && estimate_num_del * 100 >= tombstones_percent_threshold * num_entires
}

fn collect_ranges_need_compact(
    engine: &DB,
    ranges: Vec<Key>,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> Result<VecDeque<(Key, Key)>, Error> {
    // Check the SST properties for each range, and we will compact a range if the range
    // contains too much RocksDB tombstones. we will merge multiple neighbouring ranges
    // that need compacting into a single range.
    let mut ranges_need_compact = VecDeque::new();
    let cf = box_try!(rocksdb::get_cf_handle(engine, CF_WRITE));
    let mut compact_start = None;
    let mut compact_end = None;
    for range in ranges.windows(2) {
        // Get total entries and total versions in this range and check if need compacting.
        if let Some((num_ent, num_ver)) =
            get_range_entries_and_versions(engine, cf, &range[0], &range[1])
        {
            if need_compact(
                num_ent,
                num_ver,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                if compact_start.is_none() {
                    // The previous range doesn't need compacting.
                    compact_start = Some(range[0].clone());
                }
                compact_end = Some(range[1].clone());
                // Move to next range.
                continue;
            }
        }

        // Current range doesn't need compacting, save previous range that need compacting.
        if compact_start.is_some() {
            ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
            compact_start = None;
            compact_end = None;
        }
    }

    // Save the last range that need compacting.
    if compact_start.is_some() {
        ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
    }

    Ok(ranges_need_compact)
}

#[cfg(test)]
mod tests {
    use std::thread::sleep;
    use std::time::Duration;

    use tempdir::TempDir;

    use raftstore::store::keys::data_key;
    use rocksdb::{self, Writable, WriteBatch, DB};
    use storage::mvcc::{Write, WriteType};
    use storage::types::Key as MvccKey;
    use storage::{CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE};
    use util::properties::MvccPropertiesCollectorFactory;
    use util::rocksdb::new_engine;
    use util::rocksdb::stats::get_range_entries_and_versions;
    use util::rocksdb::{get_cf_handle, new_engine_opt, CFOptions};

    use super::*;

    const ROCKSDB_TOTAL_SST_FILES_SIZE: &str = "rocksdb.total-sst-files-size";

    #[test]
    fn test_compact_range() {
        let path = TempDir::new("compact-range-test").unwrap();
        let db = new_engine(path.path().to_str().unwrap(), &[CF_DEFAULT], None).unwrap();
        let db = Arc::new(db);

        let mut runner = Runner::new(Arc::clone(&db));

        let handle = get_cf_handle(&db, CF_DEFAULT).unwrap();

        // generate first sst file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // generate another sst file has the same content with first sst file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // get total sst files size.
        let old_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();

        // schedule compact range task
        runner.run(Task::Compact {
            cf_name: String::from(CF_DEFAULT),
            start_key: None,
            end_key: None,
        });
        sleep(Duration::from_secs(5));

        // get total sst files size after compact range.
        let new_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();
        assert!(old_sst_files_size > new_sst_files_size);
    }

    fn mvcc_put(db: &DB, k: &[u8], v: &[u8], start_ts: u64, commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        let w = Write::new(WriteType::Put, start_ts, Some(v.to_vec()));
        db.put_cf(cf, k.as_encoded(), &w.to_bytes()).unwrap();
    }

    fn delete(db: &DB, k: &[u8], commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        db.delete_cf(cf, k.as_encoded()).unwrap();
    }

    fn open_db(path: &str) -> DB {
        let db_opts = rocksdb::DBOptions::new();
        let mut cf_opts = rocksdb::ColumnFamilyOptions::new();
        cf_opts.set_level_zero_file_num_compaction_trigger(8);
        let f = Box::new(MvccPropertiesCollectorFactory::default());
        cf_opts.add_table_properties_collector_factory("tikv.test-collector", f);
        let cfs_opts = vec![
            CFOptions::new(CF_DEFAULT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_RAFT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_LOCK, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_WRITE, cf_opts),
        ];
        new_engine_opt(path, db_opts, cfs_opts).unwrap()
    }

    #[test]
    fn test_check_space_redundancy() {
        let p = TempDir::new("test").unwrap();
        let engine = open_db(p.path().to_str().unwrap());
        let cf = get_cf_handle(&engine, CF_WRITE).unwrap();

        // mvcc_put 0..5
        for i in 0..5 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        // gc 0..5
        for i in 0..5 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        // mvcc_put 5..10
        for i in 5..10 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 5);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        ).unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);

        // gc 5..10
        for i in 5..10 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        ).unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k9"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);
    }
}
<commit_msg>escape compact range key (#3910)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::VecDeque;
use std::error;
use std::fmt::{self, Display, Formatter};
use std::sync::Arc;
use std::time::Instant;

use rocksdb::DB;
use storage::CF_WRITE;
use util::escape;
use util::rocksdb;
use util::rocksdb::compact_range;
use util::rocksdb::stats::get_range_entries_and_versions;
use util::worker::Runnable;

use super::metrics::COMPACT_RANGE_CF;

type Key = Vec<u8>;

pub enum Task {
    Compact {
        cf_name: String,
        start_key: Option<Key>, // None means smallest key
        end_key: Option<Key>,   // None means largest key
    },

    CheckAndCompact {
        cf_names: Vec<String>,         // Column families need to compact
        ranges: Vec<Key>,              // Ranges need to check
        tombstones_num_threshold: u64, // The minimum RocksDB tombstones a range that need compacting has
        tombstones_percent_threshold: u64,
    },
}

impl Display for Task {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        match *self {
            Task::Compact {
                ref cf_name,
                ref start_key,
                ref end_key,
            } => f
                .debug_struct("Compact")
                .field("cf_name", cf_name)
                .field("start_key", &start_key.as_ref().map(|k| escape(k)))
                .field("end_key", &end_key.as_ref().map(|k| escape(k)))
                .finish(),
            Task::CheckAndCompact {
                ref cf_names,
                ref ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => f
                .debug_struct("CheckAndCompact")
                .field("cf_names", cf_names)
                .field(
                    "ranges",
                    &(
                        ranges.first().as_ref().map(|k| escape(k)),
                        ranges.last().as_ref().map(|k| escape(k)),
                    ),
                )
                .field("tombstones_num_threshold", &tombstones_num_threshold)
                .field(
                    "tombstones_percent_threshold",
                    &tombstones_percent_threshold,
                )
                .finish(),
        }
    }
}

quick_error! {
    #[derive(Debug)]
    pub enum Error {
        Other(err: Box<error::Error + Sync + Send>) {
            from()
            cause(err.as_ref())
            description(err.description())
            display("compact failed {:?}", err)
        }
    }
}

pub struct Runner {
    engine: Arc<DB>,
}

impl Runner {
    pub fn new(engine: Arc<DB>) -> Runner {
        Runner { engine }
    }

    pub fn compact_range_cf(
        &mut self,
        cf_name: String,
        start_key: Option<Vec<u8>>,
        end_key: Option<Vec<u8>>,
    ) -> Result<(), Error> {
        let handle = box_try!(rocksdb::get_cf_handle(&self.engine, &cf_name));
        let timer = Instant::now();
        let compact_range_timer = COMPACT_RANGE_CF
            .with_label_values(&[&cf_name])
            .start_coarse_timer();
        let start = start_key.as_ref().map(Vec::as_slice);
        let end = end_key.as_ref().map(Vec::as_slice);
        compact_range(
            &self.engine,
            handle,
            start,
            end,
            false,
            1, /* threads */
        );
        compact_range_timer.observe_duration();
        info!(
            "compact range ({:?}, {:?}) for cf {:?} finished, takes: {:?}",
            &start.as_ref().map(|k| escape(k)),
            &end.as_ref().map(|k| escape(k)),
            cf_name,
            timer.elapsed()
        );
        Ok(())
    }
}

impl Runnable<Task> for Runner {
    fn run(&mut self, task: Task) {
        match task {
            Task::Compact {
                cf_name,
                start_key,
                end_key,
            } => {
                let cf = cf_name.clone();
                if let Err(e) = self.compact_range_cf(cf_name, start_key, end_key) {
                    error!("execute compact range for cf {} failed, err {}", &cf, e);
                }
            }
            Task::CheckAndCompact {
                cf_names,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => match collect_ranges_need_compact(
                &self.engine,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                Ok(mut ranges) => for (start, end) in ranges.drain(..) {
                    for cf in &cf_names {
                        if let Err(e) = self.compact_range_cf(
                            cf.clone(),
                            Some(start.clone()),
                            Some(end.clone()),
                        ) {
                            error!(
                                "compact range ({:?}, {:?}) for cf {:?} failed, error {:?}",
                                start, end, cf, e
                            );
                        }
                    }
                },
                Err(e) => warn!("check ranges need reclaim failed, err: {:?}", e),
            },
        }
    }
}

fn need_compact(
    num_entires: u64,
    num_versions: u64,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> bool {
    if num_entires <= num_versions {
        return false;
    }

    // When the number of tombstones exceed threshold and ratio, this range need compacting.
    let estimate_num_del = num_entires - num_versions;
    estimate_num_del >= tombstones_num_threshold
        && estimate_num_del * 100 >= tombstones_percent_threshold * num_entires
}

fn collect_ranges_need_compact(
    engine: &DB,
    ranges: Vec<Key>,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> Result<VecDeque<(Key, Key)>, Error> {
    // Check the SST properties for each range, and we will compact a range if the range
    // contains too much RocksDB tombstones. we will merge multiple neighbouring ranges
    // that need compacting into a single range.
    let mut ranges_need_compact = VecDeque::new();
    let cf = box_try!(rocksdb::get_cf_handle(engine, CF_WRITE));
    let mut compact_start = None;
    let mut compact_end = None;
    for range in ranges.windows(2) {
        // Get total entries and total versions in this range and check if need compacting.
        if let Some((num_ent, num_ver)) =
            get_range_entries_and_versions(engine, cf, &range[0], &range[1])
        {
            if need_compact(
                num_ent,
                num_ver,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                if compact_start.is_none() {
                    // The previous range doesn't need compacting.
                    compact_start = Some(range[0].clone());
                }
                compact_end = Some(range[1].clone());
                // Move to next range.
                continue;
            }
        }

        // Current range doesn't need compacting, save previous range that need compacting.
        if compact_start.is_some() {
            ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
            compact_start = None;
            compact_end = None;
        }
    }

    // Save the last range that need compacting.
    if compact_start.is_some() {
        ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
    }

    Ok(ranges_need_compact)
}

#[cfg(test)]
mod tests {
    use std::thread::sleep;
    use std::time::Duration;

    use tempdir::TempDir;

    use raftstore::store::keys::data_key;
    use rocksdb::{self, Writable, WriteBatch, DB};
    use storage::mvcc::{Write, WriteType};
    use storage::types::Key as MvccKey;
    use storage::{CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE};
    use util::properties::MvccPropertiesCollectorFactory;
    use util::rocksdb::new_engine;
    use util::rocksdb::stats::get_range_entries_and_versions;
    use util::rocksdb::{get_cf_handle, new_engine_opt, CFOptions};

    use super::*;

    const ROCKSDB_TOTAL_SST_FILES_SIZE: &str = "rocksdb.total-sst-files-size";

    #[test]
    fn test_compact_range() {
        let path = TempDir::new("compact-range-test").unwrap();
        let db = new_engine(path.path().to_str().unwrap(), &[CF_DEFAULT], None).unwrap();
        let db = Arc::new(db);

        let mut runner = Runner::new(Arc::clone(&db));

        let handle = get_cf_handle(&db, CF_DEFAULT).unwrap();

        // generate first sst file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // generate another sst file has the same content with first sst file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // get total sst files size.
        let old_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();

        // schedule compact range task
        runner.run(Task::Compact {
            cf_name: String::from(CF_DEFAULT),
            start_key: None,
            end_key: None,
        });
        sleep(Duration::from_secs(5));

        // get total sst files size after compact range.
        let new_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();
        assert!(old_sst_files_size > new_sst_files_size);
    }

    fn mvcc_put(db: &DB, k: &[u8], v: &[u8], start_ts: u64, commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        let w = Write::new(WriteType::Put, start_ts, Some(v.to_vec()));
        db.put_cf(cf, k.as_encoded(), &w.to_bytes()).unwrap();
    }

    fn delete(db: &DB, k: &[u8], commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        db.delete_cf(cf, k.as_encoded()).unwrap();
    }

    fn open_db(path: &str) -> DB {
        let db_opts = rocksdb::DBOptions::new();
        let mut cf_opts = rocksdb::ColumnFamilyOptions::new();
        cf_opts.set_level_zero_file_num_compaction_trigger(8);
        let f = Box::new(MvccPropertiesCollectorFactory::default());
        cf_opts.add_table_properties_collector_factory("tikv.test-collector", f);
        let cfs_opts = vec![
            CFOptions::new(CF_DEFAULT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_RAFT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_LOCK, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_WRITE, cf_opts),
        ];
        new_engine_opt(path, db_opts, cfs_opts).unwrap()
    }

    #[test]
    fn test_check_space_redundancy() {
        let p = TempDir::new("test").unwrap();
        let engine = open_db(p.path().to_str().unwrap());
        let cf = get_cf_handle(&engine, CF_WRITE).unwrap();

        // mvcc_put 0..5
        for i in 0..5 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        // gc 0..5
        for i in 0..5 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        // mvcc_put 5..10
        for i in 5..10 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 5);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        ).unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);

        // gc 5..10
        for i in 5..10 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        ).unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k9"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);
    }
}
<|endoftext|>


<commit_before>// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;

const BINARY_FILTER_FILE_NAME: &str = "seccomp_filter.bpf";
const JSON_DIR: &str = "../../resources/seccomp";
const SECCOMPILER_BUILD_DIR: &str = "../../build/seccompiler";
const SECCOMPILER_SRC_DIR: &str = "../seccompiler/src";

// This script is run on every modification in the target-specific JSON file in `resources/seccomp`.
// It compiles the JSON seccomp policies into a serializable BPF format, using seccompiler.
// The generated binary code will get included in Firecracker's code, at compile-time.
fn main() {
    let target = env::var("TARGET").expect("Missing target.");
    let out_dir = env::var("OUT_DIR").expect("Missing build-level OUT_DIR.");
    let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("Missing target arch.");

    // Path to the JSON seccomp policy.
    let mut json_path = PathBuf::from(JSON_DIR);
    json_path.push(format!("{}.json", target));

    // If the current target doesn't have a default filter, use a default, empty filter.
    // This is to make sure that Firecracker builds even with libc toolchains for which we don't provide
    // a default filter. For example, GNU libc.
    if !json_path.exists() {
        json_path.pop();
        json_path.push("unimplemented.json");

        println!(
            "cargo:warning=No default seccomp policy for target: {}. \
            Defaulting to `resources/seccomp/unimplemented.json`.",
            target
        );
    }

    // Retrigger the build script if the JSON file has changed.
    let json_path = json_path.to_str().expect("Invalid bytes");
    println!("cargo:rerun-if-changed={}", json_path);

    // Also retrigger the build script on any seccompiler source code change.
    register_seccompiler_src_watchlist(Path::new(SECCOMPILER_SRC_DIR));

    // Path of the generated binary file.
    let mut bpf_out_path = PathBuf::from(&out_dir);
    bpf_out_path.push(BINARY_FILTER_FILE_NAME);
    let bpf_out_path = bpf_out_path.to_str().expect("Invalid bytes.");

    // Command for running seccompiler
    let mut command = Command::new("cargo");
    command.args(&[
        "run",
        "-p",
        "seccompiler",
        "--verbose",
        "--target",
        &target,
        // We need to specify a separate build directory for seccompiler. Otherwise, cargo will
        // deadlock waiting to acquire a lock on the build folder that the parent cargo process is
        // holding.
        "--target-dir",
        SECCOMPILER_BUILD_DIR,
        "--",
        "--input-file",
        &json_path,
        "--target-arch",
        &target_arch,
        "--output-file",
        bpf_out_path,
    ]);

    match command.output() {
        Err(error) => panic!("\nSeccompiler error: {:?}\n", error),
        Ok(result) if !result.status.success() => {
            panic!(
                "\nSeccompiler returned non-zero exit code:\nstderr: {}\nstdout: {}\n",
                String::from_utf8(result.stderr).unwrap(),
                String::from_utf8(result.stdout).unwrap(),
            );
        }
        Ok(_) => {}
    }
}

// Recursively traverse the entire seccompiler source folder and trigger a re-run of this build
// script on any modification of these files.
fn register_seccompiler_src_watchlist(src_dir: &Path) {
    let contents = fs::read_dir(src_dir).expect("Unable to read folder contents.");
    for entry in contents {
        let path = entry.unwrap().path();
        let metadata = fs::metadata(&path).expect("Unable to read file/folder metadata.");

        if metadata.is_file() {
            // Watch all source files.
            println!(
                "cargo:rerun-if-changed={}",
                path.to_str().expect("Invalid unicode bytes.")
            );
        } else if metadata.is_dir() {
            // If is a folder, recurse.
            register_seccompiler_src_watchlist(&path);
        }
    }
}
<commit_msg>build.rs: add basic seccomp filter<commit_after>// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;

const ADVANCED_BINARY_FILTER_FILE_NAME: &str = "seccomp_filter.bpf";
const BASIC_BINARY_FILTER_FILE_NAME: &str = "basic_seccomp_filter.bpf";

const JSON_DIR: &str = "../../resources/seccomp";
const SECCOMPILER_BUILD_DIR: &str = "../../build/seccompiler";
const SECCOMPILER_SRC_DIR: &str = "../seccompiler/src";

// This script is run on every modification in the target-specific JSON file in `resources/seccomp`.
// It compiles the JSON seccomp policies into a serializable BPF format, using seccompiler.
// The generated binary code will get included in Firecracker's code, at compile-time.
fn main() {
    let target = env::var("TARGET").expect("Missing target.");
    let out_dir = env::var("OUT_DIR").expect("Missing build-level OUT_DIR.");

    // Path to the JSON seccomp policy.
    let mut json_path = PathBuf::from(JSON_DIR);
    json_path.push(format!("{}.json", target));

    // If the current target doesn't have a default filter, use a default, empty filter.
    // This is to make sure that Firecracker builds even with libc toolchains for which we don't provide
    // a default filter. For example, GNU libc.
    if !json_path.exists() {
        json_path.pop();
        json_path.push("unimplemented.json");

        println!(
            "cargo:warning=No default seccomp policy for target: {}. \
            Defaulting to `resources/seccomp/unimplemented.json`.",
            target
        );
    }

    // Retrigger the build script if the JSON file has changed.
    let json_path = json_path.to_str().expect("Invalid bytes");
    println!("cargo:rerun-if-changed={}", json_path);

    // Also retrigger the build script on any seccompiler source code change.
    register_seccompiler_src_watchlist(Path::new(SECCOMPILER_SRC_DIR));

    // Run seccompiler, getting the default, advanced filter.
    let mut bpf_out_path = PathBuf::from(&out_dir);
    bpf_out_path.push(ADVANCED_BINARY_FILTER_FILE_NAME);
    run_seccompiler(
        &target,
        json_path,
        bpf_out_path.to_str().expect("Invalid bytes."),
        false,
    );

    // Run seccompiler with the `--basic` flag, getting the filter for `--seccomp-level 1`.
    let mut bpf_out_path = PathBuf::from(&out_dir);
    bpf_out_path.push(BASIC_BINARY_FILTER_FILE_NAME);
    run_seccompiler(
        &target,
        json_path,
        bpf_out_path.to_str().expect("Invalid bytes."),
        true,
    );
}

// Run seccompiler with the given arguments.
fn run_seccompiler(cargo_target: &str, json_path: &str, out_path: &str, basic: bool) {
    let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("Missing target arch.");

    // Command for running seccompiler
    let mut command = Command::new("cargo");
    command.args(&[
        "run",
        "-p",
        "seccompiler",
        "--verbose",
        "--target",
        &cargo_target,
        // We need to specify a separate build directory for seccompiler. Otherwise, cargo will
        // deadlock waiting to acquire a lock on the build folder that the parent cargo process is
        // holding.
        "--target-dir",
        SECCOMPILER_BUILD_DIR,
        "--",
        "--input-file",
        &json_path,
        "--target-arch",
        &target_arch,
        "--output-file",
        out_path,
    ]);

    if basic {
        command.arg("--basic");
    }

    match command.output() {
        Err(error) => panic!("\nSeccompiler error: {:?}\n", error),
        Ok(result) if !result.status.success() => {
            panic!(
                "\nSeccompiler returned non-zero exit code:\nstderr: {}\nstdout: {}\n",
                String::from_utf8(result.stderr).unwrap(),
                String::from_utf8(result.stdout).unwrap(),
            );
        }
        Ok(_) => {}
    }
}

// Recursively traverse the entire seccompiler source folder and trigger a re-run of this build
// script on any modification of these files.
fn register_seccompiler_src_watchlist(src_dir: &Path) {
    let contents = fs::read_dir(src_dir).expect("Unable to read folder contents.");
    for entry in contents {
        let path = entry.unwrap().path();
        let metadata = fs::metadata(&path).expect("Unable to read file/folder metadata.");

        if metadata.is_file() {
            // Watch all source files.
            println!(
                "cargo:rerun-if-changed={}",
                path.to_str().expect("Invalid unicode bytes.")
            );
        } else if metadata.is_dir() {
            // If is a folder, recurse.
            register_seccompiler_src_watchlist(&path);
        }
    }
}
<|endoftext|>


<commit_before>use std::io;

fn to_interger(string :~str) -> int {
    println!("{} {}", string, from_str::<int>(string));
    match from_str::<int>(string) {
        Some(0) => 0,
        Some(x) => x,
        _       => 0
    }
}

fn main() {
    println!("Enter width of the room:");
    let width = to_interger(io::stdin().read_line().unwrap());
    println!("Enter depth of the room:");
    let depth = to_interger(io::stdin().read_line().unwrap());
    println!("Enter price of sqrm of tiles:");
    let tile_price = to_interger(io::stdin().read_line().unwrap());

    let price = width * depth * tile_price;

    println!("The Price for the tiles would be: {}€", price);
}<commit_msg>Parsing of Inputs is now working.<commit_after>use std::io;

fn to_interger(string :~str) -> int {
    /* Complex workaround the string comes with a \n attached,
     * so I have to slice it off
     */
    match from_str::<int>(string.slice_to(string.len() - 1)) {
        Some(0) => 0,
        Some(x) => x,
        _       => 0
    }
}

fn main() {
    println!("Enter width of the room:");
    let width = to_interger(io::stdin().read_line().unwrap());
    println!("Enter depth of the room:");
    let depth = to_interger(io::stdin().read_line().unwrap());
    println!("Enter price of sqrm of tiles:");
    let tile_price = to_interger(io::stdin().read_line().unwrap());

    let price = width * depth * tile_price;

    println!("The Price for the tiles would be: {}€", price);
}<|endoftext|>


<commit_before>//! The driver trait which all drivers will implement.

use schema::Definition;
use error::{Error, ErrorCode};
use patch::Patch;
use value::{Pointer, Value};
use query::Query;

pub trait Driver {
  /// Connects to a driver and returns a driver instance. After calling this
  /// the driver is ready to roll!
  fn connect(uri: &str) -> Result<Box<Self>, Error>;

  /// Validates an Ardite Schema Definition dependending on the driver’s
  /// contracts with the developer. Note that the definition object will not be
  /// associated with the driver object. This is intentional, the driver should
  /// not be able to influence the definition without an external tool.
  fn validate_definition(_: &Definition) -> Result<(), Error> {
    Ok(())
  }

  /// Performs a complex query on the driver. Returns a value whose shape
  /// matches the shape of the query.
  fn query(&self, query: Query) -> Result<Value, Error>;

  /// Applies multiple patches to the driver. If one patch fails, all other
  /// patches must also fail. Returns a value with all of the new patched
  /// values along with any driver generated values only.
  fn patch(&self, patch: Vec<Patch>) -> Result<Value, Error>;

  /// Get’s a value in the driver at a specific point and returns exactly that
  /// value.
  fn get(&self, pointer: Pointer) -> Result<Value, Error> {
    let query = Query::from(pointer.clone());
    match try!(self.query(query)).get(pointer) {
      Some(value) => Ok(value),
      None => Err(Error::new(ErrorCode::Internal, "Driver failed to return a value with the requested data.", None))
    }
  }

  /// Set a value at a certain point in the driver. Returns nothing.
  fn set(&self, pointer: Pointer, value: Value) -> Result<(), Error> {
    try!(self.patch(vec![Patch::Set(pointer, value)]));
    Ok(())
  }
}
<commit_msg>style: split line again<commit_after>//! The driver trait which all drivers will implement.

use schema::Definition;
use error::{Error, ErrorCode};
use patch::Patch;
use value::{Pointer, Value};
use query::Query;

pub trait Driver {
  /// Connects to a driver and returns a driver instance. After calling this
  /// the driver is ready to roll!
  fn connect(uri: &str) -> Result<Box<Self>, Error>;

  /// Validates an Ardite Schema Definition dependending on the driver’s
  /// contracts with the developer. Note that the definition object will not be
  /// associated with the driver object. This is intentional, the driver should
  /// not be able to influence the definition without an external tool.
  fn validate_definition(_: &Definition) -> Result<(), Error> {
    Ok(())
  }

  /// Performs a complex query on the driver. Returns a value whose shape
  /// matches the shape of the query.
  fn query(&self, query: Query) -> Result<Value, Error>;

  /// Applies multiple patches to the driver. If one patch fails, all other
  /// patches must also fail. Returns a value with all of the new patched
  /// values along with any driver generated values only.
  fn patch(&self, patch: Vec<Patch>) -> Result<Value, Error>;

  /// Get’s a value in the driver at a specific point and returns exactly that
  /// value.
  fn get(&self, pointer: Pointer) -> Result<Value, Error> {
    let query = Query::from(pointer.clone());
    let value = try!(self.query(query));
    match value.get(pointer) {
      Some(value) => Ok(value),
      None => Err(Error::new(ErrorCode::Internal, "Driver failed to return a value with the requested data.", None))
    }
  }

  /// Set a value at a certain point in the driver. Returns nothing.
  fn set(&self, pointer: Pointer, value: Value) -> Result<(), Error> {
    try!(self.patch(vec![Patch::Set(pointer, value)]));
    Ok(())
  }
}
<|endoftext|>