<commit_before>#![cfg(not(windows))]

extern crate libc;

use screen::Key;
use screen::Key::*;
use std::io::{Read, Write};
use std::fs::{File, OpenOptions};
use std::os::unix::io::AsRawFd;
use std::path::*;
use std::process::Command;
use std::process::Stdio;
use std::str;
use std::iter::repeat;
use std::cmp::min;
use ansi;
use ::NEWLINE;

use std::{mem, ptr};
use std::thread;
use std::sync::mpsc::Sender;
use std::sync::mpsc::Receiver;
use std::sync::{mpsc, Arc, Mutex};

use self::libc::{sigaction, SIGWINCH, SA_SIGINFO, sighandler_t, c_int, c_ushort, c_ulong};

pub struct Screen {
    tty: Terminal,
    original_stty_state: Vec<u8>,
    pub height: u16,
    pub width: u16,
    pub visible_choices: u16,
    start_line: u16,
}

impl Screen {
    pub fn open_screen(desired_rows: u16) -> Screen {
        let mut tty = Terminal::open_terminal();
        let current_stty_state = tty.stty(&["-g"]);
        tty.initialize();
        let (cols, rows) = tty.winsize().unwrap();
        let visible_choices = min(desired_rows, rows - 1);
        let start_line = rows - visible_choices - 1;
        for _ in 0..visible_choices {
            tty.write(NEWLINE.as_bytes());
        }
        tty.write(&ansi::save_cursor());

        Screen {
            tty: tty,
            original_stty_state: current_stty_state,
            height: rows,
            width: cols,
            visible_choices: visible_choices,
            start_line: start_line,
        }
    }

    fn restore_tty(&mut self) {
        self.tty.stty(&[&String::from_utf8(self.original_stty_state.clone()).unwrap()]);
    }

    fn reset_cursor(&mut self) {
        self.tty.write(&ansi::restore_cursor());

        // Writing this carriage return works around a rendering bug in
        // Neovim's terminal emulation. Without it, the cursor flies
        // around all over the place while typing, and the input prompt
        // is not rendered correctly. I have no idea why this workaround
        // is effective, or why others are not, or what the root cause
        // of the issue is (likely something involving save/restore
        // support).
        self.tty.write(b"\r");

        self.tty.write(&ansi::cursor_up(self.visible_choices));
    }

    pub fn move_cursor_to_prompt_line(&mut self, col: u16) {
        self.reset_cursor();
        self.tty.write(&ansi::cursor_right(col));
    }

    pub fn blank_screen(&mut self) {
        self.reset_cursor();
        let blank_line = repeat(' ').take(self.width as usize).collect::<String>();
        for _ in 0..self.visible_choices + 1 {
            self.tty.write(blank_line.as_bytes());
        }
        self.reset_cursor();
    }

    pub fn show_cursor(&mut self) {
        self.tty.write(&ansi::show_cursor());
    }

    pub fn hide_cursor(&mut self) {
        self.tty.write(&ansi::hide_cursor());
    }

    pub fn write(&mut self, s: &str) {
        self.tty.write(s.as_bytes());
    }

    pub fn write_red_inverted(&mut self, s: &str) {
        self.tty.write(&ansi::red());
        self.tty.write(&ansi::inverse());
        self.tty.write(s.as_bytes());
        self.tty.write(&ansi::reset());
    }

    pub fn write_red(&mut self, s: &str) {
        self.tty.write(&ansi::red());
        self.tty.write(s.as_bytes());
        self.tty.write(&ansi::reset());
    }

    pub fn write_inverted(&mut self, s: &str) {
        self.tty.write(&ansi::inverse());
        self.tty.write(s.as_bytes());
        self.tty.write(&ansi::reset());
    }

    // Return all buffered keystrokes, or the next key if buffer is empty.
    pub fn get_buffered_keys(&mut self) -> Vec<Key> {
        let mut ret = Vec::new();
        while let Ok(bytes) = self.tty.input.try_recv() {
            ret.extend(bytes);
        }
        while ret.is_empty() {
            let bytes = self.tty.input.recv().unwrap();
            if bytes.len() == 1 && bytes[0] == 0 {
                self.tty.initialize();
                return vec![Nothing];
            } else {
                ret.extend(bytes);
            }
        }
        Terminal::translate_bytes(ret.clone())
    }

    fn more_bytes_needed(bytes: &[u8]) -> bool {
        if let Err(_) = str::from_utf8(bytes) { true } else { false }
    }
}

impl Drop for Screen {
    fn drop(&mut self) {
        self.restore_tty();
    }
}

struct Terminal {
    input: Receiver<Vec<u8>>,
    input_fd: i32,
    output: File,
}

static mut GLOBAL_TX: *const Arc<Mutex<Sender<Vec<u8>>>> = 0 as *const Arc<Mutex<Sender<Vec<u8>>>>;

fn set_global_tx(tx: Sender<Vec<u8>>) {
    unsafe {
        let singleton = Arc::new(Mutex::new(tx));
        GLOBAL_TX = mem::transmute(Box::new(singleton));
    }
}

fn get_global_tx() -> Sender<Vec<u8>> {
    let singleton = unsafe { (*GLOBAL_TX).clone() };
    let tx = singleton.lock().unwrap();
    tx.clone()
}

unsafe extern fn sigwinch_handler(signum: c_int) {
    if signum == SIGWINCH {
        get_global_tx().send([0].to_vec()).unwrap();
    }
}

fn register_sigwinch_handler() {
    unsafe {
        let mut action: sigaction = mem::zeroed();
        action.sa_flags = SA_SIGINFO;
        action.sa_sigaction = sigwinch_handler as sighandler_t;
        sigaction(SIGWINCH, &action, ptr::null_mut());
    }
}

impl Terminal {
    fn open_terminal() -> Terminal {
        let term_path = Path::new("/dev/tty");
        let mut input_file = File::open(&term_path).unwrap();
        let output_file = OpenOptions::new().write(true).open(&term_path).unwrap();
        let input_fd = input_file.as_raw_fd();
        let (tx, rx) = mpsc::channel();
        set_global_tx(tx);

        register_sigwinch_handler();

        thread::spawn(move || {
            loop {
                let mut buf = [0; 255];
                let tx = get_global_tx();
                if let Ok(length) = input_file.read(&mut buf) {
                    tx.send(buf[0..length].to_vec()).unwrap();
                } else {
                    tx.send([0].to_vec()).unwrap();
                }
            }
        });
        Terminal {
            input: rx,
            input_fd: input_fd,
            output: output_file,
        }
    }

    fn initialize(&mut self) {
        self.stty(&["raw", "-echo", "cbreak", "opost", "onlcr"]);
    }

    fn stty(&mut self, args: &[&str]) -> Vec<u8> {
        extern { fn dup2(src: c_int, dst: c_int) -> c_int;  }
        unsafe {
            // This is a hack until a replacement for InheritFd from old_io is available.
            dup2(self.input_fd, 0);
        }
        let mut process = match Command::new("stty")
            .args(args)
            .stdin(Stdio::inherit())
            .stdout(Stdio::piped())
            .spawn()
        {
            Ok(p) => p,
            Err(e) => panic!("Spawn failed: {}", e),
        };

        let exit = process.wait();

        let mut buf = Vec::new();
        if exit.unwrap().success() {
            process.stdout.as_mut().unwrap().read_to_end(&mut buf).unwrap();
            let mut str = String::from_utf8(buf).unwrap();

            // The output from `stty -g` may include a newline, which we have to strip off. Otherwise,
            // when we go to restore the tty, stty (on some platforms) will fail with an "invalid
            // argument" error.
            ::trim(&mut str);

            str.into_bytes()
        } else {
            process.stderr.as_mut().unwrap().read_to_end(&mut buf).unwrap();
            panic!(String::from_utf8(buf).unwrap());
        }
    }

    fn translate_bytes(bytes: Vec<u8>) -> Vec<Key> {
        if bytes == [27, b'[', b'A'] { return vec![Up] };
        if bytes == [27, b'[', b'B'] { return vec![Down] };
        if bytes == [27, b'O', b'A'] { return vec![Up] };
        if bytes == [27, b'O', b'B'] { return vec![Down] };

        let chars = String::from_utf8(bytes).unwrap().chars().collect::<Vec<char>>();
        chars.into_iter().map(Terminal::translate_char).collect()
    }

    fn translate_char(c: char) -> Key {
        let numeric_char = c as u32;
        if c == '\r' {
            Enter
        } else if numeric_char == 9 {
            Tab
        } else if numeric_char == 127 {
            Backspace
        } else if numeric_char & 96 == 0 && numeric_char <= 128u32 {
            let c = c as u8;
            Control((c + 96u8) as char)
        } else {
            Char(c)
        }
    }

    fn write(&mut self, s: &[u8]) {
        self.output.write_all(s).unwrap();
    }

    fn writeln(&mut self, s: &str) {
        self.output.write(s.as_bytes()).unwrap();
        self.output.write(b"\n").unwrap();
    }

    fn winsize(&self) -> Option<(u16, u16)> {
        extern { fn ioctl(fd: c_int, request: c_ulong, ...) -> c_int; }
        #[cfg(any(target_os = "macos", target_os = "freebsd"))]
        const TIOCGWINSZ: c_ulong = 0x40087468;

        #[cfg(any(target_os = "linux", target_os = "android"))]
        const TIOCGWINSZ: c_ulong = 0x00005413;

        #[repr(C)]
        struct TermSize {
            rows: c_ushort,
            cols: c_ushort,
            x: c_ushort,
            y: c_ushort,
        }

        let size = TermSize { rows: 0, cols: 0, x: 0, y: 0 };
        if unsafe { ioctl(self.output.as_raw_fd(), TIOCGWINSZ, &size) } == 0 {
            Some((size.cols as u16, size.rows as u16))
        } else {
            None
        }
    }
}

#[cfg(test)]
mod tests {
    use super::Terminal;

    #[test]
    fn winsize_test() {
        // Travis-CI builds run without a tty, making this test impossible.
        if option_env!("TRAVIS").is_some() {
            // TODO: It should be made obvious from the output that this test was skipped
            return;
        }
        let term = Terminal::open_terminal();
        let (cols, rows) = term.winsize().expect("Failed to get window size!");
        // We don't know the window size a priori, but we can at least
        // assert that it is within some kind of sensible range.
        assert!(cols > 40);
        assert!(rows > 40);
        assert!(cols < 1000);
        assert!(rows < 1000);
    }
}
<commit_msg>Ignore bracketed paste mode control codes<commit_after>#![cfg(not(windows))]

extern crate libc;

use screen::Key;
use screen::Key::*;
use std::io::{Read, Write};
use std::fs::{File, OpenOptions};
use std::os::unix::io::AsRawFd;
use std::path::*;
use std::process::Command;
use std::process::Stdio;
use std::str;
use std::iter::repeat;
use std::cmp::min;
use ansi;
use ::NEWLINE;

use std::{mem, ptr};
use std::thread;
use std::sync::mpsc::Sender;
use std::sync::mpsc::Receiver;
use std::sync::{mpsc, Arc, Mutex};

use self::libc::{sigaction, SIGWINCH, SA_SIGINFO, sighandler_t, c_int, c_ushort, c_ulong};

pub struct Screen {
    tty: Terminal,
    original_stty_state: Vec<u8>,
    pub height: u16,
    pub width: u16,
    pub visible_choices: u16,
    start_line: u16,
}

impl Screen {
    pub fn open_screen(desired_rows: u16) -> Screen {
        let mut tty = Terminal::open_terminal();
        let current_stty_state = tty.stty(&["-g"]);
        tty.initialize();
        let (cols, rows) = tty.winsize().unwrap();
        let visible_choices = min(desired_rows, rows - 1);
        let start_line = rows - visible_choices - 1;
        for _ in 0..visible_choices {
            tty.write(NEWLINE.as_bytes());
        }
        tty.write(&ansi::save_cursor());

        Screen {
            tty: tty,
            original_stty_state: current_stty_state,
            height: rows,
            width: cols,
            visible_choices: visible_choices,
            start_line: start_line,
        }
    }

    fn restore_tty(&mut self) {
        self.tty.stty(&[&String::from_utf8(self.original_stty_state.clone()).unwrap()]);
    }

    fn reset_cursor(&mut self) {
        self.tty.write(&ansi::restore_cursor());

        // Writing this carriage return works around a rendering bug in
        // Neovim's terminal emulation. Without it, the cursor flies
        // around all over the place while typing, and the input prompt
        // is not rendered correctly. I have no idea why this workaround
        // is effective, or why others are not, or what the root cause
        // of the issue is (likely something involving save/restore
        // support).
        self.tty.write(b"\r");

        self.tty.write(&ansi::cursor_up(self.visible_choices));
    }

    pub fn move_cursor_to_prompt_line(&mut self, col: u16) {
        self.reset_cursor();
        self.tty.write(&ansi::cursor_right(col));
    }

    pub fn blank_screen(&mut self) {
        self.reset_cursor();
        let blank_line = repeat(' ').take(self.width as usize).collect::<String>();
        for _ in 0..self.visible_choices + 1 {
            self.tty.write(blank_line.as_bytes());
        }
        self.reset_cursor();
    }

    pub fn show_cursor(&mut self) {
        self.tty.write(&ansi::show_cursor());
    }

    pub fn hide_cursor(&mut self) {
        self.tty.write(&ansi::hide_cursor());
    }

    pub fn write(&mut self, s: &str) {
        self.tty.write(s.as_bytes());
    }

    pub fn write_red_inverted(&mut self, s: &str) {
        self.tty.write(&ansi::red());
        self.tty.write(&ansi::inverse());
        self.tty.write(s.as_bytes());
        self.tty.write(&ansi::reset());
    }

    pub fn write_red(&mut self, s: &str) {
        self.tty.write(&ansi::red());
        self.tty.write(s.as_bytes());
        self.tty.write(&ansi::reset());
    }

    pub fn write_inverted(&mut self, s: &str) {
        self.tty.write(&ansi::inverse());
        self.tty.write(s.as_bytes());
        self.tty.write(&ansi::reset());
    }

    // Return all buffered keystrokes, or the next key if buffer is empty.
    pub fn get_buffered_keys(&mut self) -> Vec<Key> {
        let mut ret = Vec::new();
        while let Ok(bytes) = self.tty.input.try_recv() {
            ret.extend(bytes);
        }
        while ret.is_empty() {
            let bytes = self.tty.input.recv().unwrap();
            if bytes.len() == 1 && bytes[0] == 0 {
                self.tty.initialize();
                return vec![Nothing];
            } else {
                ret.extend(bytes);
            }
        }
        Terminal::translate_bytes(ret.clone())
    }

    fn more_bytes_needed(bytes: &[u8]) -> bool {
        if let Err(_) = str::from_utf8(bytes) { true } else { false }
    }
}

impl Drop for Screen {
    fn drop(&mut self) {
        self.restore_tty();
    }
}

struct Terminal {
    input: Receiver<Vec<u8>>,
    input_fd: i32,
    output: File,
}

static mut GLOBAL_TX: *const Arc<Mutex<Sender<Vec<u8>>>> = 0 as *const Arc<Mutex<Sender<Vec<u8>>>>;

fn set_global_tx(tx: Sender<Vec<u8>>) {
    unsafe {
        let singleton = Arc::new(Mutex::new(tx));
        GLOBAL_TX = mem::transmute(Box::new(singleton));
    }
}

fn get_global_tx() -> Sender<Vec<u8>> {
    let singleton = unsafe { (*GLOBAL_TX).clone() };
    let tx = singleton.lock().unwrap();
    tx.clone()
}

unsafe extern fn sigwinch_handler(signum: c_int) {
    if signum == SIGWINCH {
        get_global_tx().send([0].to_vec()).unwrap();
    }
}

fn register_sigwinch_handler() {
    unsafe {
        let mut action: sigaction = mem::zeroed();
        action.sa_flags = SA_SIGINFO;
        action.sa_sigaction = sigwinch_handler as sighandler_t;
        sigaction(SIGWINCH, &action, ptr::null_mut());
    }
}

impl Terminal {
    fn open_terminal() -> Terminal {
        let term_path = Path::new("/dev/tty");
        let mut input_file = File::open(&term_path).unwrap();
        let output_file = OpenOptions::new().write(true).open(&term_path).unwrap();
        let input_fd = input_file.as_raw_fd();
        let (tx, rx) = mpsc::channel();
        set_global_tx(tx);

        register_sigwinch_handler();

        thread::spawn(move || {
            loop {
                let mut buf = [0; 255];
                let tx = get_global_tx();
                if let Ok(length) = input_file.read(&mut buf) {
                    tx.send(buf[0..length].to_vec()).unwrap();
                } else {
                    tx.send([0].to_vec()).unwrap();
                }
            }
        });
        Terminal {
            input: rx,
            input_fd: input_fd,
            output: output_file,
        }
    }

    fn initialize(&mut self) {
        self.stty(&["raw", "-echo", "cbreak", "opost", "onlcr"]);
    }

    fn stty(&mut self, args: &[&str]) -> Vec<u8> {
        extern { fn dup2(src: c_int, dst: c_int) -> c_int;  }
        unsafe {
            // This is a hack until a replacement for InheritFd from old_io is available.
            dup2(self.input_fd, 0);
        }
        let mut process = match Command::new("stty")
            .args(args)
            .stdin(Stdio::inherit())
            .stdout(Stdio::piped())
            .spawn()
        {
            Ok(p) => p,
            Err(e) => panic!("Spawn failed: {}", e),
        };

        let exit = process.wait();

        let mut buf = Vec::new();
        if exit.unwrap().success() {
            process.stdout.as_mut().unwrap().read_to_end(&mut buf).unwrap();
            let mut str = String::from_utf8(buf).unwrap();

            // The output from `stty -g` may include a newline, which we have to strip off. Otherwise,
            // when we go to restore the tty, stty (on some platforms) will fail with an "invalid
            // argument" error.
            ::trim(&mut str);

            str.into_bytes()
        } else {
            process.stderr.as_mut().unwrap().read_to_end(&mut buf).unwrap();
            panic!(String::from_utf8(buf).unwrap());
        }
    }

    fn translate_bytes(bytes: Vec<u8>) -> Vec<Key> {
        const BEGIN_PASTE: &'static [u8] = b"\x1B[200~";
        const END_PASTE: &'static [u8] = b"\x1B[201~";

        if bytes == b"\x1B[A" || bytes == b"\x1BOA" { return vec![Up] };
        if bytes == b"\x1B[B" || bytes == b"\x1BOB" { return vec![Down] };

        let bs = if bytes.starts_with(BEGIN_PASTE) && bytes.ends_with(END_PASTE) {
            let start = BEGIN_PASTE.len();
            let end = bytes.len() - END_PASTE.len();
            bytes[start..end].to_vec()
        } else {
            bytes
        };

        String::from_utf8(bs)
            .unwrap()
            .chars()
            .map(Terminal::translate_char)
            .collect()
    }

    fn translate_char(c: char) -> Key {
        let numeric_char = c as u32;
        if c == '\r' {
            Enter
        } else if numeric_char == 9 {
            Tab
        } else if numeric_char == 127 {
            Backspace
        } else if numeric_char & 96 == 0 && numeric_char <= 128u32 {
            let c = c as u8;
            Control((c + 96u8) as char)
        } else {
            Char(c)
        }
    }

    fn write(&mut self, s: &[u8]) {
        self.output.write_all(s).unwrap();
    }

    fn writeln(&mut self, s: &str) {
        self.output.write(s.as_bytes()).unwrap();
        self.output.write(b"\n").unwrap();
    }

    fn winsize(&self) -> Option<(u16, u16)> {
        extern { fn ioctl(fd: c_int, request: c_ulong, ...) -> c_int; }
        #[cfg(any(target_os = "macos", target_os = "freebsd"))]
        const TIOCGWINSZ: c_ulong = 0x40087468;

        #[cfg(any(target_os = "linux", target_os = "android"))]
        const TIOCGWINSZ: c_ulong = 0x00005413;

        #[repr(C)]
        struct TermSize {
            rows: c_ushort,
            cols: c_ushort,
            x: c_ushort,
            y: c_ushort,
        }

        let size = TermSize { rows: 0, cols: 0, x: 0, y: 0 };
        if unsafe { ioctl(self.output.as_raw_fd(), TIOCGWINSZ, &size) } == 0 {
            Some((size.cols as u16, size.rows as u16))
        } else {
            None
        }
    }
}

#[cfg(test)]
mod tests {
    use super::Terminal;

    #[test]
    fn winsize_test() {
        // Travis-CI builds run without a tty, making this test impossible.
        if option_env!("TRAVIS").is_some() {
            // TODO: It should be made obvious from the output that this test was skipped
            return;
        }
        let term = Terminal::open_terminal();
        let (cols, rows) = term.winsize().expect("Failed to get window size!");
        // We don't know the window size a priori, but we can at least
        // assert that it is within some kind of sensible range.
        assert!(cols > 40);
        assert!(rows > 40);
        assert!(cols < 1000);
        assert!(rows < 1000);
    }
}
<|endoftext|>


<commit_before>//! # Custom `derive` pretending to be functional procedural macros on Rust 1.15
//!
//! This crate enables creating function-like macros (invoked as `foo!(...)`)
//! with a procedural component,
//! based on both custom `derive` (a.k.a. *Macros 1.1*) and `macro_rules!`.
//!
//! This convoluted mechanism enables such macros to run on stable Rust 1.15,
//! even though functional procedural macros (a.k.a. *Macros 2.0*) are not available yet.
//!
//! A library defining such a macro needs two crates: a “normal” one, and a `proc-macro` one.
//! In the example below we’ll call them `libfoo` and `libfoo-macros`, respectively.
//!
//! # Credits
//!
//! The trick that makes this crate work
//! is based on an idea from [David Tolnay](https://github.com/dtolnay).
//! Many thanks!
//!
//! # Example
//!
//! As a simple example, we’re going to re-implement the `stringify!` macro.
//! This is useless since `stringify!` already exists in the standard library,
//! and a bit absurd since this crate uses `stringify!` internally.
//!
//! Nevertheless, it serves as a simple example to demonstrate the use of this crate.
//!
//! ## The `proc-macro` crate
//!
//! The minimal `Cargo.toml` file is typical for Macros 1.1:
//!
//! ```toml
//! [package]
//! name = "libfoo-macros"
//! version = "1.0.0"
//!
//! [lib]
//! proc-macro = true
//! ```
//!
//! In the code, we define the procedural part of our macro in a function.
//! This function will not be used directly by end users,
//! but it still needs to be re-exported to them
//! (because of limitations in `macro_rules!`).
//!
//! To avoid name collisions, we and a long and explicit prefix in the function’s name.
//!
//! The function takes a string containing arbitrary Rust tokens,
//! and returns a string that is parsed as *items*.
//! The returned string can contain constants, statics, functions, `impl`s, etc.,
//! but not expressions directly.
//!
//! ```rust
//! #[macro_use] extern crate procedural_masquerade;
//! extern crate proc_macro;
//!
//! define_proc_macros! {
//!     #[allow(non_snake_case)]
//!     pub fn foo_internal__stringify_const(input: &str) -> String {
//!         format!("const STRINGIFIED: &'static str = {:?};", input)
//!     }
//! }
//! ```
//!
//! A less trivial macro would probably use
//! the [`syn`](https://github.com/dtolnay/syn/) crate to parse its input
//! and the [`quote`](https://github.com/dtolnay/quote) crate to generate its output.
//!
//! ## The library crate
//!
//! ```toml
//! [package]
//! name = "libfoo"
//! version = "1.0.0"
//!
//! [dependencies]
//! cssparser-macros = {path = "./macros", version = "1.0"}
//! ```
//!
//! ```rust
//! #[macro_use] extern crate libfoo_macros;  // (1)
//!
//! pub use libfoo_macros::*;  // (2)
//!
//! define_invoke_proc_macro!(libfoo__invoke_proc_macro);  // (3)
//!
//! #[macro_export]
//! macro_rules! foo_stringify {  // (4)
//!     ( $( $tts: tt ) ) => {
//!         {  // (5)
//!             libfoo__invoke_proc_macro! {  // (6)
//!                 foo_internal__stringify_const!( $( $tts ) )  // (7)
//!             }
//!             STRINGIFIED  // (8)
//!         }
//!     }
//! }
//! ```
//!
//! Let’s go trough the numbered lines one by one:
//!
//! 1. `libfoo` depends on the other `libfoo-macros`, and imports its macros.
//! 2. Everything exported by `libfoo-macros` (which is one custom `derive`)
//!    is re-exported to users of `libfoo`.
//!    They’re not expected to use it directly,
//!    but expansion of the `foo_stringify` macro needs it.
//! 3. This macro invocation defines yet another macro, called `libfoo__invoke_proc_macro`,
//!    which is also exported.
//!    This indirection is necessary
//!    because re-exporting `macro_rules!` macros doesn’t work currently,
//!    and once again it is used by the expansion of `foo_stringify`.
//!    Again, we use a long prefix to avoid name collisions.
//! 4. Finally, we define the macro that we really want.
//!    This one has a name that users will use.
//! 5. The expansion of this macro will define some items,
//!    whose names are not hygienic in `macro_rules`.
//!    So we wrap everything in an extra `{…}` block to prevent these names for leaking.
//! 6. Here we use the macro defined in (3),
//!    which allows us to write something that look like invoking a functional procedural macro,
//!    but really uses a custom `derive`.
//!    This will define a type called `ProceduralMasqueradeDummyType`,
//!    as a placeholder to use `derive`.
//!    If `libfoo__invoke_proc_macro!` is to be used more than once,
//!    each use needs to be nested in another block
//!    so that the names of multiple dummy types don’t collide.
//! 7. In addition to the dummy type,
//!    the items returned by our procedural component are inserted here.
//!    (In this case the `STRINGIFIED` constant.)
//! 8. Finally, we write the expression that we want the macro to evaluate to.
//!    This expression can use parts of `foo_stringify`’s input,
//!    it can contain control-flow statements like `return` or `continue`,
//!    and of course refer to procedurally-defined items.
//!
//! This macro can be used in an expression context.
//! It expands to a block-expression that contains some items (as an implementation detail)
//! and ends with another expression.
//!
//! ## For users
//!
//! Users of `libfoo` don’t need to worry about any of these implementation details.
//! They can use the `foo_stringify` macro as if it were a simle `macro_rules` macro:
//!
//! ```rust
//! #[macro_use] extern crate libfoo;
//!
//! fn main() {
//!     do_something(foo_stringify!(1 + 2));
//! }
//!
//! fn do_something(_: &str) { /* ... */ }
//! ```
//!
//! # More
//!
//! To see a more complex example, look at
//! [`cssparser`’s `src/macros.rs](https://github.com/servo/rust-cssparser/blob/master/src/macros.rs)
//! and
//! [`cssparser-macros`’s `macros/lib.rs](https://github.com/servo/rust-cssparser/blob/master/macros/lib.rs).

/////
///// This indirection is necessary because re-exporting `macro_rules!` macros doesn’t work.
///// Without it, when a library `libfoo` defines a macro whose expansion uses directly
///// a macro

/// This macro wraps `&str -> String` functions
/// in custom `derive` implementations with `#[proc_macro_derive]`.
///
/// See crate documentation for details.
#[macro_export]
macro_rules! define_proc_macros {
    (
        $(
            $( #[$attr:meta] )*
            pub fn $proc_macro_name: ident ($input: ident : &str) -> String
            $body: block
        )+
    ) => {
        $(
            $( #[$attr] )*
            #[proc_macro_derive($proc_macro_name)]
            pub fn $proc_macro_name(input: ::proc_macro::TokenStream) -> ::proc_macro::TokenStream {
                // Use another function to hide this one’s local variables from $block
                #[inline]
                fn implementation($input: &str) -> String $body

                let input = input.to_string();
                let mut normalized = String::with_capacity(input.len());
                for piece in input.split_whitespace() {
                    normalized.push_str(piece);
                    normalized.push(' ');
                }

                let prefix = "#[allow(unused)] enum ProceduralMasqueradeDummyType { Input = (0, stringify!(";
                let suffix = ")).0, } ";
                assert!(normalized.starts_with(prefix), "expected prefix not found in {:?}", input);
                assert!(normalized.ends_with(suffix), "expected suffix not found in {:?}", input);

                let start = prefix.len();
                let end = normalized.len() - suffix.len();
                let output = implementation(&normalized[start..end]);
                output.parse().unwrap()
            }
        )+
    }
}

/// This macro expands to the definition of another macro (whose name is given as a parameter).
///
/// See crate documentation for details.
#[macro_export]
macro_rules! define_invoke_proc_macro {
    ($macro_name: ident) => {
        #[macro_export]
        macro_rules! $macro_name {
            ($proc_macro_name: ident ! $paren: tt) => {
                #[derive($proc_macro_name)]
                #[allow(unused)]
                enum ProceduralMasqueradeDummyType {
                    // The magic happens here.
                    //
                    // We use an `enum` with an explicit discriminant
                    // because that is the only case where a type definition
                    // can contain a (const) expression.
                    //
                    // `(0, "foo").0` evalutes to 0, with the `"foo"` part ignored.
                    //
                    // By the time the `#[proc_macro_derive]` function
                    // implementing `#[derive($proc_macro_name)]` is called,
                    // `$paren` has already been replaced with the input of this inner macro,
                    // but `stringify!` has not been expanded yet.
                    //
                    // This how arbitrary tokens can be inserted
                    // in the input to the `#[proc_macro_derive]` function.
                    //
                    // Later, `stringify!(...)` is expanded into a string literal
                    // which is then ignored.
                    // Using `stringify!` enables passing arbitrary tokens
                    // rather than only what can be parsed as a const expression.
                    Input = (0, stringify! $paren ).0
                }
            }
        }
    }
}
<commit_msg>Remove leftover copy/pasted comment<commit_after>//! # Custom `derive` pretending to be functional procedural macros on Rust 1.15
//!
//! This crate enables creating function-like macros (invoked as `foo!(...)`)
//! with a procedural component,
//! based on both custom `derive` (a.k.a. *Macros 1.1*) and `macro_rules!`.
//!
//! This convoluted mechanism enables such macros to run on stable Rust 1.15,
//! even though functional procedural macros (a.k.a. *Macros 2.0*) are not available yet.
//!
//! A library defining such a macro needs two crates: a “normal” one, and a `proc-macro` one.
//! In the example below we’ll call them `libfoo` and `libfoo-macros`, respectively.
//!
//! # Credits
//!
//! The trick that makes this crate work
//! is based on an idea from [David Tolnay](https://github.com/dtolnay).
//! Many thanks!
//!
//! # Example
//!
//! As a simple example, we’re going to re-implement the `stringify!` macro.
//! This is useless since `stringify!` already exists in the standard library,
//! and a bit absurd since this crate uses `stringify!` internally.
//!
//! Nevertheless, it serves as a simple example to demonstrate the use of this crate.
//!
//! ## The `proc-macro` crate
//!
//! The minimal `Cargo.toml` file is typical for Macros 1.1:
//!
//! ```toml
//! [package]
//! name = "libfoo-macros"
//! version = "1.0.0"
//!
//! [lib]
//! proc-macro = true
//! ```
//!
//! In the code, we define the procedural part of our macro in a function.
//! This function will not be used directly by end users,
//! but it still needs to be re-exported to them
//! (because of limitations in `macro_rules!`).
//!
//! To avoid name collisions, we and a long and explicit prefix in the function’s name.
//!
//! The function takes a string containing arbitrary Rust tokens,
//! and returns a string that is parsed as *items*.
//! The returned string can contain constants, statics, functions, `impl`s, etc.,
//! but not expressions directly.
//!
//! ```rust
//! #[macro_use] extern crate procedural_masquerade;
//! extern crate proc_macro;
//!
//! define_proc_macros! {
//!     #[allow(non_snake_case)]
//!     pub fn foo_internal__stringify_const(input: &str) -> String {
//!         format!("const STRINGIFIED: &'static str = {:?};", input)
//!     }
//! }
//! ```
//!
//! A less trivial macro would probably use
//! the [`syn`](https://github.com/dtolnay/syn/) crate to parse its input
//! and the [`quote`](https://github.com/dtolnay/quote) crate to generate its output.
//!
//! ## The library crate
//!
//! ```toml
//! [package]
//! name = "libfoo"
//! version = "1.0.0"
//!
//! [dependencies]
//! cssparser-macros = {path = "./macros", version = "1.0"}
//! ```
//!
//! ```rust
//! #[macro_use] extern crate libfoo_macros;  // (1)
//!
//! pub use libfoo_macros::*;  // (2)
//!
//! define_invoke_proc_macro!(libfoo__invoke_proc_macro);  // (3)
//!
//! #[macro_export]
//! macro_rules! foo_stringify {  // (4)
//!     ( $( $tts: tt ) ) => {
//!         {  // (5)
//!             libfoo__invoke_proc_macro! {  // (6)
//!                 foo_internal__stringify_const!( $( $tts ) )  // (7)
//!             }
//!             STRINGIFIED  // (8)
//!         }
//!     }
//! }
//! ```
//!
//! Let’s go trough the numbered lines one by one:
//!
//! 1. `libfoo` depends on the other `libfoo-macros`, and imports its macros.
//! 2. Everything exported by `libfoo-macros` (which is one custom `derive`)
//!    is re-exported to users of `libfoo`.
//!    They’re not expected to use it directly,
//!    but expansion of the `foo_stringify` macro needs it.
//! 3. This macro invocation defines yet another macro, called `libfoo__invoke_proc_macro`,
//!    which is also exported.
//!    This indirection is necessary
//!    because re-exporting `macro_rules!` macros doesn’t work currently,
//!    and once again it is used by the expansion of `foo_stringify`.
//!    Again, we use a long prefix to avoid name collisions.
//! 4. Finally, we define the macro that we really want.
//!    This one has a name that users will use.
//! 5. The expansion of this macro will define some items,
//!    whose names are not hygienic in `macro_rules`.
//!    So we wrap everything in an extra `{…}` block to prevent these names for leaking.
//! 6. Here we use the macro defined in (3),
//!    which allows us to write something that look like invoking a functional procedural macro,
//!    but really uses a custom `derive`.
//!    This will define a type called `ProceduralMasqueradeDummyType`,
//!    as a placeholder to use `derive`.
//!    If `libfoo__invoke_proc_macro!` is to be used more than once,
//!    each use needs to be nested in another block
//!    so that the names of multiple dummy types don’t collide.
//! 7. In addition to the dummy type,
//!    the items returned by our procedural component are inserted here.
//!    (In this case the `STRINGIFIED` constant.)
//! 8. Finally, we write the expression that we want the macro to evaluate to.
//!    This expression can use parts of `foo_stringify`’s input,
//!    it can contain control-flow statements like `return` or `continue`,
//!    and of course refer to procedurally-defined items.
//!
//! This macro can be used in an expression context.
//! It expands to a block-expression that contains some items (as an implementation detail)
//! and ends with another expression.
//!
//! ## For users
//!
//! Users of `libfoo` don’t need to worry about any of these implementation details.
//! They can use the `foo_stringify` macro as if it were a simle `macro_rules` macro:
//!
//! ```rust
//! #[macro_use] extern crate libfoo;
//!
//! fn main() {
//!     do_something(foo_stringify!(1 + 2));
//! }
//!
//! fn do_something(_: &str) { /* ... */ }
//! ```
//!
//! # More
//!
//! To see a more complex example, look at
//! [`cssparser`’s `src/macros.rs](https://github.com/servo/rust-cssparser/blob/master/src/macros.rs)
//! and
//! [`cssparser-macros`’s `macros/lib.rs](https://github.com/servo/rust-cssparser/blob/master/macros/lib.rs).

/// This macro wraps `&str -> String` functions
/// in custom `derive` implementations with `#[proc_macro_derive]`.
///
/// See crate documentation for details.
#[macro_export]
macro_rules! define_proc_macros {
    (
        $(
            $( #[$attr:meta] )*
            pub fn $proc_macro_name: ident ($input: ident : &str) -> String
            $body: block
        )+
    ) => {
        $(
            $( #[$attr] )*
            #[proc_macro_derive($proc_macro_name)]
            pub fn $proc_macro_name(input: ::proc_macro::TokenStream) -> ::proc_macro::TokenStream {
                // Use another function to hide this one’s local variables from $block
                #[inline]
                fn implementation($input: &str) -> String $body

                let input = input.to_string();
                let mut normalized = String::with_capacity(input.len());
                for piece in input.split_whitespace() {
                    normalized.push_str(piece);
                    normalized.push(' ');
                }

                let prefix = "#[allow(unused)] enum ProceduralMasqueradeDummyType { Input = (0, stringify!(";
                let suffix = ")).0, } ";
                assert!(normalized.starts_with(prefix), "expected prefix not found in {:?}", input);
                assert!(normalized.ends_with(suffix), "expected suffix not found in {:?}", input);

                let start = prefix.len();
                let end = normalized.len() - suffix.len();
                let output = implementation(&normalized[start..end]);
                output.parse().unwrap()
            }
        )+
    }
}

/// This macro expands to the definition of another macro (whose name is given as a parameter).
///
/// See crate documentation for details.
#[macro_export]
macro_rules! define_invoke_proc_macro {
    ($macro_name: ident) => {
        #[macro_export]
        macro_rules! $macro_name {
            ($proc_macro_name: ident ! $paren: tt) => {
                #[derive($proc_macro_name)]
                #[allow(unused)]
                enum ProceduralMasqueradeDummyType {
                    // The magic happens here.
                    //
                    // We use an `enum` with an explicit discriminant
                    // because that is the only case where a type definition
                    // can contain a (const) expression.
                    //
                    // `(0, "foo").0` evalutes to 0, with the `"foo"` part ignored.
                    //
                    // By the time the `#[proc_macro_derive]` function
                    // implementing `#[derive($proc_macro_name)]` is called,
                    // `$paren` has already been replaced with the input of this inner macro,
                    // but `stringify!` has not been expanded yet.
                    //
                    // This how arbitrary tokens can be inserted
                    // in the input to the `#[proc_macro_derive]` function.
                    //
                    // Later, `stringify!(...)` is expanded into a string literal
                    // which is then ignored.
                    // Using `stringify!` enables passing arbitrary tokens
                    // rather than only what can be parsed as a const expression.
                    Input = (0, stringify! $paren ).0
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) Carl-Erwin Griffith
//
// Permission is hereby granted, free of charge, to any
// person obtaining a copy of this software and associated
// documentation files (the "Software"), to deal in the
// Software without restriction, including without
// limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software
// is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice
// shall be included in all copies or substantial portions
// of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER

// {
//   Derived from : Bjoern Hoehrmann work
//   Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
//   See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.

const UTF8_ACCEPT: u32 = 0;
const UTF8_REJECT: u32 = 1;

#[rustfmt::skip]
static UTF8D: &'static [u8] = &[
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
  8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
  0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
  0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
  0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
  1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
  1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
  1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
];

/*
    state UTF8_ACCEPT => initial state or decoding successful
    state UTF8_REJECT => error
    state other => intermediate states need more inputs
*/
#[inline]
pub fn decode_byte(state: u32, byte: u8, codep: &mut u32) -> u32 {
    let cp_type = u32::from(UTF8D[byte as usize]);

    *codep = if state != UTF8_ACCEPT {
        u32::from(byte & 0x3f) | (*codep << 6)
    } else {
        (0xff >> cp_type) & u32::from(byte)
    };

    u32::from(UTF8D[256 + (state * 16) as usize + cp_type as usize])
}

// } end-of-derived code

#[inline]
pub fn is_codepoint_start(byte: u8) -> bool {
    if byte < 0x80 {
        return true;
    }

    if byte >= 0xC2 && byte <= 0xDF {
        return true;
    }

    if byte >= 0xC2 && byte <= 0xDF {
        return true;
    }

    if byte >= 0xE0 && byte <= 0xEC {
        return true;
    }

    if byte >= 0xED && byte <= 0xEF {
        return true;
    }

    if byte >= 0xF0 && byte <= 0xF4 {
        return true;
    }

    false
}

// return 0 on error, or the number of written bytes
pub fn encode(codepoint: u32, out: &mut [u8; 4]) -> usize {
    if codepoint < 0x80 {
        out[0] = codepoint as u8;
        return 1;
    }

    if codepoint < 0x800 {
        out[0] = 0xC0 | (codepoint >> 6) as u8;
        out[1] = 0x80 | (codepoint & 0x3F) as u8;
        return 2;
    }

    if codepoint < 0xFFFF {
        out[0] = 0xE0 | (codepoint >> 12) as u8;
        out[1] = 0x80 | (codepoint >> 6) as u8;
        out[2] = 0x80 | (codepoint & 0x3F) as u8;
        return 3;
    }

    if codepoint < 0x0010_FFFF {
        out[0] = 0xF0 | (codepoint >> 18) as u8;
        out[1] = 0x80 | (codepoint >> 12) as u8;
        out[2] = 0x80 | (codepoint >> 6) as u8;
        out[3] = 0x80 | (codepoint & 0x3F) as u8;
        return 4;
    }

    0
}

// TODO: change this with temporary (cp, offset, size) until from_offset
pub fn get_previous_codepoint_start(data: &[u8], from_offset: u64) -> u64 {
    //                 cp    size   offset
    let mut cp_info: [(char, usize, u64); 8] = [
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
    ];
    let mut nr_cpinfo = 0;

    // rewind up to 4 bytes
    // and decode forward / save offset
    let mut off = if from_offset > 4 { from_offset - 4 } else { 0 };
    while off < from_offset {
        let (cp, _, size) = get_codepoint(data, off);

        cp_info[nr_cpinfo] = (cp, size, off);
        nr_cpinfo += 1;

        off += size as u64;
    }

    if nr_cpinfo != 0 {
        cp_info[nr_cpinfo - 1].2
    } else {
        from_offset
    }
}

pub fn get_next_codepoint_start(data: &[u8], from_offset: u64) -> u64 {
    let (_, _, size) = get_codepoint(data, from_offset);
    from_offset + size as u64
}

pub fn get_codepoint(data: &[u8], from_offset: u64) -> (char, u64, usize) {
    let mut state = 0;
    let mut codep = 0;
    let mut size = 0;

    for b in data.iter().skip(from_offset as usize) {
        size += 1;
        state = decode_byte(state, *b, &mut codep);
        match state {
            UTF8_ACCEPT => {
                break;
            }

            UTF8_REJECT => {
                // decode error : invalid sequence
                codep = 0xfffd;
                size = 1; // force restart @ next byte
                break;
            }
            _ => {}
        }
    }

    // TODO return Result<(char, usize), status> -> state != 1|0 -> need mode data
    (
        crate::core::codec::text::u32_to_char(codep),
        from_offset,
        size,
    )
}

pub fn get_prev_codepoint(data: &[u8], from_offset: u64) -> (char, u64, usize) {
    let offset = get_previous_codepoint_start(data, from_offset);
    get_codepoint(data, offset)
}

#[test]
fn test_codec_encode() {
    let expect_cp: [u8; 4] = [0xe2, 0x82, 0xac, 0x00];
    let mut mut_cp: [u8; 4] = [0x00, 0x00, 0x00, 0x00];

    let n = encode(0x20ac as u32, &mut mut_cp);
    assert_eq!(n, 3);
    assert_eq!(mut_cp, expect_cp);
}

#[test]
fn test_codec_decode() {
    use crate::core::codec::text::u32_to_char;

    let mut state: u32 = 0;
    let mut codep: u32 = 0;

    let sequence: [u8; 4] = [0xe2, 0x82, 0xac, 0x00];
    for b in &sequence {
        println!("decode byte '{:x}'", *b);
        state = decode_byte(state, *b, &mut codep);
        match state {
            UTF8_ACCEPT => {
                break;
            }
            UTF8_REJECT => {
                println!("invalid utf8 sequence");
                break;
            }
            _ => {
                continue;
            }
        }
    }

    println!("decoded codepoint value {:x}", codep);
    let c = u32_to_char(codep);
    println!("decoded codepoint char {}", c);

    assert_eq!(codep, 0x20ac);
}

#[test]
fn test2_codec_decode() {
    let data: [u8; 27] = [
        0xe2, 0x82, 0xac, 0xe2, 0x82, 0x61, 0x0a, 0x82, 0xac, 0xe2, 0x82, 0x61, 0x0a, 0xac, 0xe2,
        0x82, 0x61, 0x0a, 0xe2, 0x82, 0x61, 0x0a, 0x82, 0x61, 0x0a, 0x61, 0x0a,
    ];

    let mut state: u32 = 0;
    let mut codep: u32 = 0;

    for b in &data {
        println!("decode byte '{:x}'", *b);
        state = decode_byte(state, *b, &mut codep);
        println!("state  '{}'", state);
        match state {
            UTF8_ACCEPT => {
                println!("decoded cp = '{:?}'", codep);
                state = 0;
                codep = 0;
            }
            UTF8_REJECT => {
                println!("invalid utf8 sequence, restart");
                state = 0;
                codep = 0;
                state = decode_byte(state, *b, &mut codep);
                match state {
                    UTF8_ACCEPT => {
                        println!("decoded cp = '{:?}'", codep);
                        state = 0;
                        codep = 0;
                    }
                    UTF8_REJECT => {
                        println!("invalid utf8 sequence, restart");
                        state = 0;
                        codep = 0;
                    }
                    _ => {}
                }
            }

            _ => {
                continue;
            }
        }
    }
}

#[test]
fn test_backward_decode() {
    let data: [u8; 24] = [
        0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac,
        0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac,
    ];

    let mut start_offset: u64 = 6;
    println!(
        "start_offset {} byte is '{:x}'",
        start_offset, data[start_offset as usize] as u32
    );

    let off = get_previous_codepoint_start(&data, start_offset as u64);
    println!(
        "start_offset({}) - off({}) = {}",
        start_offset,
        off,
        start_offset - off
    );
    let delta = start_offset - off;
    start_offset -= delta;

    println!("new start_offset({})", start_offset);
}
<commit_msg>update utf8 decoder<commit_after>// Copyright (c) Carl-Erwin Griffith
//
// Permission is hereby granted, free of charge, to any
// person obtaining a copy of this software and associated
// documentation files (the "Software"), to deal in the
// Software without restriction, including without
// limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software
// is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice
// shall be included in all copies or substantial portions
// of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER

// {
//   Derived from : Bjoern Hoehrmann work
//   Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
//   See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.

const UTF8_ACCEPT: u32 = 0;
const UTF8_REJECT: u32 = 12;

#[rustfmt::skip]
static UTF8D: &'static [u8] = &[
  // The first part of the table maps bytes to character classes that
  // to reduce the size of the transition table and create bitmasks.
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
  10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,

  // The second part is a transition table that maps a combination
  // of a state of the automaton and a character class to a state.
  0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
  12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
  12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
  12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
  12,36,12,12,12,12,12,12,12,12,12,12,
];

/*
    state UTF8_ACCEPT => initial state or decoding successful
    state UTF8_REJECT => error
    state other => intermediate states need more inputs
*/
#[inline]
pub fn decode_byte(state: u32, byte: u8, codep: &mut u32) -> u32 {
    let cp_type = u32::from(UTF8D[byte as usize]);

    *codep = if state != UTF8_ACCEPT {
        u32::from(byte & 0x3f) | (*codep << 6)
    } else {
        (0xff >> cp_type) & u32::from(byte)
    };

    u32::from(UTF8D[(256 + state + cp_type) as usize])
}

// } end-of-derived code

#[inline]
pub fn is_codepoint_start(byte: u8) -> bool {
    if byte < 0x80 {
        return true;
    }

    if byte >= 0xC2 && byte <= 0xDF {
        return true;
    }

    if byte >= 0xC2 && byte <= 0xDF {
        return true;
    }

    if byte >= 0xE0 && byte <= 0xEC {
        return true;
    }

    if byte >= 0xED && byte <= 0xEF {
        return true;
    }

    if byte >= 0xF0 && byte <= 0xF4 {
        return true;
    }

    false
}

// return 0 on error, or the number of written bytes
pub fn encode(codepoint: u32, out: &mut [u8; 4]) -> usize {
    if codepoint < 0x80 {
        out[0] = codepoint as u8;
        return 1;
    }

    if codepoint < 0x800 {
        out[0] = 0xC0 | (codepoint >> 6) as u8;
        out[1] = 0x80 | (codepoint & 0x3F) as u8;
        return 2;
    }

    if codepoint < 0xFFFF {
        out[0] = 0xE0 | (codepoint >> 12) as u8;
        out[1] = 0x80 | (codepoint >> 6) as u8;
        out[2] = 0x80 | (codepoint & 0x3F) as u8;
        return 3;
    }

    if codepoint < 0x0010_FFFF {
        out[0] = 0xF0 | (codepoint >> 18) as u8;
        out[1] = 0x80 | (codepoint >> 12) as u8;
        out[2] = 0x80 | (codepoint >> 6) as u8;
        out[3] = 0x80 | (codepoint & 0x3F) as u8;
        return 4;
    }

    0
}

// TODO: change this with temporary (cp, offset, size) until from_offset
pub fn get_previous_codepoint_start(data: &[u8], from_offset: u64) -> u64 {
    //                 cp    size   offset
    let mut cp_info: [(char, usize, u64); 8] = [
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
        ('\0', 0, 0),
    ];
    let mut nr_cpinfo = 0;

    // rewind up to 4 bytes
    // and decode forward / save offset
    let mut off = if from_offset > 4 { from_offset - 4 } else { 0 };
    while off < from_offset {
        let (cp, _, size) = get_codepoint(data, off);

        cp_info[nr_cpinfo] = (cp, size, off);
        nr_cpinfo += 1;

        off += size as u64;
    }

    if nr_cpinfo != 0 {
        cp_info[nr_cpinfo - 1].2
    } else {
        from_offset
    }
}

pub fn get_next_codepoint_start(data: &[u8], from_offset: u64) -> u64 {
    let (_, _, size) = get_codepoint(data, from_offset);
    from_offset + size as u64
}

pub fn get_codepoint(data: &[u8], from_offset: u64) -> (char, u64, usize) {
    let mut state = 0;
    let mut codep = 0;
    let mut size = 0;

    for b in data.iter().skip(from_offset as usize) {
        size += 1;
        state = decode_byte(state, *b, &mut codep);
        match state {
            UTF8_ACCEPT => {
                break;
            }

            UTF8_REJECT => {
                // decode error : invalid sequence
                codep = 0xfffd;
                size = 1; // force restart @ next byte
                break;
            }
            _ => {}
        }
    }

    // TODO return Result<(char, usize), status> -> state != 1|0 -> need mode data
    (
        crate::core::codec::text::u32_to_char(codep),
        from_offset,
        size,
    )
}

pub fn get_prev_codepoint(data: &[u8], from_offset: u64) -> (char, u64, usize) {
    let offset = get_previous_codepoint_start(data, from_offset);
    get_codepoint(data, offset)
}

#[test]
fn test_codec_encode() {
    let expect_cp: [u8; 4] = [0xe2, 0x82, 0xac, 0x00];
    let mut mut_cp: [u8; 4] = [0x00, 0x00, 0x00, 0x00];

    let n = encode(0x20ac as u32, &mut mut_cp);
    assert_eq!(n, 3);
    assert_eq!(mut_cp, expect_cp);
}

#[test]
fn test_codec_decode() {
    use crate::core::codec::text::u32_to_char;

    let mut state: u32 = 0;
    let mut codep: u32 = 0;

    let sequence: [u8; 4] = [0xe2, 0x82, 0xac, 0x00];
    for b in &sequence {
        println!("decode byte '{:x}'", *b);
        state = decode_byte(state, *b, &mut codep);
        match state {
            UTF8_ACCEPT => {
                break;
            }
            UTF8_REJECT => {
                println!("invalid utf8 sequence");
                break;
            }
            _ => {
                continue;
            }
        }
    }

    println!("decoded codepoint value {:x}", codep);
    let c = u32_to_char(codep);
    println!("decoded codepoint char {}", c);

    assert_eq!(codep, 0x20ac);
}

#[test]
fn test2_codec_decode() {
    let data: [u8; 27] = [
        0xe2, 0x82, 0xac, 0xe2, 0x82, 0x61, 0x0a, 0x82, 0xac, 0xe2, 0x82, 0x61, 0x0a, 0xac, 0xe2,
        0x82, 0x61, 0x0a, 0xe2, 0x82, 0x61, 0x0a, 0x82, 0x61, 0x0a, 0x61, 0x0a,
    ];

    let mut state: u32 = 0;
    let mut codep: u32 = 0;

    for b in &data {
        println!("decode byte '{:x}'", *b);
        state = decode_byte(state, *b, &mut codep);
        println!("state  '{}'", state);
        match state {
            UTF8_ACCEPT => {
                println!("decoded cp = '{:?}'", codep);
                state = 0;
                codep = 0;
            }
            UTF8_REJECT => {
                println!("invalid utf8 sequence, restart");
                state = 0;
                codep = 0;
                state = decode_byte(state, *b, &mut codep);
                match state {
                    UTF8_ACCEPT => {
                        println!("decoded cp = '{:?}'", codep);
                        state = 0;
                        codep = 0;
                    }
                    UTF8_REJECT => {
                        println!("invalid utf8 sequence, restart");
                        state = 0;
                        codep = 0;
                    }
                    _ => {}
                }
            }

            _ => {
                continue;
            }
        }
    }
}

#[test]
fn test_backward_decode() {
    let data: [u8; 24] = [
        0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac,
        0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac,
    ];

    let mut start_offset: u64 = 6;
    println!(
        "start_offset {} byte is '{:x}'",
        start_offset, data[start_offset as usize] as u32
    );

    let off = get_previous_codepoint_start(&data, start_offset as u64);
    println!(
        "start_offset({}) - off({}) = {}",
        start_offset,
        off,
        start_offset - off
    );
    let delta = start_offset - off;
    start_offset -= delta;

    println!("new start_offset({})", start_offset);
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate futures;
extern crate test;

use futures::{Async, Poll, AsyncSink};
use futures::executor;
use futures::executor::{Notify, NotifyHandle};

use futures::sink::Sink;
use futures::stream::Stream;

use futures::sync::mpsc::unbounded;
use futures::sync::mpsc::channel;
use futures::sync::mpsc::Sender;
use futures::sync::mpsc::UnboundedSender;

use test::Bencher;

fn notify_noop() -> NotifyHandle {
    struct Noop;

    impl Notify for Noop {
        fn notify(&self, _id: usize) {}
    }

    const NOOP : &'static Noop = &Noop;

    NotifyHandle::from(NOOP)
}

/// Single producer, single consumer
#[bench]
fn unbounded_1_tx(b: &mut Bencher) {
    b.iter(|| {
        let (tx, rx) = unbounded();

        let mut rx = executor::spawn(rx);

        // 1000 iterations to avoid measuring overhead of initialization
        // Result should be divided by 1000
        for i in 0..1000 {

            // Poll, not ready, park
            assert_eq!(Ok(Async::NotReady), rx.poll_stream_notify(&notify_noop(), 1));

            UnboundedSender::send(&tx, i).unwrap();

            // Now poll ready
            assert_eq!(Ok(Async::Ready(Some(i))), rx.poll_stream_notify(&notify_noop(), 1));
        }
    })
}

/// 100 producers, single consumer
#[bench]
fn unbounded_100_tx(b: &mut Bencher) {
    b.iter(|| {
        let (tx, rx) = unbounded();

        let mut rx = executor::spawn(rx);

        let tx: Vec<_> = (0..100).map(|_| tx.clone()).collect();

        // 1000 send/recv operations total, result should be divided by 1000
        for _ in 0..10 {
            for i in 0..tx.len() {
                assert_eq!(Ok(Async::NotReady), rx.poll_stream_notify(&notify_noop(), 1));

                UnboundedSender::send(&tx[i], i).unwrap();

                assert_eq!(Ok(Async::Ready(Some(i))), rx.poll_stream_notify(&notify_noop(), 1));
            }
        }
    })
}

/// A Stream that continuously sends incrementing number of the queue
struct TestSender {
    tx: Sender<u32>,
    last: u32, // Last number sent
}

// Could be a Future, it doesn't matter
impl Stream for TestSender {
    type Item = u32;
    type Error = ();

    fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
        match self.tx.start_send(self.last + 1) {
            Err(_) => panic!(),
            Ok(AsyncSink::Ready) => {
                self.last += 1;
                assert_eq!(Ok(Async::Ready(())), self.tx.poll_complete());
                Ok(Async::Ready(Some(self.last)))
            }
            Ok(AsyncSink::NotReady(_)) => {
                Ok(Async::NotReady)
            }
        }
    }
}

/// Single producers, single consumer
#[bench]
fn bounded_1_tx(b: &mut Bencher) {
    b.iter(|| {
        let (tx, rx) = channel(0);

        let mut tx = executor::spawn(TestSender {
            tx: tx,
            last: 0,
        });

        let mut rx = executor::spawn(rx);

        for i in 0..1000 {
            assert_eq!(Ok(Async::Ready(Some(i + 1))), tx.poll_stream_notify(&notify_noop(), 1));
            assert_eq!(Ok(Async::NotReady), tx.poll_stream_notify(&notify_noop(), 1));
            assert_eq!(Ok(Async::Ready(Some(i + 1))), rx.poll_stream_notify(&notify_noop(), 1));
        }
    })
}

/// 100 producers, single consumer
#[bench]
fn bounded_100_tx(b: &mut Bencher) {
    b.iter(|| {
        // Each sender can send one item after specified capacity
        let (tx, rx) = channel(0);

        let mut tx: Vec<_> = (0..100).map(|_| {
            executor::spawn(TestSender {
                tx: tx.clone(),
                last: 0
            })
        }).collect();

        let mut rx = executor::spawn(rx);

        for i in 0..10 {
            for j in 0..tx.len() {
                // Send an item
                assert_eq!(Ok(Async::Ready(Some(i + 1))), tx[j].poll_stream_notify(&notify_noop(), 1));
                // Then block
                assert_eq!(Ok(Async::NotReady), tx[j].poll_stream_notify(&notify_noop(), 1));
                // Recv the item
                assert_eq!(Ok(Async::Ready(Some(i + 1))), rx.poll_stream_notify(&notify_noop(), 1));
            }
        }
    })
}
<commit_msg>Benchmark uncontended operation of sync::mpsc<commit_after>#![feature(test)]

extern crate futures;
extern crate test;

use futures::{Async, Poll, AsyncSink};
use futures::executor;
use futures::executor::{Notify, NotifyHandle};

use futures::sink::Sink;
use futures::stream::Stream;

use futures::sync::mpsc::unbounded;
use futures::sync::mpsc::channel;
use futures::sync::mpsc::Sender;
use futures::sync::mpsc::UnboundedSender;

use test::Bencher;

fn notify_noop() -> NotifyHandle {
    struct Noop;

    impl Notify for Noop {
        fn notify(&self, _id: usize) {}
    }

    const NOOP : &'static Noop = &Noop;

    NotifyHandle::from(NOOP)
}

/// Single producer, single consumer
#[bench]
fn unbounded_1_tx(b: &mut Bencher) {
    b.iter(|| {
        let (tx, rx) = unbounded();

        let mut rx = executor::spawn(rx);

        // 1000 iterations to avoid measuring overhead of initialization
        // Result should be divided by 1000
        for i in 0..1000 {

            // Poll, not ready, park
            assert_eq!(Ok(Async::NotReady), rx.poll_stream_notify(&notify_noop(), 1));

            UnboundedSender::send(&tx, i).unwrap();

            // Now poll ready
            assert_eq!(Ok(Async::Ready(Some(i))), rx.poll_stream_notify(&notify_noop(), 1));
        }
    })
}

/// 100 producers, single consumer
#[bench]
fn unbounded_100_tx(b: &mut Bencher) {
    b.iter(|| {
        let (tx, rx) = unbounded();

        let mut rx = executor::spawn(rx);

        let tx: Vec<_> = (0..100).map(|_| tx.clone()).collect();

        // 1000 send/recv operations total, result should be divided by 1000
        for _ in 0..10 {
            for i in 0..tx.len() {
                assert_eq!(Ok(Async::NotReady), rx.poll_stream_notify(&notify_noop(), 1));

                UnboundedSender::send(&tx[i], i).unwrap();

                assert_eq!(Ok(Async::Ready(Some(i))), rx.poll_stream_notify(&notify_noop(), 1));
            }
        }
    })
}

#[bench]
fn unbounded_uncontended(b: &mut Bencher) {
    b.iter(|| {
        let (tx, mut rx) = unbounded();

        for i in 0..1000 {
            UnboundedSender::send(&tx, i).expect("send");
            // No need to create a task, because poll is not going to park.
            assert_eq!(Ok(Async::Ready(Some(i))), rx.poll());
        }
    })
}

/// A Stream that continuously sends incrementing number of the queue
struct TestSender {
    tx: Sender<u32>,
    last: u32, // Last number sent
}

// Could be a Future, it doesn't matter
impl Stream for TestSender {
    type Item = u32;
    type Error = ();

    fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
        match self.tx.start_send(self.last + 1) {
            Err(_) => panic!(),
            Ok(AsyncSink::Ready) => {
                self.last += 1;
                assert_eq!(Ok(Async::Ready(())), self.tx.poll_complete());
                Ok(Async::Ready(Some(self.last)))
            }
            Ok(AsyncSink::NotReady(_)) => {
                Ok(Async::NotReady)
            }
        }
    }
}

/// Single producers, single consumer
#[bench]
fn bounded_1_tx(b: &mut Bencher) {
    b.iter(|| {
        let (tx, rx) = channel(0);

        let mut tx = executor::spawn(TestSender {
            tx: tx,
            last: 0,
        });

        let mut rx = executor::spawn(rx);

        for i in 0..1000 {
            assert_eq!(Ok(Async::Ready(Some(i + 1))), tx.poll_stream_notify(&notify_noop(), 1));
            assert_eq!(Ok(Async::NotReady), tx.poll_stream_notify(&notify_noop(), 1));
            assert_eq!(Ok(Async::Ready(Some(i + 1))), rx.poll_stream_notify(&notify_noop(), 1));
        }
    })
}

/// 100 producers, single consumer
#[bench]
fn bounded_100_tx(b: &mut Bencher) {
    b.iter(|| {
        // Each sender can send one item after specified capacity
        let (tx, rx) = channel(0);

        let mut tx: Vec<_> = (0..100).map(|_| {
            executor::spawn(TestSender {
                tx: tx.clone(),
                last: 0
            })
        }).collect();

        let mut rx = executor::spawn(rx);

        for i in 0..10 {
            for j in 0..tx.len() {
                // Send an item
                assert_eq!(Ok(Async::Ready(Some(i + 1))), tx[j].poll_stream_notify(&notify_noop(), 1));
                // Then block
                assert_eq!(Ok(Async::NotReady), tx[j].poll_stream_notify(&notify_noop(), 1));
                // Recv the item
                assert_eq!(Ok(Async::Ready(Some(i + 1))), rx.poll_stream_notify(&notify_noop(), 1));
            }
        }
    })
}
<|endoftext|>


<commit_before>mod database;
pub(crate) mod s3;

pub(crate) use self::database::DatabaseBackend;
pub(crate) use self::s3::S3Backend;
use failure::Error;
use time::Timespec;

use failure::err_msg;
use postgres::{transaction::Transaction, Connection};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};

#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) struct Blob {
    pub(crate) path: String,
    pub(crate) mime: String,
    pub(crate) date_updated: Timespec,
    pub(crate) content: Vec<u8>,
}

fn get_file_list_from_dir<P: AsRef<Path>>(path: P, files: &mut Vec<PathBuf>) -> Result<(), Error> {
    let path = path.as_ref();

    for file in path.read_dir()? {
        let file = file?;

        if file.file_type()?.is_file() {
            files.push(file.path());
        } else if file.file_type()?.is_dir() {
            get_file_list_from_dir(file.path(), files)?;
        }
    }

    Ok(())
}

pub fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>, Error> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        get_file_list_from_dir(path, &mut files)?;
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub(crate) enum Storage<'a> {
    Database(DatabaseBackend<'a>),
    S3(S3Backend<'a>),
}

impl<'a> Storage<'a> {
    pub(crate) fn new(conn: &'a Connection) -> Self {
        if let Some(c) = s3::s3_client() {
            Storage::from(S3Backend::new(c, s3::S3_BUCKET_NAME))
        } else {
            DatabaseBackend::new(conn).into()
        }
    }
    pub(crate) fn get(&self, path: &str) -> Result<Blob, Error> {
        match self {
            Self::Database(db) => db.get(path),
            Self::S3(s3) => s3.get(path),
        }
    }

    fn store_batch(&mut self, batch: &[Blob], trans: &Transaction) -> Result<(), Error> {
        match self {
            Self::Database(db) => db.store_batch(batch, trans),
            Self::S3(s3) => s3.store_batch(batch),
        }
    }

    // Store all files in `root_dir` into the backend under `prefix`.
    //
    // If the environment is configured with S3 credentials, this will upload to S3;
    // otherwise, this will store files in the database.
    //
    // This returns a HashMap<filename, mime type>.
    pub(crate) fn store_all(
        &mut self,
        conn: &Connection,
        prefix: &str,
        root_dir: &Path,
    ) -> Result<HashMap<PathBuf, String>, Error> {
        const MAX_CONCURRENT_UPLOADS: usize = 1000;

        let trans = conn.transaction()?;
        let mut file_paths_and_mimes = HashMap::new();

        get_file_list(root_dir)?
            .into_iter()
            .filter_map(|file_path| {
                // Some files have insufficient permissions
                // (like .lock file created by cargo in documentation directory).
                // Skip these files.
                fs::File::open(root_dir.join(&file_path))
                    .ok()
                    .map(|file| (file_path, file))
            })
            .map(|(file_path, mut file)| {
                let mut content: Vec<u8> = Vec::new();
                file.read_to_end(&mut content)?;

                let bucket_path = Path::new(prefix).join(&file_path);

                #[cfg(windows)] // On windows, we need to normalize \\ to / so the route logic works
                let bucket_path = path_slash::PathBufExt::to_slash(&bucket_path).unwrap();
                #[cfg(not(windows))]
                let bucket_path = bucket_path.into_os_string().into_string().unwrap();

                let mime = detect_mime(&file_path)?;

                file_paths_and_mimes.insert(file_path, mime.to_string());
                Ok(Blob {
                    path: bucket_path,
                    mime: mime.to_string(),
                    content,
                    // this field is ignored by the backend
                    date_updated: Timespec::new(0, 0),
                })
            })
            .collect::<Result<Vec<_>, Error>>()?
            .chunks(MAX_CONCURRENT_UPLOADS)
            .try_for_each(|batch| self.store_batch(batch, &trans))?;

        trans.commit()?;
        Ok(file_paths_and_mimes)
    }
}

fn detect_mime(file_path: &Path) -> Result<&'static str, Error> {
    let mime = mime_guess::from_path(file_path)
        .first_raw()
        .map(|m| m)
        .unwrap_or("text/plain");
    Ok(match mime {
        "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
            match file_path.extension().and_then(OsStr::to_str) {
                Some("md") => "text/markdown",
                Some("rs") => "text/rust",
                Some("markdown") => "text/markdown",
                Some("css") => "text/css",
                Some("toml") => "text/toml",
                Some("js") => "application/javascript",
                Some("json") => "application/json",
                _ => mime,
            }
        }
        "image/svg" => "image/svg+xml",
        _ => mime,
    })
}

impl<'a> From<DatabaseBackend<'a>> for Storage<'a> {
    fn from(db: DatabaseBackend<'a>) -> Self {
        Self::Database(db)
    }
}

impl<'a> From<S3Backend<'a>> for Storage<'a> {
    fn from(db: S3Backend<'a>) -> Self {
        Self::S3(db)
    }
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use super::*;
    use crate::test::wrapper;
    use std::env;

    #[test]
    fn test_uploads() {
        use std::fs;
        let dir = tempdir::TempDir::new("docs.rs-upload-test").unwrap();
        let files = ["Cargo.toml", "src/main.rs"];
        for &file in &files {
            let path = dir.path().join(file);
            if let Some(parent) = path.parent() {
                fs::create_dir_all(parent).unwrap();
            }
            fs::write(path, "data").expect("failed to write to file");
        }
        wrapper(|env| {
            let db = env.db();
            let conn = db.conn();
            let mut backend = Storage::Database(DatabaseBackend::new(&conn));
            let stored_files = backend.store_all(&conn, "rustdoc", dir.path()).unwrap();
            assert_eq!(stored_files.len(), files.len());
            for name in &files {
                let name = Path::new(name);
                assert!(stored_files.contains_key(name));
            }
            assert_eq!(
                stored_files.get(Path::new("Cargo.toml")).unwrap(),
                "text/toml"
            );
            assert_eq!(
                stored_files.get(Path::new("src/main.rs")).unwrap(),
                "text/rust"
            );

            let file = backend.get("rustdoc/Cargo.toml").unwrap();
            assert_eq!(file.content, b"data");
            assert_eq!(file.mime, "text/toml");
            assert_eq!(file.path, "rustdoc/Cargo.toml");

            let file = backend.get("rustdoc/src/main.rs").unwrap();
            assert_eq!(file.content, b"data");
            assert_eq!(file.mime, "text/rust");
            assert_eq!(file.path, "rustdoc/src/main.rs");
            Ok(())
        })
    }

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
    #[test]
    fn test_mime_types() {
        check_mime(".gitignore", "text/plain");
        check_mime("hello.toml", "text/toml");
        check_mime("hello.css", "text/css");
        check_mime("hello.js", "application/javascript");
        check_mime("hello.html", "text/html");
        check_mime("hello.hello.md", "text/markdown");
        check_mime("hello.markdown", "text/markdown");
        check_mime("hello.json", "application/json");
        check_mime("hello.txt", "text/plain");
        check_mime("file.rs", "text/rust");
        check_mime("important.svg", "image/svg+xml");
    }

    fn check_mime(path: &str, expected_mime: &str) {
        let detected_mime = detect_mime(Path::new(&path));
        let detected_mime = detected_mime.expect("no mime was given");
        assert_eq!(detected_mime, expected_mime);
    }
}
<commit_msg>Don't read all files into memory at once<commit_after>mod database;
pub(crate) mod s3;

pub(crate) use self::database::DatabaseBackend;
pub(crate) use self::s3::S3Backend;
use failure::Error;
use time::Timespec;

use failure::err_msg;
use postgres::{transaction::Transaction, Connection};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};

#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) struct Blob {
    pub(crate) path: String,
    pub(crate) mime: String,
    pub(crate) date_updated: Timespec,
    pub(crate) content: Vec<u8>,
}

fn get_file_list_from_dir<P: AsRef<Path>>(path: P, files: &mut Vec<PathBuf>) -> Result<(), Error> {
    let path = path.as_ref();

    for file in path.read_dir()? {
        let file = file?;

        if file.file_type()?.is_file() {
            files.push(file.path());
        } else if file.file_type()?.is_dir() {
            get_file_list_from_dir(file.path(), files)?;
        }
    }

    Ok(())
}

pub fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>, Error> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        get_file_list_from_dir(path, &mut files)?;
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub(crate) enum Storage<'a> {
    Database(DatabaseBackend<'a>),
    S3(S3Backend<'a>),
}

impl<'a> Storage<'a> {
    pub(crate) fn new(conn: &'a Connection) -> Self {
        if let Some(c) = s3::s3_client() {
            Storage::from(S3Backend::new(c, s3::S3_BUCKET_NAME))
        } else {
            DatabaseBackend::new(conn).into()
        }
    }
    pub(crate) fn get(&self, path: &str) -> Result<Blob, Error> {
        match self {
            Self::Database(db) => db.get(path),
            Self::S3(s3) => s3.get(path),
        }
    }

    fn store_batch(&mut self, batch: &[Blob], trans: &Transaction) -> Result<(), Error> {
        match self {
            Self::Database(db) => db.store_batch(batch, trans),
            Self::S3(s3) => s3.store_batch(batch),
        }
    }

    // Store all files in `root_dir` into the backend under `prefix`.
    //
    // If the environment is configured with S3 credentials, this will upload to S3;
    // otherwise, this will store files in the database.
    //
    // This returns a HashMap<filename, mime type>.
    pub(crate) fn store_all(
        &mut self,
        conn: &Connection,
        prefix: &str,
        root_dir: &Path,
    ) -> Result<HashMap<PathBuf, String>, Error> {
        const MAX_CONCURRENT_UPLOADS: usize = 1000;

        let trans = conn.transaction()?;
        let mut file_paths_and_mimes = HashMap::new();

        let mut blobs = get_file_list(root_dir)?
            .into_iter()
            .filter_map(|file_path| {
                // Some files have insufficient permissions
                // (like .lock file created by cargo in documentation directory).
                // Skip these files.
                fs::File::open(root_dir.join(&file_path))
                    .ok()
                    .map(|file| (file_path, file))
            })
            .map(|(file_path, mut file)| -> Result<_, Error> {
                let mut content: Vec<u8> = Vec::new();
                file.read_to_end(&mut content)?;

                let bucket_path = Path::new(prefix).join(&file_path);

                #[cfg(windows)] // On windows, we need to normalize \\ to / so the route logic works
                let bucket_path = path_slash::PathBufExt::to_slash(&bucket_path).unwrap();
                #[cfg(not(windows))]
                let bucket_path = bucket_path.into_os_string().into_string().unwrap();

                let mime = detect_mime(&file_path)?;

                file_paths_and_mimes.insert(file_path, mime.to_string());
                Ok(Blob {
                    path: bucket_path,
                    mime: mime.to_string(),
                    content,
                    // this field is ignored by the backend
                    date_updated: Timespec::new(0, 0),
                })
            });
        loop {
            let batch: Vec<_> = blobs
                .by_ref()
                .take(MAX_CONCURRENT_UPLOADS)
                .collect::<Result<_, Error>>()?;
            if batch.is_empty() {
                break;
            }
            self.store_batch(&batch, &trans)?;
        }

        trans.commit()?;
        Ok(file_paths_and_mimes)
    }
}

fn detect_mime(file_path: &Path) -> Result<&'static str, Error> {
    let mime = mime_guess::from_path(file_path)
        .first_raw()
        .map(|m| m)
        .unwrap_or("text/plain");
    Ok(match mime {
        "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
            match file_path.extension().and_then(OsStr::to_str) {
                Some("md") => "text/markdown",
                Some("rs") => "text/rust",
                Some("markdown") => "text/markdown",
                Some("css") => "text/css",
                Some("toml") => "text/toml",
                Some("js") => "application/javascript",
                Some("json") => "application/json",
                _ => mime,
            }
        }
        "image/svg" => "image/svg+xml",
        _ => mime,
    })
}

impl<'a> From<DatabaseBackend<'a>> for Storage<'a> {
    fn from(db: DatabaseBackend<'a>) -> Self {
        Self::Database(db)
    }
}

impl<'a> From<S3Backend<'a>> for Storage<'a> {
    fn from(db: S3Backend<'a>) -> Self {
        Self::S3(db)
    }
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use super::*;
    use crate::test::wrapper;
    use std::env;

    #[test]
    fn test_uploads() {
        use std::fs;
        let dir = tempdir::TempDir::new("docs.rs-upload-test").unwrap();
        let files = ["Cargo.toml", "src/main.rs"];
        for &file in &files {
            let path = dir.path().join(file);
            if let Some(parent) = path.parent() {
                fs::create_dir_all(parent).unwrap();
            }
            fs::write(path, "data").expect("failed to write to file");
        }
        wrapper(|env| {
            let db = env.db();
            let conn = db.conn();
            let mut backend = Storage::Database(DatabaseBackend::new(&conn));
            let stored_files = backend.store_all(&conn, "rustdoc", dir.path()).unwrap();
            assert_eq!(stored_files.len(), files.len());
            for name in &files {
                let name = Path::new(name);
                assert!(stored_files.contains_key(name));
            }
            assert_eq!(
                stored_files.get(Path::new("Cargo.toml")).unwrap(),
                "text/toml"
            );
            assert_eq!(
                stored_files.get(Path::new("src/main.rs")).unwrap(),
                "text/rust"
            );

            let file = backend.get("rustdoc/Cargo.toml").unwrap();
            assert_eq!(file.content, b"data");
            assert_eq!(file.mime, "text/toml");
            assert_eq!(file.path, "rustdoc/Cargo.toml");

            let file = backend.get("rustdoc/src/main.rs").unwrap();
            assert_eq!(file.content, b"data");
            assert_eq!(file.mime, "text/rust");
            assert_eq!(file.path, "rustdoc/src/main.rs");
            Ok(())
        })
    }

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
    #[test]
    fn test_mime_types() {
        check_mime(".gitignore", "text/plain");
        check_mime("hello.toml", "text/toml");
        check_mime("hello.css", "text/css");
        check_mime("hello.js", "application/javascript");
        check_mime("hello.html", "text/html");
        check_mime("hello.hello.md", "text/markdown");
        check_mime("hello.markdown", "text/markdown");
        check_mime("hello.json", "application/json");
        check_mime("hello.txt", "text/plain");
        check_mime("file.rs", "text/rust");
        check_mime("important.svg", "image/svg+xml");
    }

    fn check_mime(path: &str, expected_mime: &str) {
        let detected_mime = detect_mime(Path::new(&path));
        let detected_mime = detected_mime.expect("no mime was given");
        assert_eq!(detected_mime, expected_mime);
    }
}
<|endoftext|>


<commit_before>//! Simple integration tests oriented towards gravity computations

#![feature(core)]

extern crate acacia;
extern crate nalgebra;
extern crate quickcheck;

use nalgebra::{ApproxEq, Pnt2, Pnt3, FloatPnt, Vec2, Vec3, zero, Norm, Orig};
use quickcheck::{TestResult, quickcheck};
use acacia::{Tree, Node, AssociatedData, DataQuery, Positioned};
use acacia::partition::Ncube;

#[test]
fn tree_center_of_mass() {
    fn tree_center_of_mass(data: Vec<(f64, (f64, f64))>) -> TestResult {
        // Only test non-empty lists with positive masses
        if data.is_empty() || data.iter().any(|&(m, _)| m <= 0.0) {
            return TestResult::discard();
        }
        // No two points should be in the same place
        for i in 0..data.len() {
            for j in 0..i {
                let (_, pi) = data[i];
                let (_, pj) = data[j];
                if pi == pj {
                    return TestResult::discard();
                }
            }
        }
        // Compute center of mass in the traditional way
        let (mps, ms) = data.iter()
            .map(|&(m, (x, y))| (Vec2::new(x, y) * m, m))
            .fold((zero::<Vec2<f64>>(), 0.0f64), |(mps, ms), (mp, m)| (mps + mp, ms + m));
        let com = mps / ms;
        // Now use the tree
        let tree = Tree::new(
            data.iter().map(|&(m, (x, y))|
                Positioned { object: m, position: Pnt2::new(x, y) }
            ),
            Ncube::new(Orig::orig(), 200.0f64),
            (zero(), 0.0),
            &|obj| (obj.position.to_vec() * obj.object, obj.object),
            &|&(mps, ms), &(mp, m)| (mps + mp, ms + m)
        );
        let (tree_mps, tree_ms) = *tree.data();
        // …and compare
        TestResult::from_bool(ApproxEq::approx_eq(&(tree_mps / tree_ms), &com))
    }
    quickcheck(tree_center_of_mass as fn(Vec<(f64, (f64, f64))>) -> TestResult);
}

#[test]
fn tree_gravity_approx() {
    fn tree_gravity_approx(
            starfield: Vec<(f64, (f64, f64, f64))>,
            test_point: (f64, f64, f64)
        ) -> TestResult
    {
        // We want to have at least one star
        if starfield.is_empty() {
            return TestResult::discard();
        }
        // Only test positive masses
        if starfield.iter().any(|&(m, _)| m <= 0.0) {
            return TestResult::discard();
        }
        // The test point should not be in the same place as any star
        if starfield.iter().any(|&(_, p)| p == test_point) {
            return TestResult::discard();
        }
        // No two stars should be in the same place
        for i in 0..starfield.len() {
            for j in 0..i {
                let (_, pi) = starfield[i];
                let (_, pj) = starfield[j];
                if pi == pj {
                    return TestResult::discard();
                }
            }
        }
        // (T, T, T) -> Pnt3<T>
        fn pnt<T>(p: (T, T, T)) -> Pnt3<T> {
            let (x, y, z) = p;
            Pnt3::new(x, y, z)
        }
        let test_point = pnt(test_point);
        // Newton's law of gravity for two point masses (with G = 1)
        let newton = |(m, p1): (f64, Pnt3<f64>), p2| {
            let diff: Vec3<f64> = p1 - p2;
            let r = diff.norm();
            diff * (m / r.powi(3))
        };
        // Calculate gravity exactly
        let simple_gravity = starfield.iter()
            .map(|&(m, p)| newton((m, pnt(p)), test_point))
            .fold(zero(), |a: Vec3<_>, b| a + b);
        // Calculate gravity using a tree
        let orig: Pnt3<f64> = Orig::orig();
        let data_width = test_point.as_vec().norm() * 2.0;
        let width = if data_width < 200.0 { 200.0 } else { data_width };
        let tree = Tree::new(
            starfield.iter().map(|&(m, (x, y, z))|
                Positioned { object: m, position: Pnt3::new(x, y, z) }
            ),
            Ncube::new(orig, width),
            (orig, zero()),
            &|obj| (obj.position, obj.object),
            &|&(com1, m1), &(com2, m2)|
                if m1 + m2 > zero() {(
                    orig + (com1.to_vec() * m1 + com2.to_vec() * m2) / (m1 + m2),
                    m1 + m2,
                )}
                else {
                    (orig, zero())
                }
        );
        let theta = 0.5; // A bit arbitrary but this appears to work
        let tree_gravity =
            tree.query_data(|node| {
                let &(ref center_of_mass, _) = node.data();
                let d = FloatPnt::dist(&test_point, center_of_mass);
                let delta = FloatPnt::dist(&node.partition().center(), center_of_mass);
                d < node.partition().width() / theta + delta
            })
            .map(|&(com, m)| newton((m, com), test_point))
            .fold(zero::<Vec3<f64>>(), |a, b| a + b);

        // Now the tree gravity should approximate the exact one, within 10 %
        TestResult::from_bool(simple_gravity.approx_eq_eps(&tree_gravity, &(0.1 * simple_gravity.norm())))
    }
    quickcheck(tree_gravity_approx as fn(Vec<(f64, (f64, f64, f64))>, (f64, f64, f64)) -> TestResult)
}
<commit_msg>Do not use superfluous feature "core"<commit_after>//! Simple integration tests oriented towards gravity computations

extern crate acacia;
extern crate nalgebra;
extern crate quickcheck;

use nalgebra::{ApproxEq, Pnt2, Pnt3, FloatPnt, Vec2, Vec3, zero, Norm, Orig};
use quickcheck::{TestResult, quickcheck};
use acacia::{Tree, Node, AssociatedData, DataQuery, Positioned};
use acacia::partition::Ncube;

#[test]
fn tree_center_of_mass() {
    fn tree_center_of_mass(data: Vec<(f64, (f64, f64))>) -> TestResult {
        // Only test non-empty lists with positive masses
        if data.is_empty() || data.iter().any(|&(m, _)| m <= 0.0) {
            return TestResult::discard();
        }
        // No two points should be in the same place
        for i in 0..data.len() {
            for j in 0..i {
                let (_, pi) = data[i];
                let (_, pj) = data[j];
                if pi == pj {
                    return TestResult::discard();
                }
            }
        }
        // Compute center of mass in the traditional way
        let (mps, ms) = data.iter()
            .map(|&(m, (x, y))| (Vec2::new(x, y) * m, m))
            .fold((zero::<Vec2<f64>>(), 0.0f64), |(mps, ms), (mp, m)| (mps + mp, ms + m));
        let com = mps / ms;
        // Now use the tree
        let tree = Tree::new(
            data.iter().map(|&(m, (x, y))|
                Positioned { object: m, position: Pnt2::new(x, y) }
            ),
            Ncube::new(Orig::orig(), 200.0f64),
            (zero(), 0.0),
            &|obj| (obj.position.to_vec() * obj.object, obj.object),
            &|&(mps, ms), &(mp, m)| (mps + mp, ms + m)
        );
        let (tree_mps, tree_ms) = *tree.data();
        // …and compare
        TestResult::from_bool(ApproxEq::approx_eq(&(tree_mps / tree_ms), &com))
    }
    quickcheck(tree_center_of_mass as fn(Vec<(f64, (f64, f64))>) -> TestResult);
}

#[test]
fn tree_gravity_approx() {
    fn tree_gravity_approx(
            starfield: Vec<(f64, (f64, f64, f64))>,
            test_point: (f64, f64, f64)
        ) -> TestResult
    {
        // We want to have at least one star
        if starfield.is_empty() {
            return TestResult::discard();
        }
        // Only test positive masses
        if starfield.iter().any(|&(m, _)| m <= 0.0) {
            return TestResult::discard();
        }
        // The test point should not be in the same place as any star
        if starfield.iter().any(|&(_, p)| p == test_point) {
            return TestResult::discard();
        }
        // No two stars should be in the same place
        for i in 0..starfield.len() {
            for j in 0..i {
                let (_, pi) = starfield[i];
                let (_, pj) = starfield[j];
                if pi == pj {
                    return TestResult::discard();
                }
            }
        }
        // (T, T, T) -> Pnt3<T>
        fn pnt<T>(p: (T, T, T)) -> Pnt3<T> {
            let (x, y, z) = p;
            Pnt3::new(x, y, z)
        }
        let test_point = pnt(test_point);
        // Newton's law of gravity for two point masses (with G = 1)
        let newton = |(m, p1): (f64, Pnt3<f64>), p2| {
            let diff: Vec3<f64> = p1 - p2;
            let r = diff.norm();
            diff * (m / r.powi(3))
        };
        // Calculate gravity exactly
        let simple_gravity = starfield.iter()
            .map(|&(m, p)| newton((m, pnt(p)), test_point))
            .fold(zero(), |a: Vec3<_>, b| a + b);
        // Calculate gravity using a tree
        let orig: Pnt3<f64> = Orig::orig();
        let data_width = test_point.as_vec().norm() * 2.0;
        let width = if data_width < 200.0 { 200.0 } else { data_width };
        let tree = Tree::new(
            starfield.iter().map(|&(m, (x, y, z))|
                Positioned { object: m, position: Pnt3::new(x, y, z) }
            ),
            Ncube::new(orig, width),
            (orig, zero()),
            &|obj| (obj.position, obj.object),
            &|&(com1, m1), &(com2, m2)|
                if m1 + m2 > zero() {(
                    orig + (com1.to_vec() * m1 + com2.to_vec() * m2) / (m1 + m2),
                    m1 + m2,
                )}
                else {
                    (orig, zero())
                }
        );
        let theta = 0.5; // A bit arbitrary but this appears to work
        let tree_gravity =
            tree.query_data(|node| {
                let &(ref center_of_mass, _) = node.data();
                let d = FloatPnt::dist(&test_point, center_of_mass);
                let delta = FloatPnt::dist(&node.partition().center(), center_of_mass);
                d < node.partition().width() / theta + delta
            })
            .map(|&(com, m)| newton((m, com), test_point))
            .fold(zero::<Vec3<f64>>(), |a, b| a + b);

        // Now the tree gravity should approximate the exact one, within 10 %
        TestResult::from_bool(simple_gravity.approx_eq_eps(&tree_gravity, &(0.1 * simple_gravity.norm())))
    }
    quickcheck(tree_gravity_approx as fn(Vec<(f64, (f64, f64, f64))>, (f64, f64, f64)) -> TestResult)
}
<|endoftext|>