<commit_before>pub trait XmlStr {
    fn end_of_attribute(&self, quote: &str) -> Option<uint>;
    fn end_of_literal(&self, expected: &str) -> Option<uint>;
    fn end_of_version_num(&self) -> Option<uint>;
    fn end_of_char_data(&self) -> Option<uint>;
    fn end_of_cdata(&self) -> Option<uint>;
    fn end_of_decimal_chars(&self) -> Option<uint>;
    fn end_of_hex_chars(&self) -> Option<uint>;
    fn end_of_comment(&self) -> Option<uint>;
    fn end_of_pi_value(&self) -> Option<uint>;
    fn end_of_start_rest(&self, is_first: |char| -> bool, is_rest: |char| -> bool) -> Option<uint>;
    fn end_of_name(&self) -> Option<uint>;
    fn end_of_space(&self) -> Option<uint>;
    fn end_of_start_tag(&self) -> Option<uint>;
}

impl<'a> XmlStr for &'a str {
    fn end_of_attribute(&self, quote: &str) -> Option<uint> {
        if self.len() == 0 ||
           self.starts_with("&") ||
           self.starts_with("<") ||
           self.starts_with(quote)
        {
            return None;
        }

        let (quote_char, _) = quote.slice_shift_char().expect("Cant have null quote");

        let mut positions = self.char_indices().skip_while(|&(_, c)| c != '&' && c != '<' && c != quote_char);

        match positions.next() {
            Some((offset, _)) => Some(offset),
            None => Some(self.len())
        }
    }

    fn end_of_literal(&self, expected: &str) -> Option<uint> {
        if self.starts_with(expected) {
            Some(expected.len())
        } else {
            None
        }
    }

    fn end_of_version_num(&self) -> Option<uint> {
        if self.starts_with("1.") {
            let mut positions = self.char_indices().peekable();
            positions.next();
            positions.next();

            // Need at least one character
            match positions.peek() {
                Some(&(_, c)) if c.is_decimal_char() => {},
                _ => return None,
            };

            let mut positions = positions.skip_while(|&(_, c)| c.is_decimal_char());
            match positions.next() {
                Some((offset, _)) => Some(offset),
                None => Some(self.len()),
            }
        } else {
            None
        }
    }

    fn end_of_char_data(&self) -> Option<uint> {
        if self.starts_with("<") ||
           self.starts_with("&") ||
           self.starts_with("]]>")
        {
            return None
        }

        // Using a hex literal because emacs' rust-mode doesn't
        // understand ] in a char literal. :-(
        let mut positions = self.char_indices().skip_while(|&(_, c)| c != '<' && c != '&' && c != '\x5d');

        loop {
            match positions.next() {
                None => return Some(self.len()),
                Some((offset, c)) if c == '<' || c == '&' => return Some(offset),
                Some((offset, _)) => {
                    let tail = self.slice_from(offset);
                    if tail.starts_with("]]>") {
                        return Some(offset)
                    } else {
                        // False alarm, resume scanning
                        continue;
                    }
                },
            }
        }
    }

    fn end_of_cdata(&self) -> Option<uint> {
        match self.find_str("]]>") {
            None => None,
            Some(offset) => Some(offset),
        }
    }

    fn end_of_decimal_chars(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_decimal_char(),
                               |c| c.is_decimal_char())
    }

    fn end_of_hex_chars(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_hex_char(),
                               |c| c.is_hex_char())
    }

    fn end_of_comment(&self) -> Option<uint> {
        // This deliberately does not include the >. -- is not allowed
        // in a comment, so we can just test the end if it matches the
        // complete close delimiter.
        match self.find_str("--") {
            None => None,
            Some(offset) => Some(offset),
        }
    }

    fn end_of_pi_value(&self) -> Option<uint> {
        match self.find_str("?>") {
            None => None,
            Some(offset) => Some(offset),
        }
    }

    fn end_of_start_rest(&self,
                         is_first: |char| -> bool,
                         is_rest: |char| -> bool)
                         -> Option<uint>
    {
        let mut positions = self.char_indices();

        match positions.next() {
            Some((_, c)) if is_first(c) => (),
            Some((_, _)) => return None,
            None => return None,
        };

        let mut positions = positions.skip_while(|&(_, c)| is_rest(c));
        match positions.next() {
            Some((offset, _)) => Some(offset),
            None => Some(self.len()),
        }
    }

    fn end_of_name(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_name_start_char(), |c| c.is_name_char())
    }

    fn end_of_space(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_space_char(), |c| c.is_space_char())
    }

    fn end_of_start_tag(&self) -> Option<uint> {
        let mut positions = self.char_indices();

        match positions.next() {
            Some((_, c)) if '<' == c => (),
            _ => return None,
        };

        match positions.next() {
            Some((offset, c)) =>
                match c {
                    '?' | '!' | '/' => None,
                    _ => Some(offset),
                },
            None => Some(self.len()),
        }
    }
}

trait XmlChar {
    fn is_name_start_char(&self) -> bool;
    fn is_name_char(&self) -> bool;
    fn is_space_char(&self) -> bool;
    fn is_decimal_char(&self) -> bool;
    fn is_hex_char(&self) -> bool;
}

impl XmlChar for char {
    fn is_name_start_char(&self) -> bool {
        match *self {
            ':'                         |
            'A'...'Z'                   |
            '_'                         |
            'a'...'z'                   |
            '\U000000C0'...'\U000000D6' |
            '\U000000D8'...'\U000000F6' |
            '\U000000F8'...'\U000002FF' |
            '\U00000370'...'\U0000037D' |
            '\U0000037F'...'\U00001FFF' |
            '\U0000200C'...'\U0000200D' |
            '\U00002070'...'\U0000218F' |
            '\U00002C00'...'\U00002FEF' |
            '\U00003001'...'\U0000D7FF' |
            '\U0000F900'...'\U0000FDCF' |
            '\U0000FDF0'...'\U0000FFFD' |
            '\U00010000'...'\U000EFFFF' => true,
            _ => false,
        }
    }

    fn is_name_char(&self) -> bool {
        if self.is_name_start_char() { return true; }
        match *self {
            '-'                 |
            '.'                 |
            '0'...'9'           |
            '\u00B7'            |
            '\u0300'...'\u036F' |
            '\u203F'...'\u2040' => true,
            _ => false
        }
    }

    fn is_space_char(&self) -> bool {
        match *self {
            '\x20' |
            '\x09' |
            '\x0D' |
            '\x0A' => true,
            _ => false,
        }
    }

    fn is_decimal_char(&self) -> bool {
        match *self {
            '0'...'9' => true,
            _ => false,
        }
    }

    fn is_hex_char(&self) -> bool {
        match *self {
            '0'...'9' |
            'a'...'f' |
            'A'...'F' => true,
            _ => false,
        }
    }
}

#[cfg(test)]
mod test {

use super::XmlStr;

#[test]
fn end_of_char_data_leading_ampersand() {
    assert_eq!("&".end_of_char_data(), None);
}

#[test]
fn end_of_char_data_leading_less_than() {
    assert_eq!("<".end_of_char_data(), None);
}

#[test]
fn end_of_char_data_leading_cdata_end() {
    assert_eq!("]]>".end_of_char_data(), None);
}

#[test]
fn end_of_char_data_until_ampersand() {
    assert_eq!("hello&world".end_of_char_data(), Some("hello".len()));
}

#[test]
fn end_of_char_data_until_less_than() {
    assert_eq!("hello<world".end_of_char_data(), Some("hello".len()));
}

#[test]
fn end_of_char_data_until_cdata_end() {
    assert_eq!("hello]]>world".end_of_char_data(), Some("hello".len()));
}

#[test]
fn end_of_char_data_includes_right_square() {
    assert_eq!("hello]world".end_of_char_data(), Some("hello]world".len()));
}

}
<commit_msg>Update for new Unicode character literal syntax<commit_after>pub trait XmlStr {
    fn end_of_attribute(&self, quote: &str) -> Option<uint>;
    fn end_of_literal(&self, expected: &str) -> Option<uint>;
    fn end_of_version_num(&self) -> Option<uint>;
    fn end_of_char_data(&self) -> Option<uint>;
    fn end_of_cdata(&self) -> Option<uint>;
    fn end_of_decimal_chars(&self) -> Option<uint>;
    fn end_of_hex_chars(&self) -> Option<uint>;
    fn end_of_comment(&self) -> Option<uint>;
    fn end_of_pi_value(&self) -> Option<uint>;
    fn end_of_start_rest(&self, is_first: |char| -> bool, is_rest: |char| -> bool) -> Option<uint>;
    fn end_of_name(&self) -> Option<uint>;
    fn end_of_space(&self) -> Option<uint>;
    fn end_of_start_tag(&self) -> Option<uint>;
}

impl<'a> XmlStr for &'a str {
    fn end_of_attribute(&self, quote: &str) -> Option<uint> {
        if self.len() == 0 ||
           self.starts_with("&") ||
           self.starts_with("<") ||
           self.starts_with(quote)
        {
            return None;
        }

        let (quote_char, _) = quote.slice_shift_char().expect("Cant have null quote");

        let mut positions = self.char_indices().skip_while(|&(_, c)| c != '&' && c != '<' && c != quote_char);

        match positions.next() {
            Some((offset, _)) => Some(offset),
            None => Some(self.len())
        }
    }

    fn end_of_literal(&self, expected: &str) -> Option<uint> {
        if self.starts_with(expected) {
            Some(expected.len())
        } else {
            None
        }
    }

    fn end_of_version_num(&self) -> Option<uint> {
        if self.starts_with("1.") {
            let mut positions = self.char_indices().peekable();
            positions.next();
            positions.next();

            // Need at least one character
            match positions.peek() {
                Some(&(_, c)) if c.is_decimal_char() => {},
                _ => return None,
            };

            let mut positions = positions.skip_while(|&(_, c)| c.is_decimal_char());
            match positions.next() {
                Some((offset, _)) => Some(offset),
                None => Some(self.len()),
            }
        } else {
            None
        }
    }

    fn end_of_char_data(&self) -> Option<uint> {
        if self.starts_with("<") ||
           self.starts_with("&") ||
           self.starts_with("]]>")
        {
            return None
        }

        // Using a hex literal because emacs' rust-mode doesn't
        // understand ] in a char literal. :-(
        let mut positions = self.char_indices().skip_while(|&(_, c)| c != '<' && c != '&' && c != '\x5d');

        loop {
            match positions.next() {
                None => return Some(self.len()),
                Some((offset, c)) if c == '<' || c == '&' => return Some(offset),
                Some((offset, _)) => {
                    let tail = self.slice_from(offset);
                    if tail.starts_with("]]>") {
                        return Some(offset)
                    } else {
                        // False alarm, resume scanning
                        continue;
                    }
                },
            }
        }
    }

    fn end_of_cdata(&self) -> Option<uint> {
        match self.find_str("]]>") {
            None => None,
            Some(offset) => Some(offset),
        }
    }

    fn end_of_decimal_chars(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_decimal_char(),
                               |c| c.is_decimal_char())
    }

    fn end_of_hex_chars(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_hex_char(),
                               |c| c.is_hex_char())
    }

    fn end_of_comment(&self) -> Option<uint> {
        // This deliberately does not include the >. -- is not allowed
        // in a comment, so we can just test the end if it matches the
        // complete close delimiter.
        match self.find_str("--") {
            None => None,
            Some(offset) => Some(offset),
        }
    }

    fn end_of_pi_value(&self) -> Option<uint> {
        match self.find_str("?>") {
            None => None,
            Some(offset) => Some(offset),
        }
    }

    fn end_of_start_rest(&self,
                         is_first: |char| -> bool,
                         is_rest: |char| -> bool)
                         -> Option<uint>
    {
        let mut positions = self.char_indices();

        match positions.next() {
            Some((_, c)) if is_first(c) => (),
            Some((_, _)) => return None,
            None => return None,
        };

        let mut positions = positions.skip_while(|&(_, c)| is_rest(c));
        match positions.next() {
            Some((offset, _)) => Some(offset),
            None => Some(self.len()),
        }
    }

    fn end_of_name(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_name_start_char(), |c| c.is_name_char())
    }

    fn end_of_space(&self) -> Option<uint> {
        self.end_of_start_rest(|c| c.is_space_char(), |c| c.is_space_char())
    }

    fn end_of_start_tag(&self) -> Option<uint> {
        let mut positions = self.char_indices();

        match positions.next() {
            Some((_, c)) if '<' == c => (),
            _ => return None,
        };

        match positions.next() {
            Some((offset, c)) =>
                match c {
                    '?' | '!' | '/' => None,
                    _ => Some(offset),
                },
            None => Some(self.len()),
        }
    }
}

trait XmlChar {
    fn is_name_start_char(&self) -> bool;
    fn is_name_char(&self) -> bool;
    fn is_space_char(&self) -> bool;
    fn is_decimal_char(&self) -> bool;
    fn is_hex_char(&self) -> bool;
}

impl XmlChar for char {
    fn is_name_start_char(&self) -> bool {
        match *self {
            ':'                         |
            'A'...'Z'                   |
            '_'                         |
            'a'...'z'                   |
            '\u{0000C0}'...'\u{0000D6}' |
            '\u{0000D8}'...'\u{0000F6}' |
            '\u{0000F8}'...'\u{0002FF}' |
            '\u{000370}'...'\u{00037D}' |
            '\u{00037F}'...'\u{001FFF}' |
            '\u{00200C}'...'\u{00200D}' |
            '\u{002070}'...'\u{00218F}' |
            '\u{002C00}'...'\u{002FEF}' |
            '\u{003001}'...'\u{00D7FF}' |
            '\u{00F900}'...'\u{00FDCF}' |
            '\u{00FDF0}'...'\u{00FFFD}' |
            '\u{010000}'...'\u{0EFFFF}' => true,
            _ => false,
        }
    }

    fn is_name_char(&self) -> bool {
        if self.is_name_start_char() { return true; }
        match *self {
            '-'                     |
            '.'                     |
            '0'...'9'               |
            '\u{00B7}'              |
            '\u{0300}'...'\u{036F}' |
            '\u{203F}'...'\u{2040}' => true,
            _ => false
        }
    }

    fn is_space_char(&self) -> bool {
        match *self {
            '\x20' |
            '\x09' |
            '\x0D' |
            '\x0A' => true,
            _ => false,
        }
    }

    fn is_decimal_char(&self) -> bool {
        match *self {
            '0'...'9' => true,
            _ => false,
        }
    }

    fn is_hex_char(&self) -> bool {
        match *self {
            '0'...'9' |
            'a'...'f' |
            'A'...'F' => true,
            _ => false,
        }
    }
}

#[cfg(test)]
mod test {

use super::XmlStr;

#[test]
fn end_of_char_data_leading_ampersand() {
    assert_eq!("&".end_of_char_data(), None);
}

#[test]
fn end_of_char_data_leading_less_than() {
    assert_eq!("<".end_of_char_data(), None);
}

#[test]
fn end_of_char_data_leading_cdata_end() {
    assert_eq!("]]>".end_of_char_data(), None);
}

#[test]
fn end_of_char_data_until_ampersand() {
    assert_eq!("hello&world".end_of_char_data(), Some("hello".len()));
}

#[test]
fn end_of_char_data_until_less_than() {
    assert_eq!("hello<world".end_of_char_data(), Some("hello".len()));
}

#[test]
fn end_of_char_data_until_cdata_end() {
    assert_eq!("hello]]>world".end_of_char_data(), Some("hello".len()));
}

#[test]
fn end_of_char_data_includes_right_square() {
    assert_eq!("hello]world".end_of_char_data(), Some("hello]world".len()));
}

}
<|endoftext|>


<commit_before>use std::old_io::{File, Open, Read, Write};
use std::old_io::IoError;
use std::old_io::IoResult;
use super::GapBuffer;
use super::gap_buffer;
use super::Position;
use super::Range;

pub struct Buffer {
    data: GapBuffer,
    pub path: Option<Path>,
    pub cursor: Position,
}

impl Buffer {
    /// Returns the contents of the buffer as a string.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn data(&self) -> String {
        self.data.to_string()
    }

    /// Writes the contents of the buffer to its path.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::old_io::{File, Open, Read};
    ///
    /// // Set up a buffer and point it to a path.
    /// let mut buffer = scribe::buffer::new();
    /// let write_path = Path::new("my_doc");
    /// buffer.path = Some(write_path.clone());
    ///
    /// // Put some data into the buffer and save it.
    /// buffer.insert("scribe");
    /// buffer.save();
    ///
    /// # let saved_data = File::open_mode(&write_path, Open, Read)
    /// #   .unwrap().read_to_string().unwrap();
    /// # assert_eq!(saved_data, "scribe");
    ///
    /// # std::old_io::fs::unlink(&write_path);
    /// ```
    pub fn save(&self) -> Option<IoError> {
        let path = match self.path.clone() {
            Some(p) => p,
            None => Path::new(""),
        };

        // Try to open and write to the file, returning any errors encountered.
        let mut file = match File::open_mode(&path, Open, Write) {
            Ok(f) => f,
            Err(error) => return Some(error),
        };
        match file.write(self.data().as_bytes()) {
            Ok(_) => (),
            Err(error) => return Some(error),
        }

        return None
    }

    /// Inserts `data` into the buffer at the cursor position.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn insert(&mut self, data: &str) {
        self.data.insert(data, &self.cursor);
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = gap_buffer::new(String::new());
    let cursor = Position{ line: 0, offset: 0 };

    Buffer{ data: data, path: None, cursor: cursor }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::from_file(&Path::new("tests/sample/file")).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: &Path) -> IoResult<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open_mode(path, Open, Read) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = match file.read_to_string() {
        Ok(d) => d,
        Err(error) => return Err(error),
    };

    let data = gap_buffer::new(data);
    let cursor = Position{ line: 0, offset: 0 };

    // Create a new buffer using the loaded data, path, and other defaults.
    Ok(Buffer{ data: data, path: Some(path.clone()), cursor: cursor })
}
<commit_msg>Implement buffer move_cursor method.<commit_after>use std::old_io::{File, Open, Read, Write};
use std::old_io::IoError;
use std::old_io::IoResult;
use super::GapBuffer;
use super::gap_buffer;
use super::Position;
use super::Range;

pub struct Buffer {
    data: GapBuffer,
    pub path: Option<Path>,
    pub cursor: Position,
}

impl Buffer {
    /// Returns the contents of the buffer as a string.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn data(&self) -> String {
        self.data.to_string()
    }

    /// Writes the contents of the buffer to its path.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::old_io::{File, Open, Read};
    ///
    /// // Set up a buffer and point it to a path.
    /// let mut buffer = scribe::buffer::new();
    /// let write_path = Path::new("my_doc");
    /// buffer.path = Some(write_path.clone());
    ///
    /// // Put some data into the buffer and save it.
    /// buffer.insert("scribe");
    /// buffer.save();
    ///
    /// # let saved_data = File::open_mode(&write_path, Open, Read)
    /// #   .unwrap().read_to_string().unwrap();
    /// # assert_eq!(saved_data, "scribe");
    ///
    /// # std::old_io::fs::unlink(&write_path);
    /// ```
    pub fn save(&self) -> Option<IoError> {
        let path = match self.path.clone() {
            Some(p) => p,
            None => Path::new(""),
        };

        // Try to open and write to the file, returning any errors encountered.
        let mut file = match File::open_mode(&path, Open, Write) {
            Ok(f) => f,
            Err(error) => return Some(error),
        };
        match file.write(self.data().as_bytes()) {
            Ok(_) => (),
            Err(error) => return Some(error),
        }

        return None
    }

    /// Inserts `data` into the buffer at the cursor position.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn insert(&mut self, data: &str) {
        self.data.insert(data, &self.cursor);
    }

    pub fn get_cursor(&self) -> Position {
        Position{ line: self.cursor.line, offset: self.cursor.offset }
    }

    /// Moves the buffer cursor to the specified location. The location is
    /// bounds-checked against the buffer data and the cursor will not be
    /// updated if it is out-of-bounds.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// let in_bounds = scribe::buffer::Position{ line: 0, offset: 2 };
    /// let out_of_bounds = scribe::buffer::Position{ line: 2, offset: 2 };
    /// buffer.insert("scribe");
    ///
    /// buffer.move_cursor(in_bounds);
    /// assert_eq!(buffer.get_cursor().line, 0);
    /// assert_eq!(buffer.get_cursor().offset, 2);
    ///
    /// buffer.move_cursor(out_of_bounds);
    /// assert_eq!(buffer.get_cursor().line, 0);
    /// assert_eq!(buffer.get_cursor().offset, 2);
    /// ```
    pub fn move_cursor(&mut self, position: Position) {
        if self.data.in_bounds(&position) {
            self.cursor = position;
        }
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = gap_buffer::new(String::new());
    let cursor = Position{ line: 0, offset: 0 };

    Buffer{ data: data, path: None, cursor: cursor }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::from_file(&Path::new("tests/sample/file")).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: &Path) -> IoResult<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open_mode(path, Open, Read) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = match file.read_to_string() {
        Ok(d) => d,
        Err(error) => return Err(error),
    };

    let data = gap_buffer::new(data);
    let cursor = Position{ line: 0, offset: 0 };

    // Create a new buffer using the loaded data, path, and other defaults.
    Ok(Buffer{ data: data, path: Some(path.clone()), cursor: cursor })
}
<|endoftext|>


<commit_before>use std::io::{stdout, stderr, Write};

use argparse::{ArgumentParser, StoreTrue};

use config::Config;

pub fn print_list(config: &Config, mut args: Vec<String>)
    -> Result<i32, String>
{
    let mut all = false;
    let mut builtin = false;
    let mut hidden = false;
    let mut containers = false;
    {
        args.insert(0, String::from("vagga _list"));
        let mut ap = ArgumentParser::new();
        ap.refer(&mut containers)
            .add_option(&["--containers"], StoreTrue,
                "Show containers instead of commands");
        ap.refer(&mut all)
            .add_option(&["-A", "--all"], StoreTrue,
                "Show all commands");
        ap.refer(&mut builtin)
            .add_option(&["--builtin"], StoreTrue,
                "Show built-in commands (starting with underscore)");
        ap.refer(&mut hidden)
            .add_option(&["--hidden"], StoreTrue,
                "Show hidden commands");
        match ap.parse(args, &mut stdout(), &mut stderr()) {
            Ok(()) => {}
            Err(x) => return Ok(x),
        }
    }
    if containers {
        for (cname, _) in config.containers.iter() {
            println!("{}", cname);
        }
    } else {
        let mut out = stdout();
        for (k, cmd) in config.commands.iter() {
            if k.starts_with("_") && !hidden {
                continue;
            }
            out.write_all(k.as_bytes()).ok();
            match cmd.description() {
                Some(ref val) => {
                    if k.len() > 19 {
                        out.write_all(b"\n                    ").ok();
                    } else {
                        for _ in k.len()..19 {
                            out.write_all(b" ").ok();
                        }
                        out.write_all(b" ").ok();
                    }
                    if val.contains("\n") {
                        for line in val.lines() {
                            out.write_all(line.as_bytes()).ok();
                            out.write_all(b"\n                    ").ok();
                        };
                    } else {
                        out.write_all(val.as_bytes()).ok();
                    }
                }
                None => {}
            }
            out.write_all(b"\n").ok();
        }

        if all || builtin {
            // TODO(tailhook) fetch builtins from completion code
            write!(&mut out, "\
                _build              Build a container\n\
                _run                Run arbitrary command, \
                                    optionally building container\n\
                _clean              Clean containers and build artifacts\n\
                _list               List of built-in commands\n\
            ").ok();
        }
    }
    return Ok(0);
}

pub fn print_help(config: &Config)
    -> Result<i32, String>
{
    let mut err = stderr();
    writeln!(&mut err, "Available commands:").ok();
    for (k, cmd) in config.commands.iter() {
        if k.starts_with("_") {
            continue;
        }
        write!(&mut err, "    {}", k).ok();
        match cmd.description() {
            Some(ref val) => {
                if k.len() > 19 {
                    err.write_all(b"\n                        ").ok();
                } else {
                    for _ in k.len()..19 {
                        err.write_all(b" ").ok();
                    }
                    err.write_all(b" ").ok();
                }
                if val.contains("\n") {
                    for line in val.lines() {
                        err.write_all(line.as_bytes()).ok();
                        err.write_all(b"\n                        ").ok();
                    };
                } else {
                    err.write_all(val.as_bytes()).ok();
                }
            }
            None => {}
        }
        err.write_all(b"\n").ok();
    }
    Ok(127)
}
<commit_msg>fix vagga _list -A to include hidden commands<commit_after>use std::io::{stdout, stderr, Write};

use argparse::{ArgumentParser, StoreTrue};

use config::Config;

pub fn print_list(config: &Config, mut args: Vec<String>)
    -> Result<i32, String>
{
    let mut all = false;
    let mut builtin = false;
    let mut hidden = false;
    let mut containers = false;
    {
        args.insert(0, String::from("vagga _list"));
        let mut ap = ArgumentParser::new();
        ap.refer(&mut containers)
            .add_option(&["--containers"], StoreTrue,
                "Show containers instead of commands");
        ap.refer(&mut all)
            .add_option(&["-A", "--all"], StoreTrue,
                "Show all commands");
        ap.refer(&mut builtin)
            .add_option(&["--builtin"], StoreTrue,
                "Show built-in commands (starting with underscore)");
        ap.refer(&mut hidden)
            .add_option(&["--hidden"], StoreTrue,
                "Show hidden commands");
        match ap.parse(args, &mut stdout(), &mut stderr()) {
            Ok(()) => {}
            Err(x) => return Ok(x),
        }
    }
    if containers {
        for (cname, _) in config.containers.iter() {
            println!("{}", cname);
        }
    } else {
        let mut out = stdout();
        for (k, cmd) in config.commands.iter() {
            if k.starts_with("_") && !(hidden || all) {
                continue;
            }
            out.write_all(k.as_bytes()).ok();
            match cmd.description() {
                Some(ref val) => {
                    if k.len() > 19 {
                        out.write_all(b"\n                    ").ok();
                    } else {
                        for _ in k.len()..19 {
                            out.write_all(b" ").ok();
                        }
                        out.write_all(b" ").ok();
                    }
                    if val.contains("\n") {
                        for line in val.lines() {
                            out.write_all(line.as_bytes()).ok();
                            out.write_all(b"\n                    ").ok();
                        };
                    } else {
                        out.write_all(val.as_bytes()).ok();
                    }
                }
                None => {}
            }
            out.write_all(b"\n").ok();
        }

        if all || builtin {
            // TODO(tailhook) fetch builtins from completion code
            write!(&mut out, "\
                _build              Build a container\n\
                _run                Run arbitrary command, \
                                    optionally building container\n\
                _clean              Clean containers and build artifacts\n\
                _list               List of built-in commands\n\
            ").ok();
        }
    }
    return Ok(0);
}

pub fn print_help(config: &Config)
    -> Result<i32, String>
{
    let mut err = stderr();
    writeln!(&mut err, "Available commands:").ok();
    for (k, cmd) in config.commands.iter() {
        if k.starts_with("_") {
            continue;
        }
        write!(&mut err, "    {}", k).ok();
        match cmd.description() {
            Some(ref val) => {
                if k.len() > 19 {
                    err.write_all(b"\n                        ").ok();
                } else {
                    for _ in k.len()..19 {
                        err.write_all(b" ").ok();
                    }
                    err.write_all(b" ").ok();
                }
                if val.contains("\n") {
                    for line in val.lines() {
                        err.write_all(line.as_bytes()).ok();
                        err.write_all(b"\n                        ").ok();
                    };
                } else {
                    err.write_all(val.as_bytes()).ok();
                }
            }
            None => {}
        }
        err.write_all(b"\n").ok();
    }
    Ok(127)
}
<|endoftext|>


<commit_before>#![allow(non_camel_case_types)]

extern crate rand;
extern crate rayon;
extern crate time;

use rand::{Rng, SeedableRng, XorShiftRng};
use std::env;
use std::str::FromStr;

trait Joiner {
    fn is_parallel() -> bool;
    fn join<A,R_A,B,R_B>(oper_a: A, oper_b: B) -> (R_A, R_B)
        where A: FnOnce() -> R_A + Send, B: FnOnce() -> R_B + Send;
}

struct Parallel;
impl Joiner for Parallel {
    #[inline]
    fn is_parallel() -> bool {
        true
    }
    #[inline]
    fn join<A,R_A,B,R_B>(oper_a: A, oper_b: B) -> (R_A, R_B)
        where A: FnOnce() -> R_A + Send, B: FnOnce() -> R_B + Send
    {
        rayon::join(oper_a, oper_b)
    }
}

struct Sequential;
impl Joiner for Sequential {
    #[inline]
    fn is_parallel() -> bool {
        false
    }
    #[inline]
    fn join<A,R_A,B,R_B>(oper_a: A, oper_b: B) -> (R_A, R_B)
        where A: FnOnce() -> R_A + Send, B: FnOnce() -> R_B + Send
    {
        let a = oper_a();
        let b = oper_b();
        (a, b)
    }
}

fn quick_sort<J:Joiner, T:PartialOrd+Send>(v: &mut [T]) {
    if v.len() <= 1 {
        return;
    }

    if J::is_parallel() && v.len() <= 10*1024 {
        return quick_sort::<Sequential, T>(v);
    }

    let mid = partition(v);
    let (lo, hi) = v.split_at_mut(mid);
    J::join(|| quick_sort::<J,T>(lo),
            || quick_sort::<J,T>(hi));
}

fn partition<T:PartialOrd+Send>(v: &mut [T]) -> usize {
    let pivot = v.len() - 1;
    let mut i = 0;
    for j in 0..pivot {
        if v[j] <= v[pivot] {
            v.swap(i, j);
            i += 1;
        }
    }
    v.swap(i, pivot);
    i
}

fn main() {
    let mut kilo = 24;
    let mut run_par = true;
    let mut run_seq = true;
    let mut reps = 10;

    let mut args = env::args().skip(1);
    while let Some(arg) = args.next() {
        if arg == "--no-par" {
            run_par = false;
        } else if arg == "--no-seq" {
            run_seq = false;
        } else if arg == "--reps" {
            if let Some(reps_arg) = args.next() {
                match usize::from_str(&reps_arg) {
                    Ok(v) => reps = v,
                    Err(_) => {
                        println!("invalid argument `{}`, expected integer", reps_arg);
                    }
                }
            } else {
                println!("missing argument to --reps");
            }
        } else {
            match usize::from_str(&arg) {
                Ok(v) => kilo = v,
                Err(_) => {
                    println!("invalid argument `{}`, expected integer", arg);
                }
            }
        }
    }

    let mut rng = XorShiftRng::from_seed([0, 1, 2, 3]);
    let mut par_time = 0;
    let mut seq_time = 0;

    rayon::initialize();

    for _ in 0 .. reps {
        let data: Vec<_> = (0..kilo*1024).map(|_| rng.next_u32()).collect();

        if run_par {
            let mut data = data.clone();
            let par_start = time::precise_time_ns();
            quick_sort::<Parallel, u32>(&mut data);
            par_time += time::precise_time_ns() - par_start;
        }

        if run_seq {
            let mut data = data.clone();
            let seq_start = time::precise_time_ns();
            quick_sort::<Sequential, u32>(&mut data);
            seq_time += time::precise_time_ns() - seq_start;
        }
    }

    println!("Array length    : {}K 32-bit integers", kilo);
    println!("Repetitions     : {}", reps);
    if run_par { println!("Parallel time   : {}", par_time); }
    if run_seq { println!("Sequential time : {}", seq_time); }
    if run_par && run_seq {
        println!("Parallel speedup: {}", (seq_time as f64) / (par_time as f64));
    }
}
<commit_msg>change fallback to 5K<commit_after>#![allow(non_camel_case_types)]

extern crate rand;
extern crate rayon;
extern crate time;

use rand::{Rng, SeedableRng, XorShiftRng};
use std::env;
use std::str::FromStr;

trait Joiner {
    fn is_parallel() -> bool;
    fn join<A,R_A,B,R_B>(oper_a: A, oper_b: B) -> (R_A, R_B)
        where A: FnOnce() -> R_A + Send, B: FnOnce() -> R_B + Send;
}

struct Parallel;
impl Joiner for Parallel {
    #[inline]
    fn is_parallel() -> bool {
        true
    }
    #[inline]
    fn join<A,R_A,B,R_B>(oper_a: A, oper_b: B) -> (R_A, R_B)
        where A: FnOnce() -> R_A + Send, B: FnOnce() -> R_B + Send
    {
        rayon::join(oper_a, oper_b)
    }
}

struct Sequential;
impl Joiner for Sequential {
    #[inline]
    fn is_parallel() -> bool {
        false
    }
    #[inline]
    fn join<A,R_A,B,R_B>(oper_a: A, oper_b: B) -> (R_A, R_B)
        where A: FnOnce() -> R_A + Send, B: FnOnce() -> R_B + Send
    {
        let a = oper_a();
        let b = oper_b();
        (a, b)
    }
}

fn quick_sort<J:Joiner, T:PartialOrd+Send>(v: &mut [T]) {
    if v.len() <= 1 {
        return;
    }

    if J::is_parallel() && v.len() <= 5*1024 {
        return quick_sort::<Sequential, T>(v);
    }

    let mid = partition(v);
    let (lo, hi) = v.split_at_mut(mid);
    J::join(|| quick_sort::<J,T>(lo),
            || quick_sort::<J,T>(hi));
}

fn partition<T:PartialOrd+Send>(v: &mut [T]) -> usize {
    let pivot = v.len() - 1;
    let mut i = 0;
    for j in 0..pivot {
        if v[j] <= v[pivot] {
            v.swap(i, j);
            i += 1;
        }
    }
    v.swap(i, pivot);
    i
}

fn main() {
    let mut kilo = 24;
    let mut run_par = true;
    let mut run_seq = true;
    let mut reps = 10;

    let mut args = env::args().skip(1);
    while let Some(arg) = args.next() {
        if arg == "--no-par" {
            run_par = false;
        } else if arg == "--no-seq" {
            run_seq = false;
        } else if arg == "--reps" {
            if let Some(reps_arg) = args.next() {
                match usize::from_str(&reps_arg) {
                    Ok(v) => reps = v,
                    Err(_) => {
                        println!("invalid argument `{}`, expected integer", reps_arg);
                    }
                }
            } else {
                println!("missing argument to --reps");
            }
        } else {
            match usize::from_str(&arg) {
                Ok(v) => kilo = v,
                Err(_) => {
                    println!("invalid argument `{}`, expected integer", arg);
                }
            }
        }
    }

    let mut rng = XorShiftRng::from_seed([0, 1, 2, 3]);
    let mut par_time = 0;
    let mut seq_time = 0;

    rayon::initialize();

    for _ in 0 .. reps {
        let data: Vec<_> = (0..kilo*1024).map(|_| rng.next_u32()).collect();

        if run_par {
            let mut data = data.clone();
            let par_start = time::precise_time_ns();
            quick_sort::<Parallel, u32>(&mut data);
            par_time += time::precise_time_ns() - par_start;
        }

        if run_seq {
            let mut data = data.clone();
            let seq_start = time::precise_time_ns();
            quick_sort::<Sequential, u32>(&mut data);
            seq_time += time::precise_time_ns() - seq_start;
        }
    }

    println!("Array length    : {}K 32-bit integers", kilo);
    println!("Repetitions     : {}", reps);
    if run_par { println!("Parallel time   : {}", par_time); }
    if run_seq { println!("Sequential time : {}", seq_time); }
    if run_par && run_seq {
        println!("Parallel speedup: {}", (seq_time as f64) / (par_time as f64));
    }
}
<|endoftext|>


<commit_before>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the readme renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.

#![warn(clippy::all, rust_2018_idioms)]

#[macro_use]
extern crate serde;

use cargo_registry::{
    db,
    models::Version,
    render::readme_to_html,
    schema::{crates, readme_renderings, versions},
    Config,
};
use std::{io::Read, path::Path, thread};

use chrono::{TimeZone, Utc};
use diesel::{dsl::any, prelude::*};
use docopt::Docopt;
use flate2::read::GzDecoder;
use reqwest::{blocking::Client, header};
use tar::{self, Archive};

const CACHE_CONTROL_README: &str = "public,max-age=604800";
const DEFAULT_PAGE_SIZE: usize = 25;
const USAGE: &str = "
Usage: render-readmes [options]
       render-readmes --help

Options:
    -h, --help         Show this message.
    --page-size NUM    How many versions should be queried and processed at a time.
    --older-than DATE  Only rerender readmes that are older than this date.
    --crate NAME       Only rerender readmes for the specified crate.
";

#[derive(Deserialize)]
struct Args {
    flag_page_size: Option<usize>,
    flag_older_than: Option<String>,
    flag_crate: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.deserialize())
        .unwrap_or_else(|e| e.exit());
    let config = Config::default();
    let conn = db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = args.flag_older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = args.flag_crate {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids: Vec<i32> = query.load(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = args.flag_page_size.unwrap_or(DEFAULT_PAGE_SIZE);

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions: Vec<(Version, String)> = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            let config = config.clone();
            Version::record_readme_rendering(version.id, &conn).unwrap_or_else(|_| {
                panic!(
                    "[{}-{}] Couldn't record rendering time",
                    krate_name, version.num
                )
            });
            let client = client.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(&config, &client, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let mut extra_headers = header::HeaderMap::new();
                extra_headers.insert(header::CACHE_CONTROL, CACHE_CONTROL_README.parse().unwrap());
                config
                    .uploader
                    .upload(
                        &client,
                        &readme_path,
                        content,
                        content_length,
                        "text/html",
                        extra_headers,
                    )
                    .unwrap_or_else(|_| {
                        panic!(
                            "[{}-{}] Couldn't upload file to S3",
                            krate_name, version.num
                        )
                    });
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    config: &Config,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> Option<String> {
    let location = config
        .uploader
        .crate_location(krate_name, &version.num.to_string());

    let response = match client.get(&location).send() {
        Ok(r) => r,
        Err(err) => {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                krate_name, version.num, err
            );
            return None;
        }
    };

    if !response.status().is_success() {
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            krate_name,
            version.num,
            response.text().unwrap()
        );
        return None;
    }

    let reader = GzDecoder::new(response);
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Invalid tar archive entries",
            krate_name, version.num
        )
    });
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", krate_name, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        toml::from_str(&contents).unwrap_or_else(|_| {
            panic!(
                "[{}-{}] Syntax error in manifest file",
                krate_name, version.num
            )
        })
    };

    let rendered = {
        let path = format!(
            "{}-{}/{}",
            krate_name, version.num, manifest.package.readme?
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        readme_to_html(
            &contents,
            manifest
                .package
                .readme_file
                .as_ref()
                .map_or("README.md", |e| &**e),
            manifest.package.repository.as_deref(),
        )
    };
    return Some(rendered);

    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
        readme_file: Option<String>,
        repository: Option<String>,
    }

    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
    version: &Version,
    krate_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                filepath == path
            }
        })
        .unwrap_or_else(|| {
            panic!(
                "[{}-{}] couldn't open file: {}",
                krate_name,
                version.num,
                path.display()
            )
        })
        .unwrap_or_else(|_| {
            panic!(
                "[{}-{}] file is not present: {}",
                krate_name,
                version.num,
                path.display()
            )
        });
    let mut contents = String::new();
    file.read_to_string(&mut contents).unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Couldn't read file contents",
            krate_name, version.num
        )
    });
    contents
}
<commit_msg>bin/render-readmes: Use `clap` for CLI argument parsing<commit_after>#![warn(clippy::all, rust_2018_idioms)]

#[macro_use]
extern crate serde;

use cargo_registry::{
    db,
    models::Version,
    render::readme_to_html,
    schema::{crates, readme_renderings, versions},
    Config,
};
use std::{io::Read, path::Path, thread};

use chrono::{TimeZone, Utc};
use clap::Clap;
use diesel::{dsl::any, prelude::*};
use flate2::read::GzDecoder;
use reqwest::{blocking::Client, header};
use tar::{self, Archive};

const CACHE_CONTROL_README: &str = "public,max-age=604800";

#[derive(Clap, Debug)]
#[clap(
    name = "render-readmes",
    about = "Iterates over every crate versions ever uploaded and (re-)renders their \
        readme using the readme renderer from the cargo_registry crate.\n\
        \n\
        Warning: this can take a lot of time."
)]
struct Opts {
    /// How many versions should be queried and processed at a time.
    #[clap(long, default_value = "25")]
    page_size: usize,

    /// Only rerender readmes that are older than this date.
    #[clap(long)]
    older_than: Option<String>,

    /// Only rerender readmes for the specified crate.
    #[clap(long = "crate")]
    crate_name: Option<String>,
}

fn main() {
    let opts: Opts = Opts::parse();

    let config = Config::default();
    let conn = db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = opts.older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = opts.crate_name {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids: Vec<i32> = query.load(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = opts.page_size;

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions: Vec<(Version, String)> = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            let config = config.clone();
            Version::record_readme_rendering(version.id, &conn).unwrap_or_else(|_| {
                panic!(
                    "[{}-{}] Couldn't record rendering time",
                    krate_name, version.num
                )
            });
            let client = client.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(&config, &client, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let mut extra_headers = header::HeaderMap::new();
                extra_headers.insert(header::CACHE_CONTROL, CACHE_CONTROL_README.parse().unwrap());
                config
                    .uploader
                    .upload(
                        &client,
                        &readme_path,
                        content,
                        content_length,
                        "text/html",
                        extra_headers,
                    )
                    .unwrap_or_else(|_| {
                        panic!(
                            "[{}-{}] Couldn't upload file to S3",
                            krate_name, version.num
                        )
                    });
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    config: &Config,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> Option<String> {
    let location = config
        .uploader
        .crate_location(krate_name, &version.num.to_string());

    let response = match client.get(&location).send() {
        Ok(r) => r,
        Err(err) => {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                krate_name, version.num, err
            );
            return None;
        }
    };

    if !response.status().is_success() {
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            krate_name,
            version.num,
            response.text().unwrap()
        );
        return None;
    }

    let reader = GzDecoder::new(response);
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Invalid tar archive entries",
            krate_name, version.num
        )
    });
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", krate_name, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        toml::from_str(&contents).unwrap_or_else(|_| {
            panic!(
                "[{}-{}] Syntax error in manifest file",
                krate_name, version.num
            )
        })
    };

    let rendered = {
        let path = format!(
            "{}-{}/{}",
            krate_name, version.num, manifest.package.readme?
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        readme_to_html(
            &contents,
            manifest
                .package
                .readme_file
                .as_ref()
                .map_or("README.md", |e| &**e),
            manifest.package.repository.as_deref(),
        )
    };
    return Some(rendered);

    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
        readme_file: Option<String>,
        repository: Option<String>,
    }

    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
    version: &Version,
    krate_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                filepath == path
            }
        })
        .unwrap_or_else(|| {
            panic!(
                "[{}-{}] couldn't open file: {}",
                krate_name,
                version.num,
                path.display()
            )
        })
        .unwrap_or_else(|_| {
            panic!(
                "[{}-{}] file is not present: {}",
                krate_name,
                version.num,
                path.display()
            )
        });
    let mut contents = String::new();
    file.read_to_string(&mut contents).unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Couldn't read file contents",
            krate_name, version.num
        )
    });
    contents
}
<|endoftext|>


<commit_before>use memory::Memory;

static NMI_VECTOR: u16 = 0xfffa;
static RESET_VECTOR: u16 = 0xfffc;
static IRQ_VECTOR: u16 = 0xfffe;

struct Registers {
	pc: u16,				// program counter
	ac: u8,					// accumulator
	x: u8,					// x register
	y: u8,					// y register
	sr: u8,					// status register (NV-BDIZC: Negative, oVerflow, Break, Decimal, Interrupt, Zero, Carry)
	sp: u8,					// stack pointer
}

enum Operand {
	Implied,							// OPC				// operand implied
	Immediate(u8),						// OPC #$BB			// operand is byte (BB)
	Accumulator,						// OPC A			// operand is AC
	Relative(i8),						// OPC $BB			// branch target is PC + offset (BB), bit 7 signifies negative offset
	Absolute(u16),						// OPC $HHLL		// operand is address $HHLL
	AbsoluteIndexedWithX(u16),			// OPC $HHLL,X		// operand is address $HHLL incremented by X with carry
	AbsoluteIndexedWithY(u16),			// OPC $HHLL,Y		// operand is address $HHLL incremented by Y with carry
	Indirect(u16),						// OPC ($HHLL)		// operand is effective address; effective address is value of address
	ZeroPage(u8),						// OPC $LL			// operand is address $00LL
	ZeroPageIndexedWithX(u8),			// OPC $LL,X		// operand is address $00LL incremented by X, no page transition
	ZeroPageIndexedWithY(u8),			// OPC $LL,Y		// operand is address $00LL incremented by Y, no page transition
	ZeroPageIndexedWithXIndirect(u8),	// OPC ($LL,X)		// operand is effective address; effective address is $00LL incremented by X without carry
	ZeroPageIndirectIndexedWithY(u8),	// OPC ($LL),Y		// operand is effective address incremented by Y with carry; effective address is word at $00LL
}

impl Operand {
	fn addr (&self, cpu: &Mos6502) -> u16 {
		match * self {
			Implied								=> fail!("mos6510: Implied operand is never targetted to an address"),
			Immediate(_)						=> fail!("mos6510: Immediade operand is never targetted to an address"),
			Accumulator							=> fail!("mos6510: Accumulator operand is never targetted to an address"),
			Relative(offset)					=> cpu.reg.pc + offset as u16,
			Absolute(addr)						=> addr,
			AbsoluteIndexedWithX(addr)			=> addr + cpu.reg.x as u16,
			AbsoluteIndexedWithY(addr)			=> addr + cpu.reg.y as u16,
			Indirect(addr)						=> cpu.get_le(addr),
			ZeroPage(addr)						=> addr as u16,
			ZeroPageIndexedWithX(addr)			=> (addr + cpu.reg.x) as u16,				// no page transition
			ZeroPageIndexedWithY(addr)			=> (addr + cpu.reg.y) as u16,				// no page transition
			ZeroPageIndexedWithXIndirect(addr)	=> cpu.get_le((addr + cpu.reg.x) as u16),	// no page transition
			ZeroPageIndirectIndexedWithY(addr)	=> { let iaddr: u16 = cpu.get_le(addr as u16); iaddr + cpu.reg.y as u16 },
		}
	}

	fn get (&self, cpu: &Mos6502) -> u8 {
		match *self {
			Implied								=> fail!("mos6510: Implied operand never has a value"),
			Immediate(val)						=> val,
			Accumulator							=> cpu.reg.ac,
			Relative(_target)					=> fail!("mos6510: Relative operand never has a value"),
			op									=> { let addr = op.addr(cpu); cpu.get(addr) },
		}
	}

	fn set (&self, cpu: &mut Mos6502, val: u8) {
		match *self {
			Implied								=> fail!("mos6510: Implied operand never sets a value"),
			Immediate(_)						=> fail!("mos6510: Immediate operand never sets a value"),
			Accumulator							=> cpu.reg.ac = val,
			Relative(_target)					=> fail!("mos6510: Relative operand never sets a value"),
			op									=> { let addr = op.addr(cpu); cpu.set(addr, val); },
		}
	}
}

pub struct Mos6502<'self> {
	priv reg: Registers,					// internal CPU registers
	priv mem: &'self Memory<'self, u16>,	// memory as accessible to the CPU
}

impl<'self> Mos6502<'self> {
	pub fn new (mem: &'self Memory<u16>) -> Mos6502<'self> {
		Mos6502 {
			reg: Registers { pc: 0, ac: 0, x: 0, y: 0, sr: 0, sp: 0 },
			mem: mem,
		}
	}
}

pub struct Mos6510<'self> {
	priv cpu: Mos6502<'self>,				// Core CPU
	priv port_ddr: u8,						// CPU port data direction register
	priv port_dat: u8,						// CPU port data register
}

impl<'self> Mos6510<'self> {
	pub fn new (mem: &'self Memory<u16>) -> Mos6510<'self> {
		// TODO: addresses $0000 (data direction) and $0001 (data) are hardwired for the processor I/O port
		Mos6510 { cpu: Mos6502::new(mem), port_ddr: 0, port_dat: 0 }
	}
}
<commit_msg>Fixed cpu memory access without cpu being addressable itself<commit_after>use memory::Memory;

static NMI_VECTOR: u16 = 0xfffa;
static RESET_VECTOR: u16 = 0xfffc;
static IRQ_VECTOR: u16 = 0xfffe;

struct Registers {
	pc: u16,				// program counter
	ac: u8,					// accumulator
	x: u8,					// x register
	y: u8,					// y register
	sr: u8,					// status register (NV-BDIZC: Negative, oVerflow, Break, Decimal, Interrupt, Zero, Carry)
	sp: u8,					// stack pointer
}

enum Operand {
	Implied,							// OPC				// operand implied
	Immediate(u8),						// OPC #$BB			// operand is byte (BB)
	Accumulator,						// OPC A			// operand is AC
	Relative(i8),						// OPC $BB			// branch target is PC + offset (BB), bit 7 signifies negative offset
	Absolute(u16),						// OPC $HHLL		// operand is address $HHLL
	AbsoluteIndexedWithX(u16),			// OPC $HHLL,X		// operand is address $HHLL incremented by X with carry
	AbsoluteIndexedWithY(u16),			// OPC $HHLL,Y		// operand is address $HHLL incremented by Y with carry
	Indirect(u16),						// OPC ($HHLL)		// operand is effective address; effective address is value of address
	ZeroPage(u8),						// OPC $LL			// operand is address $00LL
	ZeroPageIndexedWithX(u8),			// OPC $LL,X		// operand is address $00LL incremented by X, no page transition
	ZeroPageIndexedWithY(u8),			// OPC $LL,Y		// operand is address $00LL incremented by Y, no page transition
	ZeroPageIndexedWithXIndirect(u8),	// OPC ($LL,X)		// operand is effective address; effective address is $00LL incremented by X without carry
	ZeroPageIndirectIndexedWithY(u8),	// OPC ($LL),Y		// operand is effective address incremented by Y with carry; effective address is word at $00LL
}

impl Operand {
	fn addr (&self, cpu: &Mos6502) -> u16 {
		match * self {
			Implied								=> fail!("mos6510: Implied operand is never targetted to an address"),
			Immediate(_)						=> fail!("mos6510: Immediade operand is never targetted to an address"),
			Accumulator							=> fail!("mos6510: Accumulator operand is never targetted to an address"),
			Relative(offset)					=> cpu.reg.pc + offset as u16,
			Absolute(addr)						=> addr,
			AbsoluteIndexedWithX(addr)			=> addr + cpu.reg.x as u16,
			AbsoluteIndexedWithY(addr)			=> addr + cpu.reg.y as u16,
			Indirect(addr)						=> cpu.mem.get_le(addr),
			ZeroPage(addr)						=> addr as u16,
			ZeroPageIndexedWithX(addr)			=> (addr + cpu.reg.x) as u16,					// no page transition
			ZeroPageIndexedWithY(addr)			=> (addr + cpu.reg.y) as u16,					// no page transition
			ZeroPageIndexedWithXIndirect(addr)	=> cpu.mem.get_le((addr + cpu.reg.x) as u16),	// no page transition
			ZeroPageIndirectIndexedWithY(addr)	=> { let iaddr: u16 = cpu.mem.get_le(addr as u16); iaddr + cpu.reg.y as u16 },
		}
	}

	fn get (&self, cpu: &Mos6502) -> u8 {
		match *self {
			Implied								=> fail!("mos6510: Implied operand never has a value"),
			Immediate(val)						=> val,
			Accumulator							=> cpu.reg.ac,
			Relative(_target)					=> fail!("mos6510: Relative operand never has a value"),
			op									=> { let addr = op.addr(cpu); cpu.mem.get(addr) },
		}
	}

	fn set (&self, cpu: &mut Mos6502, val: u8) {
		match *self {
			Implied								=> fail!("mos6510: Implied operand never sets a value"),
			Immediate(_)						=> fail!("mos6510: Immediate operand never sets a value"),
			Accumulator							=> cpu.reg.ac = val,
			Relative(_target)					=> fail!("mos6510: Relative operand never sets a value"),
			op									=> { let addr = op.addr(cpu); cpu.mem.set(addr, val); },
		}
	}
}

pub struct Mos6502<'self> {
	priv reg: Registers,					// internal CPU registers
	priv mem: &'self Memory<'self, u16>,	// memory as accessible to the CPU
}

impl<'self> Mos6502<'self> {
	pub fn new (mem: &'self Memory<u16>) -> Mos6502<'self> {
		Mos6502 {
			reg: Registers { pc: 0, ac: 0, x: 0, y: 0, sr: 0, sp: 0 },
			mem: mem,
		}
	}
}

pub struct Mos6510<'self> {
	priv cpu: Mos6502<'self>,				// Core CPU
	priv port_ddr: u8,						// CPU port data direction register
	priv port_dat: u8,						// CPU port data register
}

impl<'self> Mos6510<'self> {
	pub fn new (mem: &'self Memory<u16>) -> Mos6510<'self> {
		// TODO: addresses $0000 (data direction) and $0001 (data) are hardwired for the processor I/O port
		Mos6510 { cpu: Mos6502::new(mem), port_ddr: 0, port_dat: 0 }
	}
}
<|endoftext|>


<commit_before>use utils::lru_cache::LRUCache;
use dovahkiin::types::custom_types::id::Id;
use dovahkiin::types::custom_types::map::Map;
use std::cell::RefCell;
use ram::cell::Cell;
use ram::types::*;
use std::cell::Ref;
use std::cell::RefMut;
use index::btree::*;
use bifrost::utils::async_locks::{RwLock, RwLockWriteGuard, RwLockReadGuard, Mutex};
use std::ops::Deref;
use std::ops::DerefMut;
use core::borrow::BorrowMut;
use dovahkiin::types::value::ToValue;
use itertools::Itertools;
use std::collections::BTreeMap;
use std::mem;
use std::rc::Rc;
use owning_ref::{RcRef, OwningHandle, OwningRef};
use ram::schema::{Schema, Field};
use dovahkiin::types::type_id_of;
use std::sync::Arc;
use client::AsyncClient;
use futures::Future;

pub type ExtNodeCacheMap = Mutex<LRUCache<Id, RwLock<ExtNode>>>;
pub type ExtNodeCachedMut = RwLockWriteGuard<ExtNode>;
pub type ExtNodeCachedImmute = RwLockReadGuard<ExtNode>;

const PAGE_SCHEMA: &'static str = "NEB_BTREE_PAGE";
const KEYS_FIELD: &'static str = "keys";
const NEXT_FIELD: &'static str = "next";
const PREV_FIELD: &'static str = "prev";

lazy_static! {
    static ref KEYS_KEY_HASH : u64 = key_hash(KEYS_FIELD);
    static ref NEXT_PAGE_KEY_HASH : u64 = key_hash(NEXT_FIELD);
    static ref PREV_PAGE_KEY_HASH : u64 = key_hash(PREV_FIELD);
    static ref PAGE_SCHEMA_ID: u32 = key_hash(PAGE_SCHEMA) as u32;
}

#[derive(Clone)]
pub struct ExtNode {
    pub id: Id,
    pub keys: EntryKeySlice,
    pub next: Id,
    pub prev: Id,
    pub len: usize,
    pub version: u64,
    pub removed: bool,
}

pub struct ExtNodeSplit {
    pub node_2: ExtNode,
    pub keys_1_len: usize
}

impl ExtNode {
    pub fn new(id: &Id) -> ExtNode {
        ExtNode {
            id: *id,
            keys: EntryKeySlice::init(),
            next: Id::unit_id(),
            prev: Id::unit_id(),
            len: 0,
            version: 0,
            removed: false
        }
    }
    pub fn from_cell(cell: Cell) -> Self {
        let keys = &cell.data[*KEYS_KEY_HASH];
        let keys_len = keys.len().unwrap();
        let mut key_slice = EntryKeySlice::init();
        let mut key_count = 0;
        let next = cell.data[*NEXT_PAGE_KEY_HASH].Id().unwrap();
        let prev = cell.data[*PREV_PAGE_KEY_HASH].Id().unwrap();
        for (i, key_val) in keys.iter_value().unwrap().enumerate() {
            let key = if let Value::PrimArray(PrimitiveArray::U8(ref array)) = key_val {
                EntryKey::from_slice(array.as_slice())
            } else { panic!("invalid entry") };
            key_slice[i] = key;
            key_count += 1;
        }
        ExtNode {
            id: cell.id(),
            keys: key_slice,
            next: *next,
            prev: *prev,
            len: key_count,
            version: cell.header.version,
            removed: false
        }
    }
    pub fn to_cell(&self) -> Cell {
        let mut value = Value::Map(Map::new());
        value[*NEXT_PAGE_KEY_HASH] = Value::Id(self.next);
        value[*KEYS_KEY_HASH] = self
            .keys[..self.len]
            .iter()
            .map(|key| {
                key.as_slice().to_vec().value()
            })
            .collect_vec()
            .value();
        Cell::new_with_id(*PAGE_SCHEMA_ID, &self.id, value)
    }
    pub fn remove(&mut self, pos: usize) {
        let cached_len = self.len;
        self.keys.remove_at(pos, cached_len);
        self.len -= 1;
    }
    pub fn insert(
        &mut self,
        key: EntryKey,
        pos: usize,
        tree: &BPlusTree,
        bz: &CacheBufferZone)
        -> Option<(Node, Option<EntryKey>)>
    {
        let mut cached = self;
        let cached_len = cached.len;
        if cached_len >= NUM_KEYS {
            // need to split
            debug!("insert to external with split, key {:?}, pos {}", &key, pos);
            cached.dump();
            let pivot = cached_len / 2;
            let split = {
                let cached_next = *&cached.next;
                let cached_id = *&cached.id;
                let new_page_id = tree.new_page_id();
                cached.next = new_page_id;
                let mut keys_1 = &mut cached.keys;
                let mut keys_2 = keys_1.split_at_pivot(pivot, cached_len);
                let mut keys_1_len = pivot;
                let mut keys_2_len = cached_len - pivot;
                insert_into_split(
                    key,
                    keys_1, &mut keys_2,
                    &mut keys_1_len, &mut keys_2_len,
                    pos, pivot);
                ExtNodeSplit {
                    keys_1_len,
                    node_2: ExtNode {
                        id: new_page_id,
                        keys: keys_2,
                        next: cached_next,
                        prev: cached_id,
                        len: keys_2_len,
                        version: 0,
                        removed: false
                    }
                }
            };
            cached.next = split.node_2.id;
            cached.len = split.keys_1_len;
            let node_2_id = split.node_2.id;
            let node_2 = Node::External(box node_2_id);
            debug!("Split to left len {}, right len {}", cached.len, split.node_2.len);
            cached.dump();
            split.node_2.dump();
            bz.new_node(split.node_2);
            return Some((node_2, None));

        } else {
            debug!("insert to external without split at {}, key {:?}", pos, key);
            let middle_insertion = pos != cached_len;
            let mut new_cached_len = cached_len;
            cached.keys.insert_at(key, pos, &mut new_cached_len);
            cached.len = new_cached_len;
            if middle_insertion {
                cached.dump();
            }
            return None;
        }
    }
    pub fn merge_with(&mut self, right: &mut Self) {
        let self_len = self.len;
        let new_len = self.len + right.len;
        assert!(new_len <= self.keys.len());
        for i in self.len .. new_len {
            self.keys[i] = mem::replace(&mut right.keys[i - self_len], Default::default());
        }
    }
    pub fn dump(&self) {
        debug!("Dumping {:?}, keys {}", self.id, self.len);
        for i in 0..NUM_KEYS {
            println!("{}\t- {:?}", i, self.keys[i]);
        }
    }
}

pub fn rearrange_empty_extnode(node: &ExtNode, bz: &CacheBufferZone) -> Id {
    let prev = node.prev;
    let next = node.next;
    if !prev.is_unit_id() {
        let mut prev_node = bz.get_for_mut(&prev);
        prev_node.next = next;
    }
    if !next.is_unit_id() {
        let mut next_node = bz.get_for_mut(&next);
        next_node.prev = prev;
    }
    return node.id;
}

#[derive(Clone)]
pub enum CacheGuardHolder {
    Read(ExtNodeCachedImmute),
    Write(ExtNodeCachedMut)
}

type NodeRcRefCell = Rc<RefCell<ExtNode>>;

pub type RcNodeRefData<'a> = OwningHandle<RcRef<RefCell<ExtNode>>, Ref<'a, ExtNode>>;
pub type RcNodeRefMut<'a> = OwningHandle<RcRef<RefCell<ExtNode>>, RefMut<'a, ExtNode>>;

pub enum RcNodeRef<'a> {
    Data(RcNodeRefData<'a>),
    Guard(CacheGuardHolder)
}

pub struct CacheBufferZone {
    mapper: Arc<ExtNodeCacheMap>,
    storage: Arc<AsyncClient>,
    guards: RefCell<BTreeMap<Id, CacheGuardHolder>>,
    data: RefCell<BTreeMap<Id, Option<NodeRcRefCell>>>
}

impl Deref for CacheGuardHolder {
    type Target = ExtNode;

    fn deref(&self) -> &'_ <Self as Deref>::Target {
        match self {
            &CacheGuardHolder::Read(ref g) => &*g,
            &CacheGuardHolder::Write(ref g) => &*g
        }
    }
}

impl <'a> Deref for RcNodeRef<'a> {
    type Target = ExtNode;

    fn deref(&self) -> &'_ <Self as Deref>::Target {
        match self {
            &RcNodeRef::Guard(ref g) => &*g,
            &RcNodeRef::Data(ref g) => &*g
        }
    }
}

impl CacheBufferZone {
    pub fn new(mapper: &Arc<ExtNodeCacheMap>, storage: &Arc<AsyncClient>) -> CacheBufferZone {
        CacheBufferZone {
            mapper: mapper.clone() ,
            storage: storage.clone(),
            guards: RefCell::new(BTreeMap::new()),
            data: RefCell::new(BTreeMap::new())
        }
    }

    pub fn get(&self, id: &Id) -> RcNodeRef {
        debug!("Getting cached fo readonly {:?}", id);
        {
            // read from data (from write)
            let data = self.data.borrow();
            match data.get(id) {
                Some(Some(ref data)) => {
                    let cell = data.clone();
                    let cell_ref = RcRef::new(cell);
                    let handle = OwningHandle::new(cell_ref);
                    return RcNodeRef::Data(handle);
                },
                Some(None) => panic!(), // deleted
                _ => {}
            }
        }
        {
            // read from guards
            let guard = self.guards.borrow();
            if let Some(holder) = guard.get(id) {
                return RcNodeRef::Guard(holder.clone());
            }
        }
        {
            debug!("Buffer zone inserting for readonly");
            let guard = self.get_ext_node_cached(id);
            let holder = CacheGuardHolder::Read(guard);
            let mut guards = self.guards.borrow_mut();
            guards.insert(*id, holder);
            debug!("Buffer zone now have data {}", guards.len());
        }
        return self.get(id);
    }

    pub fn get_for_mut(&self, id: &Id) -> RcNodeRefMut {
        debug!("Getting cached for mutation {:?}", id);
        {
            let data = self.data.borrow();
            match data.get(id) {
                Some(Some(ref extnode)) => {
                    let guards = self.guards.borrow();
                    if let CacheGuardHolder::Read(_) = guards.get(id).unwrap() {
                        panic!("Mutating readonly buffered cache");
                    }
                    let cell = extnode.clone();
                    let cell_ref = RcRef::new(cell);
                    let handle = OwningHandle::new_mut(cell_ref);
                    debug!("Got cached, len {}, cache data {}, cache guards {}", handle.len, data.len(), guards.len());
                    return handle
                },
                Some(None) => panic!(),
                _ => {}
            }
        }
        {
            debug!("Buffer zone inserting for mutation {:?}", id);
            let guard = self.get_mut_ext_node_cached(id);
            let mut data = self.data.borrow_mut();
            data.insert(*id, Some(Rc::new(RefCell::new((&*guard).clone()))));
            let holder = CacheGuardHolder::Write(guard);
            let mut guards = self.guards.borrow_mut();
            guards.insert(*id, holder);
            debug!("Buffer zone now have data {}, guards {}", data.len(), guards.len());
        }
        self.get_for_mut(id)
    }

    pub fn delete(&self, id: &Id) {
        let mut data = self.data.borrow_mut();
        data.insert(*id, None);
    }

    pub fn new_node(&self, node: ExtNode) {
        let mut mapper = self.mapper.lock();
        if mapper.get(&node.id).is_some() {
            panic!()
        } else {
            let node_id = node.id;
            let lock = RwLock::new(ExtNode::new(&node_id));
            let guard = CacheGuardHolder::Write(lock.write());
            let mut guards = self.guards.borrow_mut();
            let mut data = self.data.borrow_mut();
            debug!("new node in buffered cache: {:?}, was guards: {}, data: {}", node_id, guards.len(), data.len());
            mapper.insert(node_id, lock);
            guards.insert(node_id, guard);
            data.insert(node_id, Some(Rc::new(RefCell::new(node))));
        }
    }

    pub fn flush(&self) {
        let mut guards = self.guards.borrow_mut();
        let data = self.data.borrow();
        debug!("Flushing cache buffer, guards: {:?}, data: {:?}", guards.keys(), data.keys());
        for ((id, data),  (gid, mut holder)) in data.iter().zip(guards.iter_mut()) {
            assert_eq!(id, gid);
            if let Some(ref node_rc) = data {
                debug!("Flushing updating node: {:?}", id);
                if let &mut CacheGuardHolder::Write(ref mut guard) = holder {
                    let mut ext_node = (*(*node_rc).borrow()).clone();
                    debug!("flushing {:?} to cache with {} keys", ext_node.id, ext_node.len);
                    **guard = ext_node
                }
            } else {
                debug!("Flushing deleting node: {:?}", id);
                self.mapper.lock().remove(id);
                self.storage.remove_cell(*id).wait().unwrap();
            }
        }
    }

    pub fn get_guard(&self, id: &Id) -> Option<CacheGuardHolder> {
        if id.is_unit_id() {
            return None;
        }
        let mut guards = self.guards.borrow_mut();
        {
            let res = guards.get(id).map(|rg| rg.clone());
            if res.is_some() { return res; }
        }
        let guard = self.get_ext_node_cached(id);
        let holder = CacheGuardHolder::Read(guard);
        guards.insert(*id, holder.clone());
        return Some(holder)
    }

    pub fn get_mut_ext_node_cached(&self, id: &Id) -> ExtNodeCachedMut {
        let mut map = self.mapper.lock();
        return map.get_or_fetch(id).unwrap().write();
    }
    pub fn get_ext_node_cached(&self, id: &Id) -> ExtNodeCachedImmute {
        let mut map = self.mapper.lock();
        return map.get_or_fetch(id).unwrap().read();
    }
}

pub fn page_schema() -> Schema {
    Schema {
        id: *PAGE_SCHEMA_ID,
        name: String::from(PAGE_SCHEMA),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new(
            "*", 0, false, false,
            Some(vec![
                Field::new(NEXT_FIELD, type_id_of(Type::Id), false, false, None),
                Field::new(PREV_FIELD, type_id_of(Type::Id), false, false, None),
                Field::new(KEYS_FIELD, type_id_of(Type::U8), false, true, None)
            ])
        )
    }
}
<commit_msg>fix insertion errors<commit_after>use utils::lru_cache::LRUCache;
use dovahkiin::types::custom_types::id::Id;
use dovahkiin::types::custom_types::map::Map;
use std::cell::RefCell;
use ram::cell::Cell;
use ram::types::*;
use std::cell::Ref;
use std::cell::RefMut;
use index::btree::*;
use bifrost::utils::async_locks::{RwLock, RwLockWriteGuard, RwLockReadGuard, Mutex};
use std::ops::Deref;
use std::ops::DerefMut;
use core::borrow::BorrowMut;
use dovahkiin::types::value::ToValue;
use itertools::Itertools;
use std::collections::BTreeMap;
use std::mem;
use std::rc::Rc;
use owning_ref::{RcRef, OwningHandle, OwningRef};
use ram::schema::{Schema, Field};
use dovahkiin::types::type_id_of;
use std::sync::Arc;
use client::AsyncClient;
use futures::Future;

pub type ExtNodeCacheMap = Mutex<LRUCache<Id, RwLock<ExtNode>>>;
pub type ExtNodeCachedMut = RwLockWriteGuard<ExtNode>;
pub type ExtNodeCachedImmute = RwLockReadGuard<ExtNode>;

const PAGE_SCHEMA: &'static str = "NEB_BTREE_PAGE";
const KEYS_FIELD: &'static str = "keys";
const NEXT_FIELD: &'static str = "next";
const PREV_FIELD: &'static str = "prev";

lazy_static! {
    static ref KEYS_KEY_HASH : u64 = key_hash(KEYS_FIELD);
    static ref NEXT_PAGE_KEY_HASH : u64 = key_hash(NEXT_FIELD);
    static ref PREV_PAGE_KEY_HASH : u64 = key_hash(PREV_FIELD);
    static ref PAGE_SCHEMA_ID: u32 = key_hash(PAGE_SCHEMA) as u32;
}

#[derive(Clone)]
pub struct ExtNode {
    pub id: Id,
    pub keys: EntryKeySlice,
    pub next: Id,
    pub prev: Id,
    pub len: usize,
    pub version: u64,
    pub removed: bool,
}

pub struct ExtNodeSplit {
    pub node_2: ExtNode,
    pub keys_1_len: usize
}

impl ExtNode {
    pub fn new(id: &Id) -> ExtNode {
        ExtNode {
            id: *id,
            keys: EntryKeySlice::init(),
            next: Id::unit_id(),
            prev: Id::unit_id(),
            len: 0,
            version: 0,
            removed: false
        }
    }
    pub fn from_cell(cell: Cell) -> Self {
        let keys = &cell.data[*KEYS_KEY_HASH];
        let keys_len = keys.len().unwrap();
        let mut key_slice = EntryKeySlice::init();
        let mut key_count = 0;
        let next = cell.data[*NEXT_PAGE_KEY_HASH].Id().unwrap();
        let prev = cell.data[*PREV_PAGE_KEY_HASH].Id().unwrap();
        for (i, key_val) in keys.iter_value().unwrap().enumerate() {
            let key = if let Value::PrimArray(PrimitiveArray::U8(ref array)) = key_val {
                EntryKey::from_slice(array.as_slice())
            } else { panic!("invalid entry") };
            key_slice[i] = key;
            key_count += 1;
        }
        ExtNode {
            id: cell.id(),
            keys: key_slice,
            next: *next,
            prev: *prev,
            len: key_count,
            version: cell.header.version,
            removed: false
        }
    }
    pub fn to_cell(&self) -> Cell {
        let mut value = Value::Map(Map::new());
        value[*NEXT_PAGE_KEY_HASH] = Value::Id(self.next);
        value[*KEYS_KEY_HASH] = self
            .keys[..self.len]
            .iter()
            .map(|key| {
                key.as_slice().to_vec().value()
            })
            .collect_vec()
            .value();
        Cell::new_with_id(*PAGE_SCHEMA_ID, &self.id, value)
    }
    pub fn remove(&mut self, pos: usize) {
        let cached_len = self.len;
        self.keys.remove_at(pos, cached_len);
        self.len -= 1;
    }
    pub fn insert(
        &mut self,
        key: EntryKey,
        pos: usize,
        tree: &BPlusTree,
        bz: &CacheBufferZone)
        -> Option<(Node, Option<EntryKey>)>
    {
        let mut cached = self;
        let cached_len = cached.len;
        if cached_len >= NUM_KEYS {
            // need to split
            debug!("insert to external with split, key {:?}, pos {}", &key, pos);
            // cached.dump();
            let pivot = cached_len / 2;
            let split = {
                let cached_next = *&cached.next;
                let cached_id = *&cached.id;
                let new_page_id = tree.new_page_id();
                cached.next = new_page_id;
                let mut keys_1 = &mut cached.keys;
                let mut keys_2 = keys_1.split_at_pivot(pivot, cached_len);
                let mut keys_1_len = pivot;
                let mut keys_2_len = cached_len - pivot;
                insert_into_split(
                    key,
                    keys_1, &mut keys_2,
                    &mut keys_1_len, &mut keys_2_len,
                    pos, pivot);
                ExtNodeSplit {
                    keys_1_len,
                    node_2: ExtNode {
                        id: new_page_id,
                        keys: keys_2,
                        next: cached_next,
                        prev: cached_id,
                        len: keys_2_len,
                        version: 0,
                        removed: false
                    }
                }
            };
            cached.next = split.node_2.id;
            cached.len = split.keys_1_len;
            let node_2_id = split.node_2.id;
            let node_2 = Node::External(box node_2_id);
            debug!("Split to left len {}, right len {}", cached.len, split.node_2.len);
            // cached.dump();
            // split.node_2.dump();
            bz.new_node(split.node_2);
            return Some((node_2, None));

        } else {
            debug!("insert to external without split at {}, key {:?}", pos, key);
            // let middle_insertion = pos != cached_len;
            let mut new_cached_len = cached_len;
            cached.keys.insert_at(key, pos, &mut new_cached_len);
            cached.len = new_cached_len;
//            if middle_insertion {
//                cached.dump();
//            }
            return None;
        }
    }
    pub fn merge_with(&mut self, right: &mut Self) {
        let self_len = self.len;
        let new_len = self.len + right.len;
        assert!(new_len <= self.keys.len());
        for i in self.len .. new_len {
            self.keys[i] = mem::replace(&mut right.keys[i - self_len], Default::default());
        }
    }
    pub fn dump(&self) {
        debug!("Dumping {:?}, keys {}", self.id, self.len);
        for i in 0..NUM_KEYS {
            println!("{}\t- {:?}", i, self.keys[i]);
        }
    }
}

pub fn rearrange_empty_extnode(node: &ExtNode, bz: &CacheBufferZone) -> Id {
    let prev = node.prev;
    let next = node.next;
    if !prev.is_unit_id() {
        let mut prev_node = bz.get_for_mut(&prev);
        prev_node.next = next;
    }
    if !next.is_unit_id() {
        let mut next_node = bz.get_for_mut(&next);
        next_node.prev = prev;
    }
    return node.id;
}

#[derive(Clone)]
pub enum CacheGuardHolder {
    Read(ExtNodeCachedImmute),
    Write(ExtNodeCachedMut)
}

type NodeRcRefCell = Rc<RefCell<ExtNode>>;

pub type RcNodeRefData<'a> = OwningHandle<RcRef<RefCell<ExtNode>>, Ref<'a, ExtNode>>;
pub type RcNodeRefMut<'a> = OwningHandle<RcRef<RefCell<ExtNode>>, RefMut<'a, ExtNode>>;

pub enum RcNodeRef<'a> {
    Data(RcNodeRefData<'a>),
    Guard(CacheGuardHolder)
}

pub struct CacheBufferZone {
    mapper: Arc<ExtNodeCacheMap>,
    storage: Arc<AsyncClient>,
    guards: RefCell<BTreeMap<Id, CacheGuardHolder>>,
    data: RefCell<BTreeMap<Id, Option<NodeRcRefCell>>>
}

impl Deref for CacheGuardHolder {
    type Target = ExtNode;

    fn deref(&self) -> &'_ <Self as Deref>::Target {
        match self {
            &CacheGuardHolder::Read(ref g) => &*g,
            &CacheGuardHolder::Write(ref g) => &*g
        }
    }
}

impl <'a> Deref for RcNodeRef<'a> {
    type Target = ExtNode;

    fn deref(&self) -> &'_ <Self as Deref>::Target {
        match self {
            &RcNodeRef::Guard(ref g) => &*g,
            &RcNodeRef::Data(ref g) => &*g
        }
    }
}

impl CacheBufferZone {
    pub fn new(mapper: &Arc<ExtNodeCacheMap>, storage: &Arc<AsyncClient>) -> CacheBufferZone {
        CacheBufferZone {
            mapper: mapper.clone() ,
            storage: storage.clone(),
            guards: RefCell::new(BTreeMap::new()),
            data: RefCell::new(BTreeMap::new())
        }
    }

    pub fn get(&self, id: &Id) -> RcNodeRef {
        debug!("Getting cached fo readonly {:?}", id);
        {
            // read from data (from write)
            let data = self.data.borrow();
            match data.get(id) {
                Some(Some(ref data)) => {
                    let cell = data.clone();
                    let cell_ref = RcRef::new(cell);
                    let handle = OwningHandle::new(cell_ref);
                    return RcNodeRef::Data(handle);
                },
                Some(None) => panic!(), // deleted
                _ => {}
            }
        }
        {
            // read from guards
            let guard = self.guards.borrow();
            if let Some(holder) = guard.get(id) {
                return RcNodeRef::Guard(holder.clone());
            }
        }
        {
            debug!("Buffer zone inserting for readonly");
            let guard = self.get_ext_node_cached(id);
            let holder = CacheGuardHolder::Read(guard);
            let mut guards = self.guards.borrow_mut();
            guards.insert(*id, holder);
            debug!("Buffer zone now have data {}", guards.len());
        }
        return self.get(id);
    }

    pub fn get_for_mut(&self, id: &Id) -> RcNodeRefMut {
        debug!("Getting cached for mutation {:?}", id);
        {
            let data = self.data.borrow();
            match data.get(id) {
                Some(Some(ref extnode)) => {
                    let guards = self.guards.borrow();
                    if let CacheGuardHolder::Read(_) = guards.get(id).unwrap() {
                        panic!("Mutating readonly buffered cache");
                    }
                    let cell = extnode.clone();
                    let cell_ref = RcRef::new(cell);
                    let handle = OwningHandle::new_mut(cell_ref);
                    debug!("Got cached, len {}, cache data {}, cache guards {}", handle.len, data.len(), guards.len());
                    return handle
                },
                Some(None) => panic!(),
                _ => {}
            }
        }
        {
            debug!("Buffer zone inserting for mutation {:?}", id);
            let guard = self.get_mut_ext_node_cached(id);
            let mut data = self.data.borrow_mut();
            data.insert(*id, Some(Rc::new(RefCell::new((&*guard).clone()))));
            let holder = CacheGuardHolder::Write(guard);
            let mut guards = self.guards.borrow_mut();
            guards.insert(*id, holder);
            debug!("Buffer zone now have data {}, guards {}", data.len(), guards.len());
        }
        self.get_for_mut(id)
    }

    pub fn delete(&self, id: &Id) {
        let mut data = self.data.borrow_mut();
        data.insert(*id, None);
    }

    pub fn new_node(&self, node: ExtNode) {
        let mut mapper = self.mapper.lock();
        if mapper.get(&node.id).is_some() {
            panic!()
        } else {
            let node_id = node.id;
            let lock = RwLock::new(ExtNode::new(&node_id));
            let guard = CacheGuardHolder::Write(lock.write());
            let mut guards = self.guards.borrow_mut();
            let mut data = self.data.borrow_mut();
            debug!("new node in buffered cache: {:?}, was guards: {}, data: {}", node_id, guards.len(), data.len());
            mapper.insert(node_id, lock);
            guards.insert(node_id, guard);
            data.insert(node_id, Some(Rc::new(RefCell::new(node))));
        }
    }

    pub fn flush(&self) {
        let mut guards = self.guards.borrow_mut();
        let data = self.data.borrow();
        debug!("Flushing cache buffer, guards: {:?}, data: {:?}", guards.keys(), data.keys());
        for ((id, data),  (gid, mut holder)) in data.iter().zip(guards.iter_mut()) {
            assert_eq!(id, gid);
            if let Some(ref node_rc) = data {
                debug!("Flushing updating node: {:?}", id);
                if let &mut CacheGuardHolder::Write(ref mut guard) = holder {
                    let mut ext_node = (*(*node_rc).borrow()).clone();
                    debug!("flushing {:?} to cache with {} keys", ext_node.id, ext_node.len);
                    **guard = ext_node
                }
            } else {
                debug!("Flushing deleting node: {:?}", id);
                self.mapper.lock().remove(id);
                self.storage.remove_cell(*id).wait().unwrap();
            }
        }
    }

    pub fn get_guard(&self, id: &Id) -> Option<CacheGuardHolder> {
        if id.is_unit_id() {
            return None;
        }
        let mut guards = self.guards.borrow_mut();
        {
            let res = guards.get(id).map(|rg| rg.clone());
            if res.is_some() { return res; }
        }
        let guard = self.get_ext_node_cached(id);
        let holder = CacheGuardHolder::Read(guard);
        guards.insert(*id, holder.clone());
        return Some(holder)
    }

    pub fn get_mut_ext_node_cached(&self, id: &Id) -> ExtNodeCachedMut {
        let mut map = self.mapper.lock();
        return map.get_or_fetch(id).unwrap().write();
    }
    pub fn get_ext_node_cached(&self, id: &Id) -> ExtNodeCachedImmute {
        let mut map = self.mapper.lock();
        return map.get_or_fetch(id).unwrap().read();
    }
}

pub fn page_schema() -> Schema {
    Schema {
        id: *PAGE_SCHEMA_ID,
        name: String::from(PAGE_SCHEMA),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new(
            "*", 0, false, false,
            Some(vec![
                Field::new(NEXT_FIELD, type_id_of(Type::Id), false, false, None),
                Field::new(PREV_FIELD, type_id_of(Type::Id), false, false, None),
                Field::new(KEYS_FIELD, type_id_of(Type::U8), false, true, None)
            ])
        )
    }
}
<|endoftext|>