<commit_before>use unicode_width::{UnicodeWidthStr,UnicodeWidthChar};
use super::Renderer;

/// State corresponding to a partially constructed line.
struct PartialLine {
    text: String,
    /// The width in character cells of the text, or
    /// current position.
    pos: usize,
}

impl PartialLine {
    pub fn new() -> PartialLine {
        PartialLine {
            text: String::new(),
            pos: 0,
        }
    }
}

/// A renderer which just outputs plain text.
pub struct TextRenderer {
    width: usize,
    lines: Vec<String>,
    /// True at the end of a block, meaning we should add
    /// a blank line if any other text is added.
    at_block_end: bool,
    partial_line: Option<PartialLine>,
    /// Set to true when adding text (eg markup) that doesn't require
    /// a space before the next word.
    no_space_needed: bool,
    links: Vec<String>,
}

impl TextRenderer {
    /// Construct a new empty TextRenderer.
    pub fn new(width: usize) -> TextRenderer {
        html_trace!("new({})", width);
        TextRenderer {
            width: width,
            lines: Vec::new(),
            at_block_end: false,
            partial_line: None,
            links: Vec::new(),
            no_space_needed: false,
        }
    }

    /// Take the partial line out of self, or create one
    /// if there wasn't one.
    fn take_partial_line(&mut self) -> PartialLine {
        if let Some(p) = self.partial_line.take() {
            p
        } else {
            PartialLine::new()
        }
    }

    pub fn add_subblock(&mut self, s: &str) {
        html_trace!("add_subblock({}, {})", self.width, s);
        self.lines.extend(s.lines().map(|l| l.into()));
    }

    fn push_line(&mut self, s: String) {
        self.lines.push(s);
    }
    fn flush_line(&mut self) {
        if let Some(s) = self.partial_line.take() {
            if s.text.len() > 0 {
                self.push_line(s.text);
            }
        }
    }

    pub fn into_string(mut self) -> String {
        let mut result = String::new();
        for s in self.into_lines() {
            result.push_str(&s);
            result.push('\n');
        }
        html_trace!("into_string({}, {:?})", self.width, result);
        result
    }

    fn into_lines(mut self) -> Vec<String> {
        self.flush_line();
        // And add the links
        if self.links.len() > 0 {
            self.start_block();
            for (idx, target) in self.links.drain((0..)).enumerate() {
                self.lines.push(format!("[{}] {}", idx+1, target));
            }
        }
        self.lines
    }
}

impl Renderer for TextRenderer {
    type Sub = TextRenderer;

    fn add_empty_line(&mut self) {
        html_trace!("add_empty_line()");
        self.flush_line();
        self.lines.push("".into());
        html_trace_quiet!("add_empty_line: at_block_end <- false");
        self.at_block_end = false;
        html_trace_quiet!("add_empty_line: new lines: {:?}", self.lines);
    }

    fn new_sub_renderer(&self, width: usize) -> Self::Sub {
        TextRenderer::new(width)
    }

    fn start_block(&mut self) {
        html_trace!("start_block({})", self.width);
        self.flush_line();
        if self.lines.len() > 0 {
            self.add_empty_line();
        }
        html_trace_quiet!("start_block; at_block_end <- false");
        self.at_block_end = false;
    }

    fn add_preformatted_block(&mut self, text: &str) {
        html_trace!("add_block({}, {})", self.width, text);
        self.start_block();
        self.add_subblock(text);
        html_trace_quiet!("add_block: at_block_end <- true");
        self.at_block_end = true;
    }

    fn end_block(&mut self) {
        self.at_block_end = true;
    }

    fn add_inline_text(&mut self, text: &str) {
        html_trace!("add_inline_text({}, {})", self.width, text);
        let mut partial = self.take_partial_line();
        for word in text.split_whitespace() {
            let mut need_space = (partial.pos > 0) && !(self.no_space_needed);
            let space_space = if need_space { 1 } else { 0 };
            // We've used up the "no space needed" condition now.
            self.no_space_needed = false;
            if self.width <= (partial.pos + space_space) {
                self.push_line(partial.text);
                partial = self.take_partial_line();
                need_space = false;
            }
            let space_left = self.width - partial.pos - space_space;
            let word_width = UnicodeWidthStr::width(word);
            if word_width <= space_left {
                /* It fits; no problem.  Add a space if not at the
                 * start of line.*/
                if need_space {
                    partial.text.push(' ');
                    partial.pos += 1;
                }
                partial.text.push_str(word);
                partial.pos += word_width;
                continue;
            }

            /* It doesn't fit.  If we're not at the start of the line,
             * then go to a new line. */
            if partial.pos > 0 {
                self.push_line(partial.text);
                partial = self.take_partial_line();
            }

            /* We're now at the start of a line. */
            if word_width > self.width {
                /* It doesn't fit at all on the line, so break it. */
                for c in word.chars() {
                    let c_width = UnicodeWidthChar::width(c).unwrap();
                    if c_width + partial.pos > self.width {
                        /* Break here */
                        self.push_line(partial.text);
                        partial = self.take_partial_line();
                    }
                    /* This might happen with really narrow spaces... */
                    assert!(c_width <= self.width);

                    partial.text.push(c);
                    partial.pos += c_width;
                }
            } else {
                partial.text.push_str(word);
                partial.pos += word_width;
            }
        }
        self.partial_line = Some(partial);
    }

    fn width(&self) -> usize {
        self.width
    }

    fn add_block_line(&mut self, line: &str)
    {
        self.add_subblock(line);
    }

    fn append_subrender<'a, I>(&mut self, other: Self::Sub,
                               prefixes: I)
                           where I:Iterator<Item=&'a str>
    {
        self.flush_line();
        self.lines.extend(other.into_lines()
                               .into_iter()
                               .zip(prefixes)
                               .map(|(line, prefix)| prefix.to_string() + &line));
    }

    fn append_columns<I>(&mut self, cols: I, separator: &str)
                           where I:IntoIterator<Item=Self::Sub> {
        self.flush_line();

        let line_sets = cols.into_iter()
                            .map(|sub_r| {
                                let width = sub_r.width;
                                (width, sub_r.into_lines()
                                             .into_iter()
                                             .map(|line| format!("{: <width$}", line, width=width))
                                             .collect())
                                 })
                            .collect::<Vec<(usize, Vec<String>)>>();

        let cell_height = line_sets.iter()
                                   .map(|&(_, ref v)| v.len())
                                   .max().unwrap_or(0);
        let spaces:String = (0..self.width).map(|_| ' ').collect();
        for i in 0..cell_height {
            let mut line = String::new();
            for (cellno, &(width, ref ls)) in line_sets.iter().enumerate() {
                let piece = ls.get(i).map(|s| s.as_str()).unwrap_or(&spaces[0..width]);
                line.push_str(piece);
                if cellno != line_sets.len()-1 {
                    line.push('|')
                }
            }
            self.add_block_line(&line);
        }
    }

    fn empty(&self) -> bool {
        self.lines.len() == 0 && self.partial_line.is_none()
    }

    fn text_len(&self) -> usize {
        let mut result = 0;
        for line in &self.lines {
            result += UnicodeWidthStr::width(line.as_str());
        }
        if let Some(ref line) = self.partial_line {
            result += line.pos;
        }
        result
    }

    fn start_link(&mut self, target: &str)
    {
        self.links.push(target.to_string());
        self.add_inline_text("[");
        self.no_space_needed = true;
    }
    fn end_link(&mut self)
    {
        let idx = self.links.len();
        self.no_space_needed = true;
        self.add_inline_text(&format!("][{}]", idx));
    }
}<commit_msg>Wrap link URLS.<commit_after>use unicode_width::{UnicodeWidthStr,UnicodeWidthChar};
use super::Renderer;
use std::mem;

/// State corresponding to a partially constructed line.
struct PartialLine {
    text: String,
    /// The width in character cells of the text, or
    /// current position.
    pos: usize,
}

impl PartialLine {
    pub fn new() -> PartialLine {
        PartialLine {
            text: String::new(),
            pos: 0,
        }
    }
}

/// A renderer which just outputs plain text.
pub struct TextRenderer {
    width: usize,
    lines: Vec<String>,
    /// True at the end of a block, meaning we should add
    /// a blank line if any other text is added.
    at_block_end: bool,
    partial_line: Option<PartialLine>,
    /// Set to true when adding text (eg markup) that doesn't require
    /// a space before the next word.
    no_space_needed: bool,
    links: Vec<String>,
}

impl TextRenderer {
    /// Construct a new empty TextRenderer.
    pub fn new(width: usize) -> TextRenderer {
        html_trace!("new({})", width);
        TextRenderer {
            width: width,
            lines: Vec::new(),
            at_block_end: false,
            partial_line: None,
            links: Vec::new(),
            no_space_needed: false,
        }
    }

    /// Take the partial line out of self, or create one
    /// if there wasn't one.
    fn take_partial_line(&mut self) -> PartialLine {
        if let Some(p) = self.partial_line.take() {
            p
        } else {
            PartialLine::new()
        }
    }

    pub fn add_subblock(&mut self, s: &str) {
        html_trace!("add_subblock({}, {})", self.width, s);
        self.lines.extend(s.lines().map(|l| l.into()));
    }

    fn push_line(&mut self, s: String) {
        self.lines.push(s);
    }
    fn flush_line(&mut self) {
        if let Some(s) = self.partial_line.take() {
            if s.text.len() > 0 {
                self.push_line(s.text);
            }
        }
    }

    pub fn into_string(mut self) -> String {
        let mut result = String::new();
        for s in self.into_lines() {
            result.push_str(&s);
            result.push('\n');
        }
        html_trace!("into_string({}, {:?})", self.width, result);
        result
    }

    fn into_lines(mut self) -> Vec<String> {
        self.flush_line();
        // And add the links
        if self.links.len() > 0 {
            self.start_block();
            for (idx, target) in self.links.drain((0..)).enumerate() {
                /* Hard wrap */
                let mut output = String::new();
                let mut pos = 0;
                for c in format!("[{}] {}", idx+1, target).chars() {
                    let c_width = UnicodeWidthChar::width(c).unwrap();
                    if pos + c_width > self.width {
                        let mut tmp_s = String::new();
                        mem::swap(&mut output, &mut tmp_s);
                        self.lines.push(tmp_s);
                        output.push(c);
                        pos = c_width;
                    } else {
                        output.push(c);
                        pos += c_width;
                    }
                }
                self.lines.push(output);
            }
        }
        self.lines
    }
}

impl Renderer for TextRenderer {
    type Sub = TextRenderer;

    fn add_empty_line(&mut self) {
        html_trace!("add_empty_line()");
        self.flush_line();
        self.lines.push("".into());
        html_trace_quiet!("add_empty_line: at_block_end <- false");
        self.at_block_end = false;
        html_trace_quiet!("add_empty_line: new lines: {:?}", self.lines);
    }

    fn new_sub_renderer(&self, width: usize) -> Self::Sub {
        TextRenderer::new(width)
    }

    fn start_block(&mut self) {
        html_trace!("start_block({})", self.width);
        self.flush_line();
        if self.lines.len() > 0 {
            self.add_empty_line();
        }
        html_trace_quiet!("start_block; at_block_end <- false");
        self.at_block_end = false;
    }

    fn add_preformatted_block(&mut self, text: &str) {
        html_trace!("add_block({}, {})", self.width, text);
        self.start_block();
        self.add_subblock(text);
        html_trace_quiet!("add_block: at_block_end <- true");
        self.at_block_end = true;
    }

    fn end_block(&mut self) {
        self.at_block_end = true;
    }

    fn add_inline_text(&mut self, text: &str) {
        html_trace!("add_inline_text({}, {})", self.width, text);
        let mut partial = self.take_partial_line();
        for word in text.split_whitespace() {
            let mut need_space = (partial.pos > 0) && !(self.no_space_needed);
            let space_space = if need_space { 1 } else { 0 };
            // We've used up the "no space needed" condition now.
            self.no_space_needed = false;
            if self.width <= (partial.pos + space_space) {
                self.push_line(partial.text);
                partial = self.take_partial_line();
                need_space = false;
            }
            let space_left = self.width - partial.pos - space_space;
            let word_width = UnicodeWidthStr::width(word);
            if word_width <= space_left {
                /* It fits; no problem.  Add a space if not at the
                 * start of line.*/
                if need_space {
                    partial.text.push(' ');
                    partial.pos += 1;
                }
                partial.text.push_str(word);
                partial.pos += word_width;
                continue;
            }

            /* It doesn't fit.  If we're not at the start of the line,
             * then go to a new line. */
            if partial.pos > 0 {
                self.push_line(partial.text);
                partial = self.take_partial_line();
            }

            /* We're now at the start of a line. */
            if word_width > self.width {
                /* It doesn't fit at all on the line, so break it. */
                for c in word.chars() {
                    let c_width = UnicodeWidthChar::width(c).unwrap();
                    if c_width + partial.pos > self.width {
                        /* Break here */
                        self.push_line(partial.text);
                        partial = self.take_partial_line();
                    }
                    /* This might happen with really narrow spaces... */
                    assert!(c_width <= self.width);

                    partial.text.push(c);
                    partial.pos += c_width;
                }
            } else {
                partial.text.push_str(word);
                partial.pos += word_width;
            }
        }
        self.partial_line = Some(partial);
    }

    fn width(&self) -> usize {
        self.width
    }

    fn add_block_line(&mut self, line: &str)
    {
        self.add_subblock(line);
    }

    fn append_subrender<'a, I>(&mut self, other: Self::Sub,
                               prefixes: I)
                           where I:Iterator<Item=&'a str>
    {
        self.flush_line();
        self.lines.extend(other.into_lines()
                               .into_iter()
                               .zip(prefixes)
                               .map(|(line, prefix)| prefix.to_string() + &line));
    }

    fn append_columns<I>(&mut self, cols: I, separator: &str)
                           where I:IntoIterator<Item=Self::Sub> {
        self.flush_line();

        let line_sets = cols.into_iter()
                            .map(|sub_r| {
                                let width = sub_r.width;
                                (width, sub_r.into_lines()
                                             .into_iter()
                                             .map(|line| format!("{: <width$}", line, width=width))
                                             .collect())
                                 })
                            .collect::<Vec<(usize, Vec<String>)>>();

        let cell_height = line_sets.iter()
                                   .map(|&(_, ref v)| v.len())
                                   .max().unwrap_or(0);
        let spaces:String = (0..self.width).map(|_| ' ').collect();
        for i in 0..cell_height {
            let mut line = String::new();
            for (cellno, &(width, ref ls)) in line_sets.iter().enumerate() {
                let piece = ls.get(i).map(|s| s.as_str()).unwrap_or(&spaces[0..width]);
                line.push_str(piece);
                if cellno != line_sets.len()-1 {
                    line.push('|')
                }
            }
            self.add_block_line(&line);
        }
    }

    fn empty(&self) -> bool {
        self.lines.len() == 0 && self.partial_line.is_none()
    }

    fn text_len(&self) -> usize {
        let mut result = 0;
        for line in &self.lines {
            result += UnicodeWidthStr::width(line.as_str());
        }
        if let Some(ref line) = self.partial_line {
            result += line.pos;
        }
        result
    }

    fn start_link(&mut self, target: &str)
    {
        self.links.push(target.to_string());
        self.add_inline_text("[");
        self.no_space_needed = true;
    }
    fn end_link(&mut self)
    {
        let idx = self.links.len();
        self.no_space_needed = true;
        self.add_inline_text(&format!("][{}]", idx));
    }
}<|endoftext|>


<commit_before>use std::io::{FileAccess, FileMode, Open, Read, Truncate};
use std::io::{ReadWrite};
use std::io::{SeekEnd, SeekSet};
use std::io::fs::File;
use std::mem::size_of;
use std::io::Decorator;

static version: u8 = 1;

// some definitions from kissdb.h

struct Kissdb {
    hash_table_size : u64,
    key_size : u64,
    value_size : u64,
    num_hash_tables : u64,
    hash_tables : ~[u64],
    f : std::io::fs::File
}

enum Error {
  ErrorIO,
  ErrorMalloc,
  InvalidParameters,
  CorruptDbFile
}

enum OpenMode {
  ReadOnly,
  RW,
  RWCreate,
  RWReplace
}

fn kissdb_hash(b : &~[u8]) -> u64
{
    let mut hash : u64 = 5381;
    for &bx in b.iter() {
        hash = ((hash << 5) + hash) + (bx as u64);
    }
    return hash
}

// either
// 1. use the c95 based alloc, realloc and free
// 2. use more rustic something
// maybe trying 2. gives me more knowledge
fn kissdb_open(
    path : &Path,
    orig_mode : OpenMode,
    mut hash_table_size : u64,
    mut key_size : u64,
    mut value_size : u64
    ) -> Option<~ Kissdb>
{
    let (mode, access) : (FileMode, FileAccess) =
        match orig_mode {
            ReadOnly =>  (Open, Read), // rb
            RW => (Open, ReadWrite), // r+b
            RWCreate =>  (Open, ReadWrite), // r+b
            RWReplace => (Truncate, ReadWrite) // w+b
        };
    let mut f_ : Option<File>
        = std::io::fs::File::open_mode(path, mode, access);

    if f_.is_none() {
        match (orig_mode) {
            RWCreate =>
            f_ = std::io::fs::File::open_mode(path, Truncate, ReadWrite), // w+b
            _ => f_ = f_
        }
    };

    match f_ {
        None => return None,
        Some (mut f) =>
        {let kissdb_header_size : u64 = ((size_of::<u64>() * 3) + 4) as u64;

         f.seek(0, SeekEnd);
         if (f.tell() < kissdb_header_size) {
             if hash_table_size > 0 && key_size > 0
                 && value_size >0
             {
                 f.seek(0, SeekSet);
                 let tmp2 : [u8, ..4] = ['K' as u8, 'd' as u8, 'R' as u8, version];
                 f.write(tmp2);
                 f.write_le_u64(hash_table_size);
                 f.write_le_u64(key_size);
                 f.write_le_u64(value_size);
                 f.flush();
             }
             else {
                 fail!()
             }
         }
         else {
             f.seek(0, SeekSet);
             let mut tmp2 : [u8, ..4] = [0,0,0,0];
             if f.read(tmp2) != Some(4) { fail! (); }
             if tmp2 != ['K' as u8, 'd' as u8, 'R' as u8, version] {fail! ();}
             hash_table_size = f.read_le_u64();
             key_size = f.read_le_u64();
             value_size = f.read_le_u64();
         }

         // let hash_table_size_bytes : uint = size_of::<u64>() * (hash_table_size + 1) as uint;
         // let mut httmp = std::vec::from_elem(hash_table_size_bytes, 0 as u8);
         //         let httmp = do std::at_vec::build::<u64>(None) |push| { for i in range(0, hash_table_size + 1)
//                  { push (0); }
        //};

         let mut num_hash_tables = 0;

         // should use at_vec, actually
         // no, std::vec is OK.  It has appends
         let mut hash_tables : ~[u64] = std::vec::from_elem(0, 0 as u64);
         let mut h : Option<~[u64]>;

         // do we do realloc in rust? with append, apparently.
         while({h = f.read_one_hash_table(hash_table_size); h.is_some()})
         {
             let httmp = h.unwrap();
             hash_tables = std::vec::append(hash_tables, httmp);
             num_hash_tables = num_hash_tables + 1;
             if (httmp[hash_table_size] > 0) {
                 f.seek(httmp[hash_table_size] as i64, SeekSet)
             }
             else {
                 break;
             }
         }

         debug!("key_size = {:s}", key_size.to_str());

         // construct h somehow
         return Some(~Kissdb {hash_table_size : hash_table_size,
                              key_size : key_size,
                              value_size : value_size,
                              num_hash_tables : num_hash_tables,
                              hash_tables : hash_tables,
                              f : f})
        }
    }
}

trait File2 {
    fn read_one_hash_table(&mut self, u64) -> Option<~[u64]>;
    fn write_one_hash_table(&mut self, &[u64]);
}

impl File2 for File {
    fn read_one_hash_table(&mut self,
                           hash_table_size : u64) -> Option<~[u64]> {
        let bs = (hash_table_size + 1) as uint * size_of::<u64>();
        let mut buf = std::vec::from_elem(bs, 0 as u8);
        let mut result = std::vec::from_elem((hash_table_size + 1) as uint, 0 as u64);
        match self.read(buf) {
            None => None,
            Some(buf_size) if buf_size == bs => {
                let mut br = std::io::mem::BufReader::new(buf);
                for i in range(0, hash_table_size + 1) {
                    result[i] = br.read_le_u64();
                }
                Some((result))
            },
            _ => None
        }
    }
    fn write_one_hash_table(&mut self,
                            ht : &[u64]) {
        for &x in ht.iter() {
            self.write_le_u64(x);
        }
    }
}

fn kissdb_close(~db : ~Kissdb) {
    //throwing away
}

trait Kdb {
    fn kissdb_get(&mut self, &~[u8]) -> Option<~[u8]>;
    fn kissdb_put(&mut self, key : &~[u8], value : &[u8]) -> bool;
}

impl Kdb for Kissdb {
    fn kissdb_get(&mut self, key : &~[u8]) -> Option<~[u8]> {
        if key.len() != self.key_size as uint { return None }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let hash_tables = &self.hash_tables;
        for i in range(0, self.num_hash_tables) {
            let offset : i64 = hash_tables[i * (self.hash_table_size + 1) + hash] as i64;
            if (offset == 0) { return None }; // yeah, 0 is being special. never used normally.
            self.f.seek(offset as i64, SeekSet);

            let klen = self.key_size as uint;
            // isn't compare available?? read_bytes available
            let tmp = self.f.read_bytes(klen);

            if std::vec::bytes::memcmp(key, &tmp) == 0 {
                return Some(self.f.read_bytes(self.value_size as uint))
            }
        }
        return None
    }

    fn kissdb_put(&mut self, key : &~[u8], value : &[u8]) -> bool
    {
        if key.len() != self.key_size as uint { fail!() }
        if value.len() != self.value_size as uint { fail!() }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let mut lasthtoffset : u64 = ((size_of::<u64>() * 3) + 4) as u64;
        let mut htoffset = lasthtoffset;

        for i in range(0, self.num_hash_tables) {
            let offset : i64 = self.hash_tables[(self.hash_table_size + 1) * i + hash] as i64;
		    if (offset != 0) { // yes, 0 is treated special. 0 will never used normally
			    /* rewrite if already exists */
                self.f.seek(offset, SeekSet);

			    let klen = self.key_size as uint;
                let tmp = self.f.read_bytes(klen);
                if std::vec::bytes::memcmp(key, &tmp) == 0 {
                    self.f.write(value);
				    self.f.flush();
				    return true; /* success */
                }
                // put_no_match_next_hash_table:
        	    lasthtoffset = htoffset;
		        htoffset = self.hash_tables[(self.hash_table_size + 1) * i + self.hash_table_size];
                //cmp != , should be put_no_match_next_hash_table
		    } else {
			    /* add if an empty hash table slot is discovered */
                self.f.seek(0, SeekEnd);
	            let endoffset : u64 = self.f.tell();

                // debug!("endoffset = {:s}",endoffset.to_str());
                self.f.write(*key);
                self.f.write(value);

                self.f.seek(htoffset as i64 + (size_of::<u64>() as i64 * hash as i64), SeekSet);
                self.f.write_le_u64(endoffset);
			    self.hash_tables[(self.hash_table_size + 1) * i + hash] = endoffset;

			    self.f.flush();

			    return true; /* success */
		    }
	    }

	    /* if no existing slots, add a new page of hash table entries */
        debug!("add new page");
        self.f.seek(0, SeekEnd);
	    let endoffset = self.f.tell();

        let mut new_table : ~[u64] = std::vec::from_elem(self.hash_table_size as uint + 1, 0 as u64);
        let hash_table_size_bytes : u64 = size_of::<u64>() as u64 * (self.hash_table_size + 1);
        new_table[hash] = endoffset + hash_table_size_bytes; // where key, value are to be written
        self.f.write_one_hash_table(new_table);
        let longer = std::vec::append(self.hash_tables.clone(), new_table); // slow!!
        self.hash_tables = longer;

        debug!("writing at {:s}",self.f.tell().to_str());
        self.f.write(*key);
        self.f.write(value);

        // what is happening here??, aha, keep track to find this bit.
	    if (self.num_hash_tables > 0) {
            self.f.seek(lasthtoffset as i64 + (size_of::<u64>() as i64 * self.hash_table_size as i64), SeekSet);
            self.f.write_le_u64(endoffset);
            // mimick rereading
            self.hash_tables[((self.hash_table_size + 1) * (self.num_hash_tables - 1)) + self.hash_table_size] = endoffset;
	    }

        self.num_hash_tables = self.num_hash_tables + 1;
        self.f.flush();
	    return true; /* success */
    }

}

fn main()
{
    println("Opening new empty database test.db...");
    let mut db = kissdb_open(&Path::new("test.db"), RWReplace, 1024, 8, size_of::<u64>() as u64).unwrap();

    println("Adding and then re-getting 10000 64-byte values...");

    let mut v : [u8, .. 8] = [0,0,0,0,0,0,0,0];

    for i in range(0, 10000) {
        for j in range(0, 8) {
            v[j] = i as u8;
        }
        let mut key = std::io::mem::MemWriter::new();
        key.write_le_u64(i as u64);
        db.kissdb_put(key.inner_ref(), v);
        let gotten = db.kissdb_get(key.inner_ref()).unwrap();
        for j in range(0, 8) {
            if gotten[j] != i as u8 { fail!() }
        }
    }

    println("Getting 10000 64-byte values...");

    for i in range(0, 10000) {
        let mut key = std::io::mem::MemWriter::new();
        key.write_le_u64(i as u64);
        let gotten = db.kissdb_get(key.inner_ref()).unwrap();
        for j in range(0,8) {
            if gotten[j] != i as u8 { fail!() }
        }
    }

    println("closing and re-opening database in read-only mode...");

    kissdb_close(db);

    let mut db = kissdb_open(&Path::new("test.db"), ReadOnly, 1024, 8, size_of::<u64>() as u64).unwrap();

    println("Getting 10000 64-byte values...");

    for i in range(0, 10000) {
        let mut key = std::io::mem::MemWriter::new();
        key.write_le_u64(i as u64);
        let gotten = db.kissdb_get(key.inner_ref()).unwrap();
        for j in range(0,8) {
            if gotten[j] != i as u8 { fail!() }
        }
    }

	println("Iterator not implemented ...");

    kissdb_close(db);

    println("All tests OK!");
}
<commit_msg>some refactoring<commit_after>use std::io::{FileAccess, FileMode, Open, Read, Truncate};
use std::io::{ReadWrite};
use std::io::{SeekEnd, SeekSet};
use std::io::fs::File;
use std::mem::size_of;
use std::io::Decorator;

static version: u8 = 1;

// some definitions from kissdb.h

struct Kissdb {
    hash_table_size : u64,
    key_size : u64,
    value_size : u64,
    num_hash_tables : u64,
    hash_tables : ~[u64],
    f : std::io::fs::File
}

enum OpenMode {
  ReadOnly,
  RW,
  RWCreate,
  RWReplace
}

fn kissdb_hash(b : &~[u8]) -> u64
{
    let mut hash : u64 = 5381;
    for &bx in b.iter() {
        hash = ((hash << 5) + hash) + (bx as u64);
    }
    return hash
}

fn kissdb_open(
    path : &Path,
    orig_mode : OpenMode,
    mut hash_table_size : u64,
    mut key_size : u64,
    mut value_size : u64
    ) -> Option<~ Kissdb>
{
    let (mode, access) : (FileMode, FileAccess) =
        match orig_mode {
            ReadOnly =>  (Open, Read), // rb
            RW => (Open, ReadWrite), // r+b
            RWCreate =>  (Open, ReadWrite), // r+b
            RWReplace => (Truncate, ReadWrite) // w+b
        };
    let mut f_ : Option<File>
        = std::io::fs::File::open_mode(path, mode, access);

    if f_.is_none() {
        match (orig_mode) {
            RWCreate =>
            f_ = std::io::fs::File::open_mode(path, Truncate, ReadWrite), // w+b
            _ => f_ = f_
        }
    };

    let mut f =
        match f_ {
            None => return None,
            Some (f) => f
        };
    let kissdb_header_size : u64 = ((size_of::<u64>() * 3) + 4) as u64;

    f.seek(0, SeekEnd);
    if (f.tell() < kissdb_header_size) {
        if hash_table_size > 0 && key_size > 0
            && value_size >0
        {
            f.seek(0, SeekSet);
            let tmp2 : [u8, ..4] = ['K' as u8, 'd' as u8, 'R' as u8, version];
            f.write(tmp2);
            f.write_le_u64(hash_table_size);
            f.write_le_u64(key_size);
            f.write_le_u64(value_size);
            f.flush();
        }
        else {
            fail!()
        }
    }
    else {
        f.seek(0, SeekSet);
        let mut tmp2 : [u8, ..4] = [0,0,0,0];
        if f.read(tmp2) != Some(4) { fail! (); }
        if tmp2 != ['K' as u8, 'd' as u8, 'R' as u8, version] {fail! ();}
        hash_table_size = f.read_le_u64();
        key_size = f.read_le_u64();
        value_size = f.read_le_u64();
    }

    let mut num_hash_tables = 0;

    let mut hash_tables : ~[u64] = std::vec::from_elem(0, 0 as u64);
    let mut h : Option<~[u64]>;

    // do we do realloc in rust? with append, apparently.
    while({h = f.read_one_hash_table(hash_table_size); h.is_some()})
    {
        let httmp = h.unwrap();
        hash_tables = std::vec::append(hash_tables, httmp);
        num_hash_tables = num_hash_tables + 1;
        if (httmp[hash_table_size] > 0) {
            f.seek(httmp[hash_table_size] as i64, SeekSet)
        }
        else {
            break;
        }
    }

    debug!("key_size = {:s}", key_size.to_str());

    return Some(~Kissdb {hash_table_size : hash_table_size,
                         key_size : key_size,
                         value_size : value_size,
                         num_hash_tables : num_hash_tables,
                         hash_tables : hash_tables,
                         f : f})
}

trait File2 {
    fn read_one_hash_table(&mut self, u64) -> Option<~[u64]>;
    fn write_one_hash_table(&mut self, &[u64]);
}

impl File2 for File {
    fn read_one_hash_table(&mut self,
                           hash_table_size : u64) -> Option<~[u64]> {
        let bs = (hash_table_size + 1) as uint * size_of::<u64>();
        let mut buf = std::vec::from_elem(bs, 0 as u8);
        let mut result = std::vec::from_elem((hash_table_size + 1) as uint, 0 as u64);
        match self.read(buf) {
            None => None,
            Some(buf_size) if buf_size == bs => {
                let mut br = std::io::mem::BufReader::new(buf);
                for i in range(0, hash_table_size + 1) {
                    result[i] = br.read_le_u64();
                }
                Some((result))
            },
            _ => None
        }
    }
    fn write_one_hash_table(&mut self,
                            ht : &[u64]) {
        for &x in ht.iter() {
            self.write_le_u64(x);
        }
    }
}

fn kissdb_close(~db : ~Kissdb) {
    //throwing away
}

trait Kdb {
    fn kissdb_get(&mut self, &~[u8]) -> Option<~[u8]>;
    fn kissdb_put(&mut self, key : &~[u8], value : &[u8]) -> bool;
}

impl Kdb for Kissdb {
    fn kissdb_get(&mut self, key : &~[u8]) -> Option<~[u8]> {
        if key.len() != self.key_size as uint { return None }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let hash_tables = &self.hash_tables;
        for i in range(0, self.num_hash_tables) {
            let offset : i64 = hash_tables[i * (self.hash_table_size + 1) + hash] as i64;
            if (offset == 0) { return None }; // yeah, 0 is being special. never used normally.
            self.f.seek(offset as i64, SeekSet);

            let klen = self.key_size as uint;
            // isn't compare available?? read_bytes available
            let tmp = self.f.read_bytes(klen);

            if std::vec::bytes::memcmp(key, &tmp) == 0 {
                return Some(self.f.read_bytes(self.value_size as uint))
            }
        }
        return None
    }

    fn kissdb_put(&mut self, key : &~[u8], value : &[u8]) -> bool
    {
        if key.len() != self.key_size as uint { fail!() }
        if value.len() != self.value_size as uint { fail!() }
        let hash = kissdb_hash(key) % self.hash_table_size;
        let mut lasthtoffset : u64 = ((size_of::<u64>() * 3) + 4) as u64;
        let mut htoffset = lasthtoffset;

        for i in range(0, self.num_hash_tables) {
            let offset : i64 = self.hash_tables[(self.hash_table_size + 1) * i + hash] as i64;
		    if (offset != 0) { // yes, 0 is treated special. 0 will never used normally
			    /* rewrite if already exists */
                self.f.seek(offset, SeekSet);

			    let klen = self.key_size as uint;
                let tmp = self.f.read_bytes(klen);
                if std::vec::bytes::memcmp(key, &tmp) == 0 {
                    self.f.write(value);
				    self.f.flush();
				    return true; /* success */
                }
                // put_no_match_next_hash_table:
        	    lasthtoffset = htoffset;
		        htoffset = self.hash_tables[(self.hash_table_size + 1) * i + self.hash_table_size];
                //cmp != , should be put_no_match_next_hash_table
		    } else {
			    /* add if an empty hash table slot is discovered */
                self.f.seek(0, SeekEnd);
	            let endoffset : u64 = self.f.tell();

                // debug!("endoffset = {:s}",endoffset.to_str());
                self.f.write(*key);
                self.f.write(value);

                self.f.seek(htoffset as i64 + (size_of::<u64>() as i64 * hash as i64), SeekSet);
                self.f.write_le_u64(endoffset);
			    self.hash_tables[(self.hash_table_size + 1) * i + hash] = endoffset;

			    self.f.flush();

			    return true; /* success */
		    }
	    }

	    /* if no existing slots, add a new page of hash table entries */
        debug!("add new page");
        self.f.seek(0, SeekEnd);
	    let endoffset = self.f.tell();

        let mut new_table : ~[u64] = std::vec::from_elem(self.hash_table_size as uint + 1, 0 as u64);
        let hash_table_size_bytes : u64 = size_of::<u64>() as u64 * (self.hash_table_size + 1);
        new_table[hash] = endoffset + hash_table_size_bytes; // where key, value are to be written
        self.f.write_one_hash_table(new_table);
        let longer = std::vec::append(self.hash_tables.clone(), new_table); // slow!!
        self.hash_tables = longer;

        debug!("writing at {:s}",self.f.tell().to_str());
        self.f.write(*key);
        self.f.write(value);

	    if (self.num_hash_tables > 0) {
            self.f.seek(lasthtoffset as i64 + (size_of::<u64>() as i64 * self.hash_table_size as i64), SeekSet);
            self.f.write_le_u64(endoffset);
            // mimick rereading
            self.hash_tables[((self.hash_table_size + 1) * (self.num_hash_tables - 1)) + self.hash_table_size] = endoffset;
	    }

        self.num_hash_tables = self.num_hash_tables + 1;
        self.f.flush();
	    return true; /* success */
    }

}

fn main()
{
    println("Opening new empty database test.db...");
    let mut db = kissdb_open(&Path::new("test.db"), RWReplace, 1024, 8, size_of::<u64>() as u64).unwrap();

    println("Adding and then re-getting 10000 64-byte values...");

    let mut v : [u8, .. 8] = [0,0,0,0,0,0,0,0];

    for i in range(0, 10000) {
        for j in range(0, 8) {
            v[j] = i as u8;
        }
        let mut key = std::io::mem::MemWriter::new();
        key.write_le_u64(i as u64);
        db.kissdb_put(key.inner_ref(), v);
        let gotten = db.kissdb_get(key.inner_ref()).unwrap();
        for j in range(0, 8) {
            if gotten[j] != i as u8 { fail!() }
        }
    }

    println("Getting 10000 64-byte values...");

    for i in range(0, 10000) {
        let mut key = std::io::mem::MemWriter::new();
        key.write_le_u64(i as u64);
        let gotten = db.kissdb_get(key.inner_ref()).unwrap();
        for j in range(0,8) {
            if gotten[j] != i as u8 { fail!() }
        }
    }

    println("closing and re-opening database in read-only mode...");

    kissdb_close(db);

    let mut db = kissdb_open(&Path::new("test.db"), ReadOnly, 1024, 8, size_of::<u64>() as u64).unwrap();

    println("Getting 10000 64-byte values...");

    for i in range(0, 10000) {
        let mut key = std::io::mem::MemWriter::new();
        key.write_le_u64(i as u64);
        let gotten = db.kissdb_get(key.inner_ref()).unwrap();
        for j in range(0,8) {
            if gotten[j] != i as u8 { fail!() }
        }
    }

	println("Iterator not implemented ...");

    kissdb_close(db);

    println("All tests OK!");
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

//! Processes that are directly spawned by shared mempool runtime initialization
use crate::{
    core_mempool::{CoreMempool, TimelineState},
    counters,
    logging::{LogEntry, LogEvent, LogSchema},
    network::{MempoolNetworkEvents, MempoolSyncMsg},
    shared_mempool::{
        tasks,
        tasks::process_committed_transactions,
        types::{notify_subscribers, ScheduledBroadcast, SharedMempool, SharedMempoolNotification},
    },
    MempoolEventsReceiver, QuorumStoreRequest,
};
use ::network::protocols::network::Event;
use aptos_config::network_id::{NetworkId, PeerNetworkId};
use aptos_infallible::Mutex;
use aptos_logger::prelude::*;
use aptos_types::on_chain_config::OnChainConfigPayload;
use bounded_executor::BoundedExecutor;
use consensus_types::common::TransactionSummary;
use event_notifications::ReconfigNotificationListener;
use futures::{
    channel::mpsc,
    stream::{select_all, FuturesUnordered},
    StreamExt,
};
use mempool_notifications::{MempoolCommitNotification, MempoolNotificationListener};
use std::{
    sync::Arc,
    time::{Duration, Instant, SystemTime},
};
use tokio::{runtime::Handle, time::interval};
use tokio_stream::wrappers::IntervalStream;
use vm_validator::vm_validator::TransactionValidation;

use super::types::MempoolClientRequest;

/// Coordinator that handles inbound network events and outbound txn broadcasts.
pub(crate) async fn coordinator<V>(
    mut smp: SharedMempool<V>,
    executor: Handle,
    network_events: Vec<(NetworkId, MempoolNetworkEvents)>,
    mut client_events: MempoolEventsReceiver,
    mut quorum_store_requests: mpsc::Receiver<QuorumStoreRequest>,
    mut mempool_listener: MempoolNotificationListener,
    mut mempool_reconfig_events: ReconfigNotificationListener,
) where
    V: TransactionValidation,
{
    info!(LogSchema::event_log(
        LogEntry::CoordinatorRuntime,
        LogEvent::Start
    ));
    // Combine `NetworkEvents` for each `NetworkId` into one stream
    let smp_events: Vec<_> = network_events
        .into_iter()
        .map(|(network_id, events)| events.map(move |e| (network_id, e)))
        .collect();
    let mut events = select_all(smp_events).fuse();
    let mut scheduled_broadcasts = FuturesUnordered::new();

    // Use a BoundedExecutor to restrict only `workers_available` concurrent
    // worker tasks that can process incoming transactions.
    let workers_available = smp.config.shared_mempool_max_concurrent_inbound_syncs;
    let bounded_executor = BoundedExecutor::new(workers_available, executor.clone());

    loop {
        let _timer = counters::MAIN_LOOP.start_timer();
        ::futures::select! {
            msg = client_events.select_next_some() => {
                handle_client_request(&mut smp, &bounded_executor, msg).await;
            },
            msg = quorum_store_requests.select_next_some() => {
                tasks::process_quorum_store_request(&smp, msg);
            },
            msg = mempool_listener.select_next_some() => {
                handle_commit_notification(&mut smp, msg, &mut mempool_listener);
            },
            reconfig_notification = mempool_reconfig_events.select_next_some() => {
                handle_mempool_reconfig_event(&mut smp, &bounded_executor, reconfig_notification.on_chain_configs).await;
            },
            (peer, backoff) = scheduled_broadcasts.select_next_some() => {
                tasks::execute_broadcast(peer, backoff, &mut smp, &mut scheduled_broadcasts, executor.clone()).await;
            },
            (network_id, event) = events.select_next_some() => {
                handle_network_event(&executor, &bounded_executor, &mut scheduled_broadcasts, &mut smp, network_id, event).await;
            },
            complete => break,
        }
    }
    error!(LogSchema::event_log(
        LogEntry::CoordinatorRuntime,
        LogEvent::Terminated
    ));
}

/// Spawn a task for processing `MempoolClientRequest`s from a client such as API service
async fn handle_client_request<V>(
    smp: &mut SharedMempool<V>,
    bounded_executor: &BoundedExecutor,
    request: MempoolClientRequest,
) where
    V: TransactionValidation,
{
    match request {
        MempoolClientRequest::SubmitTransaction(txn, callback) => {
            // This timer measures how long it took for the bounded executor to *schedule* the
            // task.
            let _timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_LABEL,
                counters::SPAWN_LABEL,
            );
            // This timer measures how long it took for the task to go from scheduled to started.
            let task_start_timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_LABEL,
                counters::START_LABEL,
            );
            bounded_executor
                .spawn(tasks::process_client_transaction_submission(
                    smp.clone(),
                    txn,
                    callback,
                    task_start_timer,
                ))
                .await;
        }
        MempoolClientRequest::GetTransactionByHash(hash, callback) => {
            // This timer measures how long it took for the bounded executor to *schedule* the
            // task.
            let _timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_GET_TXN_LABEL,
                counters::SPAWN_LABEL,
            );
            // This timer measures how long it took for the task to go from scheduled to started.
            let task_start_timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_GET_TXN_LABEL,
                counters::START_LABEL,
            );
            bounded_executor
                .spawn(tasks::process_client_get_transaction(
                    smp.clone(),
                    hash,
                    callback,
                    task_start_timer,
                ))
                .await;
        }
    }
}

/// Handle removing committed transactions from local mempool immediately.  This should be done
/// immediately to ensure broadcasts of committed transactions stop as soon as possible.
fn handle_commit_notification<V>(
    smp: &mut SharedMempool<V>,
    msg: MempoolCommitNotification,
    mempool_listener: &mut MempoolNotificationListener,
) where
    V: TransactionValidation,
{
    debug!(
        LogSchema::event_log(LogEntry::StateSyncCommit, LogEvent::Received).state_sync_msg(&msg)
    );

    // Process and time committed user transactions.
    let start_time = Instant::now();
    counters::mempool_service_transactions(
        counters::COMMIT_STATE_SYNC_LABEL,
        msg.transactions.len(),
    );
    process_committed_transactions(
        &smp.mempool,
        msg.transactions
            .iter()
            .map(|txn| TransactionSummary {
                sender: txn.sender,
                sequence_number: txn.sequence_number,
            })
            .collect(),
        msg.block_timestamp_usecs,
        false,
    );
    smp.validator.write().notify_commit();
    let counter_result = if mempool_listener.ack_commit_notification(msg).is_err() {
        error!(LogSchema::event_log(
            LogEntry::StateSyncCommit,
            LogEvent::CallbackFail
        ));
        counters::REQUEST_FAIL_LABEL
    } else {
        counters::REQUEST_SUCCESS_LABEL
    };
    let latency = start_time.elapsed();
    counters::mempool_service_latency(counters::COMMIT_STATE_SYNC_LABEL, counter_result, latency);
}

/// Spawn a task to restart the transaction validator with the new reconfig data.
async fn handle_mempool_reconfig_event<V>(
    smp: &mut SharedMempool<V>,
    bounded_executor: &BoundedExecutor,
    config_update: OnChainConfigPayload,
) where
    V: TransactionValidation,
{
    info!(LogSchema::event_log(
        LogEntry::ReconfigUpdate,
        LogEvent::Received
    ));
    let _timer =
        counters::task_spawn_latency_timer(counters::RECONFIG_EVENT_LABEL, counters::SPAWN_LABEL);

    bounded_executor
        .spawn(tasks::process_config_update(
            config_update,
            smp.validator.clone(),
        ))
        .await;
}

/// Handles all NewPeer, LostPeer, and network messages.
/// - NewPeer events start new automatic broadcasts if the peer is upstream. If the peer is not upstream, we ignore it.
/// - LostPeer events disable the upstream peer, which will cancel ongoing broadcasts.
/// - Network messages follow a simple Request/Response framework to accept new transactions
/// TODO: Move to RPC off of DirectSend
async fn handle_network_event<V>(
    executor: &Handle,
    bounded_executor: &BoundedExecutor,
    scheduled_broadcasts: &mut FuturesUnordered<ScheduledBroadcast>,
    smp: &mut SharedMempool<V>,
    network_id: NetworkId,
    event: Event<MempoolSyncMsg>,
) where
    V: TransactionValidation,
{
    match event {
        Event::NewPeer(metadata) => {
            counters::shared_mempool_event_inc("new_peer");
            let peer = PeerNetworkId::new(network_id, metadata.remote_peer_id);
            let is_new_peer = smp.network_interface.add_peer(peer, metadata.clone());
            let is_upstream_peer = smp
                .network_interface
                .is_upstream_peer(&peer, Some(&metadata));
            debug!(LogSchema::new(LogEntry::NewPeer)
                .peer(&peer)
                .is_upstream_peer(is_upstream_peer));
            notify_subscribers(SharedMempoolNotification::PeerStateChange, &smp.subscribers);
            if is_new_peer && is_upstream_peer {
                tasks::execute_broadcast(peer, false, smp, scheduled_broadcasts, executor.clone())
                    .await;
            }
        }
        Event::LostPeer(metadata) => {
            counters::shared_mempool_event_inc("lost_peer");
            let peer = PeerNetworkId::new(network_id, metadata.remote_peer_id);
            debug!(LogSchema::new(LogEntry::LostPeer)
                .peer(&peer)
                .is_upstream_peer(
                    smp.network_interface
                        .is_upstream_peer(&peer, Some(&metadata))
                ));
            smp.network_interface.disable_peer(peer);
            notify_subscribers(SharedMempoolNotification::PeerStateChange, &smp.subscribers);
        }
        Event::Message(peer_id, msg) => {
            counters::shared_mempool_event_inc("message");
            match msg {
                MempoolSyncMsg::BroadcastTransactionsRequest {
                    request_id,
                    transactions,
                } => {
                    let smp_clone = smp.clone();
                    let peer = PeerNetworkId::new(network_id, peer_id);
                    let ineligible_for_broadcast = (!smp.broadcast_within_validator_network()
                        && smp.network_interface.is_validator())
                        || smp.network_interface.is_upstream_peer(&peer, None);
                    let timeline_state = if ineligible_for_broadcast {
                        TimelineState::NonQualified
                    } else {
                        TimelineState::NotReady
                    };
                    // This timer measures how long it took for the bounded executor to
                    // *schedule* the task.
                    let _timer = counters::task_spawn_latency_timer(
                        counters::PEER_BROADCAST_EVENT_LABEL,
                        counters::SPAWN_LABEL,
                    );
                    // This timer measures how long it took for the task to go from scheduled
                    // to started.
                    let task_start_timer = counters::task_spawn_latency_timer(
                        counters::PEER_BROADCAST_EVENT_LABEL,
                        counters::START_LABEL,
                    );
                    bounded_executor
                        .spawn(tasks::process_transaction_broadcast(
                            smp_clone,
                            transactions,
                            request_id,
                            timeline_state,
                            peer,
                            task_start_timer,
                        ))
                        .await;
                }
                MempoolSyncMsg::BroadcastTransactionsResponse {
                    request_id,
                    retry,
                    backoff,
                } => {
                    let ack_timestamp = SystemTime::now();
                    smp.network_interface.process_broadcast_ack(
                        PeerNetworkId::new(network_id, peer_id),
                        request_id,
                        retry,
                        backoff,
                        ack_timestamp,
                    );
                }
            }
        }
        Event::RpcRequest(peer_id, _msg, _, _res_tx) => {
            counters::unexpected_msg_count_inc(&network_id);
            sample!(
                SampleRate::Duration(Duration::from_secs(60)),
                warn!(LogSchema::new(LogEntry::UnexpectedNetworkMsg)
                    .peer(&PeerNetworkId::new(network_id, peer_id)))
            );
        }
    }
}

/// Garbage collect all expired transactions by SystemTTL.
pub(crate) async fn gc_coordinator(mempool: Arc<Mutex<CoreMempool>>, gc_interval_ms: u64) {
    info!(LogSchema::event_log(LogEntry::GCRuntime, LogEvent::Start));
    let mut interval = IntervalStream::new(interval(Duration::from_millis(gc_interval_ms)));
    while let Some(_interval) = interval.next().await {
        sample!(
            SampleRate::Duration(Duration::from_secs(60)),
            info!(LogSchema::event_log(LogEntry::GCRuntime, LogEvent::Live))
        );
        mempool.lock().gc();
    }

    error!(LogSchema::event_log(
        LogEntry::GCRuntime,
        LogEvent::Terminated
    ));
}

/// Periodically logs a snapshot of transactions in core mempool.
/// In the future we may want an interactive way to directly query mempool's internal state.
/// For now, we will rely on this periodic snapshot to observe the internal state.
pub(crate) async fn snapshot_job(mempool: Arc<Mutex<CoreMempool>>, snapshot_interval_secs: u64) {
    let mut interval = IntervalStream::new(interval(Duration::from_secs(snapshot_interval_secs)));
    while let Some(_interval) = interval.next().await {
        let snapshot = mempool.lock().gen_snapshot();
        debug!(LogSchema::new(LogEntry::MempoolSnapshot).txns(snapshot));
    }
}
<commit_msg>trivial: remove a spammy debug log<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

//! Processes that are directly spawned by shared mempool runtime initialization
use crate::{
    core_mempool::{CoreMempool, TimelineState},
    counters,
    logging::{LogEntry, LogEvent, LogSchema},
    network::{MempoolNetworkEvents, MempoolSyncMsg},
    shared_mempool::{
        tasks,
        tasks::process_committed_transactions,
        types::{notify_subscribers, ScheduledBroadcast, SharedMempool, SharedMempoolNotification},
    },
    MempoolEventsReceiver, QuorumStoreRequest,
};
use ::network::protocols::network::Event;
use aptos_config::network_id::{NetworkId, PeerNetworkId};
use aptos_infallible::Mutex;
use aptos_logger::prelude::*;
use aptos_types::on_chain_config::OnChainConfigPayload;
use bounded_executor::BoundedExecutor;
use consensus_types::common::TransactionSummary;
use event_notifications::ReconfigNotificationListener;
use futures::{
    channel::mpsc,
    stream::{select_all, FuturesUnordered},
    StreamExt,
};
use mempool_notifications::{MempoolCommitNotification, MempoolNotificationListener};
use std::{
    sync::Arc,
    time::{Duration, Instant, SystemTime},
};
use tokio::{runtime::Handle, time::interval};
use tokio_stream::wrappers::IntervalStream;
use vm_validator::vm_validator::TransactionValidation;

use super::types::MempoolClientRequest;

/// Coordinator that handles inbound network events and outbound txn broadcasts.
pub(crate) async fn coordinator<V>(
    mut smp: SharedMempool<V>,
    executor: Handle,
    network_events: Vec<(NetworkId, MempoolNetworkEvents)>,
    mut client_events: MempoolEventsReceiver,
    mut quorum_store_requests: mpsc::Receiver<QuorumStoreRequest>,
    mut mempool_listener: MempoolNotificationListener,
    mut mempool_reconfig_events: ReconfigNotificationListener,
) where
    V: TransactionValidation,
{
    info!(LogSchema::event_log(
        LogEntry::CoordinatorRuntime,
        LogEvent::Start
    ));
    // Combine `NetworkEvents` for each `NetworkId` into one stream
    let smp_events: Vec<_> = network_events
        .into_iter()
        .map(|(network_id, events)| events.map(move |e| (network_id, e)))
        .collect();
    let mut events = select_all(smp_events).fuse();
    let mut scheduled_broadcasts = FuturesUnordered::new();

    // Use a BoundedExecutor to restrict only `workers_available` concurrent
    // worker tasks that can process incoming transactions.
    let workers_available = smp.config.shared_mempool_max_concurrent_inbound_syncs;
    let bounded_executor = BoundedExecutor::new(workers_available, executor.clone());

    loop {
        let _timer = counters::MAIN_LOOP.start_timer();
        ::futures::select! {
            msg = client_events.select_next_some() => {
                handle_client_request(&mut smp, &bounded_executor, msg).await;
            },
            msg = quorum_store_requests.select_next_some() => {
                tasks::process_quorum_store_request(&smp, msg);
            },
            msg = mempool_listener.select_next_some() => {
                handle_commit_notification(&mut smp, msg, &mut mempool_listener);
            },
            reconfig_notification = mempool_reconfig_events.select_next_some() => {
                handle_mempool_reconfig_event(&mut smp, &bounded_executor, reconfig_notification.on_chain_configs).await;
            },
            (peer, backoff) = scheduled_broadcasts.select_next_some() => {
                tasks::execute_broadcast(peer, backoff, &mut smp, &mut scheduled_broadcasts, executor.clone()).await;
            },
            (network_id, event) = events.select_next_some() => {
                handle_network_event(&executor, &bounded_executor, &mut scheduled_broadcasts, &mut smp, network_id, event).await;
            },
            complete => break,
        }
    }
    error!(LogSchema::event_log(
        LogEntry::CoordinatorRuntime,
        LogEvent::Terminated
    ));
}

/// Spawn a task for processing `MempoolClientRequest`s from a client such as API service
async fn handle_client_request<V>(
    smp: &mut SharedMempool<V>,
    bounded_executor: &BoundedExecutor,
    request: MempoolClientRequest,
) where
    V: TransactionValidation,
{
    match request {
        MempoolClientRequest::SubmitTransaction(txn, callback) => {
            // This timer measures how long it took for the bounded executor to *schedule* the
            // task.
            let _timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_LABEL,
                counters::SPAWN_LABEL,
            );
            // This timer measures how long it took for the task to go from scheduled to started.
            let task_start_timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_LABEL,
                counters::START_LABEL,
            );
            bounded_executor
                .spawn(tasks::process_client_transaction_submission(
                    smp.clone(),
                    txn,
                    callback,
                    task_start_timer,
                ))
                .await;
        }
        MempoolClientRequest::GetTransactionByHash(hash, callback) => {
            // This timer measures how long it took for the bounded executor to *schedule* the
            // task.
            let _timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_GET_TXN_LABEL,
                counters::SPAWN_LABEL,
            );
            // This timer measures how long it took for the task to go from scheduled to started.
            let task_start_timer = counters::task_spawn_latency_timer(
                counters::CLIENT_EVENT_GET_TXN_LABEL,
                counters::START_LABEL,
            );
            bounded_executor
                .spawn(tasks::process_client_get_transaction(
                    smp.clone(),
                    hash,
                    callback,
                    task_start_timer,
                ))
                .await;
        }
    }
}

/// Handle removing committed transactions from local mempool immediately.  This should be done
/// immediately to ensure broadcasts of committed transactions stop as soon as possible.
fn handle_commit_notification<V>(
    smp: &mut SharedMempool<V>,
    msg: MempoolCommitNotification,
    mempool_listener: &mut MempoolNotificationListener,
) where
    V: TransactionValidation,
{
    debug!(
        block_timestamp_usecs = msg.block_timestamp_usecs,
        num_committed_txns = msg.transactions.len(),
        LogSchema::event_log(LogEntry::StateSyncCommit, LogEvent::Received),
    );

    // Process and time committed user transactions.
    let start_time = Instant::now();
    counters::mempool_service_transactions(
        counters::COMMIT_STATE_SYNC_LABEL,
        msg.transactions.len(),
    );
    process_committed_transactions(
        &smp.mempool,
        msg.transactions
            .iter()
            .map(|txn| TransactionSummary {
                sender: txn.sender,
                sequence_number: txn.sequence_number,
            })
            .collect(),
        msg.block_timestamp_usecs,
        false,
    );
    smp.validator.write().notify_commit();
    let counter_result = if mempool_listener.ack_commit_notification(msg).is_err() {
        error!(LogSchema::event_log(
            LogEntry::StateSyncCommit,
            LogEvent::CallbackFail
        ));
        counters::REQUEST_FAIL_LABEL
    } else {
        counters::REQUEST_SUCCESS_LABEL
    };
    let latency = start_time.elapsed();
    counters::mempool_service_latency(counters::COMMIT_STATE_SYNC_LABEL, counter_result, latency);
}

/// Spawn a task to restart the transaction validator with the new reconfig data.
async fn handle_mempool_reconfig_event<V>(
    smp: &mut SharedMempool<V>,
    bounded_executor: &BoundedExecutor,
    config_update: OnChainConfigPayload,
) where
    V: TransactionValidation,
{
    info!(LogSchema::event_log(
        LogEntry::ReconfigUpdate,
        LogEvent::Received
    ));
    let _timer =
        counters::task_spawn_latency_timer(counters::RECONFIG_EVENT_LABEL, counters::SPAWN_LABEL);

    bounded_executor
        .spawn(tasks::process_config_update(
            config_update,
            smp.validator.clone(),
        ))
        .await;
}

/// Handles all NewPeer, LostPeer, and network messages.
/// - NewPeer events start new automatic broadcasts if the peer is upstream. If the peer is not upstream, we ignore it.
/// - LostPeer events disable the upstream peer, which will cancel ongoing broadcasts.
/// - Network messages follow a simple Request/Response framework to accept new transactions
/// TODO: Move to RPC off of DirectSend
async fn handle_network_event<V>(
    executor: &Handle,
    bounded_executor: &BoundedExecutor,
    scheduled_broadcasts: &mut FuturesUnordered<ScheduledBroadcast>,
    smp: &mut SharedMempool<V>,
    network_id: NetworkId,
    event: Event<MempoolSyncMsg>,
) where
    V: TransactionValidation,
{
    match event {
        Event::NewPeer(metadata) => {
            counters::shared_mempool_event_inc("new_peer");
            let peer = PeerNetworkId::new(network_id, metadata.remote_peer_id);
            let is_new_peer = smp.network_interface.add_peer(peer, metadata.clone());
            let is_upstream_peer = smp
                .network_interface
                .is_upstream_peer(&peer, Some(&metadata));
            debug!(LogSchema::new(LogEntry::NewPeer)
                .peer(&peer)
                .is_upstream_peer(is_upstream_peer));
            notify_subscribers(SharedMempoolNotification::PeerStateChange, &smp.subscribers);
            if is_new_peer && is_upstream_peer {
                tasks::execute_broadcast(peer, false, smp, scheduled_broadcasts, executor.clone())
                    .await;
            }
        }
        Event::LostPeer(metadata) => {
            counters::shared_mempool_event_inc("lost_peer");
            let peer = PeerNetworkId::new(network_id, metadata.remote_peer_id);
            debug!(LogSchema::new(LogEntry::LostPeer)
                .peer(&peer)
                .is_upstream_peer(
                    smp.network_interface
                        .is_upstream_peer(&peer, Some(&metadata))
                ));
            smp.network_interface.disable_peer(peer);
            notify_subscribers(SharedMempoolNotification::PeerStateChange, &smp.subscribers);
        }
        Event::Message(peer_id, msg) => {
            counters::shared_mempool_event_inc("message");
            match msg {
                MempoolSyncMsg::BroadcastTransactionsRequest {
                    request_id,
                    transactions,
                } => {
                    let smp_clone = smp.clone();
                    let peer = PeerNetworkId::new(network_id, peer_id);
                    let ineligible_for_broadcast = (!smp.broadcast_within_validator_network()
                        && smp.network_interface.is_validator())
                        || smp.network_interface.is_upstream_peer(&peer, None);
                    let timeline_state = if ineligible_for_broadcast {
                        TimelineState::NonQualified
                    } else {
                        TimelineState::NotReady
                    };
                    // This timer measures how long it took for the bounded executor to
                    // *schedule* the task.
                    let _timer = counters::task_spawn_latency_timer(
                        counters::PEER_BROADCAST_EVENT_LABEL,
                        counters::SPAWN_LABEL,
                    );
                    // This timer measures how long it took for the task to go from scheduled
                    // to started.
                    let task_start_timer = counters::task_spawn_latency_timer(
                        counters::PEER_BROADCAST_EVENT_LABEL,
                        counters::START_LABEL,
                    );
                    bounded_executor
                        .spawn(tasks::process_transaction_broadcast(
                            smp_clone,
                            transactions,
                            request_id,
                            timeline_state,
                            peer,
                            task_start_timer,
                        ))
                        .await;
                }
                MempoolSyncMsg::BroadcastTransactionsResponse {
                    request_id,
                    retry,
                    backoff,
                } => {
                    let ack_timestamp = SystemTime::now();
                    smp.network_interface.process_broadcast_ack(
                        PeerNetworkId::new(network_id, peer_id),
                        request_id,
                        retry,
                        backoff,
                        ack_timestamp,
                    );
                }
            }
        }
        Event::RpcRequest(peer_id, _msg, _, _res_tx) => {
            counters::unexpected_msg_count_inc(&network_id);
            sample!(
                SampleRate::Duration(Duration::from_secs(60)),
                warn!(LogSchema::new(LogEntry::UnexpectedNetworkMsg)
                    .peer(&PeerNetworkId::new(network_id, peer_id)))
            );
        }
    }
}

/// Garbage collect all expired transactions by SystemTTL.
pub(crate) async fn gc_coordinator(mempool: Arc<Mutex<CoreMempool>>, gc_interval_ms: u64) {
    info!(LogSchema::event_log(LogEntry::GCRuntime, LogEvent::Start));
    let mut interval = IntervalStream::new(interval(Duration::from_millis(gc_interval_ms)));
    while let Some(_interval) = interval.next().await {
        sample!(
            SampleRate::Duration(Duration::from_secs(60)),
            info!(LogSchema::event_log(LogEntry::GCRuntime, LogEvent::Live))
        );
        mempool.lock().gc();
    }

    error!(LogSchema::event_log(
        LogEntry::GCRuntime,
        LogEvent::Terminated
    ));
}

/// Periodically logs a snapshot of transactions in core mempool.
/// In the future we may want an interactive way to directly query mempool's internal state.
/// For now, we will rely on this periodic snapshot to observe the internal state.
pub(crate) async fn snapshot_job(mempool: Arc<Mutex<CoreMempool>>, snapshot_interval_secs: u64) {
    let mut interval = IntervalStream::new(interval(Duration::from_secs(snapshot_interval_secs)));
    while let Some(_interval) = interval.next().await {
        let snapshot = mempool.lock().gen_snapshot();
        debug!(LogSchema::new(LogEntry::MempoolSnapshot).txns(snapshot));
    }
}
<|endoftext|>


<commit_before>use std::convert::{From, Into};
use std::error::Error;

#[derive(Debug)]
#[derive(Clone)]
#[derive(PartialEq)]
#[derive(Eq)]
// #[derive(Display)]
pub enum FileIDType {
    UUID,
}

impl FileIDType {

    fn parse(s: &str) -> Option<FileIDType> {
        unimplemented!()
    }

}

<commit_msg>impl Into<String> for FileIDType<commit_after>use std::convert::{From, Into};
use std::error::Error;

#[derive(Debug)]
#[derive(Clone)]
#[derive(PartialEq)]
#[derive(Eq)]
// #[derive(Display)]
pub enum FileIDType {
    UUID,
}

impl FileIDType {

    fn parse(s: &str) -> Option<FileIDType> {
        unimplemented!()
    }

}

impl Into<String> for FileIDType {

    fn into(self) -> String {
        match self {
            FileIDType::UUID    => String::from("UUID"),
        }
    }
}

<|endoftext|>


<commit_before>#![allow(non_upper_case_globals)]
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(dead_code)]
#![allow(unused_variables)]

include!("bindings.rs");

use std::os::raw::c_uint;
use std :: os :: raw :: c_char;
use std::io::Write;

enum PrintState {
    OUT,
    IN,
    DISPLY
}

fn print_ptr_rec(mut port: std::io::Stdout, x: ptr, state: PrintState) {
    if  (x & fx_mask) == fx_tag {
        write!(port, "{}", (x as c_uint) >> fx_shift);
    } else {
        write!(port, "TODO");
    }
}
fn print_ptr(x: ptr) {
    print_ptr_rec(std::io::stdout(), x, PrintState::OUT);
    println!("");
}

fn allocate_protected_space(size: usize) -> *mut c_char {
    let mut v = Vec::with_capacity(size);
    let ptr = v.as_mut_ptr();
    std::mem::forget(v);
    ptr
}

fn deallocate_protected_space(p: *mut c_char, size: usize) {
    unsafe { std::mem::drop(Vec::from_raw_parts(p, 0, size)) };
}

fn main() {
    let stack_size = 16 * 4096;
    let heap_size = 4 * 16 * 4096;
    let global_size = 16 * 4096;
    let scratch_size = 16 * 4096;

    let stack_top = allocate_protected_space(stack_size);
    let stack_base = unsafe { stack_top.offset(stack_size as isize) };

    let heap = allocate_protected_space(heap_size);
    let global = allocate_protected_space(global_size);
    let scratch = allocate_protected_space(scratch_size);

    let uninit = 0 as (*mut std::os::raw::c_void);

    let mut ctxt = context {
        eax : uninit,
        ebx : uninit,
        ecx : uninit,
        edx : uninit,
        esi : uninit,
        edi : uninit,
        ebp : uninit,
        esp : uninit,
    };

    let heap_top = unsafe { heap.offset((heap_size as isize)/2) };
    let mut mem = memory {
        heap_next : heap,
        global_next : global,
        heap_base : heap,
        heap_top : heap_top,
        heap_base_alt : heap_top,
        heap_top_alt : unsafe { heap.offset(heap_size as isize) },
        global_base : global,
        stack_base : stack_base,
        scratch_base : scratch,
        edi : 0
    };

    print_ptr(unsafe {
        scheme_entry(&mut ctxt, stack_base, &mut mem)
    });

    deallocate_protected_space(stack_top, stack_size);
    deallocate_protected_space(heap, stack_size);
}
<commit_msg>one more warning, it works now<commit_after>#![allow(non_upper_case_globals)]
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(dead_code)]
#![allow(unused_variables)]
#![allow(unused_must_use)]

include!("bindings.rs");

use std::os::raw::c_uint;
use std :: os :: raw :: c_char;
use std::io::Write;

enum PrintState {
    OUT,
    IN,
    DISPLY
}

fn print_ptr_rec(mut port: std::io::Stdout, x: ptr, state: PrintState) {
    if  (x & fx_mask) == fx_tag {
        write!(port, "{}", (x as c_uint) >> fx_shift);
    } else {
        write!(port, "TODO");
    }
}
fn print_ptr(x: ptr) {
    print_ptr_rec(std::io::stdout(), x, PrintState::OUT);
    println!("");
}

fn allocate_protected_space(size: usize) -> *mut c_char {
    let mut v = Vec::with_capacity(size);
    let ptr = v.as_mut_ptr();
    std::mem::forget(v);
    ptr
}

fn deallocate_protected_space(p: *mut c_char, size: usize) {
    unsafe { std::mem::drop(Vec::from_raw_parts(p, 0, size)) };
}

fn main() {
    let stack_size = 16 * 4096;
    let heap_size = 4 * 16 * 4096;
    let global_size = 16 * 4096;
    let scratch_size = 16 * 4096;

    let stack_top = allocate_protected_space(stack_size);
    let stack_base = unsafe { stack_top.offset(stack_size as isize) };

    let heap = allocate_protected_space(heap_size);
    let global = allocate_protected_space(global_size);
    let scratch = allocate_protected_space(scratch_size);

    let uninit = 0 as (*mut std::os::raw::c_void);

    let mut ctxt = context {
        eax : uninit,
        ebx : uninit,
        ecx : uninit,
        edx : uninit,
        esi : uninit,
        edi : uninit,
        ebp : uninit,
        esp : uninit,
    };

    let heap_top = unsafe { heap.offset((heap_size as isize)/2) };
    let mut mem = memory {
        heap_next : heap,
        global_next : global,
        heap_base : heap,
        heap_top : heap_top,
        heap_base_alt : heap_top,
        heap_top_alt : unsafe { heap.offset(heap_size as isize) },
        global_base : global,
        stack_base : stack_base,
        scratch_base : scratch,
        edi : 0
    };

    print_ptr(unsafe {
        scheme_entry(&mut ctxt, stack_base, &mut mem)
    });

    deallocate_protected_space(stack_top, stack_size);
    deallocate_protected_space(heap, stack_size);
}
<|endoftext|>


<commit_before>use super::*;

// Are ASTs encodable?
#[test]
fn check_asts_encodable() {
    fn assert_encodable<T: rustc_serialize::Encodable>() {}
    assert_encodable::<Crate>();
}
<commit_msg>Fix rustc_ast unit test<commit_after>use super::*;

// Are ASTs encodable?
#[test]
fn check_asts_encodable() {
    fn assert_encodable<
        T: for<'a> rustc_serialize::Encodable<rustc_serialize::json::Encoder<'a>>,
    >() {
    }
    assert_encodable::<Crate>();
}
<|endoftext|>


<commit_before>// Copyright (C) 2015, Alberto Corona <alberto@0x1a.us>
// All rights reserved. This file is part of rpf, distributed under the
// BSD 3-Clause license. For full terms please see the LICENSE file.

use std::path::{PathBuf,Path};

/// Adds some useful functions for manipulating and retrieving information from
/// paths
pub trait PathMod {
    /// Returns a `PathBuf` of `&self`'s last component
    ///
    /// # Example
    ///
    /// ```
    /// use rpf::PathMod;
    /// use std::path::Path;
    ///
    /// let path = Path::new("/tmp/test/mod");
    /// let last = path.last_component();
    /// ```
    fn last_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self`'s first component
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/tmp/test/mod");
    /// let first = path.first_component();
    /// assert_eq!(first, PathBuf::from("/"));
    /// ```
    fn first_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self` relative from `rel_from`
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("mod");
    /// let parent = PathBuf::from("/tmp/test/mod");
    /// let rel = path.rel_to(&parent);
    /// ```
    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf;

    /// Returns a `&str` for a path, returns a blank string if unable to
    /// get a string for the path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/usr/share");
    /// let path_str = &path.as_str();
    /// assert_eq!(path_str, &"/usr/share");
    /// ```
    fn as_str(&self) -> &str;

    /// Returns a `String` for a path, returns an empty string if unable to get
    /// a string for a path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path_string = PathBuf::from("/var/log/test").as_string();
    /// assert_eq!(path_string, "/var/log/test".to_string());
    /// ```
    fn as_string(&self) -> String;
}

impl PathMod for PathBuf {
    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf {
        self.relative_from(&rel_from).unwrap_or(&PathBuf::new()).to_path_buf()
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }
}

impl PathMod for Path {
    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf {
        self.relative_from(&rel_from).unwrap_or(&PathBuf::new()).to_path_buf()
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }
}

#[test]
fn test_pathmod_first_comp() {
    let comp = Path::new("/etc/test").first_component();
    assert_eq!(comp, PathBuf::from("/"));
}

#[test]
fn test_pathmod_last_comp() {
    let comp = Path::new("/etc/test").last_component();
    assert_eq!(comp, PathBuf::from("test"));
}

#[test]
fn test_pathmod_as_str() {
    let string = Path::new("/var/log/test").as_str();
    assert_eq!(string, "/var/log/test");
}

#[test]
fn test_pathmod_as_string() {
    let string = Path::new("/var/log/test").as_string();
    assert_eq!(string, "/var/log/test".to_string());
}
<commit_msg>pathmod: change `rel_to`, add `is_dot`<commit_after>// Copyright (C) 2015, Alberto Corona <alberto@0x1a.us>
// All rights reserved. This file is part of rpf, distributed under the
// BSD 3-Clause license. For full terms please see the LICENSE file.

use std::path::{PathBuf,Path};

/// Adds some useful functions for manipulating and retrieving information from
/// paths
pub trait PathMod {
    /// Returns true if the path's file name starts with a "."
    ///
    /// # Example
    ///
    /// ```
    /// use rpf::PathMod;
    /// use std::path::Path;
    ///
    /// let path = Path::new("/test/dot/.dotfile");
    /// assert_eq!(path.is_dot(), true);
    fn is_dot(&self) -> bool;

    /// Returns a `PathBuf` of `&self`'s last component
    ///
    /// # Example
    ///
    /// ```
    /// use rpf::PathMod;
    /// use std::path::Path;
    ///
    /// let path = Path::new("/tmp/test/mod");
    /// let last = path.last_component();
    /// ```
    fn last_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self`'s first component
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/tmp/test/mod");
    /// let first = path.first_component();
    /// assert_eq!(first, PathBuf::from("/"));
    /// ```
    fn first_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self` relative from its parent
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("mod");
    /// let parent = PathBuf::from("/tmp/test/mod");
    /// let rel = path.rel_to_parent();
    /// ```
    fn rel_to_parent(&self) -> PathBuf;

    /// Returns a `&str` for a path, returns a blank string if unable to
    /// get a string for the path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/usr/share");
    /// let path_str = &path.as_str();
    /// assert_eq!(path_str, &"/usr/share");
    /// ```
    fn as_str(&self) -> &str;

    /// Returns a `String` for a path, returns an empty string if unable to get
    /// a string for a path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path_string = PathBuf::from("/var/log/test").as_string();
    /// assert_eq!(path_string, "/var/log/test".to_string());
    /// ```
    fn as_string(&self) -> String;
}

impl PathMod for PathBuf {
    fn is_dot(&self) -> bool {
        let file_name = match self.file_name() {
            Some(s) => {
                match s.to_str() {
                    Some(k) => { k },
                    None => { "" }
                }
            },
            None => { "" }
        };
        if file_name.starts_with(".") { return true; }
        else { return false; }
    }

    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to_parent(&self) -> PathBuf {
        let parent = match self.parent() {
            Some(p) => { p },
            None => { Path::new("") }
        };
        match self.relative_from(parent) {
            Some(s) => { PathBuf::from(s) },
            None => { PathBuf::from("") }
        }
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }
}

impl PathMod for Path {
    fn is_dot(&self) -> bool {
        let file_name = match self.file_name() {
            Some(s) => {
                match s.to_str() {
                    Some(k) => { k },
                    None => { "" }
                }
            },
            None => { "" }
        };
        if file_name.starts_with(".") { return true; }
        else { return false; }
    }

    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to_parent(&self) -> PathBuf {
        let parent = match self.parent() {
            Some(p) => { p },
            None => { Path::new("") }
        };
        match self.relative_from(parent) {
            Some(s) => { PathBuf::from(s) },
            None => { PathBuf::from("") }
        }
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }
}

#[test]
fn test_pathmod_first_comp() {
    let comp = Path::new("/etc/test").first_component();
    assert_eq!(comp, PathBuf::from("/"));
}

#[test]
fn test_pathmod_last_comp() {
    let comp = Path::new("/etc/test").last_component();
    assert_eq!(comp, PathBuf::from("test"));
}

#[test]
fn test_pathmod_as_str() {
    let string = Path::new("/var/log/test").as_str();
    assert_eq!(string, "/var/log/test");
}

#[test]
fn test_pathmod_as_string() {
    let string = Path::new("/var/log/test").as_string();
    assert_eq!(string, "/var/log/test".to_string());
}

#[test]
fn test_pathmod_is_dot() {
    let path = Path::new("/dir/test/.test");
    assert_eq!(path.is_dot(), true);
}
<|endoftext|>


<commit_before>//! Metamethods for accesssing the registry values!

use super::super::super::registry;
use registry::RegistryAccess;

use hlua::any::AnyLuaValue;

use rustc_serialize::json::{Json, ToJson};
use std::ops::Deref;

pub fn index(table: AnyLuaValue, lua_key: AnyLuaValue)
             -> Result<AnyLuaValue, &'static str> {
    if let AnyLuaValue::LuaString(key) = lua_key {
        let maybe_json = registry::get_json(&key);
        if let Some(json_pair) = maybe_json {
            let (access, json) = json_pair;
            if access == RegistryAccess::Private {
                // TODO In lua_async: return nil
                Err("No value found for that key!")
            }
            else {
                Ok(convert_json(json.deref().to_json()))
            }
        }
        else {
            // Should be nil, nil doesn't seem to be included
            Err("No value found for that key!")
        }
    }
    else {
        Err("The registry can only be indexed with keys!")
    }
}

// Prevent lua from changing the registry?
pub fn new_index(table: AnyLuaValue, key: AnyLuaValue, val: AnyLuaValue)
                 -> Result<(), &'static str> {
    //Err("You can't set the registry!")
}

fn convert_json(json: Json) -> AnyLuaValue {
    match json {
        Json::String(val) => AnyLuaValue::LuaString(val),
        Json::Boolean(val) => AnyLuaValue::LuaBoolean(val),
        Json::F64(val) => AnyLuaValue::LuaNumber(val),
        Json::I64(val) => AnyLuaValue::LuaNumber((val as i32) as f64),
        Json::U64(val) => AnyLuaValue::LuaNumber((val as u32) as f64),
        Json::Null => AnyLuaValue::LuaNil,
        Json::Array(vals) => {
            let mut count = 0f64;
            // Gotta love that 1-based indexing. Start at zero but increment for
            // the first one. It works here at least.
            AnyLuaValue::LuaArray(vals.into_iter().map(|v| {
                count += 1.0;
                (AnyLuaValue::LuaNumber(count), convert_json(v))
            }).collect())
        },
        Json::Object(vals) => {
            AnyLuaValue::LuaArray(vals.into_iter().map(|(key, val)| {
                (AnyLuaValue::LuaString(key), convert_json(val))
            }).collect())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::convert_json;
    use hlua::any::AnyLuaValue::*;
    use rustc_serialize::json::{Json, ToJson, Object};
    use std::collections::BTreeMap;

    #[test]
    fn convert_string() {
        let input = "Hello World".to_string();
        let json_input = Json::String(input.clone());
        assert_eq!(convert_json(json_input), LuaString(input));
    }
    #[test]
    fn convert_numbers() {
        let lua_number = LuaNumber(42f64);
        assert_eq!(convert_json(Json::F64(42f64)), lua_number);
        assert_eq!(convert_json(Json::I64(42i64)), lua_number);
        assert_eq!(convert_json(Json::U64(42u64)), lua_number);
    }

    #[test]
    fn convert_bool() {
        assert_eq!(convert_json(Json::Boolean(true)), LuaBoolean(true));
        assert_eq!(convert_json(Json::Boolean(false)), LuaBoolean(false));
    }

    #[test]
    fn convert_array() {
        let json = Json::Array(
            vec![Json::Boolean(true), Json::F64(11f64),
                 Json::String("Foo".to_string())]
        );
        let lua = LuaArray(vec![
                (LuaNumber(1f64), LuaBoolean(true)),
                (LuaNumber(2f64), LuaNumber(11f64)),
                (LuaNumber(3f64), LuaString("Foo".to_string()))
            ]);
        assert_eq!(convert_json(json), lua);
    }

    #[test]
    fn convert_object() {
        // { name="foo", point: {x=0, y=1}, valid=false }
        // The fields are emitted in alphabetical order thanks to Json's
        // internal BTreeMap representation.

        let mut json_tree: Object = BTreeMap::new();
        let mut point_tree: Object = BTreeMap::new();
        point_tree.insert("x".to_string(), Json::U64(0u64));
        point_tree.insert("y".to_string(), Json::U64(1u64));
        json_tree.insert("point".to_string(), Json::Object(point_tree));
        json_tree.insert("name".to_string(), Json::String("foo".to_string()));
        json_tree.insert("valid".to_string(), Json::Boolean(false));

        let lua = LuaArray(vec![
            (LuaString("name".to_string()), LuaString("foo".to_string())),
            (LuaString("point".to_string()), LuaArray(vec![
                (LuaString("x".to_string()), LuaNumber(0f64)),
                (LuaString("y".to_string()), LuaNumber(1f64))
            ])),
            (LuaString("valid".to_string()), LuaBoolean(false)),
        ]);

        assert_eq!(convert_json(Json::Object(json_tree)), lua);
    }
}
<commit_msg>Fix registry compile errors<commit_after>//! Metamethods for accesssing the registry values!

use super::super::super::registry;
use registry::RegistryAccess;

use hlua::any::AnyLuaValue;

use rustc_serialize::json::{Json, ToJson};
use std::ops::Deref;

pub fn index(table: AnyLuaValue, lua_key: AnyLuaValue)
             -> Result<AnyLuaValue, &'static str> {
    if let AnyLuaValue::LuaString(key) = lua_key {
        let maybe_json = registry::get_json(&key);
        if let Some(json_pair) = maybe_json {
            let (access, json) = json_pair;
            if access == RegistryAccess::Private {
                // TODO In lua_async: return nil
                Err("No value found for that key!")
            }
            else {
                Ok(convert_json(json.deref().to_json()))
            }
        }
        else {
            // Should be nil, nil doesn't seem to be included
            Err("No value found for that key!")
        }
    }
    else {
        Err("The registry can only be indexed with keys!")
    }
}

// Prevent lua from changing the registry?
pub fn new_index(table: AnyLuaValue, key: AnyLuaValue, val: AnyLuaValue)
                 -> Result<(), &'static str> {
    Err("You can't set the registry!")
}

fn convert_json(json: Json) -> AnyLuaValue {
    match json {
        Json::String(val) => AnyLuaValue::LuaString(val),
        Json::Boolean(val) => AnyLuaValue::LuaBoolean(val),
        Json::F64(val) => AnyLuaValue::LuaNumber(val),
        Json::I64(val) => AnyLuaValue::LuaNumber((val as i32) as f64),
        Json::U64(val) => AnyLuaValue::LuaNumber((val as u32) as f64),
        Json::Null => AnyLuaValue::LuaNil,
        Json::Array(vals) => {
            let mut count = 0f64;
            // Gotta love that 1-based indexing. Start at zero but increment for
            // the first one. It works here at least.
            AnyLuaValue::LuaArray(vals.into_iter().map(|v| {
                count += 1.0;
                (AnyLuaValue::LuaNumber(count), convert_json(v))
            }).collect())
        },
        Json::Object(vals) => {
            AnyLuaValue::LuaArray(vals.into_iter().map(|(key, val)| {
                (AnyLuaValue::LuaString(key), convert_json(val))
            }).collect())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::convert_json;
    use hlua::any::AnyLuaValue::*;
    use rustc_serialize::json::{Json, ToJson, Object};
    use std::collections::BTreeMap;

    #[test]
    fn convert_string() {
        let input = "Hello World".to_string();
        let json_input = Json::String(input.clone());
        assert_eq!(convert_json(json_input), LuaString(input));
    }
    #[test]
    fn convert_numbers() {
        let lua_number = LuaNumber(42f64);
        assert_eq!(convert_json(Json::F64(42f64)), lua_number);
        assert_eq!(convert_json(Json::I64(42i64)), lua_number);
        assert_eq!(convert_json(Json::U64(42u64)), lua_number);
    }

    #[test]
    fn convert_bool() {
        assert_eq!(convert_json(Json::Boolean(true)), LuaBoolean(true));
        assert_eq!(convert_json(Json::Boolean(false)), LuaBoolean(false));
    }

    #[test]
    fn convert_array() {
        let json = Json::Array(
            vec![Json::Boolean(true), Json::F64(11f64),
                 Json::String("Foo".to_string())]
        );
        let lua = LuaArray(vec![
                (LuaNumber(1f64), LuaBoolean(true)),
                (LuaNumber(2f64), LuaNumber(11f64)),
                (LuaNumber(3f64), LuaString("Foo".to_string()))
            ]);
        assert_eq!(convert_json(json), lua);
    }

    #[test]
    fn convert_object() {
        // { name="foo", point: {x=0, y=1}, valid=false }
        // The fields are emitted in alphabetical order thanks to Json's
        // internal BTreeMap representation.

        let mut json_tree: Object = BTreeMap::new();
        let mut point_tree: Object = BTreeMap::new();
        point_tree.insert("x".to_string(), Json::U64(0u64));
        point_tree.insert("y".to_string(), Json::U64(1u64));
        json_tree.insert("point".to_string(), Json::Object(point_tree));
        json_tree.insert("name".to_string(), Json::String("foo".to_string()));
        json_tree.insert("valid".to_string(), Json::Boolean(false));

        let lua = LuaArray(vec![
            (LuaString("name".to_string()), LuaString("foo".to_string())),
            (LuaString("point".to_string()), LuaArray(vec![
                (LuaString("x".to_string()), LuaNumber(0f64)),
                (LuaString("y".to_string()), LuaNumber(1f64))
            ])),
            (LuaString("valid".to_string()), LuaBoolean(false)),
        ]);

        assert_eq!(convert_json(Json::Object(json_tree)), lua);
    }
}
<|endoftext|>


<commit_before>use crate::codegen_runtime::{NifReturnable, NifReturned};
use crate::{types, Encoder, Env};
use std::fmt;

/// Represents usual errors that can happen in a nif. This enables you
/// to return an error from anywhere, even places where you don't have
/// an Env available.
pub enum Error {
    /// Returned when the NIF has been called with the wrong number or type of
    /// arguments.
    BadArg,
    /// Encodes the string into an atom and returns it from the NIF.
    Atom(&'static str),
    RaiseAtom(&'static str),
    RaiseTerm(Box<dyn Encoder>),
}

unsafe impl NifReturnable for crate::error::Error {
    unsafe fn as_returned(self, env: Env) -> NifReturned {
        match self {
            Error::BadArg => NifReturned::BadArg,
            Error::Atom(atom_str) => {
                let atom = types::atom::Atom::from_str(env, atom_str)
                    .ok()
                    .expect("Error::Atom: bad atom")
                    .to_term(env);
                NifReturned::Term(atom.as_c_arg())
            }
            Error::RaiseAtom(atom_str) => {
                let atom = types::atom::Atom::from_str(env, atom_str)
                    .ok()
                    .expect("Error::RaiseAtom: bad argument");
                NifReturned::Raise(atom.as_c_arg())
            }
            Error::RaiseTerm(ref term_unencoded) => {
                let term = term_unencoded.encode(env);
                NifReturned::Raise(term.as_c_arg())
            }
        }
    }
}

impl fmt::Debug for Error {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match self {
            Error::BadArg => write!(fmt, "{{error, badarg}}"),
            Error::Atom(ref s) => write!(fmt, "{{error, {}}}", s),
            Error::RaiseAtom(ref s) => write!(fmt, "throw({})", s),
            Error::RaiseTerm(_) => write!(fmt, "throw(<term>)"),
        }
    }
}<commit_msg>Remove unrelated obsolete ok() call<commit_after>use crate::codegen_runtime::{NifReturnable, NifReturned};
use crate::{types, Encoder, Env};
use std::fmt;

/// Represents usual errors that can happen in a nif. This enables you
/// to return an error from anywhere, even places where you don't have
/// an Env available.
pub enum Error {
    /// Returned when the NIF has been called with the wrong number or type of
    /// arguments.
    BadArg,
    /// Encodes the string into an atom and returns it from the NIF.
    Atom(&'static str),
    RaiseAtom(&'static str),
    RaiseTerm(Box<dyn Encoder>),
}

unsafe impl NifReturnable for crate::error::Error {
    unsafe fn as_returned(self, env: Env) -> NifReturned {
        match self {
            Error::BadArg => NifReturned::BadArg,
            Error::Atom(atom_str) => {
                let atom = types::atom::Atom::from_str(env, atom_str)
                    .expect("Error::Atom: bad atom")
                    .to_term(env);
                NifReturned::Term(atom.as_c_arg())
            }
            Error::RaiseAtom(atom_str) => {
                let atom = types::atom::Atom::from_str(env, atom_str)
                    .expect("Error::RaiseAtom: bad argument");
                NifReturned::Raise(atom.as_c_arg())
            }
            Error::RaiseTerm(ref term_unencoded) => {
                let term = term_unencoded.encode(env);
                NifReturned::Raise(term.as_c_arg())
            }
        }
    }
}

impl fmt::Debug for Error {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match self {
            Error::BadArg => write!(fmt, "{{error, badarg}}"),
            Error::Atom(ref s) => write!(fmt, "{{error, {}}}", s),
            Error::RaiseAtom(ref s) => write!(fmt, "throw({})", s),
            Error::RaiseTerm(_) => write!(fmt, "throw(<term>)"),
        }
    }
}
<|endoftext|>