<commit_before>use crate::proto::grpc_frame::parse_grpc_frame_header;
use crate::error;
use crate::result;
use bytes::Buf;
use bytes::Bytes;
use httpbis::BufGetBytes;
use httpbis::BytesDeque;

#[derive(Default)]
pub(crate) struct GrpcFrameParser {
    /// Next frame length; stripped prefix from the queue
    next_frame_len: Option<u32>,
    /// Unparsed bytes
    buffer: BytesDeque,
}

impl GrpcFrameParser {
    /// Add `Bytes` object to the queue.
    pub fn enqueue(&mut self, bytes: Bytes) {
        self.buffer.extend(bytes);
    }

    /// Try populating `next_frame_len` field from the queue.
    fn fill_next_frame_len(&mut self) -> result::Result<Option<u32>> {
        if let None = self.next_frame_len {
            self.next_frame_len = parse_grpc_frame_header(&mut self.buffer)?;
        }
        Ok(self.next_frame_len)
    }

    /// Parse next frame if buffer has full frame.
    pub fn next_frame(&mut self) -> result::Result<Option<Bytes>> {
        if let Some(len) = self.fill_next_frame_len()? {
            if self.buffer.remaining() > len as usize {
                self.next_frame_len = None;
                return Ok(Some(BufGetBytes::get_bytes(&mut self.buffer, len as usize)));
            }
        }
        Ok(None)
    }

    /// Parse all frames from buffer.
    pub fn next_frames(&mut self) -> result::Result<Vec<Bytes>> {
        let mut r = Vec::new();
        while let Some(frame) = self.next_frame()? {
            r.push(frame);
        }
        Ok(r)
    }

    /// Buffered data is empty.
    pub fn is_empty(&self) -> bool {
        self.next_frame_len.is_none() && !self.buffer.has_remaining()
    }

    pub fn check_empty(&self) -> result::Result<()> {
        if !self.is_empty() {
            return Err(error::Error::Other("partial frame"));
        }
        Ok(())
    }
}
<commit_msg>Fix frame parsing<commit_after>use crate::error;
use crate::proto::grpc_frame::parse_grpc_frame_header;
use crate::result;
use bytes::Buf;
use bytes::Bytes;
use httpbis::BufGetBytes;
use httpbis::BytesDeque;

#[derive(Default, Debug)]
pub(crate) struct GrpcFrameParser {
    /// Next frame length; stripped prefix from the queue
    next_frame_len: Option<u32>,
    /// Unparsed bytes
    buffer: BytesDeque,
}

impl GrpcFrameParser {
    /// Add `Bytes` object to the queue.
    pub fn enqueue(&mut self, bytes: Bytes) {
        self.buffer.extend(bytes);
    }

    /// Try populating `next_frame_len` field from the queue.
    fn fill_next_frame_len(&mut self) -> result::Result<Option<u32>> {
        if let None = self.next_frame_len {
            self.next_frame_len = parse_grpc_frame_header(&mut self.buffer)?;
        }
        Ok(self.next_frame_len)
    }

    /// Parse next frame if buffer has full frame.
    pub fn next_frame(&mut self) -> result::Result<Option<Bytes>> {
        if let Some(len) = self.fill_next_frame_len()? {
            if self.buffer.remaining() >= len as usize {
                self.next_frame_len = None;
                return Ok(Some(BufGetBytes::get_bytes(&mut self.buffer, len as usize)));
            }
        }
        Ok(None)
    }

    /// Parse all frames from buffer.
    pub fn next_frames(&mut self) -> result::Result<Vec<Bytes>> {
        let mut r = Vec::new();
        while let Some(frame) = self.next_frame()? {
            r.push(frame);
        }
        Ok(r)
    }

    /// Buffered data is empty.
    pub fn is_empty(&self) -> bool {
        self.next_frame_len.is_none() && !self.buffer.has_remaining()
    }

    pub fn check_empty(&self) -> result::Result<()> {
        if !self.is_empty() {
            return Err(error::Error::Other("partial frame"));
        }
        Ok(())
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use bytes::Bytes;

    #[test]
    fn test() {
        fn parse(data: &[u8]) -> result::Result<Option<Bytes>> {
            let mut parser = GrpcFrameParser::default();
            parser.enqueue(Bytes::copy_from_slice(data));
            parser.next_frame()
        }

        assert_eq!(None, parse(b"").unwrap());
        assert_eq!(None, parse(b"1").unwrap());
        assert_eq!(None, parse(b"14sc").unwrap());
        assert_eq!(None, parse(b"\x00\x00\x00\x00\x07\x0a\x05wo").unwrap());
        assert_eq!(
            Some(Bytes::copy_from_slice(b"\x0a\x05world")),
            parse(b"\x00\x00\x00\x00\x07\x0a\x05world").unwrap()
        );
    }
}
<|endoftext|>


<commit_before>use std::fmt::Show;
use std::hash::{hash,Hash};
use std::collections::HashMap;
use std::thunk::Invoke;
use std::mem::replace;
//use std::sync::Arc;
use std::rc::Rc;
use std::fmt;

// "One big memo table" design

pub trait Adapton {
    fn new () -> Self ;

    // Names
    fn name_of_string (self:&mut Self, String) -> Name ;
    fn name_of_u64 (self:&mut Self, u64) -> Name ;
    fn name_pair (self: &Self, Name, Name) -> Name ;
    fn name_fork (self:&mut Self, Name) -> (Name, Name) ;

    fn ns<T,F> (self: &mut Self, Name, body:F) -> T
        where F:FnOnce(&mut Self) -> T ;

    fn put<T:Eq+Clone+Show> (self:&mut Self, T) -> Art<T> ;

    // Mutable cells
    fn cell<T:Eq+Clone+Show> (self:&mut Self, ArtId, T) -> MutArt<T> ;
    fn set<T:Eq+Clone+Show> (self:&mut Self, MutArt<T>, T) ;

    fn thunk<Arg:Eq+Hash+Clone+Show,T:Eq+Clone+Show>
        (self:&mut Self, id:ArtId,
         fn_body:Box<Invoke<(&mut Self, Arg), T> + 'static>, arg:Arg) -> Art<T> ;

    fn force<T:Eq+Clone+Show> (self:&mut Self, Art<T>) -> T ;
}

#[derive(Hash,Show,PartialEq,Eq)]
pub enum Lineage {
    Unknown, Structural,
    Root,
    Pair(Rc<Lineage>,Rc<Lineage>),
    ForkL(Rc<Lineage>),
    ForkR(Rc<Lineage>),
    String(String),
    U64(u64),
    Rc(Rc<Lineage>),
}

#[derive(Hash,Show,PartialEq,Eq)]
pub enum Path {
    Empty,
    Child(Rc<Path>,Name),
}

#[derive(Hash,Show,PartialEq,Eq)]
pub struct Name {
    hash : u64,
    lineage : Rc<Lineage>,
}

#[derive(Hash,Show,PartialEq,Eq)]
pub struct Loc {
    hash : u64,
    name : Rc<Name>,
    path : Rc<Path>
}

#[derive(Hash,Show,PartialEq,Eq)]
pub enum ArtId {
    None,            // Identifies an Art::Box. No dependency tracking.
    Structural(u64), // Identifies an Art::Loc.
    Nominal(Name),   // Identifies an Art::Loc.
}

#[derive(Hash,Show,PartialEq,Eq)]
pub enum Art<T> {
    Box(Box<T>),  // No entry in table. No dependency tracking.
    Loc(Rc<Loc>), // Location in table.
}

#[derive(Hash,Show,PartialEq,Eq)]
pub enum MutArt<T> {
    MutArt(Art<T>)
}

#[derive(Show)]
enum Node<Res,Comp:Computer<Res>> {
    Pure(PureNode<Res>),
    Mut(MutNode<Res>),
    Compute(ComputeNode<Res,Comp>),
}

#[derive(Show)]
struct PureNode<T> {
    loc : Rc<Loc>,
    val : T,
}

#[derive(Show)]
struct MutNode<T> {
    loc : Rc<Loc>,
    creators  : Vec<DemPrec>,
    dem_precs : Vec<DemPrec>,
    val : T,
}

trait Computer<Res> {
    type Arg;
    fn compute(self:&Self, st:&mut AdaptonState, arg:Self::Arg) -> Res;
    //fn size_hint(&self) -> (usize, Option<usize>) { ... }
}

//#[derive(Show)]
struct ComputeNode<Res,Comp:Computer<Res>> {
    loc : Rc<Loc>,
    creators  : Vec<DemPrec>,
    dem_precs : Vec<DemPrec>,
    dem_succs : Vec<DemSucc>,
    arg : Comp::Arg,
    res : Option<Res>,
    //body : Box<Invoke<(&'static mut A, Arg),Res> + 'static>,
    comp : Comp,
}

impl<Res,Comp:Computer<Res>> fmt::Show for ComputeNode<Res,Comp> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "(ComputeNode)")
    }
}

#[derive(Show)]
struct DemPrec {
    loc : Rc<Loc>,
}

#[derive(Show)]
struct DemSucc {
    loc : Rc<Loc>,
    dirty : bool
}

pub struct Frame {
    path : Rc<Path>,
    name : Rc<Name>,
    succs : Vec<DemSucc>,
}

pub struct AdaptonState {
    table : HashMap<Rc<Loc>, *mut ()>,
    stack : Vec<Frame>,
}

impl fmt::Show for AdaptonState {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "(AdaptonState)")
    }
}

fn node<'x,Res,Comp:Computer<Res>> (st: &'x mut AdaptonState, loc:&Rc<Loc>) -> &'x mut Node<Res,Comp> {
    let node = st.table.get(loc) ;
    match node {
        None => panic!("dangling pointer: {}", loc),
        Some(ptr) => {
            let ptr = unsafe {
                std::mem::transmute::<
                    *mut (), &mut Node<Res,Comp>
                    >(*ptr) } ;
            ptr
        }
    }
}

fn dem_precs<'x,Res:'x,Comp:Computer<Res>+'x> (st: &'x mut AdaptonState, loc: &Rc<Loc>) -> &'x mut Vec<DemPrec> {
    match *(node::<Res,Comp>(st,loc)) {
        Node::Pure(_) => {
            panic!("Node::Pure: no precs")
        },
        Node::Compute(ref mut nd) => {
            &mut nd.dem_precs
        },
        Node::Mut(ref mut nd) => {
            &mut nd.dem_precs
        }
    }
}

fn revoke_demand<'x,Res:'x,Comp:Computer<Res>+'x> (st:&mut AdaptonState, src:&Rc<Loc>, succs:&Vec<DemSucc>) {
    for succ in succs.iter() {
        let precs = dem_precs::<Res,Comp>(st, &succ.loc);
        precs.retain(|ref prec| &prec.loc != src);
    }
}

fn invoke_demand<'x,Res:'x,Comp:Computer<Res>+'x> (st:&mut AdaptonState, src:Rc<Loc>, succs:& Vec<DemSucc>) {
    for succ in succs.iter() {
        let precs = dem_precs::<Res,Comp>(st, &succ.loc);
        precs.push(DemPrec{loc:src.clone()})
    }
}

impl Adapton for AdaptonState {

    fn new () -> AdaptonState {
        let empty = Rc::new(Path::Empty);
        let root = Rc::new(Name{hash:0, lineage:Rc::new(Lineage::Root) });
        let mut stack = Vec::new();
        stack.push( Frame{path:empty, name:root, succs:Vec::new()} ) ;
        AdaptonState {
            table : HashMap::new (),
            stack : stack,
        }
    }

    fn name_of_string (self:&mut AdaptonState, sym:String) -> Name {
        let h = hash::<_>(&sym) ;
        let s = Lineage::String(sym) ;
        Name{ hash:h, lineage:Rc::new(s) }
    }

    fn name_of_u64 (self:&mut AdaptonState, sym:u64) -> Name {
        let h = hash::<_>(&sym) ;
        let s = Lineage::U64(sym) ;
        Name{ hash:h, lineage:Rc::new(s) }
    }

    fn name_pair (self: &AdaptonState, fst: Name, snd: Name) -> Name {
        let h = hash::<_>( &(fst.hash,snd.hash) ) ;
        let p = Lineage::Pair(fst.lineage, snd.lineage) ;
        Name{ hash:h, lineage:Rc::new(p) }
    }

    fn name_fork (self:&mut AdaptonState, nm:Name) -> (Name, Name) {
        let h1 = hash::<_>( &(&nm, 11111111) ) ; // TODO: make this hashing better.
        let h2 = hash::<_>( &(&nm, 22222222) ) ;
        ( Name{ hash:h1,
                lineage:Rc::new(Lineage::ForkL(nm.lineage.clone())) } ,
          Name{ hash:h2,
                lineage:Rc::new(Lineage::ForkR(nm.lineage)) } )
    }

    fn ns<T,F> (self: &mut Self, nm:Name, body:F) -> T where F:FnOnce(&mut Self) -> T {
        let path_body = Rc::new(Path::Child(self.stack[0].path.clone(), nm)) ;
        let path_pre = replace(&mut self.stack[0].path, path_body ) ;
        let x = body(self) ;
        let path_body = replace(&mut self.stack[0].path, path_pre) ;
        drop(path_body);
        x
    }

    fn put<T:Eq> (self:&mut AdaptonState, x:T) -> Art<T> {
        Art::Box(box x)
    }

    fn cell<T:Eq+Clone+Show,Comp:Computer<T>> (self:&mut AdaptonState, id:ArtId, val:T) -> MutArt<T> {
        let loc = match id {
            ArtId::None => panic!("a cell requires a unique identity"),
            ArtId::Structural(hash) => {
                Rc::new(Loc{
                    hash:hash,
                    name:Rc::new(Name{hash:hash,
                                      lineage:Rc::new(Lineage::Structural)}),
                    path:self.stack[0].path.clone()
                })
            }
            ArtId::Nominal(nm) => {
                Rc::new(Loc{
                    hash:nm.hash,
                    name:Rc::new(nm),
                    path:self.stack[0].path.clone() })
            }} ;
        let mut node = Node::Mut(MutNode{
            loc:loc.clone(),
            dem_precs:Vec::new(),
            creators:Vec::new(),
            val:val
        }) ;
        let ptr = unsafe { std::mem::transmute::<
                           *mut Node<T,Comp>,
                           *mut ()
                           >(&mut node) } ;
        // TODO: Check to see if the cell exists;
        // check if its content has changed.
        // dirty its precs if so.
        self.table.insert(loc.clone(), ptr) ;
        MutArt::MutArt(Art::Loc(loc))
    }

    fn set<T:Eq+Clone+Show> (self:&mut Self, cell:MutArt<T>, val:T) {
        match cell {
            MutArt::MutArt(Art::Box(b)) => {
                panic!("cannot set pure value");
            }
            MutArt::MutArt(Art::Loc(loc)) => {
                let node = self.table.get(&loc) ;
                match node {
                    None => panic!("dangling pointer: {}", loc),
                    Some(ptr) => {
                        let node : &mut MutNode<T> = unsafe {
                            let node = std::mem::transmute::<
                                *mut (), *mut Node<T,Computer<T>>
                                >(*ptr) ;
                            match *node {
                                Node::Mut(ref mut nd) => nd,
                                ref nd => panic!("impossible: {}", nd)
                            }} ;
                        if (node.val == val) {
                            // Nothing.
                        }
                        else {
                            node.val = val;
                            // TODO: Dirty traversal. Notify/mark demand precs.
                        }
                    }
                }
            }
        }
    }

    fn thunk<Arg:Eq+Hash+Clone+Show,T:Eq+Clone+Show>
        (self:&mut AdaptonState,
         id:ArtId, fn_body:Box<Invoke<(&mut Self, Arg),T>+'static>, arg:Arg) -> Art<T>
    {
        match id {
            ArtId::None => {
                Art::Box(box fn_body.invoke((self,arg)))
            },
            ArtId::Structural(hash) => {
                panic!("")
            },
            ArtId::Nominal(nm) => {
                panic!("")
            }
        }
    }

    fn force<T:Eq+Clone+Show> (self:&mut AdaptonState, art:Art<T>) -> T {
        match art {
            Art::Box(b) => *b,
            Art::Loc(loc) => {
                let node = self.table.get(&loc) ;
                match node {
                    None => panic!("dangling pointer: {}", loc),
                    Some(ptr) => {
                        let node : &mut Node<T,Computer<T>> = unsafe {
                            let node = std::mem::transmute::<*mut (), *mut Node<T,Computer<T>>>(*ptr) ;
                            &mut ( *node ) } ;
                        match *node {
                            Node::Pure(ref mut nd) => {
                                nd.val.clone()
                            },
                            Node::Mut(ref mut nd) => {
                                if self.stack.is_empty() { } else {
                                    self.stack[0].succs.push(DemSucc{loc:loc.clone(),dirty:false});
                                } ;
                                nd.val.clone()
                            },
                            Node::Compute(ref mut nd) => {
                                self.stack.push ( Frame{name:loc.name.clone(),
                                                        path:loc.path.clone(),
                                                        succs:Vec::new(), } );
                                let val = panic!("TODO: run compute node body") ;
                                let mut frame = match
                                    self.stack.pop() { None => panic!(""), Some(frame) => frame } ;
                                revoke_demand( self, &nd.loc, &nd.dem_succs );
                                invoke_demand( self, nd.loc.clone(), &frame.succs );
                                nd.dem_succs = frame.succs;
                                if self.stack.is_empty() { } else {
                                    self.stack[0].succs.push(DemSucc{loc:loc.clone(),dirty:false});
                                };
                                val
                            }
                        }
                    }
                }
            }
        }
    }

}

pub fn main () {

}
<commit_msg>rustup, still have ICE<commit_after>#![feature(box_syntax)]

use std::fmt::Debug;
use std::hash::{hash,Hash,SipHasher};
use std::collections::HashMap;
use std::thunk::Invoke;
use std::mem::replace;
//use std::sync::Arc;
use std::rc::Rc;
use std::fmt;

// "One big memo table" design

pub trait Adapton {
    fn new () -> Self ;

    // Names
    fn name_of_string (self:&mut Self, String) -> Name ;
    fn name_of_u64 (self:&mut Self, u64) -> Name ;
    fn name_pair (self: &Self, Name, Name) -> Name ;
    fn name_fork (self:&mut Self, Name) -> (Name, Name) ;

    fn ns<T,F> (self: &mut Self, Name, body:F) -> T
        where F:FnOnce(&mut Self) -> T ;

    fn put<T:Eq+Clone+Debug> (self:&mut Self, T) -> Art<T> ;

    // Mutable cells
    fn cell<T:Eq+Clone+Debug> (self:&mut Self, ArtId, T) -> MutArt<T> ;
    fn set<T:Eq+Clone+Debug> (self:&mut Self, MutArt<T>, T) ;

    fn thunk<Arg:Eq+Hash<SipHasher>+Clone+Debug,T:Eq+Clone+Debug>
        (self:&mut Self, id:ArtId,
         fn_body:Box<Invoke<(&mut Self, Arg), T> + 'static>, arg:Arg) -> Art<T> ;

    fn force<T:Eq+Clone+Debug> (self:&mut Self, Art<T>) -> T ;
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub enum Lineage {
    Unknown, Structural,
    Root,
    Pair(Rc<Lineage>,Rc<Lineage>),
    ForkL(Rc<Lineage>),
    ForkR(Rc<Lineage>),
    String(String),
    U64(u64),
    Rc(Rc<Lineage>),
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub enum Path {
    Empty,
    Child(Rc<Path>,Name),
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub struct Name {
    hash : u64,
    lineage : Rc<Lineage>,
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub struct Loc {
    hash : u64,
    name : Rc<Name>,
    path : Rc<Path>
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub enum ArtId {
    None,            // Identifies an Art::Box. No dependency tracking.
    Structural(u64), // Identifies an Art::Loc.
    Nominal(Name),   // Identifies an Art::Loc.
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub enum Art<T> {
    Box(Box<T>),  // No entry in table. No dependency tracking.
    Loc(Rc<Loc>), // Location in table.
}

#[derive(Hash,Debug,PartialEq,Eq)]
pub enum MutArt<T> {
    MutArt(Art<T>)
}

#[derive(Debug)]
enum Node<Res> {
    Pure(PureNode<Res>),
    Mut(MutNode<Res>),
    Compute(Box<Computer<Res>>),
}

#[derive(Debug)]
struct PureNode<T> {
    loc : Rc<Loc>,
    val : T,
}

#[derive(Debug)]
struct MutNode<T> {
    loc : Rc<Loc>,
    creators  : Vec<DemPrec>,
    dem_precs : Vec<DemPrec>,
    val : T,
}

trait Clos where Self::Comp : Computer<Self::Res> {
    type Res;
    type Comp;
    fn get_node<'x>(self:&'x mut Self) ->
        &'x mut ComputeNode<Self::Res,Self::Comp> ;
}

//#[derive(Debug)]
struct ComputeNode<Res,Comp:Computer<Res>> {
    loc : Rc<Loc>,
    creators  : Vec<DemPrec>,
    dem_precs : Vec<DemPrec>,
    dem_succs : Vec<DemSucc>,
    arg : Comp::Arg,
    res : Option<Res>,
    //body : Box<Invoke<(&'static mut A, Arg),Res> + 'static>,
    comp : Box<Comp>,
}

trait Computer<Res> {
    type Arg;
    fn compute(self:&Self, st:&mut AdaptonState, arg:Self::Arg) -> Res;
    //fn size_hint(&self) -> (usize, Option<usize>) { ... }
}

impl<Res,Comp:Computer<Res>> fmt::Debug for ComputeNode<Res,Comp> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "(ComputeNode)")
    }
}

#[derive(Debug)]
struct DemPrec {
    loc : Rc<Loc>,
}

#[derive(Debug)]
struct DemSucc {
    loc : Rc<Loc>,
    dirty : bool
}

pub struct Frame {
    path : Rc<Path>,
    name : Rc<Name>,
    succs : Vec<DemSucc>,
}

pub struct AdaptonState {
    table : HashMap<Rc<Loc>, *mut ()>,
    stack : Vec<Frame>,
}

impl fmt::Debug for AdaptonState {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "(AdaptonState)")
    }
}

fn node<'x,Res,Comp:Computer<Res>> (st: &'x mut AdaptonState, loc:&Rc<Loc>) -> &'x mut Node<Res,Comp> {
    let node = st.table.get(loc) ;
    match node {
        None => panic!("dangling pointer"),
        Some(ptr) => {
            let ptr = unsafe {
                std::mem::transmute::<
                    *mut (), &mut Node<Res,Comp>
                    >(*ptr) } ;
            ptr
        }
    }
}

fn dem_precs<'x,Res:'x,Comp:Computer<Res>+'x> (st: &'x mut AdaptonState, loc: &Rc<Loc>) -> &'x mut Vec<DemPrec> {
    match *(node::<Res,Comp>(st,loc)) {
        Node::Pure(_) => {
            panic!("Node::Pure: no precs")
        },
        Node::Compute(ref mut nd) => {
            &mut nd.dem_precs
        },
        Node::Mut(ref mut nd) => {
            &mut nd.dem_precs
        }
    }
}

fn revoke_demand<'x,Res:'x,Comp:Computer<Res>+'x> (st:&mut AdaptonState, src:&Rc<Loc>, succs:&Vec<DemSucc>) {
    for succ in succs.iter() {
        let precs = dem_precs::<Res,Comp>(st, &succ.loc);
        precs.retain(|ref prec| &prec.loc != src);
    }
}

fn invoke_demand<'x,Res:'x,Comp:Computer<Res>+'x> (st:&mut AdaptonState, src:Rc<Loc>, succs:& Vec<DemSucc>) {
    for succ in succs.iter() {
        let precs = dem_precs::<Res,Comp>(st, &succ.loc);
        precs.push(DemPrec{loc:src.clone()})
    }
}

impl Adapton for AdaptonState {

    fn new () -> AdaptonState {
        let empty = Rc::new(Path::Empty);
        let root = Rc::new(Name{hash:0, lineage:Rc::new(Lineage::Root) });
        let mut stack = Vec::new();
        stack.push( Frame{path:empty, name:root, succs:Vec::new()} ) ;
        AdaptonState {
            table : HashMap::new (),
            stack : stack,
        }
    }

    fn name_of_string (self:&mut AdaptonState, sym:String) -> Name {
        let h = hash::<_,SipHasher>(&sym) ;
        let s = Lineage::String(sym) ;
        Name{ hash:h, lineage:Rc::new(s) }
    }

    fn name_of_u64 (self:&mut AdaptonState, sym:u64) -> Name {
        let h = hash::<_,SipHasher>(&sym) ;
        let s = Lineage::U64(sym) ;
        Name{ hash:h, lineage:Rc::new(s) }
    }

    fn name_pair (self: &AdaptonState, fst: Name, snd: Name) -> Name {
        let h = hash::<_,SipHasher>( &(fst.hash,snd.hash) ) ;
        let p = Lineage::Pair(fst.lineage, snd.lineage) ;
        Name{ hash:h, lineage:Rc::new(p) }
    }

    fn name_fork (self:&mut AdaptonState, nm:Name) -> (Name, Name) {
        let h1 = hash::<_,SipHasher>( &(&nm, 11111111) ) ; // TODO: make this hashing better.
        let h2 = hash::<_,SipHasher>( &(&nm, 22222222) ) ;
        ( Name{ hash:h1,
                lineage:Rc::new(Lineage::ForkL(nm.lineage.clone())) } ,
          Name{ hash:h2,
                lineage:Rc::new(Lineage::ForkR(nm.lineage)) } )
    }

    fn ns<T,F> (self: &mut Self, nm:Name, body:F) -> T where F:FnOnce(&mut Self) -> T {
        let path_body = Rc::new(Path::Child(self.stack[0].path.clone(), nm)) ;
        let path_pre = replace(&mut self.stack[0].path, path_body ) ;
        let x = body(self) ;
        let path_body = replace(&mut self.stack[0].path, path_pre) ;
        drop(path_body);
        x
    }

    fn put<T:Eq> (self:&mut AdaptonState, x:T) -> Art<T> {
        Art::Box(Box::new(x))
    }

    fn cell<T:Eq+Clone+Debug> (self:&mut AdaptonState, id:ArtId, val:T) -> MutArt<T> {
        let loc = match id {
            ArtId::None => panic!("a cell requires a unique identity"),
            ArtId::Structural(hash) => {
                Rc::new(Loc{
                    hash:hash,
                    name:Rc::new(Name{hash:hash,
                                      lineage:Rc::new(Lineage::Structural)}),
                    path:self.stack[0].path.clone()
                })
            }
            ArtId::Nominal(nm) => {
                Rc::new(Loc{
                    hash:nm.hash,
                    name:Rc::new(nm),
                    path:self.stack[0].path.clone() })
            }} ;
        let mut node = Node::Mut(MutNode{
            loc:loc.clone(),
            dem_precs:Vec::new(),
            creators:Vec::new(),
            val:val
        }) ;
        let ptr = panic!("") ;
            // unsafe { std::mem::transmute::<
            // *mut Node<T,Comp>,
            // *mut ()
            // >(&mut node) } ;
        // TODO: Check to see if the cell exists;
        // check if its content has changed.
        // dirty its precs if so.
        self.table.insert(loc.clone(), ptr) ;
        MutArt::MutArt(Art::Loc(loc))
    }

    fn set<T:Eq+Clone+Debug> (self:&mut Self, cell:MutArt<T>, val:T) {
        match cell {
            MutArt::MutArt(Art::Box(b)) => {
                panic!("cannot set pure value");
            }
            MutArt::MutArt(Art::Loc(loc)) => {
                let node = self.table.get(&loc) ;
                match node {
                    None => panic!("dangling pointer"),
                    Some(ptr) => {
                        let node : &mut MutNode<T> = unsafe {
                            let node = std::mem::transmute::<
                                *mut (), *mut Node<T,Computer<T>>
                                >(*ptr) ;
                            match *node {
                                Node::Mut(ref mut nd) => nd,
                                ref nd => panic!("impossible")
                            }} ;
                        if (node.val == val) {
                            // Nothing.
                        }
                        else {
                            node.val = val;
                            // TODO: Dirty traversal. Notify/mark demand precs.
                        }
                    }
                }
            }
        }
    }

    fn thunk<Arg:Eq+Hash<SipHasher>+Clone+Debug,T:Eq+Clone+Debug>
        (self:&mut AdaptonState,
         id:ArtId, fn_body:Box<Invoke<(&mut Self, Arg),T>+'static>, arg:Arg) -> Art<T>
    {
        match id {
            ArtId::None => {
                Art::Box(Box::new(fn_body.invoke((self,arg))))
            },
            ArtId::Structural(hash) => {
                panic!("")
            },
            ArtId::Nominal(nm) => {
                panic!("")
            }
        }
    }

    fn force<T:Eq+Clone+Debug> (self:&mut AdaptonState, art:Art<T>) -> T {
        match art {
            Art::Box(b) => *b,
            Art::Loc(loc) => {
                let node = self.table.get(&loc) ;
                match node {
                    None => panic!("dangling pointer"),
                    Some(ptr) => {
                        let node : &mut Node<T,Computer<T>> = unsafe {
                            let node = std::mem::transmute::<*mut (), *mut Node<T,Computer<T>>>(*ptr) ;
                            &mut ( *node ) } ;
                        match *node {
                            Node::Pure(ref mut nd) => {
                                nd.val.clone()
                            },
                            Node::Mut(ref mut nd) => {
                                if self.stack.is_empty() { } else {
                                    self.stack[0].succs.push(DemSucc{loc:loc.clone(),dirty:false});
                                } ;
                                nd.val.clone()
                            },
                            Node::Compute(ref mut nd) => {
                                self.stack.push ( Frame{name:loc.name.clone(),
                                                        path:loc.path.clone(),
                                                        succs:Vec::new(), } );
                                let val = panic!("TODO: run compute node body") ;
                                let mut frame = match
                                    self.stack.pop() { None => panic!(""), Some(frame) => frame } ;
                                revoke_demand( self, &nd.loc, &nd.dem_succs );
                                invoke_demand( self, nd.loc.clone(), &frame.succs );
                                nd.dem_succs = frame.succs;
                                if self.stack.is_empty() { } else {
                                    self.stack[0].succs.push(DemSucc{loc:loc.clone(),dirty:false});
                                };
                                val
                            }
                        }
                    }
                }
            }
        }
    }

}

pub fn main () {

}
<|endoftext|>


<commit_before>use std::ops::Deref;

use crate::*;

/// The `sled` embedded database! Implements
/// `Deref<Target = sled::Tree>` to refer to
/// a default keyspace.
#[derive(Clone)]
pub struct Db {
    #[doc(hidden)]
    pub context: Context,
    pub(crate) default: Tree,
    tenants: Arc<RwLock<FastMap8<IVec, Tree>>>,
}

/// Opens a `Db` with a default configuration at the
/// specified path. This will create a new storage
/// directory at the specified path if it does
/// not already exist. You can use the `Db::was_recovered`
/// method to determine if your database was recovered
/// from a previous instance. You can use `Config::create_new`
/// if you want to increase the chances that the database
/// will be freshly created.
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Db> {
    Config::new().path(path).open()
}

#[allow(unsafe_code)]
unsafe impl Send for Db {}

#[allow(unsafe_code)]
unsafe impl Sync for Db {}

impl Deref for Db {
    type Target = Tree;

    fn deref(&self) -> &Tree {
        &self.default
    }
}

impl Debug for Db {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        let tenants = self.tenants.read();
        writeln!(f, "Db {{")?;
        for (raw_name, tree) in tenants.iter() {
            let name = std::str::from_utf8(raw_name)
                .ok()
                .map_or_else(|| format!("{:?}", raw_name), String::from);
            write!(f, "    Tree: {:?} contents: {:?}", name, tree)?;
        }
        write!(f, "}}")?;
        Ok(())
    }
}

impl Db {
    #[doc(hidden)]
    #[deprecated(since = "0.30.2", note = "replaced by `sled::open`")]
    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
        Config::new().path(path).open()
    }

    pub(crate) fn start_inner(config: RunningConfig) -> Result<Self> {
        let _measure = Measure::new(&M.tree_start);

        let context = Context::start(config)?;

        #[cfg(all(
            not(miri),
            any(
                windows,
                target_os = "linux",
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "openbsd",
                target_os = "netbsd",
            )
        ))]
        {
            let flusher_pagecache = context.pagecache.clone();
            let flusher = context.flush_every_ms.map(move |fem| {
                flusher::Flusher::new(
                    "log flusher".to_owned(),
                    flusher_pagecache,
                    fem,
                )
            });
            *context.flusher.lock() = flusher;
        }

        // create or open the default tree
        let guard = pin();
        let default =
            meta::open_tree(&context, DEFAULT_TREE_ID.to_vec(), &guard)?;

        let ret = Self {
            context: context.clone(),
            default,
            tenants: Arc::new(RwLock::new(FastMap8::default())),
        };

        let mut tenants = ret.tenants.write();

        for (id, root) in context.pagecache.get_meta(&guard)?.tenants() {
            let tree = Tree(Arc::new(TreeInner {
                tree_id: id.clone(),
                subscribers: Subscribers::default(),
                context: context.clone(),
                root: AtomicU64::new(root),
                merge_operator: RwLock::new(None),
            }));
            assert!(tenants.insert(id, tree).is_none());
        }

        drop(tenants);

        #[cfg(feature = "event_log")]
        {
            for (_name, tree) in ret.tenants.read().iter() {
                tree.verify_integrity().unwrap();
            }
            ret.context.event_log.verify();
        }

        Ok(ret)
    }

    /// Open or create a new disk-backed Tree with its own keyspace,
    /// accessible from the `Db` via the provided identifier.
    pub fn open_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<Tree> {
        let name_ref = name.as_ref();
        let tenants = self.tenants.read();
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }
        drop(tenants);

        let guard = pin();

        let mut tenants = self.tenants.write();

        // we need to check this again in case another
        // thread opened it concurrently.
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }

        let tree = meta::open_tree(&self.context, name_ref.to_vec(), &guard)?;

        assert!(tenants.insert(name_ref.into(), tree.clone()).is_none());

        Ok(tree)
    }

    /// Remove a disk-backed collection.
    pub fn drop_tree(&self, name: &[u8]) -> Result<bool> {
        if name == DEFAULT_TREE_ID {
            return Err(Error::Unsupported(
                "cannot remove the core structures".into(),
            ));
        }
        trace!("dropping tree {:?}", name,);

        let mut tenants = self.tenants.write();

        let tree = if let Some(tree) = tenants.remove(&*name) {
            tree
        } else {
            return Ok(false);
        };

        let guard = pin();

        let mut root_id =
            Some(self.context.pagecache.meta_pid_for_name(name, &guard)?);

        let mut leftmost_chain: Vec<PageId> = vec![root_id.unwrap()];
        let mut cursor = root_id.unwrap();
        while let Some(view) = self.view_for_pid(cursor, &guard)? {
            if let Some(index) = view.data.index_ref() {
                let leftmost_child = index.pointers[0];
                leftmost_chain.push(leftmost_child);
                cursor = leftmost_child;
            } else {
                break;
            }
        }

        loop {
            let res = self
                .context
                .pagecache
                .cas_root_in_meta(name, root_id, None, &guard)?;

            if let Err(actual_root) = res {
                root_id = actual_root;
            } else {
                break;
            }
        }

        tree.root.store(u64::max_value(), SeqCst);

        // drop writer lock
        drop(tenants);

        tree.gc_pages(leftmost_chain)?;

        guard.flush();

        Ok(true)
    }

    /// Returns the trees names saved in this Db.
    pub fn tree_names(&self) -> Vec<IVec> {
        let tenants = self.tenants.read();
        tenants.iter().map(|(name, _)| name.clone()).collect()
    }

    /// Returns `true` if the database was
    /// recovered from a previous process.
    /// Note that database state is only
    /// guaranteed to be present up to the
    /// last call to `flush`! Otherwise state
    /// is synced to disk periodically if the
    /// `sync_every_ms` configuration option
    /// is set to `Some(number_of_ms_between_syncs)`
    /// or if the IO buffer gets filled to
    /// capacity before being rotated.
    pub fn was_recovered(&self) -> bool {
        self.context.was_recovered()
    }

    /// Generate a monotonic ID. Not guaranteed to be
    /// contiguous. Written to disk every `idgen_persist_interval`
    /// operations, followed by a blocking flush. During recovery, we
    /// take the last recovered generated ID and add 2x
    /// the `idgen_persist_interval` to it. While persisting, if the
    /// previous persisted counter wasn't synced to disk yet, we will do
    /// a blocking flush to fsync the latest counter, ensuring
    /// that we will never give out the same counter twice.
    pub fn generate_id(&self) -> Result<u64> {
        self.context.generate_id()
    }

    /// A database export method for all collections in the `Db`,
    /// for use in sled version upgrades. Can be used in combination
    /// with the `import` method below on a database running a later
    /// version.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the export.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn export(
        &self,
    ) -> Vec<(CollectionType, CollectionName, impl Iterator<Item = Vec<Vec<u8>>>)>
    {
        let tenants = self.tenants.read();

        let mut ret = vec![];

        for (name, tree) in tenants.iter() {
            ret.push((
                b"tree".to_vec(),
                name.to_vec(),
                tree.iter().map(|kv_opt| {
                    let kv = kv_opt.unwrap();
                    vec![kv.0.to_vec(), kv.1.to_vec()]
                }),
            ));
        }

        ret
    }

    /// Imports the collections from a previous database.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the import.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn import(
        &self,
        export: Vec<(
            CollectionType,
            CollectionName,
            impl Iterator<Item = Vec<Vec<u8>>>,
        )>,
    ) {
        for (collection_type, collection_name, collection_iter) in export {
            match collection_type {
                ref t if t == b"tree" => {
                    let tree = self
                        .open_tree(collection_name)
                        .expect("failed to open new tree during import");
                    for mut kv in collection_iter {
                        let v = kv
                            .pop()
                            .expect("failed to get value from tree export");
                        let k = kv
                            .pop()
                            .expect("failed to get key from tree export");
                        let old = tree.insert(k, v).expect(
                            "failed to insert value during tree import",
                        );
                        assert!(
                            old.is_none(),
                            "import is overwriting existing data"
                        );
                    }
                }
                other => panic!("unknown collection type {:?}", other),
            }
        }
    }

    /// Returns the CRC32 of all keys and values
    /// in this Db.
    ///
    /// This is O(N) and locks all underlying Trees
    /// for the duration of the entire scan.
    pub fn checksum(&self) -> Result<u32> {
        let tenants_mu = self.tenants.write();

        // we use a btreemap to ensure lexicographic
        // iteration over tree names to have consistent
        // checksums.
        let tenants: BTreeMap<_, _> = tenants_mu.iter().collect();

        let mut hasher = crc32fast::Hasher::new();
        let mut locks = vec![];

        locks.push(concurrency_control::write());

        for (name, tree) in &tenants {
            hasher.update(name);

            let mut iter = tree.iter();
            while let Some(kv_res) = iter.next_inner() {
                let (k, v) = kv_res?;
                hasher.update(&k);
                hasher.update(&v);
            }
        }

        Ok(hasher.finalize())
    }

    /// Returns the on-disk size of the storage files
    /// for this database.
    pub fn size_on_disk(&self) -> Result<u64> {
        self.context.pagecache.size_on_disk()
    }

    /// Traverses all files and calculates their total physical
    /// size, then traverses all pages and calculates their
    /// total logical size, then divides the physical size
    /// by the logical size.
    #[doc(hidden)]
    pub fn space_amplification(&self) -> Result<f64> {
        self.context.pagecache.space_amplification()
    }
}

/// These types provide the information that allows an entire
/// system to be exported and imported to facilitate
/// major upgrades. It is comprised entirely
/// of standard library types to be forward compatible.
/// NB this definitions are expensive to change, because
/// they impact the migration path.
type CollectionType = Vec<u8>;
type CollectionName = Vec<u8>;
<commit_msg>Make Db::drop_tree parameters match open_tree parameters.<commit_after>use std::ops::Deref;

use crate::*;

/// The `sled` embedded database! Implements
/// `Deref<Target = sled::Tree>` to refer to
/// a default keyspace.
#[derive(Clone)]
pub struct Db {
    #[doc(hidden)]
    pub context: Context,
    pub(crate) default: Tree,
    tenants: Arc<RwLock<FastMap8<IVec, Tree>>>,
}

/// Opens a `Db` with a default configuration at the
/// specified path. This will create a new storage
/// directory at the specified path if it does
/// not already exist. You can use the `Db::was_recovered`
/// method to determine if your database was recovered
/// from a previous instance. You can use `Config::create_new`
/// if you want to increase the chances that the database
/// will be freshly created.
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Db> {
    Config::new().path(path).open()
}

#[allow(unsafe_code)]
unsafe impl Send for Db {}

#[allow(unsafe_code)]
unsafe impl Sync for Db {}

impl Deref for Db {
    type Target = Tree;

    fn deref(&self) -> &Tree {
        &self.default
    }
}

impl Debug for Db {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        let tenants = self.tenants.read();
        writeln!(f, "Db {{")?;
        for (raw_name, tree) in tenants.iter() {
            let name = std::str::from_utf8(raw_name)
                .ok()
                .map_or_else(|| format!("{:?}", raw_name), String::from);
            write!(f, "    Tree: {:?} contents: {:?}", name, tree)?;
        }
        write!(f, "}}")?;
        Ok(())
    }
}

impl Db {
    #[doc(hidden)]
    #[deprecated(since = "0.30.2", note = "replaced by `sled::open`")]
    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
        Config::new().path(path).open()
    }

    pub(crate) fn start_inner(config: RunningConfig) -> Result<Self> {
        let _measure = Measure::new(&M.tree_start);

        let context = Context::start(config)?;

        #[cfg(all(
            not(miri),
            any(
                windows,
                target_os = "linux",
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "openbsd",
                target_os = "netbsd",
            )
        ))]
        {
            let flusher_pagecache = context.pagecache.clone();
            let flusher = context.flush_every_ms.map(move |fem| {
                flusher::Flusher::new(
                    "log flusher".to_owned(),
                    flusher_pagecache,
                    fem,
                )
            });
            *context.flusher.lock() = flusher;
        }

        // create or open the default tree
        let guard = pin();
        let default =
            meta::open_tree(&context, DEFAULT_TREE_ID.to_vec(), &guard)?;

        let ret = Self {
            context: context.clone(),
            default,
            tenants: Arc::new(RwLock::new(FastMap8::default())),
        };

        let mut tenants = ret.tenants.write();

        for (id, root) in context.pagecache.get_meta(&guard)?.tenants() {
            let tree = Tree(Arc::new(TreeInner {
                tree_id: id.clone(),
                subscribers: Subscribers::default(),
                context: context.clone(),
                root: AtomicU64::new(root),
                merge_operator: RwLock::new(None),
            }));
            assert!(tenants.insert(id, tree).is_none());
        }

        drop(tenants);

        #[cfg(feature = "event_log")]
        {
            for (_name, tree) in ret.tenants.read().iter() {
                tree.verify_integrity().unwrap();
            }
            ret.context.event_log.verify();
        }

        Ok(ret)
    }

    /// Open or create a new disk-backed Tree with its own keyspace,
    /// accessible from the `Db` via the provided identifier.
    pub fn open_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<Tree> {
        let name_ref = name.as_ref();
        let tenants = self.tenants.read();
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }
        drop(tenants);

        let guard = pin();

        let mut tenants = self.tenants.write();

        // we need to check this again in case another
        // thread opened it concurrently.
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }

        let tree = meta::open_tree(&self.context, name_ref.to_vec(), &guard)?;

        assert!(tenants.insert(name_ref.into(), tree.clone()).is_none());

        Ok(tree)
    }

    /// Remove a disk-backed collection.
    pub fn drop_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<bool> {
        let name = name.as_ref();
        if name == DEFAULT_TREE_ID {
            return Err(Error::Unsupported(
                "cannot remove the core structures".into(),
            ));
        }
        trace!("dropping tree {:?}", name,);

        let mut tenants = self.tenants.write();

        let tree = if let Some(tree) = tenants.remove(&*name) {
            tree
        } else {
            return Ok(false);
        };

        let guard = pin();

        let mut root_id =
            Some(self.context.pagecache.meta_pid_for_name(name, &guard)?);

        let mut leftmost_chain: Vec<PageId> = vec![root_id.unwrap()];
        let mut cursor = root_id.unwrap();
        while let Some(view) = self.view_for_pid(cursor, &guard)? {
            if let Some(index) = view.data.index_ref() {
                let leftmost_child = index.pointers[0];
                leftmost_chain.push(leftmost_child);
                cursor = leftmost_child;
            } else {
                break;
            }
        }

        loop {
            let res = self
                .context
                .pagecache
                .cas_root_in_meta(name, root_id, None, &guard)?;

            if let Err(actual_root) = res {
                root_id = actual_root;
            } else {
                break;
            }
        }

        tree.root.store(u64::max_value(), SeqCst);

        // drop writer lock
        drop(tenants);

        tree.gc_pages(leftmost_chain)?;

        guard.flush();

        Ok(true)
    }

    /// Returns the trees names saved in this Db.
    pub fn tree_names(&self) -> Vec<IVec> {
        let tenants = self.tenants.read();
        tenants.iter().map(|(name, _)| name.clone()).collect()
    }

    /// Returns `true` if the database was
    /// recovered from a previous process.
    /// Note that database state is only
    /// guaranteed to be present up to the
    /// last call to `flush`! Otherwise state
    /// is synced to disk periodically if the
    /// `sync_every_ms` configuration option
    /// is set to `Some(number_of_ms_between_syncs)`
    /// or if the IO buffer gets filled to
    /// capacity before being rotated.
    pub fn was_recovered(&self) -> bool {
        self.context.was_recovered()
    }

    /// Generate a monotonic ID. Not guaranteed to be
    /// contiguous. Written to disk every `idgen_persist_interval`
    /// operations, followed by a blocking flush. During recovery, we
    /// take the last recovered generated ID and add 2x
    /// the `idgen_persist_interval` to it. While persisting, if the
    /// previous persisted counter wasn't synced to disk yet, we will do
    /// a blocking flush to fsync the latest counter, ensuring
    /// that we will never give out the same counter twice.
    pub fn generate_id(&self) -> Result<u64> {
        self.context.generate_id()
    }

    /// A database export method for all collections in the `Db`,
    /// for use in sled version upgrades. Can be used in combination
    /// with the `import` method below on a database running a later
    /// version.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the export.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn export(
        &self,
    ) -> Vec<(CollectionType, CollectionName, impl Iterator<Item = Vec<Vec<u8>>>)>
    {
        let tenants = self.tenants.read();

        let mut ret = vec![];

        for (name, tree) in tenants.iter() {
            ret.push((
                b"tree".to_vec(),
                name.to_vec(),
                tree.iter().map(|kv_opt| {
                    let kv = kv_opt.unwrap();
                    vec![kv.0.to_vec(), kv.1.to_vec()]
                }),
            ));
        }

        ret
    }

    /// Imports the collections from a previous database.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the import.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn import(
        &self,
        export: Vec<(
            CollectionType,
            CollectionName,
            impl Iterator<Item = Vec<Vec<u8>>>,
        )>,
    ) {
        for (collection_type, collection_name, collection_iter) in export {
            match collection_type {
                ref t if t == b"tree" => {
                    let tree = self
                        .open_tree(collection_name)
                        .expect("failed to open new tree during import");
                    for mut kv in collection_iter {
                        let v = kv
                            .pop()
                            .expect("failed to get value from tree export");
                        let k = kv
                            .pop()
                            .expect("failed to get key from tree export");
                        let old = tree.insert(k, v).expect(
                            "failed to insert value during tree import",
                        );
                        assert!(
                            old.is_none(),
                            "import is overwriting existing data"
                        );
                    }
                }
                other => panic!("unknown collection type {:?}", other),
            }
        }
    }

    /// Returns the CRC32 of all keys and values
    /// in this Db.
    ///
    /// This is O(N) and locks all underlying Trees
    /// for the duration of the entire scan.
    pub fn checksum(&self) -> Result<u32> {
        let tenants_mu = self.tenants.write();

        // we use a btreemap to ensure lexicographic
        // iteration over tree names to have consistent
        // checksums.
        let tenants: BTreeMap<_, _> = tenants_mu.iter().collect();

        let mut hasher = crc32fast::Hasher::new();
        let mut locks = vec![];

        locks.push(concurrency_control::write());

        for (name, tree) in &tenants {
            hasher.update(name);

            let mut iter = tree.iter();
            while let Some(kv_res) = iter.next_inner() {
                let (k, v) = kv_res?;
                hasher.update(&k);
                hasher.update(&v);
            }
        }

        Ok(hasher.finalize())
    }

    /// Returns the on-disk size of the storage files
    /// for this database.
    pub fn size_on_disk(&self) -> Result<u64> {
        self.context.pagecache.size_on_disk()
    }

    /// Traverses all files and calculates their total physical
    /// size, then traverses all pages and calculates their
    /// total logical size, then divides the physical size
    /// by the logical size.
    #[doc(hidden)]
    pub fn space_amplification(&self) -> Result<f64> {
        self.context.pagecache.space_amplification()
    }
}

/// These types provide the information that allows an entire
/// system to be exported and imported to facilitate
/// major upgrades. It is comprised entirely
/// of standard library types to be forward compatible.
/// NB this definitions are expensive to change, because
/// they impact the migration path.
type CollectionType = Vec<u8>;
type CollectionName = Vec<u8>;
<|endoftext|>


<commit_before>use std::ops::Deref;

use crate::*;

/// The `sled` embedded database! Implements
/// `Deref<Target = sled::Tree>` to refer to
/// a default keyspace / namespace / bucket.
#[derive(Clone)]
pub struct Db {
    #[doc(hidden)]
    pub context: Context,
    pub(crate) default: Tree,
    tenants: Arc<RwLock<FastMap8<IVec, Tree>>>,
}

/// Opens a `Db` with a default configuration at the
/// specified path. This will create a new storage
/// directory at the specified path if it does
/// not already exist. You can use the `Db::was_recovered`
/// method to determine if your database was recovered
/// from a previous instance. You can use `Config::create_new`
/// if you want to increase the chances that the database
/// will be freshly created.
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Db> {
    Config::new().path(path).open()
}

impl Deref for Db {
    type Target = Tree;

    fn deref(&self) -> &Tree {
        &self.default
    }
}

impl Debug for Db {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        let tenants = self.tenants.read();
        writeln!(f, "Db {{")?;
        for (raw_name, tree) in tenants.iter() {
            let name = std::str::from_utf8(raw_name)
                .ok()
                .map_or_else(|| format!("{:?}", raw_name), String::from);
            write!(f, "    Tree: {:?} contents: {:?}", name, tree)?;
        }
        write!(f, "}}")?;
        Ok(())
    }
}

impl Db {
    #[doc(hidden)]
    #[deprecated(since = "0.30.2", note = "replaced by `sled::open`")]
    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
        Config::new().path(path).open()
    }

    pub(crate) fn start_inner(config: RunningConfig) -> Result<Self> {
        let _measure = Measure::new(&M.tree_start);

        let context = Context::start(config)?;

        #[cfg(all(
            not(miri),
            any(
                windows,
                target_os = "linux",
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "openbsd",
                target_os = "netbsd",
            )
        ))]
        {
            let flusher_pagecache = context.pagecache.clone();
            let flusher = context.flush_every_ms.map(move |fem| {
                flusher::Flusher::new(
                    "log flusher".to_owned(),
                    flusher_pagecache,
                    fem,
                )
            });
            *context.flusher.lock() = flusher;
        }

        // create or open the default tree
        let guard = pin();
        let default =
            meta::open_tree(&context, DEFAULT_TREE_ID.to_vec(), &guard)?;

        let ret = Self {
            context: context.clone(),
            default,
            tenants: Arc::new(RwLock::new(FastMap8::default())),
        };

        let mut tenants = ret.tenants.write();

        for (id, root) in context.pagecache.get_meta(&guard)?.tenants() {
            let tree = Tree(Arc::new(TreeInner {
                tree_id: id.clone(),
                subscribers: Subscribers::default(),
                context: context.clone(),
                root: AtomicU64::new(root),
                merge_operator: RwLock::new(None),
            }));
            assert!(tenants.insert(id, tree).is_none());
        }

        drop(tenants);

        #[cfg(feature = "event_log")]
        {
            for (_name, tree) in ret.tenants.read().iter() {
                tree.verify_integrity()?;
            }
            ret.context.event_log.verify();
        }

        Ok(ret)
    }

    /// Open or create a new disk-backed Tree with its own keyspace,
    /// accessible from the `Db` via the provided identifier.
    pub fn open_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<Tree> {
        let name_ref = name.as_ref();
        let tenants = self.tenants.read();
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }
        drop(tenants);

        let guard = pin();

        let mut tenants = self.tenants.write();

        // we need to check this again in case another
        // thread opened it concurrently.
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }

        let tree = meta::open_tree(&self.context, name_ref.to_vec(), &guard)?;

        assert!(tenants.insert(name_ref.into(), tree.clone()).is_none());

        Ok(tree)
    }

    /// Remove a disk-backed collection.
    pub fn drop_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<bool> {
        let name_ref = name.as_ref();
        if name_ref == DEFAULT_TREE_ID {
            return Err(Error::Unsupported(
                "cannot remove the core structures".into(),
            ));
        }
        trace!("dropping tree {:?}", name_ref,);

        let mut tenants = self.tenants.write();

        let tree = if let Some(tree) = tenants.remove(&*name_ref) {
            tree
        } else {
            return Ok(false);
        };

        let guard = pin();

        let mut root_id =
            Some(self.context.pagecache.meta_pid_for_name(name_ref, &guard)?);

        let mut leftmost_chain: Vec<PageId> = vec![root_id.unwrap()];
        let mut cursor = root_id.unwrap();
        while let Some(view) = self.view_for_pid(cursor, &guard)? {
            if let Some(index) = view.data.index_ref() {
                let leftmost_child = index.pointers[0];
                leftmost_chain.push(leftmost_child);
                cursor = leftmost_child;
            } else {
                break;
            }
        }

        loop {
            let res = self
                .context
                .pagecache
                .cas_root_in_meta(name_ref, root_id, None, &guard)?;

            if let Err(actual_root) = res {
                root_id = actual_root;
            } else {
                break;
            }
        }

        tree.root.store(u64::max_value(), SeqCst);

        // drop writer lock
        drop(tenants);

        tree.gc_pages(leftmost_chain)?;

        guard.flush();

        Ok(true)
    }

    /// Returns the trees names saved in this Db.
    pub fn tree_names(&self) -> Vec<IVec> {
        let tenants = self.tenants.read();
        tenants.iter().map(|(name, _)| name.clone()).collect()
    }

    /// Returns `true` if the database was
    /// recovered from a previous process.
    /// Note that database state is only
    /// guaranteed to be present up to the
    /// last call to `flush`! Otherwise state
    /// is synced to disk periodically if the
    /// `sync_every_ms` configuration option
    /// is set to `Some(number_of_ms_between_syncs)`
    /// or if the IO buffer gets filled to
    /// capacity before being rotated.
    pub fn was_recovered(&self) -> bool {
        self.context.was_recovered()
    }

    /// Generate a monotonic ID. Not guaranteed to be
    /// contiguous. Written to disk every `idgen_persist_interval`
    /// operations, followed by a blocking flush. During recovery, we
    /// take the last recovered generated ID and add 2x
    /// the `idgen_persist_interval` to it. While persisting, if the
    /// previous persisted counter wasn't synced to disk yet, we will do
    /// a blocking flush to fsync the latest counter, ensuring
    /// that we will never give out the same counter twice.
    pub fn generate_id(&self) -> Result<u64> {
        self.context.generate_id()
    }

    /// A database export method for all collections in the `Db`,
    /// for use in sled version upgrades. Can be used in combination
    /// with the `import` method below on a database running a later
    /// version.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the export.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn export(
        &self,
    ) -> Vec<(CollectionType, CollectionName, impl Iterator<Item = Vec<Vec<u8>>>)>
    {
        let tenants = self.tenants.read();

        let mut ret = vec![];

        for (name, tree) in tenants.iter() {
            ret.push((
                b"tree".to_vec(),
                name.to_vec(),
                tree.iter().map(|kv_opt| {
                    let kv = kv_opt.unwrap();
                    vec![kv.0.to_vec(), kv.1.to_vec()]
                }),
            ));
        }

        ret
    }

    /// Imports the collections from a previous database.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the import.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn import(
        &self,
        export: Vec<(
            CollectionType,
            CollectionName,
            impl Iterator<Item = Vec<Vec<u8>>>,
        )>,
    ) {
        for (collection_type, collection_name, collection_iter) in export {
            match collection_type {
                ref t if t == b"tree" => {
                    let tree = self
                        .open_tree(collection_name)
                        .expect("failed to open new tree during import");
                    for mut kv in collection_iter {
                        let v = kv
                            .pop()
                            .expect("failed to get value from tree export");
                        let k = kv
                            .pop()
                            .expect("failed to get key from tree export");
                        let old = tree.insert(k, v).expect(
                            "failed to insert value during tree import",
                        );
                        assert!(
                            old.is_none(),
                            "import is overwriting existing data"
                        );
                    }
                }
                other => panic!("unknown collection type {:?}", other),
            }
        }
    }

    /// Returns the CRC32 of all keys and values
    /// in this Db.
    ///
    /// This is O(N) and locks all underlying Trees
    /// for the duration of the entire scan.
    pub fn checksum(&self) -> Result<u32> {
        let tenants_mu = self.tenants.write();

        // we use a btreemap to ensure lexicographic
        // iteration over tree names to have consistent
        // checksums.
        let tenants: BTreeMap<_, _> = tenants_mu.iter().collect();

        let mut hasher = crc32fast::Hasher::new();
        let mut locks = vec![];

        locks.push(concurrency_control::write());

        for (name, tree) in &tenants {
            hasher.update(name);

            let mut iter = tree.iter();
            while let Some(kv_res) = iter.next_inner() {
                let (k, v) = kv_res?;
                hasher.update(&k);
                hasher.update(&v);
            }
        }

        Ok(hasher.finalize())
    }

    /// Returns the on-disk size of the storage files
    /// for this database.
    pub fn size_on_disk(&self) -> Result<u64> {
        self.context.pagecache.size_on_disk()
    }

    /// Traverses all files and calculates their total physical
    /// size, then traverses all pages and calculates their
    /// total logical size, then divides the physical size
    /// by the logical size.
    #[doc(hidden)]
    pub fn space_amplification(&self) -> Result<f64> {
        self.context.pagecache.space_amplification()
    }
}

/// These types provide the information that allows an entire
/// system to be exported and imported to facilitate
/// major upgrades. It is comprised entirely
/// of standard library types to be forward compatible.
/// NB this definitions are expensive to change, because
/// they impact the migration path.
type CollectionType = Vec<u8>;
type CollectionName = Vec<u8>;
<commit_msg>Use exclusive concurrency control + atomic write batch while dropping trees<commit_after>use std::ops::Deref;

use crate::*;

/// The `sled` embedded database! Implements
/// `Deref<Target = sled::Tree>` to refer to
/// a default keyspace / namespace / bucket.
#[derive(Clone)]
pub struct Db {
    #[doc(hidden)]
    pub context: Context,
    pub(crate) default: Tree,
    tenants: Arc<RwLock<FastMap8<IVec, Tree>>>,
}

/// Opens a `Db` with a default configuration at the
/// specified path. This will create a new storage
/// directory at the specified path if it does
/// not already exist. You can use the `Db::was_recovered`
/// method to determine if your database was recovered
/// from a previous instance. You can use `Config::create_new`
/// if you want to increase the chances that the database
/// will be freshly created.
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Db> {
    Config::new().path(path).open()
}

impl Deref for Db {
    type Target = Tree;

    fn deref(&self) -> &Tree {
        &self.default
    }
}

impl Debug for Db {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        let tenants = self.tenants.read();
        writeln!(f, "Db {{")?;
        for (raw_name, tree) in tenants.iter() {
            let name = std::str::from_utf8(raw_name)
                .ok()
                .map_or_else(|| format!("{:?}", raw_name), String::from);
            write!(f, "    Tree: {:?} contents: {:?}", name, tree)?;
        }
        write!(f, "}}")?;
        Ok(())
    }
}

impl Db {
    #[doc(hidden)]
    #[deprecated(since = "0.30.2", note = "replaced by `sled::open`")]
    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
        Config::new().path(path).open()
    }

    pub(crate) fn start_inner(config: RunningConfig) -> Result<Self> {
        let _measure = Measure::new(&M.tree_start);

        let context = Context::start(config)?;

        #[cfg(all(
            not(miri),
            any(
                windows,
                target_os = "linux",
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "openbsd",
                target_os = "netbsd",
            )
        ))]
        {
            let flusher_pagecache = context.pagecache.clone();
            let flusher = context.flush_every_ms.map(move |fem| {
                flusher::Flusher::new(
                    "log flusher".to_owned(),
                    flusher_pagecache,
                    fem,
                )
            });
            *context.flusher.lock() = flusher;
        }

        // create or open the default tree
        let guard = pin();
        let default =
            meta::open_tree(&context, DEFAULT_TREE_ID.to_vec(), &guard)?;

        let ret = Self {
            context: context.clone(),
            default,
            tenants: Arc::new(RwLock::new(FastMap8::default())),
        };

        let mut tenants = ret.tenants.write();

        for (id, root) in context.pagecache.get_meta(&guard)?.tenants() {
            let tree = Tree(Arc::new(TreeInner {
                tree_id: id.clone(),
                subscribers: Subscribers::default(),
                context: context.clone(),
                root: AtomicU64::new(root),
                merge_operator: RwLock::new(None),
            }));
            assert!(tenants.insert(id, tree).is_none());
        }

        drop(tenants);

        #[cfg(feature = "event_log")]
        {
            for (_name, tree) in ret.tenants.read().iter() {
                tree.verify_integrity()?;
            }
            ret.context.event_log.verify();
        }

        Ok(ret)
    }

    /// Open or create a new disk-backed Tree with its own keyspace,
    /// accessible from the `Db` via the provided identifier.
    pub fn open_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<Tree> {
        let name_ref = name.as_ref();
        let tenants = self.tenants.read();
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }
        drop(tenants);

        let guard = pin();

        let mut tenants = self.tenants.write();

        // we need to check this again in case another
        // thread opened it concurrently.
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }

        let tree = meta::open_tree(&self.context, name_ref.to_vec(), &guard)?;

        assert!(tenants.insert(name_ref.into(), tree.clone()).is_none());

        Ok(tree)
    }

    /// Remove a disk-backed collection. This is blocking and fairly slow.
    pub fn drop_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<bool> {
        let name_ref = name.as_ref();
        if name_ref == DEFAULT_TREE_ID {
            return Err(Error::Unsupported(
                "cannot remove the core structures".into(),
            ));
        }
        trace!("dropping tree {:?}", name_ref,);

        let mut tenants = self.tenants.write();

        let tree = if let Some(tree) = tenants.remove(&*name_ref) {
            tree
        } else {
            return Ok(false);
        };

        let _cc = concurrency_control::write();
        let guard = pin();

        // peg is for atomic recovery in case we crash
        // half-way through this cleaning operation.
        let peg = self.context.pin_log(&guard)?;

        // signal to all threads that this tree is no longer valid
        tree.root.store(u64::max_value(), SeqCst);

        let mut root_id =
            Some(self.context.pagecache.meta_pid_for_name(name_ref, &guard)?);

        let mut leftmost_chain: Vec<PageId> = vec![root_id.unwrap()];
        let mut cursor = root_id.unwrap();
        while let Some(view) = self.view_for_pid(cursor, &guard)? {
            if let Some(index) = view.data.index_ref() {
                let leftmost_child = index.pointers[0];
                leftmost_chain.push(leftmost_child);
                cursor = leftmost_child;
            } else {
                break;
            }
        }

        loop {
            let res = self
                .context
                .pagecache
                .cas_root_in_meta(name_ref, root_id, None, &guard)?;

            if let Err(actual_root) = res {
                root_id = actual_root;
            } else {
                break;
            }
        }

        // record atomic recovery information into the log
        peg.seal_batch()?;

        // drop writer lock
        drop(tenants);

        tree.gc_pages(leftmost_chain)?;

        guard.flush();

        Ok(true)
    }

    /// Returns the trees names saved in this Db.
    pub fn tree_names(&self) -> Vec<IVec> {
        let tenants = self.tenants.read();
        tenants.iter().map(|(name, _)| name.clone()).collect()
    }

    /// Returns `true` if the database was
    /// recovered from a previous process.
    /// Note that database state is only
    /// guaranteed to be present up to the
    /// last call to `flush`! Otherwise state
    /// is synced to disk periodically if the
    /// `sync_every_ms` configuration option
    /// is set to `Some(number_of_ms_between_syncs)`
    /// or if the IO buffer gets filled to
    /// capacity before being rotated.
    pub fn was_recovered(&self) -> bool {
        self.context.was_recovered()
    }

    /// Generate a monotonic ID. Not guaranteed to be
    /// contiguous. Written to disk every `idgen_persist_interval`
    /// operations, followed by a blocking flush. During recovery, we
    /// take the last recovered generated ID and add 2x
    /// the `idgen_persist_interval` to it. While persisting, if the
    /// previous persisted counter wasn't synced to disk yet, we will do
    /// a blocking flush to fsync the latest counter, ensuring
    /// that we will never give out the same counter twice.
    pub fn generate_id(&self) -> Result<u64> {
        self.context.generate_id()
    }

    /// A database export method for all collections in the `Db`,
    /// for use in sled version upgrades. Can be used in combination
    /// with the `import` method below on a database running a later
    /// version.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the export.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn export(
        &self,
    ) -> Vec<(CollectionType, CollectionName, impl Iterator<Item = Vec<Vec<u8>>>)>
    {
        let tenants = self.tenants.read();

        let mut ret = vec![];

        for (name, tree) in tenants.iter() {
            ret.push((
                b"tree".to_vec(),
                name.to_vec(),
                tree.iter().map(|kv_opt| {
                    let kv = kv_opt.unwrap();
                    vec![kv.0.to_vec(), kv.1.to_vec()]
                }),
            ));
        }

        ret
    }

    /// Imports the collections from a previous database.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the import.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn import(
        &self,
        export: Vec<(
            CollectionType,
            CollectionName,
            impl Iterator<Item = Vec<Vec<u8>>>,
        )>,
    ) {
        for (collection_type, collection_name, collection_iter) in export {
            match collection_type {
                ref t if t == b"tree" => {
                    let tree = self
                        .open_tree(collection_name)
                        .expect("failed to open new tree during import");
                    for mut kv in collection_iter {
                        let v = kv
                            .pop()
                            .expect("failed to get value from tree export");
                        let k = kv
                            .pop()
                            .expect("failed to get key from tree export");
                        let old = tree.insert(k, v).expect(
                            "failed to insert value during tree import",
                        );
                        assert!(
                            old.is_none(),
                            "import is overwriting existing data"
                        );
                    }
                }
                other => panic!("unknown collection type {:?}", other),
            }
        }
    }

    /// Returns the CRC32 of all keys and values
    /// in this Db.
    ///
    /// This is O(N) and locks all underlying Trees
    /// for the duration of the entire scan.
    pub fn checksum(&self) -> Result<u32> {
        let tenants_mu = self.tenants.write();

        // we use a btreemap to ensure lexicographic
        // iteration over tree names to have consistent
        // checksums.
        let tenants: BTreeMap<_, _> = tenants_mu.iter().collect();

        let mut hasher = crc32fast::Hasher::new();
        let _cc = concurrency_control::write();

        for (name, tree) in &tenants {
            hasher.update(name);

            let mut iter = tree.iter();
            while let Some(kv_res) = iter.next_inner() {
                let (k, v) = kv_res?;
                hasher.update(&k);
                hasher.update(&v);
            }
        }

        Ok(hasher.finalize())
    }

    /// Returns the on-disk size of the storage files
    /// for this database.
    pub fn size_on_disk(&self) -> Result<u64> {
        self.context.pagecache.size_on_disk()
    }

    /// Traverses all files and calculates their total physical
    /// size, then traverses all pages and calculates their
    /// total logical size, then divides the physical size
    /// by the logical size.
    #[doc(hidden)]
    pub fn space_amplification(&self) -> Result<f64> {
        self.context.pagecache.space_amplification()
    }
}

/// These types provide the information that allows an entire
/// system to be exported and imported to facilitate
/// major upgrades. It is comprised entirely
/// of standard library types to be forward compatible.
/// NB this definitions are expensive to change, because
/// they impact the migration path.
type CollectionType = Vec<u8>;
type CollectionName = Vec<u8>;
<|endoftext|>