<commit_before>use super::*;

use std::{collections::HashSet, convert::TryFrom, ops::Bound};

#[derive(Debug, Clone)]
pub(crate) struct View<'a> {
    pub(crate) pid: PageId,
    pub(crate) lo: &'a IVec,
    pub(crate) hi: &'a IVec,
    pub(crate) is_index: bool,
    pub(crate) next: Option<PageId>,
    pub(crate) ptr: TreePtr<'a>,
    frags: Vec<&'a Frag>,
    base_offset: usize,
    pub(crate) base_data: &'a Data,
    pub(crate) merging_child: Option<PageId>,
    pub(crate) merging: bool,
    min_children: usize,
}

impl<'a> View<'a> {
    pub(crate) fn new(
        pid: PageId,
        ptr: TreePtr<'a>,
        frags: Vec<&'a Frag>,
    ) -> View<'a> {
        let mut merging = false;
        let mut merging_child = None;
        let mut merge_confirmed = false;
        let mut min_children: isize = 0;

        for (offset, frag) in frags.iter().enumerate() {
            match frag {
                Frag::Base(node) => {
                    // NB if we re-add ParentSplit, we must
                    // handle the hi & next members differently
                    // here

                    min_children += node.data.len() as isize;

                    return View {
                        hi: &node.hi,
                        lo: &node.lo,
                        is_index: node.data.index_ref().is_some(),
                        next: node.next,
                        base_offset: offset,
                        base_data: &node.data,
                        pid,
                        ptr,
                        frags,
                        merging,
                        merging_child,
                        min_children: usize::try_from(std::cmp::max(
                            0,
                            min_children,
                        ))
                        .unwrap(),
                    };
                }
                Frag::ParentMergeIntention(pid) => {
                    if merge_confirmed {
                        merge_confirmed = false;
                    } else {
                        assert!(merging_child.is_none());
                        merging_child = Some(*pid);
                    }
                }
                Frag::ParentMergeConfirm => {
                    assert!(!merge_confirmed);
                    merge_confirmed = true;
                }
                Frag::ChildMergeCap => {
                    assert!(!merging);
                    assert_eq!(offset, 0);
                    merging = true;
                }
                Frag::Del(..) => min_children -= 1,
                _ => {}
            }
        }

        panic!("view was never initialized with a base")
    }

    pub(crate) fn contains_upper_bound(&self, bound: &Bound<IVec>) -> bool {
        match bound {
            Bound::Excluded(bound) if self.hi >= bound => true,
            Bound::Included(bound) if self.hi > bound => true,
            _ => self.hi.is_empty(),
        }
    }

    pub(crate) fn contains_lower_bound(
        &self,
        bound: &Bound<IVec>,
        is_forward: bool,
    ) -> bool {
        match bound {
            Bound::Excluded(bound)
                if self.lo < bound || (is_forward && bound == self.lo) =>
            {
                true
            }
            Bound::Included(bound) if self.lo <= bound => true,
            _ => self.lo.is_empty(),
        }
    }

    fn keys(&self) -> Vec<&IVec> {
        let mut keys: HashSet<&IVec> = self
            .base_data
            .leaf_ref()
            .unwrap()
            .iter()
            .map(|(k, _v)| k)
            .collect();

        for offset in (0..self.base_offset).rev() {
            match self.frags[offset] {
                Frag::Set(k, _) => {
                    keys.insert(k);
                }
                Frag::Del(k) => {
                    keys.remove(k);
                }
                Frag::Merge(k, _) => {
                    keys.insert(k);
                }
                Frag::Base(_) => {
                    panic!(
                        "somehow hit 2 base nodes while \
                         searching for a successor"
                    );
                }
                Frag::ChildMergeCap => {}
                Frag::ParentMergeIntention(_) | Frag::ParentMergeConfirm => {
                    panic!(
                        "somehow hit parent merge \
                         frags while searching for a \
                         successor"
                    )
                }
            }
        }

        let mut keys: Vec<_> = keys.into_iter().collect();
        keys.sort_unstable_by(|a, b| prefix_cmp(&a, &b));

        keys
    }

    pub(crate) fn successor(
        &self,
        bound: &Bound<IVec>,
        config: &Config,
    ) -> Option<(IVec, IVec)> {
        assert!(!self.is_index);

        // This encoding happens this way because
        // keys cannot be lower than the node's lo key.
        let predecessor_key = match bound {
            Bound::Unbounded => prefix_encode(self.lo, self.lo),
            Bound::Included(b) => {
                let max = std::cmp::max(b, self.lo);
                prefix_encode(self.lo, max)
            }
            Bound::Excluded(b) => {
                let max = std::cmp::max(b, self.lo);
                prefix_encode(self.lo, max)
            }
        };

        let keys = self.keys();

        let successor_keys = keys.iter().filter(|k| {
            let ord = prefix_cmp(k, &predecessor_key);

            ord == std::cmp::Ordering::Greater
                || ord == std::cmp::Ordering::Equal
        });

        for encoded_key in successor_keys {
            let decoded_key = prefix_decode(self.lo, &encoded_key);

            if let Bound::Excluded(e) = bound {
                if e == &decoded_key {
                    // skip this excluded key
                    continue;
                }
            }

            // try to get this key until it works
            if let Some(value) = self.leaf_value_for_key(&decoded_key, config) {
                return Some((IVec::from(decoded_key), value));
            }
        }

        None
    }

    pub(crate) fn predecessor(
        &self,
        bound: &Bound<IVec>,
        config: &Config,
    ) -> Option<(IVec, IVec)> {
        assert!(!self.is_index);

        // This encoding happens this way because
        // the rightmost (unbounded) node has
        // a hi key represented by the empty slice
        let successor_key = match bound {
            Bound::Unbounded => {
                if self.hi.is_empty() {
                    prefix_encode(self.lo, &[255; 1024 * 1024])
                } else {
                    prefix_encode(self.lo, self.hi)
                }
            }
            Bound::Included(b) => {
                let min = if self.hi.is_empty() {
                    b
                } else {
                    std::cmp::min(b, self.hi)
                };
                prefix_encode(self.lo, min)
            }
            Bound::Excluded(b) => {
                let min = if self.hi.is_empty() {
                    b
                } else {
                    std::cmp::min(b, self.hi)
                };
                prefix_encode(self.lo, min)
            }
        };

        let keys = self.keys();

        let predecessor_keys = keys.iter().filter(|k| {
            let ord = prefix_cmp(k, &successor_key);

            ord == std::cmp::Ordering::Less || ord == std::cmp::Ordering::Equal
        });

        for encoded_key in predecessor_keys.rev() {
            let decoded_key = prefix_decode(self.lo, &encoded_key);

            if let Bound::Excluded(e) = bound {
                if e == &decoded_key {
                    // skip this excluded key
                    continue;
                }
            }

            // try to get this key until it works
            if let Some(value) = self.leaf_value_for_key(&decoded_key, config) {
                return Some((IVec::from(decoded_key), value));
            }
        }

        None
    }

    pub(crate) fn leaf_value_for_key(
        &self,
        key: &[u8],
        config: &Config,
    ) -> Option<IVec> {
        assert!(!self.is_index);

        let mut merge_base = None;
        let mut merges = vec![];

        for frag in self.frags[..=self.base_offset].iter() {
            match frag {
                Frag::Set(k, val) if self.key_eq(k, key) => {
                    if merges.is_empty() {
                        return Some(val.clone());
                    } else {
                        merge_base = Some(val);
                        break;
                    }
                }
                Frag::Del(k) if self.key_eq(k, key) => {
                    // we should ignore "earlier" Frag's for
                    // this key, but still need to handle
                    // merges we encountered "after" this
                    // deletion
                    break;
                }
                Frag::Merge(k, val) if self.key_eq(k, key) => merges.push(val),
                Frag::Base(node) => {
                    let data = &node.data;
                    let items =
                        data.leaf_ref().expect("last_node should be a leaf");
                    let search = items
                        .binary_search_by(|&(ref k, ref _v)| {
                            prefix_cmp_encoded(k, key, &self.lo)
                        })
                        .ok();

                    let val = search.map(|idx| &items[idx].1);
                    if merges.is_empty() {
                        return val.cloned();
                    } else {
                        merge_base = val;
                    }
                }
                _ => {}
            }
        }

        if merges.is_empty() {
            None
        } else {
            let merge_fn_ptr = config
                .merge_operator
                .expect("must have a merge operator set");

            unsafe {
                let merge_fn: MergeOperator = std::mem::transmute(merge_fn_ptr);

                let mut ret = merge_fn(
                    key,
                    merge_base.map(|iv| &**iv),
                    &merges.pop().unwrap(),
                );
                                       ;
                for merge in merges.into_iter().rev() {
                    if let Some(v) = ret {
                        ret = merge_fn(key, Some(&*v), merge);
                    } else {
                        ret = merge_fn(key, None, merge);
                    }
                }

                ret.map(IVec::from)
            }
        }
    }

    #[inline]
    fn key_eq(&self, encoded: &[u8], not_encoded: &[u8]) -> bool {
        prefix_cmp_encoded(encoded, not_encoded, self.lo)
            == std::cmp::Ordering::Equal
    }

    pub(crate) fn index_next_node(&self, key: &[u8]) -> (usize, PageId) {
        assert!(self.is_index);

        for frag in self.frags[..=self.base_offset].iter() {
            match frag {
                Frag::Set(..) => unimplemented!(),
                Frag::Del(..) => unimplemented!(),
                Frag::Merge(..) => unimplemented!(),
                Frag::Base(node) => {
                    let data = &node.data;
                    let items =
                        data.index_ref().expect("last_node should be a leaf");
                    let search =
                        binary_search_lub(items, |&(ref k, ref _v)| {
                            prefix_cmp_encoded(k, key, &node.lo)
                        });

                    // This might be none if ord is Less and we're
                    // searching for the empty key
                    let index = search.expect("failed to traverse index");

                    return (index, items[index].1);
                }
                Frag::ParentMergeIntention(_)
                | Frag::ParentMergeConfirm
                | Frag::ChildMergeCap => {
                    // nothing to do for these frags
                }
            }
        }
        panic!("no index found")
    }

    pub(crate) fn should_split(&self, max_sz: u64) -> bool {
        self.min_children > 2
            && self.size_in_bytes() > max_sz
            && self.merging_child.is_none()
            && !self.merging
    }

    pub(crate) fn should_merge(&self, min_sz: u64) -> bool {
        self.size_in_bytes() < min_sz
            && self.merging_child.is_none()
            && !self.merging
    }

    pub(crate) fn can_merge_child(&self) -> bool {
        self.merging_child.is_none() && !self.merging
    }

    pub(crate) fn compact(&self, config: &Config) -> Node {
        let mut lhs = self.frags[self.base_offset].unwrap_base().clone();
        for offset in (0..self.base_offset).rev() {
            let frag = self.frags[offset];
            lhs.apply(frag, config.merge_operator);
        }
        lhs
    }

    pub(crate) fn split(&self, config: &Config) -> (Node, Node) {
        let mut lhs = self.compact(config);
        let rhs = lhs.split();

        lhs.data.drop_gte(&rhs.lo, &lhs.lo);
        lhs.hi = rhs.lo.clone();

        // intentionally make this the end to make
        // any issues pop out with setting it
        // correctly after the split.
        lhs.next = None;

        (lhs, rhs)
    }

    #[inline]
    pub(crate) fn size_in_bytes(&self) -> u64 {
        // TODO needs to better account for
        // sizes that don't actually fall under
        // a merge threshold once we support one.
        self.frags[..=self.base_offset]
            .iter()
            .map(|f| f.size_in_bytes())
            .sum()
    }
}
<commit_msg>Fix bug with reverse iteration in the presence of merges<commit_after>use super::*;

use std::{collections::HashSet, convert::TryFrom, ops::Bound};

#[derive(Debug, Clone)]
pub(crate) struct View<'a> {
    pub(crate) pid: PageId,
    pub(crate) lo: &'a IVec,
    pub(crate) hi: &'a IVec,
    pub(crate) is_index: bool,
    pub(crate) next: Option<PageId>,
    pub(crate) ptr: TreePtr<'a>,
    frags: Vec<&'a Frag>,
    base_offset: usize,
    pub(crate) base_data: &'a Data,
    pub(crate) merging_child: Option<PageId>,
    pub(crate) merging: bool,
    min_children: usize,
}

impl<'a> View<'a> {
    pub(crate) fn new(
        pid: PageId,
        ptr: TreePtr<'a>,
        frags: Vec<&'a Frag>,
    ) -> View<'a> {
        let mut merging = false;
        let mut merging_child = None;
        let mut merge_confirmed = false;
        let mut min_children: isize = 0;

        for (offset, frag) in frags.iter().enumerate() {
            match frag {
                Frag::Base(node) => {
                    // NB if we re-add ParentSplit, we must
                    // handle the hi & next members differently
                    // here

                    min_children += node.data.len() as isize;

                    return View {
                        hi: &node.hi,
                        lo: &node.lo,
                        is_index: node.data.index_ref().is_some(),
                        next: node.next,
                        base_offset: offset,
                        base_data: &node.data,
                        pid,
                        ptr,
                        frags,
                        merging,
                        merging_child,
                        min_children: usize::try_from(std::cmp::max(
                            0,
                            min_children,
                        ))
                        .unwrap(),
                    };
                }
                Frag::ParentMergeIntention(pid) => {
                    if merge_confirmed {
                        merge_confirmed = false;
                    } else {
                        assert!(merging_child.is_none());
                        merging_child = Some(*pid);
                    }
                }
                Frag::ParentMergeConfirm => {
                    assert!(!merge_confirmed);
                    merge_confirmed = true;
                }
                Frag::ChildMergeCap => {
                    assert!(!merging);
                    assert_eq!(offset, 0);
                    merging = true;
                }
                Frag::Del(..) => min_children -= 1,
                _ => {}
            }
        }

        panic!("view was never initialized with a base")
    }

    pub(crate) fn contains_upper_bound(&self, bound: &Bound<IVec>) -> bool {
        match bound {
            Bound::Excluded(bound) if self.hi >= bound => true,
            Bound::Included(bound) if self.hi > bound => true,
            _ => self.hi.is_empty(),
        }
    }

    pub(crate) fn contains_lower_bound(
        &self,
        bound: &Bound<IVec>,
        is_forward: bool,
    ) -> bool {
        match bound {
            Bound::Excluded(bound)
                if self.lo < bound || (is_forward && bound == self.lo) =>
            {
                true
            }
            Bound::Included(bound) if self.lo <= bound => true,
            Bound::Unbounded if !is_forward => self.hi.is_empty(),
            _ => self.lo.is_empty(),
        }
    }

    fn keys(&self) -> Vec<&IVec> {
        let mut keys: HashSet<&IVec> = self
            .base_data
            .leaf_ref()
            .unwrap()
            .iter()
            .map(|(k, _v)| k)
            .collect();

        for offset in (0..self.base_offset).rev() {
            match self.frags[offset] {
                Frag::Set(k, _) => {
                    keys.insert(k);
                }
                Frag::Del(k) => {
                    keys.remove(k);
                }
                Frag::Merge(k, _) => {
                    keys.insert(k);
                }
                Frag::Base(_) => {
                    panic!(
                        "somehow hit 2 base nodes while \
                         searching for a successor"
                    );
                }
                Frag::ChildMergeCap => {}
                Frag::ParentMergeIntention(_) | Frag::ParentMergeConfirm => {
                    panic!(
                        "somehow hit parent merge \
                         frags while searching for a \
                         successor"
                    )
                }
            }
        }

        let mut keys: Vec<_> = keys.into_iter().collect();
        keys.sort_unstable_by(|a, b| prefix_cmp(&a, &b));

        keys
    }

    pub(crate) fn successor(
        &self,
        bound: &Bound<IVec>,
        config: &Config,
    ) -> Option<(IVec, IVec)> {
        assert!(!self.is_index);

        // This encoding happens this way because
        // keys cannot be lower than the node's lo key.
        let predecessor_key = match bound {
            Bound::Unbounded => prefix_encode(self.lo, self.lo),
            Bound::Included(b) => {
                let max = std::cmp::max(b, self.lo);
                prefix_encode(self.lo, max)
            }
            Bound::Excluded(b) => {
                let max = std::cmp::max(b, self.lo);
                prefix_encode(self.lo, max)
            }
        };

        let keys = self.keys();

        let successor_keys = keys.iter().filter(|k| {
            let ord = prefix_cmp(k, &predecessor_key);

            ord == std::cmp::Ordering::Greater
                || ord == std::cmp::Ordering::Equal
        });

        for encoded_key in successor_keys {
            let decoded_key = prefix_decode(self.lo, &encoded_key);

            if let Bound::Excluded(e) = bound {
                if e == &decoded_key {
                    // skip this excluded key
                    continue;
                }
            }

            // try to get this key until it works
            if let Some(value) = self.leaf_value_for_key(&decoded_key, config) {
                return Some((IVec::from(decoded_key), value));
            }
        }

        None
    }

    pub(crate) fn predecessor(
        &self,
        bound: &Bound<IVec>,
        config: &Config,
    ) -> Option<(IVec, IVec)> {
        assert!(!self.is_index);

        // This encoding happens this way because
        // the rightmost (unbounded) node has
        // a hi key represented by the empty slice
        let successor_key = match bound {
            Bound::Unbounded => {
                if self.hi.is_empty() {
                    prefix_encode(self.lo, &[255; 1024 * 1024])
                } else {
                    prefix_encode(self.lo, self.hi)
                }
            }
            Bound::Included(b) => {
                let min = if self.hi.is_empty() {
                    b
                } else {
                    std::cmp::min(b, self.hi)
                };
                prefix_encode(self.lo, min)
            }
            Bound::Excluded(b) => {
                let min = if self.hi.is_empty() {
                    b
                } else {
                    std::cmp::min(b, self.hi)
                };
                prefix_encode(self.lo, min)
            }
        };

        let keys = self.keys();

        let predecessor_keys = keys.iter().filter(|k| {
            let ord = prefix_cmp(k, &successor_key);

            ord == std::cmp::Ordering::Less || ord == std::cmp::Ordering::Equal
        });

        for encoded_key in predecessor_keys.rev() {
            let decoded_key = prefix_decode(self.lo, &encoded_key);

            if let Bound::Excluded(e) = bound {
                if e == &decoded_key {
                    // skip this excluded key
                    continue;
                }
            }

            // try to get this key until it works
            if let Some(value) = self.leaf_value_for_key(&decoded_key, config) {
                return Some((IVec::from(decoded_key), value));
            }
        }

        None
    }

    pub(crate) fn leaf_value_for_key(
        &self,
        key: &[u8],
        config: &Config,
    ) -> Option<IVec> {
        assert!(!self.is_index);

        let mut merge_base = None;
        let mut merges = vec![];

        for frag in self.frags[..=self.base_offset].iter() {
            match frag {
                Frag::Set(k, val) if self.key_eq(k, key) => {
                    if merges.is_empty() {
                        return Some(val.clone());
                    } else {
                        merge_base = Some(val);
                        break;
                    }
                }
                Frag::Del(k) if self.key_eq(k, key) => {
                    // we should ignore "earlier" Frag's for
                    // this key, but still need to handle
                    // merges we encountered "after" this
                    // deletion
                    break;
                }
                Frag::Merge(k, val) if self.key_eq(k, key) => merges.push(val),
                Frag::Base(node) => {
                    let data = &node.data;
                    let items =
                        data.leaf_ref().expect("last_node should be a leaf");
                    let search = items
                        .binary_search_by(|&(ref k, ref _v)| {
                            prefix_cmp_encoded(k, key, &self.lo)
                        })
                        .ok();

                    let val = search.map(|idx| &items[idx].1);
                    if merges.is_empty() {
                        return val.cloned();
                    } else {
                        merge_base = val;
                    }
                }
                _ => {}
            }
        }

        if merges.is_empty() {
            None
        } else {
            let merge_fn_ptr = config
                .merge_operator
                .expect("must have a merge operator set");

            unsafe {
                let merge_fn: MergeOperator = std::mem::transmute(merge_fn_ptr);

                let mut ret = merge_fn(
                    key,
                    merge_base.map(|iv| &**iv),
                    &merges.pop().unwrap(),
                );
                                       ;
                for merge in merges.into_iter().rev() {
                    if let Some(v) = ret {
                        ret = merge_fn(key, Some(&*v), merge);
                    } else {
                        ret = merge_fn(key, None, merge);
                    }
                }

                ret.map(IVec::from)
            }
        }
    }

    #[inline]
    fn key_eq(&self, encoded: &[u8], not_encoded: &[u8]) -> bool {
        prefix_cmp_encoded(encoded, not_encoded, self.lo)
            == std::cmp::Ordering::Equal
    }

    pub(crate) fn index_next_node(&self, key: &[u8]) -> (usize, PageId) {
        assert!(self.is_index);

        for frag in self.frags[..=self.base_offset].iter() {
            match frag {
                Frag::Set(..) => unimplemented!(),
                Frag::Del(..) => unimplemented!(),
                Frag::Merge(..) => unimplemented!(),
                Frag::Base(node) => {
                    let data = &node.data;
                    let items =
                        data.index_ref().expect("last_node should be a leaf");
                    let search =
                        binary_search_lub(items, |&(ref k, ref _v)| {
                            prefix_cmp_encoded(k, key, &node.lo)
                        });

                    // This might be none if ord is Less and we're
                    // searching for the empty key
                    let index = search.expect("failed to traverse index");

                    return (index, items[index].1);
                }
                Frag::ParentMergeIntention(_)
                | Frag::ParentMergeConfirm
                | Frag::ChildMergeCap => {
                    // nothing to do for these frags
                }
            }
        }
        panic!("no index found")
    }

    pub(crate) fn should_split(&self, max_sz: u64) -> bool {
        self.min_children > 2
            && self.size_in_bytes() > max_sz
            && self.merging_child.is_none()
            && !self.merging
    }

    pub(crate) fn should_merge(&self, min_sz: u64) -> bool {
        self.size_in_bytes() < min_sz
            && self.merging_child.is_none()
            && !self.merging
    }

    pub(crate) fn can_merge_child(&self) -> bool {
        self.merging_child.is_none() && !self.merging
    }

    pub(crate) fn compact(&self, config: &Config) -> Node {
        let mut lhs = self.frags[self.base_offset].unwrap_base().clone();
        for offset in (0..self.base_offset).rev() {
            let frag = self.frags[offset];
            lhs.apply(frag, config.merge_operator);
        }
        lhs
    }

    pub(crate) fn split(&self, config: &Config) -> (Node, Node) {
        let mut lhs = self.compact(config);
        let rhs = lhs.split();

        lhs.data.drop_gte(&rhs.lo, &lhs.lo);
        lhs.hi = rhs.lo.clone();

        // intentionally make this the end to make
        // any issues pop out with setting it
        // correctly after the split.
        lhs.next = None;

        (lhs, rhs)
    }

    #[inline]
    pub(crate) fn size_in_bytes(&self) -> u64 {
        // TODO needs to better account for
        // sizes that don't actually fall under
        // a merge threshold once we support one.
        self.frags[..=self.base_offset]
            .iter()
            .map(|f| f.size_in_bytes())
            .sum()
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! cmake wrapper for build

extern crate cmake;
extern crate fs_extra;

use cmake::Config;
use std::env;
use std::path::{PathBuf};
use fs_extra::dir::*;

fn main() {
	let path_str = env::var("OUT_DIR").unwrap();
	let mut out_path = PathBuf::from(&path_str);
	out_path.pop();
	out_path.pop();
	out_path.pop();
	let mut plugin_path = PathBuf::from(&path_str);
	plugin_path.push("build");
	plugin_path.push("plugins");
	println!("cargo:rerun-if-changed=build.rs");
	println!("cargo:rerun-if-changed=plugins");
	println!("cargo:rerun-if-changed=plugins/cmake");
	println!("cargo:rerun-if-changed=plugins/cuckoo/src");
	let dst = Config::new("plugins")
	                      //.define("FOO","BAR") //whatever flags go here
	                      //.cflag("-foo") //and here
	                      .build_target("")
	                      .build();

	println!("Plugin path: {:?}", plugin_path);
	println!("OUT PATH: {:?}", out_path);
	let options = CopyOptions::new();
	if let Err(e) = copy(plugin_path, out_path, &options) {
		println!("{:?}", e);
	}

	println!("cargo:rustc-link-search=native={}", dst.display());

}<commit_msg>rebuilding if included files are touched<commit_after>// Copyright 2017 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! cmake wrapper for build

extern crate cmake;
extern crate fs_extra;

use cmake::Config;
use std::env;
use std::path::{PathBuf};
use fs_extra::dir::*;

fn main() {
	let path_str = env::var("OUT_DIR").unwrap();
	let mut out_path = PathBuf::from(&path_str);
	out_path.pop();
	out_path.pop();
	out_path.pop();
	let mut plugin_path = PathBuf::from(&path_str);
	plugin_path.push("build");
	plugin_path.push("plugins");
	//Collect the files and directories we care about
	let dir_content = get_dir_content("plugins").unwrap();
	for d in dir_content.directories {
		let file_content = get_dir_content(d).unwrap();
		for f in file_content.files {
			println!("cargo:rerun-if-changed={}",f);
		}
	}
	for f in dir_content.files{
		println!("cargo:rerun-if-changed={}",f);
	}
	//panic!("marp");
	let dst = Config::new("plugins")
	                      //.define("FOO","BAR") //whatever flags go here
	                      //.cflag("-foo") //and here
	                      .build_target("")
	                      .build();

	println!("Plugin path: {:?}", plugin_path);
	println!("OUT PATH: {:?}", out_path);
	let mut options = CopyOptions::new();
	options.overwrite=true;
	if let Err(e) = copy(plugin_path, out_path, &options) {
		println!("{:?}", e);
	}

	println!("cargo:rustc-link-search=native={}", dst.display());

}<|endoftext|>


<commit_before>use ::anyhow::Result;
use ::async_std::net::TcpStream;
use ::async_std::prelude::*;
use ::byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use ::log::debug;
use ::std::io::Cursor;

use crate::message::*;

/// Convenience wrapper around send_bytes
/// Deserialize a message and feed the bytes into send_bytes
pub async fn send_message(message: Message, socket: &mut TcpStream) -> Result<()> {
    debug!("Sending message: {:?}", message);
    // Prepare command for transfer and determine message byte size
    let payload = bincode::serialize(&message).expect("Failed to serialize message.");

    send_bytes(payload, socket).await
}

/// Send a Vec of bytes. Before the actual bytes are send, the size of the message
/// is transmitted in an header of fixed size (u64).
pub async fn send_bytes(payload: Vec<u8>, socket: &mut TcpStream) -> Result<()> {
    let message_size = payload.len() as u64;

    let mut header = vec![];
    header.write_u64::<BigEndian>(message_size).unwrap();

    // Send the request size header first.
    // Afterwards send the request.
    socket.write_all(&header).await?;

    // Split the payload into 1.5kbyte chunks (MUT for TCP)
    let mut iter = payload.chunks(1500);
    while let Some(chunk) = iter.next() {
        socket.write_all(chunk).await?;
    }

    Ok(())
}

/// Receive a byte stream depending on a given header
/// This is the basic protocol beneath all pueue communication
pub async fn receive_bytes(socket: &mut TcpStream) -> Result<Vec<u8>> {
    // Receive the header with the overall message size
    let mut header = vec![0; 8];
    socket.read(&mut header).await?;
    let mut header = Cursor::new(header);
    let message_size = header.read_u64::<BigEndian>()? as usize;

    // Buffer for the whole payload
    let mut payload_bytes = Vec::with_capacity(message_size);

    // Receive chunks until we reached the expected message size
    while payload_bytes.len() < message_size {
        // Calculate the amount of bytes left
        // By default try a buffer size of 1024 bytes
        let mut chunk_size = message_size - payload_bytes.len();
        if chunk_size > 1024 {
            chunk_size = 1024;
        }

        // Read data and get the amount of received bytes
        let mut chunk = vec![0; chunk_size];
        let received_bytes = socket.read(&mut chunk).await?;

        // If we received less bytes than the chunk buffer size,
        // split the unneeded bytes, since they are filled with zeros
        if received_bytes < chunk_size {
            chunk.split_off(received_bytes);
        }

        payload_bytes.append(&mut chunk);
    }

    Ok(payload_bytes)
}

/// Convenience wrapper that receives a message and converts it into a Message
pub async fn receive_message(socket: &mut TcpStream) -> Result<Message> {
    let payload_bytes = receive_bytes(socket).await?;

    // Deserialize the message
    let message: Message = bincode::deserialize(&payload_bytes)?;
    debug!("Received message: {:?}", message);

    Ok(message)
}
<commit_msg>Add hint about possibly needed restart after upgrade<commit_after>use ::anyhow::{Context, Result};
use ::async_std::net::TcpStream;
use ::async_std::prelude::*;
use ::byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use ::log::debug;
use ::std::io::Cursor;

use crate::message::*;

/// Convenience wrapper around send_bytes
/// Deserialize a message and feed the bytes into send_bytes
pub async fn send_message(message: Message, socket: &mut TcpStream) -> Result<()> {
    debug!("Sending message: {:?}", message);
    // Prepare command for transfer and determine message byte size
    let payload = bincode::serialize(&message).expect("Failed to serialize message.");

    send_bytes(payload, socket).await
}

/// Send a Vec of bytes. Before the actual bytes are send, the size of the message
/// is transmitted in an header of fixed size (u64).
pub async fn send_bytes(payload: Vec<u8>, socket: &mut TcpStream) -> Result<()> {
    let message_size = payload.len() as u64;

    let mut header = vec![];
    header.write_u64::<BigEndian>(message_size).unwrap();

    // Send the request size header first.
    // Afterwards send the request.
    socket.write_all(&header).await?;

    // Split the payload into 1.5kbyte chunks (MUT for TCP)
    let mut iter = payload.chunks(1500);
    while let Some(chunk) = iter.next() {
        socket.write_all(chunk).await?;
    }

    Ok(())
}

/// Receive a byte stream depending on a given header
/// This is the basic protocol beneath all pueue communication
pub async fn receive_bytes(socket: &mut TcpStream) -> Result<Vec<u8>> {
    // Receive the header with the overall message size
    let mut header = vec![0; 8];
    socket.read(&mut header).await?;
    let mut header = Cursor::new(header);
    let message_size = header.read_u64::<BigEndian>()? as usize;

    // Buffer for the whole payload
    let mut payload_bytes = Vec::with_capacity(message_size);

    // Receive chunks until we reached the expected message size
    while payload_bytes.len() < message_size {
        // Calculate the amount of bytes left
        // By default try a buffer size of 1024 bytes
        let mut chunk_size = message_size - payload_bytes.len();
        if chunk_size > 1024 {
            chunk_size = 1024;
        }

        // Read data and get the amount of received bytes
        let mut chunk = vec![0; chunk_size];
        let received_bytes = socket.read(&mut chunk).await?;

        // If we received less bytes than the chunk buffer size,
        // split the unneeded bytes, since they are filled with zeros
        if received_bytes < chunk_size {
            chunk.split_off(received_bytes);
        }

        payload_bytes.append(&mut chunk);
    }

    Ok(payload_bytes)
}

/// Convenience wrapper that receives a message and converts it into a Message
pub async fn receive_message(socket: &mut TcpStream) -> Result<Message> {
    let payload_bytes = receive_bytes(socket).await?;

    // Deserialize the message
    let message: Message = bincode::deserialize(&payload_bytes)
        .context("In case you updated Pueue, try restarting the daemon. Otherwise please report this")?;
    debug!("Received message: {:?}", message);

    Ok(message)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use base64::DecodeError;
use std::process::ExitStatus;
use std::io;
use std::result;
use std::string::FromUtf8Error;
use rusoto_ecr::GetAuthorizationTokenError;

use common;
use hab;
use handlebars;
use hcore;
use failure;

pub type Result<T> = result::Result<T, failure::Error>;

#[derive(Debug, Fail)]
pub enum Error {
    #[fail(display = "{}", _0)]
    Base64DecodeError(DecodeError),
    #[fail(display = "Docker build failed with exit code: {}", _0)]
    BuildFailed(ExitStatus),
    #[fail(display = "Could not determine Docker image ID for image: {}", _0)]
    DockerImageIdNotFound(String),
    #[fail(display = "Switch to Windows containers to export Docker images on Windows. \
                     Current Docker Server OS is set to: {}",
           _0)]
    DockerNotInWindowsMode(String),
    #[fail(display = "{}", _0)]
    InvalidToken(FromUtf8Error),
    #[fail(display = "{}", _0)]
    Hab(hab::error::Error),
    #[fail(display = "{}", _0)]
    HabitatCommon(common::Error),
    #[fail(display = "{}", _0)]
    HabitatCore(hcore::Error),
    #[fail(display = "Docker login failed with exit code: {}", _0)]
    LoginFailed(ExitStatus),
    #[fail(display = "Docker logout failed with exit code: {}", _0)]
    LogoutFailed(ExitStatus),
    #[fail(display = "No ECR Tokens returned")]
    NoECRTokensReturned,
    #[fail(display = "{}", _0)]
    TokenFetchFailed(GetAuthorizationTokenError),
    #[fail(display = "A primary service package could not be determined from: {:?}. \
                     At least one package with a run hook must be provided.",
           _0)]
    PrimaryServicePackageNotFound(Vec<String>),
    #[fail(display = "Docker image push failed with exit code: {}", _0)]
    PushImageFailed(ExitStatus),
    #[fail(display = "Removing Docker local images failed with exit code: {}", _0)]
    RemoveImageFailed(ExitStatus),
    #[fail(display = "{}", _0)]
    TemplateRenderError(handlebars::TemplateRenderError),
    #[fail(display = "{}", _0)]
    IO(io::Error),
}
<commit_msg>pkg-export-docker: Remove redundant error enums<commit_after>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use base64::DecodeError;
use std::process::ExitStatus;
use std::result;
use std::string::FromUtf8Error;
use rusoto_ecr::GetAuthorizationTokenError;

use failure;

pub type Result<T> = result::Result<T, failure::Error>;

#[derive(Debug, Fail)]
pub enum Error {
    #[fail(display = "{}", _0)]
    Base64DecodeError(DecodeError),
    #[fail(display = "Docker build failed with exit code: {}", _0)]
    BuildFailed(ExitStatus),
    #[fail(display = "Could not determine Docker image ID for image: {}", _0)]
    DockerImageIdNotFound(String),
    #[fail(display = "Switch to Windows containers to export Docker images on Windows. \
                     Current Docker Server OS is set to: {}",
           _0)]
    DockerNotInWindowsMode(String),
    #[fail(display = "{}", _0)]
    InvalidToken(FromUtf8Error),
    #[fail(display = "Docker login failed with exit code: {}", _0)]
    LoginFailed(ExitStatus),
    #[fail(display = "Docker logout failed with exit code: {}", _0)]
    LogoutFailed(ExitStatus),
    #[fail(display = "No ECR Tokens returned")]
    NoECRTokensReturned,
    #[fail(display = "{}", _0)]
    TokenFetchFailed(GetAuthorizationTokenError),
    #[fail(display = "A primary service package could not be determined from: {:?}. \
                     At least one package with a run hook must be provided.",
           _0)]
    PrimaryServicePackageNotFound(Vec<String>),
    #[fail(display = "Docker image push failed with exit code: {}", _0)]
    PushImageFailed(ExitStatus),
    #[fail(display = "Removing Docker local images failed with exit code: {}", _0)]
    RemoveImageFailed(ExitStatus),
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::time::{SystemTime, Duration, UNIX_EPOCH};
use time::{self, Timespec, Tm};
use std::fs::{self, OpenOptions, File};
use std::io::{self, Write};
use std::path::Path;
use std::sync::Mutex;
use log::{self, LogLevelFilter, Log, LogMetadata, LogRecord, SetLoggerError};

use super::logger;

const ONE_DAY_SECONDS: u64 = 60 * 60 * 24;
const NANOSECONDS_PER_MILLISECOND: i32 = 1_000_000;

fn systemtime_to_tm(t: SystemTime) -> Tm {
    let duration = t.duration_since(UNIX_EPOCH).unwrap();
    let spec = Timespec::new(duration.as_secs() as i64, duration.subsec_nanos() as i32);
    time::at(spec)
}

fn compute_rollover_time(tm: Tm) -> Tm {
    let day_start_tm = Tm {
        tm_hour: 0,
        tm_min: 0,
        tm_sec: 0,
        tm_nsec: 0,
        ..tm
    };
    let duration = time::Duration::from_std(Duration::new(ONE_DAY_SECONDS, 0)).unwrap();
    (day_start_tm.to_utc() + duration).to_local()
}

fn one_day_before(tm: Tm) -> Tm {
    let duration = time::Duration::from_std(Duration::new(ONE_DAY_SECONDS, 0)).unwrap();
    (tm.to_utc() - duration).to_local()
}

fn open_log_file(path: &str) -> io::Result<File> {
    let p = Path::new(path);
    let parent = p.parent().unwrap();
    if !parent.is_dir() {
        try!(fs::create_dir_all(parent))
    }
    OpenOptions::new()
        .append(true)
        .create(true)
        .open(path)
}

struct RotatingFileLoggerCore {
    rollover_time: Tm,
    file_path: String,
    file: File,
}

impl RotatingFileLoggerCore {
    fn new(path: &str) -> io::Result<RotatingFileLoggerCore> {
        let file = try!(open_log_file(path));
        let file_attr = fs::metadata(path).unwrap();
        let file_modified_time = file_attr.modified().unwrap();
        let rollover_time = compute_rollover_time(systemtime_to_tm(file_modified_time));
        let ret = RotatingFileLoggerCore {
            rollover_time: rollover_time,
            file_path: path.to_string(),
            file: file,
        };
        Ok(ret)
    }

    fn open(&mut self) {
        self.file = open_log_file(&self.file_path).unwrap()
    }

    fn should_rollover(&mut self) -> bool {
        time::now() > self.rollover_time
    }

    fn do_rollover(&mut self) {
        self.close();
        let mut s = self.file_path.clone();
        s.push_str(".");
        s.push_str(&time::strftime("%Y%m%d", &one_day_before(self.rollover_time)).unwrap());
        fs::rename(&self.file_path, &s).unwrap();
        self.update_rollover_time();
        self.open()
    }

    fn update_rollover_time(&mut self) {
        let now = time::now();
        self.rollover_time = compute_rollover_time(now);
    }

    fn close(&mut self) {
        self.file.flush().unwrap()
    }
}

pub fn init(level: &str, file_path: &str) -> Result<(), SetLoggerError> {
    let l = logger::get_level_by_string(level);
    log::set_logger(|max_log_level| {
        max_log_level.set(l);
        Box::new(RotatingFileLogger::new(level, file_path).unwrap())
    })
}

/// A log implemetation which writes to file and rotates by day.
pub struct RotatingFileLogger {
    level: LogLevelFilter,
    core: Mutex<RotatingFileLoggerCore>,
}

impl RotatingFileLogger {
    pub fn new(level: &str, file_path: &str) -> io::Result<RotatingFileLogger> {
        let core = try!(RotatingFileLoggerCore::new(file_path));
        let ret = RotatingFileLogger {
            level: logger::get_level_by_string(level),
            core: Mutex::new(core),
        };
        Ok(ret)
    }
}

impl Log for RotatingFileLogger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= self.level
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            let mut core = self.core.lock().unwrap();
            if core.should_rollover() {
                core.do_rollover()
            };
            let now = time::now();
            let _ = write!(core.file,
                           "{},{:03} {}:{} - {:5} - {}\n",
                           time::strftime("%Y-%m-%d %H:%M:%S", &now).unwrap(),
                           now.tm_nsec / NANOSECONDS_PER_MILLISECOND,
                           record.location().file().rsplit('/').nth(0).unwrap(),
                           record.location().line(),
                           record.level(),
                           record.args());
        }
    }
}

impl Drop for RotatingFileLogger {
    fn drop(&mut self) {
        let mut core = self.core.lock().unwrap();
        core.close()
    }
}

#[cfg(test)]
mod tests {
    extern crate log;
    extern crate rand;
    extern crate utime;
    use time::{self, Timespec};
    use std::io::prelude::*;
    use std::fs::OpenOptions;
    use std::path::Path;
    use tempdir::TempDir;
    use super::{RotatingFileLoggerCore, ONE_DAY_SECONDS};

    fn file_exists(file: &str) -> bool {
        let path = Path::new(file);
        path.exists() && path.is_file()
    }

    #[test]
    fn test_rotating_file_logger() {
        let tmp_dir = TempDir::new("").unwrap();
        let log_file =
            tmp_dir.path().join("test_rotating_file_logger.log").to_str().unwrap().to_string();
        // create a file with mtime == one day ago
        {
            let mut file = OpenOptions::new()
                .append(true)
                .create(true)
                .open(&log_file)
                .unwrap();
            file.write_all(b"hello world!").unwrap();
        }
        let ts = time::now().to_timespec();
        let one_day_ago = Timespec::new(ts.sec - ONE_DAY_SECONDS as i64, ts.nsec);
        let time_in_sec = one_day_ago.sec as u64;
        utime::set_file_times(&log_file, time_in_sec, time_in_sec).unwrap();
        // initialize the logger
        let mut core = RotatingFileLoggerCore::new(&log_file).unwrap();
        assert!(core.should_rollover());
        core.do_rollover();
        // check the rotated file exist
        let mut rotated_file = log_file.clone();
        rotated_file.push_str(".");
        let file_suffix_time =
            super::one_day_before(super::compute_rollover_time(time::at(one_day_ago)));
        rotated_file.push_str(&time::strftime("%Y%m%d", &file_suffix_time).unwrap());
        assert!(file_exists(&rotated_file));
        assert!(!core.should_rollover());
    }
}
<commit_msg>add doc and test case for util function<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::time::{SystemTime, Duration, UNIX_EPOCH};
use time::{self, Timespec, Tm};
use std::fs::{self, OpenOptions, File};
use std::io::{self, Write};
use std::path::Path;
use std::sync::Mutex;
use log::{self, LogLevelFilter, Log, LogMetadata, LogRecord, SetLoggerError};

use super::logger;

const ONE_DAY_SECONDS: u64 = 60 * 60 * 24;
const NANOSECONDS_PER_MILLISECOND: i32 = 1_000_000;

fn systemtime_to_tm(t: SystemTime) -> Tm {
    let duration = t.duration_since(UNIX_EPOCH).unwrap();
    let spec = Timespec::new(duration.as_secs() as i64, duration.subsec_nanos() as i32);
    time::at(spec)
}

fn compute_rollover_time(tm: Tm) -> Tm {
    let day_start_tm = Tm {
        tm_hour: 0,
        tm_min: 0,
        tm_sec: 0,
        tm_nsec: 0,
        ..tm
    };
    let duration = time::Duration::from_std(Duration::new(ONE_DAY_SECONDS, 0)).unwrap();
    (day_start_tm.to_utc() + duration).to_local()
}

/// Returns a Tm at the time one day before the given Tm.
/// It expects the argument `tm` to be in local timezone. The resulting Tm is in local timezone.
fn one_day_before(tm: Tm) -> Tm {
    let duration = time::Duration::from_std(Duration::new(ONE_DAY_SECONDS, 0)).unwrap();
    (tm.to_utc() - duration).to_local()
}

fn open_log_file(path: &str) -> io::Result<File> {
    let p = Path::new(path);
    let parent = p.parent().unwrap();
    if !parent.is_dir() {
        try!(fs::create_dir_all(parent))
    }
    OpenOptions::new()
        .append(true)
        .create(true)
        .open(path)
}

struct RotatingFileLoggerCore {
    rollover_time: Tm,
    file_path: String,
    file: File,
}

impl RotatingFileLoggerCore {
    fn new(path: &str) -> io::Result<RotatingFileLoggerCore> {
        let file = try!(open_log_file(path));
        let file_attr = fs::metadata(path).unwrap();
        let file_modified_time = file_attr.modified().unwrap();
        let rollover_time = compute_rollover_time(systemtime_to_tm(file_modified_time));
        let ret = RotatingFileLoggerCore {
            rollover_time: rollover_time,
            file_path: path.to_string(),
            file: file,
        };
        Ok(ret)
    }

    fn open(&mut self) {
        self.file = open_log_file(&self.file_path).unwrap()
    }

    fn should_rollover(&mut self) -> bool {
        time::now() > self.rollover_time
    }

    fn do_rollover(&mut self) {
        self.close();
        let mut s = self.file_path.clone();
        s.push_str(".");
        s.push_str(&time::strftime("%Y%m%d", &one_day_before(self.rollover_time)).unwrap());
        fs::rename(&self.file_path, &s).unwrap();
        self.update_rollover_time();
        self.open()
    }

    fn update_rollover_time(&mut self) {
        let now = time::now();
        self.rollover_time = compute_rollover_time(now);
    }

    fn close(&mut self) {
        self.file.flush().unwrap()
    }
}

pub fn init(level: &str, file_path: &str) -> Result<(), SetLoggerError> {
    let l = logger::get_level_by_string(level);
    log::set_logger(|max_log_level| {
        max_log_level.set(l);
        Box::new(RotatingFileLogger::new(level, file_path).unwrap())
    })
}

/// A log implemetation which writes to file and rotates by day.
pub struct RotatingFileLogger {
    level: LogLevelFilter,
    core: Mutex<RotatingFileLoggerCore>,
}

impl RotatingFileLogger {
    pub fn new(level: &str, file_path: &str) -> io::Result<RotatingFileLogger> {
        let core = try!(RotatingFileLoggerCore::new(file_path));
        let ret = RotatingFileLogger {
            level: logger::get_level_by_string(level),
            core: Mutex::new(core),
        };
        Ok(ret)
    }
}

impl Log for RotatingFileLogger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= self.level
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            let mut core = self.core.lock().unwrap();
            if core.should_rollover() {
                core.do_rollover()
            };
            let now = time::now();
            let _ = write!(core.file,
                           "{},{:03} {}:{} - {:5} - {}\n",
                           time::strftime("%Y-%m-%d %H:%M:%S", &now).unwrap(),
                           now.tm_nsec / NANOSECONDS_PER_MILLISECOND,
                           record.location().file().rsplit('/').nth(0).unwrap(),
                           record.location().line(),
                           record.level(),
                           record.args());
        }
    }
}

impl Drop for RotatingFileLogger {
    fn drop(&mut self) {
        let mut core = self.core.lock().unwrap();
        core.close()
    }
}

#[cfg(test)]
mod tests {
    extern crate log;
    extern crate rand;
    extern crate utime;
    use time::{self, Timespec};
    use std::io::prelude::*;
    use std::fs::OpenOptions;
    use std::path::Path;
    use tempdir::TempDir;
    use super::{RotatingFileLoggerCore, ONE_DAY_SECONDS};

    #[test]
    fn test_one_day_before() {
        let tm = time::strptime("2016-08-30", "%Y-%m-%d").unwrap().to_local();
        let one_day_ago = time::strptime("2016-08-29", "%Y-%m-%d").unwrap().to_local();
        assert_eq!(one_day_ago, super::one_day_before(tm));
    }

    fn file_exists(file: &str) -> bool {
        let path = Path::new(file);
        path.exists() && path.is_file()
    }

    #[test]
    fn test_rotating_file_logger() {
        let tmp_dir = TempDir::new("").unwrap();
        let log_file =
            tmp_dir.path().join("test_rotating_file_logger.log").to_str().unwrap().to_string();
        // create a file with mtime == one day ago
        {
            let mut file = OpenOptions::new()
                .append(true)
                .create(true)
                .open(&log_file)
                .unwrap();
            file.write_all(b"hello world!").unwrap();
        }
        let ts = time::now().to_timespec();
        let one_day_ago = Timespec::new(ts.sec - ONE_DAY_SECONDS as i64, ts.nsec);
        let time_in_sec = one_day_ago.sec as u64;
        utime::set_file_times(&log_file, time_in_sec, time_in_sec).unwrap();
        // initialize the logger
        let mut core = RotatingFileLoggerCore::new(&log_file).unwrap();
        assert!(core.should_rollover());
        core.do_rollover();
        // check the rotated file exist
        let mut rotated_file = log_file.clone();
        rotated_file.push_str(".");
        let file_suffix_time =
            super::one_day_before(super::compute_rollover_time(time::at(one_day_ago)));
        rotated_file.push_str(&time::strftime("%Y%m%d", &file_suffix_time).unwrap());
        assert!(file_exists(&rotated_file));
        assert!(!core.should_rollover());
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(json_macros)]

#[cfg(target_family="unix")]
extern crate fuse;
extern crate time;
extern crate libc;
extern crate rustc_serialize;

use std::collections::BTreeMap;
use std::env;
use std::path::Path;
use libc::{ENOENT};
use time::Timespec;
#[cfg(target_family="unix")]
use fuse::{FileAttr, FileType, Filesystem, Request, ReplyAttr, ReplyData, ReplyEntry, ReplyDirectory};
use rustc_serialize::json;

#[cfg(target_family="unix")]
struct JsonFilesystem {
    tree: json::Object,
    attrs: BTreeMap<u64, FileAttr>,
    inodes: BTreeMap<String, u64>,
}

#[cfg(target_family="unix")]
impl JsonFilesystem {
    fn new(tree: &json::Object) -> JsonFilesystem {
        let mut attrs = BTreeMap::new();
        let mut inodes = BTreeMap::new();
        let ts = time::now().to_timespec();
        let attr = FileAttr {
            ino: 1,
            size: 0,
            blocks: 0,
            atime: ts,
            mtime: ts,
            ctime: ts,
            crtime: ts,
            kind: FileType::Directory,
            perm: 0o755,
            nlink: 0,
            uid: 0,
            gid: 0,
            rdev: 0,
            flags: 0,
        };
        attrs.insert(1, attr);
        inodes.insert("/".to_owned(), 1);
        for (i, (key, value)) in tree.iter().enumerate() {
            let attr = FileAttr {
                ino: i as u64 + 2,
                size: value.pretty().to_string().len() as u64,
                blocks: 0,
                atime: ts,
                mtime: ts,
                ctime: ts,
                crtime: ts,
                kind: FileType::RegularFile,
                perm: 0o644,
                nlink: 0,
                uid: 0,
                gid: 0,
                rdev: 0,
                flags: 0,
            };
            attrs.insert(attr.ino, attr);
            inodes.insert(key.clone(), attr.ino);
        }
        JsonFilesystem { tree: tree.clone(), attrs: attrs, inodes: inodes }
    }
}

#[cfg(target_family="unix")]
impl Filesystem for JsonFilesystem {
    fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
        println!("getattr(ino={})", ino);
        match self.attrs.get(&ino) {
            Some(attr) => {
                let ttl = Timespec::new(1, 0);
                reply.attr(&ttl, attr);
            },
            None => reply.error(ENOENT),
        };
    }

    fn lookup(&mut self, _req: &Request, parent: u64, name: &Path, reply: ReplyEntry) {
        println!("lookup(parent={}, name={})", parent, name.display());
        let inode = match self.inodes.get(name.to_str().unwrap()) {
            Some(inode) => inode,
            None => {
                reply.error(ENOENT);
                return;
            },
        };
        match self.attrs.get(inode) {
            Some(attr) => {
                let ttl = Timespec::new(1, 0);
                reply.entry(&ttl, attr, 0);
            },
            None => reply.error(ENOENT),
        };
    }

    fn read(&mut self, _req: &Request, ino: u64, fh: u64, offset: u64, size: u32, reply: ReplyData) {
        println!("read(ino={}, fh={}, offset={}, size={})", ino, fh, offset, size);
        for (key, &inode) in &self.inodes {
            if inode == ino {
                let value = self.tree.get(key).unwrap();
                reply.data(value.pretty().to_string().as_bytes());
                return;
            }
        }
        reply.error(ENOENT);
    }

    fn readdir(&mut self, _req: &Request, ino: u64, fh: u64, offset: u64, mut reply: ReplyDirectory) {
        println!("readdir(ino={}, fh={}, offset={})", ino, fh, offset);
        if ino == 1 {
            if offset == 0 {
                reply.add(1, 0, FileType::Directory, &Path::new("."));
                reply.add(1, 1, FileType::Directory, &Path::new(".."));
                for (key, &inode) in &self.inodes {
                    if inode == 1 { continue; }
                    let offset = inode; // hack
                    println!("\tkey={}, inode={}, offset={}", key, inode, offset);
                    reply.add(inode, offset, FileType::RegularFile, &Path::new(key));
                }
            }
            reply.ok();
        } else {
            reply.error(ENOENT);
        }
    }
}

#[cfg(target_family="windows")]
fn main() {
    println!("24 days of Rust - fuse (days 15 & 16)");
    println!("This example does not work on Windows :(");
}

#[cfg(target_family="unix")]
fn main() {
    println!("24 days of Rust - fuse (days 15 & 16)");
    let data = json!({
        "foo": "bar",
        "answer": 42,
    });
    let tree = data.as_object().unwrap();
    let fs = JsonFilesystem::new(tree);
    let mountpoint = match env::args().nth(1) {
        Some(path) => path,
        None => {
            println!("Usage: {} <MOUNTPOINT>", env::args().nth(0).unwrap());
            return;
        }
    };
    fuse::mount(fs, &mountpoint, &[]);
}
<commit_msg>Use JSON string instead of json! macro in vol1, day 15.<commit_after>#[cfg(target_family="unix")]
extern crate fuse;
extern crate time;
extern crate libc;
extern crate rustc_serialize;

use std::collections::BTreeMap;
use std::env;
use std::path::Path;
use libc::{ENOENT};
use time::Timespec;
#[cfg(target_family="unix")]
use fuse::{FileAttr, FileType, Filesystem, Request, ReplyAttr, ReplyData, ReplyEntry, ReplyDirectory};
use rustc_serialize::json;

#[cfg(target_family="unix")]
struct JsonFilesystem {
    tree: json::Object,
    attrs: BTreeMap<u64, FileAttr>,
    inodes: BTreeMap<String, u64>,
}

#[cfg(target_family="unix")]
impl JsonFilesystem {
    fn new(tree: &json::Object) -> JsonFilesystem {
        let mut attrs = BTreeMap::new();
        let mut inodes = BTreeMap::new();
        let ts = time::now().to_timespec();
        let attr = FileAttr {
            ino: 1,
            size: 0,
            blocks: 0,
            atime: ts,
            mtime: ts,
            ctime: ts,
            crtime: ts,
            kind: FileType::Directory,
            perm: 0o755,
            nlink: 0,
            uid: 0,
            gid: 0,
            rdev: 0,
            flags: 0,
        };
        attrs.insert(1, attr);
        inodes.insert("/".to_owned(), 1);
        for (i, (key, value)) in tree.iter().enumerate() {
            let attr = FileAttr {
                ino: i as u64 + 2,
                size: value.pretty().to_string().len() as u64,
                blocks: 0,
                atime: ts,
                mtime: ts,
                ctime: ts,
                crtime: ts,
                kind: FileType::RegularFile,
                perm: 0o644,
                nlink: 0,
                uid: 0,
                gid: 0,
                rdev: 0,
                flags: 0,
            };
            attrs.insert(attr.ino, attr);
            inodes.insert(key.clone(), attr.ino);
        }
        JsonFilesystem { tree: tree.clone(), attrs: attrs, inodes: inodes }
    }
}

#[cfg(target_family="unix")]
impl Filesystem for JsonFilesystem {
    fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
        println!("getattr(ino={})", ino);
        match self.attrs.get(&ino) {
            Some(attr) => {
                let ttl = Timespec::new(1, 0);
                reply.attr(&ttl, attr);
            },
            None => reply.error(ENOENT),
        };
    }

    fn lookup(&mut self, _req: &Request, parent: u64, name: &Path, reply: ReplyEntry) {
        println!("lookup(parent={}, name={})", parent, name.display());
        let inode = match self.inodes.get(name.to_str().unwrap()) {
            Some(inode) => inode,
            None => {
                reply.error(ENOENT);
                return;
            },
        };
        match self.attrs.get(inode) {
            Some(attr) => {
                let ttl = Timespec::new(1, 0);
                reply.entry(&ttl, attr, 0);
            },
            None => reply.error(ENOENT),
        };
    }

    fn read(&mut self, _req: &Request, ino: u64, fh: u64, offset: u64, size: u32, reply: ReplyData) {
        println!("read(ino={}, fh={}, offset={}, size={})", ino, fh, offset, size);
        for (key, &inode) in &self.inodes {
            if inode == ino {
                let value = self.tree.get(key).unwrap();
                reply.data(value.pretty().to_string().as_bytes());
                return;
            }
        }
        reply.error(ENOENT);
    }

    fn readdir(&mut self, _req: &Request, ino: u64, fh: u64, offset: u64, mut reply: ReplyDirectory) {
        println!("readdir(ino={}, fh={}, offset={})", ino, fh, offset);
        if ino == 1 {
            if offset == 0 {
                reply.add(1, 0, FileType::Directory, &Path::new("."));
                reply.add(1, 1, FileType::Directory, &Path::new(".."));
                for (key, &inode) in &self.inodes {
                    if inode == 1 { continue; }
                    let offset = inode; // hack
                    println!("\tkey={}, inode={}, offset={}", key, inode, offset);
                    reply.add(inode, offset, FileType::RegularFile, &Path::new(key));
                }
            }
            reply.ok();
        } else {
            reply.error(ENOENT);
        }
    }
}

#[cfg(target_family="windows")]
fn main() {
    println!("24 days of Rust - fuse (days 15 & 16)");
    println!("This example does not work on Windows :(");
}

#[cfg(target_family="unix")]
fn main() {
    println!("24 days of Rust - fuse (days 15 & 16)");
    let data = json::Json::from_str("{\"foo\": \"bar\", \"answer\": 42}").unwrap();
    let tree = data.as_object().unwrap();
    let fs = JsonFilesystem::new(tree);
    let mountpoint = match env::args().nth(1) {
        Some(path) => path,
        None => {
            println!("Usage: {} <MOUNTPOINT>", env::args().nth(0).unwrap());
            return;
        }
    };
    fuse::mount(fs, &mountpoint, &[]);
}
<|endoftext|>


<commit_before>#[cfg(target_family="unix")]
extern crate lzma;
extern crate zip;

use std::fs::{File, metadata};
use std::io::{Read, Seek, Write};
use zip::read::{ZipArchive, ZipFile};
use zip::result::ZipResult;
use zip::write::{FileOptions, ZipWriter};

#[cfg(target_family="unix")]
use lzma::{LzmaWriter, LzmaError};

static FILE_CONTENTS: &'static [u8] = include_bytes!("../Cargo.lock");

fn create_zip_archive<T: Seek + Write>(buf: &mut T) -> ZipResult<()> {
    let mut writer = ZipWriter::new(buf);
    writer.start_file("example.txt", FileOptions::default())?;
    writer.write(FILE_CONTENTS)?;
    writer.finish()?;
    Ok(())
}

fn browse_zip_archive<T, F, U>(buf: &mut T, browse_func: F) -> ZipResult<Vec<U>>
    where T: Read + Seek,
          F: Fn(&ZipFile) -> ZipResult<U>
{
    let mut archive = ZipArchive::new(buf)?;
    (0..archive.len())
        .map(|i| archive.by_index(i).and_then(|file| browse_func(&file)))
        .collect()
}

fn create_bz2_archive<T: Seek + Write>(buf: &mut T) -> ZipResult<()> {
    let mut writer = ZipWriter::new(buf);
    writer.start_file("example.txt",
                    FileOptions::default().compression_method(zip::CompressionMethod::Bzip2))?;
    writer.write(FILE_CONTENTS)?;
    writer.finish()?;
    Ok(())
}

#[cfg(target_family="unix")]
fn create_xz_archive<T: Write>(buf: &mut T) -> Result<(), LzmaError> {
    let mut writer = LzmaWriter::new_compressor(buf, 6)?;
    writer.write(FILE_CONTENTS)?;
    writer.finish()?;
    Ok(())
}

#[cfg(target_family="windows")]
fn create_xz_archive<T: Write>(_: &mut T) -> Result<(), ()> {
    Ok(())
}

fn main() {
    println!("24 Days of Rust vol. 2 - zip");
    let mut file = File::create("example.zip").expect("Couldn't create file");
    create_zip_archive(&mut file).expect("Couldn't create archive");

    let mut file = File::open("example.zip").expect("Couldn't open file");
    let files = browse_zip_archive(&mut file, |f| {
        Ok(format!("{}: {} -> {}", f.name(), f.size(), f.compressed_size()))
    });
    println!("{:?}", files);

    let mut file = File::create("example.bz2").expect("Couldn't create file");
    create_bz2_archive(&mut file).expect("Couldn't create archive");

    let mut file = File::create("example.xz").expect("Couldn't create file");
    create_xz_archive(&mut file).expect("Couldn't create archive");

    if let Ok(meta) = metadata("example.zip") {
        println!("ZIP file size: {} bytes", meta.len());
    }
    if let Ok(meta) = metadata("example.bz2") {
        println!("BZ2 file size: {} bytes", meta.len());
    }
    if let Ok(meta) = metadata("example.xz") {
        println!("XZ file size: {} bytes", meta.len());
    }
}
<commit_msg>Use write_all in archive examples.<commit_after>#[cfg(target_family="unix")]
extern crate lzma;
extern crate zip;

use std::fs::{File, metadata};
use std::io::{Read, Seek, Write};
use zip::read::{ZipArchive, ZipFile};
use zip::result::ZipResult;
use zip::write::{FileOptions, ZipWriter};

#[cfg(target_family="unix")]
use lzma::{LzmaWriter, LzmaError};

static FILE_CONTENTS: &'static [u8] = include_bytes!("../Cargo.lock");

fn create_zip_archive<T: Seek + Write>(buf: &mut T) -> ZipResult<()> {
    let mut writer = ZipWriter::new(buf);
    writer.start_file("example.txt", FileOptions::default())?;
    writer.write_all(FILE_CONTENTS)?;
    writer.finish()?;
    Ok(())
}

fn browse_zip_archive<T, F, U>(buf: &mut T, browse_func: F) -> ZipResult<Vec<U>>
    where T: Read + Seek,
          F: Fn(&ZipFile) -> ZipResult<U>
{
    let mut archive = ZipArchive::new(buf)?;
    (0..archive.len())
        .map(|i| archive.by_index(i).and_then(|file| browse_func(&file)))
        .collect()
}

fn create_bz2_archive<T: Seek + Write>(buf: &mut T) -> ZipResult<()> {
    let mut writer = ZipWriter::new(buf);
    writer.start_file("example.txt",
                    FileOptions::default().compression_method(zip::CompressionMethod::Bzip2))?;
    writer.write_all(FILE_CONTENTS)?;
    writer.finish()?;
    Ok(())
}

#[cfg(target_family="unix")]
fn create_xz_archive<T: Write>(buf: &mut T) -> Result<(), LzmaError> {
    let mut writer = LzmaWriter::new_compressor(buf, 6)?;
    writer.write_all(FILE_CONTENTS)?;
    writer.finish()?;
    Ok(())
}

#[cfg(target_family="windows")]
fn create_xz_archive<T: Write>(_: &mut T) -> Result<(), ()> {
    Ok(())
}

fn main() {
    println!("24 Days of Rust vol. 2 - zip");
    let mut file = File::create("example.zip").expect("Couldn't create file");
    create_zip_archive(&mut file).expect("Couldn't create archive");

    let mut file = File::open("example.zip").expect("Couldn't open file");
    let files = browse_zip_archive(&mut file, |f| {
        Ok(format!("{}: {} -> {}", f.name(), f.size(), f.compressed_size()))
    });
    println!("{:?}", files);

    let mut file = File::create("example.bz2").expect("Couldn't create file");
    create_bz2_archive(&mut file).expect("Couldn't create archive");

    let mut file = File::create("example.xz").expect("Couldn't create file");
    create_xz_archive(&mut file).expect("Couldn't create archive");

    if let Ok(meta) = metadata("example.zip") {
        println!("ZIP file size: {} bytes", meta.len());
    }
    if let Ok(meta) = metadata("example.bz2") {
        println!("BZ2 file size: {} bytes", meta.len());
    }
    if let Ok(meta) = metadata("example.xz") {
        println!("XZ file size: {} bytes", meta.len());
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    chained_bft::{
        block_storage::{BlockReader, BlockStore},
        network::NetworkSender,
        persistent_liveness_storage::{PersistentLivenessStorage, RecoveryData},
    },
    counters,
    state_replication::StateComputer,
};
use anyhow::{bail, format_err};
use consensus_types::{
    block::Block,
    block_retrieval::{BlockRetrievalRequest, BlockRetrievalStatus},
    common::{Author, Payload},
    quorum_cert::QuorumCert,
    sync_info::SyncInfo,
};
use libra_logger::prelude::*;
use libra_types::{account_address::AccountAddress, validator_change::ValidatorChangeProof};
use mirai_annotations::checked_precondition;
use rand::{prelude::*, Rng};
use std::{
    clone::Clone,
    sync::Arc,
    time::{Duration, Instant},
};
use termion::color::*;

#[derive(Debug, PartialEq)]
/// Whether we need to do block retrieval if we want to insert a Quorum Cert.
pub enum NeedFetchResult {
    QCRoundBeforeRoot,
    QCAlreadyExist,
    QCBlockExist,
    NeedFetch,
}

impl<T: Payload> BlockStore<T> {
    /// Check if we're far away from this ledger info and need to sync.
    /// Returns false if we have this block in the tree or the root's round is higher than the
    /// block.
    pub fn need_sync_for_quorum_cert(&self, qc: &QuorumCert) -> bool {
        // This precondition ensures that the check in the following lines
        // does not result in an addition overflow.
        checked_precondition!(self.root().round() < std::u64::MAX - 1);

        // If we have the block locally, we're not far from this QC thus don't need to sync.
        // In case root().round() is greater than that the committed
        // block carried by LI is older than my current commit.
        !(self.block_exists(qc.commit_info().id())
            || self.root().round() >= qc.commit_info().round())
    }

    /// Checks if quorum certificate can be inserted in block store without RPC
    /// Returns the enum to indicate the detailed status.
    pub fn need_fetch_for_quorum_cert(&self, qc: &QuorumCert) -> NeedFetchResult {
        if qc.certified_block().round() < self.root().round() {
            return NeedFetchResult::QCRoundBeforeRoot;
        }
        if self
            .get_quorum_cert_for_block(qc.certified_block().id())
            .is_some()
        {
            return NeedFetchResult::QCAlreadyExist;
        }
        if self.block_exists(qc.certified_block().id()) {
            return NeedFetchResult::QCBlockExist;
        }
        NeedFetchResult::NeedFetch
    }

    /// Fetches dependencies for given sync_info.quorum_cert
    /// If gap is large, performs state sync using process_highest_commit_cert
    /// Inserts sync_info.quorum_cert into block store as the last step
    pub async fn sync_to(
        &self,
        sync_info: &SyncInfo,
        mut retriever: BlockRetriever<T>,
    ) -> anyhow::Result<()> {
        self.process_highest_commit_cert(sync_info.highest_commit_cert().clone(), &mut retriever)
            .await?;

        match self.need_fetch_for_quorum_cert(sync_info.highest_quorum_cert()) {
            NeedFetchResult::NeedFetch => {
                self.fetch_quorum_cert(sync_info.highest_quorum_cert().clone(), retriever)
                    .await?
            }
            NeedFetchResult::QCBlockExist => {
                self.insert_single_quorum_cert(sync_info.highest_quorum_cert().clone())?
            }
            _ => (),
        }
        Ok(())
    }

    /// Insert the quorum certificate separately from the block, used to split the processing of
    /// updating the consensus state(with qc) and deciding whether to vote(with block)
    /// The missing ancestors are going to be retrieved from the given peer. If a given peer
    /// fails to provide the missing ancestors, the qc is not going to be added.
    async fn fetch_quorum_cert(
        &self,
        qc: QuorumCert,
        mut retriever: BlockRetriever<T>,
    ) -> anyhow::Result<()> {
        let mut pending = vec![];
        let mut retrieve_qc = qc.clone();
        loop {
            if self.block_exists(retrieve_qc.certified_block().id()) {
                break;
            }
            let mut blocks = retriever.retrieve_block_for_qc(&retrieve_qc, 1).await?;
            // retrieve_block_for_qc guarantees that blocks has exactly 1 element
            let block = blocks.remove(0);
            retrieve_qc = block.quorum_cert().clone();
            pending.push(block);
        }
        // insert the qc <- block pair
        while let Some(block) = pending.pop() {
            let block_qc = block.quorum_cert().clone();
            self.insert_single_quorum_cert(block_qc)?;
            self.execute_and_insert_block(block)?;
        }
        self.insert_single_quorum_cert(qc)
    }

    /// Check the highest commit cert sent by peer to see if we're behind and start a fast
    /// forward sync if the committed block doesn't exist in our tree.
    /// It works as follows:
    /// 1. request the committed 3-chain from the peer, if C2 is the highest_commit_cert
    /// we request for B0 <- C0 <- B1 <- C1 <- B2 (<- C2)
    /// 2. We persist the 3-chain to storage before start sync to ensure we could restart if we
    /// crash in the middle of the sync.
    /// 3. We prune the old tree and replace with a new tree built with the 3-chain.
    async fn process_highest_commit_cert(
        &self,
        highest_commit_cert: QuorumCert,
        retriever: &mut BlockRetriever<T>,
    ) -> anyhow::Result<()> {
        if !self.need_sync_for_quorum_cert(&highest_commit_cert) {
            return Ok(());
        }
        let (root, root_metadata, blocks, quorum_certs) = Self::fast_forward_sync(
            &highest_commit_cert,
            retriever,
            self.storage.clone(),
            self.state_computer.clone(),
        )
        .await?
        .take();
        debug!("{}Sync to{} {}", Fg(Blue), Fg(Reset), root.0);
        self.rebuild(root, root_metadata, blocks, quorum_certs)
            .await;

        if highest_commit_cert.ends_epoch() {
            retriever
                .network
                .notify_epoch_change(ValidatorChangeProof::new(
                    vec![highest_commit_cert.ledger_info().clone()],
                    /* more = */ false,
                ))
                .await;
        }
        Ok(())
    }

    pub async fn fast_forward_sync<'a>(
        highest_commit_cert: &'a QuorumCert,
        retriever: &'a mut BlockRetriever<T>,
        storage: Arc<dyn PersistentLivenessStorage<T>>,
        state_computer: Arc<dyn StateComputer<Payload = T>>,
    ) -> anyhow::Result<RecoveryData<T>> {
        debug!(
            "Start state sync with peer: {}, to block: {}",
            retriever.preferred_peer.short_str(),
            highest_commit_cert.commit_info(),
        );

        let blocks = retriever
            .retrieve_block_for_qc(&highest_commit_cert, 3)
            .await?;
        assert_eq!(
            blocks.last().expect("should have 3-chain").id(),
            highest_commit_cert.commit_info().id(),
        );
        let mut quorum_certs = vec![];
        quorum_certs.push(highest_commit_cert.clone());
        quorum_certs.extend(blocks.iter().map(|block| block.quorum_cert().clone()));

        // If a node restarts in the middle of state synchronization, it is going to try to catch up
        // to the stored quorum certs as the new root.
        storage.save_tree(blocks.clone(), quorum_certs.clone())?;
        let pre_sync_instance = Instant::now();
        state_computer
            .sync_to(highest_commit_cert.ledger_info().clone())
            .await?;
        counters::STATE_SYNC_DURATION_S.observe_duration(pre_sync_instance.elapsed());
        let recovery_data = storage
            .start()
            .expect_recovery_data("Failed to construct recovery data after fast forward sync");

        Ok(recovery_data)
    }
}

/// BlockRetriever is used internally to retrieve blocks
pub struct BlockRetriever<T> {
    network: NetworkSender<T>,
    deadline: Instant,
    preferred_peer: Author,
}

impl<T: Payload> BlockRetriever<T> {
    pub fn new(network: NetworkSender<T>, deadline: Instant, preferred_peer: Author) -> Self {
        Self {
            network,
            deadline,
            preferred_peer,
        }
    }
    /// Retrieve chain of n blocks for given QC
    ///
    /// Returns Result with Vec that has a guaranteed size of num_blocks
    /// This guarantee is based on BlockRetrievalResponse::verify that ensures that number of
    /// blocks in response is equal to number of blocks requested.  This method will
    /// continue until either the round deadline is reached or the quorum certificate members all
    /// fail to return the missing chain.
    ///
    /// The first attempt of block retrieval will always be sent to preferred_peer to allow the
    /// leader to drive quorum certificate creation The other peers from the quorum certificate
    /// will be randomly tried next.  If all members of the quorum certificate are exhausted, an
    /// error is returned
    async fn retrieve_block_for_qc<'a>(
        &'a mut self,
        qc: &'a QuorumCert,
        num_blocks: u64,
    ) -> anyhow::Result<Vec<Block<T>>> {
        let block_id = qc.certified_block().id();
        let mut peers: Vec<&AccountAddress> = qc.ledger_info().signatures().keys().collect();
        let mut attempt = 0_u32;
        loop {
            if peers.is_empty() {
                bail!(
                    "Failed to fetch block {} in {} attempts: no more peers available",
                    block_id,
                    attempt
                );
            }
            let peer = self.pick_peer(attempt, &mut peers);
            attempt += 1;

            let timeout = retrieval_timeout(&self.deadline, attempt);
            let timeout = timeout.ok_or_else(|| {
                format_err!("Failed to fetch block {} from {}, attempt {}: round deadline was reached, won't make more attempts", block_id, peer, attempt)
            })?;
            debug!(
                "Fetching {} from {}, attempt {}",
                block_id,
                peer.short_str(),
                attempt
            );
            let response = self
                .network
                .request_block(
                    BlockRetrievalRequest::new(block_id, num_blocks),
                    peer,
                    timeout,
                )
                .await;
            let response = match response {
                Err(e) => {
                    warn!(
                        "Failed to fetch block {} from {}: {:?}, trying another peer",
                        block_id,
                        peer.short_str(),
                        e
                    );
                    continue;
                }
                Ok(response) => response,
            };
            if response.status() != BlockRetrievalStatus::Succeeded {
                warn!(
                    "Failed to fetch block {} from {}: {:?}, trying another peer",
                    block_id,
                    peer.short_str(),
                    response.status()
                );
                continue;
            }
            return Ok(response.blocks().clone());
        }
    }

    fn pick_peer(&self, attempt: u32, peers: &mut Vec<&AccountAddress>) -> AccountAddress {
        assert!(!peers.is_empty(), "pick_peer on empty peer list");

        if attempt == 0 {
            // remove preferred_peer if its in list of peers
            // (strictly speaking it is not required to be there)
            for i in 0..peers.len() {
                if *peers[i] == self.preferred_peer {
                    peers.remove(i);
                    break;
                }
            }
            return self.preferred_peer;
        }

        let peer_idx = thread_rng().gen_range(0, peers.len());
        *peers.remove(peer_idx)
    }
}

// Max timeout is 16s=RETRIEVAL_INITIAL_TIMEOUT*(2^RETRIEVAL_MAX_EXP)
const RETRIEVAL_INITIAL_TIMEOUT: Duration = Duration::from_secs(1);
const RETRIEVAL_MAX_EXP: u32 = 4;

/// Returns exponentially increasing timeout with
/// limit of RETRIEVAL_INITIAL_TIMEOUT*(2^RETRIEVAL_MAX_EXP)
#[allow(clippy::trivially_copy_pass_by_ref)]
fn retrieval_timeout(deadline: &Instant, attempt: u32) -> Option<Duration> {
    assert!(attempt > 0, "retrieval_timeout attempt can't be 0");
    let exp = RETRIEVAL_MAX_EXP.min(attempt - 1); // [0..RETRIEVAL_MAX_EXP]
    let request_timeout = RETRIEVAL_INITIAL_TIMEOUT * 2_u32.pow(exp);
    let now = Instant::now();
    let deadline_timeout = if *deadline >= now {
        Some(deadline.duration_since(now))
    } else {
        None
    };
    deadline_timeout.map(|delay| request_timeout.min(delay))
}
<commit_msg>[consensus] fix the leaking quorum cert in consensusdb<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    chained_bft::{
        block_storage::{BlockReader, BlockStore},
        network::NetworkSender,
        persistent_liveness_storage::{PersistentLivenessStorage, RecoveryData},
    },
    counters,
    state_replication::StateComputer,
};
use anyhow::{bail, format_err};
use consensus_types::{
    block::Block,
    block_retrieval::{BlockRetrievalRequest, BlockRetrievalStatus},
    common::{Author, Payload},
    quorum_cert::QuorumCert,
    sync_info::SyncInfo,
};
use libra_logger::prelude::*;
use libra_types::{account_address::AccountAddress, validator_change::ValidatorChangeProof};
use mirai_annotations::checked_precondition;
use rand::{prelude::*, Rng};
use std::{
    clone::Clone,
    sync::Arc,
    time::{Duration, Instant},
};
use termion::color::*;

#[derive(Debug, PartialEq)]
/// Whether we need to do block retrieval if we want to insert a Quorum Cert.
pub enum NeedFetchResult {
    QCRoundBeforeRoot,
    QCAlreadyExist,
    QCBlockExist,
    NeedFetch,
}

impl<T: Payload> BlockStore<T> {
    /// Check if we're far away from this ledger info and need to sync.
    /// Returns false if we have this block in the tree or the root's round is higher than the
    /// block.
    pub fn need_sync_for_quorum_cert(&self, qc: &QuorumCert) -> bool {
        // This precondition ensures that the check in the following lines
        // does not result in an addition overflow.
        checked_precondition!(self.root().round() < std::u64::MAX - 1);

        // If we have the block locally, we're not far from this QC thus don't need to sync.
        // In case root().round() is greater than that the committed
        // block carried by LI is older than my current commit.
        !(self.block_exists(qc.commit_info().id())
            || self.root().round() >= qc.commit_info().round())
    }

    /// Checks if quorum certificate can be inserted in block store without RPC
    /// Returns the enum to indicate the detailed status.
    pub fn need_fetch_for_quorum_cert(&self, qc: &QuorumCert) -> NeedFetchResult {
        if qc.certified_block().round() < self.root().round() {
            return NeedFetchResult::QCRoundBeforeRoot;
        }
        if self
            .get_quorum_cert_for_block(qc.certified_block().id())
            .is_some()
        {
            return NeedFetchResult::QCAlreadyExist;
        }
        if self.block_exists(qc.certified_block().id()) {
            return NeedFetchResult::QCBlockExist;
        }
        NeedFetchResult::NeedFetch
    }

    /// Fetches dependencies for given sync_info.quorum_cert
    /// If gap is large, performs state sync using process_highest_commit_cert
    /// Inserts sync_info.quorum_cert into block store as the last step
    pub async fn sync_to(
        &self,
        sync_info: &SyncInfo,
        mut retriever: BlockRetriever<T>,
    ) -> anyhow::Result<()> {
        self.process_highest_commit_cert(sync_info.highest_commit_cert().clone(), &mut retriever)
            .await?;

        match self.need_fetch_for_quorum_cert(sync_info.highest_quorum_cert()) {
            NeedFetchResult::NeedFetch => {
                self.fetch_quorum_cert(sync_info.highest_quorum_cert().clone(), retriever)
                    .await?
            }
            NeedFetchResult::QCBlockExist => {
                self.insert_single_quorum_cert(sync_info.highest_quorum_cert().clone())?
            }
            _ => (),
        }
        Ok(())
    }

    /// Insert the quorum certificate separately from the block, used to split the processing of
    /// updating the consensus state(with qc) and deciding whether to vote(with block)
    /// The missing ancestors are going to be retrieved from the given peer. If a given peer
    /// fails to provide the missing ancestors, the qc is not going to be added.
    async fn fetch_quorum_cert(
        &self,
        qc: QuorumCert,
        mut retriever: BlockRetriever<T>,
    ) -> anyhow::Result<()> {
        let mut pending = vec![];
        let mut retrieve_qc = qc.clone();
        loop {
            if self.block_exists(retrieve_qc.certified_block().id()) {
                break;
            }
            let mut blocks = retriever.retrieve_block_for_qc(&retrieve_qc, 1).await?;
            // retrieve_block_for_qc guarantees that blocks has exactly 1 element
            let block = blocks.remove(0);
            retrieve_qc = block.quorum_cert().clone();
            pending.push(block);
        }
        // insert the qc <- block pair
        while let Some(block) = pending.pop() {
            let block_qc = block.quorum_cert().clone();
            self.insert_single_quorum_cert(block_qc)?;
            self.execute_and_insert_block(block)?;
        }
        self.insert_single_quorum_cert(qc)
    }

    /// Check the highest commit cert sent by peer to see if we're behind and start a fast
    /// forward sync if the committed block doesn't exist in our tree.
    /// It works as follows:
    /// 1. request the committed 3-chain from the peer, if C2 is the highest_commit_cert
    /// we request for B0 <- C0 <- B1 <- C1 <- B2 (<- C2)
    /// 2. We persist the 3-chain to storage before start sync to ensure we could restart if we
    /// crash in the middle of the sync.
    /// 3. We prune the old tree and replace with a new tree built with the 3-chain.
    async fn process_highest_commit_cert(
        &self,
        highest_commit_cert: QuorumCert,
        retriever: &mut BlockRetriever<T>,
    ) -> anyhow::Result<()> {
        if !self.need_sync_for_quorum_cert(&highest_commit_cert) {
            return Ok(());
        }
        let (root, root_metadata, blocks, quorum_certs) = Self::fast_forward_sync(
            &highest_commit_cert,
            retriever,
            self.storage.clone(),
            self.state_computer.clone(),
        )
        .await?
        .take();
        debug!("{}Sync to{} {}", Fg(Blue), Fg(Reset), root.0);
        self.rebuild(root, root_metadata, blocks, quorum_certs)
            .await;

        if highest_commit_cert.ends_epoch() {
            retriever
                .network
                .notify_epoch_change(ValidatorChangeProof::new(
                    vec![highest_commit_cert.ledger_info().clone()],
                    /* more = */ false,
                ))
                .await;
        }
        Ok(())
    }

    pub async fn fast_forward_sync<'a>(
        highest_commit_cert: &'a QuorumCert,
        retriever: &'a mut BlockRetriever<T>,
        storage: Arc<dyn PersistentLivenessStorage<T>>,
        state_computer: Arc<dyn StateComputer<Payload = T>>,
    ) -> anyhow::Result<RecoveryData<T>> {
        debug!(
            "Start state sync with peer: {}, to block: {}",
            retriever.preferred_peer.short_str(),
            highest_commit_cert.commit_info(),
        );

        let blocks = retriever
            .retrieve_block_for_qc(&highest_commit_cert, 3)
            .await?;
        assert_eq!(
            blocks.last().expect("should have 3-chain").id(),
            highest_commit_cert.commit_info().id(),
        );
        let mut quorum_certs = vec![];
        quorum_certs.push(highest_commit_cert.clone());
        quorum_certs.extend(
            blocks
                .iter()
                .take(2)
                .map(|block| block.quorum_cert().clone()),
        );
        for (i, block) in blocks.iter().enumerate() {
            assert_eq!(block.id(), quorum_certs[i].certified_block().id());
        }

        // If a node restarts in the middle of state synchronization, it is going to try to catch up
        // to the stored quorum certs as the new root.
        storage.save_tree(blocks.clone(), quorum_certs.clone())?;
        let pre_sync_instance = Instant::now();
        state_computer
            .sync_to(highest_commit_cert.ledger_info().clone())
            .await?;
        counters::STATE_SYNC_DURATION_S.observe_duration(pre_sync_instance.elapsed());
        let recovery_data = storage
            .start()
            .expect_recovery_data("Failed to construct recovery data after fast forward sync");

        Ok(recovery_data)
    }
}

/// BlockRetriever is used internally to retrieve blocks
pub struct BlockRetriever<T> {
    network: NetworkSender<T>,
    deadline: Instant,
    preferred_peer: Author,
}

impl<T: Payload> BlockRetriever<T> {
    pub fn new(network: NetworkSender<T>, deadline: Instant, preferred_peer: Author) -> Self {
        Self {
            network,
            deadline,
            preferred_peer,
        }
    }
    /// Retrieve chain of n blocks for given QC
    ///
    /// Returns Result with Vec that has a guaranteed size of num_blocks
    /// This guarantee is based on BlockRetrievalResponse::verify that ensures that number of
    /// blocks in response is equal to number of blocks requested.  This method will
    /// continue until either the round deadline is reached or the quorum certificate members all
    /// fail to return the missing chain.
    ///
    /// The first attempt of block retrieval will always be sent to preferred_peer to allow the
    /// leader to drive quorum certificate creation The other peers from the quorum certificate
    /// will be randomly tried next.  If all members of the quorum certificate are exhausted, an
    /// error is returned
    async fn retrieve_block_for_qc<'a>(
        &'a mut self,
        qc: &'a QuorumCert,
        num_blocks: u64,
    ) -> anyhow::Result<Vec<Block<T>>> {
        let block_id = qc.certified_block().id();
        let mut peers: Vec<&AccountAddress> = qc.ledger_info().signatures().keys().collect();
        let mut attempt = 0_u32;
        loop {
            if peers.is_empty() {
                bail!(
                    "Failed to fetch block {} in {} attempts: no more peers available",
                    block_id,
                    attempt
                );
            }
            let peer = self.pick_peer(attempt, &mut peers);
            attempt += 1;

            let timeout = retrieval_timeout(&self.deadline, attempt);
            let timeout = timeout.ok_or_else(|| {
                format_err!("Failed to fetch block {} from {}, attempt {}: round deadline was reached, won't make more attempts", block_id, peer, attempt)
            })?;
            debug!(
                "Fetching {} from {}, attempt {}",
                block_id,
                peer.short_str(),
                attempt
            );
            let response = self
                .network
                .request_block(
                    BlockRetrievalRequest::new(block_id, num_blocks),
                    peer,
                    timeout,
                )
                .await;
            let response = match response {
                Err(e) => {
                    warn!(
                        "Failed to fetch block {} from {}: {:?}, trying another peer",
                        block_id,
                        peer.short_str(),
                        e
                    );
                    continue;
                }
                Ok(response) => response,
            };
            if response.status() != BlockRetrievalStatus::Succeeded {
                warn!(
                    "Failed to fetch block {} from {}: {:?}, trying another peer",
                    block_id,
                    peer.short_str(),
                    response.status()
                );
                continue;
            }
            return Ok(response.blocks().clone());
        }
    }

    fn pick_peer(&self, attempt: u32, peers: &mut Vec<&AccountAddress>) -> AccountAddress {
        assert!(!peers.is_empty(), "pick_peer on empty peer list");

        if attempt == 0 {
            // remove preferred_peer if its in list of peers
            // (strictly speaking it is not required to be there)
            for i in 0..peers.len() {
                if *peers[i] == self.preferred_peer {
                    peers.remove(i);
                    break;
                }
            }
            return self.preferred_peer;
        }

        let peer_idx = thread_rng().gen_range(0, peers.len());
        *peers.remove(peer_idx)
    }
}

// Max timeout is 16s=RETRIEVAL_INITIAL_TIMEOUT*(2^RETRIEVAL_MAX_EXP)
const RETRIEVAL_INITIAL_TIMEOUT: Duration = Duration::from_secs(1);
const RETRIEVAL_MAX_EXP: u32 = 4;

/// Returns exponentially increasing timeout with
/// limit of RETRIEVAL_INITIAL_TIMEOUT*(2^RETRIEVAL_MAX_EXP)
#[allow(clippy::trivially_copy_pass_by_ref)]
fn retrieval_timeout(deadline: &Instant, attempt: u32) -> Option<Duration> {
    assert!(attempt > 0, "retrieval_timeout attempt can't be 0");
    let exp = RETRIEVAL_MAX_EXP.min(attempt - 1); // [0..RETRIEVAL_MAX_EXP]
    let request_timeout = RETRIEVAL_INITIAL_TIMEOUT * 2_u32.pow(exp);
    let now = Instant::now();
    let deadline_timeout = if *deadline >= now {
        Some(deadline.duration_since(now))
    } else {
        None
    };
    deadline_timeout.map(|delay| request_timeout.min(delay))
}
<|endoftext|>