<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

use crate::geom::flow_relative::{Rect, Sides};
use crate::geom::{PhysicalPoint, PhysicalRect};
#[cfg(debug_assertions)]
use crate::layout_debug;
use crate::positioned::HoistedFragmentId;
use gfx::text::glyph::GlyphStore;
use gfx_traits::print_tree::PrintTree;
#[cfg(not(debug_assertions))]
use serde::ser::{Serialize, Serializer};
use servo_arc::Arc as ServoArc;
use std::sync::Arc;
use style::computed_values::overflow_x::T as ComputedOverflow;
use style::dom::OpaqueNode;
use style::logical_geometry::WritingMode;
use style::properties::ComputedValues;
use style::values::computed::Length;
use style::Zero;
use webrender_api::{FontInstanceKey, ImageKey};

#[derive(Serialize)]
pub(crate) enum Fragment {
    Box(BoxFragment),
    Anonymous(AnonymousFragment),
    AbsoluteOrFixedPositioned(AbsoluteOrFixedPositionedFragment),
    Text(TextFragment),
    Image(ImageFragment),
}

#[derive(Serialize)]
pub(crate) struct AbsoluteOrFixedPositionedFragment(pub HoistedFragmentId);

#[derive(Serialize)]
pub(crate) struct BoxFragment {
    pub tag: OpaqueNode,
    pub debug_id: DebugId,
    #[serde(skip_serializing)]
    pub style: ServoArc<ComputedValues>,
    pub children: Vec<Fragment>,

    /// From the containing block’s start corner…?
    /// This might be broken when the containing block is in a different writing mode:
    /// https://drafts.csswg.org/css-writing-modes/#orthogonal-flows
    pub content_rect: Rect<Length>,

    pub padding: Sides<Length>,
    pub border: Sides<Length>,
    pub margin: Sides<Length>,

    pub block_margins_collapsed_with_children: CollapsedBlockMargins,

    /// The scrollable overflow of this box fragment.
    pub scrollable_overflow_from_children: PhysicalRect<Length>,

    /// XXX Add thsi
    pub hoisted_fragment_id: Option<HoistedFragmentId>,
}

#[derive(Serialize)]
pub(crate) struct CollapsedBlockMargins {
    pub collapsed_through: bool,
    pub start: CollapsedMargin,
    pub end: CollapsedMargin,
}

#[derive(Clone, Copy, Serialize)]
pub(crate) struct CollapsedMargin {
    max_positive: Length,
    min_negative: Length,
}

/// Can contain child fragments with relative coordinates, but does not contribute to painting itself.
#[derive(Serialize)]
pub(crate) struct AnonymousFragment {
    pub debug_id: DebugId,
    pub rect: Rect<Length>,
    pub children: Vec<Fragment>,
    pub mode: WritingMode,

    /// The scrollable overflow of this anonymous fragment's children.
    pub scrollable_overflow: PhysicalRect<Length>,
}

#[derive(Serialize)]
pub(crate) struct TextFragment {
    pub debug_id: DebugId,
    pub tag: OpaqueNode,
    #[serde(skip_serializing)]
    pub parent_style: ServoArc<ComputedValues>,
    pub rect: Rect<Length>,
    pub ascent: Length,
    #[serde(skip_serializing)]
    pub font_key: FontInstanceKey,
    pub glyphs: Vec<Arc<GlyphStore>>,
}

#[derive(Serialize)]
pub(crate) struct ImageFragment {
    pub debug_id: DebugId,
    #[serde(skip_serializing)]
    pub style: ServoArc<ComputedValues>,
    pub rect: Rect<Length>,
    #[serde(skip_serializing)]
    pub image_key: ImageKey,
}

impl Fragment {
    pub fn offset_inline(&mut self, offset: &Length) {
        let position = match self {
            Fragment::Box(f) => &mut f.content_rect.start_corner,
            Fragment::AbsoluteOrFixedPositioned(_) => return,
            Fragment::Anonymous(f) => &mut f.rect.start_corner,
            Fragment::Text(f) => &mut f.rect.start_corner,
            Fragment::Image(f) => &mut f.rect.start_corner,
        };

        position.inline += *offset;
    }

    pub fn print(&self, tree: &mut PrintTree) {
        match self {
            Fragment::Box(fragment) => fragment.print(tree),
            Fragment::AbsoluteOrFixedPositioned(fragment) => fragment.print(tree),
            Fragment::Anonymous(fragment) => fragment.print(tree),
            Fragment::Text(fragment) => fragment.print(tree),
            Fragment::Image(fragment) => fragment.print(tree),
        }
    }

    pub fn scrollable_overflow(&self) -> PhysicalRect<Length> {
        // FIXME(mrobinson, bug 25564): We should be using the containing block
        // here to properly convert scrollable overflow to physical geometry.
        let containing_block = PhysicalRect::zero();
        match self {
            Fragment::Box(fragment) => fragment.scrollable_overflow_for_parent(&containing_block),
            Fragment::AbsoluteOrFixedPositioned(_) => PhysicalRect::zero(),
            Fragment::Anonymous(fragment) => fragment.scrollable_overflow.clone(),
            Fragment::Text(fragment) => fragment
                .rect
                .to_physical(fragment.parent_style.writing_mode, &containing_block),
            Fragment::Image(fragment) => fragment
                .rect
                .to_physical(fragment.style.writing_mode, &containing_block),
        }
    }

    pub fn is_hoisted(&self) -> bool {
        match self {
            Fragment::Box(fragment) if fragment.hoisted_fragment_id.is_some() => true,
            _ => false,
        }
    }

    pub fn hoisted_fragment_id(&self) -> Option<&HoistedFragmentId> {
        match self {
            Fragment::Box(fragment) => fragment.hoisted_fragment_id.as_ref(),
            _ => None,
        }
    }
}

impl AbsoluteOrFixedPositionedFragment {
    pub fn print(&self, tree: &mut PrintTree) {
        tree.add_item(format!("AbsoluteOrFixedPositionedFragment({:?})", self.0));
    }
}

impl AnonymousFragment {
    pub fn no_op(mode: WritingMode) -> Self {
        Self {
            debug_id: DebugId::new(),
            children: vec![],
            rect: Rect::zero(),
            mode,
            scrollable_overflow: PhysicalRect::zero(),
        }
    }

    pub fn new(rect: Rect<Length>, children: Vec<Fragment>, mode: WritingMode) -> Self {
        let content_origin = rect.start_corner.to_physical(mode);
        let scrollable_overflow = children.iter().fold(PhysicalRect::zero(), |acc, child| {
            acc.union(
                &child
                    .scrollable_overflow()
                    .translate(content_origin.to_vector()),
            )
        });
        AnonymousFragment {
            debug_id: DebugId::new(),
            rect,
            children,
            mode,
            scrollable_overflow,
        }
    }

    pub fn print(&self, tree: &mut PrintTree) {
        tree.new_level(format!(
            "Anonymous\
                \nrect={:?}\
                \nscrollable_overflow={:?}",
            self.rect, self.scrollable_overflow
        ));

        for child in &self.children {
            child.print(tree);
        }
        tree.end_level();
    }
}

impl BoxFragment {
    pub fn new(
        tag: OpaqueNode,
        style: ServoArc<ComputedValues>,
        children: Vec<Fragment>,
        content_rect: Rect<Length>,
        padding: Sides<Length>,
        border: Sides<Length>,
        margin: Sides<Length>,
        block_margins_collapsed_with_children: CollapsedBlockMargins,
        hoisted_fragment_id: Option<HoistedFragmentId>,
    ) -> BoxFragment {
        let scrollable_overflow_from_children =
            children.iter().fold(PhysicalRect::zero(), |acc, child| {
                acc.union(&child.scrollable_overflow())
            });
        BoxFragment {
            tag,
            debug_id: DebugId::new(),
            style,
            children,
            content_rect,
            padding,
            border,
            margin,
            block_margins_collapsed_with_children,
            scrollable_overflow_from_children,
            hoisted_fragment_id,
        }
    }

    pub fn scrollable_overflow(&self) -> PhysicalRect<Length> {
        // FIXME(mrobinson, bug 25564): We should be using the containing block
        // here to properly convert scrollable overflow to physical geometry.
        let physical_padding_rect = self
            .padding_rect()
            .to_physical(self.style.writing_mode, &PhysicalRect::zero());

        let content_origin = self
            .content_rect
            .start_corner
            .to_physical(self.style.writing_mode);
        physical_padding_rect.union(
            &self
                .scrollable_overflow_from_children
                .translate(content_origin.to_vector()),
        )
    }

    pub fn padding_rect(&self) -> Rect<Length> {
        self.content_rect.inflate(&self.padding)
    }

    pub fn border_rect(&self) -> Rect<Length> {
        self.padding_rect().inflate(&self.border)
    }

    pub fn print(&self, tree: &mut PrintTree) {
        tree.new_level(format!(
            "Box\
                \ncontent={:?}\
                \npadding rect={:?}\
                \nborder rect={:?}\
                \nscrollable_overflow={:?}\
                \noverflow={:?} / {:?}\
                \nstyle={:p}\
                \nhoisted_id={:?}",
            self.content_rect,
            self.padding_rect(),
            self.border_rect(),
            self.scrollable_overflow(),
            self.style.get_box().overflow_x,
            self.style.get_box().overflow_y,
            self.style,
            self.hoisted_fragment_id,
        ));

        for child in &self.children {
            child.print(tree);
        }
        tree.end_level();
    }

    pub fn scrollable_overflow_for_parent(
        &self,
        containing_block: &PhysicalRect<Length>,
    ) -> PhysicalRect<Length> {
        let mut overflow = self
            .border_rect()
            .to_physical(self.style.writing_mode, containing_block);

        if self.style.get_box().overflow_y != ComputedOverflow::Visible &&
            self.style.get_box().overflow_x != ComputedOverflow::Visible
        {
            return overflow;
        }

        // https://www.w3.org/TR/css-overflow-3/#scrollable
        // Only include the scrollable overflow of a child box if it has overflow: visible.
        let scrollable_overflow = self.scrollable_overflow();
        let bottom_right = PhysicalPoint::new(
            overflow.max_x().max(scrollable_overflow.max_x()),
            overflow.max_y().max(scrollable_overflow.max_y()),
        );

        if self.style.get_box().overflow_y == ComputedOverflow::Visible {
            overflow.origin.y = overflow.origin.y.min(scrollable_overflow.origin.y);
            overflow.size.height = bottom_right.y - overflow.origin.y;
        }

        if self.style.get_box().overflow_x == ComputedOverflow::Visible {
            overflow.origin.x = overflow.origin.x.min(scrollable_overflow.origin.x);
            overflow.size.width = bottom_right.x - overflow.origin.x;
        }

        overflow
    }
}

impl TextFragment {
    pub fn print(&self, tree: &mut PrintTree) {
        tree.add_item(format!(
            "Text num_glyphs={}",
            self.glyphs
                .iter()
                .map(|glyph_store| glyph_store.len().0)
                .sum::<isize>()
        ));
    }
}

impl ImageFragment {
    pub fn print(&self, tree: &mut PrintTree) {
        tree.add_item(format!(
            "Image\
                \nrect={:?}",
            self.rect
        ));
    }
}

impl CollapsedBlockMargins {
    pub fn from_margin(margin: &Sides<Length>) -> Self {
        Self {
            collapsed_through: false,
            start: CollapsedMargin::new(margin.block_start),
            end: CollapsedMargin::new(margin.block_end),
        }
    }

    pub fn zero() -> Self {
        Self {
            collapsed_through: false,
            start: CollapsedMargin::zero(),
            end: CollapsedMargin::zero(),
        }
    }
}

impl CollapsedMargin {
    pub fn zero() -> Self {
        Self {
            max_positive: Length::zero(),
            min_negative: Length::zero(),
        }
    }

    pub fn new(margin: Length) -> Self {
        Self {
            max_positive: margin.max(Length::zero()),
            min_negative: margin.min(Length::zero()),
        }
    }

    pub fn adjoin(&self, other: &Self) -> Self {
        Self {
            max_positive: self.max_positive.max(other.max_positive),
            min_negative: self.min_negative.min(other.min_negative),
        }
    }

    pub fn adjoin_assign(&mut self, other: &Self) {
        *self = self.adjoin(other);
    }

    pub fn solve(&self) -> Length {
        self.max_positive + self.min_negative
    }
}

#[cfg(not(debug_assertions))]
#[derive(Clone)]
pub struct DebugId;

#[cfg(debug_assertions)]
#[derive(Clone, Serialize)]
#[serde(transparent)]
pub struct DebugId(u16);

#[cfg(not(debug_assertions))]
impl DebugId {
    pub fn new() -> DebugId {
        DebugId
    }
}

#[cfg(debug_assertions)]
impl DebugId {
    pub fn new() -> DebugId {
        DebugId(layout_debug::generate_unique_debug_id())
    }
}

#[cfg(not(debug_assertions))]
impl Serialize for DebugId {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        serializer.serialize_str(&format!("{:p}", &self))
    }
}
<commit_msg>layout_2020: Fix a documentation comment<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

use crate::geom::flow_relative::{Rect, Sides};
use crate::geom::{PhysicalPoint, PhysicalRect};
#[cfg(debug_assertions)]
use crate::layout_debug;
use crate::positioned::HoistedFragmentId;
use gfx::text::glyph::GlyphStore;
use gfx_traits::print_tree::PrintTree;
#[cfg(not(debug_assertions))]
use serde::ser::{Serialize, Serializer};
use servo_arc::Arc as ServoArc;
use std::sync::Arc;
use style::computed_values::overflow_x::T as ComputedOverflow;
use style::dom::OpaqueNode;
use style::logical_geometry::WritingMode;
use style::properties::ComputedValues;
use style::values::computed::Length;
use style::Zero;
use webrender_api::{FontInstanceKey, ImageKey};

#[derive(Serialize)]
pub(crate) enum Fragment {
    Box(BoxFragment),
    Anonymous(AnonymousFragment),
    AbsoluteOrFixedPositioned(AbsoluteOrFixedPositionedFragment),
    Text(TextFragment),
    Image(ImageFragment),
}

#[derive(Serialize)]
pub(crate) struct AbsoluteOrFixedPositionedFragment(pub HoistedFragmentId);

#[derive(Serialize)]
pub(crate) struct BoxFragment {
    pub tag: OpaqueNode,
    pub debug_id: DebugId,
    #[serde(skip_serializing)]
    pub style: ServoArc<ComputedValues>,
    pub children: Vec<Fragment>,

    /// From the containing block’s start corner…?
    /// This might be broken when the containing block is in a different writing mode:
    /// https://drafts.csswg.org/css-writing-modes/#orthogonal-flows
    pub content_rect: Rect<Length>,

    pub padding: Sides<Length>,
    pub border: Sides<Length>,
    pub margin: Sides<Length>,

    pub block_margins_collapsed_with_children: CollapsedBlockMargins,

    /// The scrollable overflow of this box fragment.
    pub scrollable_overflow_from_children: PhysicalRect<Length>,

    /// If this fragment was laid out from a hoisted box, this id corresponds to the id stored in
    /// the AbsoluteOrFixedPositionedFragment left as a placeholder in the tree position of the
    /// box.
    pub hoisted_fragment_id: Option<HoistedFragmentId>,
}

#[derive(Serialize)]
pub(crate) struct CollapsedBlockMargins {
    pub collapsed_through: bool,
    pub start: CollapsedMargin,
    pub end: CollapsedMargin,
}

#[derive(Clone, Copy, Serialize)]
pub(crate) struct CollapsedMargin {
    max_positive: Length,
    min_negative: Length,
}

/// Can contain child fragments with relative coordinates, but does not contribute to painting itself.
#[derive(Serialize)]
pub(crate) struct AnonymousFragment {
    pub debug_id: DebugId,
    pub rect: Rect<Length>,
    pub children: Vec<Fragment>,
    pub mode: WritingMode,

    /// The scrollable overflow of this anonymous fragment's children.
    pub scrollable_overflow: PhysicalRect<Length>,
}

#[derive(Serialize)]
pub(crate) struct TextFragment {
    pub debug_id: DebugId,
    pub tag: OpaqueNode,
    #[serde(skip_serializing)]
    pub parent_style: ServoArc<ComputedValues>,
    pub rect: Rect<Length>,
    pub ascent: Length,
    #[serde(skip_serializing)]
    pub font_key: FontInstanceKey,
    pub glyphs: Vec<Arc<GlyphStore>>,
}

#[derive(Serialize)]
pub(crate) struct ImageFragment {
    pub debug_id: DebugId,
    #[serde(skip_serializing)]
    pub style: ServoArc<ComputedValues>,
    pub rect: Rect<Length>,
    #[serde(skip_serializing)]
    pub image_key: ImageKey,
}

impl Fragment {
    pub fn offset_inline(&mut self, offset: &Length) {
        let position = match self {
            Fragment::Box(f) => &mut f.content_rect.start_corner,
            Fragment::AbsoluteOrFixedPositioned(_) => return,
            Fragment::Anonymous(f) => &mut f.rect.start_corner,
            Fragment::Text(f) => &mut f.rect.start_corner,
            Fragment::Image(f) => &mut f.rect.start_corner,
        };

        position.inline += *offset;
    }

    pub fn print(&self, tree: &mut PrintTree) {
        match self {
            Fragment::Box(fragment) => fragment.print(tree),
            Fragment::AbsoluteOrFixedPositioned(fragment) => fragment.print(tree),
            Fragment::Anonymous(fragment) => fragment.print(tree),
            Fragment::Text(fragment) => fragment.print(tree),
            Fragment::Image(fragment) => fragment.print(tree),
        }
    }

    pub fn scrollable_overflow(&self) -> PhysicalRect<Length> {
        // FIXME(mrobinson, bug 25564): We should be using the containing block
        // here to properly convert scrollable overflow to physical geometry.
        let containing_block = PhysicalRect::zero();
        match self {
            Fragment::Box(fragment) => fragment.scrollable_overflow_for_parent(&containing_block),
            Fragment::AbsoluteOrFixedPositioned(_) => PhysicalRect::zero(),
            Fragment::Anonymous(fragment) => fragment.scrollable_overflow.clone(),
            Fragment::Text(fragment) => fragment
                .rect
                .to_physical(fragment.parent_style.writing_mode, &containing_block),
            Fragment::Image(fragment) => fragment
                .rect
                .to_physical(fragment.style.writing_mode, &containing_block),
        }
    }

    pub fn is_hoisted(&self) -> bool {
        match self {
            Fragment::Box(fragment) if fragment.hoisted_fragment_id.is_some() => true,
            _ => false,
        }
    }

    pub fn hoisted_fragment_id(&self) -> Option<&HoistedFragmentId> {
        match self {
            Fragment::Box(fragment) => fragment.hoisted_fragment_id.as_ref(),
            _ => None,
        }
    }
}

impl AbsoluteOrFixedPositionedFragment {
    pub fn print(&self, tree: &mut PrintTree) {
        tree.add_item(format!("AbsoluteOrFixedPositionedFragment({:?})", self.0));
    }
}

impl AnonymousFragment {
    pub fn no_op(mode: WritingMode) -> Self {
        Self {
            debug_id: DebugId::new(),
            children: vec![],
            rect: Rect::zero(),
            mode,
            scrollable_overflow: PhysicalRect::zero(),
        }
    }

    pub fn new(rect: Rect<Length>, children: Vec<Fragment>, mode: WritingMode) -> Self {
        let content_origin = rect.start_corner.to_physical(mode);
        let scrollable_overflow = children.iter().fold(PhysicalRect::zero(), |acc, child| {
            acc.union(
                &child
                    .scrollable_overflow()
                    .translate(content_origin.to_vector()),
            )
        });
        AnonymousFragment {
            debug_id: DebugId::new(),
            rect,
            children,
            mode,
            scrollable_overflow,
        }
    }

    pub fn print(&self, tree: &mut PrintTree) {
        tree.new_level(format!(
            "Anonymous\
                \nrect={:?}\
                \nscrollable_overflow={:?}",
            self.rect, self.scrollable_overflow
        ));

        for child in &self.children {
            child.print(tree);
        }
        tree.end_level();
    }
}

impl BoxFragment {
    pub fn new(
        tag: OpaqueNode,
        style: ServoArc<ComputedValues>,
        children: Vec<Fragment>,
        content_rect: Rect<Length>,
        padding: Sides<Length>,
        border: Sides<Length>,
        margin: Sides<Length>,
        block_margins_collapsed_with_children: CollapsedBlockMargins,
        hoisted_fragment_id: Option<HoistedFragmentId>,
    ) -> BoxFragment {
        let scrollable_overflow_from_children =
            children.iter().fold(PhysicalRect::zero(), |acc, child| {
                acc.union(&child.scrollable_overflow())
            });
        BoxFragment {
            tag,
            debug_id: DebugId::new(),
            style,
            children,
            content_rect,
            padding,
            border,
            margin,
            block_margins_collapsed_with_children,
            scrollable_overflow_from_children,
            hoisted_fragment_id,
        }
    }

    pub fn scrollable_overflow(&self) -> PhysicalRect<Length> {
        // FIXME(mrobinson, bug 25564): We should be using the containing block
        // here to properly convert scrollable overflow to physical geometry.
        let physical_padding_rect = self
            .padding_rect()
            .to_physical(self.style.writing_mode, &PhysicalRect::zero());

        let content_origin = self
            .content_rect
            .start_corner
            .to_physical(self.style.writing_mode);
        physical_padding_rect.union(
            &self
                .scrollable_overflow_from_children
                .translate(content_origin.to_vector()),
        )
    }

    pub fn padding_rect(&self) -> Rect<Length> {
        self.content_rect.inflate(&self.padding)
    }

    pub fn border_rect(&self) -> Rect<Length> {
        self.padding_rect().inflate(&self.border)
    }

    pub fn print(&self, tree: &mut PrintTree) {
        tree.new_level(format!(
            "Box\
                \ncontent={:?}\
                \npadding rect={:?}\
                \nborder rect={:?}\
                \nscrollable_overflow={:?}\
                \noverflow={:?} / {:?}\
                \nstyle={:p}\
                \nhoisted_id={:?}",
            self.content_rect,
            self.padding_rect(),
            self.border_rect(),
            self.scrollable_overflow(),
            self.style.get_box().overflow_x,
            self.style.get_box().overflow_y,
            self.style,
            self.hoisted_fragment_id,
        ));

        for child in &self.children {
            child.print(tree);
        }
        tree.end_level();
    }

    pub fn scrollable_overflow_for_parent(
        &self,
        containing_block: &PhysicalRect<Length>,
    ) -> PhysicalRect<Length> {
        let mut overflow = self
            .border_rect()
            .to_physical(self.style.writing_mode, containing_block);

        if self.style.get_box().overflow_y != ComputedOverflow::Visible &&
            self.style.get_box().overflow_x != ComputedOverflow::Visible
        {
            return overflow;
        }

        // https://www.w3.org/TR/css-overflow-3/#scrollable
        // Only include the scrollable overflow of a child box if it has overflow: visible.
        let scrollable_overflow = self.scrollable_overflow();
        let bottom_right = PhysicalPoint::new(
            overflow.max_x().max(scrollable_overflow.max_x()),
            overflow.max_y().max(scrollable_overflow.max_y()),
        );

        if self.style.get_box().overflow_y == ComputedOverflow::Visible {
            overflow.origin.y = overflow.origin.y.min(scrollable_overflow.origin.y);
            overflow.size.height = bottom_right.y - overflow.origin.y;
        }

        if self.style.get_box().overflow_x == ComputedOverflow::Visible {
            overflow.origin.x = overflow.origin.x.min(scrollable_overflow.origin.x);
            overflow.size.width = bottom_right.x - overflow.origin.x;
        }

        overflow
    }
}

impl TextFragment {
    pub fn print(&self, tree: &mut PrintTree) {
        tree.add_item(format!(
            "Text num_glyphs={}",
            self.glyphs
                .iter()
                .map(|glyph_store| glyph_store.len().0)
                .sum::<isize>()
        ));
    }
}

impl ImageFragment {
    pub fn print(&self, tree: &mut PrintTree) {
        tree.add_item(format!(
            "Image\
                \nrect={:?}",
            self.rect
        ));
    }
}

impl CollapsedBlockMargins {
    pub fn from_margin(margin: &Sides<Length>) -> Self {
        Self {
            collapsed_through: false,
            start: CollapsedMargin::new(margin.block_start),
            end: CollapsedMargin::new(margin.block_end),
        }
    }

    pub fn zero() -> Self {
        Self {
            collapsed_through: false,
            start: CollapsedMargin::zero(),
            end: CollapsedMargin::zero(),
        }
    }
}

impl CollapsedMargin {
    pub fn zero() -> Self {
        Self {
            max_positive: Length::zero(),
            min_negative: Length::zero(),
        }
    }

    pub fn new(margin: Length) -> Self {
        Self {
            max_positive: margin.max(Length::zero()),
            min_negative: margin.min(Length::zero()),
        }
    }

    pub fn adjoin(&self, other: &Self) -> Self {
        Self {
            max_positive: self.max_positive.max(other.max_positive),
            min_negative: self.min_negative.min(other.min_negative),
        }
    }

    pub fn adjoin_assign(&mut self, other: &Self) {
        *self = self.adjoin(other);
    }

    pub fn solve(&self) -> Length {
        self.max_positive + self.min_negative
    }
}

#[cfg(not(debug_assertions))]
#[derive(Clone)]
pub struct DebugId;

#[cfg(debug_assertions)]
#[derive(Clone, Serialize)]
#[serde(transparent)]
pub struct DebugId(u16);

#[cfg(not(debug_assertions))]
impl DebugId {
    pub fn new() -> DebugId {
        DebugId
    }
}

#[cfg(debug_assertions)]
impl DebugId {
    pub fn new() -> DebugId {
        DebugId(layout_debug::generate_unique_debug_id())
    }
}

#[cfg(not(debug_assertions))]
impl Serialize for DebugId {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        serializer.serialize_str(&format!("{:p}", &self))
    }
}
<|endoftext|>


<commit_before>
use super::*;
use types::*;
use core::fmt;

define_path_attr!(MpReachNlri, doc="Multi Protocol Network Layer Reachability Information");

impl<'a> MpReachNlri<'a> {

    pub fn from_bytes(bytes: &'a [u8]) -> MpReachNlri {
        MpReachNlri {
            inner: bytes,
        }
    }

    fn afi(&self) -> Afi {
        Afi::from((self.value()[0] as u16) << 8 | self.value()[1] as u16)
    }

    fn safi(&self) -> Safi {
        Safi::from(self.value()[2])
    }

    fn nexthop_len(&self) -> usize {
        self.value()[3] as usize
    }

    pub fn nexthop(&self) -> MpNextHop<'a> {
        MpNextHop {
            inner: &self.value()[4..self.nexthop_len()],
            afi: self.afi(),
        }
    }

    pub fn nlris(&self) -> MpNlriIter<'a> {
        let offset = 2 + 1 + 1 + self.nexthop_len() + 1;
        match (self.afi(), self.safi()) {
            (AFI_IPV4, SAFI_UNICAST) =>
                MpNlriIter::Ipv4Unicast(Ipv4UnicastNlriIter{inner: &self.value()[offset..], error: false}),
            (AFI_IPV4, SAFI_MULTICAST) =>
                MpNlriIter::Ipv4Multicast(Ipv4MulticastNlriIter{inner: &self.value()[offset..], error: false}),
            (AFI_IPV6, SAFI_UNICAST) =>
                MpNlriIter::Ipv6Unicast(Ipv6UnicastNlriIter{inner: &self.value()[offset..], error: false}),
            (AFI_IPV6, SAFI_MULTICAST) =>
                MpNlriIter::Ipv6Multicast(Ipv6MulticastNlriIter{inner: &self.value()[offset..], error: false}),
            _ => MpNlriIter::Other(OtherNlriIter{inner: &self.value()[offset..], error: false}),
        }
    }
}

impl<'a> fmt::Debug for MpReachNlri<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("MpReachNlri")
            .field("nexthop", &self.nexthop())
            .field("nlris", &self.nlris())
            // .field("inner", &self.inner)
            .finish()
    }
}

#[derive(Debug)]
pub enum MpNlriIter<'a> {
    Ipv4Unicast(Ipv4UnicastNlriIter<'a>),
    Ipv6Unicast(Ipv6UnicastNlriIter<'a>),
    Ipv4Multicast(Ipv4MulticastNlriIter<'a>),
    Ipv6Multicast(Ipv6MulticastNlriIter<'a>),
    Other(OtherNlriIter<'a>)
}

#[derive(Debug)]
pub struct MpNextHop<'a> {
    inner: &'a [u8],
    afi: Afi,
}

macro_rules! impl_mp_nlri {
    ($item:ident, $iter:ident) => {

        #[derive(Clone)]
        pub struct $item<'a> {
            inner: &'a [u8],
        }

        impl<'a> fmt::Debug for $item<'a> {
            fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
                self.prefix().fmt(fmt)
            }
        }

        impl<'a> $item<'a> {
            pub fn prefix(&self) -> Prefix<'a> {
                Prefix{inner: self.inner}
            }
        }

        #[derive(Clone)]
        pub struct $iter<'a> {
            inner: &'a [u8],
            error: bool,
        }

        impl<'a> Iterator for $iter<'a> {
            type Item = Result<$item<'a>>;

            fn next(&mut self) -> Option<Result<$item<'a>>> {
                if self.error || self.inner.is_empty() {
                    return None;
                }

                let mask_len = self.inner[0] as usize;
                assert!(mask_len <= 128);
                let byte_len = (mask_len+15) / 8;
                if self.inner.len() < byte_len {
                    self.error = true;
                    return Some(Err(BgpError::BadLength));
                }
                let slice = &self.inner[..byte_len];
                let nlri = $item{inner: slice};
                self.inner = &self.inner[byte_len..];
                Some(Ok(nlri))
            }
        }

        impl<'a> fmt::Debug for $iter<'a> {
            fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
                fmt.debug_list().entries(self.clone()).finish()
            }
        }

    }
}

impl_mp_nlri!(Ipv4UnicastNlri, Ipv4UnicastNlriIter);
impl_mp_nlri!(Ipv6UnicastNlri, Ipv6UnicastNlriIter);
impl_mp_nlri!(Ipv4MulticastNlri, Ipv4MulticastNlriIter);
impl_mp_nlri!(Ipv6MulticastNlri, Ipv6MulticastNlriIter);
impl_mp_nlri!(OtherNlri, OtherNlriIter);

#[cfg(test)]
mod test {

    use super::*;
    use types::*;
    #[test]
    fn parse_mp_nlri_multicast() {
	      // path_attrs: [Origin(Igp),
        //              AsPath(Ok([3549, 3356, 137, 137, 137, 8978])),
        //              MultiExitDisc(13814),
        //              Communities(Ok([3549:2017, 3549:30840])),
        //              MpReachNlri(MpReachNlri {
        //                  nexthop: MpNextHop {
        //                      inner: [], afi: ipv4
        //                  },
        //                  nlris: Ipv4Multicast(193.43.128.19.212.77.0/22),
        //                  inner: [144, 14, 0, 17, 0, 1, 2, 4, 208, 51, 134, 246, 0, 22, 193, 43, 128, 19, 212, 77, 0] })]

        let bytes = &[//144,   // flags
                      //14,    // type
                      //0, 17, // length
                      // 0, 1,  // afi
                      // 2,     // safi
                      // 4,     // nexthop length
                      // 208, 51, 134, 246, // nexthop
                      // 0,     // reserved
                      22,    // prefixlength 1
                      193, 43, 128, // prefix 1
                      19,    // prefixlength 2
                      212, 77, 0 // prefix 2
        ];
        let mut iter = Ipv4MulticastNlriIter{inner: bytes, error: false};
        assert_eq!(iter.next().unwrap().unwrap().prefix(), Prefix{inner: &[22, 193, 43, 128]});
        assert_eq!(iter.next().unwrap().unwrap().prefix(), Prefix{inner: &[19, 212, 77, 0]});
        assert!(iter.next().is_none());
    }
}
<commit_msg>fix offset in MpNextHop<commit_after>
use super::*;
use types::*;
use core::fmt;

define_path_attr!(MpReachNlri, doc="Multi Protocol Network Layer Reachability Information");

impl<'a> MpReachNlri<'a> {

    pub fn from_bytes(bytes: &'a [u8]) -> MpReachNlri {
        MpReachNlri {
            inner: bytes,
        }
    }

    fn afi(&self) -> Afi {
        Afi::from((self.value()[0] as u16) << 8 | self.value()[1] as u16)
    }

    fn safi(&self) -> Safi {
        Safi::from(self.value()[2])
    }

    fn nexthop_len(&self) -> usize {
        self.value()[3] as usize
    }

    pub fn nexthop(&self) -> MpNextHop<'a> {
        MpNextHop {
            inner: &self.value()[4..][..self.nexthop_len()],
            afi: self.afi(),
        }
    }

    pub fn nlris(&self) -> MpNlriIter<'a> {
        let offset = 2 + 1 + 1 + self.nexthop_len() + 1;
        match (self.afi(), self.safi()) {
            (AFI_IPV4, SAFI_UNICAST) =>
                MpNlriIter::Ipv4Unicast(Ipv4UnicastNlriIter{inner: &self.value()[offset..], error: false}),
            (AFI_IPV4, SAFI_MULTICAST) =>
                MpNlriIter::Ipv4Multicast(Ipv4MulticastNlriIter{inner: &self.value()[offset..], error: false}),
            (AFI_IPV6, SAFI_UNICAST) =>
                MpNlriIter::Ipv6Unicast(Ipv6UnicastNlriIter{inner: &self.value()[offset..], error: false}),
            (AFI_IPV6, SAFI_MULTICAST) =>
                MpNlriIter::Ipv6Multicast(Ipv6MulticastNlriIter{inner: &self.value()[offset..], error: false}),
            _ => MpNlriIter::Other(OtherNlriIter{inner: &self.value()[offset..], error: false}),
        }
    }
}

impl<'a> fmt::Debug for MpReachNlri<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("MpReachNlri")
            .field("nexthop", &self.nexthop())
            .field("nlris", &self.nlris())
            // .field("inner", &self.inner)
            .finish()
    }
}

#[derive(Debug)]
pub enum MpNlriIter<'a> {
    Ipv4Unicast(Ipv4UnicastNlriIter<'a>),
    Ipv6Unicast(Ipv6UnicastNlriIter<'a>),
    Ipv4Multicast(Ipv4MulticastNlriIter<'a>),
    Ipv6Multicast(Ipv6MulticastNlriIter<'a>),
    Other(OtherNlriIter<'a>)
}

#[derive(Debug)]
pub struct MpNextHop<'a> {
    inner: &'a [u8],
    afi: Afi,
}

macro_rules! impl_mp_nlri {
    ($item:ident, $iter:ident) => {

        #[derive(Clone)]
        pub struct $item<'a> {
            inner: &'a [u8],
        }

        impl<'a> fmt::Debug for $item<'a> {
            fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
                self.prefix().fmt(fmt)
            }
        }

        impl<'a> $item<'a> {
            pub fn prefix(&self) -> Prefix<'a> {
                Prefix{inner: self.inner}
            }
        }

        #[derive(Clone)]
        pub struct $iter<'a> {
            inner: &'a [u8],
            error: bool,
        }

        impl<'a> Iterator for $iter<'a> {
            type Item = Result<$item<'a>>;

            fn next(&mut self) -> Option<Result<$item<'a>>> {
                if self.error || self.inner.is_empty() {
                    return None;
                }

                let mask_len = self.inner[0] as usize;
                assert!(mask_len <= 128);
                let byte_len = (mask_len+15) / 8;
                if self.inner.len() < byte_len {
                    self.error = true;
                    return Some(Err(BgpError::BadLength));
                }
                let slice = &self.inner[..byte_len];
                let nlri = $item{inner: slice};
                self.inner = &self.inner[byte_len..];
                Some(Ok(nlri))
            }
        }

        impl<'a> fmt::Debug for $iter<'a> {
            fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
                fmt.debug_list().entries(self.clone()).finish()
            }
        }

    }
}

impl_mp_nlri!(Ipv4UnicastNlri, Ipv4UnicastNlriIter);
impl_mp_nlri!(Ipv6UnicastNlri, Ipv6UnicastNlriIter);
impl_mp_nlri!(Ipv4MulticastNlri, Ipv4MulticastNlriIter);
impl_mp_nlri!(Ipv6MulticastNlri, Ipv6MulticastNlriIter);
impl_mp_nlri!(OtherNlri, OtherNlriIter);

#[cfg(test)]
mod test {

    use super::*;
    use types::*;
    #[test]
    fn parse_mp_nlri_multicast() {
	      // path_attrs: [Origin(Igp),
        //              AsPath(Ok([3549, 3356, 137, 137, 137, 8978])),
        //              MultiExitDisc(13814),
        //              Communities(Ok([3549:2017, 3549:30840])),
        //              MpReachNlri(MpReachNlri {
        //                  nexthop: MpNextHop {
        //                      inner: [], afi: ipv4
        //                  },
        //                  nlris: Ipv4Multicast(193.43.128.19.212.77.0/22),
        //                  inner: [144, 14, 0, 17, 0, 1, 2, 4, 208, 51, 134, 246, 0, 22, 193, 43, 128, 19, 212, 77, 0] })]

        let bytes = &[//144,   // flags
                      //14,    // type
                      //0, 17, // length
                      // 0, 1,  // afi
                      // 2,     // safi
                      // 4,     // nexthop length
                      // 208, 51, 134, 246, // nexthop
                      // 0,     // reserved
                      22,    // prefixlength 1
                      193, 43, 128, // prefix 1
                      19,    // prefixlength 2
                      212, 77, 0 // prefix 2
        ];
        let mut iter = Ipv4MulticastNlriIter{inner: bytes, error: false};
        assert_eq!(iter.next().unwrap().unwrap().prefix(), Prefix{inner: &[22, 193, 43, 128]});
        assert_eq!(iter.next().unwrap().unwrap().prefix(), Prefix{inner: &[19, 212, 77, 0]});
        assert!(iter.next().is_none());
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! A transaction can have multiple operations on state. For example, it might update values
//! for a few existing keys. Imagine that we have the following tree.
//!
//! ```text
//!                 root0
//!                 /    \
//!                /      \
//!  key1 => value11        key2 => value21
//! ```
//!
//! The next transaction updates `key1`'s value to `value12` and `key2`'s value to `value22`.
//! Let's assume we update key2 first. Then the tree becomes:
//!
//! ```text
//!                   (on disk)              (in memory)
//!                     root0                  root1'
//!                    /     \                /     \
//!                   /   ___ \ _____________/       \
//!                  /  _/     \                      \
//!                 / _/        \                      \
//!                / /           \                      \
//!   key1 => value11           key2 => value21       key2 => value22
//!      (on disk)                 (on disk)            (in memory)
//! ```
//!
//! Note that
//!   1) we created a new version of the tree with `root1'` and the new `key2` node generated;
//!   2) both `root1'` and the new `key2` node are still held in memory within a batch of nodes
//!      that will be written into db atomically.
//!
//! Next, we need to update `key1`'s value. This time we are dealing with the tree starting from
//! the new root. Part of the tree is in memory and the rest of it is in database. We'll update the
//! left child and the new root. We should
//!   1) create a new version for `key1` child.
//!   2) update `root1'` directly instead of making another version.
//! The resulting tree should look like:
//!
//! ```text
//!                   (on disk)                                     (in memory)
//!                     root0                                         root1''
//!                    /     \                                       /     \
//!                   /       \                                     /       \
//!                  /         \                                   /         \
//!                 /           \                                 /           \
//!                /             \                               /             \
//!   key1 => value11             key2 => value21  key1 => value12              key2 => value22
//!      (on disk)                   (on disk)       (in memory)                  (in memory)
//! ```
//!
//! This means that we need to be able to tell whether to create a new version of a node or to
//! update an existing node by deleting it and creating a new node directly. `TreeCache` provides
//! APIs to cache intermediate nodes and blobs in memory and simplify the actual tree
//! implementation.
//!
//! If we are dealing with a single-version tree, any complex tree operation can be seen as a
//! collection of the following operations:
//!   - Put a new node.
//!   - Delete a node.
//! When we apply these operations on a multi-version tree:
//!   1) Put a new node.
//!   2) When we remove a node, if the node is in the previous on-disk version, we don't need to do
//!      anything. Otherwise we delete it from the tree cache.
//! Updating node could be operated as deletion of the node followed by insertion of the updated
//! node.

#[cfg(test)]
mod tree_cache_test;

use crate::{
    node_type::{Node, NodeKey},
    StaleNodeIndex, TreeReader, TreeUpdateBatch,
};
use anyhow::{bail, Result};
use libra_crypto::HashValue;
use libra_types::transaction::Version;
use std::{
    collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet},
    convert::Into,
};

/// `FrozenTreeCache` is used as a field of `TreeCache` storing all the nodes and blobs that are
/// are generated by earlier transactions so they have to be immutable. The motivation of
/// `FrozenTreeCache` is to let `TreeCache` freeze intermediate results from each transaction to
/// help commit more than one transaction in a row atomically.
#[derive(Default)]
struct FrozenTreeCache {
    /// Immutable node_cache.
    node_cache: BTreeMap<NodeKey, Node>,

    /// # of leaves in the `node_cache`,
    num_new_leaves: usize,

    /// Immutable stale_node_index_cache.
    stale_node_index_cache: BTreeSet<StaleNodeIndex>,

    /// # of leaves in the `stale_node_index_cache`,
    num_stale_leaves: usize,

    /// Frozen root hashes after each earlier transaction.
    root_hashes: Vec<HashValue>,
}

/// `TreeCache` is a in-memory cache for per-transaction updates of sparse Merkle nodes and value
/// blobs.
pub struct TreeCache<'a, R: 'a + TreeReader> {
    /// `NodeKey` of the current root node in cache.
    root_node_key: NodeKey,

    /// The version of the transaction to which the upcoming `put`s will be related.
    next_version: Version,

    /// Intermediate nodes keyed by node hash
    node_cache: HashMap<NodeKey, Node>,

    /// # of leaves in the `node_cache`,
    num_new_leaves: usize,

    /// Partial stale log. `NodeKey` to identify the stale record.
    stale_node_index_cache: HashSet<NodeKey>,

    /// # of leaves in the `stale_node_index_cache`,
    num_stale_leaves: usize,

    /// The immutable part of this cache, which will be committed to the underlying storage.
    frozen_cache: FrozenTreeCache,

    /// The underlying persistent storage.
    reader: &'a R,
}

impl<'a, R> TreeCache<'a, R>
where
    R: 'a + TreeReader,
{
    /// Constructs a new `TreeCache` instance.
    pub fn new(reader: &'a R, next_version: Version) -> Self {
        let mut node_cache = HashMap::new();
        let root_node_key = if next_version == 0 {
            // If the first version is 0, it means we need to start from an empty tree so we insert
            // a null node beforehand deliberately to deal with this corner case.
            node_cache.insert(NodeKey::new_empty_path(0), Node::new_null());
            NodeKey::new_empty_path(0)
        } else {
            NodeKey::new_empty_path(next_version - 1)
        };
        Self {
            node_cache,
            stale_node_index_cache: HashSet::new(),
            frozen_cache: FrozenTreeCache::default(),
            root_node_key,
            next_version,
            reader,
            num_stale_leaves: 0,
            num_new_leaves: 0,
        }
    }

    /// Gets a node with given node key. If it doesn't exist in node cache, read from `reader`.
    pub fn get_node(&self, node_key: &NodeKey) -> Result<Node> {
        Ok(if let Some(node) = self.node_cache.get(node_key) {
            node.clone()
        } else if let Some(node) = self.frozen_cache.node_cache.get(node_key) {
            node.clone()
        } else {
            self.reader.get_node(node_key)?
        })
    }

    /// Gets the current root node key.
    pub fn get_root_node_key(&self) -> &NodeKey {
        &self.root_node_key
    }

    /// Set roots `node_key`.
    pub fn set_root_node_key(&mut self, root_node_key: NodeKey) {
        self.root_node_key = root_node_key;
    }

    /// Puts the node with given hash as key into node_cache.
    pub fn put_node(&mut self, node_key: NodeKey, new_node: Node) -> Result<()> {
        match self.node_cache.entry(node_key) {
            Entry::Vacant(o) => {
                if new_node.is_leaf() {
                    self.num_new_leaves += 1
                }
                o.insert(new_node);
            }
            Entry::Occupied(o) => bail!("Node with key {:?} already exists in NodeBatch", o.key()),
        };
        Ok(())
    }

    /// Deletes a node with given hash.
    pub fn delete_node(&mut self, old_node_key: &NodeKey, is_leaf: bool) {
        // If node cache doesn't have this node, it means the node is in the previous version of
        // the tree on the disk.
        if self.node_cache.remove(&old_node_key).is_none() {
            let is_new_entry = self.stale_node_index_cache.insert(old_node_key.clone());
            assert!(is_new_entry, "Node gets stale twice unexpectedly.");
            if is_leaf {
                self.num_stale_leaves += 1;
            }
        } else if is_leaf {
            self.num_new_leaves -= 1;
        }
    }

    /// Freezes all the contents in cache to be immutable and clear `node_cache`.
    pub fn freeze(&mut self) {
        let root_node_key = self.get_root_node_key();
        let root_hash = self
            .get_node(root_node_key)
            .unwrap_or_else(|_| unreachable!("Root node with key {:?} must exist", root_node_key))
            .hash();
        self.frozen_cache.root_hashes.push(root_hash);
        self.frozen_cache.node_cache.extend(self.node_cache.drain());

        let stale_since_version = self.next_version;
        self.frozen_cache
            .stale_node_index_cache
            .extend(
                self.stale_node_index_cache
                    .drain()
                    .map(|node_key| StaleNodeIndex {
                        stale_since_version,
                        node_key,
                    }),
            );
        self.frozen_cache.num_stale_leaves += self.num_stale_leaves;
        self.num_stale_leaves = 0;
        self.frozen_cache.num_new_leaves += self.num_new_leaves;
        self.num_new_leaves = 0;

        self.next_version += 1;
    }
}

impl<'a, R> Into<(Vec<HashValue>, TreeUpdateBatch)> for TreeCache<'a, R>
where
    R: 'a + TreeReader,
{
    fn into(self) -> (Vec<HashValue>, TreeUpdateBatch) {
        (
            self.frozen_cache.root_hashes,
            TreeUpdateBatch {
                node_batch: self.frozen_cache.node_cache,
                stale_node_index_batch: self.frozen_cache.stale_node_index_cache,
                num_new_leaves: self.frozen_cache.num_new_leaves,
                num_stale_leaves: self.frozen_cache.num_stale_leaves,
            },
        )
    }
}
<commit_msg>[storage] fix typo<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! A transaction can have multiple operations on state. For example, it might update values
//! for a few existing keys. Imagine that we have the following tree.
//!
//! ```text
//!                 root0
//!                 /    \
//!                /      \
//!  key1 => value11        key2 => value21
//! ```
//!
//! The next transaction updates `key1`'s value to `value12` and `key2`'s value to `value22`.
//! Let's assume we update key2 first. Then the tree becomes:
//!
//! ```text
//!                   (on disk)              (in memory)
//!                     root0                  root1'
//!                    /     \                /     \
//!                   /   ___ \ _____________/       \
//!                  /  _/     \                      \
//!                 / _/        \                      \
//!                / /           \                      \
//!   key1 => value11           key2 => value21       key2 => value22
//!      (on disk)                 (on disk)            (in memory)
//! ```
//!
//! Note that
//!   1) we created a new version of the tree with `root1'` and the new `key2` node generated;
//!   2) both `root1'` and the new `key2` node are still held in memory within a batch of nodes
//!      that will be written into db atomically.
//!
//! Next, we need to update `key1`'s value. This time we are dealing with the tree starting from
//! the new root. Part of the tree is in memory and the rest of it is in database. We'll update the
//! left child and the new root. We should
//!   1) create a new version for `key1` child.
//!   2) update `root1'` directly instead of making another version.
//! The resulting tree should look like:
//!
//! ```text
//!                   (on disk)                                     (in memory)
//!                     root0                                         root1''
//!                    /     \                                       /     \
//!                   /       \                                     /       \
//!                  /         \                                   /         \
//!                 /           \                                 /           \
//!                /             \                               /             \
//!   key1 => value11             key2 => value21  key1 => value12              key2 => value22
//!      (on disk)                   (on disk)       (in memory)                  (in memory)
//! ```
//!
//! This means that we need to be able to tell whether to create a new version of a node or to
//! update an existing node by deleting it and creating a new node directly. `TreeCache` provides
//! APIs to cache intermediate nodes and blobs in memory and simplify the actual tree
//! implementation.
//!
//! If we are dealing with a single-version tree, any complex tree operation can be seen as a
//! collection of the following operations:
//!   - Put a new node.
//!   - Delete a node.
//! When we apply these operations on a multi-version tree:
//!   1) Put a new node.
//!   2) When we remove a node, if the node is in the previous on-disk version, we don't need to do
//!      anything. Otherwise we delete it from the tree cache.
//! Updating node could be operated as deletion of the node followed by insertion of the updated
//! node.

#[cfg(test)]
mod tree_cache_test;

use crate::{
    node_type::{Node, NodeKey},
    StaleNodeIndex, TreeReader, TreeUpdateBatch,
};
use anyhow::{bail, Result};
use libra_crypto::HashValue;
use libra_types::transaction::Version;
use std::{
    collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet},
    convert::Into,
};

/// `FrozenTreeCache` is used as a field of `TreeCache` storing all the nodes and blobs that
/// are generated by earlier transactions so they have to be immutable. The motivation of
/// `FrozenTreeCache` is to let `TreeCache` freeze intermediate results from each transaction to
/// help commit more than one transaction in a row atomically.
#[derive(Default)]
struct FrozenTreeCache {
    /// Immutable node_cache.
    node_cache: BTreeMap<NodeKey, Node>,

    /// # of leaves in the `node_cache`,
    num_new_leaves: usize,

    /// Immutable stale_node_index_cache.
    stale_node_index_cache: BTreeSet<StaleNodeIndex>,

    /// # of leaves in the `stale_node_index_cache`,
    num_stale_leaves: usize,

    /// Frozen root hashes after each earlier transaction.
    root_hashes: Vec<HashValue>,
}

/// `TreeCache` is a in-memory cache for per-transaction updates of sparse Merkle nodes and value
/// blobs.
pub struct TreeCache<'a, R: 'a + TreeReader> {
    /// `NodeKey` of the current root node in cache.
    root_node_key: NodeKey,

    /// The version of the transaction to which the upcoming `put`s will be related.
    next_version: Version,

    /// Intermediate nodes keyed by node hash
    node_cache: HashMap<NodeKey, Node>,

    /// # of leaves in the `node_cache`,
    num_new_leaves: usize,

    /// Partial stale log. `NodeKey` to identify the stale record.
    stale_node_index_cache: HashSet<NodeKey>,

    /// # of leaves in the `stale_node_index_cache`,
    num_stale_leaves: usize,

    /// The immutable part of this cache, which will be committed to the underlying storage.
    frozen_cache: FrozenTreeCache,

    /// The underlying persistent storage.
    reader: &'a R,
}

impl<'a, R> TreeCache<'a, R>
where
    R: 'a + TreeReader,
{
    /// Constructs a new `TreeCache` instance.
    pub fn new(reader: &'a R, next_version: Version) -> Self {
        let mut node_cache = HashMap::new();
        let root_node_key = if next_version == 0 {
            // If the first version is 0, it means we need to start from an empty tree so we insert
            // a null node beforehand deliberately to deal with this corner case.
            node_cache.insert(NodeKey::new_empty_path(0), Node::new_null());
            NodeKey::new_empty_path(0)
        } else {
            NodeKey::new_empty_path(next_version - 1)
        };
        Self {
            node_cache,
            stale_node_index_cache: HashSet::new(),
            frozen_cache: FrozenTreeCache::default(),
            root_node_key,
            next_version,
            reader,
            num_stale_leaves: 0,
            num_new_leaves: 0,
        }
    }

    /// Gets a node with given node key. If it doesn't exist in node cache, read from `reader`.
    pub fn get_node(&self, node_key: &NodeKey) -> Result<Node> {
        Ok(if let Some(node) = self.node_cache.get(node_key) {
            node.clone()
        } else if let Some(node) = self.frozen_cache.node_cache.get(node_key) {
            node.clone()
        } else {
            self.reader.get_node(node_key)?
        })
    }

    /// Gets the current root node key.
    pub fn get_root_node_key(&self) -> &NodeKey {
        &self.root_node_key
    }

    /// Set roots `node_key`.
    pub fn set_root_node_key(&mut self, root_node_key: NodeKey) {
        self.root_node_key = root_node_key;
    }

    /// Puts the node with given hash as key into node_cache.
    pub fn put_node(&mut self, node_key: NodeKey, new_node: Node) -> Result<()> {
        match self.node_cache.entry(node_key) {
            Entry::Vacant(o) => {
                if new_node.is_leaf() {
                    self.num_new_leaves += 1
                }
                o.insert(new_node);
            }
            Entry::Occupied(o) => bail!("Node with key {:?} already exists in NodeBatch", o.key()),
        };
        Ok(())
    }

    /// Deletes a node with given hash.
    pub fn delete_node(&mut self, old_node_key: &NodeKey, is_leaf: bool) {
        // If node cache doesn't have this node, it means the node is in the previous version of
        // the tree on the disk.
        if self.node_cache.remove(&old_node_key).is_none() {
            let is_new_entry = self.stale_node_index_cache.insert(old_node_key.clone());
            assert!(is_new_entry, "Node gets stale twice unexpectedly.");
            if is_leaf {
                self.num_stale_leaves += 1;
            }
        } else if is_leaf {
            self.num_new_leaves -= 1;
        }
    }

    /// Freezes all the contents in cache to be immutable and clear `node_cache`.
    pub fn freeze(&mut self) {
        let root_node_key = self.get_root_node_key();
        let root_hash = self
            .get_node(root_node_key)
            .unwrap_or_else(|_| unreachable!("Root node with key {:?} must exist", root_node_key))
            .hash();
        self.frozen_cache.root_hashes.push(root_hash);
        self.frozen_cache.node_cache.extend(self.node_cache.drain());

        let stale_since_version = self.next_version;
        self.frozen_cache
            .stale_node_index_cache
            .extend(
                self.stale_node_index_cache
                    .drain()
                    .map(|node_key| StaleNodeIndex {
                        stale_since_version,
                        node_key,
                    }),
            );
        self.frozen_cache.num_stale_leaves += self.num_stale_leaves;
        self.num_stale_leaves = 0;
        self.frozen_cache.num_new_leaves += self.num_new_leaves;
        self.num_new_leaves = 0;

        self.next_version += 1;
    }
}

impl<'a, R> Into<(Vec<HashValue>, TreeUpdateBatch)> for TreeCache<'a, R>
where
    R: 'a + TreeReader,
{
    fn into(self) -> (Vec<HashValue>, TreeUpdateBatch) {
        (
            self.frozen_cache.root_hashes,
            TreeUpdateBatch {
                node_batch: self.frozen_cache.node_cache,
                stale_node_index_batch: self.frozen_cache.stale_node_index_cache,
                num_new_leaves: self.frozen_cache.num_new_leaves,
                num_stale_leaves: self.frozen_cache.num_stale_leaves,
            },
        )
    }
}
<|endoftext|>


<commit_before><commit_msg>Add tests to test backend replacing<commit_after><|endoftext|>


<commit_before>#[macro_use]
extern crate linxal;

#[macro_use]
extern crate ndarray;
extern crate num_traits;

use ndarray::{Array, ArrayBase, arr1, Data, Ix2};
use linxal::factorization::{QRFactors};
use linxal::prelude::{Magnitude};

/// Check that all the properties of the qr factorization are
/// reasonable.
fn check_qr<D1: Data<Elem=f32>>(m: &ArrayBase<D1, Ix2>, qr: &QRFactors<f32>) {
    let q: Array<f32, Ix2> = qr.q().unwrap();
    let r: Array<f32, Ix2> = qr.r().unwrap();

    let a1 = q.dot(&r);

    // let mut a1 = Array::zeros((q.dim().0, r.dim().1));
    // general_mat_mul(1.0, &q, &r, 0.0, &mut a1);

    // Q * R needs to match QR.
    println!("{:?}\n{:?}\n", q, r);
    assert_eq_within_tol!(a1, m, 0.001);

    // partial results for Q yield the same value as Q.
    for i in 1 .. q.dim().1 + 1 {
        let qk = qr.qk(i).unwrap();
        assert_eq_within_tol!(qk, q.slice(s![.., ..i as isize]), 0.001);
    }

    // partial results for R yield the same value as R.
    for i in 1 .. r.dim().0 + 1 {
        let rk = qr.rk(i).unwrap();
        assert_eq_within_tol!(rk, r.slice(s![..i as isize, ..]), 0.001);
    }

    // R is upper-triangular/trapezoidal
    for (i, x) in r.indexed_iter() {
        if i.0 > i.1 {
            assert_eq!(*x, 0.0);
        }
    }

    // Q is orthogonal
    let qtq = q.t().dot(&q);
    let eye: Array<f32, Ix2> = Array::eye(q.dim().1);
    assert_eq_within_tol!(qtq, eye, 0.001);
}

#[test]
fn qr_basic_tall() {
    let m = arr1(&[0.0, 2.0, 2.0, -1.0, 2.0, -1.0, 0.0, 1.5, 2.0, -1.0, 2.0, -1.0]).into_shape((6, 2)).unwrap();

    let qr = QRFactors::compute(&m);
    assert!(qr.is_ok());

    check_qr(&m, &qr.unwrap());
}

#[test]
fn qr_basic_wide() {
    let m = arr1(&[0.0, 2.0, 2.0, -1.0, 2.0, -1.0,
                   0.0, 1.5, 2.0, -1.0, 2.0, -1.0]).into_shape((2, 6)).unwrap();

    let qr = QRFactors::compute(&m);
    assert!(qr.is_ok());

    check_qr(&m, &qr.unwrap());
}
<commit_msg>qr: add tests for generated matrices<commit_after>#[macro_use]
extern crate linxal;

#[macro_use]
extern crate ndarray;
extern crate num_traits;

use ndarray::{Array, ArrayBase, arr1, Data, Ix2};
use linxal::factorization::{QRFactors};
use linxal::prelude::{Magnitude};

/// Check that all the properties of the qr factorization are
/// reasonable.
fn check_qr<D1: Data<Elem=f32>>(m: &ArrayBase<D1, Ix2>, qr: &QRFactors<f32>) {
    let q: Array<f32, Ix2> = qr.q().unwrap();
    let r: Array<f32, Ix2> = qr.r().unwrap();

    assert_eq!(qr.rows(), q.rows());
    assert_eq!(qr.cols(), r.cols());

    let a1 = q.dot(&r);

    // let mut a1 = Array::zeros((q.dim().0, r.dim().1));
    // general_mat_mul(1.0, &q, &r, 0.0, &mut a1);

    // Q * R needs to match QR.
    println!("{:?}\n{:?}\n", q, r);
    assert_eq_within_tol!(a1, m, 0.001);

    // partial results for Q yield the same value as Q.
    for i in 1 .. q.dim().1 + 1 {
        let qk = qr.qk(i).unwrap();
        assert_eq_within_tol!(qk, q.slice(s![.., ..i as isize]), 0.001);
    }

    // partial results for R yield the same value as R.
    for i in 1 .. r.dim().0 + 1 {
        let rk = qr.rk(i).unwrap();
        assert_eq_within_tol!(rk, r.slice(s![..i as isize, ..]), 0.001);
    }

    // R is upper-triangular/trapezoidal
    for (i, x) in r.indexed_iter() {
        if i.0 > i.1 {
            assert_eq!(*x, 0.0);
        }
    }

    // Q is orthogonal
    let qtq = q.t().dot(&q);
    let eye: Array<f32, Ix2> = Array::eye(q.dim().1);
    assert_eq_within_tol!(qtq, eye, 0.001);
}

#[test]
fn qr_basic_tall() {
    let m = arr1(&[0.0, 2.0, 2.0, -1.0, 2.0, -1.0, 0.0, 1.5, 2.0, -1.0, 2.0, -1.0]).into_shape((6, 2)).unwrap();

    let qr = QRFactors::compute(&m);
    assert!(qr.is_ok());

    check_qr(&m, &qr.unwrap());
}

#[test]
fn qr_basic_wide() {
    let m = arr1(&[0.0, 2.0, 2.0, -1.0, 2.0, -1.0,
                   0.0, 1.5, 2.0, -1.0, 2.0, -1.0]).into_shape((2, 6)).unwrap();

    let qr = QRFactors::compute(&m);
    assert!(qr.is_ok());

    check_qr(&m, &qr.unwrap());
}

#[test]
fn qr_basic_linspace() {
    for m in 1..6 {
        for n in 1..6 {
            let mat = Array::linspace(1.0, (m*n) as f32, m*n).into_shape((m, n)).unwrap();

            let qr = QRFactors::compute(&mat);
            assert!(qr.is_ok());

            check_qr(&mat, &qr.unwrap());
        }
    }

}
<|endoftext|>


<commit_before>#![cfg(feature = "s3")]

extern crate env_logger;
#[macro_use]
extern crate log;
extern crate time;

#[macro_use]
extern crate rusoto;

use std::io::Read;
use std::fs::File;
use rusoto::{DefaultCredentialsProvider, Region};
use rusoto::s3::{S3Helper, S3Client, ListObjectsRequest, HeadObjectRequest};

#[test]
fn list_buckets_tests() {
    let _ = env_logger::init();
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    let response = s3.list_buckets().unwrap();
    info!("Got list of buckets: {:?}", response);
    for q in response.buckets {
        info!("Existing bucket: {:?}", q.name);
    }
}

#[test]
fn put_object_test() {
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    let mut f = File::open("tests/sample-data/no_credentials").unwrap();
    let mut contents : Vec<u8> = Vec::new();
    match f.read_to_end(&mut contents) {
        Err(why) => panic!("Error opening file to send to S3: {}", why),
        Ok(_) => {
            s3.put_object("rusototester", "no_credentials", &contents).unwrap();
        }
    }
}

#[test]
fn put_and_fetch_timestamp_named_object_test() {
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    let mut f = File::open("tests/sample-data/no_credentials").unwrap();
    let mut contents : Vec<u8> = Vec::new();
    match f.read_to_end(&mut contents) {
        Err(why) => panic!("Error opening file to send to S3: {}", why),
        Ok(_) => {
            s3.put_object("rusototester", "2016-10-07T23:30:38Z", &contents).unwrap();
        }
    }
    let get_response = s3.get_object("rusototester", "2016-10-07T23:30:38Z").unwrap();
    println!("Got object back: {:?}", get_response);
}

#[test]
fn list_objects_test() {
    let _ = env_logger::init();
    let bare_s3 = S3Client::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    let mut list_request = ListObjectsRequest::default(); // need to set bucket
    list_request.bucket = "rusototester".to_string();
    let result = bare_s3.list_objects(&list_request).unwrap();
    println!("result is {:?}", result);
}

// Dependent on the file being there or it'll break.
#[test]
fn get_object_test() {
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    s3.get_object("rusototester", "no_credentials2").unwrap();
}

// Dependent on the file being there or it'll break.
#[test]
fn head_object_test() {
    let _ = env_logger::init();
    let s3 = S3Client::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    let size_req = HeadObjectRequest{
      bucket: "rusototester".to_string(),
      key: "no_credentials2".to_string(),
      ..Default::default()
    };
    println!("{:?}", s3.head_object(&size_req).unwrap());
}

#[test]
fn delete_object_test() {
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), Region::UsWest2);
    s3.delete_object("rusototester", "no_credentials").unwrap();
}
<commit_msg>Make s3 integration test bucket configurable.  Collapse tests that depend on luck into a single test that sets up necessary environment and tears it down<commit_after>#![cfg(feature = "s3")]

extern crate env_logger;
#[macro_use]
extern crate log;
extern crate time;

#[macro_use]
extern crate rusoto;

use std::io::Read;
use std::fs::File;
use std::env::var;
use rusoto::{DefaultCredentialsProvider, Region};
use rusoto::s3::{S3Helper, S3Client, ListObjectsRequest, HeadObjectRequest};

fn test_bucket() -> String {
    match var("S3_TEST_BUCKET") {
        Ok(val) => val.to_owned(),
        Err(_) => "rusototester".to_owned()
    }
}

fn test_bucket_region() -> Region {
    match var("S3_TEST_BUCKET_REGION") {
        Ok(val) => val.parse().unwrap(),
        Err(_) => "us-west-2".parse().unwrap()
    }
}

#[test]
fn list_buckets_tests() {
    let _ = env_logger::init();
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), test_bucket_region());
    let response = s3.list_buckets().unwrap();
    info!("Got list of buckets: {:?}", response);
    for q in response.buckets {
        info!("Existing bucket: {:?}", q.name);
    }
}

#[test]
fn object_lifecycle_test() {
    // PUT an object
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), test_bucket_region());
    let mut f = File::open("tests/sample-data/no_credentials").unwrap();
    let mut contents : Vec<u8> = Vec::new();
    match f.read_to_end(&mut contents) {
        Err(why) => panic!("Error opening file to send to S3: {}", why),
        Ok(_) => {
            s3.put_object(&test_bucket(), "no_credentials", &contents).unwrap();
        }
    }

    let client = S3Client::new(DefaultCredentialsProvider::new().unwrap(), test_bucket_region());
    // HEAD the object that was PUT
    let size_req = HeadObjectRequest{
      bucket: test_bucket(),
      key: "no_credentials".to_string(),
      ..Default::default()
    };

    println!("{:?}", client.head_object(&size_req).unwrap());

    // GET the object
    s3.get_object(&test_bucket(), "no_credentials").unwrap();

    // DELETE the object
    s3.delete_object(&test_bucket(), "no_credentials").unwrap();
}

#[test]
fn put_and_fetch_timestamp_named_object_test() {
    let s3 = S3Helper::new(DefaultCredentialsProvider::new().unwrap(), test_bucket_region());
    let mut f = File::open("tests/sample-data/no_credentials").unwrap();
    let mut contents : Vec<u8> = Vec::new();
    match f.read_to_end(&mut contents) {
        Err(why) => panic!("Error opening file to send to S3: {}", why),
        Ok(_) => {
            s3.put_object(&test_bucket(), "2016-10-07T23:30:38Z", &contents).unwrap();
        }
    }
    let get_response = s3.get_object(&test_bucket(), "2016-10-07T23:30:38Z").unwrap();
    println!("Got object back: {:?}", get_response);
}

#[test]
fn list_objects_test() {
    let _ = env_logger::init();
    let bare_s3 = S3Client::new(DefaultCredentialsProvider::new().unwrap(), test_bucket_region());
    let mut list_request = ListObjectsRequest::default(); // need to set bucket
    list_request.bucket = test_bucket();
    let result = bare_s3.list_objects(&list_request).unwrap();
    println!("result is {:?}", result);
}

<|endoftext|>


<commit_before>use bls::{
    verify_proof_of_possession,
};
use types::{
    ValidatorRecord,
    ValidatorStatus,
    ValidatorRegistration,
};

/// The size of a validators deposit in GWei.
pub const DEPOSIT_GWEI: u64 = 32_000_000_000;

/// Inducts validators into a `CrystallizedState`.
pub struct ValidatorInductor {
    pub current_slot: u64,
    pub shard_count: u16,
    validators: Vec<ValidatorRecord>,
    empty_validator_start: usize,
}

#[derive(Debug, PartialEq, Clone)]
pub enum ValidatorInductionError {
    InvalidShard,
    InvaidProofOfPossession,
}

impl ValidatorInductor {
    pub fn new(current_slot: u64, shard_count: u16, validators: Vec<ValidatorRecord>)
        -> Self
    {
        Self {
            current_slot,
            shard_count,
            validators,
            empty_validator_start: 0,
        }
    }

    /// Attempt to induct a validator into the CrystallizedState.
    ///
    /// Returns an error if the registration is invalid, otherwise returns the index of the
    /// validator in `CrystallizedState.validators`.
    pub fn induct(&mut self, rego: &ValidatorRegistration)
        -> Result<usize, ValidatorInductionError>
    {
        let v = self.process_registration(rego)?;
        Ok(self.add_validator(v))
    }

    /// Verify a `ValidatorRegistration` and return a `ValidatorRecord` if valid.
    fn process_registration(&self, r: &ValidatorRegistration)
        -> Result<ValidatorRecord, ValidatorInductionError>
    {
        /*
         * Ensure withdrawal shard is not too high.
         */
        if r.withdrawal_shard > self.shard_count {
            return Err(ValidatorInductionError::InvalidShard)
        }

        /*
         * Prove validator has knowledge of their secret key.
         */
        if !verify_proof_of_possession(&r.proof_of_possession, &r.pubkey) {
            return Err(ValidatorInductionError::InvaidProofOfPossession)
        }

        Ok(ValidatorRecord {
            pubkey: r.pubkey.clone(),
            withdrawal_shard: r.withdrawal_shard,
            withdrawal_address: r.withdrawal_address,
            randao_commitment: r.randao_commitment,
            randao_last_change: self.current_slot,
            balance: DEPOSIT_GWEI,
            status: ValidatorStatus::PendingActivation as u8,
            exit_slot: 0,
        })
    }

    /// Returns the index of the first `ValidatorRecord` in the `CrystallizedState` where
    /// `validator.status == Withdrawn`. If no such record exists, `None` is returned.
    fn first_withdrawn_validator(&mut self)
        -> Option<usize>
    {
        for i in self.empty_validator_start..self.validators.len() {
            if self.validators[i].status == ValidatorStatus::Withdrawn as u8 {
                self.empty_validator_start = i + 1;
                return Some(i)
            }
        }
        None
    }

    /// Adds a `ValidatorRecord` to the `CrystallizedState` by replacing first validator where
    /// `validator.status == Withdraw`. If no such withdrawn validator exists, adds the new
    /// validator to the end of the list.
    fn add_validator(&mut self, v: ValidatorRecord)
        -> usize
    {
        match self.first_withdrawn_validator() {
            Some(i) => {
                self.validators[i] = v;
                i
            }
            None => {
                self.validators.push(v);
                self.validators.len() - 1
            }
        }
    }

    pub fn to_vec(self)
        -> Vec<ValidatorRecord>
    {
        self.validators
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use bls::{
        Keypair,
        Signature,
    };
    use types::{
        Address,
        Hash256,
    };
    use hashing::proof_of_possession_hash;

    fn registration_equals_record(reg: &ValidatorRegistration, rec: &ValidatorRecord)
        -> bool
    {
        (reg.pubkey == rec.pubkey) &
        (reg.withdrawal_shard == rec.withdrawal_shard) &
        (reg.withdrawal_address == rec.withdrawal_address) &
        (reg.randao_commitment == rec.randao_commitment) &
        (verify_proof_of_possession(&reg.proof_of_possession, &rec.pubkey))
    }

    /// Generate a proof of possession for some keypair.
    fn get_proof_of_possession(kp: &Keypair) -> Signature {
        let pop_message = proof_of_possession_hash(&kp.pk.as_bytes());
        Signature::new_hashed(&pop_message, &kp.sk)
    }

    /// Generate a basic working ValidatorRegistration for use in tests.
    fn get_registration() -> ValidatorRegistration {
        let kp = Keypair::random();
        ValidatorRegistration {
            pubkey: kp.pk.clone(),
            withdrawal_shard: 0,
            withdrawal_address: Address::zero(),
            randao_commitment: Hash256::zero(),
            proof_of_possession: get_proof_of_possession(&kp),
        }
    }

    #[test]
    fn test_validator_inductor_valid_empty_validators() {
        let validators = vec![];

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r);
        let validators = inductor.to_vec();

        assert_eq!(result.unwrap(), 0);
        assert!(registration_equals_record(&r, &validators[0]));
        assert_eq!(validators.len(), 1);
    }

    #[test]
    fn test_validator_inductor_valid_all_active_validators() {
        let mut validators = vec![];
        for _ in 0..5 {
            let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
            v.status = ValidatorStatus::Active as u8;
            validators.push(v);
        }

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r);
        let validators = inductor.to_vec();

        assert_eq!(result.unwrap(), 5);
        assert!(registration_equals_record(&r, &validators[5]));
        assert_eq!(validators.len(), 6);
    }

    #[test]
    fn test_validator_inductor_valid_all_second_validator_withdrawn() {
        let mut validators = vec![];
        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
        v.status = ValidatorStatus::Active as u8;
        validators.push(v);
        for _ in 0..4 {
            let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
            v.status = ValidatorStatus::Withdrawn as u8;
            validators.push(v);
        }

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r);
        let validators = inductor.to_vec();

        assert_eq!(result.unwrap(), 1);
        assert!(registration_equals_record(&r, &validators[1]));
        assert_eq!(validators.len(), 5);
    }

    #[test]
    fn test_validator_inductor_valid_all_withdrawn_validators() {
        let mut validators = vec![];
        for _ in 0..5 {
            let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
            v.status = ValidatorStatus::Withdrawn as u8;
            validators.push(v);
        }

        /*
         * Ensure the first validator gets the 0'th slot
         */
        let r = get_registration();
        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r);
        let validators = inductor.to_vec();
        assert_eq!(result.unwrap(), 0);
        assert!(registration_equals_record(&r, &validators[0]));

        /*
         * Ensure the second validator gets the 1'st slot
         */
        let r_two = get_registration();
        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r_two);
        let validators = inductor.to_vec();
        assert_eq!(result.unwrap(), 1);
        assert!(registration_equals_record(&r_two, &validators[1]));
        assert_eq!(validators.len(), 5);
    }

    #[test]
    fn test_validator_inductor_shard_too_high() {
        let validators = vec![];

        let mut r = get_registration();
        r.withdrawal_shard = 1025;

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r);
        let validators = inductor.to_vec();

        assert_eq!(result, Err(ValidatorInductionError::InvalidShard));
        assert_eq!(validators.len(), 0);
    }

    #[test]
    fn test_validator_inductor_shard_proof_of_possession_failure() {
        let validators = vec![];

        let mut r = get_registration();
        let kp = Keypair::random();
        r.proof_of_possession = get_proof_of_possession(&kp);

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r);
        let validators = inductor.to_vec();

        assert_eq!(result, Err(ValidatorInductionError::InvaidProofOfPossession));
        assert_eq!(validators.len(), 0);
    }
}
<commit_msg>Add `status` to validator induction.<commit_after>use bls::{
    verify_proof_of_possession,
};
use types::{
    ValidatorRecord,
    ValidatorStatus,
    ValidatorRegistration,
};

/// The size of a validators deposit in GWei.
pub const DEPOSIT_GWEI: u64 = 32_000_000_000;

/// Inducts validators into a `CrystallizedState`.
pub struct ValidatorInductor {
    pub current_slot: u64,
    pub shard_count: u16,
    validators: Vec<ValidatorRecord>,
    empty_validator_start: usize,
}

#[derive(Debug, PartialEq, Clone)]
pub enum ValidatorInductionError {
    InvalidShard,
    InvaidProofOfPossession,
}

impl ValidatorInductor {
    pub fn new(current_slot: u64, shard_count: u16, validators: Vec<ValidatorRecord>)
        -> Self
    {
        Self {
            current_slot,
            shard_count,
            validators,
            empty_validator_start: 0,
        }
    }

    /// Attempt to induct a validator into the CrystallizedState.
    ///
    /// Returns an error if the registration is invalid, otherwise returns the index of the
    /// validator in `CrystallizedState.validators`.
    pub fn induct(&mut self, rego: &ValidatorRegistration, status: ValidatorStatus)
        -> Result<usize, ValidatorInductionError>
    {
        let v = self.process_registration(rego, status)?;
        Ok(self.add_validator(v))
    }

    /// Verify a `ValidatorRegistration` and return a `ValidatorRecord` if valid.
    fn process_registration(&self, r: &ValidatorRegistration, status: ValidatorStatus)
        -> Result<ValidatorRecord, ValidatorInductionError>
    {
        /*
         * Ensure withdrawal shard is not too high.
         */
        if r.withdrawal_shard > self.shard_count {
            return Err(ValidatorInductionError::InvalidShard)
        }

        /*
         * Prove validator has knowledge of their secret key.
         */
        if !verify_proof_of_possession(&r.proof_of_possession, &r.pubkey) {
            return Err(ValidatorInductionError::InvaidProofOfPossession)
        }

        Ok(ValidatorRecord {
            pubkey: r.pubkey.clone(),
            withdrawal_shard: r.withdrawal_shard,
            withdrawal_address: r.withdrawal_address,
            randao_commitment: r.randao_commitment,
            randao_last_change: self.current_slot,
            balance: DEPOSIT_GWEI,
            status: status as u8,
            exit_slot: 0,
        })
    }

    /// Returns the index of the first `ValidatorRecord` in the `CrystallizedState` where
    /// `validator.status == Withdrawn`. If no such record exists, `None` is returned.
    fn first_withdrawn_validator(&mut self)
        -> Option<usize>
    {
        for i in self.empty_validator_start..self.validators.len() {
            if self.validators[i].status == ValidatorStatus::Withdrawn as u8 {
                self.empty_validator_start = i + 1;
                return Some(i)
            }
        }
        None
    }

    /// Adds a `ValidatorRecord` to the `CrystallizedState` by replacing first validator where
    /// `validator.status == Withdraw`. If no such withdrawn validator exists, adds the new
    /// validator to the end of the list.
    fn add_validator(&mut self, v: ValidatorRecord)
        -> usize
    {
        match self.first_withdrawn_validator() {
            Some(i) => {
                self.validators[i] = v;
                i
            }
            None => {
                self.validators.push(v);
                self.validators.len() - 1
            }
        }
    }

    pub fn to_vec(self)
        -> Vec<ValidatorRecord>
    {
        self.validators
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use bls::{
        Keypair,
        Signature,
    };
    use types::{
        Address,
        Hash256,
    };
    use hashing::proof_of_possession_hash;

    fn registration_equals_record(reg: &ValidatorRegistration, rec: &ValidatorRecord)
        -> bool
    {
        (reg.pubkey == rec.pubkey) &
        (reg.withdrawal_shard == rec.withdrawal_shard) &
        (reg.withdrawal_address == rec.withdrawal_address) &
        (reg.randao_commitment == rec.randao_commitment) &
        (verify_proof_of_possession(&reg.proof_of_possession, &rec.pubkey))
    }

    /// Generate a proof of possession for some keypair.
    fn get_proof_of_possession(kp: &Keypair) -> Signature {
        let pop_message = proof_of_possession_hash(&kp.pk.as_bytes());
        Signature::new_hashed(&pop_message, &kp.sk)
    }

    /// Generate a basic working ValidatorRegistration for use in tests.
    fn get_registration() -> ValidatorRegistration {
        let kp = Keypair::random();
        ValidatorRegistration {
            pubkey: kp.pk.clone(),
            withdrawal_shard: 0,
            withdrawal_address: Address::zero(),
            randao_commitment: Hash256::zero(),
            proof_of_possession: get_proof_of_possession(&kp),
        }
    }

    #[test]
    fn test_validator_inductor_valid_empty_validators() {
        let validators = vec![];

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();

        assert_eq!(result.unwrap(), 0);
        assert!(registration_equals_record(&r, &validators[0]));
        assert_eq!(validators.len(), 1);
    }

    #[test]
    fn test_validator_inductor_status() {
        let validators = vec![];

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let _ = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let _ = inductor.induct(&r, ValidatorStatus::Active);
        let validators = inductor.to_vec();

        assert!(validators[0].status == ValidatorStatus::PendingActivation as u8);
        assert!(validators[1].status == ValidatorStatus::Active as u8);
        assert_eq!(validators.len(), 2);
    }

    #[test]
    fn test_validator_inductor_valid_all_active_validators() {
        let mut validators = vec![];
        for _ in 0..5 {
            let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
            v.status = ValidatorStatus::Active as u8;
            validators.push(v);
        }

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();

        assert_eq!(result.unwrap(), 5);
        assert!(registration_equals_record(&r, &validators[5]));
        assert_eq!(validators.len(), 6);
    }

    #[test]
    fn test_validator_inductor_valid_all_second_validator_withdrawn() {
        let mut validators = vec![];
        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
        v.status = ValidatorStatus::Active as u8;
        validators.push(v);
        for _ in 0..4 {
            let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
            v.status = ValidatorStatus::Withdrawn as u8;
            validators.push(v);
        }

        let r = get_registration();

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();

        assert_eq!(result.unwrap(), 1);
        assert!(registration_equals_record(&r, &validators[1]));
        assert_eq!(validators.len(), 5);
    }

    #[test]
    fn test_validator_inductor_valid_all_withdrawn_validators() {
        let mut validators = vec![];
        for _ in 0..5 {
            let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
            v.status = ValidatorStatus::Withdrawn as u8;
            validators.push(v);
        }

        /*
         * Ensure the first validator gets the 0'th slot
         */
        let r = get_registration();
        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();
        assert_eq!(result.unwrap(), 0);
        assert!(registration_equals_record(&r, &validators[0]));

        /*
         * Ensure the second validator gets the 1'st slot
         */
        let r_two = get_registration();
        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r_two, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();
        assert_eq!(result.unwrap(), 1);
        assert!(registration_equals_record(&r_two, &validators[1]));
        assert_eq!(validators.len(), 5);
    }

    #[test]
    fn test_validator_inductor_shard_too_high() {
        let validators = vec![];

        let mut r = get_registration();
        r.withdrawal_shard = 1025;

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();

        assert_eq!(result, Err(ValidatorInductionError::InvalidShard));
        assert_eq!(validators.len(), 0);
    }

    #[test]
    fn test_validator_inductor_shard_proof_of_possession_failure() {
        let validators = vec![];

        let mut r = get_registration();
        let kp = Keypair::random();
        r.proof_of_possession = get_proof_of_possession(&kp);

        let mut inductor = ValidatorInductor::new(0, 1024, validators);
        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
        let validators = inductor.to_vec();

        assert_eq!(result, Err(ValidatorInductionError::InvaidProofOfPossession));
        assert_eq!(validators.len(), 0);
    }
}
<|endoftext|>


<commit_before>use std::iter;

pub fn verse(n: int) -> ~str {
    match n {
        0 =>
            ~"No more bottles of beer on the wall, no more bottles of beer.\nGo to the store and buy some more, 99 bottles of beer on the wall.\n",
        1 =>
            ~"1 bottle of beer on the wall, 1 bottle of beer.\nTake it down and pass it around, no more bottles of beer on the wall.\n",
        2 =>
            ~"2 bottles of beer on the wall, 2 bottles of beer.\nTake one down and pass it around, 1 bottle of beer on the wall.\n",
        n if n > 2 && n <= 99 =>
            [n.to_str(), ~" bottles of beer on the wall, ",
             n.to_str(), ~" bottles of beer.\n",
             ~"Take one down and pass it around, ",
             (n - 1).to_str(), ~" bottles of beer on the wall.\n"].concat(),
        _ =>
            fail!(),
    }
}

pub fn sing(start: int, end: int) -> ~str {
    iter::range_inclusive(end, start)
        .invert()
        .map(verse)
        .collect::<~[~str]>()
        .connect("\n") + "\n"
}
<commit_msg>rust - clean up beer-song<commit_after>use std::iter;

pub fn verse(n: int) -> ~str {
    match n {
        0 =>
            ~"No more bottles of beer on the wall, no more bottles of beer.\nGo to the store and buy some more, 99 bottles of beer on the wall.\n",
        1 =>
            ~"1 bottle of beer on the wall, 1 bottle of beer.\nTake it down and pass it around, no more bottles of beer on the wall.\n",
        2 =>
            ~"2 bottles of beer on the wall, 2 bottles of beer.\nTake one down and pass it around, 1 bottle of beer on the wall.\n",
        n if n > 2 && n <= 99 =>
            format!("{n} bottles of beer on the wall, {n} bottles of beer.\nTake one down and pass it around, {n_minus_1} bottles of beer on the wall.\n", n=n, n_minus_1=n - 1),
        _ =>
            fail!(),
    }
}

pub fn sing(start: int, end: int) -> ~str {
    let mut s = iter::range_inclusive(end, start)
        .invert()
        .map(verse)
        .to_owned_vec()
        .connect("\n");
    s.push_char('\n');
    s
}
<|endoftext|>


<commit_before>use std::io;
use std::io::fs::mkdir_recursive;

use csv;
use csv::index::Indexed;
use docopt;

use types::{CliError, CsvConfig, Delimiter};
use util;

docopt!(Args, "
Splits the given CSV data into chunks.

The files are written to the directory given with the name '{start}-{end}.csv',
where {start} and {end} is the half-open interval corresponding to the records
in the chunk.

Usage:
    xsv split [options] <outdir> [<input>]

split options:
    -s, --size <arg>       The number of records to write into each chunk.
                           [default: 500]
    -j, --jobs <arg>       The number of spliting jobs to run in parallel.
                           Note that this only works when the given CSV data
                           has an index already created.
                           [default: 4]

Common options:
    -h, --help             Display this message
    -o, --output <file>    Write output to <file> instead of stdout.
    -n, --no-headers       When set, the first row will NOT be interpreted
                           as column names. Note that this has no effect when
                           concatenating columns.
    -d, --delimiter <arg>  The field delimiter for reading CSV data.
                           Must be a single character. [default: ,]
", arg_input: Option<String>, arg_outdir: String, flag_output: Option<String>,
   flag_delimiter: Delimiter, flag_size: u64, flag_jobs: uint)

pub fn main() -> Result<(), CliError> {
    let args: Args = try!(util::get_args());
    try!(io| mkdir_recursive(&Path::new(args.arg_outdir[]),
                             io::AllPermissions));

    match try!(args.rconfig().indexed()) {
        Some(idx) => args.parallel_split(idx),
        None => args.sequential_split(),
    }
}

impl Args {
    fn sequential_split(&self) -> Result<(), CliError> {
        let rconfig = self.rconfig();
        let mut rdr = try!(io| rconfig.reader());
        let headers = try!(csv| rdr.byte_headers());

        let mut wtr = try!(self.new_writer(headers[], 0));
        for (i, row) in rdr.byte_records().enumerate() {
            if i > 0 && i as u64 % self.flag_size == 0 {
                try!(csv| wtr.flush());
                wtr = try!(self.new_writer(headers[], i as u64));
            }
            let row = try!(csv| row);
            try!(csv| wtr.write_bytes(row.into_iter()));
        }
        try!(csv| wtr.flush());
        Ok(())
    }

    fn parallel_split(&self, idx: Indexed<io::File, io::File>)
                     -> Result<(), CliError> {
        use std::sync::TaskPool;

        let mut nchunks = idx.count() / self.flag_size;
        if idx.count() % self.flag_size != 0 {
            nchunks += 1;
        }
        let mut pool = TaskPool::new(self.flag_jobs, || { proc(i) i });
        for i in range(0, nchunks) {
            let conf = self.rconfig();
            let args = self.clone();
            pool.execute(proc(_) {
                let mut idx = conf.indexed().unwrap().unwrap();
                let headers = idx.csv().byte_headers().unwrap();
                let mut wtr = args.new_writer(headers[], i * args.flag_size)
                                  .unwrap();

                idx.seek(i * args.flag_size).unwrap();
                let writenum = args.flag_size as uint;
                for row in idx.csv().byte_records().take(writenum) {
                    let row = row.unwrap();
                    wtr.write_bytes(row.into_iter()).unwrap();
                }
                wtr.flush().unwrap();
            });
        }
        Ok(())
    }

    fn new_writer(&self, headers: &[csv::ByteString], start: u64)
                 -> Result<csv::Writer<Box<io::Writer+'static>>, CliError> {
        let dir = Path::new(self.arg_outdir.clone());
        let path = dir.join(format!("{}.csv", start));
        let spath = Some(path.display().to_string());
        let mut wtr = try!(io| CsvConfig::new(spath).writer());
        if !self.flag_no_headers {
            try!(csv| wtr.write_bytes(headers.iter().map(|f| f[])));
        }
        Ok(wtr)
    }

    fn rconfig(&self) -> CsvConfig {
        CsvConfig::new(self.arg_input.clone())
                  .delimiter(self.flag_delimiter)
                  .no_headers(self.flag_no_headers)
    }
}

// This is gross. Any way around it?
impl Clone for Args {
    fn clone(&self) -> Args {
        Args {
            flag_size: self.flag_size,
            cmd_split: self.cmd_split,
            flag_output: self.flag_output.clone(),
            flag_no_headers: self.flag_no_headers,
            flag_delimiter: self.flag_delimiter.clone(),
            flag_help: self.flag_help,
            arg_input: self.arg_input.clone(),
            arg_outdir: self.arg_outdir.clone(),
            flag_jobs: self.flag_jobs,
        }
    }
}
<commit_msg>Some touchups.<commit_after>use std::io;
use std::io::fs::mkdir_recursive;

use csv;
use csv::index::Indexed;
use docopt;

use types::{CliError, CsvConfig, Delimiter};
use util;

docopt!(Args, "
Splits the given CSV data into chunks.

The files are written to the directory given with the name '{start}.csv',
where {start} is the index of the first record of the chunk (starting at 0).

Usage:
    xsv split [options] <outdir> [<input>]

split options:
    -s, --size <arg>       The number of records to write into each chunk.
                           [default: 500]
    -j, --jobs <arg>       The number of spliting jobs to run in parallel.
                           This only works when the given CSV data has
                           an index already created. Note that a file handle
                           is opened for each job.
                           [default: 12]

Common options:
    -h, --help             Display this message
    -o, --output <file>    Write output to <file> instead of stdout.
    -n, --no-headers       When set, the first row will NOT be interpreted
                           as column names. Note that this has no effect when
                           concatenating columns.
    -d, --delimiter <arg>  The field delimiter for reading CSV data.
                           Must be a single character. [default: ,]
", arg_input: Option<String>, arg_outdir: String, flag_output: Option<String>,
   flag_delimiter: Delimiter, flag_size: u64, flag_jobs: uint)

pub fn main() -> Result<(), CliError> {
    let args: Args = try!(util::get_args());
    try!(io| mkdir_recursive(&Path::new(args.arg_outdir[]),
                             io::AllPermissions));

    match try!(args.rconfig().indexed()) {
        Some(idx) => args.parallel_split(idx),
        None => args.sequential_split(),
    }
}

impl Args {
    fn sequential_split(&self) -> Result<(), CliError> {
        let rconfig = self.rconfig();
        let mut rdr = try!(io| rconfig.reader());
        let headers = try!(csv| rdr.byte_headers());

        let mut wtr = try!(self.new_writer(headers[], 0));
        for (i, row) in rdr.byte_records().enumerate() {
            if i > 0 && i as u64 % self.flag_size == 0 {
                try!(csv| wtr.flush());
                wtr = try!(self.new_writer(headers[], i as u64));
            }
            let row = try!(csv| row);
            try!(csv| wtr.write_bytes(row.into_iter()));
        }
        try!(csv| wtr.flush());
        Ok(())
    }

    fn parallel_split(&self, idx: Indexed<io::File, io::File>)
                     -> Result<(), CliError> {
        use std::sync::TaskPool;

        let mut nchunks = idx.count() / self.flag_size;
        if idx.count() % self.flag_size != 0 {
            nchunks += 1;
        }
        let mut pool = TaskPool::new(self.flag_jobs, || { proc(i) i });
        for i in range(0, nchunks) {
            let args = self.clone();
            pool.execute(proc(_) {
                let conf = args.rconfig();
                let mut idx = conf.indexed().unwrap().unwrap();
                let headers = idx.csv().byte_headers().unwrap();
                let mut wtr = args.new_writer(headers[], i * args.flag_size)
                                  .unwrap();

                idx.seek(i * args.flag_size).unwrap();
                let writenum = args.flag_size as uint;
                for row in idx.csv().byte_records().take(writenum) {
                    let row = row.unwrap();
                    wtr.write_bytes(row.into_iter()).unwrap();
                }
                wtr.flush().unwrap();
            });
        }
        Ok(())
    }

    fn new_writer(&self, headers: &[csv::ByteString], start: u64)
                 -> Result<csv::Writer<Box<io::Writer+'static>>, CliError> {
        let dir = Path::new(self.arg_outdir.clone());
        let path = dir.join(format!("{}.csv", start));
        let spath = Some(path.display().to_string());
        let mut wtr = try!(io| CsvConfig::new(spath).writer());
        if !self.flag_no_headers {
            try!(csv| wtr.write_bytes(headers.iter().map(|f| f[])));
        }
        Ok(wtr)
    }

    fn rconfig(&self) -> CsvConfig {
        CsvConfig::new(self.arg_input.clone())
                  .delimiter(self.flag_delimiter)
                  .no_headers(self.flag_no_headers)
    }
}

// This is gross. Any way around it?
impl Clone for Args {
    fn clone(&self) -> Args {
        Args {
            flag_size: self.flag_size,
            cmd_split: self.cmd_split,
            flag_output: self.flag_output.clone(),
            flag_no_headers: self.flag_no_headers,
            flag_delimiter: self.flag_delimiter.clone(),
            flag_help: self.flag_help,
            arg_input: self.arg_input.clone(),
            arg_outdir: self.arg_outdir.clone(),
            flag_jobs: self.flag_jobs,
        }
    }
}
<|endoftext|>