<commit_before>// Copyright 2020 The Tink-Rust Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

use prost::Message;
use tink_core::{utils::wrap_err, TinkError};

#[test]
fn test_templates() {
    tink_mac::init();
    let template = tink_mac::hmac_sha256_tag128_key_template();
    check_hmac_template(template, 32, 16, tink_proto::HashType::Sha256)
        .expect("incorrect HMACSHA256Tag128KeyTemplate");
    let template = tink_mac::hmac_sha256_tag256_key_template();
    check_hmac_template(template, 32, 32, tink_proto::HashType::Sha256)
        .expect("incorrect HMACSHA256Tag256KeyTemplate");
    let template = tink_mac::hmac_sha512_tag256_key_template();
    check_hmac_template(template, 64, 32, tink_proto::HashType::Sha512)
        .expect("incorrect HMACSHA512Tag256KeyTemplate");
    let template = tink_mac::hmac_sha512_tag512_key_template();
    check_hmac_template(template, 64, 64, tink_proto::HashType::Sha512)
        .expect("incorrect HMACSHA512Tag512KeyTemplate");
    let template = tink_mac::aes_cmac_tag128_key_template();
    check_cmac_template(template, 32, 16).expect("incorrect AESCMACTag128KeyTemplate");
}

fn check_hmac_template(
    template: tink_proto::KeyTemplate,
    key_size: usize,
    tag_size: usize,
    hash_type: tink_proto::HashType,
) -> Result<(), TinkError> {
    if template.type_url != tink_tests::HMAC_TYPE_URL {
        return Err("type_url is incorrect".into());
    }
    let format = tink_proto::HmacKeyFormat::decode(template.value.as_ref())
        .map_err(|e| wrap_err("unable to unmarshal serialized key format", e))?;
    if format.key_size as usize != key_size
        || format.params.as_ref().unwrap().hash != hash_type as i32
        || format.params.as_ref().unwrap().tag_size as usize != tag_size
    {
        return Err("KeyFormat is incorrect".into());
    }
    let keymanager = tink_core::registry::get_key_manager(tink_tests::HMAC_TYPE_URL)
        .expect("HMAC key manager not found");
    keymanager
        .new_key(&template.value)
        .expect("HMAC key manager cannot create key");
    Ok(())
}

fn check_cmac_template(
    template: tink_proto::KeyTemplate,
    key_size: usize,
    tag_size: usize,
) -> Result<(), TinkError> {
    if template.type_url != tink_tests::AES_CMAC_TYPE_URL {
        return Err("TypeUrl is incorrect".into());
    }
    let format = tink_proto::AesCmacKeyFormat::decode(template.value.as_ref())
        .map_err(|e| wrap_err("unable to unmarshal serialized key format", e))?;
    if format.key_size as usize != key_size
        || format.params.as_ref().unwrap().tag_size as usize != tag_size
    {
        return Err("KeyFormat is incorrect".into());
    }
    let keymanager = tink_core::registry::get_key_manager(tink_tests::AES_CMAC_TYPE_URL)
        .expect("AES CMAC key manager not found");
    keymanager
        .new_key(&template.value)
        .expect("AES CMAC key manager cannot create key");
    Ok(())
}
<commit_msg>tink-mac: add key template tests<commit_after>// Copyright 2020 The Tink-Rust Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

use prost::Message;
use tink_core::{utils::wrap_err, TinkError};

#[test]
fn test_key_templates() {
    tink_mac::init();
    let test_cases = vec![
        (
            "HMAC_SHA256_128BITTAG",
            tink_mac::hmac_sha256_tag128_key_template(),
        ),
        (
            "HMAC_SHA256_256BITTAG",
            tink_mac::hmac_sha256_tag256_key_template(),
        ),
        (
            "HMAC_SHA512_256BITTAG",
            tink_mac::hmac_sha512_tag256_key_template(),
        ),
        (
            "HMAC_SHA512_512BITTAG",
            tink_mac::hmac_sha512_tag512_key_template(),
        ),
        ("AES_CMAC", tink_mac::aes_cmac_tag128_key_template()),
    ];
    for (name, template) in test_cases {
        let want = tink_tests::key_template_proto("mac", name).unwrap();
        assert_eq!(want, template);

        let handle = tink_core::keyset::Handle::new(&template).unwrap();
        let primitive = tink_mac::new(&handle).unwrap();

        let msg = b"this data needs to be authenticated";
        let tag = primitive.compute_mac(msg).unwrap();
        assert!(primitive.verify_mac(&tag, msg).is_ok());
    }
}

#[test]
fn test_templates() {
    tink_mac::init();
    let template = tink_mac::hmac_sha256_tag128_key_template();
    check_hmac_template(template, 32, 16, tink_proto::HashType::Sha256)
        .expect("incorrect HMACSHA256Tag128KeyTemplate");
    let template = tink_mac::hmac_sha256_tag256_key_template();
    check_hmac_template(template, 32, 32, tink_proto::HashType::Sha256)
        .expect("incorrect HMACSHA256Tag256KeyTemplate");
    let template = tink_mac::hmac_sha512_tag256_key_template();
    check_hmac_template(template, 64, 32, tink_proto::HashType::Sha512)
        .expect("incorrect HMACSHA512Tag256KeyTemplate");
    let template = tink_mac::hmac_sha512_tag512_key_template();
    check_hmac_template(template, 64, 64, tink_proto::HashType::Sha512)
        .expect("incorrect HMACSHA512Tag512KeyTemplate");
    let template = tink_mac::aes_cmac_tag128_key_template();
    check_cmac_template(template, 32, 16).expect("incorrect AESCMACTag128KeyTemplate");
}

fn check_hmac_template(
    template: tink_proto::KeyTemplate,
    key_size: usize,
    tag_size: usize,
    hash_type: tink_proto::HashType,
) -> Result<(), TinkError> {
    if template.type_url != tink_tests::HMAC_TYPE_URL {
        return Err("type_url is incorrect".into());
    }
    let format = tink_proto::HmacKeyFormat::decode(template.value.as_ref())
        .map_err(|e| wrap_err("unable to unmarshal serialized key format", e))?;
    if format.key_size as usize != key_size
        || format.params.as_ref().unwrap().hash != hash_type as i32
        || format.params.as_ref().unwrap().tag_size as usize != tag_size
    {
        return Err("KeyFormat is incorrect".into());
    }
    let keymanager = tink_core::registry::get_key_manager(tink_tests::HMAC_TYPE_URL)
        .expect("HMAC key manager not found");
    keymanager
        .new_key(&template.value)
        .expect("HMAC key manager cannot create key");
    Ok(())
}

fn check_cmac_template(
    template: tink_proto::KeyTemplate,
    key_size: usize,
    tag_size: usize,
) -> Result<(), TinkError> {
    if template.type_url != tink_tests::AES_CMAC_TYPE_URL {
        return Err("TypeUrl is incorrect".into());
    }
    let format = tink_proto::AesCmacKeyFormat::decode(template.value.as_ref())
        .map_err(|e| wrap_err("unable to unmarshal serialized key format", e))?;
    if format.key_size as usize != key_size
        || format.params.as_ref().unwrap().tag_size as usize != tag_size
    {
        return Err("KeyFormat is incorrect".into());
    }
    let keymanager = tink_core::registry::get_key_manager(tink_tests::AES_CMAC_TYPE_URL)
        .expect("AES CMAC key manager not found");
    keymanager
        .new_key(&template.value)
        .expect("AES CMAC key manager cannot create key");
    Ok(())
}
<|endoftext|>


<commit_before>use elapsed::measure_time;

use fall_tree::{NodeType, ERROR, TextRange, FileStats, INode, TextUnit, Language};
use lex::{Token, LexRule, tokenize};

#[derive(Clone, Copy, Debug)]
pub struct TokenSequence<'a> {
    text: &'a str,
    start: usize,
    non_ws_indexes: &'a [usize],
    original_tokens: &'a [Token],
}

#[derive(Debug)]
pub enum Node {
    Leaf(NodeType, usize),
    Composite {
        ty: Option<NodeType>,
        children: Vec<Node>,
    }
}

impl Node {
    pub fn error() -> Node {
        Self::composite(Some(ERROR))
    }

    pub fn composite(ty: Option<NodeType>) -> Node {
        Node::Composite { ty: ty, children: Vec::new() }
    }

    pub fn success<'t>(ts: TokenSequence<'t>) -> (Node, TokenSequence<'t>) {
        (Self::composite(None), ts)
    }
}

impl<'a> TokenSequence<'a> {
    pub fn prefix(&self, suffix: TokenSequence<'a>) -> TokenSequence<'a> {
        TokenSequence {
            text: self.text,
            start: self.start,
            non_ws_indexes: &self.non_ws_indexes[..suffix.start - self.start],
            original_tokens: self.original_tokens
        }
    }

    pub fn current(&self) -> Option<Token> {
        self.non_ws_indexes.first().map(|&idx| {
            self.original_tokens[idx]
        })
    }

    pub fn bump(&self) -> (Node, TokenSequence<'a>) {
        let token = self.current().expect("Can't bump an empty token sequence");
        let node = Node::Leaf(token.ty, self.non_ws_indexes[0]);
        let mut text_len = 0;
        let next_idx = self.non_ws_indexes.get(1).map(|&i| i).unwrap_or(self.original_tokens.len());
        for i in self.non_ws_indexes[0] ..next_idx {
            text_len += self.original_tokens[i].len.as_u32() as usize;
        }
        let rest = TokenSequence {
            text: &self.text[text_len..],
            start: self.start + 1,
            non_ws_indexes: &self.non_ws_indexes[1..],
            original_tokens: self.original_tokens,
        };
        (node, rest)
    }

    pub fn bump_by_text(&self, ty: NodeType, text: &str) -> Option<(Node, TokenSequence<'a>)> {
        if let Some(t) = self.current_text() {
            if t == text {
                let node = Node::Leaf(ty, self.non_ws_indexes[0]);
                let (_, rest) = self.bump();
                return Some((node, rest));
            }
        }
        None
    }

    fn current_text(&self) -> Option<&'a str> {
        match self.current() {
            Some(token) => Some(&self.text[..token.len.as_u32() as usize]),
            None => None,
        }
    }
}

impl Node {
    pub fn push_child(&mut self, child: Node) {
        match *self {
            Node::Composite { ref mut children, .. } => children.push(child),
            Node::Leaf(..) => panic!("Can't add children to a leaf node"),
        }
    }

    pub fn debug(&self, tokens: &TokenSequence) -> String {
        let (l, r) = match (self.left_idx(), self.right_idx()) {
            (Some(l), Some(r)) => (l, r),
            _ => return "EMPTY-NODE".to_owned()
        };
        let mut result = String::new();
        let mut start = tokens.original_tokens[..l].iter().map(|t| t.len).sum::<TextUnit>();
        for t in tokens.original_tokens[l..r].iter() {
            result += &tokens.text[TextRange::from_len(start, t.len)];
            start += t.len;
        }
        result
    }

    fn right_idx(&self) -> Option<usize> {
        match *self {
            Node::Leaf(_, idx) => Some(idx),
            Node::Composite { ref children, .. } => children.iter().rev().filter_map(|n| {
                n.right_idx()
            }).next(),
        }
    }

    fn left_idx(&self) -> Option<usize> {
        match *self {
            Node::Leaf(_, idx) => Some(idx),
            Node::Composite { ref children, .. } => match children.first() {
                None => None,
                Some(child) => child.left_idx()
            },
        }
    }
}

fn is_ws(lang: &Language, token: Token) -> bool {
    lang.node_type_info(token.ty).whitespace_like
}

pub fn parse(
    text: &str,
    lang: &Language,
    tokenizer: &[LexRule],
    parser: &Fn(TokenSequence, &mut FileStats) -> Node
) -> (FileStats, INode) {
    let mut stats = FileStats::new();
    let (lex_time, owned_tokens) = measure_time(|| tokenize(&text, tokenizer).collect::<Vec<_>>());
    stats.lexing_time = lex_time.duration();
    stats.reparsed_region = TextRange::from_to(TextUnit::zero(), TextUnit::from_usize(text.len()));
    let non_ws_indexes: Vec<usize> = owned_tokens.iter().enumerate().filter_map(|(i, &t)| {
        if is_ws(lang, t) { None } else { Some(i) }
    }).collect();

    let ws_len = owned_tokens.iter()
        .take_while(|&&t| is_ws(lang, t))
        .map(|t| t.len.as_u32() as usize)
        .sum();
    let (parse_time, node) = {
        let tokens = TokenSequence {
            text: &text[ws_len..],
            start: 0,
            non_ws_indexes: &non_ws_indexes,
            original_tokens: &owned_tokens,
        };
        measure_time(|| parser(tokens, &mut stats))
    };
    stats.parsing_time = parse_time.duration();

    let ws_node = to_ws_node(lang, node, &owned_tokens);
    let inode = ws_node.into_inode().unwrap();
    (stats, inode)
}

#[derive(Debug)]
struct WsNode {
    ty: Option<NodeType>,
    len: TextUnit,
    children: Vec<WsNode>,
    first: Option<usize>,
    last: Option<usize>
}

impl WsNode {
    fn push_child(&mut self, lang: &Language, child: WsNode, tokens: &[Token]) {
        match (self.last, child.first) {
            (Some(l), Some(r)) if l + 1 < r => {
                for idx in l + 1..r {
                    let t = tokens[idx];
                    assert!(is_ws(lang, t), "expected whitespace, got {:?}", t.ty);
                    self.push_child_raw(token_ws_node(idx, t))
                }
            }
            _ => {}
        }
        self.push_child_raw(child)
    }

    fn push_child_raw(&mut self, child: WsNode) {
        self.last = child.last.or(self.last);
        self.first = self.first.or(child.first);
        self.len += child.len;
        self.children.push(child);
    }

    fn attach_to_inode(self, parent: &mut INode) {
        if self.children.is_empty() {
            parent.push_child(INode::new_leaf(self.ty.unwrap(), self.len));
            return;
        }
        match self.into_inode() {
            Ok(node) => parent.push_child(node),
            Err(this) => {
                for child in this.children {
                    child.attach_to_inode(parent);
                }
            }
        }
    }

    fn into_inode(self) -> Result<INode, WsNode> {
        if let Some(ty) = self.ty {
            let mut node = INode::new(ty);
            for child in self.children {
                child.attach_to_inode(&mut node);
            }
            Ok(node)
        } else {
            Err(self)
        }
    }
}

fn token_ws_node(idx: usize, t: Token) -> WsNode {
    WsNode {
        ty: Some(t.ty),
        len: t.len,
        children: Vec::new(),
        first: Some(idx),
        last: Some(idx),
    }
}

fn to_ws_node(lang: &Language, file_node: Node, tokens: &[Token]) -> WsNode {
    let (ty, children) = match file_node {
        Node::Composite { ty, children, .. } => (ty.unwrap(), children),
        _ => panic!("Root node must be composite")
    };
    let mut result = WsNode {
        ty: Some(ty),
        len: TextUnit::zero(),
        children: Vec::new(),
        first: None,
        last: None,
    };

    for (i, &t) in tokens.iter().enumerate() {
        if !is_ws(lang, t) {
            break
        }
        result.push_child_raw(token_ws_node(i, t))
    }

    for child in children {
        add_child(lang, &mut result, &child, tokens)
    }
    if let Some(idx) = result.last {
        for idx in idx + 1..tokens.len() {
            let t = tokens[idx];
            assert!(is_ws(lang, t));
            result.push_child_raw(token_ws_node(idx, t))
        }
    }
    result
}

fn add_child(lang: &Language, parent: &mut WsNode, node: &Node, tokens: &[Token]) {
    match *node {
        Node::Leaf(ty, idx) => {
            let mut node = token_ws_node(idx, tokens[idx]);
            node.ty = Some(ty);
            parent.push_child(lang, node, tokens)
        }
        Node::Composite { ty, ref children } => {
            let mut p = WsNode {
                ty: ty,
                len: TextUnit::zero(),
                children: Vec::new(),
                first: None,
                last: None,
            };
            for child in children {
                add_child(lang, &mut p, child, tokens);
            }
            parent.push_child(lang, p, tokens)
        }
    }
}
<commit_msg>Refactor<commit_after>use elapsed::measure_time;

use fall_tree::{NodeType, ERROR, TextRange, FileStats, INode, TextUnit, Language};
use lex::{Token, LexRule, tokenize};

#[derive(Clone, Copy, Debug)]
pub struct TokenSequence<'a> {
    text: &'a str,
    start: usize,
    non_ws_indexes: &'a [usize],
    original_tokens: &'a [Token],
}

#[derive(Debug)]
pub enum Node {
    Leaf {
        ty: NodeType,
        token_idx: usize
    },
    Composite {
        ty: Option<NodeType>,
        children: Vec<Node>,
    }
}

impl Node {
    pub fn error() -> Node {
        Self::composite(Some(ERROR))
    }

    pub fn composite(ty: Option<NodeType>) -> Node {
        Node::Composite { ty: ty, children: Vec::new() }
    }

    pub fn success<'t>(ts: TokenSequence<'t>) -> (Node, TokenSequence<'t>) {
        (Self::composite(None), ts)
    }
}

impl<'a> TokenSequence<'a> {
    pub fn prefix(&self, suffix: TokenSequence<'a>) -> TokenSequence<'a> {
        TokenSequence {
            text: self.text,
            start: self.start,
            non_ws_indexes: &self.non_ws_indexes[..suffix.start - self.start],
            original_tokens: self.original_tokens
        }
    }

    pub fn current(&self) -> Option<Token> {
        self.non_ws_indexes.first().map(|&idx| {
            self.original_tokens[idx]
        })
    }

    pub fn bump(&self) -> (Node, TokenSequence<'a>) {
        let token = self.current().expect("Can't bump an empty token sequence");
        let node = Node::Leaf { ty: token.ty, token_idx: self.non_ws_indexes[0] };
        let mut text_len = 0;
        let next_idx = self.non_ws_indexes.get(1).map(|&i| i).unwrap_or(self.original_tokens.len());
        for i in self.non_ws_indexes[0]..next_idx {
            text_len += self.original_tokens[i].len.as_u32() as usize;
        }
        let rest = TokenSequence {
            text: &self.text[text_len..],
            start: self.start + 1,
            non_ws_indexes: &self.non_ws_indexes[1..],
            original_tokens: self.original_tokens,
        };
        (node, rest)
    }

    pub fn bump_by_text(&self, ty: NodeType, text: &str) -> Option<(Node, TokenSequence<'a>)> {
        if let Some(t) = self.current_text() {
            if t == text {
                let node = Node::Leaf { ty: ty, token_idx: self.non_ws_indexes[0] };
                let (_, rest) = self.bump();
                return Some((node, rest));
            }
        }
        None
    }

    fn current_text(&self) -> Option<&'a str> {
        match self.current() {
            Some(token) => Some(&self.text[..token.len.as_u32() as usize]),
            None => None,
        }
    }
}

impl Node {
    pub fn push_child(&mut self, child: Node) {
        match *self {
            Node::Composite { ref mut children, .. } => children.push(child),
            Node::Leaf { .. } => panic!("Can't add children to a leaf node"),
        }
    }

    pub fn debug(&self, tokens: &TokenSequence) -> String {
        let (l, r) = match (self.left_idx(), self.right_idx()) {
            (Some(l), Some(r)) => (l, r),
            _ => return "EMPTY-NODE".to_owned()
        };
        let mut result = String::new();
        let mut start = tokens.original_tokens[..l].iter().map(|t| t.len).sum::<TextUnit>();
        for t in tokens.original_tokens[l..r].iter() {
            result += &tokens.text[TextRange::from_len(start, t.len)];
            start += t.len;
        }
        result
    }

    fn right_idx(&self) -> Option<usize> {
        match *self {
            Node::Leaf { token_idx, .. } => Some(token_idx),
            Node::Composite { ref children, .. } => children.iter().rev().filter_map(|n| {
                n.right_idx()
            }).next(),
        }
    }

    fn left_idx(&self) -> Option<usize> {
        match *self {
            Node::Leaf { token_idx, .. } => Some(token_idx),
            Node::Composite { ref children, .. } => match children.first() {
                None => None,
                Some(child) => child.left_idx()
            },
        }
    }
}

fn is_ws(lang: &Language, token: Token) -> bool {
    lang.node_type_info(token.ty).whitespace_like
}

pub fn parse(
    text: &str,
    lang: &Language,
    tokenizer: &[LexRule],
    parser: &Fn(TokenSequence, &mut FileStats) -> Node
) -> (FileStats, INode) {
    let mut stats = FileStats::new();
    let (lex_time, owned_tokens) = measure_time(|| tokenize(&text, tokenizer).collect::<Vec<_>>());
    stats.lexing_time = lex_time.duration();
    stats.reparsed_region = TextRange::from_to(TextUnit::zero(), TextUnit::from_usize(text.len()));
    let non_ws_indexes: Vec<usize> = owned_tokens.iter().enumerate().filter_map(|(i, &t)| {
        if is_ws(lang, t) { None } else { Some(i) }
    }).collect();

    let ws_len = owned_tokens.iter()
        .take_while(|&&t| is_ws(lang, t))
        .map(|t| t.len.as_u32() as usize)
        .sum();
    let (parse_time, node) = {
        let tokens = TokenSequence {
            text: &text[ws_len..],
            start: 0,
            non_ws_indexes: &non_ws_indexes,
            original_tokens: &owned_tokens,
        };
        measure_time(|| parser(tokens, &mut stats))
    };
    stats.parsing_time = parse_time.duration();

    let ws_node = to_ws_node(lang, node, &owned_tokens);
    let inode = ws_node.into_inode().unwrap();
    (stats, inode)
}

#[derive(Debug)]
struct WsNode {
    ty: Option<NodeType>,
    len: TextUnit,
    children: Vec<WsNode>,
    first: Option<usize>,
    last: Option<usize>
}

impl WsNode {
    fn push_child(&mut self, lang: &Language, child: WsNode, tokens: &[Token]) {
        match (self.last, child.first) {
            (Some(l), Some(r)) if l + 1 < r => {
                for idx in l + 1..r {
                    let t = tokens[idx];
                    assert!(is_ws(lang, t), "expected whitespace, got {:?}", t.ty);
                    self.push_child_raw(token_ws_node(idx, t))
                }
            }
            _ => {}
        }
        self.push_child_raw(child)
    }

    fn push_child_raw(&mut self, child: WsNode) {
        self.last = child.last.or(self.last);
        self.first = self.first.or(child.first);
        self.len += child.len;
        self.children.push(child);
    }

    fn attach_to_inode(self, parent: &mut INode) {
        if self.children.is_empty() {
            parent.push_child(INode::new_leaf(self.ty.unwrap(), self.len));
            return;
        }
        match self.into_inode() {
            Ok(node) => parent.push_child(node),
            Err(this) => {
                for child in this.children {
                    child.attach_to_inode(parent);
                }
            }
        }
    }

    fn into_inode(self) -> Result<INode, WsNode> {
        if let Some(ty) = self.ty {
            let mut node = INode::new(ty);
            for child in self.children {
                child.attach_to_inode(&mut node);
            }
            Ok(node)
        } else {
            Err(self)
        }
    }
}

fn token_ws_node(idx: usize, t: Token) -> WsNode {
    WsNode {
        ty: Some(t.ty),
        len: t.len,
        children: Vec::new(),
        first: Some(idx),
        last: Some(idx),
    }
}

fn to_ws_node(lang: &Language, file_node: Node, tokens: &[Token]) -> WsNode {
    let (ty, children) = match file_node {
        Node::Composite { ty, children, .. } => (ty.unwrap(), children),
        _ => panic!("Root node must be composite")
    };
    let mut result = WsNode {
        ty: Some(ty),
        len: TextUnit::zero(),
        children: Vec::new(),
        first: None,
        last: None,
    };

    for (i, &t) in tokens.iter().enumerate() {
        if !is_ws(lang, t) {
            break
        }
        result.push_child_raw(token_ws_node(i, t))
    }

    for child in children {
        add_child(lang, &mut result, &child, tokens)
    }
    if let Some(idx) = result.last {
        for idx in idx + 1..tokens.len() {
            let t = tokens[idx];
            assert!(is_ws(lang, t));
            result.push_child_raw(token_ws_node(idx, t))
        }
    }
    result
}

fn add_child(lang: &Language, parent: &mut WsNode, node: &Node, tokens: &[Token]) {
    match *node {
        Node::Leaf { ty, token_idx } => {
            let mut node = token_ws_node(token_idx, tokens[token_idx]);
            node.ty = Some(ty);
            parent.push_child(lang, node, tokens)
        }
        Node::Composite { ty, ref children } => {
            let mut p = WsNode {
                ty: ty,
                len: TextUnit::zero(),
                children: Vec::new(),
                first: None,
                last: None,
            };
            for child in children {
                add_child(lang, &mut p, child, tokens);
            }
            parent.push_child(lang, p, tokens)
        }
    }
}
<|endoftext|>


<commit_before>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

//! BookmarkCollection module
//!
//! A BookmarkCollection is nothing more than a simple store entry. One can simply call functions
//! from the libimagentrylink::external::ExternalLinker trait on this to generate external links.
//!
//! The BookmarkCollection type offers helper functions to get all links or such things.
use std::ops::Deref;
use std::ops::DerefMut;

use regex::Regex;

use error::BookmarkErrorKind as BEK;
use error::MapErrInto;
use result::Result;
use module_path::ModuleEntryPath;

use libimagstore::store::Store;
use libimagstore::storeid::IntoStoreId;
use libimagstore::store::FileLockEntry;
use libimagentrylink::external::ExternalLinker;
use libimagentrylink::external::iter::UrlIter;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::internal::Link as StoreLink;
use libimagerror::into::IntoError;
use url::Url;

use link::Link;

pub struct BookmarkCollection<'a> {
    fle: FileLockEntry<'a>,
    store: &'a Store,
}

/// {Internal, External}Linker is implemented as Deref is implemented
impl<'a> Deref for BookmarkCollection<'a> {
    type Target = FileLockEntry<'a>;

    fn deref(&self) -> &FileLockEntry<'a> {
        &self.fle
    }

}

impl<'a> DerefMut for BookmarkCollection<'a> {

    fn deref_mut(&mut self) -> &mut FileLockEntry<'a> {
        &mut self.fle
    }

}

impl<'a> BookmarkCollection<'a> {

    pub fn new(store: &'a Store, name: &str) -> Result<BookmarkCollection<'a>> {
        ModuleEntryPath::new(name)
            .into_storeid()
            .and_then(|id| store.create(id))
            .map(|fle| {
                BookmarkCollection {
                    fle: fle,
                    store: store,
                }
            })
            .map_err_into(BEK::StoreReadError)
    }

    pub fn get(store: &'a Store, name: &str) -> Result<BookmarkCollection<'a>> {
        ModuleEntryPath::new(name)
            .into_storeid()
            .and_then(|id| store.get(id))
            .map_err_into(BEK::StoreReadError)
            .and_then(|fle| {
                match fle {
                    None => Err(BEK::CollectionNotFound.into_error()),
                    Some(e) => Ok(BookmarkCollection {
                        fle: e,
                        store: store,
                    }),
                }
            })
    }

    pub fn delete(store: &Store, name: &str) -> Result<()> {
        ModuleEntryPath::new(name)
            .into_storeid()
            .and_then(|id| store.delete(id))
            .map_err_into(BEK::StoreReadError)
    }

    pub fn links(&self) -> Result<UrlIter> {
        self.fle.get_external_links(&self.store).map_err_into(BEK::LinkError)
    }

    pub fn link_entries(&self) -> Result<Vec<StoreLink>> {
        use libimagentrylink::external::is_external_link_storeid;

        self.fle
            .get_internal_links()
            .map(|v| v.filter(|id| is_external_link_storeid(id)).collect())
            .map_err_into(BEK::StoreReadError)
    }

    pub fn add_link(&mut self, l: Link) -> Result<()> {
        use link::IntoUrl;

        l.into_url()
            .and_then(|url| self.add_external_link(self.store, url).map_err_into(BEK::LinkingError))
            .map_err_into(BEK::LinkError)
    }

    pub fn get_links_matching(&self, r: Regex) -> Result<UrlIter> {
        use std::result::Result as RResult;

        self.get_external_links(self.store)
            .map_err_into(BEK::LinkError)
            .map(|v| {
                v.filter_map(RResult::ok) // TODO: Do not ignore errors here
                    .map(Url::into_string)
                    .filter(|urlstr| r.is_match(&urlstr[..]))
                    .map(Link::from)
                    .collect()
            })
    }

    pub fn remove_link(&mut self, l: Link) -> Result<()> {
        use link::IntoUrl;

        l.into_url()
            .and_then(|url| {
                self.remove_external_link(self.store, url).map_err_into(BEK::LinkingError)
            })
            .map_err_into(BEK::LinkError)
    }

}

<commit_msg>Rewrite BookmarkCollection::get_links_matching() to return Iterator<commit_after>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

//! BookmarkCollection module
//!
//! A BookmarkCollection is nothing more than a simple store entry. One can simply call functions
//! from the libimagentrylink::external::ExternalLinker trait on this to generate external links.
//!
//! The BookmarkCollection type offers helper functions to get all links or such things.
use std::ops::Deref;
use std::ops::DerefMut;

use regex::Regex;

use error::BookmarkErrorKind as BEK;
use error::MapErrInto;
use result::Result;
use module_path::ModuleEntryPath;

use libimagstore::store::Store;
use libimagstore::storeid::IntoStoreId;
use libimagstore::store::FileLockEntry;
use libimagentrylink::external::ExternalLinker;
use libimagentrylink::external::iter::UrlIter;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::internal::Link as StoreLink;
use libimagerror::into::IntoError;

use link::Link;

use self::iter::LinksMatchingRegexIter;

pub struct BookmarkCollection<'a> {
    fle: FileLockEntry<'a>,
    store: &'a Store,
}

/// {Internal, External}Linker is implemented as Deref is implemented
impl<'a> Deref for BookmarkCollection<'a> {
    type Target = FileLockEntry<'a>;

    fn deref(&self) -> &FileLockEntry<'a> {
        &self.fle
    }

}

impl<'a> DerefMut for BookmarkCollection<'a> {

    fn deref_mut(&mut self) -> &mut FileLockEntry<'a> {
        &mut self.fle
    }

}

impl<'a> BookmarkCollection<'a> {

    pub fn new(store: &'a Store, name: &str) -> Result<BookmarkCollection<'a>> {
        ModuleEntryPath::new(name)
            .into_storeid()
            .and_then(|id| store.create(id))
            .map(|fle| {
                BookmarkCollection {
                    fle: fle,
                    store: store,
                }
            })
            .map_err_into(BEK::StoreReadError)
    }

    pub fn get(store: &'a Store, name: &str) -> Result<BookmarkCollection<'a>> {
        ModuleEntryPath::new(name)
            .into_storeid()
            .and_then(|id| store.get(id))
            .map_err_into(BEK::StoreReadError)
            .and_then(|fle| {
                match fle {
                    None => Err(BEK::CollectionNotFound.into_error()),
                    Some(e) => Ok(BookmarkCollection {
                        fle: e,
                        store: store,
                    }),
                }
            })
    }

    pub fn delete(store: &Store, name: &str) -> Result<()> {
        ModuleEntryPath::new(name)
            .into_storeid()
            .and_then(|id| store.delete(id))
            .map_err_into(BEK::StoreReadError)
    }

    pub fn links(&self) -> Result<UrlIter> {
        self.fle.get_external_links(&self.store).map_err_into(BEK::LinkError)
    }

    pub fn link_entries(&self) -> Result<Vec<StoreLink>> {
        use libimagentrylink::external::is_external_link_storeid;

        self.fle
            .get_internal_links()
            .map(|v| v.filter(|id| is_external_link_storeid(id)).collect())
            .map_err_into(BEK::StoreReadError)
    }

    pub fn add_link(&mut self, l: Link) -> Result<()> {
        use link::IntoUrl;

        l.into_url()
            .and_then(|url| self.add_external_link(self.store, url).map_err_into(BEK::LinkingError))
            .map_err_into(BEK::LinkError)
    }

    pub fn get_links_matching(&self, r: Regex) -> Result<LinksMatchingRegexIter<'a>> {
        use self::iter::IntoLinksMatchingRegexIter;

        self.get_external_links(self.store)
            .map_err_into(BEK::LinkError)
            .map(|iter| iter.matching_regex(r))
    }

    pub fn remove_link(&mut self, l: Link) -> Result<()> {
        use link::IntoUrl;

        l.into_url()
            .and_then(|url| {
                self.remove_external_link(self.store, url).map_err_into(BEK::LinkingError)
            })
            .map_err_into(BEK::LinkError)
    }

}

pub mod iter {
    use link::Link;
    use result::Result;
    use error::{MapErrInto, BookmarkErrorKind as BEK};

    pub struct LinkIter<I>(I)
        where I: Iterator<Item = Link>;

    impl<I: Iterator<Item = Link>> LinkIter<I> {
        pub fn new(i: I) -> LinkIter<I> {
            LinkIter(i)
        }
    }

    impl<I: Iterator<Item = Link>> Iterator for LinkIter<I> {
        type Item = Link;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next()
        }
    }

    impl<I> From<I> for LinkIter<I> where I: Iterator<Item = Link> {
        fn from(i: I) -> LinkIter<I> {
            LinkIter(i)
        }
    }

    use libimagentrylink::external::iter::UrlIter;
    use regex::Regex;

    pub struct LinksMatchingRegexIter<'a>(UrlIter<'a>, Regex);

    impl<'a> LinksMatchingRegexIter<'a> {
        pub fn new(i: UrlIter<'a>, r: Regex) -> LinksMatchingRegexIter<'a> {
            LinksMatchingRegexIter(i, r)
        }
    }

    impl<'a> Iterator for LinksMatchingRegexIter<'a> {
        type Item = Result<Link>;

        fn next(&mut self) -> Option<Self::Item> {
            loop {
                let n = match self.0.next() {
                    Some(Ok(n))  => n,
                    Some(Err(e)) => return Some(Err(e).map_err_into(BEK::LinkError)),
                    None         => return None,
                };

                let s = n.into_string();
                if self.1.is_match(&s[..]) {
                    return Some(Ok(Link::from(s)))
                } else {
                    continue;
                }
            }
        }
    }

    pub trait IntoLinksMatchingRegexIter<'a> {
        fn matching_regex(self, Regex) -> LinksMatchingRegexIter<'a>;
    }

    impl<'a> IntoLinksMatchingRegexIter<'a> for UrlIter<'a> {
        fn matching_regex(self, r: Regex) -> LinksMatchingRegexIter<'a> {
            LinksMatchingRegexIter(self, r)
        }
    }

}

<|endoftext|>


<commit_before>//!Bindings and rewrites of include/linux/time.h and include/linux/time64.h

extern crate linux_api;

use linux_api::*;

///Checks to see if two timespecs are equal
pub fn timespec_equal(a: &timespec, b: &timespec) -> c_int {
    return ((a.tv_sec == b.tv_sec) && (a.tv_nsec == b.tv_nsec)) as c_int;
}

///Compares two timespecs
///
///If lhs < rhs, return < 0
///If lhs == rhs, return 0
///If lhs > rhs, return > 0
pub fn timespec_compare(lhs: &timespec, rhs: &timespec) -> c_int {
    if lhs.tv_sec < rhs.tv_sec { return -1; }
    if lhs.tv_sec > rhs.tv_sec { return 1; }
    return (lhs.tv_nsec - rhs.tv_nsec) as c_int;
}

///Compares two timevals
///
///If lhs < rhs, return < 0
///If lhs == rhs, return 0
///If lhs > rhs, return > 0
pub fn timeval_compare(lhs: &timeval, rhs: &timeval) -> c_int {
    if lhs.tv_sec < rhs.tv_sec { return -1; }
    if lhs.tv_sec > rhs.tv_sec { return 1; }
    return (lhs.tv_usec - rhs.tv_usec) as c_int;
}

//TODO: mktime64
//TODO: mktime

///Sets the data contained in the borrowed `timespec` ts to normalized values
pub fn set_normalized_timespec(ts: &mut timespec, sec: time_t, nsec: i64) {
    let mut realsec = sec;
    let mut realnsec = nsec;

    while realnsec >= NSEC_PER_SEC {
        //TODO: Check to see if this gets optimized and, if so, force it not to as per kernel definition
        realnsec -= NSEC_PER_SEC;
        realsec += 1;
    }

    while realnsec < 0 {
        //Ditto
        realnsec += NSEC_PER_SEC;
        realsec -= 1;
    }

    ts.tv_sec = realsec;
    ts.tv_nsec = realnsec;
}

//TODO: timespec_add_safe, timespec_sub

///Adds two timespecs together, returning the result as a new timespec
pub fn timespec_add(lhs: &timespec, rhs: &timespec) -> timespec {
    let mut ts_delta: timespec = timespec { tv_sec:0, tv_nsec: 0 };
    set_normalized_timespec(&mut ts_delta, lhs.tv_sec + rhs.tv_sec, lhs.tv_nsec + rhs.tv_nsec);
    ts_delta
}

///Subtracts rhs from lhs, returning the difference as a new timespec
pub fn timespec_sub(lhs: &timespec, rhs: &timespec) -> timespec {
    let mut ts_delta: timespec = timespec { tv_sec: 0, tv_nsec: 0 };
    set_normalized_timespec(&mut ts_delta, lhs.tv_sec - rhs.tv_sec, lhs.tv_nsec - rhs.tv_nsec);
    ts_delta
}

///Checks to see if a timespec is valid
pub fn timespec_valid(ts: &timespec) -> bool {
    //Dates before 1970 are bogus
    if ts.tv_sec < 0 {
        return false;
    }

    //Can't have more nanoseconds than a second
    if ts.tv_nsec >= NSEC_PER_SEC {
        return false;
    }

    //Okay, all looks good
    return true;
}

///Checks to see if a timespec is valid according to stricter rules
pub fn timespec_valid_strict(ts: &timespec) -> bool {
    if !timespec_valid(ts) {
        return false;
    }

    //Okay, we've validated it quickly
    //Disallow values that could overflow ktime_t
    if ts.tv_sec >= KTIME_SEC_MAX {
        return false;
    }

    return true;
}

///Checks to see if a timeval is valid
pub fn timeval_valid(tv: &timeval) -> bool {
    //Dates before 1970 are bogus
    if tv.tv_sec < 0 {
        return false;
    }

    //Can't have more microseconds than a second
    if tv.tv_usec >= USEC_PER_SEC {
        return false;
    }

    //Okay, all looks good
    return true;
}

///Creates a time64_t from the supplied year, month, day, hour, minute, and second
pub fn mktime64(year0: c_uint, mon0: c_uint, day: c_uint, hour: c_uint, min: c_uint, sec: c_uint) -> time64_t {

    let mut mon = mon0;
    let mut year = year0;

    //The kernel tells us to do this. It's basically treating Jan and Feb as months 11 and 12, then pulling the others up by two. Leap year, perhaps?

    let mut _mon = mon as c_int;

    _mon -= 2;

    if _mon <= 0 {
        mon += 10; //Because we didn't take the two off)
        year -= 1;
    } else {
        mon -= 2; //Because we can
    }

    //I'm trusting you here, kernel developers! This is ugly but it had better work

    let mut result: time64_t = 0;

    result = (year/4 - year/100 + year/400 + 367*mon/12 + day) as i64;
    result += year as i64 * 365;
    result -= 719499;
    result *= 24;
    result += hour as i64;
    result *= 60;
    result += min as i64;
    result *= 60;
    result += sec as i64;

    //result += (((((year / 4) - (year / 100) + (year / 400) + (367*mon/12) + day + (year * 365)) * 24 + hour) * 60 + min) * 60 + sec) as time64_t;

    return result;

}

#[cfg(test)]
mod test {
    use super::*;
    use linux_api::*;

    #[test]
    fn test_timespec_valid() {
        let atime: timespec = timespec { tv_sec: 0, tv_nsec: 4262 };

        assert!(timespec_valid(&atime));
    }

    #[test]
    fn test_timespec_too_many_seconds_valid() {
        let atime: timespec = timespec { tv_sec: KTIME_SEC_MAX + 365, tv_nsec: 4262 };

        assert!(!timespec_valid_strict(&atime));
    }

    #[test]
    fn test_timespec_before_1970_invalid() {
        let atime: timespec = timespec { tv_sec: -63, tv_nsec: 4262 };

        assert!(!timespec_valid(&atime));
    }

    #[test]
    fn test_timespec_too_many_nanoseconds_invalid() {
        let atime: timespec = timespec { tv_sec: 0, tv_nsec: NSEC_PER_SEC + 2 };

        assert!(!timespec_valid(&atime));
    }

    #[test]
    fn test_timeval_valid() {
        let atime: timeval = timeval { tv_sec: 0, tv_usec: 4262 };

        assert!(timeval_valid(&atime));
    }

    #[test]
    fn test_timeval_before_1970_invalid() {
        let atime: timeval = timeval { tv_sec: -63, tv_usec: 4262 };

        assert!(!timeval_valid(&atime));
    }

    #[test]
    fn test_timeval_too_many_nanoseconds_invalid() {
        let atime: timeval = timeval { tv_sec: 0, tv_usec: USEC_PER_SEC + 2 };

        assert!(!timeval_valid(&atime));
    }

    #[test]
    fn test_timespec_equal_when_equal() {
        let one: timespec = timespec { tv_sec: 77, tv_nsec: 77 };
        let two: timespec = timespec { tv_sec: 77, tv_nsec: 77 };

        assert_eq!(1, timespec_equal(&one, &two));
    }

    #[test]
    fn test_timespec_equal_when_inequal() {
        let one: timespec = timespec { tv_sec: 0, tv_nsec: 266 };
        let two: timespec = timespec { tv_sec: 4, tv_nsec: 0 };

        assert_eq!(0, timespec_equal(&one, &two));
    }

    #[test]
    fn test_timespec_compare_same() {
        let one = timespec { tv_sec: 10, tv_nsec: 0 };
        let two = timespec { tv_sec: 10, tv_nsec: 0 };

        assert_eq!(0, timespec_compare(&one, &two));
    }

    #[test]
    fn test_timespec_compare_larger() {
        let one = timespec { tv_sec: 11, tv_nsec: 0 };
        let two = timespec { tv_sec: 10, tv_nsec: 0 };

        assert_eq!(1, timespec_compare(&one, &two));
    }

    #[test]
    fn test_timespec_compare_smaller() {
        let one = timespec { tv_sec: 9, tv_nsec: 0 };
        let two = timespec { tv_sec: 10, tv_nsec: 0 };

        assert_eq!(-1, timespec_compare(&one, &two));
    }

    #[test]
    fn test_timeval_compare_same() {
        let one = timeval { tv_sec: 10, tv_usec: 0 };
        let two = timeval { tv_sec: 10, tv_usec: 0 };

        assert_eq!(0, timeval_compare(&one, &two));
    }

    #[test]
    fn test_timeval_compare_larger() {
        let one = timeval { tv_sec: 11, tv_usec: 0 };
        let two = timeval { tv_sec: 10, tv_usec: 0 };

        assert_eq!(1, timeval_compare(&one, &two));
    }

    #[test]
    fn test_timeval_compare_smaller() {
        let one = timeval { tv_sec: 9, tv_usec: 0 };
        let two = timeval { tv_sec: 10, tv_usec: 0 };

        assert_eq!(-1, timeval_compare(&one, &two));
    }

    #[test]
    fn test_normalized_timespec_many_nanos() {
        let mut spec: timespec = timespec { tv_sec: 0, tv_nsec: 0 };

        set_normalized_timespec(&mut spec, 10, NSEC_PER_SEC * 20);

        assert_eq!(30, spec.tv_sec);
        assert_eq!(0, spec.tv_nsec);
    }

    #[test]
    fn test_normalized_timespec_negative_nanos() {
        let mut spec: timespec = timespec { tv_sec: 0, tv_nsec: 0 };

        set_normalized_timespec(&mut spec, 5, -800);

        assert_eq!(4, spec.tv_sec);
        assert_eq!(NSEC_PER_SEC - 800, spec.tv_nsec);
    }

    #[test]
    fn test_timespec_add_simple() {
        let lhs = timespec { tv_sec: 80, tv_nsec: 500 };
        let rhs = timespec { tv_sec: 30, tv_nsec: 580 };

        let delta = timespec_add(&lhs, &rhs);

        assert_eq!(110, delta.tv_sec);
        assert_eq!(1080, delta.tv_nsec);
    }

    #[test]
    fn test_timespec_add_manynanos() {
        let lhs = timespec { tv_sec: 80, tv_nsec: NSEC_PER_SEC / 2 };
        let rhs = timespec { tv_sec: 30, tv_nsec: (NSEC_PER_SEC / 2) + 400 };

        let delta = timespec_add(&lhs, &rhs);

        assert_eq!(111, delta.tv_sec);
        assert_eq!(400, delta.tv_nsec);
    }

    #[test]
    fn test_timespec_sub_simple() {
        let lhs = timespec { tv_sec: 80, tv_nsec: 500 };
        let rhs = timespec { tv_sec: 30, tv_nsec: 580 };

        let delta = timespec_sub(&lhs, &rhs);

        assert_eq!(49, delta.tv_sec);
        assert_eq!(NSEC_PER_SEC - 80, delta.tv_nsec);
    }

    #[test]
    fn test_timespec_sub_bad() {
        let lhs = timespec { tv_sec: 80, tv_nsec: 500 };
        let rhs = timespec { tv_sec: 1030, tv_nsec: 580 };

        let delta = timespec_sub(&lhs, &rhs);

        assert_eq!(-951, delta.tv_sec);
        assert_eq!(NSEC_PER_SEC - 80, delta.tv_nsec);
    }

    #[test]
    fn test_mktime64_known_dates() {
        let apocalypse2012: time64_t = 1356048000;
        let sequential: time64_t = 1234567890;

        assert_eq!(apocalypse2012, mktime64(2012, 12, 21, 00, 0, 0));
        assert_eq!(sequential, mktime64(2009, 2, 13, 23, 31, 30));
    }

}<commit_msg>Add and test FFI functions time() and difftime()<commit_after>//!Bindings and rewrites of include/linux/time.h and include/linux/time64.h

extern crate linux_api;

use linux_api::*;

#[link(name="rt")]
extern "C" {
    //Dummy section to link RT, but probably shouldn't link everything to it
}

//First, we do external functions
extern "C" {

    ///Gets the current time as seconds from the UNIX epoch
    pub fn time(t: *mut time_t) -> time_t;

    pub fn difftime(time1: time_t, time0: time_t) -> c_double;

}

///Checks to see if two timespecs are equal
pub fn timespec_equal(a: &timespec, b: &timespec) -> c_int {
    return ((a.tv_sec == b.tv_sec) && (a.tv_nsec == b.tv_nsec)) as c_int;
}

///Compares two timespecs
///
///If lhs < rhs, return < 0
///If lhs == rhs, return 0
///If lhs > rhs, return > 0
pub fn timespec_compare(lhs: &timespec, rhs: &timespec) -> c_int {
    if lhs.tv_sec < rhs.tv_sec { return -1; }
    if lhs.tv_sec > rhs.tv_sec { return 1; }
    return (lhs.tv_nsec - rhs.tv_nsec) as c_int;
}

///Compares two timevals
///
///If lhs < rhs, return < 0
///If lhs == rhs, return 0
///If lhs > rhs, return > 0
pub fn timeval_compare(lhs: &timeval, rhs: &timeval) -> c_int {
    if lhs.tv_sec < rhs.tv_sec { return -1; }
    if lhs.tv_sec > rhs.tv_sec { return 1; }
    return (lhs.tv_usec - rhs.tv_usec) as c_int;
}

//TODO: mktime64
//TODO: mktime

///Sets the data contained in the borrowed `timespec` ts to normalized values
pub fn set_normalized_timespec(ts: &mut timespec, sec: time_t, nsec: i64) {
    let mut realsec = sec;
    let mut realnsec = nsec;

    while realnsec >= NSEC_PER_SEC {
        //TODO: Check to see if this gets optimized and, if so, force it not to as per kernel definition
        realnsec -= NSEC_PER_SEC;
        realsec += 1;
    }

    while realnsec < 0 {
        //Ditto
        realnsec += NSEC_PER_SEC;
        realsec -= 1;
    }

    ts.tv_sec = realsec;
    ts.tv_nsec = realnsec;
}

//TODO: timespec_add_safe, timespec_sub

///Adds two timespecs together, returning the result as a new timespec
pub fn timespec_add(lhs: &timespec, rhs: &timespec) -> timespec {
    let mut ts_delta: timespec = timespec { tv_sec:0, tv_nsec: 0 };
    set_normalized_timespec(&mut ts_delta, lhs.tv_sec + rhs.tv_sec, lhs.tv_nsec + rhs.tv_nsec);
    ts_delta
}

///Subtracts rhs from lhs, returning the difference as a new timespec
pub fn timespec_sub(lhs: &timespec, rhs: &timespec) -> timespec {
    let mut ts_delta: timespec = timespec { tv_sec: 0, tv_nsec: 0 };
    set_normalized_timespec(&mut ts_delta, lhs.tv_sec - rhs.tv_sec, lhs.tv_nsec - rhs.tv_nsec);
    ts_delta
}

///Checks to see if a timespec is valid
pub fn timespec_valid(ts: &timespec) -> bool {
    //Dates before 1970 are bogus
    if ts.tv_sec < 0 {
        return false;
    }

    //Can't have more nanoseconds than a second
    if ts.tv_nsec >= NSEC_PER_SEC {
        return false;
    }

    //Okay, all looks good
    return true;
}

///Checks to see if a timespec is valid according to stricter rules
pub fn timespec_valid_strict(ts: &timespec) -> bool {
    if !timespec_valid(ts) {
        return false;
    }

    //Okay, we've validated it quickly
    //Disallow values that could overflow ktime_t
    if ts.tv_sec >= KTIME_SEC_MAX {
        return false;
    }

    return true;
}

///Checks to see if a timeval is valid
pub fn timeval_valid(tv: &timeval) -> bool {
    //Dates before 1970 are bogus
    if tv.tv_sec < 0 {
        return false;
    }

    //Can't have more microseconds than a second
    if tv.tv_usec >= USEC_PER_SEC {
        return false;
    }

    //Okay, all looks good
    return true;
}

///Creates a time64_t from the supplied year, month, day, hour, minute, and second
pub fn mktime64(year0: c_uint, mon0: c_uint, day: c_uint, hour: c_uint, min: c_uint, sec: c_uint) -> time64_t {

    let mut mon = mon0;
    let mut year = year0;

    //The kernel tells us to do this. It's basically treating Jan and Feb as months 11 and 12, then pulling the others up by two. Leap year, perhaps?

    let mut _mon = mon as c_int;

    _mon -= 2;

    if _mon <= 0 {
        mon += 10; //Because we didn't take the two off)
        year -= 1;
    } else {
        mon -= 2; //Because we can
    }

    //I'm trusting you here, kernel developers! This is ugly but it had better work

    let mut result: time64_t = 0;

    result = (year/4 - year/100 + year/400 + 367*mon/12 + day) as i64;
    result += year as i64 * 365;
    result -= 719499;
    result *= 24;
    result += hour as i64;
    result *= 60;
    result += min as i64;
    result *= 60;
    result += sec as i64;

    //result += (((((year / 4) - (year / 100) + (year / 400) + (367*mon/12) + day + (year * 365)) * 24 + hour) * 60 + min) * 60 + sec) as time64_t;

    return result;

}

#[cfg(test)]
mod test {
    use super::*;
    use linux_api::*;

    #[test]
    fn test_time_ffi() {
        let mut atime: time_t = 0;

        unsafe { atime = time(::std::ptr::null_mut()); }

        assert!(atime >= 1437309103); //Time of first testing
    }

    #[test]
    fn test_difftime_ffi() {
        let mut atime: time_t = 0;

        unsafe { atime = time(::std::ptr::null_mut()); }

        let anothertime: time_t = atime - 100;

        let mut result: c_double = 0f64;

        unsafe { result = difftime(atime, anothertime); }

        assert_eq!(100f64, result);

        unsafe { result = difftime(anothertime, atime); }

        assert_eq!(-100f64, result);
    }

    #[test]
    fn test_timespec_valid() {
        let atime: timespec = timespec { tv_sec: 0, tv_nsec: 4262 };

        assert!(timespec_valid(&atime));
    }

    #[test]
    fn test_timespec_too_many_seconds_valid() {
        let atime: timespec = timespec { tv_sec: KTIME_SEC_MAX + 365, tv_nsec: 4262 };

        assert!(!timespec_valid_strict(&atime));
    }

    #[test]
    fn test_timespec_before_1970_invalid() {
        let atime: timespec = timespec { tv_sec: -63, tv_nsec: 4262 };

        assert!(!timespec_valid(&atime));
    }

    #[test]
    fn test_timespec_too_many_nanoseconds_invalid() {
        let atime: timespec = timespec { tv_sec: 0, tv_nsec: NSEC_PER_SEC + 2 };

        assert!(!timespec_valid(&atime));
    }

    #[test]
    fn test_timeval_valid() {
        let atime: timeval = timeval { tv_sec: 0, tv_usec: 4262 };

        assert!(timeval_valid(&atime));
    }

    #[test]
    fn test_timeval_before_1970_invalid() {
        let atime: timeval = timeval { tv_sec: -63, tv_usec: 4262 };

        assert!(!timeval_valid(&atime));
    }

    #[test]
    fn test_timeval_too_many_nanoseconds_invalid() {
        let atime: timeval = timeval { tv_sec: 0, tv_usec: USEC_PER_SEC + 2 };

        assert!(!timeval_valid(&atime));
    }

    #[test]
    fn test_timespec_equal_when_equal() {
        let one: timespec = timespec { tv_sec: 77, tv_nsec: 77 };
        let two: timespec = timespec { tv_sec: 77, tv_nsec: 77 };

        assert_eq!(1, timespec_equal(&one, &two));
    }

    #[test]
    fn test_timespec_equal_when_inequal() {
        let one: timespec = timespec { tv_sec: 0, tv_nsec: 266 };
        let two: timespec = timespec { tv_sec: 4, tv_nsec: 0 };

        assert_eq!(0, timespec_equal(&one, &two));
    }

    #[test]
    fn test_timespec_compare_same() {
        let one = timespec { tv_sec: 10, tv_nsec: 0 };
        let two = timespec { tv_sec: 10, tv_nsec: 0 };

        assert_eq!(0, timespec_compare(&one, &two));
    }

    #[test]
    fn test_timespec_compare_larger() {
        let one = timespec { tv_sec: 11, tv_nsec: 0 };
        let two = timespec { tv_sec: 10, tv_nsec: 0 };

        assert_eq!(1, timespec_compare(&one, &two));
    }

    #[test]
    fn test_timespec_compare_smaller() {
        let one = timespec { tv_sec: 9, tv_nsec: 0 };
        let two = timespec { tv_sec: 10, tv_nsec: 0 };

        assert_eq!(-1, timespec_compare(&one, &two));
    }

    #[test]
    fn test_timeval_compare_same() {
        let one = timeval { tv_sec: 10, tv_usec: 0 };
        let two = timeval { tv_sec: 10, tv_usec: 0 };

        assert_eq!(0, timeval_compare(&one, &two));
    }

    #[test]
    fn test_timeval_compare_larger() {
        let one = timeval { tv_sec: 11, tv_usec: 0 };
        let two = timeval { tv_sec: 10, tv_usec: 0 };

        assert_eq!(1, timeval_compare(&one, &two));
    }

    #[test]
    fn test_timeval_compare_smaller() {
        let one = timeval { tv_sec: 9, tv_usec: 0 };
        let two = timeval { tv_sec: 10, tv_usec: 0 };

        assert_eq!(-1, timeval_compare(&one, &two));
    }

    #[test]
    fn test_normalized_timespec_many_nanos() {
        let mut spec: timespec = timespec { tv_sec: 0, tv_nsec: 0 };

        set_normalized_timespec(&mut spec, 10, NSEC_PER_SEC * 20);

        assert_eq!(30, spec.tv_sec);
        assert_eq!(0, spec.tv_nsec);
    }

    #[test]
    fn test_normalized_timespec_negative_nanos() {
        let mut spec: timespec = timespec { tv_sec: 0, tv_nsec: 0 };

        set_normalized_timespec(&mut spec, 5, -800);

        assert_eq!(4, spec.tv_sec);
        assert_eq!(NSEC_PER_SEC - 800, spec.tv_nsec);
    }

    #[test]
    fn test_timespec_add_simple() {
        let lhs = timespec { tv_sec: 80, tv_nsec: 500 };
        let rhs = timespec { tv_sec: 30, tv_nsec: 580 };

        let delta = timespec_add(&lhs, &rhs);

        assert_eq!(110, delta.tv_sec);
        assert_eq!(1080, delta.tv_nsec);
    }

    #[test]
    fn test_timespec_add_manynanos() {
        let lhs = timespec { tv_sec: 80, tv_nsec: NSEC_PER_SEC / 2 };
        let rhs = timespec { tv_sec: 30, tv_nsec: (NSEC_PER_SEC / 2) + 400 };

        let delta = timespec_add(&lhs, &rhs);

        assert_eq!(111, delta.tv_sec);
        assert_eq!(400, delta.tv_nsec);
    }

    #[test]
    fn test_timespec_sub_simple() {
        let lhs = timespec { tv_sec: 80, tv_nsec: 500 };
        let rhs = timespec { tv_sec: 30, tv_nsec: 580 };

        let delta = timespec_sub(&lhs, &rhs);

        assert_eq!(49, delta.tv_sec);
        assert_eq!(NSEC_PER_SEC - 80, delta.tv_nsec);
    }

    #[test]
    fn test_timespec_sub_bad() {
        let lhs = timespec { tv_sec: 80, tv_nsec: 500 };
        let rhs = timespec { tv_sec: 1030, tv_nsec: 580 };

        let delta = timespec_sub(&lhs, &rhs);

        assert_eq!(-951, delta.tv_sec);
        assert_eq!(NSEC_PER_SEC - 80, delta.tv_nsec);
    }

    #[test]
    fn test_mktime64_known_dates() {
        let apocalypse2012: time64_t = 1356048000;
        let sequential: time64_t = 1234567890;

        assert_eq!(apocalypse2012, mktime64(2012, 12, 21, 00, 0, 0));
        assert_eq!(sequential, mktime64(2009, 2, 13, 23, 31, 30));
    }

}<|endoftext|>


<commit_before>extern crate rmp_serde;

use std::cmp;
use std::collections::HashMap;

use commands::{Command, CommandExecutor};
use commands::pool::PoolCommand;
use errors::common::CommonError;
use errors::pool::PoolError;
use super::{
    MerkleTree,
    RemoteNode,
};
use super::rust_base58::{FromBase58, ToBase58};
use super::types::*;

enum CatchupStepResult {
    Finished,
    Continue,
    FailedAtNode(usize),
}

enum CatchupProgress {
    ShouldBeStarted,
    NotNeeded,
    Finished(MerkleTree),
    InProgress,
}

pub struct CatchupHandler {
    pub f: usize,
    pub ledger_status_same: usize,
    pub merkle_tree: MerkleTree,
    pub target_mt_size: usize,
    pub target_mt_root: Vec<u8>,
    pub new_mt_vote: usize,
    pub nodes: Vec<RemoteNode>,
    pub initiate_cmd_id: i32,
    pub is_refresh: bool,
    pub pending_catchup: Option<CatchUpProcess>,
    pub pool_id: i32,
    pub nodes_votes: Vec<Option<(String, usize)>>,
}

impl Default for CatchupHandler {
    fn default() -> Self {
        CatchupHandler {
            f: 0,
            ledger_status_same: 0,
            merkle_tree: MerkleTree::from_vec(Vec::new()).unwrap(),
            nodes: Vec::new(),
            target_mt_size: 0,
            new_mt_vote: 0,
            target_mt_root: Vec::new(),
            pending_catchup: None,
            initiate_cmd_id: 0,
            is_refresh: false,
            pool_id: 0,
            nodes_votes: Vec::new(),
        }
    }
}

impl CatchupHandler {
    pub fn process_msg(&mut self, msg: Message, raw_msg: &String, src_ind: usize) -> Result<Option<MerkleTree>, PoolError> {
        let catchup_status: CatchupProgress = match msg {
            Message::Pong => {
                //sending ledger status
                //TODO not send ledger status directly as response on ping, wait pongs from all nodes?
                let ls: LedgerStatus = LedgerStatus {
                    txnSeqNo: self.nodes.len(),
                    merkleRoot: self.merkle_tree.root_hash().as_slice().to_base58(),
                    ledgerId: 0,
                    ppSeqNo: None,
                    viewNo: None,
                };
                let resp_msg: Message = Message::LedgerStatus(ls);
                self.nodes[src_ind].send_msg(&resp_msg)?;
                CatchupProgress::InProgress
            }
            Message::LedgerStatus(ledger_status) => {
                self.nodes_votes[src_ind] = Some((ledger_status.merkleRoot, ledger_status.txnSeqNo));
                self.check_nodes_responses_on_status()?
            }
            Message::ConsistencyProof(cons_proof) => {
                self.nodes_votes[src_ind] = Some((cons_proof.newMerkleRoot, cons_proof.seqNoEnd));
                self.check_nodes_responses_on_status()?
            }
            Message::CatchupRep(catchup) => {
                self.process_catchup_rep(catchup, src_ind)?
            }
            _ => {
                warn!("unhandled msg {:?}", msg);
                CatchupProgress::InProgress
            }
        };

        match catchup_status {
            CatchupProgress::Finished(mt) => return Ok(Some(mt)),
            CatchupProgress::NotNeeded => return Ok(Some(self.merkle_tree.clone())),
            CatchupProgress::ShouldBeStarted => self.start_catchup()?,
            CatchupProgress::InProgress => { /* nothing to do */ }
        }
        Ok(None)
    }

    fn check_nodes_responses_on_status(&mut self) -> Result<CatchupProgress, PoolError> {
        if self.pending_catchup.is_some() {
            return Ok(CatchupProgress::InProgress);
        }
        let mut votes: HashMap<(String, usize), usize> = HashMap::new();
        for node_vote in &self.nodes_votes {
            if let &Some(ref node_vote) = node_vote {
                let cnt = *votes.get(&node_vote).unwrap_or(&0) + 1;
                votes.insert((node_vote.0.clone(), node_vote.1), cnt);
            }
        }
        if let Some((most_popular_vote, votes_cnt)) = votes.iter().max_by_key(|entry| entry.1) {
            if *votes_cnt == self.nodes.len() - self.f /* TODO N-f consensus */ {
                let &(ref target_mt_root, target_mt_size) = most_popular_vote;
                let cur_mt_size = self.merkle_tree.count();
                let cur_mt_hash = self.merkle_tree.root_hash().to_base58();
                if target_mt_size == cur_mt_size {
                    if cur_mt_hash.eq(target_mt_root) {
                        return Ok(CatchupProgress::NotNeeded);
                    } else {
                        return Err(PoolError::CommonError(CommonError::InvalidState(
                            "Ledger merkle tree doesn't acceptable for current tree.".to_string())));
                    }
                } else if target_mt_size > cur_mt_size {
                    self.target_mt_size = target_mt_size;
                    self.target_mt_root = target_mt_root.from_base58().map_err(|_|
                        CommonError::InvalidStructure(
                            "Can't parse target MerkleTree hash from nodes responses".to_string()))?;
                    return Ok(CatchupProgress::ShouldBeStarted);
                } else {
                    return Err(PoolError::CommonError(CommonError::InvalidState(
                        "Local merkle tree greater than mt from ledger".to_string())));
                }
            }
        }
        Ok(CatchupProgress::InProgress)
    }

    pub fn reset_nodes_votes(&mut self) {
        self.nodes_votes.clear();
        self.nodes_votes.resize(self.nodes.len(), None);
    }

    pub fn start_catchup(&mut self) -> Result<(), PoolError> {
        trace!("start_catchup");
        if self.pending_catchup.is_some() {
            return Err(PoolError::CommonError(
                CommonError::InvalidState(
                    "CatchUp already started for the pool".to_string())));
        }
        if self.merkle_tree.count() != self.nodes.len() {
            return Err(PoolError::CommonError(
                CommonError::InvalidState(
                    "Merkle tree doesn't equal nodes count".to_string())));
        }

        let node_cnt = self.nodes.iter().filter(|node| !node.is_blacklisted).count();
        let cnt_to_catchup = self.target_mt_size - self.merkle_tree.count();
        if cnt_to_catchup <= 0 {
            return Err(PoolError::CommonError(CommonError::InvalidState(
                "Nothing to CatchUp, but started".to_string())));
        }

        self.pending_catchup = Some(CatchUpProcess {
            merkle_tree: self.merkle_tree.clone(),
            pending_reps: Vec::new(),
        });

        let portion = (cnt_to_catchup + node_cnt - 1) / node_cnt; //TODO check standard round up div
        let mut catchup_req = CatchupReq {
            ledgerId: 0,
            seqNoStart: node_cnt + 1,
            seqNoEnd: node_cnt + 1 + portion - 1,
            catchupTill: self.target_mt_size,
        };
        for node in &self.nodes {
            if node.is_blacklisted {
                continue;
            }
            node.send_msg(&Message::CatchupReq(catchup_req.clone()))?;
            catchup_req.seqNoStart += portion;
            catchup_req.seqNoEnd = cmp::min(catchup_req.seqNoStart + portion - 1,
                                            catchup_req.catchupTill);
        }
        Ok(())
    }

    fn process_catchup_rep(&mut self, catchup: CatchupRep, node_idx: usize) -> Result<CatchupProgress, PoolError> {
        trace!("append {:?}", catchup);
        let catchup_finished = self.catchup_step(catchup, node_idx)?;
        match catchup_finished {
            CatchupStepResult::Finished => return Ok(CatchupProgress::Finished(self.finish_catchup()?)),
            CatchupStepResult::Continue => { /* nothing to do */ }
            CatchupStepResult::FailedAtNode(failed_node_idx) => {
                warn!("Fail to continue catch-up by response from node with idx {}. Node will be blacklisted and catchup will be restarted", failed_node_idx);
                self.nodes[failed_node_idx].is_blacklisted = true;
                self.pending_catchup = None;
                // TODO may be send ledger status again and re-obtain target MerkleTree params
                self.start_catchup()?
            }
        }
        Ok(CatchupProgress::InProgress)
    }

    fn catchup_step(&mut self, catchup: CatchupRep, node_idx: usize) -> Result<CatchupStepResult, PoolError> {
        let mut process = self.pending_catchup.as_mut()
            .ok_or(CommonError::InvalidState("Process non-existing CatchUp".to_string()))?;
        process.pending_reps.push((catchup, node_idx));

        while !process.pending_reps.is_empty() {
            let index = process.pending_reps.get_min_index()?;
            let (mut first_resp, node_idx) = process.pending_reps.remove(index);
            if first_resp.min_tx()? - 1 != process.merkle_tree.count() { break; }
            let mut temp_mt = process.merkle_tree.clone();
            while !first_resp.txns.is_empty() {
                let key = first_resp.min_tx()?.to_string();
                let new_gen_tx = first_resp.txns.remove(&key).unwrap();
                if let Ok(new_get_txn_bytes) = rmp_serde::to_vec_named(&new_gen_tx) {
                    temp_mt.append(new_get_txn_bytes)?;
                } else {
                    return Ok(CatchupStepResult::FailedAtNode(node_idx));
                }
            }

            if CatchupHandler::check_cons_proofs(&temp_mt, &first_resp.consProof, &self.target_mt_root, self.target_mt_size)
                .map_err(map_err_err!()).is_err() {
                return Ok(CatchupStepResult::FailedAtNode(node_idx));
            }

            process.merkle_tree = temp_mt;
        }
        trace!("updated mt hash {}, tree {:?}", process.merkle_tree.root_hash().as_slice().to_base58(), process.merkle_tree);
        if &process.merkle_tree.count() == &self.target_mt_size {
            if process.merkle_tree.root_hash().ne(&self.target_mt_root) {
                return Err(PoolError::CommonError(CommonError::InvalidState(
                    "CatchUp failed: all transactions added, proofs checked, but root hash differ with target".to_string())));
            }
            return Ok(CatchupStepResult::Finished);
        } else {
            return Ok(CatchupStepResult::Continue);
        }
    }

    fn check_cons_proofs(mt: &MerkleTree, cons_proofs: &Vec<String>, target_mt_root: &Vec<u8>, target_mt_size: usize) -> Result<(), CommonError> {
        let mut bytes_proofs: Vec<Vec<u8>> = Vec::new();
        for cons_proof in cons_proofs {
            let cons_proof: &String = cons_proof;
            bytes_proofs.push(cons_proof.from_base58().map_err(|err|
                CommonError::InvalidStructure(
                    format!("Can't decode node consistency proof: {}", err)))?)
        }
        if !mt.consistency_proof(target_mt_root, target_mt_size, &bytes_proofs)? {
            return Err(CommonError::InvalidStructure("Consistency proof verification failed".to_string()));
        }
        Ok(())
    }

    fn finish_catchup(&mut self) -> Result<MerkleTree, PoolError> {
        Ok(self.pending_catchup.take().
            ok_or(CommonError::InvalidState("Try to finish non-existing CatchUp".to_string()))?
            .merkle_tree)
    }

    pub fn flush_requests(&mut self, status: Result<(), PoolError>) -> Result<(), PoolError> {
        let cmd = if self.is_refresh {
            PoolCommand::RefreshAck(self.initiate_cmd_id, status)
        } else {
            PoolCommand::OpenAck(self.initiate_cmd_id, status.map(|()| self.pool_id))
        };
        CommandExecutor::instance()
            .send(Command::Pool(cmd))
            .map_err(|err|
                PoolError::CommonError(
                    CommonError::InvalidState("Can't send ACK cmd".to_string())))
    }
}
<commit_msg>Fixed error with early deleting<commit_after>extern crate rmp_serde;

use std::cmp;
use std::collections::HashMap;

use commands::{Command, CommandExecutor};
use commands::pool::PoolCommand;
use errors::common::CommonError;
use errors::pool::PoolError;
use super::{
    MerkleTree,
    RemoteNode,
};
use super::rust_base58::{FromBase58, ToBase58};
use super::types::*;

enum CatchupStepResult {
    Finished,
    Continue,
    FailedAtNode(usize),
}

enum CatchupProgress {
    ShouldBeStarted,
    NotNeeded,
    Finished(MerkleTree),
    InProgress,
}

pub struct CatchupHandler {
    pub f: usize,
    pub ledger_status_same: usize,
    pub merkle_tree: MerkleTree,
    pub target_mt_size: usize,
    pub target_mt_root: Vec<u8>,
    pub new_mt_vote: usize,
    pub nodes: Vec<RemoteNode>,
    pub initiate_cmd_id: i32,
    pub is_refresh: bool,
    pub pending_catchup: Option<CatchUpProcess>,
    pub pool_id: i32,
    pub nodes_votes: Vec<Option<(String, usize)>>,
}

impl Default for CatchupHandler {
    fn default() -> Self {
        CatchupHandler {
            f: 0,
            ledger_status_same: 0,
            merkle_tree: MerkleTree::from_vec(Vec::new()).unwrap(),
            nodes: Vec::new(),
            target_mt_size: 0,
            new_mt_vote: 0,
            target_mt_root: Vec::new(),
            pending_catchup: None,
            initiate_cmd_id: 0,
            is_refresh: false,
            pool_id: 0,
            nodes_votes: Vec::new(),
        }
    }
}

impl CatchupHandler {
    pub fn process_msg(&mut self, msg: Message, raw_msg: &String, src_ind: usize) -> Result<Option<MerkleTree>, PoolError> {
        let catchup_status: CatchupProgress = match msg {
            Message::Pong => {
                //sending ledger status
                //TODO not send ledger status directly as response on ping, wait pongs from all nodes?
                let ls: LedgerStatus = LedgerStatus {
                    txnSeqNo: self.nodes.len(),
                    merkleRoot: self.merkle_tree.root_hash().as_slice().to_base58(),
                    ledgerId: 0,
                    ppSeqNo: None,
                    viewNo: None,
                };
                let resp_msg: Message = Message::LedgerStatus(ls);
                self.nodes[src_ind].send_msg(&resp_msg)?;
                CatchupProgress::InProgress
            }
            Message::LedgerStatus(ledger_status) => {
                self.nodes_votes[src_ind] = Some((ledger_status.merkleRoot, ledger_status.txnSeqNo));
                self.check_nodes_responses_on_status()?
            }
            Message::ConsistencyProof(cons_proof) => {
                self.nodes_votes[src_ind] = Some((cons_proof.newMerkleRoot, cons_proof.seqNoEnd));
                self.check_nodes_responses_on_status()?
            }
            Message::CatchupRep(catchup) => {
                self.process_catchup_rep(catchup, src_ind)?
            }
            _ => {
                warn!("unhandled msg {:?}", msg);
                CatchupProgress::InProgress
            }
        };

        match catchup_status {
            CatchupProgress::Finished(mt) => return Ok(Some(mt)),
            CatchupProgress::NotNeeded => return Ok(Some(self.merkle_tree.clone())),
            CatchupProgress::ShouldBeStarted => self.start_catchup()?,
            CatchupProgress::InProgress => { /* nothing to do */ }
        }
        Ok(None)
    }

    fn check_nodes_responses_on_status(&mut self) -> Result<CatchupProgress, PoolError> {
        if self.pending_catchup.is_some() {
            return Ok(CatchupProgress::InProgress);
        }
        let mut votes: HashMap<(String, usize), usize> = HashMap::new();
        for node_vote in &self.nodes_votes {
            if let &Some(ref node_vote) = node_vote {
                let cnt = *votes.get(&node_vote).unwrap_or(&0) + 1;
                votes.insert((node_vote.0.clone(), node_vote.1), cnt);
            }
        }
        if let Some((most_popular_vote, votes_cnt)) = votes.iter().max_by_key(|entry| entry.1) {
            if *votes_cnt == self.nodes.len() - self.f /* TODO N-f consensus */ {
                let &(ref target_mt_root, target_mt_size) = most_popular_vote;
                let cur_mt_size = self.merkle_tree.count();
                let cur_mt_hash = self.merkle_tree.root_hash().to_base58();
                if target_mt_size == cur_mt_size {
                    if cur_mt_hash.eq(target_mt_root) {
                        return Ok(CatchupProgress::NotNeeded);
                    } else {
                        return Err(PoolError::CommonError(CommonError::InvalidState(
                            "Ledger merkle tree doesn't acceptable for current tree.".to_string())));
                    }
                } else if target_mt_size > cur_mt_size {
                    self.target_mt_size = target_mt_size;
                    self.target_mt_root = target_mt_root.from_base58().map_err(|_|
                        CommonError::InvalidStructure(
                            "Can't parse target MerkleTree hash from nodes responses".to_string()))?;
                    return Ok(CatchupProgress::ShouldBeStarted);
                } else {
                    return Err(PoolError::CommonError(CommonError::InvalidState(
                        "Local merkle tree greater than mt from ledger".to_string())));
                }
            }
        }
        Ok(CatchupProgress::InProgress)
    }

    pub fn reset_nodes_votes(&mut self) {
        self.nodes_votes.clear();
        self.nodes_votes.resize(self.nodes.len(), None);
    }

    pub fn start_catchup(&mut self) -> Result<(), PoolError> {
        trace!("start_catchup");
        if self.pending_catchup.is_some() {
            return Err(PoolError::CommonError(
                CommonError::InvalidState(
                    "CatchUp already started for the pool".to_string())));
        }
        if self.merkle_tree.count() != self.nodes.len() {
            return Err(PoolError::CommonError(
                CommonError::InvalidState(
                    "Merkle tree doesn't equal nodes count".to_string())));
        }

        let node_cnt = self.nodes.iter().filter(|node| !node.is_blacklisted).count();
        let cnt_to_catchup = self.target_mt_size - self.merkle_tree.count();
        if cnt_to_catchup <= 0 {
            return Err(PoolError::CommonError(CommonError::InvalidState(
                "Nothing to CatchUp, but started".to_string())));
        }

        self.pending_catchup = Some(CatchUpProcess {
            merkle_tree: self.merkle_tree.clone(),
            pending_reps: Vec::new(),
        });

        let portion = (cnt_to_catchup + node_cnt - 1) / node_cnt; //TODO check standard round up div
        let mut catchup_req = CatchupReq {
            ledgerId: 0,
            seqNoStart: node_cnt + 1,
            seqNoEnd: node_cnt + 1 + portion - 1,
            catchupTill: self.target_mt_size,
        };
        for node in &self.nodes {
            if node.is_blacklisted {
                continue;
            }
            node.send_msg(&Message::CatchupReq(catchup_req.clone()))?;
            catchup_req.seqNoStart += portion;
            catchup_req.seqNoEnd = cmp::min(catchup_req.seqNoStart + portion - 1,
                                            catchup_req.catchupTill);
        }
        Ok(())
    }

    fn process_catchup_rep(&mut self, catchup: CatchupRep, node_idx: usize) -> Result<CatchupProgress, PoolError> {
        trace!("append {:?}", catchup);
        let catchup_finished = self.catchup_step(catchup, node_idx)?;
        match catchup_finished {
            CatchupStepResult::Finished => return Ok(CatchupProgress::Finished(self.finish_catchup()?)),
            CatchupStepResult::Continue => { /* nothing to do */ }
            CatchupStepResult::FailedAtNode(failed_node_idx) => {
                warn!("Fail to continue catch-up by response from node with idx {}. Node will be blacklisted and catchup will be restarted", failed_node_idx);
                self.nodes[failed_node_idx].is_blacklisted = true;
                self.pending_catchup = None;
                // TODO may be send ledger status again and re-obtain target MerkleTree params
                self.start_catchup()?
            }
        }
        Ok(CatchupProgress::InProgress)
    }

    fn catchup_step(&mut self, catchup: CatchupRep, node_idx: usize) -> Result<CatchupStepResult, PoolError> {
        let mut process = self.pending_catchup.as_mut()
            .ok_or(CommonError::InvalidState("Process non-existing CatchUp".to_string()))?;
        process.pending_reps.push((catchup, node_idx));

        while !process.pending_reps.is_empty() {
            let index = process.pending_reps.get_min_index()?;
            {
                let &mut (ref mut first_resp, node_idx) = process.pending_reps.get_mut(index)
                    .ok_or(CommonError::InvalidStructure(format!("Element not Found")))?;
                if first_resp.min_tx()? - 1 != process.merkle_tree.count() { break; }

                let mut temp_mt = process.merkle_tree.clone();
                while !first_resp.txns.is_empty() {
                    let key = first_resp.min_tx()?.to_string();
                    let new_gen_tx = first_resp.txns.remove(&key).unwrap();
                    if let Ok(new_get_txn_bytes) = rmp_serde::to_vec_named(&new_gen_tx) {
                        temp_mt.append(new_get_txn_bytes)?;
                    } else {
                        return Ok(CatchupStepResult::FailedAtNode(node_idx));
                    }
                }

                if CatchupHandler::check_cons_proofs(&temp_mt, &first_resp.consProof, &self.target_mt_root, self.target_mt_size)
                    .map_err(map_err_err!()).is_err() {
                    return Ok(CatchupStepResult::FailedAtNode(node_idx));
                }

                process.merkle_tree = temp_mt;
            }
            process.pending_reps.remove(index);
        }
        trace!("updated mt hash {}, tree {:?}", process.merkle_tree.root_hash().as_slice().to_base58(), process.merkle_tree);
        if &process.merkle_tree.count() == &self.target_mt_size {
            if process.merkle_tree.root_hash().ne(&self.target_mt_root) {
                return Err(PoolError::CommonError(CommonError::InvalidState(
                    "CatchUp failed: all transactions added, proofs checked, but root hash differ with target".to_string())));
            }
            return Ok(CatchupStepResult::Finished);
        } else {
            return Ok(CatchupStepResult::Continue);
        }
    }

    fn check_cons_proofs(mt: &MerkleTree, cons_proofs: &Vec<String>, target_mt_root: &Vec<u8>, target_mt_size: usize) -> Result<(), CommonError> {
        let mut bytes_proofs: Vec<Vec<u8>> = Vec::new();
        for cons_proof in cons_proofs {
            let cons_proof: &String = cons_proof;
            bytes_proofs.push(cons_proof.from_base58().map_err(|err|
                CommonError::InvalidStructure(
                    format!("Can't decode node consistency proof: {}", err)))?)
        }
        if !mt.consistency_proof(target_mt_root, target_mt_size, &bytes_proofs)? {
            return Err(CommonError::InvalidStructure("Consistency proof verification failed".to_string()));
        }
        Ok(())
    }

    fn finish_catchup(&mut self) -> Result<MerkleTree, PoolError> {
        Ok(self.pending_catchup.take().
            ok_or(CommonError::InvalidState("Try to finish non-existing CatchUp".to_string()))?
            .merkle_tree)
    }

    pub fn flush_requests(&mut self, status: Result<(), PoolError>) -> Result<(), PoolError> {
        let cmd = if self.is_refresh {
            PoolCommand::RefreshAck(self.initiate_cmd_id, status)
        } else {
            PoolCommand::OpenAck(self.initiate_cmd_id, status.map(|()| self.pool_id))
        };
        CommandExecutor::instance()
            .send(Command::Pool(cmd))
            .map_err(|err|
                PoolError::CommonError(
                    CommonError::InvalidState("Can't send ACK cmd".to_string())))
    }
}
<|endoftext|>


<commit_before>//! Abstraction over blocking and unblocking the current thread.
//!
//! Provides an abstraction over blocking the current thread. This is similar to
//! the park / unpark constructs provided by [`std`] but made generic. This
//! allows embedding custom functionality to perform when the thread is blocked.
//!
//! A blocked [`Park`][p] instance is unblocked by calling [`unpark`] on its
//! [`Unpark`][up] handle.
//!
//! The [`ParkThread`] struct implements [`Park`][p] using
//! [`thread::park`][`std`] to put the thread to sleep. The Tokio reactor also
//! implements park, but uses [`mio::Poll`][mio] to block the thread instead.
//!
//! The [`Park`][p] trait is composable. A timer implementation might decorate a
//! [`Park`][p] implementation by checking if any timeouts have elapsed after
//! the inner [`Park`][p] implementation unblocks.
//!
//! # Model
//!
//! Conceptually, each [`Park`][p] instance has an associated token, which is
//! initially not present:
//!
//! * The [`park`] method blocks the current thread unless or until the token
//!   is available, at which point it atomically consumes the token.
//! * The [`unpark`] method atomically makes the token available if it wasn't
//!   already.
//!
//! Some things to note:
//!
//! * If [`unpark`] is called before [`park`], the next call to [`park`] will
//! **not** block the thread.
//! * **Spurious** wakeups are permitted, i.e., the [`park`] method may unblock
//!   even if [`unpark`] was not called.
//! * [`park_timeout`] does the same as [`park`] but allows specifying a maximum
//!   time to block the thread for.
//!
//! [`std`]: https://doc.rust-lang.org/std/thread/fn.park.html
//! [`thread::park`]: https://doc.rust-lang.org/std/thread/fn.park.html
//! [`ParkThread`]: struct.ParkThread.html
//! [p]: trait.Park.html
//! [`park`]: trait.Park.html#tymethod.park
//! [`park_timeout`]: trait.Park.html#tymethod.park_timeout
//! [`unpark`]: trait.Unpark.html#tymethod.unpark
//! [up]: trait.Unpark.html
//! [mio]: https://docs.rs/mio/0.6/mio/struct.Poll.html

use std::marker::PhantomData;
use std::rc::Rc;
use std::time::Duration;

use crossbeam_utils::sync::{Parker, Unparker};

/// Block the current thread.
///
/// See [module documentation][mod] for more details.
///
/// [mod]: ../index.html
pub trait Park {
    /// Unpark handle type for the `Park` implementation.
    type Unpark: Unpark;

    /// Error returned by `park`
    type Error;

    /// Get a new `Unpark` handle associated with this `Park` instance.
    fn unpark(&self) -> Self::Unpark;

    /// Block the current thread unless or until the token is available.
    ///
    /// A call to `park` does not guarantee that the thread will remain blocked
    /// forever, and callers should be prepared for this possibility. This
    /// function may wakeup spuriously for any reason.
    ///
    /// See [module documentation][mod] for more details.
    ///
    /// # Panics
    ///
    /// This function **should** not panic, but ultimately, panics are left as
    /// an implementation detail. Refer to the documentation for the specific
    /// `Park` implementation
    ///
    /// [mod]: ../index.html
    fn park(&mut self) -> Result<(), Self::Error>;

    /// Park the current thread for at most `duration`.
    ///
    /// This function is the same as `park` but allows specifying a maximum time
    /// to block the thread for.
    ///
    /// Same as `park`, there is no guarantee that the thread will remain
    /// blocked for any amount of time. Spurious wakeups are permitted for any
    /// reason.
    ///
    /// See [module documentation][mod] for more details.
    ///
    /// # Panics
    ///
    /// This function **should** not panic, but ultimately, panics are left as
    /// an implementation detail. Refer to the documentation for the specific
    /// `Park` implementation
    ///
    /// [mod]: ../index.html
    fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error>;
}

/// Unblock a thread blocked by the associated [`Park`] instance.
///
/// See [module documentation][mod] for more details.
///
/// [mod]: ../index.html
/// [`Park`]: trait.Park.html
pub trait Unpark: Sync + Send + 'static {
    /// Unblock a thread that is blocked by the associated `Park` handle.
    ///
    /// Calling `unpark` atomically makes available the unpark token, if it is
    /// not already available.
    ///
    /// See [module documentation][mod] for more details.
    ///
    /// # Panics
    ///
    /// This function **should** not panic, but ultimately, panics are left as
    /// an implementation detail. Refer to the documentation for the specific
    /// `Unpark` implementation
    ///
    /// [mod]: ../index.html
    fn unpark(&self);
}

impl Unpark for Box<Unpark> {
    fn unpark(&self) {
        (**self).unpark()
    }
}

impl Unpark for Arc<Unpark> {
    fn unpark(&self) {
        (**self).unpark()
    }
}

/// Blocks the current thread using a condition variable.
///
/// Implements the [`Park`] functionality by using a condition variable. An
/// atomic variable is also used to avoid using the condition variable if
/// possible.
///
/// The condition variable is cached in a thread-local variable and is shared
/// across all `ParkThread` instances created on the same thread. This also
/// means that an instance of `ParkThread` might be unblocked by a handle
/// associated with a different `ParkThread` instance.
#[derive(Debug)]
pub struct ParkThread {
    _anchor: PhantomData<Rc<()>>,
}

/// Error returned by [`ParkThread`]
///
/// This currently is never returned, but might at some point in the future.
///
/// [`ParkThread`]: struct.ParkThread.html
#[derive(Debug)]
pub struct ParkError {
    _p: (),
}

/// Unblocks a thread that was blocked by `ParkThread`.
#[derive(Clone, Debug)]
pub struct UnparkThread {
    inner: Unparker,
}

thread_local! {
    static CURRENT_PARKER: Parker = Parker::new();
}

// ===== impl ParkThread =====

impl ParkThread {
    /// Create a new `ParkThread` handle for the current thread.
    ///
    /// This type cannot be moved to other threads, so it should be created on
    /// the thread that the caller intends to park.
    pub fn new() -> ParkThread {
        ParkThread {
            _anchor: PhantomData,
        }
    }

    /// Get a reference to the `ParkThread` handle for this thread.
    fn with_current<F, R>(&self, f: F) -> R
        where F: FnOnce(&Parker) -> R,
    {
        CURRENT_PARKER.with(|inner| f(inner))
    }
}

impl Park for ParkThread {
    type Unpark = UnparkThread;
    type Error = ParkError;

    fn unpark(&self) -> Self::Unpark {
        let inner = self.with_current(|inner| inner.unparker().clone());
        UnparkThread { inner }
    }

    fn park(&mut self) -> Result<(), Self::Error> {
        self.with_current(|inner| inner.park());
        Ok(())
    }

    fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
        self.with_current(|inner| inner.park_timeout(duration));
        Ok(())
    }
}

// ===== impl UnparkThread =====

impl Unpark for UnparkThread {
    fn unpark(&self) {
        self.inner.unpark();
    }
}
<commit_msg>executor: fix build (#825)<commit_after>//! Abstraction over blocking and unblocking the current thread.
//!
//! Provides an abstraction over blocking the current thread. This is similar to
//! the park / unpark constructs provided by [`std`] but made generic. This
//! allows embedding custom functionality to perform when the thread is blocked.
//!
//! A blocked [`Park`][p] instance is unblocked by calling [`unpark`] on its
//! [`Unpark`][up] handle.
//!
//! The [`ParkThread`] struct implements [`Park`][p] using
//! [`thread::park`][`std`] to put the thread to sleep. The Tokio reactor also
//! implements park, but uses [`mio::Poll`][mio] to block the thread instead.
//!
//! The [`Park`][p] trait is composable. A timer implementation might decorate a
//! [`Park`][p] implementation by checking if any timeouts have elapsed after
//! the inner [`Park`][p] implementation unblocks.
//!
//! # Model
//!
//! Conceptually, each [`Park`][p] instance has an associated token, which is
//! initially not present:
//!
//! * The [`park`] method blocks the current thread unless or until the token
//!   is available, at which point it atomically consumes the token.
//! * The [`unpark`] method atomically makes the token available if it wasn't
//!   already.
//!
//! Some things to note:
//!
//! * If [`unpark`] is called before [`park`], the next call to [`park`] will
//! **not** block the thread.
//! * **Spurious** wakeups are permitted, i.e., the [`park`] method may unblock
//!   even if [`unpark`] was not called.
//! * [`park_timeout`] does the same as [`park`] but allows specifying a maximum
//!   time to block the thread for.
//!
//! [`std`]: https://doc.rust-lang.org/std/thread/fn.park.html
//! [`thread::park`]: https://doc.rust-lang.org/std/thread/fn.park.html
//! [`ParkThread`]: struct.ParkThread.html
//! [p]: trait.Park.html
//! [`park`]: trait.Park.html#tymethod.park
//! [`park_timeout`]: trait.Park.html#tymethod.park_timeout
//! [`unpark`]: trait.Unpark.html#tymethod.unpark
//! [up]: trait.Unpark.html
//! [mio]: https://docs.rs/mio/0.6/mio/struct.Poll.html

use std::marker::PhantomData;
use std::rc::Rc;
use std::sync::Arc;
use std::time::Duration;

use crossbeam_utils::sync::{Parker, Unparker};

/// Block the current thread.
///
/// See [module documentation][mod] for more details.
///
/// [mod]: ../index.html
pub trait Park {
    /// Unpark handle type for the `Park` implementation.
    type Unpark: Unpark;

    /// Error returned by `park`
    type Error;

    /// Get a new `Unpark` handle associated with this `Park` instance.
    fn unpark(&self) -> Self::Unpark;

    /// Block the current thread unless or until the token is available.
    ///
    /// A call to `park` does not guarantee that the thread will remain blocked
    /// forever, and callers should be prepared for this possibility. This
    /// function may wakeup spuriously for any reason.
    ///
    /// See [module documentation][mod] for more details.
    ///
    /// # Panics
    ///
    /// This function **should** not panic, but ultimately, panics are left as
    /// an implementation detail. Refer to the documentation for the specific
    /// `Park` implementation
    ///
    /// [mod]: ../index.html
    fn park(&mut self) -> Result<(), Self::Error>;

    /// Park the current thread for at most `duration`.
    ///
    /// This function is the same as `park` but allows specifying a maximum time
    /// to block the thread for.
    ///
    /// Same as `park`, there is no guarantee that the thread will remain
    /// blocked for any amount of time. Spurious wakeups are permitted for any
    /// reason.
    ///
    /// See [module documentation][mod] for more details.
    ///
    /// # Panics
    ///
    /// This function **should** not panic, but ultimately, panics are left as
    /// an implementation detail. Refer to the documentation for the specific
    /// `Park` implementation
    ///
    /// [mod]: ../index.html
    fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error>;
}

/// Unblock a thread blocked by the associated [`Park`] instance.
///
/// See [module documentation][mod] for more details.
///
/// [mod]: ../index.html
/// [`Park`]: trait.Park.html
pub trait Unpark: Sync + Send + 'static {
    /// Unblock a thread that is blocked by the associated `Park` handle.
    ///
    /// Calling `unpark` atomically makes available the unpark token, if it is
    /// not already available.
    ///
    /// See [module documentation][mod] for more details.
    ///
    /// # Panics
    ///
    /// This function **should** not panic, but ultimately, panics are left as
    /// an implementation detail. Refer to the documentation for the specific
    /// `Unpark` implementation
    ///
    /// [mod]: ../index.html
    fn unpark(&self);
}

impl Unpark for Box<Unpark> {
    fn unpark(&self) {
        (**self).unpark()
    }
}

impl Unpark for Arc<Unpark> {
    fn unpark(&self) {
        (**self).unpark()
    }
}

/// Blocks the current thread using a condition variable.
///
/// Implements the [`Park`] functionality by using a condition variable. An
/// atomic variable is also used to avoid using the condition variable if
/// possible.
///
/// The condition variable is cached in a thread-local variable and is shared
/// across all `ParkThread` instances created on the same thread. This also
/// means that an instance of `ParkThread` might be unblocked by a handle
/// associated with a different `ParkThread` instance.
#[derive(Debug)]
pub struct ParkThread {
    _anchor: PhantomData<Rc<()>>,
}

/// Error returned by [`ParkThread`]
///
/// This currently is never returned, but might at some point in the future.
///
/// [`ParkThread`]: struct.ParkThread.html
#[derive(Debug)]
pub struct ParkError {
    _p: (),
}

/// Unblocks a thread that was blocked by `ParkThread`.
#[derive(Clone, Debug)]
pub struct UnparkThread {
    inner: Unparker,
}

thread_local! {
    static CURRENT_PARKER: Parker = Parker::new();
}

// ===== impl ParkThread =====

impl ParkThread {
    /// Create a new `ParkThread` handle for the current thread.
    ///
    /// This type cannot be moved to other threads, so it should be created on
    /// the thread that the caller intends to park.
    pub fn new() -> ParkThread {
        ParkThread {
            _anchor: PhantomData,
        }
    }

    /// Get a reference to the `ParkThread` handle for this thread.
    fn with_current<F, R>(&self, f: F) -> R
        where F: FnOnce(&Parker) -> R,
    {
        CURRENT_PARKER.with(|inner| f(inner))
    }
}

impl Park for ParkThread {
    type Unpark = UnparkThread;
    type Error = ParkError;

    fn unpark(&self) -> Self::Unpark {
        let inner = self.with_current(|inner| inner.unparker().clone());
        UnparkThread { inner }
    }

    fn park(&mut self) -> Result<(), Self::Error> {
        self.with_current(|inner| inner.park());
        Ok(())
    }

    fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
        self.with_current(|inner| inner.park_timeout(duration));
        Ok(())
    }
}

// ===== impl UnparkThread =====

impl Unpark for UnparkThread {
    fn unpark(&self) {
        self.inner.unpark();
    }
}
<|endoftext|>